diff --git a/Cargo.lock b/Cargo.lock index 79946e6cd0..f2a825f4eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -530,6 +530,8 @@ version = "0.61.0" dependencies = [ "anyhow", "clap", + "diskann-utils", + "erased-serde", "half", "indicatif", "serde", @@ -934,6 +936,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e59b668157f32d3459d4bd634861a06ffa2503a461ca5f0916b9f0252605e66" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + [[package]] name = "errno" version = "0.3.14" @@ -2722,6 +2735,12 @@ dependencies = [ "toml 1.1.6+spec-1.1.0", ] +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + [[package]] name = "unarray" version = "0.1.4" diff --git a/diskann-benchmark-runner/Cargo.toml b/diskann-benchmark-runner/Cargo.toml index 041f16df33..b381c8ae2d 100644 --- a/diskann-benchmark-runner/Cargo.toml +++ b/diskann-benchmark-runner/Cargo.toml @@ -10,6 +10,7 @@ edition = "2024" [dependencies] anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } +erased-serde = "0.4.11" half = { workspace = true } indicatif = "0.18.3" serde = { workspace = true, features = ["derive"] } @@ -25,6 +26,7 @@ expect_used = "warn" panic = "warn" [dev-dependencies] +diskann-utils = { workspace = true, features = ["testing"], default-features = false } tempfile.workspace = true [features] diff --git a/diskann-benchmark-runner/src/lib.rs b/diskann-benchmark-runner/src/lib.rs index 403f15fc4e..8e857ff190 100644 --- a/diskann-benchmark-runner/src/lib.rs +++ b/diskann-benchmark-runner/src/lib.rs @@ -17,6 +17,7 @@ pub mod files; pub mod input; pub mod output; pub mod registry; +pub mod report; pub mod utils; pub use app::App; diff --git a/diskann-benchmark-runner/src/report/mod.rs b/diskann-benchmark-runner/src/report/mod.rs new file mode 100644 index 0000000000..125c1c08e2 --- /dev/null +++ b/diskann-benchmark-runner/src/report/mod.rs @@ -0,0 +1,1400 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +//! # Baseline Checking +//! +//! The [`Regression`](crate::benchmark::Regression) provides a means +//! of performing before/after comparisons against previously generated results. However, +//! presentation of these results is largely left to the devices of the implementors. +//! +//! This module provides a means of aggregating all match failures (if any) and presenting +//! all failures as a single unit. + +use std::borrow::Cow; + +use serde::{Serialize, Serializer}; + +pub mod tolerance; + +/// A [`Check`], [`Table`], or nested grouping of multiple [`Report`]s used to gather +/// regression checks across arbitrary benchamrks and reporting the overall success or failure. +/// +/// On success, all attributes are printed. +/// ``` +/// use diskann_benchmark_runner::report::{Report, Check}; +/// +/// let mut builder = Report::builder(); +/// { +/// let mut nested = Report::builder(); +/// nested.push("foo", Report::equal(1, 1)); +/// nested.push("bar", Report::info("cat")); +/// builder.push("nested", nested.finish()); +/// } +/// +/// let expected = " +/// nested: +/// foo: 1 +/// bar: cat +/// ".trim(); +/// +/// let report = builder.finish(); +/// assert_eq!(report.to_string(), expected); +/// ``` +/// +/// On failure, only the failing subset will be displayed. +#[derive(Debug, Serialize)] +#[serde(transparent)] +#[must_use = "reports are result types"] +pub struct Report { + inner: ReportInner, +} + +impl Report { + /// Construct an empty [`Builder`] for constructing hierarchical reports. + /// ``` + /// use diskann_benchmark_runner::report::Report; + /// + /// let mut builder = Report::builder(); + /// builder.push("foo", Report::info(10)); + /// builder.push("baz", Report::equal(10, 10)); + /// builder.push("bar", Report::equal(10, 20)); + /// let report = builder.finish(); + /// + /// let s = report.to_string(); + /// let expected = " + /// foo: 10 + /// baz: 10 + /// bar: (old) 10 != (new) 20 | FAIL + /// ".trim(); + /// assert_eq!(s, expected); + /// assert!(report.status().is_fail()); + /// ``` + pub fn builder() -> Builder { + Builder::new() + } + + /// Construct a [`Table`] using the provided list of headers. + /// ``` + /// use diskann_benchmark_runner::report::{Report, Check}; + /// + /// // Convenience for testing two renders. + /// fn process(x: &str) -> &str { + /// let x = x.strip_prefix('\n').unwrap_or(x); + /// x.strip_suffix('\n').unwrap_or(x) + /// } + /// + /// let mut table = Report::table(["hello", "world"]); + /// + /// // Tables are built one row at a time. + /// // + /// // Rows must be `finish`ed to be committed. Otherwise, + /// // their contents are // dropped. + /// let mut row = table.new_row(); + /// row.push(Check::equal(10, 20)); // Failing check. + /// row.push(Check::info("bar")); + /// table.push_row(row).unwrap(); + /// let report = table.finish(); + /// + /// let s = report.to_string(); + /// let expected = process(" + /// hello, world, status + /// ========================================= + /// (old) 10 != (new) 20 !, bar, FAIL + /// "); + /// assert_eq!(process(&s), expected); + /// ``` + pub fn table(headers: I) -> Table + where + I: IntoIterator>>, + { + Table::new(headers) + } + + /// Report `info` for diagnostic purposes. + /// + /// The information will be displayed and serialized unchanged. + pub fn info(info: T) -> Self + where + T: std::fmt::Display + Serialize + 'static, + { + Check::info(info).into() + } + + /// Report an `error`. + /// + /// Errors are considered a failure for diagnostic purposes. + /// + /// The error will be displayed and serialized using its [`ToString`] representation. + pub fn error(error: T) -> Self + where + T: ToString, + { + Check::error(error).into() + } + + /// Assert that `old == new`. + /// + /// If the check succeeds, one argument will be retained and displayed for informational + /// purposes. + /// + /// If the check fails, both values will be retained and the report fails. + pub fn equal(old: T, new: T) -> Self + where + T: std::fmt::Display + Serialize + PartialEq + 'static, + { + Check::equal(old, new).into() + } + + /// Construct a new [`Report`] using [`tolerance::HigherIsBetter`]. + pub fn higher_is_better(tolerance: &T, old: U, new: U, spec: tolerance::FmtSpec) -> Self + where + T: tolerance::HigherIsBetter, + { + tolerance.higher_is_better(old, new, spec).into() + } + + /// Construct a new [`Report`] using [`tolerance::LowerIsBetter`]. + pub fn lower_is_better(tolerance: &T, old: U, new: U, spec: tolerance::FmtSpec) -> Self + where + T: tolerance::LowerIsBetter, + { + tolerance.lower_is_better(old, new, spec).into() + } + + /// Return the [`Status`] of this report. + /// + /// [`Status::Pass`] will only be reported if all nested [`Report`]s, [`Check`]s, and + /// [`Table`]s report a pass. + pub fn status(&self) -> Status { + self.inner.status() + } + + /// Convert `self` into a [`Finished`]. + /// + /// Is this benchmark is a fail, then constructed `Finished` will only render the paths + /// that failed. Otherwise, all paths will be rendered. + pub fn pass_fail(self) -> crate::benchmark::PassFail { + let status = self.status(); + let rendering = match status { + Status::Pass => Rendering::All, + Status::Fail => Rendering::FailOnly, + }; + + let finished = Finished::new(self, rendering); + match status { + Status::Pass => crate::benchmark::PassFail::Pass(finished), + Status::Fail => crate::benchmark::PassFail::Fail(finished), + } + } +} + +impl std::fmt::Display for Report { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + Render::new(self, Rendering::All).fmt(f) + } +} + +impl From for Report { + fn from(table: Table) -> Self { + Self { + inner: ReportInner::Table(table), + } + } +} + +impl From for Report { + fn from(check: Check) -> Self { + Self { + inner: ReportInner::Check(check), + } + } +} + +/// The success or failure of checks. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Status { + Pass, + Fail, +} + +impl Status { + /// Return `true` if `self == Status::Fail`. + pub fn is_fail(self) -> bool { + self == Self::Fail + } + + /// Return `true` if `self == Status::Success`. + pub fn is_success(self) -> bool { + self == Self::Pass + } + + fn merge(&mut self, status: Self) { + if status == Status::Fail { + *self = Status::Fail; + } + } + + fn as_str(self) -> &'static str { + match self { + Self::Pass => "PASS", + Self::Fail => "FAIL", + } + } +} + +impl std::fmt::Display for Status { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl Serialize for Status { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +/// A finished [`Report`]. +/// +/// This applies a dynamic [`std::fmt::Display`] policy depending on the final status of +/// the report. +#[derive(Debug, Serialize)] +#[serde(transparent)] +pub struct Finished { + report: Report, + #[serde(skip)] + rendering: Rendering, +} + +impl Finished { + fn new(report: Report, rendering: Rendering) -> Self { + Self { report, rendering } + } +} + +impl std::fmt::Display for Finished { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + Render::new(&self.report, self.rendering).fmt(f) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Rendering { + All, + FailOnly, +} + +fn render(report: &Report, policy: Rendering) -> Option { + let ctx = FmtContext::Tree; + match &report.inner { + ReportInner::Table(table) => match policy { + Rendering::All => Some(table.to_string()), + Rendering::FailOnly => { + if table.status() == Status::Fail { + Some(table.to_string()) + } else { + None + } + } + }, + ReportInner::Check(check) => match policy { + Rendering::All => Some(check.render(ctx)), + Rendering::FailOnly => { + if check.status() == Status::Fail { + Some(check.render(ctx)) + } else { + None + } + } + }, + ReportInner::Nested { children, status } => { + if policy == Rendering::FailOnly && *status == Status::Pass { + return None; + } + + let mut kv = crate::utils::fmt::KeyValue::new(); + + // Allocate numbuffers to format all the integer keys in `children`. + let keys: Vec<_> = children.iter().map(|c| c.0.remove_position()).collect(); + + for (key, child) in std::iter::zip(keys.iter(), children.iter()) { + if let Some(rendered) = render(&child.1, policy) { + let force_nest = match &child.1.inner { + ReportInner::Table(_) => true, + ReportInner::Check(_) => false, + ReportInner::Nested { .. } => true, + }; + + if force_nest { + kv.push_eager_nested(key, rendered); + } else { + kv.push_eager(key, rendered); + } + } + } + + if kv.is_empty() { + None + } else { + Some(kv.to_string()) + } + } + } +} + +struct Render<'a> { + report: &'a Report, + policy: Rendering, +} + +impl<'a> Render<'a> { + fn new(report: &'a Report, policy: Rendering) -> Self { + Self { report, policy } + } +} + +impl std::fmt::Display for Render<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match render(self.report, self.policy) { + Some(s) => f.write_str(&s), + None => Ok(()), + } + } +} + +#[derive(Debug, Serialize)] +enum ReportInner { + /// Report is formatted as a table. + /// + /// Table must be inspected for success/failure. + Table(Table), + + /// An equal check. + Check(Check), + + /// Report is formatted as a nested report. + Nested { + children: Vec<(Key, Report)>, + status: Status, + }, +} + +impl ReportInner { + fn status(&self) -> Status { + match self { + Self::Table(table) => table.status(), + Self::Check(check) => check.status(), + Self::Nested { status, .. } => *status, + } + } +} + +///////////// +// Builder // +///////////// + +/// A builder for constructing nested [`Report`]s. +pub struct Builder { + children: Vec<(Key, Report)>, + status: Status, +} + +impl Builder { + /// Construct a new empty builder. + pub fn new() -> Self { + Self { + children: Vec::new(), + status: Status::Pass, + } + } + + /// Add `report` to the collection and associate it with `key`. + /// + /// Reports are collected in the order they are pushed and no uniqueness requirements + /// exist for `key`. If building reports positionally, [`Builder::len`] can be used to + /// get the current number of items. + pub fn push(&mut self, key: impl Into, report: Report) { + self.status.merge(report.status()); + self.children.push((key.into(), report)) + } + + /// Finish building the [`Report`]. + pub fn finish(self) -> Report { + let Self { children, status } = self; + Report { + inner: ReportInner::Nested { children, status }, + } + } + + /// Return the current number of [`Report`]s. + pub fn len(&self) -> usize { + self.children.len() + } + + /// Return `true` only if [`Builder::len`] return 0. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +impl Default for Builder { + fn default() -> Self { + Self::new() + } +} + +/// A key for a nested [`Report`] when given to [`Builder::push`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Key { + Str(&'static str), + Position(usize), + String(String), +} + +impl Key { + fn remove_position(&self) -> Cow<'_, str> { + match self { + Self::Str(s) => Cow::Borrowed(s), + Self::Position(p) => Cow::Owned(p.to_string()), + Self::String(s) => Cow::Borrowed(s), + } + } +} + +impl std::fmt::Display for Key { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Str(s) => f.write_str(s), + Self::Position(i) => write!(f, "{}", i), + Self::String(s) => f.write_str(s), + } + } +} + +impl Serialize for Key { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self { + Self::Str(s) => serializer.serialize_str(s), + Self::Position(i) => serializer.serialize_u64(*i as u64), + Self::String(s) => serializer.serialize_str(s), + } + } +} + +impl From<&'static str> for Key { + fn from(s: &'static str) -> Key { + Key::Str(s) + } +} + +impl From for Key { + fn from(i: usize) -> Key { + Key::Position(i) + } +} + +impl From for Key { + fn from(s: String) -> Key { + Key::String(s) + } +} + +/////////// +// Table // +/////////// + +/// A more horizontally compact leaf for [`Report`]. +#[derive(Debug)] +pub struct Table { + headers: Vec>, + rows: Vec, + status: Status, +} + +impl Table { + /// Create a new empty [`Table`] with the specified headers. + /// + /// Trying to add a new row via [`Table::push_row`] will fail is the row does not + /// contain the same number of entries as `headers`. + pub fn new(headers: I) -> Self + where + I: IntoIterator>>, + { + Self { + headers: headers.into_iter().map(|h| h.into()).collect(), + rows: Vec::new(), + status: Status::Pass, + } + } + + /// Return the overall status of the table. + /// + /// [`Status::Fail`] will be returned if any internal check is a failure. + pub fn status(&self) -> Status { + self.status + } + + /// Return the number of columns in the table. + /// + /// This is equal to the number of headers provided to [`Table::new`]. + pub fn ncols(&self) -> usize { + self.headers.len() + } + + /// Return the number of rows in the table. + pub fn nrows(&self) -> usize { + self.rows.len() + } + + /// Construct a new [`Row`]. + /// + /// The returned [`Row`] will not be automatically added to the table. Use + /// [`Table::push_row`] to add it. + pub fn new_row(&self) -> Row { + Row::new() + } + + /// Add `row` to the `self`. + /// + /// Returns an error if `row.len() != self.ncols()`. + pub fn push_row(&mut self, row: Row) -> anyhow::Result<()> { + if row.len() != self.ncols() { + anyhow::bail!( + "finished row has {} columns when {} were expected", + row.len(), + self.ncols(), + ); + } + + self.status.merge(row.status); + self.rows.push(row); + Ok(()) + } + + /// Complete the table. + pub fn finish(self) -> Report { + self.into() + } +} + +impl Serialize for Table { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + use serde::ser::SerializeStruct; + + let columns: Vec> = (0..self.ncols()) + .map(|column| self.rows.iter().map(|row| &row.checks[column]).collect()) + .collect(); + + let row_status: Vec = self.rows.iter().map(|row| row.status).collect(); + + let mut state = serializer.serialize_struct("Table", 4)?; + state.serialize_field("headers", &self.headers)?; + state.serialize_field("columns", &columns)?; + state.serialize_field("row_status", &row_status)?; + state.serialize_field("status", &self.status)?; + state.end() + } +} + +const EMPTY_TABLE: &str = ""; + +impl std::fmt::Display for Table { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Nothing to write for empty tables. + if self.ncols() == 0 { + return f.write_str(EMPTY_TABLE); + } + + // If this table is the source of a failure, we render an additional column to + // identify which rows caused the overall failure. + // + // To cut down on visual noise: + // + // * We don't print out "PASS". This makes "FAIL" easier to identify. + // * We do not render this column when the entire table is a pass. + let nrows = self.nrows(); + let mut table = if self.status == Status::Pass { + crate::utils::fmt::Table::new(self.headers.iter().cloned(), nrows) + } else { + let last_column = Cow::Borrowed("status"); + crate::utils::fmt::Table::new( + self.headers + .iter() + .cloned() + .chain(std::iter::once(last_column)), + nrows, + ) + }; + + for (row_index, row) in self.rows.iter().enumerate() { + let mut out = table.row(row_index); + let mut col_index = 0; + for check in row.checks.iter() { + out.insert(check.render(FmtContext::Table), col_index); + col_index += 1; + } + + if row.status == Status::Fail { + out.insert(row.status, col_index); + } + } + + write!(f, "{}", table) + } +} + +/// A new row for [`Table`]. +/// +/// Use [`Table::push_row`] to add a row to a [`Table`]. +#[derive(Debug)] +pub struct Row { + checks: Vec, + status: Status, +} + +impl Row { + /// Create a new empty [`Row`]. + pub fn new() -> Self { + Self { + checks: Vec::new(), + status: Status::Pass, + } + } + + /// Add a new [`Check`] to the row. This increases the column count of the row by 1. + pub fn push(&mut self, check: Check) { + self.status.merge(check.status()); + self.checks.push(check); + } + + /// Return the current number of columns. + pub fn len(&self) -> usize { + self.checks.len() + } + + /// Return `true` only if the row currently has no columns. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +impl Default for Row { + fn default() -> Self { + Self::new() + } +} + +/////////// +// Check // +/////////// + +/// A singular regression check. +/// +/// This is one of the leaf elements of [`Report`]. +#[must_use = "checks are result types"] +pub struct Check { + inner: Box, +} + +impl Check { + /// Return the [`Status`] of the check. + pub fn status(&self) -> Status { + self.inner.status() + } + + /// Report `info` for diagnostic purposes. + /// + /// See also: [`Report::info`]. + pub fn info(info: T) -> Self + where + T: std::fmt::Display + Serialize + 'static, + { + Self::new(Info(info)) + } + + /// Report an `error`. + /// + /// See also: [`Report::error`]. + pub fn error(error: T) -> Self + where + T: ToString, + { + Self::new(Error(error.to_string())) + } + + /// Assert that `old == new`. + /// + /// See also: [`Report::error`]. + pub fn equal(old: T, new: T) -> Self + where + T: std::fmt::Display + Serialize + PartialEq + 'static, + { + if old == new { + Self::new(Equal(old)) + } else { + Self::new(NotEqual { old, new }) + } + } + + /// Construct a new [`Check`] using [`tolerance::HigherIsBetter`]. + /// + /// See also: [`Report::higher_is_better`]. + pub fn higher_is_better(tolerance: &T, old: U, new: U, spec: tolerance::FmtSpec) -> Self + where + T: tolerance::HigherIsBetter, + { + tolerance.higher_is_better(old, new, spec) + } + + /// Construct a new [`Report`] using [`tolerance::LowerIsBetter`]. + /// + /// See also: [`Report::lower_is_better`]. + pub fn lower_is_better(tolerance: &T, old: U, new: U, spec: tolerance::FmtSpec) -> Self + where + T: tolerance::LowerIsBetter, + { + tolerance.lower_is_better(old, new, spec) + } + + fn new(inner: T) -> Self + where + T: Checkable + 'static, + { + Self { + inner: Box::new(inner), + } + } + + fn render(&self, ctx: FmtContext) -> String { + let mut s = String::new(); + + #[expect(clippy::expect_used)] + self.inner + .fmt(&mut s, ctx) + .expect("rust treats string formatting as infallible"); + s + } +} + +impl std::fmt::Debug for Check { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let inner = self.render(FmtContext::Tree); + let status = self.status(); + f.debug_struct("Check") + .field("kind", &inner) + .field("status", &status) + .finish() + } +} + +impl Serialize for Check { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut serializer = Some(serializer); + let mut result = None; + + self.inner.serialize(&mut |value| { + #[expect(clippy::expect_used)] + let serializer = serializer + .take() + .expect("serialization callback called more than once"); + result = Some(value.serialize(serializer)); + }); + + #[expect(clippy::expect_used)] + result.expect("serialization callback was not called") + } +} + +//-------// +// Equal // +//-------// + +#[derive(Debug)] +struct Equal(T); + +impl Checkable for Equal +where + T: std::fmt::Display + Serialize, +{ + fn status(&self) -> Status { + Status::Pass + } + + fn fmt(&self, f: &mut dyn std::fmt::Write, _ctx: FmtContext) -> std::fmt::Result { + write!(f, "{}", self.0) + } + + fn serialize(&self, f: &mut dyn FnMut(&dyn erased_serde::Serialize)) { + f(&SerEqual::pass(&self.0)) + } +} + +#[derive(Debug)] +struct NotEqual { + old: T, + new: T, +} + +impl Checkable for NotEqual +where + T: std::fmt::Display + Serialize, +{ + fn status(&self) -> Status { + Status::Fail + } + + fn fmt(&self, f: &mut dyn std::fmt::Write, ctx: FmtContext) -> std::fmt::Result { + write!( + f, + "(old) {} != (new) {} {}", + self.old, + self.new, + ctx.fail_marker() + ) + } + + fn serialize(&self, f: &mut dyn FnMut(&dyn erased_serde::Serialize)) { + f(&SerEqual::fail(&self.old, &self.new)) + } +} + +/// Serializer for [`Equal`] and [`NotEqual`]. +/// +/// If `new` is present, this is assumed to be a failure. +struct SerEqual<'a, T> { + old: &'a T, + new: Option<&'a T>, +} + +impl<'a, T> SerEqual<'a, T> { + fn pass(value: &'a T) -> Self { + Self { + old: value, + new: None, + } + } + + fn fail(old: &'a T, new: &'a T) -> Self { + Self { + old, + new: Some(new), + } + } +} + +impl Serialize for SerEqual<'_, T> +where + T: Serialize, +{ + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + use serde::ser::SerializeStruct; + + let (new, status) = match self.new { + Some(new) => (new, Status::Fail), + None => (self.old, Status::Pass), + }; + + let mut state = serializer.serialize_struct("Equal", 3)?; + state.serialize_field("old", self.old)?; + state.serialize_field("new", new)?; + state.serialize_field("status", &status)?; + state.end() + } +} + +//------// +// Info // +//------// + +#[derive(Debug)] +struct Info(T); + +impl Checkable for Info +where + T: std::fmt::Display + Serialize, +{ + fn status(&self) -> Status { + Status::Pass + } + + fn fmt(&self, f: &mut dyn std::fmt::Write, _ctx: FmtContext) -> std::fmt::Result { + write!(f, "{}", self.0) + } + + fn serialize(&self, f: &mut dyn FnMut(&dyn erased_serde::Serialize)) { + f(&self.0) + } +} + +//-------// +// Error // +//-------// + +#[derive(Debug)] +struct Error(String); + +impl Checkable for Error { + fn status(&self) -> Status { + Status::Fail + } + + fn fmt(&self, f: &mut dyn std::fmt::Write, ctx: FmtContext) -> std::fmt::Result { + write!(f, "{} {}", self.0, ctx.fail_marker()) + } + + fn serialize(&self, f: &mut dyn FnMut(&dyn erased_serde::Serialize)) { + struct Ser<'a>(&'a str); + + impl Serialize for Ser<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + use serde::ser::SerializeStruct; + + let mut state = serializer.serialize_struct("Error", 2)?; + state.serialize_field("error", self.0)?; + state.serialize_field("status", &Status::Fail)?; + state.end() + } + } + + f(&Ser(&self.0)) + } +} + +/// Are we rendering in a [`Table`] or as part of the normal tree rendering in [`Report`]? +#[derive(Debug, Clone, Copy)] +enum FmtContext { + Tree, + Table, +} + +impl FmtContext { + // Select how to mark failures depending on the context. + // + // In a table, we just use `!` since we already attach a row-level `FAIL`. + // + // This keeps things a little more compact horizontally. + fn fail_marker(&self) -> &'static str { + match self { + Self::Tree => "| FAIL", + Self::Table => "!", + } + } +} + +/// Trait object for checks. +trait Checkable { + fn status(&self) -> Status; + fn fmt(&self, f: &mut dyn std::fmt::Write, ctx: FmtContext) -> std::fmt::Result; + fn serialize(&self, f: &mut dyn FnMut(&dyn erased_serde::Serialize)); +} + +/////////// +// Tests // +/////////// + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_key_display_and_serialization() { + let cases = [ + (Key::Str("name"), "name", serde_json::json!("name")), + (Key::Position(42), "42", serde_json::json!(42)), + ( + Key::String(String::from("owned")), + "owned", + serde_json::json!("owned"), + ), + ]; + + for (key, rendered, serialized) in cases { + assert_eq!(key.to_string(), rendered); + assert_eq!(serde_json::to_value(&key).unwrap(), serialized); + } + } + + use diskann_utils::assert_contains; + + //-------// + // Check // + //-------// + + #[test] + fn test_check_info() { + let cases = [ + ( + Check::info(u64::MAX), + "18446744073709551615", + serde_json::json!(u64::MAX), + ), + (Check::info(0.25), "0.25", serde_json::json!(0.25)), + (Check::info(false), "false", serde_json::json!(false)), + ( + Check::info(String::from("first\nsecond")), + "first\nsecond", + serde_json::json!("first\nsecond"), + ), + ]; + + for (check, rendered, serialized) in cases { + assert_eq!(check.status(), Status::Pass); + assert_eq!(check.render(FmtContext::Tree), rendered); + assert_eq!(check.render(FmtContext::Table), rendered); + assert_eq!(serde_json::to_value(&check).unwrap(), serialized); + } + } + + #[test] + fn test_check_equal() { + let cases = [ + ( + Check::equal(u64::MAX, u64::MAX), + Status::Pass, + "18446744073709551615", + "18446744073709551615", + serde_json::json!({ + "old": u64::MAX, + "new": u64::MAX, + "status": "PASS", + }), + ), + ( + Check::equal(-1, 2), + Status::Fail, + "(old) -1 != (new) 2 | FAIL", + "(old) -1 != (new) 2 !", + serde_json::json!({ "old": -1, "new": 2, "status": "FAIL" }), + ), + ( + Check::equal("same", "same"), + Status::Pass, + "same", + "same", + serde_json::json!({ "old": "same", "new": "same", "status": "PASS" }), + ), + ( + Check::equal("before", "after"), + Status::Fail, + "(old) before != (new) after | FAIL", + "(old) before != (new) after !", + serde_json::json!({ "old": "before", "new": "after", "status": "FAIL" }), + ), + ]; + + for (check, status, tree, table, serialized) in cases { + assert_eq!(check.status(), status); + assert_eq!(check.render(FmtContext::Tree), tree); + assert_eq!(check.render(FmtContext::Table), table); + assert_eq!(serde_json::to_value(&check).unwrap(), serialized); + } + } + + #[test] + fn test_check_error() { + let cases = [ + ( + Check::error("invalid input"), + "invalid input | FAIL", + "invalid input !", + serde_json::json!({ "error": "invalid input", "status": "FAIL" }), + ), + ( + Check::error("invalid \"value\"\ntry again"), + "invalid \"value\"\ntry again | FAIL", + "invalid \"value\"\ntry again !", + serde_json::json!({ + "error": "invalid \"value\"\ntry again", + "status": "FAIL", + }), + ), + ]; + + for (check, tree, table, serialized) in cases { + assert_eq!(check.status(), Status::Fail); + assert_eq!(check.render(FmtContext::Tree), tree); + assert_eq!(check.render(FmtContext::Table), table); + assert_eq!(serde_json::to_value(&check).unwrap(), serialized); + } + } + + //-------// + // Table // + //-------// + + fn check_render(expected: &str, rendered: &str) { + // To support `expected` coming from raw strings, we remove leading and trailing + // newlines. + let expected = { + let temp = expected.strip_prefix('\n').unwrap_or(expected); + temp.strip_suffix('\n').unwrap_or(temp) + }; + + struct Pretty<'a> { + expected: &'a str, + rendered: &'a str, + } + + impl std::fmt::Display for Pretty<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!( + f, + "EXPECTED\n\n{}\n\nRENDERED\n\n{}", + self.expected, self.rendered + ) + } + } + + let pretty = Pretty { expected, rendered }; + + // Check number of lines first for better error diagnostics. + let expected_lines: Vec<&str> = expected.lines().collect(); + let rendered_lines: Vec<&str> = rendered.lines().collect(); + assert_eq!( + expected_lines.len(), + rendered_lines.len(), + "expected {} lines - got {}\n\n{}", + expected_lines.len(), + rendered_lines.len(), + pretty + ); + + for (i, (expected, rendered)) in + std::iter::zip(expected_lines.iter(), rendered_lines.iter()).enumerate() + { + // We trim the end because the expected tables trim trailing whitespace. + // + // We do keep leading whitespace, though, for slightly more accurate checks. + // + // Also, in the error message, we display the lines horizontally aligned to help + // find whitespace woes. + let expected = expected.trim_end(); + let rendered = rendered.trim_end(); + assert_eq!( + expected, + rendered, + "mismatch on line {} of {}\n\n{}", + i + 1, + expected_lines.len(), + pretty + ); + } + } + + #[test] + fn test_table_serialization() { + let mut table = Table::new(["name", "value"]); + for (name, check) in [("good", Check::info(3)), ("bad", Check::equal(1, 2))] { + let mut row = table.new_row(); + row.push(Check::info(name)); + row.push(check); + table.push_row(row).unwrap(); + } + + assert_eq!( + serde_json::to_value(&table).unwrap(), + serde_json::json!({ + "headers": ["name", "value"], + "columns": [ + ["good", "bad"], + [3, { "old": 1, "new": 2, "status": "FAIL" }], + ], + "row_status": ["PASS", "FAIL"], + "status": "FAIL", + }), + ); + } + + #[test] + fn test_table_rendering() { + let mut table = Table::new(["foo", "bar"]); + + let mut row = table.new_row(); + assert!(row.is_empty()); + assert_eq!(row.len(), 0); + + row.push(Check::info(10)); + assert!(!row.is_empty()); + assert_eq!(row.len(), 1); + + row.push(Check::info("hello")); + assert!(!row.is_empty()); + assert_eq!(row.len(), 2); + + table.push_row(row).unwrap(); + + let mut row = table.new_row(); + row.push(Check::info("world")); + row.push(Check::info(20)); + table.push_row(row).unwrap(); + + // Check that incorrect table lenghts are rejected. + assert_contains!( + table.push_row(Row::new()).unwrap_err().to_string(), + "finished row has 0 columns when 2 were expected" + ); + + assert!(table.status().is_success()); + let expected = " + foo, bar +============== + 10, hello +world, 20"; + + check_render(expected, &table.to_string()); + } + + #[test] + fn test_table_rendering_mixed_status() { + let mut table = Table::new(["value"]); + for check in [Check::equal(1, 2), Check::error("invalid"), Check::info(3)] { + let mut row = table.new_row(); + row.push(check); + table.push_row(row).unwrap(); + } + + assert!(table.status().is_fail()); + let expected = " + value, status +============================== +(old) 1 != (new) 2 !, FAIL + invalid !, FAIL + 3,"; + + check_render(expected, &table.to_string()); + check_render(expected, &table.finish().to_string()); + } + + #[test] + fn test_table_rendering_zero_rows() { + let table = Table::new(["foo", "bar"]); + + assert!(table.status().is_success()); + let expected = " +foo, bar +=========="; + + check_render(expected, &table.to_string()); + check_render(expected, &table.finish().to_string()); + } + + #[test] + fn test_table_rendering_zero_columns() { + let mut table = Table::new([] as [&str; 0]); + table.push_row(table.new_row()).unwrap(); + + assert!(table.status().is_success()); + check_render(EMPTY_TABLE, &table.to_string()); + check_render(EMPTY_TABLE, &table.finish().to_string()); + } + + #[test] + fn test_table_rendering_empty() { + let table = Table::new([] as [&str; 0]); + + assert!(table.status().is_success()); + check_render(EMPTY_TABLE, &table.to_string()); + check_render(EMPTY_TABLE, &table.finish().to_string()); + } + + #[test] + fn test_report_rendering() { + let mut good = Report::builder(); + good.push(0, Report::info("ready")); + good.push(String::from("count"), Report::equal(2, 2)); + + let mut mixed = Report::builder(); + mixed.push("keep", Report::info(7)); + mixed.push("bad", Report::equal(1, 2)); + mixed.push("err", Report::error("invalid")); + + { + let mut nested = Report::builder(); + nested.push("should be newline", Report::info(true)); + mixed.push("nested", nested.finish()); + } + + let mut table = Report::table(["value"]); + for check in [Check::equal(3, 3), Check::equal(1, 2)] { + let mut row = table.new_row(); + row.push(check); + table.push_row(row).unwrap(); + } + + let mut builder = Report::builder(); + builder.push("info", Report::info("example")); + builder.push("good", good.finish()); + builder.push("mixed", mixed.finish()); + builder.push("table", table.finish()); + builder.push("omitted", Builder::new().finish()); + let report = builder.finish(); + + assert!(report.status().is_fail()); + check_render( + " +info: example +good: + 0: ready + count: 2 +mixed: + keep: 7 + bad: (old) 1 != (new) 2 | FAIL + err: invalid | FAIL + nested: + should be newline: true +table: + value, status + ============================== + 3, + (old) 1 != (new) 2 !, FAIL", + &report.to_string(), + ); + check_render( + " +mixed: + bad: (old) 1 != (new) 2 | FAIL + err: invalid | FAIL +table: + value, status + ============================== + 3, + (old) 1 != (new) 2 !, FAIL", + &Finished::new(report, Rendering::FailOnly).to_string(), + ); + } + + #[test] + fn test_finished_report_rendering_success() { + let mut builder = Report::builder(); + builder.push("info", Report::info("ready")); + builder.push("same", Report::equal(2, 2)); + let report = builder.finish(); + + assert!(report.status().is_success()); + let expected = " +info: ready +same: 2"; + + check_render(expected, &Finished::new(report, Rendering::All).to_string()); + } +} diff --git a/diskann-benchmark-runner/src/report/tolerance.rs b/diskann-benchmark-runner/src/report/tolerance.rs new file mode 100644 index 0000000000..edbc46b53f --- /dev/null +++ b/diskann-benchmark-runner/src/report/tolerance.rs @@ -0,0 +1,1709 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + */ + +//! Tools to prevent regressions by checking against increases or decreases in values. +//! +//! Since benchmarks can be inherently noisy, these tools allow for some fuzziness in values. + +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use thiserror::Error; + +use crate::utils::num::NonNegativeFinite; + +use super::{Check, Status}; + +/// Perform a check against `old` and `new` with the semantics that "lower is better". +pub trait LowerIsBetter { + fn lower_is_better(&self, old: T, new: T, spec: FmtSpec) -> Check; +} + +/// Perform a check against `old` and `new` with the semantics that "higher is better". +pub trait HigherIsBetter { + fn higher_is_better(&self, old: T, new: T, spec: FmtSpec) -> Check; +} + +macro_rules! implement { + ($trait:ident, $method:ident, $implementor:ty) => { + implement!( + @all, + $trait, + $method, + $implementor, + { u8, u16, u32, u64, usize, i8, i16, i32, i64, isize, f32, f64 }, + ); + }; + (@all, $trait:ident, $method:ident, $implementor:ty, { $($types:ty),+ $(,)? } $(,)?) => { + $(implement!(@instance, $trait, $method, $implementor, $types);)+ + }; + (@instance, $trait:ident, $method:ident, $implementor:ty, $type:ty) => { + impl $trait<$type> for $implementor { + fn $method(&self, old: $type, new: $type, spec: FmtSpec) -> Check { + match new.change_from(&old) { + Ok(difference) => Tolerance::from(*self).check(difference, spec), + Err(err) => Check::error(&err), + } + } + } + }; +} + +macro_rules! higher_is_better { + ($T:ty) => { + implement!(HigherIsBetter, higher_is_better, $T); + }; +} + +macro_rules! lower_is_better { + ($T:ty) => { + implement!(LowerIsBetter, lower_is_better, $T); + }; +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct FmtSpec { + // The precision to use when displaying before and after values. + precision: Option, +} + +impl FmtSpec { + pub fn new() -> Self { + Self::default() + } + + pub fn with_precision(mut self, precision: u8) -> Self { + self.precision = Some(precision); + self + } +} + +trait FmtWith { + fn fmt_with(&self, f: &mut dyn std::fmt::Write, spec: &FmtSpec) -> std::fmt::Result; +} + +struct AsFmtWith<'a, T>(&'a T, &'a FmtSpec); + +impl std::fmt::Display for AsFmtWith<'_, T> +where + T: FmtWith, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt_with(f, self.1) + } +} + +impl FmtWith for f64 { + fn fmt_with(&self, f: &mut dyn std::fmt::Write, spec: &FmtSpec) -> std::fmt::Result { + let FmtSpec { precision } = spec; + match precision { + Some(p) => { + let p = usize::from(*p); + write!(f, "{:.p$}", self) + } + None => write!(f, "{}", self), + } + } +} + +macro_rules! fmt_with { + ($T:ty) => { + impl FmtWith for $T { + fn fmt_with(&self, f: &mut dyn std::fmt::Write, spec: &FmtSpec) -> std::fmt::Result { + let FmtSpec { precision: _ } = spec; + write!(f, "{}", self) + } + } + }; + ($($Ts:ty),+ $(,)?) => { + $(fmt_with!($Ts);)+ + } +} + +fmt_with!(u64, i64); + +//////////// +// Checks // +//////////// + +/// A limit on the relative decrease allowed. +/// +/// The limit is a floating point number in `[0, 1]` and a check from `old` to `new` is +/// successful if +/// ```math +/// new >= old * (1 - limit) +/// ``` +/// Supports Rust's primitive integer types up to 64 bits, including `usize`/`isize`, +/// and `f32`/`f64`. +/// +/// This check requires that `old` and `new` are positive and finite. +/// +/// # Precision +/// +/// Integer absolute differences are computed exactly. Floating-point differences are +/// computed in `f64` and are subject to floating-point rounding. +/// +/// The relative change magnitude (`absolute / old`) is computed by converting both +/// values to `f64` before division. +/// +/// If either `absolute` or `old` is an integer that cannot be represented exactly as +/// an `f64`, the conversion can lose precision. The division itself is subject to +/// floating-point rounding. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MaxRelativeDecrease { + #[serde(rename = "max-relative-decrease")] + limit: Fraction, +} + +impl MaxRelativeDecrease { + /// Construct a new [`MaxRelativeDecrease`]. + /// + /// Returns `None` if `limit` is not in the range `[0, 1]`. + /// + /// # Note + /// + /// This returns an `Option` to be more easily usable in const contexts. + pub const fn new(limit: f64) -> Option { + match Fraction::new(limit) { + Some(limit) => Some(Self { limit }), + None => None, + } + } + + const fn limit(&self) -> Fraction { + self.limit + } + + /// Return the numeric limit. + pub const fn as_f64(&self) -> f64 { + self.limit().value() + } +} + +higher_is_better!(MaxRelativeDecrease); + +/// A limit on the relative increase allowed. +/// +/// The limit is a floating point number in `[0, f64::MAX]` and a check from `old` to `new` +/// is successful if: +/// ```math +/// new <= old * (1 + limit) +/// ``` +/// Supports Rust's primitive integer types up to 64 bits, including `usize`/`isize`, +/// and `f32`/`f64`. +/// +/// This check requires that `old` and `new` are positive and finite. +/// +/// # Precision +/// +/// Integer absolute differences are computed exactly. Floating-point differences are +/// computed in `f64` and are subject to floating-point rounding. +/// +/// The relative change magnitude (`absolute / old`) is computed by converting both +/// values to `f64` before division. +/// +/// If either `absolute` or `old` is an integer that cannot be represented exactly as +/// an `f64`, the conversion can lose precision. The division itself is subject to +/// floating-point rounding. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MaxRelativeIncrease { + #[serde(rename = "max-relative-increase")] + limit: NonNegativeFinite, +} + +impl MaxRelativeIncrease { + /// Construct a new [`MaxRelativeIncrease`]. + /// + /// Returns `None` if `limit` is negative, infinite, or NaN. + /// + /// # Note + /// + /// This returns an `Option` to be unwrappable in a const context. + pub const fn new(limit: f64) -> Option { + match NonNegativeFinite::new(limit) { + Ok(limit) => Some(Self { limit }), + Err(_) => None, + } + } + + /// Return the numeric limit. + pub const fn limit(&self) -> NonNegativeFinite { + self.limit + } + + /// Return the numeric limit. + pub const fn as_f64(&self) -> f64 { + self.limit().get() + } +} + +lower_is_better!(MaxRelativeIncrease); + +/// A limit on the absolute decrease allowed. +/// +/// The limit is either a `u64` or a floating point number in `[0, f64::MAX]` and a check +/// from `old` to `new` is successful if: +/// ```math +/// new >= (old - limit) +/// ``` +/// Supports Rust's primitive integer types up to 64 bits, including `usize`/`isize`, +/// and `f32`/`f64`. +/// +/// # Precision +/// +/// Integer absolute differences are computed exactly. Floating-point differences are +/// an `f64`, the conversion can lose precision. The division itself is subject to +/// floating-point rounding. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MaxAbsoluteDecrease { + #[serde(rename = "max-absolute-decrease")] + limit: Absolute, +} + +impl MaxAbsoluteDecrease { + /// Construct a new [`MaxAbsoluteDecrease`] from `limit`. + pub const fn from_u64(limit: u64) -> Self { + Self::new(Absolute::Int(limit)) + } + + /// Construct a new [`MaxAbsoluteDecrease`] from `limit`. + /// + /// Returns `None` if `limit` is not in `[0, f64::MAX]`. + pub const fn try_from_f64(limit: f64) -> Option { + match NonNegativeFinite::new(limit) { + Ok(limit) => Some(Self::new(Absolute::Float(limit))), + Err(_) => None, + } + } + + /// Return the limit if it is an integer. + pub const fn as_u64(&self) -> Option { + self.limit.as_u64() + } + + /// Return the limit if it is a float. + pub const fn as_f64(&self) -> Option { + self.limit.as_f64() + } + + const fn new(limit: Absolute) -> Self { + Self { limit } + } + + const fn limit(&self) -> Absolute { + self.limit + } +} + +higher_is_better!(MaxAbsoluteDecrease); + +/// A limit on the absolute increase allowed. +/// +/// The limit is either a `u64` or a floating point number in `[0, f64::MAX]` and a check +/// from `old` to `new` is successful if: +/// ```math +/// new <= (old + limit) +/// ``` +/// Supports Rust's primitive integer types up to 64 bits, including `usize`/`isize`, +/// and `f32`/`f64`. +/// +/// # Precision +/// +/// Integer absolute differences are computed exactly. Floating-point differences are +/// an `f64`, the conversion can lose precision. The division itself is subject to +/// floating-point rounding. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MaxAbsoluteIncrease { + #[serde(rename = "max-absolute-increase")] + limit: Absolute, +} + +impl MaxAbsoluteIncrease { + /// Construct a new [`MaxAbsoluteIncrease`] from `limit`. + pub const fn from_u64(limit: u64) -> Self { + Self::new(Absolute::Int(limit)) + } + + /// Construct a new [`MaxAbsoluteIncrease`] from `limit`. + /// + /// Returns `None` if `limit` is not in `[0, f64::MAX]`. + pub const fn try_from_f64(limit: f64) -> Option { + match NonNegativeFinite::new(limit) { + Ok(limit) => Some(Self::new(Absolute::Float(limit))), + Err(_) => None, + } + } + + /// Return the limit if it is an integer. + pub const fn as_u64(&self) -> Option { + self.limit.as_u64() + } + + /// Return the limit if it is a float. + pub const fn as_f64(&self) -> Option { + self.limit.as_f64() + } + + const fn new(limit: Absolute) -> Self { + Self { limit } + } + + const fn limit(&self) -> Absolute { + self.limit + } +} + +lower_is_better!(MaxAbsoluteIncrease); + +//////////////////// +// Implementation // +//////////////////// + +#[derive(Debug, Clone, Copy, Serialize)] +#[serde(untagged)] +enum Tolerance { + RelativeDecrease(MaxRelativeDecrease), + AbsoluteDecrease(MaxAbsoluteDecrease), + RelativeIncrease(MaxRelativeIncrease), + AbsoluteIncrease(MaxAbsoluteIncrease), +} + +impl Tolerance { + /// Checking rules: + /// + /// * Max relative decrease: Succeeds if + /// ```math + /// new >= old * (1 - limit) --> limit >= -( (new - old) / old ) + /// ------------------- <-- Relative Change + /// ``` + /// Requires: + /// - `limit` is in `[0, 1]` (guaranteed by [`Fraction`]). + /// - `new` and `old` are both strictly positive and the relative change is finite + /// (guaranteed by [`Difference::relative`]). + /// + /// * Max absolute decrease: Succeeds if the absolute change is not decreasing or if it + /// decreases by no more than `limit`. + /// + /// Requires: + /// - `limit` to be non-negative and finite (guaranteed by [`Absolute`]). + /// + /// * Max relative increase: Succeeds if + /// ```math + /// new <= old * (1 + limit) --> limit >= ( (new - old) / old ) + /// ------------------- <-- Relative Change + /// ``` + /// Requires: + /// - `limit` is non-negative and finite (guaranteed by [`NonNegativeFinite`]). + /// - `new` and `old` are both strictly positive and the relative change is finite + /// (guaranteed by [`Difference::relative`]). + /// + /// * Max absolute increase: Succeeds if the absolute change is not increasing or if it + /// increases by no more than `limit`. + /// + /// Requires: + /// - `limit` to be non-negative and finite (guaranteed by [`Absolute`]). + fn check(self, difference: Difference, spec: FmtSpec) -> Check { + // Lazily evaluate the relative change, returning an error if it is not well formed. + macro_rules! check_relative { + ($difference:ident) => { + match $difference.relative() { + Ok(relative) => relative, + Err(err) => return Check::error(&err), + } + }; + } + + let (old, new, absolute) = difference.values(); + let (passed, relative) = match self { + Self::RelativeDecrease(limit) => { + let relative = check_relative!(difference); + let passed = limit.as_f64() >= -relative; + (passed, Some(relative)) + } + Self::AbsoluteDecrease(limit) => { + let passed = + absolute.direction != Direction::Decrease || !absolute.value.gt(limit.limit()); + (passed, None) + } + Self::RelativeIncrease(limit) => { + let relative = check_relative!(difference); + let passed = limit.as_f64() >= relative; + (passed, Some(relative)) + } + Self::AbsoluteIncrease(limit) => { + let passed = + absolute.direction != Direction::Increase || !absolute.value.gt(limit.limit()); + (passed, None) + } + }; + + let status = if passed { Status::Pass } else { Status::Fail }; + let comparison = Comparison { + old, + new, + absolute, + relative, + tolerance: self, + status, + spec, + }; + + Check::new(comparison) + } +} + +impl std::fmt::Display for Tolerance { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::RelativeDecrease(v) => write!(f, "min {:+}%", -v.as_f64() * 100.0), + Self::AbsoluteDecrease(v) => write!(f, "min -{}", v.limit()), + Self::RelativeIncrease(v) => write!(f, "max {:+}%", v.as_f64() * 100.0), + Self::AbsoluteIncrease(v) => write!(f, "max +{}", v.limit()), + } + } +} + +impl From for Tolerance { + fn from(tol: MaxRelativeDecrease) -> Self { + Self::RelativeDecrease(tol) + } +} + +impl From for Tolerance { + fn from(tol: MaxAbsoluteDecrease) -> Self { + Self::AbsoluteDecrease(tol) + } +} + +impl From for Tolerance { + fn from(tol: MaxRelativeIncrease) -> Self { + Self::RelativeIncrease(tol) + } +} + +impl From for Tolerance { + fn from(tol: MaxAbsoluteIncrease) -> Self { + Self::AbsoluteIncrease(tol) + } +} + +//----------// +// Fraction // +//----------// + +/// A number in `[0, 1]`. +#[derive(Debug, Clone, Copy)] +struct Fraction(f64); + +impl Fraction { + const fn new(value: f64) -> Option { + if value.is_nan() || value < 0.0 || value > 1.0 { + None + } else { + Some(Self(value)) + } + } + + const fn value(&self) -> f64 { + self.0 + } +} + +impl Serialize for Fraction { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_f64(self.0) + } +} + +impl<'de> Deserialize<'de> for Fraction { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = f64::deserialize(deserializer)?; + Self::new(value).ok_or_else(|| serde::de::Error::custom("fractions must be in [0, 1]")) + } +} + +//----------// +// Absolute // +//----------// + +/// A non-negative number. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(untagged)] +enum Absolute { + Int(u64), + Float(NonNegativeFinite), +} + +impl Absolute { + #[cfg(test)] + const fn try_from_f64(value: f64) -> Option { + match NonNegativeFinite::new(value) { + Ok(v) => Some(Self::Float(v)), + Err(_) => None, + } + } + + const fn as_u64(&self) -> Option { + if let Self::Int(v) = *self { + Some(v) + } else { + None + } + } + + const fn as_f64(&self) -> Option { + if let Self::Float(v) = *self { + Some(v.get()) + } else { + None + } + } + + /// Return `true` if the value of `self` is strictly greater than the value of `rhs`. + fn gt(self, rhs: Self) -> bool { + const _2_TO_THE_64: f64 = 18_446_744_073_709_551_616.0; + + match (self, rhs) { + (Self::Int(lhs), Self::Int(rhs)) => lhs > rhs, + (Self::Float(lhs), Self::Float(rhs)) => lhs.get() > rhs.get(), + (Self::Int(lhs), Self::Float(rhs)) => { + // We already know that `rhs` is non-negative and finite. + let rhs = rhs.get(); + + // If the floating point number is 2^64 or greater, there's no way `integer` + // can be greater than it. + if rhs >= _2_TO_THE_64 { + false + } else { + // Using truncation is correct. If the spacing between floating point + // numbers is fractional, it won't matter. + // + // If the spacing between floating point numbers is an integer, then + // truncating doesn't change the numeric value (provided we're less + // than 2^64, which we've already checked). + lhs > rhs as u64 + } + } + (Self::Float(lhs), Self::Int(rhs)) => { + // We already know that `lhs` is non-negative and finite. + let lhs = lhs.get(); + + // If the floating point number is 2^64 or greater, there's no way `integer` + // can be greater than or equal to it. + if lhs >= _2_TO_THE_64 { + true + } else { + // This bit is a little tricky. We can't throw away the fractional part. + // Instead, first compare the `truncated` `lhs` with the integer `rhs`. + // + // If that's true, we're done. Otherwise if `truncated` and `rhs` are + // equal, we need to check that the fractional part is non-zero, which + // would make `lhs` larger than `rhs`. + let truncated = lhs as u64; + (truncated > rhs) | (truncated == rhs && lhs.fract() != 0.0) + } + } + } + } +} + +impl std::fmt::Display for Absolute { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Int(v) => write!(f, "{}", v), + Self::Float(v) => write!(f, "{}", v.get()), + } + } +} + +impl From for Absolute { + fn from(value: u64) -> Self { + Self::Int(value) + } +} + +impl From for Absolute { + fn from(value: NonNegativeFinite) -> Self { + Self::Float(value) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize)] +enum Direction { + /// Closer to negative infinity. + Decrease, + /// The same. + Equal, + /// Closer to positive infinity. + Increase, +} + +impl Direction { + fn from_cmp(old: &T, new: &T) -> Self + where + T: Ord, + { + match new.cmp(old) { + std::cmp::Ordering::Less => Self::Decrease, + std::cmp::Ordering::Equal => Self::Equal, + std::cmp::Ordering::Greater => Self::Increase, + } + } +} + +#[derive(Debug, Clone, Copy)] +enum Difference { + PosInt { + absolute: u64, + direction: Direction, + old: u64, + new: u64, + }, + NegInt { + absolute: u64, + direction: Direction, + old: i64, + new: i64, + }, + Float { + absolute: NonNegativeFinite, + direction: Direction, + old: f64, + new: f64, + }, +} + +impl Difference { + /// Return the values of `old`, `new`, and absolute change. + fn values(&self) -> (Number, Number, AbsoluteChange) { + match *self { + Self::PosInt { + absolute, + direction, + old, + new, + } => ( + old.into(), + new.into(), + AbsoluteChange::new(absolute.into(), direction), + ), + Self::NegInt { + absolute, + direction, + old, + new, + } => ( + old.into(), + new.into(), + AbsoluteChange::new(absolute.into(), direction), + ), + Self::Float { + absolute, + direction, + old, + new, + } => ( + old.into(), + new.into(), + AbsoluteChange::new(absolute.into(), direction), + ), + } + } + + /// Return the relative difference between the old and new values by: + /// + /// ```math + /// (new - old) + /// ----------- + /// old + /// ``` + /// + /// This will generate an error under the following situations: + /// + /// * `old == 0`: This makes the computation ill-defined. + /// + /// * `new == 0`: While technically allowed mathematically, accepting `new == 0` in + /// checked-in baselines would cause future relative comparisons to fail due to + /// the first check. + /// + /// * Either `new` or `old` is negative: This messes with the interpretation of the + /// relative tolerance checks defined in [`self`] and so are rejected to ensure + /// consistent semantics. + /// + /// * Any of `new`, `old`, or the result are infinity or NaN. + fn relative(&self) -> Result { + fn check_sign_and_zero(old: T, new: T) -> Result<(), RelativeError> + where + T: Default + PartialOrd + Into, + { + if old == T::default() { + Err(RelativeError::OldZero) + } else if new == T::default() { + Err(RelativeError::NewZero) + } else if old < T::default() { + Err(RelativeError::OldNegative(old.into())) + } else if new < T::default() { + Err(RelativeError::NewNegative(new.into())) + } else { + Ok(()) + } + } + + fn check_finite(old: f64, new: f64) -> Result<(), RelativeError> { + if !old.is_finite() { + Err(RelativeError::OldNotFinite(old)) + } else if !new.is_finite() { + Err(RelativeError::NewNotFinite(new)) + } else { + Ok(()) + } + } + + fn relative(absolute: f64, old: f64, direction: Direction) -> Result { + let value = absolute / old; + if !value.is_finite() { + Err(RelativeError::RelativeNotFinite(value)) + } else { + match direction { + Direction::Decrease => Ok(-value), + Direction::Equal | Direction::Increase => Ok(value), + } + } + } + + match *self { + Self::PosInt { + absolute, + direction, + old, + new, + } => { + check_sign_and_zero(old, new)?; + relative(absolute as f64, old as f64, direction) + } + Self::NegInt { + absolute, + direction, + old, + new, + } => { + check_sign_and_zero(old, new)?; + relative(absolute as f64, old as f64, direction) + } + Self::Float { + absolute, + direction, + old, + new, + } => { + check_finite(old, new)?; + check_sign_and_zero(old, new)?; + relative(absolute.get(), old, direction) + } + } + } +} + +#[derive(Debug, Error)] +enum RelativeError { + #[error("old must be > 0 (got 0)")] + OldZero, + #[error("new must be > 0 (got 0)")] + NewZero, + #[error("old must be > 0 (got {0})")] + OldNegative(Number), + #[error("new must be > 0 (got {0})")] + NewNegative(Number), + #[error("old must be finite (got {0})")] + OldNotFinite(f64), + #[error("new must be finite (got {0})")] + NewNotFinite(f64), + #[error("relative change overflowed (got {0})")] + RelativeNotFinite(f64), +} + +trait ChangeFrom { + type Error: std::fmt::Display; + fn change_from(&self, old: &Self) -> Result; +} + +macro_rules! difference { + (@int $T:ident) => { + impl ChangeFrom for $T { + type Error = std::convert::Infallible; + fn change_from(&self, old: &Self) -> Result { + let old: i64 = *old as i64; + let new: i64 = *self as i64; + + Ok(Difference::NegInt { + absolute: new.abs_diff(old), + direction: Direction::from_cmp(&old, &new), + old, + new, + }) + } + } + }; + (@uint $T:ident) => { + impl ChangeFrom for $T { + type Error = std::convert::Infallible; + fn change_from(&self, old: &Self) -> Result { + let old: u64 = *old as u64; + let new: u64 = *self as u64; + + Ok(Difference::PosInt { + absolute: new.abs_diff(old), + direction: Direction::from_cmp(&old, &new), + old, + new, + }) + } + } + }; + (@float $T:ident) => { + impl ChangeFrom for $T { + type Error = FloatError; + fn change_from(&self, old: &Self) -> Result { + let old = *old as f64; + let new = *self as f64; + + if !old.is_finite() { + return Err(FloatError::Old(old)); + } + if !new.is_finite() { + return Err(FloatError::New(new)); + } + + let absolute = (new - old); + let direction = if absolute == 0.0 { + Direction::Equal + } else if absolute > 0.0 { + Direction::Increase + } else { + Direction::Decrease + }; + + let absolute = match NonNegativeFinite::new(absolute.abs()) { + Ok(absolute) => absolute, + Err(_) => return Err(FloatError::Diff(old, new)), + }; + + Ok(Difference::Float { + absolute, + direction, + old, + new, + }) + } + } + }; +} + +#[derive(Debug, Error)] +enum FloatError { + #[error("old must be finite (got {0})")] + Old(f64), + #[error("new must be finite (got {0})")] + New(f64), + #[error("absolute change overflowed (old {0}, new {1})")] + Diff(f64, f64), +} + +difference!(@int i8); +difference!(@int i16); +difference!(@int i32); +difference!(@int i64); +difference!(@int isize); + +difference!(@uint u8); +difference!(@uint u16); +difference!(@uint u32); +difference!(@uint u64); +difference!(@uint usize); + +difference!(@float f32); +difference!(@float f64); + +#[derive(Debug, Serialize)] +#[serde(untagged)] +enum Number { + PosInt(u64), + NegInt(i64), + Float(f64), +} + +impl std::fmt::Display for Number { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.fmt_with(f, &FmtSpec::default()) + } +} + +impl FmtWith for Number { + fn fmt_with(&self, f: &mut dyn std::fmt::Write, spec: &FmtSpec) -> std::fmt::Result { + match self { + Self::PosInt(v) => v.fmt_with(f, spec), + Self::NegInt(v) => v.fmt_with(f, spec), + Self::Float(v) => v.fmt_with(f, spec), + } + } +} + +impl From for Number { + fn from(value: u64) -> Self { + Self::PosInt(value) + } +} + +impl From for Number { + fn from(value: i64) -> Self { + Self::NegInt(value) + } +} + +impl From for Number { + fn from(value: f64) -> Self { + Self::Float(value) + } +} + +#[derive(Debug, Serialize)] +struct AbsoluteChange { + value: Absolute, + direction: Direction, +} + +impl AbsoluteChange { + fn new(value: Absolute, direction: Direction) -> Self { + Self { value, direction } + } +} + +impl FmtWith for AbsoluteChange { + fn fmt_with(&self, f: &mut dyn std::fmt::Write, spec: &FmtSpec) -> std::fmt::Result { + let sign = match self.direction { + Direction::Equal | Direction::Increase => "+", + Direction::Decrease => "-", + }; + + match &self.value { + Absolute::Int(v) => write!(f, "{}{}", sign, AsFmtWith(v, spec)), + Absolute::Float(v) => write!(f, "{}{}", sign, AsFmtWith(&v.get(), spec)), + } + } +} + +impl std::fmt::Display for AbsoluteChange { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.fmt_with(f, &FmtSpec::default()) + } +} + +//------------// +// Comparison // +//------------// + +#[derive(Debug, Serialize)] +struct Comparison { + old: Number, + new: Number, + absolute: AbsoluteChange, + relative: Option, + tolerance: Tolerance, + status: Status, + #[serde(skip)] + spec: FmtSpec, +} + +impl super::Checkable for Comparison { + fn status(&self) -> Status { + self.status + } + + fn fmt(&self, f: &mut dyn std::fmt::Write, ctx: super::FmtContext) -> std::fmt::Result { + let spec = &self.spec; + write!( + f, + "{} => {} (", + AsFmtWith(&self.old, spec), + AsFmtWith(&self.new, spec), + )?; + + match self.relative { + Some(relative) => write!(f, "{:+.3}%", relative * 100.0)?, + None => self.absolute.fmt_with(f, spec)?, + } + write!(f, ")")?; + + if self.status == Status::Fail { + write!(f, " [{}] {}", self.tolerance, ctx.fail_marker())?; + } + + Ok(()) + } + + fn serialize(&self, f: &mut dyn FnMut(&dyn erased_serde::Serialize)) { + f(&self) + } +} + +/////////// +// Tests // +/////////// + +#[cfg(test)] +mod tests { + use super::*; + + use std::assert_matches; + + // We test the public API using the resulting `Check`. + // + // This provides a nice way of ensuring the implementations are working as expected, + // generating the proper report, and viewing the layout of the reports. + // + // Tests begin with such integration style checks - more unit-style tests exercising + // numeric corner cases come later. + + #[test] + fn test_max_relative_decrease() { + use Status::{Fail, Pass}; + + // Constructors/Accessors + assert_eq!(MaxRelativeDecrease::new(-0.0).unwrap().as_f64(), 0.0); + assert!(MaxRelativeDecrease::new(-1.0).is_none()); + assert!(MaxRelativeDecrease::new(f64::INFINITY).is_none()); + assert!(MaxRelativeDecrease::new(f64::NEG_INFINITY).is_none()); + assert!(MaxRelativeDecrease::new(f64::NAN).is_none()); + assert_eq!(MaxRelativeDecrease::new(1.0).unwrap().as_f64(), 1.0); + assert!(MaxRelativeDecrease::new(1.1).is_none()); + + // Reports + let tolerance = MaxRelativeDecrease::new(0.25).unwrap(); + let spec = FmtSpec::new().with_precision(2); + let cases = [ + (8.0, 10.0, Pass, "8.00 => 10.00 (+25.000%)"), + (8.0, 8.0, Pass, "8.00 => 8.00 (+0.000%)"), + (8.0, 7.0, Pass, "8.00 => 7.00 (-12.500%)"), + (8.0, 6.0, Pass, "8.00 => 6.00 (-25.000%)"), + (8.0, 5.0, Fail, "8.00 => 5.00 (-37.500%) [min -25%] | FAIL"), + (0.0, 1.0, Fail, "old must be > 0 (got 0) | FAIL"), + (1.0, 0.0, Fail, "new must be > 0 (got 0) | FAIL"), + (-1.0, 1.0, Fail, "old must be > 0 (got -1) | FAIL"), + (1.0, -1.0, Fail, "new must be > 0 (got -1) | FAIL"), + ( + f64::INFINITY, + 1.0, + Fail, + "old must be finite (got inf) | FAIL", + ), + (1.0, f64::NAN, Fail, "new must be finite (got NaN) | FAIL"), + ( + f64::from_bits(1), + 1.0, + Fail, + "relative change overflowed (got inf) | FAIL", + ), + ]; + + for (old, new, status, expected) in cases { + let check = Check::higher_is_better(&tolerance, old, new, spec); + assert_eq!(check.status(), status, "old = {old}, new = {new}"); + assert_eq!(check.render(super::super::FmtContext::Tree), expected); + + let report = super::super::Report::higher_is_better(&tolerance, old, new, spec); + assert_eq!(report.status(), status, "old = {old}, new = {new}"); + assert_eq!(report.to_string(), expected); + } + } + + #[test] + fn test_max_relative_increase() { + use Status::{Fail, Pass}; + + // Constructors/Accessors + assert_eq!(MaxRelativeIncrease::new(-0.0).unwrap().as_f64(), 0.0); + assert!(MaxRelativeIncrease::new(-1.0).is_none()); + assert!(MaxRelativeIncrease::new(f64::INFINITY).is_none()); + assert!(MaxRelativeIncrease::new(f64::NEG_INFINITY).is_none()); + assert!(MaxRelativeIncrease::new(f64::NAN).is_none()); + assert_eq!(MaxRelativeIncrease::new(1.0).unwrap().as_f64(), 1.0); + assert_eq!(MaxRelativeIncrease::new(10.0).unwrap().as_f64(), 10.0); + + // Reports + let tolerance = MaxRelativeIncrease::new(0.25).unwrap(); + let spec = FmtSpec::new().with_precision(2); + let cases = [ + (8.0, 6.0, Pass, "8.00 => 6.00 (-25.000%)"), + (8.0, 8.0, Pass, "8.00 => 8.00 (+0.000%)"), + (8.0, 9.0, Pass, "8.00 => 9.00 (+12.500%)"), + (8.0, 10.0, Pass, "8.00 => 10.00 (+25.000%)"), + ( + 8.0, + 11.0, + Fail, + "8.00 => 11.00 (+37.500%) [max +25%] | FAIL", + ), + (0.0, 1.0, Fail, "old must be > 0 (got 0) | FAIL"), + (1.0, 0.0, Fail, "new must be > 0 (got 0) | FAIL"), + (-1.0, 1.0, Fail, "old must be > 0 (got -1) | FAIL"), + (1.0, -1.0, Fail, "new must be > 0 (got -1) | FAIL"), + (f64::NAN, 1.0, Fail, "old must be finite (got NaN) | FAIL"), + ( + 1.0, + f64::INFINITY, + Fail, + "new must be finite (got inf) | FAIL", + ), + ( + f64::from_bits(1), + 1.0, + Fail, + "relative change overflowed (got inf) | FAIL", + ), + ]; + + for (old, new, status, expected) in cases { + let check = Check::lower_is_better(&tolerance, old, new, spec); + assert_eq!(check.status(), status, "old = {old}, new = {new}"); + assert_eq!(check.render(super::super::FmtContext::Tree), expected); + + let report = super::super::Report::lower_is_better(&tolerance, old, new, spec); + assert_eq!(report.status(), status, "old = {old}, new = {new}"); + assert_eq!(report.to_string(), expected); + } + } + + #[test] + fn test_max_absolute_decrease() { + use Status::{Fail, Pass}; + + // Constructors/Accessors + assert_eq!( + MaxAbsoluteDecrease::try_from_f64(-0.0) + .unwrap() + .as_f64() + .unwrap(), + 0.0 + ); + assert!(MaxAbsoluteDecrease::try_from_f64(-1.0).is_none()); + assert!(MaxAbsoluteDecrease::try_from_f64(f64::INFINITY).is_none()); + assert!(MaxAbsoluteDecrease::try_from_f64(f64::NEG_INFINITY).is_none()); + assert!(MaxAbsoluteDecrease::try_from_f64(f64::NAN).is_none()); + assert!( + MaxAbsoluteDecrease::try_from_f64(1.0) + .unwrap() + .as_u64() + .is_none() + ); + + assert_eq!(MaxAbsoluteDecrease::from_u64(100).as_u64().unwrap(), 100); + assert!(MaxAbsoluteDecrease::from_u64(100).as_f64().is_none()); + + // Reports + let tolerance = MaxAbsoluteDecrease::try_from_f64(0.5).unwrap(); + let spec = FmtSpec::new().with_precision(2); + let cases = [ + (10.0, 11.0, Pass, "10.00 => 11.00 (+1.00)"), + (10.0, 10.0, Pass, "10.00 => 10.00 (+0.00)"), + (10.0, 9.75, Pass, "10.00 => 9.75 (-0.25)"), + (10.0, 9.5, Pass, "10.00 => 9.50 (-0.50)"), + (10.0, 9.25, Fail, "10.00 => 9.25 (-0.75) [min -0.5] | FAIL"), + (-1.0, -1.5, Pass, "-1.00 => -1.50 (-0.50)"), + (0.25, -0.25, Pass, "0.25 => -0.25 (-0.50)"), + (0.0, -0.5, Pass, "0.00 => -0.50 (-0.50)"), + (0.5, 0.0, Pass, "0.50 => 0.00 (-0.50)"), + ( + f64::INFINITY, + 1.0, + Fail, + "old must be finite (got inf) | FAIL", + ), + (1.0, f64::NAN, Fail, "new must be finite (got NaN) | FAIL"), + ]; + + for (old, new, status, expected) in cases { + let check = Check::higher_is_better(&tolerance, old, new, spec); + assert_eq!(check.status(), status, "old = {old}, new = {new}"); + assert_eq!(check.render(super::super::FmtContext::Tree), expected); + } + } + + #[test] + fn test_max_absolute_increase() { + use Status::{Fail, Pass}; + + // Constructors/Accessors + assert_eq!( + MaxAbsoluteIncrease::try_from_f64(-0.0) + .unwrap() + .as_f64() + .unwrap(), + 0.0 + ); + assert!(MaxAbsoluteIncrease::try_from_f64(-1.0).is_none()); + assert!(MaxAbsoluteIncrease::try_from_f64(f64::INFINITY).is_none()); + assert!(MaxAbsoluteIncrease::try_from_f64(f64::NEG_INFINITY).is_none()); + assert!(MaxAbsoluteIncrease::try_from_f64(f64::NAN).is_none()); + assert!( + MaxAbsoluteIncrease::try_from_f64(1.0) + .unwrap() + .as_u64() + .is_none() + ); + + assert_eq!(MaxAbsoluteIncrease::from_u64(100).as_u64().unwrap(), 100); + assert!(MaxAbsoluteIncrease::from_u64(100).as_f64().is_none()); + + // Reports + let tolerance = MaxAbsoluteIncrease::try_from_f64(0.5).unwrap(); + let spec = FmtSpec::new().with_precision(2); + let cases = [ + (10.0, 9.0, Pass, "10.00 => 9.00 (-1.00)"), + (10.0, 10.0, Pass, "10.00 => 10.00 (+0.00)"), + (10.0, 10.25, Pass, "10.00 => 10.25 (+0.25)"), + (10.0, 10.5, Pass, "10.00 => 10.50 (+0.50)"), + ( + 10.0, + 10.75, + Fail, + "10.00 => 10.75 (+0.75) [max +0.5] | FAIL", + ), + (-1.0, -0.5, Pass, "-1.00 => -0.50 (+0.50)"), + (-0.25, 0.25, Pass, "-0.25 => 0.25 (+0.50)"), + (0.0, 0.5, Pass, "0.00 => 0.50 (+0.50)"), + (-0.5, 0.0, Pass, "-0.50 => 0.00 (+0.50)"), + (f64::NAN, 1.0, Fail, "old must be finite (got NaN) | FAIL"), + ( + 1.0, + f64::INFINITY, + Fail, + "new must be finite (got inf) | FAIL", + ), + ]; + + for (old, new, status, expected) in cases { + let check = Check::lower_is_better(&tolerance, old, new, spec); + assert_eq!(check.status(), status, "old = {old}, new = {new}"); + assert_eq!(check.render(super::super::FmtContext::Tree), expected); + } + } + + //------------// + // Unit Tests // + //------------// + + #[test] + fn test_absolute_gt() { + use std::cmp::Ordering::{Equal, Greater, Less}; + + let int = Absolute::Int; + let float = |value| Absolute::try_from_f64(value).unwrap(); + let two_to_53 = 1u64 << 53; + let two_to_64 = 18_446_744_073_709_551_616.0f64; + let below_two_to_64 = two_to_64.next_down(); + + let cases = [ + (int(0), int(0), Equal), + (int(0), int(u64::MAX), Less), + (int(u64::MAX - 1), int(u64::MAX), Less), + (float(0.0), float(-0.0), Equal), + (float(0.0), float(f64::from_bits(1)), Less), + (float(1.0), float(1.0f64.next_up()), Less), + (int(0), float(-0.0), Equal), + (int(0), float(f64::from_bits(1)), Less), + (int(1), float(0.5), Greater), + (int(3), float(3.0), Equal), + (int(3), float(3.5), Less), + (int(4), float(3.5), Greater), + // Adjacent integers must remain distinct above f64's exact-integer range. + (int(two_to_53), float(two_to_53 as f64), Equal), + (int(two_to_53 + 1), float(two_to_53 as f64), Greater), + (int(two_to_53 + 1), float((two_to_53 + 2) as f64), Less), + // The last f64 below 2^64 is exactly 2^64 - 2048. + (int(u64::MAX - 2047), float(below_two_to_64), Equal), + (int(u64::MAX - 2048), float(below_two_to_64), Less), + (int(u64::MAX), float(below_two_to_64), Greater), + (int(u64::MAX), float(two_to_64), Less), + (int(u64::MAX), float(f64::MAX), Less), + ]; + + for (lhs, rhs, ordering) in cases { + assert_eq!(lhs.gt(rhs), ordering == Greater, "{lhs:?} > {rhs:?}"); + assert_eq!(rhs.gt(lhs), ordering == Less, "{rhs:?} > {lhs:?}"); + } + } + + #[test] + fn test_absolute_display() { + assert_eq!(Absolute::Int(10).to_string(), "10"); + assert_eq!(Absolute::try_from_f64(10.5).unwrap().to_string(), "10.5"); + } + + #[test] + fn test_absolute_change() { + let int = Absolute::Int; + let float = |value| Absolute::try_from_f64(value).unwrap(); + + let spec = FmtSpec::default().with_precision(1); + let cases = [ + (int(10), Direction::Increase, None, "+10"), + (int(0), Direction::Equal, None, "+0"), + // No formatting precision + (int(10), Direction::Decrease, None, "-10"), + (float(1.75), Direction::Increase, None, "+1.75"), + (float(0.0), Direction::Equal, None, "+0"), + (float(1.75), Direction::Decrease, None, "-1.75"), + // limit formatting precision + (float(1.75), Direction::Increase, Some(spec), "+1.8"), + (float(0.0), Direction::Equal, Some(spec), "+0.0"), + (float(1.75), Direction::Decrease, Some(spec), "-1.8"), + ]; + + for (value, direction, spec, expected) in cases { + let change = AbsoluteChange::new(value, direction); + match spec { + Some(spec) => assert_eq!(AsFmtWith(&change, &spec).to_string(), expected), + None => assert_eq!(change.to_string(), expected), + } + } + } + + #[test] + fn test_difference_integers() { + for (old, new, magnitude, direction) in [ + (0u64, 0u64, 0, Direction::Equal), + (0, u64::MAX, u64::MAX, Direction::Increase), + (u64::MAX, 0, u64::MAX, Direction::Decrease), + (u64::MAX - 1, u64::MAX, 1, Direction::Increase), + ] { + let difference = new.change_from(&old).unwrap(); + let Difference::PosInt { + absolute, + direction: got_direction, + old: got_old, + new: got_new, + } = difference + else { + panic!("expected unsigned difference, got {difference:?}"); + }; + assert_eq!((got_old, got_new, absolute), (old, new, magnitude)); + assert_eq!(got_direction, direction); + } + + for (old, new, magnitude, direction) in [ + (-5i64, -5i64, 0, Direction::Equal), + (-5, -2, 3, Direction::Increase), + (2, -5, 7, Direction::Decrease), + (i64::MIN, i64::MAX, u64::MAX, Direction::Increase), + (i64::MAX, i64::MIN, u64::MAX, Direction::Decrease), + ] { + let difference = new.change_from(&old).unwrap(); + let Difference::NegInt { + absolute, + direction: got_direction, + old: got_old, + new: got_new, + } = difference + else { + panic!("expected signed difference, got {difference:?}"); + }; + assert_eq!((got_old, got_new, absolute), (old, new, magnitude)); + assert_eq!(got_direction, direction); + } + } + + #[test] + fn test_difference_floats() { + for (old, new, magnitude, direction) in [ + (0.0f64, -0.0f64, 0.0, Direction::Equal), + (-1.5, 2.0, 3.5, Direction::Increase), + (2.0, -1.5, 3.5, Direction::Decrease), + ( + 0.0, + f64::from_bits(1), + f64::from_bits(1), + Direction::Increase, + ), + ] { + let difference = new.change_from(&old).unwrap(); + let Difference::Float { + absolute, + direction: got_direction, + old: got_old, + new: got_new, + } = difference + else { + panic!("expected floating-point difference, got {difference:?}"); + }; + assert_eq!((got_old, got_new, absolute.get()), (old, new, magnitude)); + assert_eq!(got_direction, direction); + } + + // f32 inputs must be widened before subtraction. + let difference = f32::MAX.change_from(&-f32::MAX).unwrap(); + let (_, _, absolute) = difference.values(); + assert_eq!(absolute.value.as_f64(), Some(2.0 * f64::from(f32::MAX))); + } + + #[test] + fn test_difference_float_errors() { + for invalid in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + assert_matches!(1.0f64.change_from(&invalid), Err(FloatError::Old(_))); + assert_matches!(invalid.change_from(&1.0), Err(FloatError::New(_))); + } + for (old, new) in [(-f64::MAX, f64::MAX), (f64::MAX, -f64::MAX)] { + assert_matches!(new.change_from(&old), Err(FloatError::Diff(..))); + } + } + + #[test] + fn test_difference_relative() { + for (old, new, expected) in [(8u64, 10u64, 0.25), (8, 6, -0.25), (8, 8, 0.0)] { + assert_eq!(new.change_from(&old).unwrap().relative().unwrap(), expected); + assert_eq!( + (new as i64) + .change_from(&(old as i64)) + .unwrap() + .relative() + .unwrap(), + expected + ); + assert_eq!( + (new as f64) + .change_from(&(old as f64)) + .unwrap() + .relative() + .unwrap(), + expected + ); + } + + // Compute the difference before converting large integers to f64. + let old = 1u64 << 53; + assert_eq!( + (old + 1).change_from(&old).unwrap().relative().unwrap(), + 1.0 / old as f64 + ); + } + + #[test] + fn test_difference_relative_errors() { + assert_matches!( + 1u64.change_from(&0).unwrap().relative(), + Err(RelativeError::OldZero) + ); + assert_matches!( + 0u64.change_from(&1).unwrap().relative(), + Err(RelativeError::NewZero) + ); + for (old, new) in [(-1i64, 1i64), (-2, -1)] { + assert_matches!( + new.change_from(&old).unwrap().relative(), + Err(RelativeError::OldNegative(_)) + ); + } + assert_matches!( + (-1i64).change_from(&1).unwrap().relative(), + Err(RelativeError::NewNegative(_)) + ); + + for zero in [0.0f64, -0.0] { + assert_matches!( + 1.0f64.change_from(&zero).unwrap().relative(), + Err(RelativeError::OldZero) + ); + assert_matches!( + zero.change_from(&1.0).unwrap().relative(), + Err(RelativeError::NewZero) + ); + } + assert_matches!( + 1.0f64.change_from(&-1.0).unwrap().relative(), + Err(RelativeError::OldNegative(_)) + ); + assert_matches!( + (-1.0f64).change_from(&1.0).unwrap().relative(), + Err(RelativeError::NewNegative(_)) + ); + assert_matches!( + f64::MAX.change_from(&f64::from_bits(1)).unwrap().relative(), + Err(RelativeError::RelativeNotFinite(_)) + ); + } + + #[test] + fn test_comparison_rendering() { + let spec = FmtSpec::new().with_precision(2); + let relative = MaxRelativeIncrease::new(0.05).unwrap(); + let absolute = MaxAbsoluteDecrease::try_from_f64(0.5).unwrap(); + let cases = [ + ( + relative.lower_is_better(100.0, 104.0, spec), + "100.00 => 104.00 (+4.000%)", + "100.00 => 104.00 (+4.000%)", + ), + ( + relative.lower_is_better(100.0, 106.0, spec), + "100.00 => 106.00 (+6.000%) [max +5%] | FAIL", + "100.00 => 106.00 (+6.000%) [max +5%] !", + ), + ( + absolute.higher_is_better(10.0, 9.75, spec), + "10.00 => 9.75 (-0.25)", + "10.00 => 9.75 (-0.25)", + ), + ( + absolute.higher_is_better(10.0, 9.0, spec), + "10.00 => 9.00 (-1.00) [min -0.5] | FAIL", + "10.00 => 9.00 (-1.00) [min -0.5] !", + ), + ]; + + for (check, tree, table) in cases { + assert_eq!(check.render(super::super::FmtContext::Tree), tree); + assert_eq!(check.render(super::super::FmtContext::Table), table); + } + } + + //---------------// + // Serialization // + //---------------// + + #[test] + fn test_fraction_deserialization() { + for (input, expected) in [ + (serde_json::json!(0), 0.0), + (serde_json::json!(-0.0), -0.0), + (serde_json::json!(0.25), 0.25), + (serde_json::json!(1), 1.0), + ] { + let fraction: Fraction = serde_json::from_value(input).unwrap(); + assert_eq!(fraction.value(), expected); + } + + for value in [-1.0, -f64::from_bits(1), 1.0f64.next_up(), 2.0] { + let error = serde_json::from_value::(serde_json::json!(value)).unwrap_err(); + assert!( + error.to_string().contains("fractions must be in [0, 1]"), + "value = {value}, error = {error}" + ); + } + + // JSON cannot represent nonfinite numbers, so deserialize them directly. + for value in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + let deserializer = + serde::de::value::F64Deserializer::::new(value); + let error = Fraction::deserialize(deserializer).unwrap_err(); + assert_eq!(error.to_string(), "fractions must be in [0, 1]"); + } + } + + #[test] + fn test_tolerance_serialization() { + let cases = [ + ( + Tolerance::from(MaxRelativeDecrease::new(0.25).unwrap()), + serde_json::json!({ "max-relative-decrease": 0.25 }), + ), + ( + Tolerance::from(MaxRelativeIncrease::new(0.5).unwrap()), + serde_json::json!({ "max-relative-increase": 0.5 }), + ), + ( + Tolerance::from(MaxAbsoluteDecrease::from_u64(u64::MAX)), + serde_json::json!({ "max-absolute-decrease": u64::MAX }), + ), + ( + Tolerance::from(MaxAbsoluteDecrease::try_from_f64(0.5).unwrap()), + serde_json::json!({ "max-absolute-decrease": 0.5 }), + ), + ( + Tolerance::from(MaxAbsoluteIncrease::from_u64(u64::MAX)), + serde_json::json!({ "max-absolute-increase": u64::MAX }), + ), + ( + Tolerance::from(MaxAbsoluteIncrease::try_from_f64(0.5).unwrap()), + serde_json::json!({ "max-absolute-increase": 0.5 }), + ), + ]; + + for (tolerance, expected) in cases { + assert_eq!(serde_json::to_value(tolerance).unwrap(), expected); + } + } + + #[test] + fn test_comparison_serialization() { + let spec = FmtSpec::new().with_precision(0); + let cases = [ + ( + MaxRelativeIncrease::new(0.25) + .unwrap() + .lower_is_better(8.0f64, 10.0, spec), + serde_json::json!({ + "old": 8.0, + "new": 10.0, + "absolute": { "value": 2.0, "direction": "Increase" }, + "relative": 0.25, + "tolerance": { "max-relative-increase": 0.25 }, + "status": "PASS", + }), + ), + ( + MaxRelativeDecrease::new(0.25) + .unwrap() + .higher_is_better(8u64, 5, spec), + serde_json::json!({ + "old": 8, + "new": 5, + "absolute": { "value": 3, "direction": "Decrease" }, + "relative": -0.375, + "tolerance": { "max-relative-decrease": 0.25 }, + "status": "FAIL", + }), + ), + ( + MaxAbsoluteDecrease::from_u64(u64::MAX - 1).higher_is_better( + i64::MAX, + i64::MIN, + spec, + ), + serde_json::json!({ + "old": i64::MAX, + "new": i64::MIN, + "absolute": { "value": u64::MAX, "direction": "Decrease" }, + "relative": null, + "tolerance": { "max-absolute-decrease": u64::MAX - 1 }, + "status": "FAIL", + }), + ), + ( + MaxAbsoluteIncrease::try_from_f64(0.5) + .unwrap() + .lower_is_better(1.25f64, 1.5, spec), + serde_json::json!({ + "old": 1.25, + "new": 1.5, + "absolute": { "value": 0.25, "direction": "Increase" }, + "relative": null, + "tolerance": { "max-absolute-increase": 0.5 }, + "status": "PASS", + }), + ), + ]; + + for (check, expected) in cases { + assert_eq!(serde_json::to_value(check).unwrap(), expected); + } + } +} diff --git a/diskann-benchmark-runner/src/utils/fmt.rs b/diskann-benchmark-runner/src/utils/fmt.rs index 3570af4eac..f7d9c9c1eb 100644 --- a/diskann-benchmark-runner/src/utils/fmt.rs +++ b/diskann-benchmark-runner/src/utils/fmt.rs @@ -462,10 +462,16 @@ impl std::fmt::Debug for MaybeLazy<'_> { /// ``` #[derive(Debug, Default)] pub struct KeyValue<'a> { - kv: Vec<(&'a str, MaybeLazy<'a>)>, + kv: Vec<(&'a str, Nesting, MaybeLazy<'a>)>, max_key_length: usize, } +#[derive(Debug, Clone, Copy)] +enum Nesting { + Auto, + Forced, +} + impl<'a> KeyValue<'a> { /// Create a new empty [`KeyValue`] formatter. pub fn new() -> Self { @@ -478,13 +484,13 @@ impl<'a> KeyValue<'a> { /// Push the key-value pair to `self` for formatting. pub fn push(&mut self, key: &'a str, value: &'a dyn std::fmt::Display) { self.max_key_length = self.max_key_length.max(key.len()); - self.kv.push((key, MaybeLazy::Lazy(value))) + self.kv.push((key, Nesting::Auto, MaybeLazy::Lazy(value))) } /// Push the key-value pair to `self` for formatting via [`std::fmt::Debug`]. pub fn push_debug(&mut self, key: &'a str, value: &'a dyn std::fmt::Debug) { self.max_key_length = self.max_key_length.max(key.len()); - self.kv.push((key, MaybeLazy::Debug(value))) + self.kv.push((key, Nesting::Auto, MaybeLazy::Debug(value))) } /// Push the key-value pair to `self` for formatting - eagerly formatting `value`. @@ -493,7 +499,28 @@ impl<'a> KeyValue<'a> { D: std::fmt::Display, { self.max_key_length = self.max_key_length.max(key.len()); - self.kv.push((key, MaybeLazy::Eager(value.to_string()))) + self.kv + .push((key, Nesting::Auto, MaybeLazy::Eager(value.to_string()))) + } + + /// Push the key-value pair to `self` for formatting - eagerly formatting `value`. + pub fn push_eager_nested(&mut self, key: &'a str, value: D) + where + D: std::fmt::Display, + { + self.max_key_length = self.max_key_length.max(key.len()); + self.kv + .push((key, Nesting::Forced, MaybeLazy::Eager(value.to_string()))) + } + + /// Return the number of pending key-value pairs. + pub fn len(&self) -> usize { + self.kv.len() + } + + /// Return `true` if there are no key-value pairs. + pub fn is_empty(&self) -> bool { + self.kv.is_empty() } } @@ -501,9 +528,14 @@ impl std::fmt::Display for KeyValue<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let width = self.max_key_length; let mut prefix = ""; - for (k, v) in self.kv.iter() { + for (k, nesting, v) in self.kv.iter() { let rendered = v.to_string(); - if rendered.contains('\n') { + let nest = match nesting { + Nesting::Forced => true, + Nesting::Auto => rendered.contains('\n'), + }; + + if nest { write!(f, "{}{}:\n{}", prefix, k, Indent::new(&rendered, 2))? } else { // Left-align the key and pad so that all single-line values line up in a diff --git a/diskann-benchmark/src/index/inmem2/benchmark/build.rs b/diskann-benchmark/src/index/inmem2/benchmark/build.rs index 05eec2ab45..8805176fa6 100644 --- a/diskann-benchmark/src/index/inmem2/benchmark/build.rs +++ b/diskann-benchmark/src/index/inmem2/benchmark/build.rs @@ -16,6 +16,10 @@ use std::{io::Write, num::NonZeroUsize, sync::Arc}; use diskann_benchmark_core::{self as dbc}; use diskann_benchmark_runner::{ self as dbr, + report::{ + tolerance::{FmtSpec, MaxRelativeIncrease}, + Report, + }, utils::{percentiles, MicroSeconds}, }; use diskann_inmem::{self as di}; @@ -58,6 +62,23 @@ pub(in crate::index::inmem2) mod dto { } } } + + #[derive(Debug, Clone, Copy, Serialize, Deserialize)] + #[serde(deny_unknown_fields)] + pub(in crate::index::inmem2) struct Tolerance { + /// This is the maximum amount of slow-down accepted. + pub(super) build_time: MaxRelativeIncrease, + } + + impl Tolerance { + pub(in crate::index::inmem2) fn example() -> Self { + const BUILD_TIME: MaxRelativeIncrease = MaxRelativeIncrease::new(0.05).unwrap(); + + Self { + build_time: BUILD_TIME, + } + } + } } /// A parsed [`diskann::graph::Config`] and more. @@ -328,6 +349,38 @@ impl std::fmt::Display for Stats { } } +//-------------// +// Regresssion // +//-------------// + +#[derive(Debug)] +pub(in crate::index::inmem2) struct Tolerance { + build_time: MaxRelativeIncrease, +} + +impl Tolerance { + pub(in crate::index::inmem2) fn from_raw(raw: &dto::Tolerance) -> Self { + let dto::Tolerance { build_time } = *raw; + Self { build_time } + } + + pub(in crate::index::inmem2) fn check(&self, old: &Stats, new: &Stats) -> Report { + let Self { build_time } = self; + let mut builder = Report::builder(); + builder.push("vectors", Report::equal(old.inserted, new.inserted)); + builder.push( + "build_time (s)", + Report::lower_is_better( + build_time, + old.total_time.as_seconds(), + new.total_time.as_seconds(), + FmtSpec::new().with_precision(3), + ), + ); + builder.finish() + } +} + /////////// // Tests // /////////// diff --git a/diskann-benchmark/src/index/inmem2/benchmark/knn.rs b/diskann-benchmark/src/index/inmem2/benchmark/knn.rs index 9e5b8cd545..6b8b276811 100644 --- a/diskann-benchmark/src/index/inmem2/benchmark/knn.rs +++ b/diskann-benchmark/src/index/inmem2/benchmark/knn.rs @@ -23,7 +23,15 @@ use std::{future::Future, io::Write, num::NonZeroUsize, sync::Arc}; use diskann::{utils::IntoUsize, ANNResult}; use diskann_benchmark_core::{self as dbc}; -use diskann_benchmark_runner::{self as dbr, files::InputFile, utils::MicroSeconds}; +use diskann_benchmark_runner::{ + self as dbr, + files::InputFile, + report::{ + tolerance::{FmtSpec, MaxAbsoluteDecrease, MaxRelativeDecrease}, + Check, Report, + }, + utils::MicroSeconds, +}; use diskann_inmem::{self as di}; use diskann_utils::views::rowmajor::{self, Matrix}; @@ -116,6 +124,27 @@ pub(in crate::index::inmem2) mod dto { } } } + + #[derive(Debug, Clone, Copy, Serialize, Deserialize)] + #[serde(deny_unknown_fields)] + pub(in crate::index::inmem2) struct Tolerance { + /// The amount of decrease in QPS that is acceptable. + pub(super) qps: MaxRelativeDecrease, + /// The amount of decrease in recall that is acceptable. + pub(super) recall: MaxAbsoluteDecrease, + } + + impl Tolerance { + pub(in crate::index::inmem2) fn example() -> Self { + const QPS: MaxRelativeDecrease = MaxRelativeDecrease::new(0.05).unwrap(); + const RECALL: MaxAbsoluteDecrease = MaxAbsoluteDecrease::try_from_f64(0.01).unwrap(); + + Self { + qps: QPS, + recall: RECALL, + } + } + } } /// Parameters for a single instance of KNN search and recall computation. @@ -679,6 +708,68 @@ impl dbc::search::Aggregate for Aggregator<'_> { } } +//------------// +// Regression // +//------------// + +#[derive(Debug)] +pub(in crate::index::inmem2) struct Tolerance { + qps: MaxRelativeDecrease, + recall: MaxAbsoluteDecrease, +} + +impl Tolerance { + pub(in crate::index::inmem2) fn from_raw(raw: &dto::Tolerance) -> Self { + let dto::Tolerance { qps, recall } = *raw; + Self { qps, recall } + } + + pub(in crate::index::inmem2) fn check(&self, old: &Stats, new: &Stats) -> Report { + let Stats(old) = old; + let Stats(new) = new; + + if old.len() != new.len() { + return Report::error(format_args!( + "mismatch number of runs - old = {}, new = {}", + old.len(), + new.len(), + )); + } + + let Self { qps, recall } = self; + let qps_spec = FmtSpec::new().with_precision(0); + let recall_spec = FmtSpec::new().with_precision(4); + + let headers = ["L", "K", "N", "Thr", "QPS (max)", "Recall"]; + let mut table = Report::table(headers); + for (old, new) in std::iter::zip(old.iter(), new.iter()) { + let mut row = table.new_row(); + row.push(Check::equal(old.search_l, new.search_l)); + row.push(Check::equal(old.recall_k, new.recall_k)); + row.push(Check::equal(old.search_n, new.search_n)); + row.push(Check::equal(old.num_tasks, new.num_tasks)); + + // We expect these fields to be non-empty. If they are empty, we receive 0.0, + // which gets propagated to the check and results in an error. + let old_qps = old.reps.iter().fold(0.0f64, |max, s| max.max(s.qps)); + let new_qps = new.reps.iter().fold(0.0f64, |max, s| max.max(s.qps)); + row.push(Check::higher_is_better(qps, old_qps, new_qps, qps_spec)); + + // Recall + row.push(Check::higher_is_better( + recall, + old.recall.average, + new.recall.average, + recall_spec, + )); + + table.push_row(row).unwrap(); + } + + table.finish() + } +} + /////////// // tests // /////////// diff --git a/diskann-benchmark/src/index/inmem2/benchmark/mod.rs b/diskann-benchmark/src/index/inmem2/benchmark/mod.rs index 49e3cd3688..8356a5605a 100644 --- a/diskann-benchmark/src/index/inmem2/benchmark/mod.rs +++ b/diskann-benchmark/src/index/inmem2/benchmark/mod.rs @@ -3,6 +3,8 @@ * Licensed under the MIT license. */ +use diskann_benchmark_runner::{self as dbr, report::Report}; + pub(in crate::index::inmem2) mod bigann; pub(in crate::index::inmem2) mod build; pub(in crate::index::inmem2) mod knn; @@ -16,3 +18,69 @@ pub(in crate::index::inmem2) struct BuildAndSearchStats { pub(in crate::index::inmem2) build: build::Stats, pub(in crate::index::inmem2) search: knn::Stats, } + +mod dto { + use super::*; + + #[derive(Debug, serde::Serialize, serde::Deserialize)] + #[serde(deny_unknown_fields)] + pub(in crate::index::inmem2) struct Tolerance { + pub(super) build: build::dto::Tolerance, + pub(super) search: knn::dto::Tolerance, + } + + impl Tolerance { + pub(in crate::index::inmem2) fn example() -> Self { + Self { + build: build::dto::Tolerance::example(), + search: knn::dto::Tolerance::example(), + } + } + } +} + +#[derive(Debug)] +pub(in crate::index::inmem2) struct Tolerance { + build: build::Tolerance, + search: knn::Tolerance, + // Used for serialization. + raw: dto::Tolerance, +} + +impl Tolerance { + pub(in crate::index::inmem2) fn check( + &self, + old: &BuildAndSearchStats, + new: &BuildAndSearchStats, + ) -> Report { + let mut builder = Report::builder(); + builder.push("build", self.build.check(&old.build, &new.build)); + builder.push("search", self.search.check(&old.search, &new.search)); + builder.finish() + } +} + +impl dbr::Input for Tolerance { + type Raw = dto::Tolerance; + + fn tag() -> &'static str { + "inmem2-build-and-search-tolerance" + } + + fn from_raw(raw: dto::Tolerance, _checker: &mut dbr::Checker) -> anyhow::Result { + let dto::Tolerance { build, search } = &raw; + Ok(Self { + build: build::Tolerance::from_raw(build), + search: knn::Tolerance::from_raw(search), + raw, + }) + } + + fn serialize(&self) -> anyhow::Result { + Ok(serde_json::to_value(&self.raw)?) + } + + fn example() -> Self::Raw { + dto::Tolerance::example() + } +} diff --git a/diskann-benchmark/src/index/inmem2/full.rs b/diskann-benchmark/src/index/inmem2/full.rs index 2ec9081bf2..984eec5c62 100644 --- a/diskann-benchmark/src/index/inmem2/full.rs +++ b/diskann-benchmark/src/index/inmem2/full.rs @@ -7,6 +7,7 @@ use std::io::Write; use diskann_benchmark_runner::{ self as dbr, + report::Finished, utils::{datatype::AsDataType, fmt::Quote}, }; use diskann_inmem::{self as di}; @@ -18,9 +19,10 @@ use crate::{ }; pub(super) fn register_benchmarks(registry: &mut dbr::Registry) -> anyhow::Result<()> { - registry.register("inmem2-f32", RunBuildAndSearch::::new())?; - registry.register("inmem2-f16", RunBuildAndSearch::::new())?; - registry.register("inmem2-u8", RunBuildAndSearch::::new())?; + registry.register_regression("inmem2-f32", RunBuildAndSearch::::new())?; + registry.register_regression("inmem2-f16", RunBuildAndSearch::::new())?; + registry.register_regression("inmem2-u8", RunBuildAndSearch::::new())?; + registry.register("inmem2-f32-stream", RunStream::::new())?; Ok(()) } @@ -194,6 +196,25 @@ where } } +impl dbr::benchmark::Regression for RunBuildAndSearch +where + T: di::repr::FullPrecision + diskann::graph::SampleableForStart + AsDataType, +{ + type Tolerances = inmem2::benchmark::Tolerance; + type Pass = Finished; + type Fail = Finished; + + fn check( + &self, + tolerances: &Self::Tolerances, + _input: &Self::Input, + before: &Self::Output, + after: &Self::Output, + ) -> anyhow::Result> { + Ok(tolerances.check(before, after).pass_fail()) + } +} + //-----------// // Streaming // //-----------// diff --git a/diskann-benchmark/src/index/inmem2/quantized/mod.rs b/diskann-benchmark/src/index/inmem2/quantized/mod.rs index 5aea80071b..bf4e1cf14a 100644 --- a/diskann-benchmark/src/index/inmem2/quantized/mod.rs +++ b/diskann-benchmark/src/index/inmem2/quantized/mod.rs @@ -9,7 +9,10 @@ use std::{io::Write, marker::PhantomData, sync::Arc}; -use diskann_benchmark_runner::{self as dbr}; +use diskann_benchmark_runner::{ + self as dbr, + report::{Finished, Report}, +}; use diskann_inmem::{self as di}; use diskann_utils::views::rowmajor::{self, Matrix}; use diskann_vector::distance::Metric; @@ -22,15 +25,15 @@ mod scalar; mod spherical; pub(super) fn register_benchmarks(registry: &mut dbr::Registry) -> anyhow::Result<()> { - registry.register( + registry.register_regression( "inmem2-sphq-static", RunBuildAndSearch::::new(), )?; - registry.register( + registry.register_regression( "inmem2-pq-static", RunBuildAndSearch::::new(), )?; - registry.register( + registry.register_regression( "inmem2-scq-static", RunBuildAndSearch::::new(), )?; @@ -260,6 +263,28 @@ fn run_quantized( Ok(inmem2::benchmark::BuildAndSearchStats { build, search }) } +impl dbr::benchmark::Regression for RunBuildAndSearch +where + Q: Quantizer, +{ + type Tolerances = inmem2::benchmark::Tolerance; + type Pass = Finished; + type Fail = Finished; + + fn check( + &self, + tolerances: &inmem2::benchmark::Tolerance, + _input: &BuildAndSearch, + before: &inmem2::benchmark::BuildAndSearchStats, + after: &inmem2::benchmark::BuildAndSearchStats, + ) -> anyhow::Result> { + let mut builder = Report::builder(); + builder.push("quantization", Report::info(Q::KIND)); + builder.push("result", tolerances.check(before, after)); + Ok(builder.finish().pass_fail()) + } +} + /////////// // Tests // /////////// diff --git a/diskann-inmem/integration/index/object.rs b/diskann-inmem/integration/index/object.rs index 560a4b4fc1..073f3579b3 100644 --- a/diskann-inmem/integration/index/object.rs +++ b/diskann-inmem/integration/index/object.rs @@ -10,14 +10,14 @@ use diskann::{ neighbor::Neighbor, utils::IntoUsize, }; -use diskann_benchmark_runner::utils::fmt::KeyValue; +use diskann_benchmark_runner::{report::Report, utils::fmt::KeyValue}; use serde::{Deserialize, Serialize}; use thiserror::Error; use diskann_inmem::{Context, Provider, Strategy, integration, repr}; use crate::support::{ - check::{CheckMatch, Match, check_all_fields}, + check::{CheckMatch, check_all_fields}, datatype::{AsDataType, FromSlice, Slice}, }; @@ -73,16 +73,13 @@ impl std::fmt::Display for KnnSearch { } impl CheckMatch for KnnSearch { - fn check_match(&self, previous: &Self) -> Match { + fn check_match(&self, previous: &Self) -> Report { let builder = check_all_fields!( self, previous, { hops, cmps }, ); - - builder.finish_with_remark(Some( - "check assumes deterministic (usually single-threaded) execution".into(), - )) + builder.finish() } } @@ -161,7 +158,7 @@ impl From for Counters { } impl CheckMatch for Counters { - fn check_match(&self, previous: &Self) -> Match { + fn check_match(&self, previous: &Self) -> Report { let builder = check_all_fields!( self, previous, @@ -175,10 +172,7 @@ impl CheckMatch for Counters { append_neighbors } ); - - builder.finish_with_remark(Some( - "check assumes deterministic (usually single-threaded) execution".into(), - )) + builder.finish() } } diff --git a/diskann-inmem/integration/index/runner.rs b/diskann-inmem/integration/index/runner.rs index d66aea4c07..a478010a60 100644 --- a/diskann-inmem/integration/index/runner.rs +++ b/diskann-inmem/integration/index/runner.rs @@ -11,6 +11,7 @@ use diskann_benchmark_runner::{ Checker, Checkpoint, Output, Registry, RegistryError, benchmark::{MatchContext, PassFail, Regression, Score}, files::InputFile, + report::{Finished, Report}, utils::fmt::Indent, }; use diskann_utils::views::rowmajor::{self, Matrix}; @@ -26,7 +27,7 @@ use diskann_inmem::{ use crate::{ index::{Counters, Index}, support::{ - check::{CheckMatch, Match, check_all_fields}, + check::{CheckMatch, check_all_fields}, datatype::{self, DataType, Dataset, DatasetView}, io::load_and_convert, tolerance, @@ -952,8 +953,8 @@ impl diskann_benchmark_runner::Benchmark for FullPrecision { impl Regression for FullPrecision { type Tolerances = tolerance::Empty; - type Pass = Match; - type Fail = Match; + type Pass = Finished; + type Fail = Finished; fn check( &self, @@ -990,7 +991,7 @@ impl std::fmt::Display for BuildAndSearch { } impl CheckMatch for BuildAndSearch { - fn check_match(&self, previous: &Self) -> Match { + fn check_match(&self, previous: &Self) -> Report { let builder = check_all_fields!( self, previous, diff --git a/diskann-inmem/integration/index/tests.rs b/diskann-inmem/integration/index/tests.rs index 52c2e6e6cd..52bb938d7a 100644 --- a/diskann-inmem/integration/index/tests.rs +++ b/diskann-inmem/integration/index/tests.rs @@ -5,14 +5,14 @@ use diskann::graph::search::Knn; use diskann_benchmark_core::recall::{RecallMetrics, Rows}; -use diskann_benchmark_runner::utils::fmt::KeyValue; +use diskann_benchmark_runner::{report::Report, utils::fmt::KeyValue}; use diskann_utils::views::rowmajor::{self, Matrix, MatrixMut}; use serde::{Deserialize, Serialize}; use crate::{ index::{Counters, Index, KnnSearch}, support::{ - check::{CheckMatch, Match, check_all_fields}, + check::{CheckMatch, check_all_fields}, datatype::DatasetView, }, }; @@ -95,7 +95,7 @@ impl From for KnnRecall { } impl CheckMatch for KnnRecall { - fn check_match(&self, previous: &Self) -> Match { + fn check_match(&self, previous: &Self) -> Report { let builder = check_all_fields!( self, previous, @@ -133,7 +133,7 @@ impl std::fmt::Display for KnnStats { } impl CheckMatch for KnnStats { - fn check_match(&self, previous: &Self) -> Match { + fn check_match(&self, previous: &Self) -> Report { let builder = check_all_fields!( self, previous, diff --git a/diskann-inmem/integration/support/check.rs b/diskann-inmem/integration/support/check.rs index 8e8ce2121c..a8fea75085 100644 --- a/diskann-inmem/integration/support/check.rs +++ b/diskann-inmem/integration/support/check.rs @@ -12,317 +12,33 @@ //! This module provides a means of aggregating all match failures (if any) and presenting //! all failures as a single unit. -use std::{ - borrow::Cow, - fmt::{Display, Write}, -}; - -use diskann_benchmark_runner::{benchmark::PassFail, utils::fmt::Table}; -use serde::{Serialize, Serializer}; +use diskann_benchmark_runner::report::{Check, Report}; /// Perform a baseline check on `self` and a previously saved result. pub(crate) trait CheckMatch { - fn check_match(&self, previous: &Self) -> Match; -} - -/// The result of a baseline check. -#[must_use = "this is a result type"] -#[derive(Debug, Serialize)] -#[serde(rename_all = "kebab-case")] -pub(crate) enum Match { - /// Successful match. - Ok, - - /// A mismatch on a specific field. - Mismatch { - got: String, - expected: String, - remark: Option>, - }, - - /// A collection of mismatches for an aggregate data type or collection. - /// - /// Use [`MatchBuilder`] to easier construction. - Nested { - children: Vec<(Key, Match)>, - remark: Option>, - }, -} - -impl Match { - /// Return `true` if `self` is [`Match::Ok`]. - #[must_use = "this has no side-effects"] - pub(crate) fn is_ok(&self) -> bool { - matches!(self, Self::Ok) - } - - /// Record a single mismatch between the retrieved value `got` and the `expected` result. - pub(crate) fn mismatch(got: &dyn Display, expected: &dyn Display) -> Self { - Self::mismatch_with_remark(got, expected, None) - } - - /// Record a single mismatch between the retrieved value `got` and the `expected` result - /// with an additional optional remark. - /// - /// The remark can be used for contexts where matches are more complex than simple - /// equality. - pub(crate) fn mismatch_with_remark( - got: &dyn Display, - expected: &dyn Display, - remark: Option>, - ) -> Self { - Self::Mismatch { - expected: expected.to_string(), - got: got.to_string(), - remark, - } - } - - /// Convert `self` into a [`PassFail`] for regression checks. - /// - /// Returns `PassFail::Pass` only if `self.is_ok`. - pub(crate) fn pass_fail(self) -> PassFail { - if self.is_ok() { - PassFail::Pass(self) - } else { - PassFail::Fail(self) - } - } -} - -impl std::fmt::Display for Match { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Ok => f.write_str("ok"), - Self::Mismatch { - got, - expected, - remark, - } => { - let header = ["got", "expected", "remark"]; - let mut table = Table::new(header, 1); - let mut row = table.row(0); - row.insert(got.clone(), 0); - row.insert(expected.clone(), 1); - if let Some(remark) = remark { - row.insert(remark.clone(), 2); - } - - table.fmt(f) - } - Self::Nested { children, remark } => { - let mut records = Vec::new(); - if let Some(remark) = remark { - records.push(Record { - path: String::new(), - got: "", - expected: "", - remark, - }); - } - - let mut buf = String::new(); - gather_mismatches(children, &mut records, Stack::new(&mut buf)); - - let mut table = Table::new(["path", "got", "expected", "remark"], records.len()); - for (i, r) in records.into_iter().enumerate() { - let mut row = table.row(i); - row.insert(r.path, 0); - row.insert(r.got.to_owned(), 1); - row.insert(r.expected.to_owned(), 2); - row.insert(r.remark.to_owned(), 3); - } - - table.fmt(f) - } - } - } -} - -fn gather_mismatches<'a>( - mismatches: &'a [(Key, Match)], - records: &mut Vec>, - mut path: Stack<'_>, -) { - for (k, m) in mismatches.iter() { - match m { - Match::Ok => continue, - Match::Mismatch { - got, - expected, - remark, - } => { - let record = Record { - path: path.push(k).get(), - got, - expected, - remark: remark.as_deref().unwrap_or(""), - }; - records.push(record); - } - Match::Nested { children, remark } => { - let path = path.push(k); - - if let Some(remark) = remark { - records.push(Record { - path: path.get(), - got: "", - expected: "", - remark, - }) - } - - gather_mismatches(children, records, path) - } - } - } -} - -#[derive(Debug)] -struct Stack<'a> { - s: &'a mut String, - len: usize, -} - -impl<'a> Stack<'a> { - fn new(s: &'a mut String) -> Self { - s.clear(); - Self { s, len: 0 } - } - - #[expect(clippy::unwrap_used, reason = "formatting shouldn't be failing here")] - fn push(&mut self, key: &Key) -> Stack<'_> { - let len = self.s.len(); - if len == 0 { - write!(self.s, "{}", key).unwrap(); - } else { - write!(self.s, ".{}", key).unwrap(); - } - - Stack { s: self.s, len } - } - - fn get(&self) -> String { - self.s.clone() - } -} - -impl Drop for Stack<'_> { - fn drop(&mut self) { - self.s.truncate(self.len) - } + fn check_match(&self, previous: &Self) -> Report; } -#[derive(Debug)] -struct Record<'a> { - path: String, - got: &'a str, - expected: &'a str, - remark: &'a str, -} - -///////// -// Key // -///////// - -/// A key to develop the full hierarchical path for a match. -/// -/// Keys can either be strings or positional indices. The latter are used when traversing -/// arrays. -#[derive(Debug, Clone, Eq, PartialEq)] -pub(crate) enum Key { - Str(&'static str), - Position(usize), - String(String), -} - -impl std::fmt::Display for Key { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Str(s) => f.write_str(s), - Self::Position(i) => write!(f, "{}", i), - Self::String(s) => f.write_str(s), - } - } -} - -impl Serialize for Key { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - match self { - Self::Str(s) => serializer.serialize_str(s), - Self::Position(i) => serializer.serialize_u64(*i as u64), - Self::String(s) => serializer.serialize_str(s), - } - } -} - -impl From<&'static str> for Key { - fn from(s: &'static str) -> Key { - Key::Str(s) - } -} - -impl From for Key { - fn from(i: usize) -> Key { - Key::Position(i) - } -} - -impl From for Key { - fn from(s: String) -> Key { - Key::String(s) - } -} - -///////////// -// Builder // -///////////// +macro_rules! check_all_fields { + ($self:expr, $prev:expr, { $($field:ident),+ $(,)? } $(,)?) => {{ + use diskann_benchmark_runner as dbr; -/// A utility for building a nested [`Match`]. -#[derive(Debug)] -pub(crate) struct MatchBuilder { - children: Vec<(Key, Match)>, + let Self { $($field),+ } = $self; + let mut builder = dbr::report::Report::builder(); + $( + builder.push( + stringify!($field), + <_ as $crate::support::check::CheckMatch>::check_match( + $field, + &$prev.$field, + ) + ); + )+ + builder + }}; } -impl MatchBuilder { - /// Construct a new empty collection of matches. - pub(crate) fn new() -> Self { - Self { - children: Vec::new(), - } - } - - /// Push the [`Match`] into the collection only if [`Match::is_ok`] fails. - pub(crate) fn push(&mut self, key: Key, child: Match) { - if !child.is_ok() { - self.children.push((key, child)); - } - } - - /// Package the collection of matches into a single [`Match`]. - /// - /// If no failing matches have been aggregated, returns [`Match::Ok`]. - pub(crate) fn finish(self) -> Match { - self.finish_with_remark(None) - } - - /// Package the collection of matches into a single [`Match`] with a remark. - /// - /// If no failing matches have been aggregated, returns [`Match::Ok`]. - pub(crate) fn finish_with_remark(self, remark: Option>) -> Match { - if self.children.is_empty() { - Match::Ok - } else { - Match::Nested { - children: self.children, - remark, - } - } - } -} +pub(crate) use check_all_fields; macro_rules! check_match_impl { ($T:ty) => { @@ -330,12 +46,8 @@ macro_rules! check_match_impl { fn check_match( &self, previous: &Self, - ) -> Match { - if self == previous { - Match::Ok - } else { - Match::mismatch(self, previous) - } + ) -> Report { + Check::equal(previous.clone(), self.clone()).into() } } }; @@ -345,25 +57,26 @@ macro_rules! check_match_impl { } check_match_impl!( - bool, u8, u16, u32, u64, usize, i8, i16, i32, i64, isize, f32, f64, &str, String + bool, u8, u16, u32, u64, usize, i8, i16, i32, i64, isize, f32, f64 ); impl CheckMatch for [T] where T: CheckMatch, { - fn check_match(&self, previous: &[T]) -> Match { + fn check_match(&self, previous: &[T]) -> Report { if self.len() != previous.len() { - return Match::mismatch_with_remark( - &self.len(), - &previous.len(), - Some("number of results is different between runs".into()), - ); + return Check::error(format_args!( + "number of results is different - old = {}, new = {}", + previous.len(), + self.len(), + )) + .into(); } - let mut builder = MatchBuilder::new(); + let mut builder = Report::builder(); for (i, (got, expected)) in std::iter::zip(self.iter(), previous.iter()).enumerate() { - builder.push(Key::from(i), got.check_match(expected)); + builder.push(i, got.check_match(expected)); } builder.finish() @@ -374,337 +87,7 @@ impl CheckMatch for Vec where T: CheckMatch, { - fn check_match(&self, previous: &Vec) -> Match { + fn check_match(&self, previous: &Vec) -> Report { self.as_slice().check_match(previous.as_slice()) } } - -//--------// -// Macros // -//--------// - -macro_rules! check_all_fields { - ($self:expr, $prev:expr, { $($field:ident),+ $(,)? } $(,)?) => {{ - let Self { $($field),+ } = $self; - let mut builder = $crate::support::check::MatchBuilder::new(); - $( - builder.push( - stringify!($field).into(), - <_ as $crate::support::check::CheckMatch>::check_match( - $field, - &$prev.$field - ), - ); - )+ - builder - }}; -} - -pub(crate) use check_all_fields; - -/////////// -// Tests // -/////////// - -#[cfg(test)] -mod tests { - use super::*; - - //-------// - // Match // - //-------// - - #[test] - fn match_is_ok() { - assert!(Match::Ok.is_ok()); - assert!(!Match::mismatch(&1, &2).is_ok()); - } - - #[test] - fn mismatch_records_got_and_expected() { - match Match::mismatch(&1, &2) { - Match::Mismatch { - got, - expected, - remark, - } => { - assert_eq!(got, "1"); - assert_eq!(expected, "2"); - assert!(remark.is_none()); - } - other => panic!("expected Mismatch, got {other:?}"), - } - } - - #[test] - fn mismatch_with_remark_records_remark() { - match Match::mismatch_with_remark(&"a", &"b", Some("note".into())) { - Match::Mismatch { - got, - expected, - remark, - } => { - assert_eq!(got, "a"); - assert_eq!(expected, "b"); - assert_eq!(remark.as_deref(), Some("note")); - } - other => panic!("expected Mismatch, got {other:?}"), - } - } - - #[test] - fn pass_fail_follows_is_ok() { - assert!(matches!(Match::Ok.pass_fail(), PassFail::Pass(Match::Ok))); - - assert!(matches!( - Match::mismatch(&1, &2).pass_fail(), - PassFail::Fail(Match::Mismatch { .. }) - )); - - let mut builder = MatchBuilder::new(); - builder.push(Key::from("test"), Match::mismatch(&1, &2)); - builder.push(Key::from("test2"), Match::mismatch(&2, &3)); - let mismatch = builder.finish(); - - assert!(matches!(mismatch, Match::Nested { .. })); - assert!(matches!( - mismatch.pass_fail(), - PassFail::Fail(Match::Nested { .. }) - )); - } - - //------------// - // CheckMatch // - //------------// - - #[test] - fn primitive_check_match() { - assert!(1u32.check_match(&1u32).is_ok()); - assert!(!2u32.check_match(&3u32).is_ok()); - assert!("x".check_match(&"x").is_ok()); - assert!(!"x".check_match(&"y").is_ok()); - } - - #[test] - fn slice_check_match_equal() { - let a = vec![1u32, 2, 3]; - let b = vec![1u32, 2, 3]; - assert!(a.check_match(&b).is_ok()); - } - - #[test] - fn slice_check_match_length_mismatch() { - let a = vec![1u32, 2, 3]; - let b = vec![1u32, 2]; - match a.check_match(&b) { - Match::Mismatch { - got, - expected, - remark, - } => { - assert_eq!(got, "3"); - assert_eq!(expected, "2"); - assert!(remark.is_some()); - } - other => panic!("expected length Mismatch, got {other:?}"), - } - } - - #[test] - fn slice_check_match_element_mismatch() { - let a = vec![1u32, 9, 3]; - let b = vec![1u32, 2, 3]; - match a.check_match(&b) { - Match::Nested { children, .. } => { - assert_eq!(children.len(), 1); - assert!(matches!(children[0].0, Key::Position(1))); - } - other => panic!("expected Nested, got {other:?}"), - } - } - - //--------------// - // MatchBuilder // - //--------------// - - #[test] - fn builder_empty_is_ok() { - assert!(MatchBuilder::new().finish().is_ok()); - } - - #[test] - fn builder_skips_ok_matches() { - let mut builder = MatchBuilder::new(); - builder.push("a".into(), Match::Ok); - builder.push("b".into(), Match::Ok); - assert!(builder.finish().is_ok()); - } - - #[test] - fn builder_collects_failures() { - let mut builder = MatchBuilder::new(); - builder.push("a".into(), Match::Ok); - builder.push("b".into(), Match::mismatch(&1, &2)); - match builder.finish() { - Match::Nested { children, remark } => { - assert_eq!(children.len(), 1); - assert!(remark.is_none()); - } - other => panic!("expected Nested, got {other:?}"), - } - } - - #[test] - fn builder_finish_with_remark() { - let mut builder = MatchBuilder::new(); - builder.push("b".into(), Match::mismatch(&1, &2)); - match builder.finish_with_remark(Some("ctx".into())) { - Match::Nested { remark, .. } => assert_eq!(remark.as_deref(), Some("ctx")), - other => panic!("expected Nested, got {other:?}"), - } - } - - //-----// - // Key // - //-----// - - #[test] - fn key_display() { - assert_eq!(Key::from("field").to_string(), "field"); - assert_eq!(Key::from(7usize).to_string(), "7"); - assert_eq!(Key::from(String::from("owned")).to_string(), "owned"); - } - - #[test] - fn key_serde() { - let k = serde_json::to_value(Key::Str("field")).unwrap(); - assert_eq!(k, serde_json::Value::String("field".into())); - - let k = serde_json::to_value(Key::Position(10)).unwrap(); - assert_eq!(k, serde_json::Value::Number(10.into())); - - let k = serde_json::to_value(Key::String("world".into())).unwrap(); - assert_eq!(k, serde_json::Value::String("world".into())); - } - - //---------// - // Display // - //---------// - - #[test] - fn display_ok() { - assert_eq!(Match::Ok.to_string(), "ok"); - } - - #[test] - fn display_nonnested() { - let mismatch = Match::mismatch_with_remark(&"hello", &1, Some("word".into())); - let rendered = mismatch.to_string(); - - let expected = r#" - got, expected, remark -=========================== -hello, 1, word -"#; - let expected = expected.strip_prefix('\n').unwrap(); - - println!("rendered = {:?}", rendered); - - let mut count = 0; - for (line, (got, expected)) in - std::iter::zip(rendered.lines(), expected.lines()).enumerate() - { - count += 1; - assert_eq!(got.trim(), expected.trim(), "failed on line {line}",); - } - assert_eq!(count, 3); - } - - #[test] - fn display_nested() { - // Build a nested match and ensure the hierarchical path is rendered. - let mut inner = MatchBuilder::new(); - inner.push(1usize.into(), Match::mismatch(&9, &2)); - inner.push( - "test".into(), - Match::mismatch_with_remark(&9, &2, Some("hello".into())), - ); - let nested = inner.finish_with_remark(Some("some remark".into())); - - let mut outer = MatchBuilder::new(); - outer.push("results".into(), nested); - let rendered = outer - .finish_with_remark(Some("final remarks".into())) - .to_string(); - - let expected = r#" - path, got, expected, remark - ================================================ - , , , final remarks - results, , , some remark - results.1, 9, 2, - results.test, 9, 2, hello - "#; - - let expected = expected.strip_prefix('\n').unwrap(); - - println!("rendered = {:?}", rendered); - - let mut count = 0; - for (line, (got, expected)) in - std::iter::zip(rendered.lines(), expected.lines()).enumerate() - { - count += 1; - assert_eq!(got.trim(), expected.trim(), "failed on line {line}",); - } - assert_eq!(count, 6); - } - - //-------------------// - // check_all_fields! // - //-------------------// - - #[derive(Debug)] - struct Sample { - a: u32, - b: String, - } - - impl CheckMatch for Sample { - fn check_match(&self, previous: &Self) -> Match { - check_all_fields!(self, previous, { a, b }).finish() - } - } - - #[test] - fn check_all_fields_equal() { - let x = Sample { - a: 1, - b: "hi".into(), - }; - let y = Sample { - a: 1, - b: "hi".into(), - }; - assert!(x.check_match(&y).is_ok()); - } - - #[test] - fn check_all_fields_reports_changed_field() { - let x = Sample { - a: 1, - b: "hi".into(), - }; - let y = Sample { - a: 1, - b: "bye".into(), - }; - match x.check_match(&y) { - Match::Nested { children, .. } => { - assert_eq!(children.len(), 1); - assert_eq!(children[0].0.to_string(), "b"); - } - other => panic!("expected Nested, got {other:?}"), - } - } -}