From ec68950d6900754edde98bf86a83a15c4f23eb45 Mon Sep 17 00:00:00 2001 From: Chimi6 <43803253+Chimi6@users.noreply.github.com> Date: Wed, 21 Oct 2026 02:54:55 -0600 Subject: [PATCH 1/2] perf: the registry is data; image pairs only for image conversions Each converter's name, formats, fidelity, and tier move from trait methods into a const fn entry() evaluated at compile time, so planning reads a static table and runs no converter's code. The 195 generated image pairs are built only when an image format is at one end (or the waypoint); a test checks every plan between formats is unchanged without them. sublime check z64 v64: 2.24 -> 1.95 MB; a 40 MB z64 -> v64 conversion: 2.66 -> 2.42 MB. --- src/cli/commands/batch.rs | 3 +- src/cli/commands/check.rs | 2 +- src/cli/commands/convert.rs | 9 +- src/cli/commands/merge.rs | 2 +- src/cli/commands/paths.rs | 29 +- src/converter/mod.rs | 38 +- src/converters/csv_to_json.rs | 45 +-- src/converters/csv_to_xlsx.rs | 46 ++- src/converters/docx_to_html.rs | 44 +-- src/converters/docx_to_markdown.rs | 47 ++- src/converters/docx_to_pages.rs | 42 +-- src/converters/docx_to_text.rs | 44 +-- src/converters/epub.rs | 80 ++--- src/converters/html_to_docx.rs | 47 ++- src/converters/html_to_markdown.rs | 47 ++- src/converters/html_to_pages.rs | 42 +-- src/converters/html_to_text.rs | 44 +-- src/converters/hub.rs | 41 +-- src/converters/image.rs | 62 ++-- src/converters/json_to_csv.rs | 80 ++--- src/converters/json_to_pages.rs | 44 +-- src/converters/json_to_toml.rs | 47 ++- src/converters/json_to_xml.rs | 47 ++- src/converters/json_to_yaml.rs | 44 +-- src/converters/markdown_json_to_markdown.rs | 36 +- src/converters/markdown_to_docx.rs | 44 +-- src/converters/markdown_to_html.rs | 48 ++- src/converters/markdown_to_json.rs | 46 +-- src/converters/markdown_to_pages.rs | 47 ++- src/converters/markdown_to_text.rs | 48 ++- src/converters/n64.rs | 45 ++- src/converters/numbers.rs | 41 +-- src/converters/ods.rs | 84 ++--- src/converters/odt.rs | 80 ++--- src/converters/pages_to_docx.rs | 47 ++- src/converters/pages_to_html.rs | 44 +-- src/converters/pages_to_json.rs | 44 +-- src/converters/pages_to_markdown.rs | 47 ++- src/converters/pages_to_text.rs | 44 +-- src/converters/pdf_to_document.rs | 35 +- src/converters/pdf_to_text.rs | 36 +- src/converters/rows.rs | 105 +++--- src/converters/rows_document.rs | 184 ++++------ src/converters/rtf.rs | 80 ++--- src/converters/text_to_docx.rs | 47 ++- src/converters/text_to_html.rs | 47 ++- src/converters/text_to_markdown.rs | 44 +-- src/converters/text_to_pages.rs | 42 +-- src/converters/to_numbers.rs | 47 ++- src/converters/to_pdf.rs | 35 +- src/converters/toml_to_json.rs | 47 ++- src/converters/xlsx_to_csv.rs | 79 ++--- src/converters/xml_to_json.rs | 47 ++- src/converters/yaml_to_json.rs | 47 ++- src/planner/chain.rs | 6 +- src/planner/execute.rs | 160 ++++----- src/planner/mod.rs | 139 ++++---- src/registry.rs | 373 ++++++++++++-------- tests/hub_pairs.rs | 22 +- tests/to_pdf.rs | 8 +- wasm/src/lib.rs | 3 +- 61 files changed, 1512 insertions(+), 1842 deletions(-) diff --git a/src/cli/commands/batch.rs b/src/cli/commands/batch.rs index 5ad48247..7c0f24a3 100644 --- a/src/cli/commands/batch.rs +++ b/src/cli/commands/batch.rs @@ -135,7 +135,8 @@ pub fn run(args: &ConvertArgs, renderer: &mut dyn Sink) -> Result render_human(&plan), diff --git a/src/cli/commands/convert.rs b/src/cli/commands/convert.rs index 399ca590..d1d6a21d 100644 --- a/src/cli/commands/convert.rs +++ b/src/cli/commands/convert.rs @@ -52,7 +52,12 @@ pub fn run( strict: args.strict, via, }; - let plan = planner::plan(registry::all_converters(), from, to, &plan_options)?; + let plan = planner::plan( + registry::converters_for(from, to, plan_options.via), + from, + to, + &plan_options, + )?; super::check_unused_options( args, @@ -225,7 +230,7 @@ fn convert_to_stdout( && let Some(last) = plan.hops.last() { context.loss( - last.name(), + last.name, crate::converter::Location::default(), format!( "stdout holds one part: wrote '{}' and left out {} more ({}); give an output file to write each, or pick one with --sheet or --page", diff --git a/src/cli/commands/merge.rs b/src/cli/commands/merge.rs index 6653ef3e..778886e8 100644 --- a/src/cli/commands/merge.rs +++ b/src/cli/commands/merge.rs @@ -107,7 +107,7 @@ fn write_pages( converter: converter.name, from: converter.from.id, to: converter.to.id, - fidelity: converter.fidelity.clone(), + fidelity: converter.fidelity, tier: Tier::Native, }], }); diff --git a/src/cli/commands/paths.rs b/src/cli/commands/paths.rs index 9f020024..52f20790 100644 --- a/src/cli/commands/paths.rs +++ b/src/cli/commands/paths.rs @@ -59,7 +59,7 @@ fn all_plans() -> Vec { } fn path_names(plan: &Plan) -> String { - let names: Vec<&str> = plan.hops.iter().map(|hop| hop.name()).collect(); + let names: Vec<&str> = plan.hops.iter().map(|hop| hop.name).collect(); names.join(" -> ") } @@ -133,11 +133,11 @@ pub fn render_markdown() -> String { let mut produced_by: Vec<&str> = Vec::new(); let mut consumed_by: Vec<&str> = Vec::new(); for converter in registry::all_converters() { - if converter.to().id == format.id { - produced_by.push(converter.name()); + if converter.to.id == format.id { + produced_by.push(converter.name); } - if converter.from().id == format.id { - consumed_by.push(converter.name()); + if converter.from.id == format.id { + consumed_by.push(converter.name); } } let line = format!( @@ -169,13 +169,13 @@ pub fn render_conversions() -> String { text.push_str("| Name | From | To | Tier | Fidelity | Notes |\n"); text.push_str("|---|---|---|---|---|---|\n"); for converter in registry::all_converters() { - let fidelity = converter.fidelity(); + let fidelity = converter.fidelity; let line = format!( "| {} | {} | {} | {} | {} | {} |\n", - converter.name(), - converter.from().id, - converter.to().id, - converter.tier().label(), + converter.name, + converter.from.id, + converter.to.id, + converter.tier.label(), fidelity.kind().label(), cell(fidelity.description().unwrap_or("")) ); @@ -221,10 +221,9 @@ fn render_mermaid() -> String { text.push_str("\n\n"); let mut lines: Vec<(String, String)> = Vec::new(); for converter in converters { - let inside = - converter.from().category == *category && converter.to().category == *category; + let inside = converter.from.category == *category && converter.to.category == *category; if inside { - add_line(&mut lines, converter.from().id, converter.to().id); + add_line(&mut lines, converter.from.id, converter.to.id); } } text.push_str("```mermaid\ngraph LR\n"); @@ -235,8 +234,8 @@ fn render_mermaid() -> String { // Across categories: the other side drawn as its category. let mut crossing: Vec<(String, String)> = Vec::new(); for converter in converters { - let from = converter.from(); - let to = converter.to(); + let from = converter.from; + let to = converter.to; if from.category == to.category { continue; } diff --git a/src/converter/mod.rs b/src/converter/mod.rs index 8d235da0..1392c761 100644 --- a/src/converter/mod.rs +++ b/src/converter/mod.rs @@ -8,7 +8,7 @@ use crate::event::Context; use crate::format::Format; /// The declared fidelity contract of a converter. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Fidelity { Lossless, /// Lossless for some inputs, lossy for others. The text says when. @@ -289,8 +289,6 @@ pub struct ConvertOptions { pub base: Option, } -/// One edge in the format graph. Implement this and add one line to -/// `src/registry.rs`. /// Where a converter that splits its input (a workbook's sheets, a /// document's tables, a PDF's pages) writes each part. pub trait Parts { @@ -299,12 +297,30 @@ pub trait Parts { fn part(&mut self, name: &str) -> io::Result<&mut dyn Write>; } +/// One edge in the format graph as the planner reads it: plain data, so +/// planning a conversion runs none of the converters' code. Each converter +/// builds its own at compile time (`entry`), and `src/registry.rs` lists +/// them. +#[derive(Clone, Copy)] +pub struct Entry { + pub name: &'static str, + pub from: &'static Format, + pub to: &'static Format, + pub fidelity: Fidelity, + pub tier: Tier, + pub converter: &'static dyn Converter, +} + +impl fmt::Debug for Entry { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.name) + } +} + +/// What a conversion does. Implement this, give the type a `const fn +/// entry` naming its formats and fidelity, and add the entry to +/// `src/registry.rs`. pub trait Converter: Sync { - fn name(&self) -> &'static str; - fn from(&self) -> &'static Format; - fn to(&self) -> &'static Format; - fn fidelity(&self) -> Fidelity; - fn tier(&self) -> Tier; fn convert( &self, input: Input<'_>, @@ -315,9 +331,9 @@ pub trait Converter: Sync { /// The options this conversion honours: by default what its input /// format honours read and its output format written. A pair that /// ignores one of them says so here. - fn options(&self) -> Vec { - let mut options = self.from().read_options.to_vec(); - options.extend_from_slice(self.to().write_options); + fn options(&self, from: &'static Format, to: &'static Format) -> Vec { + let mut options = from.read_options.to_vec(); + options.extend_from_slice(to.write_options); options } diff --git a/src/converters/csv_to_json.rs b/src/converters/csv_to_json.rs index 9a1c0caa..09165a7e 100644 --- a/src/converters/csv_to_json.rs +++ b/src/converters/csv_to_json.rs @@ -4,7 +4,7 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Tier}; use crate::event::Context; use crate::format::Format; use crate::format::formats; @@ -56,26 +56,6 @@ pub static TSV_TO_JSONL: CsvToJson = CsvToJson { }; impl Converter for CsvToJson { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - self.from - } - - fn to(&self) -> &'static Format { - self.to - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossless - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, input: Input<'_>, @@ -128,6 +108,19 @@ impl Converter for CsvToJson { } } +impl CsvToJson { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: self.from, + to: self.to, + fidelity: Fidelity::Lossless, + tier: Tier::Native, + converter: self, + } + } +} + fn write_object( name: &'static str, keys: &[PreparedKey], @@ -277,10 +270,10 @@ mod tests { #[test] fn declares_contract() { let converter = &CSV_TO_JSON; - assert_eq!(converter.name(), "csv-to-json"); - assert_eq!(converter.from().id, "csv"); - assert_eq!(converter.to().id, "json"); - assert_eq!(converter.fidelity(), Fidelity::Lossless); - assert_eq!(converter.tier(), Tier::Native); + assert_eq!(converter.entry().name, "csv-to-json"); + assert_eq!(converter.entry().from.id, "csv"); + assert_eq!(converter.entry().to.id, "json"); + assert_eq!(converter.entry().fidelity, Fidelity::Lossless); + assert_eq!(converter.entry().tier, Tier::Native); } } diff --git a/src/converters/csv_to_xlsx.rs b/src/converters/csv_to_xlsx.rs index 667a470f..d4cf44db 100644 --- a/src/converters/csv_to_xlsx.rs +++ b/src/converters/csv_to_xlsx.rs @@ -2,7 +2,7 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::event::Context; use crate::format::Format; use crate::format::formats; @@ -31,26 +31,6 @@ pub static TSV_TO_XLSX: CsvToXlsx = CsvToXlsx { }; impl Converter for CsvToXlsx { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - self.from - } - - fn to(&self) -> &'static Format { - &formats::XLSX - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, input: Input<'_>, @@ -82,15 +62,31 @@ impl Converter for CsvToXlsx { } } +impl CsvToXlsx { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: self.from, + to: &formats::XLSX, + fidelity: Fidelity::Conditional(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(CSV_TO_XLSX.name(), "csv-to-xlsx"); - assert_eq!(TSV_TO_XLSX.from().id, "tsv"); - assert_eq!(CSV_TO_XLSX.to().id, "xlsx"); - assert_eq!(CSV_TO_XLSX.fidelity(), Fidelity::Conditional(FIDELITY_NOTE)); + assert_eq!(CSV_TO_XLSX.entry().name, "csv-to-xlsx"); + assert_eq!(TSV_TO_XLSX.entry().from.id, "tsv"); + assert_eq!(CSV_TO_XLSX.entry().to.id, "xlsx"); + assert_eq!( + CSV_TO_XLSX.entry().fidelity, + Fidelity::Conditional(FIDELITY_NOTE) + ); } } diff --git a/src/converters/docx_to_html.rs b/src/converters/docx_to_html.rs index 6875cc78..8e4be892 100644 --- a/src/converters/docx_to_html.rs +++ b/src/converters/docx_to_html.rs @@ -3,10 +3,9 @@ use std::io::{Read, Write}; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Tier}; use crate::document::markdown::emit_events; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::docx::read_docx; use crate::io::html::HtmlWriter; @@ -17,26 +16,6 @@ const FIDELITY_NOTE: &str = "page layout, headers and footers, fonts, sizes, and pub struct DocxToHtml; impl Converter for DocxToHtml { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::DOCX - } - - fn to(&self) -> &'static Format { - &formats::HTML - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -57,15 +36,28 @@ impl Converter for DocxToHtml { } } +impl DocxToHtml { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::DOCX, + to: &formats::HTML, + fidelity: Fidelity::Lossy(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(DocxToHtml.name(), "docx-to-html"); - assert_eq!(DocxToHtml.from().id, "docx"); - assert_eq!(DocxToHtml.to().id, "html"); - assert_eq!(DocxToHtml.fidelity(), Fidelity::Lossy(FIDELITY_NOTE)); + assert_eq!(DocxToHtml.entry().name, "docx-to-html"); + assert_eq!(DocxToHtml.entry().from.id, "docx"); + assert_eq!(DocxToHtml.entry().to.id, "html"); + assert_eq!(DocxToHtml.entry().fidelity, Fidelity::Lossy(FIDELITY_NOTE)); } } diff --git a/src/converters/docx_to_markdown.rs b/src/converters/docx_to_markdown.rs index 5a322732..f88372fa 100644 --- a/src/converters/docx_to_markdown.rs +++ b/src/converters/docx_to_markdown.rs @@ -3,10 +3,9 @@ use std::io::{Read, Write}; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Tier}; use crate::document::markdown::emit_events; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::docx::read_docx; use crate::io::markdown::MarkdownWriter; @@ -17,26 +16,6 @@ const FIDELITY_NOTE: &str = "page layout, headers and footers, fonts, sizes, and pub struct DocxToMarkdown; impl Converter for DocxToMarkdown { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::DOCX - } - - fn to(&self) -> &'static Format { - &formats::MARKDOWN - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -57,15 +36,31 @@ impl Converter for DocxToMarkdown { } } +impl DocxToMarkdown { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::DOCX, + to: &formats::MARKDOWN, + fidelity: Fidelity::Lossy(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(DocxToMarkdown.name(), "docx-to-markdown"); - assert_eq!(DocxToMarkdown.from().id, "docx"); - assert_eq!(DocxToMarkdown.to().id, "markdown"); - assert_eq!(DocxToMarkdown.fidelity(), Fidelity::Lossy(FIDELITY_NOTE)); + assert_eq!(DocxToMarkdown.entry().name, "docx-to-markdown"); + assert_eq!(DocxToMarkdown.entry().from.id, "docx"); + assert_eq!(DocxToMarkdown.entry().to.id, "markdown"); + assert_eq!( + DocxToMarkdown.entry().fidelity, + Fidelity::Lossy(FIDELITY_NOTE) + ); } } diff --git a/src/converters/docx_to_pages.rs b/src/converters/docx_to_pages.rs index ae7c87bc..08697d8e 100644 --- a/src/converters/docx_to_pages.rs +++ b/src/converters/docx_to_pages.rs @@ -3,10 +3,9 @@ use std::io::{Read, Write}; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Tier}; use crate::converters::pages_to_json::package_error; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::docx::read_docx; use crate::io::pages::write_package_to; @@ -17,26 +16,6 @@ const FIDELITY_NOTE: &str = "paragraphs, headings, bulleted and numbered lists, pub struct DocxToPages; impl Converter for DocxToPages { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::DOCX - } - - fn to(&self) -> &'static Format { - &formats::PAGES - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -59,14 +38,27 @@ impl Converter for DocxToPages { } } +impl DocxToPages { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::DOCX, + to: &formats::PAGES, + fidelity: Fidelity::Lossy(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(DocxToPages.name(), "docx-to-pages"); - assert_eq!(DocxToPages.from().id, "docx"); - assert_eq!(DocxToPages.to().id, "pages"); + assert_eq!(DocxToPages.entry().name, "docx-to-pages"); + assert_eq!(DocxToPages.entry().from.id, "docx"); + assert_eq!(DocxToPages.entry().to.id, "pages"); } } diff --git a/src/converters/docx_to_text.rs b/src/converters/docx_to_text.rs index 070f4dee..13905c26 100644 --- a/src/converters/docx_to_text.rs +++ b/src/converters/docx_to_text.rs @@ -3,10 +3,9 @@ use std::io::{Read, Write}; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Tier}; use crate::document::markdown::emit_events; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::docx::read_docx; use crate::io::text::TextWriter; @@ -18,26 +17,6 @@ const FIDELITY_NOTE: &str = pub struct DocxToText; impl Converter for DocxToText { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::DOCX - } - - fn to(&self) -> &'static Format { - &formats::TEXT - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -58,15 +37,28 @@ impl Converter for DocxToText { } } +impl DocxToText { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::DOCX, + to: &formats::TEXT, + fidelity: Fidelity::Lossy(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(DocxToText.name(), "docx-to-text"); - assert_eq!(DocxToText.from().id, "docx"); - assert_eq!(DocxToText.to().id, "text"); - assert_eq!(DocxToText.fidelity(), Fidelity::Lossy(FIDELITY_NOTE)); + assert_eq!(DocxToText.entry().name, "docx-to-text"); + assert_eq!(DocxToText.entry().from.id, "docx"); + assert_eq!(DocxToText.entry().to.id, "text"); + assert_eq!(DocxToText.entry().fidelity, Fidelity::Lossy(FIDELITY_NOTE)); } } diff --git a/src/converters/epub.rs b/src/converters/epub.rs index 5a5b49b4..dab5303c 100644 --- a/src/converters/epub.rs +++ b/src/converters/epub.rs @@ -4,7 +4,7 @@ use std::io::{Read, Write}; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Tier}; use crate::converters::input::read_text_document; use crate::converters::pages_to_json::package_error; use crate::document::Document; @@ -92,26 +92,6 @@ fn malformed(message: String) -> ConvertError { } impl Converter for FromEpub { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - &formats::EPUB - } - - fn to(&self) -> &'static Format { - self.to - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(self.note) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -123,6 +103,19 @@ impl Converter for FromEpub { } } +impl FromEpub { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: &formats::EPUB, + to: self.to, + fidelity: Fidelity::Lossy(self.note), + tier: Tier::Native, + converter: self, + } + } +} + /// Renders `document` in `target`'s format. fn write_other( target: Other, @@ -244,26 +237,6 @@ to_epub!(HTML_TO_EPUB, "html-to-epub", HTML, Html, FROM_TEXT_NOTE); to_epub!(TEXT_TO_EPUB, "text-to-epub", TEXT, Text, FROM_TEXT_NOTE); impl Converter for ToEpub { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - self.from - } - - fn to(&self) -> &'static Format { - &formats::EPUB - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(self.note) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -281,6 +254,19 @@ impl Converter for ToEpub { } } +impl ToEpub { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: self.from, + to: &formats::EPUB, + fidelity: Fidelity::Lossy(self.note), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -296,9 +282,9 @@ mod tests { (&EPUB_TO_HTML, "html"), (&EPUB_TO_TEXT, "text"), ] { - assert_eq!(converter.from().id, "epub"); - assert_eq!(converter.to().id, to); - assert_eq!(converter.name(), format!("epub-to-{to}")); + assert_eq!(converter.entry().from.id, "epub"); + assert_eq!(converter.entry().to.id, to); + assert_eq!(converter.entry().name, format!("epub-to-{to}")); } for (converter, from) in [ (&DOCX_TO_EPUB, "docx"), @@ -309,9 +295,9 @@ mod tests { (&HTML_TO_EPUB, "html"), (&TEXT_TO_EPUB, "text"), ] { - assert_eq!(converter.from().id, from); - assert_eq!(converter.to().id, "epub"); - assert_eq!(converter.name(), format!("{from}-to-epub")); + assert_eq!(converter.entry().from.id, from); + assert_eq!(converter.entry().to.id, "epub"); + assert_eq!(converter.entry().name, format!("{from}-to-epub")); } } } diff --git a/src/converters/html_to_docx.rs b/src/converters/html_to_docx.rs index 9367d109..1787c05a 100644 --- a/src/converters/html_to_docx.rs +++ b/src/converters/html_to_docx.rs @@ -4,11 +4,10 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::converters::input::read_text_document; use crate::document::from_events::DocumentBuilder; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::docx::DocxStream; use crate::io::html::reader::parse_into; @@ -19,26 +18,6 @@ const FIDELITY_NOTE: &str = "lossless for headings, paragraphs, lists, quotes, c pub struct HtmlToDocx; impl Converter for HtmlToDocx { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::HTML - } - - fn to(&self) -> &'static Format { - &formats::DOCX - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -57,15 +36,31 @@ impl Converter for HtmlToDocx { } } +impl HtmlToDocx { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::HTML, + to: &formats::DOCX, + fidelity: Fidelity::Conditional(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(HtmlToDocx.name(), "html-to-docx"); - assert_eq!(HtmlToDocx.from().id, "html"); - assert_eq!(HtmlToDocx.to().id, "docx"); - assert_eq!(HtmlToDocx.fidelity(), Fidelity::Conditional(FIDELITY_NOTE)); + assert_eq!(HtmlToDocx.entry().name, "html-to-docx"); + assert_eq!(HtmlToDocx.entry().from.id, "html"); + assert_eq!(HtmlToDocx.entry().to.id, "docx"); + assert_eq!( + HtmlToDocx.entry().fidelity, + Fidelity::Conditional(FIDELITY_NOTE) + ); } } diff --git a/src/converters/html_to_markdown.rs b/src/converters/html_to_markdown.rs index f909e38b..40fccbd5 100644 --- a/src/converters/html_to_markdown.rs +++ b/src/converters/html_to_markdown.rs @@ -3,10 +3,9 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::converters::input::read_text_document; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::html::reader::parse_into; use crate::io::markdown::MarkdownWriter; @@ -17,26 +16,6 @@ const FIDELITY_NOTE: &str = "scripts, styles, forms, and layout are dropped; ele pub struct HtmlToMarkdown; impl Converter for HtmlToMarkdown { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::HTML - } - - fn to(&self) -> &'static Format { - &formats::MARKDOWN - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -52,15 +31,31 @@ impl Converter for HtmlToMarkdown { } } +impl HtmlToMarkdown { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::HTML, + to: &formats::MARKDOWN, + fidelity: Fidelity::Lossy(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(HtmlToMarkdown.name(), "html-to-markdown"); - assert_eq!(HtmlToMarkdown.from().id, "html"); - assert_eq!(HtmlToMarkdown.to().id, "markdown"); - assert_eq!(HtmlToMarkdown.fidelity(), Fidelity::Lossy(FIDELITY_NOTE)); + assert_eq!(HtmlToMarkdown.entry().name, "html-to-markdown"); + assert_eq!(HtmlToMarkdown.entry().from.id, "html"); + assert_eq!(HtmlToMarkdown.entry().to.id, "markdown"); + assert_eq!( + HtmlToMarkdown.entry().fidelity, + Fidelity::Lossy(FIDELITY_NOTE) + ); } } diff --git a/src/converters/html_to_pages.rs b/src/converters/html_to_pages.rs index fe7b7cda..61515792 100644 --- a/src/converters/html_to_pages.rs +++ b/src/converters/html_to_pages.rs @@ -3,12 +3,11 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::converters::input::read_text_document; use crate::converters::pages_to_json::package_error; use crate::document::from_events::DocumentBuilder; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::html::reader::parse_into; use crate::io::pages::write_package_to; @@ -19,26 +18,6 @@ const FIDELITY_NOTE: &str = "paragraphs, headings, bulleted and numbered lists, pub struct HtmlToPages; impl Converter for HtmlToPages { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::HTML - } - - fn to(&self) -> &'static Format { - &formats::PAGES - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -60,14 +39,27 @@ impl Converter for HtmlToPages { } } +impl HtmlToPages { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::HTML, + to: &formats::PAGES, + fidelity: Fidelity::Lossy(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(HtmlToPages.name(), "html-to-pages"); - assert_eq!(HtmlToPages.from().id, "html"); - assert_eq!(HtmlToPages.to().id, "pages"); + assert_eq!(HtmlToPages.entry().name, "html-to-pages"); + assert_eq!(HtmlToPages.entry().from.id, "html"); + assert_eq!(HtmlToPages.entry().to.id, "pages"); } } diff --git a/src/converters/html_to_text.rs b/src/converters/html_to_text.rs index 0d9a5ee0..825f3e4e 100644 --- a/src/converters/html_to_text.rs +++ b/src/converters/html_to_text.rs @@ -3,10 +3,9 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::converters::input::read_text_document; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::html::reader::parse_into; use crate::io::text::TextWriter; @@ -17,26 +16,6 @@ const FIDELITY_NOTE: &str = "markup, scripts, styles, and image sources are drop pub struct HtmlToText; impl Converter for HtmlToText { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::HTML - } - - fn to(&self) -> &'static Format { - &formats::TEXT - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -52,15 +31,28 @@ impl Converter for HtmlToText { } } +impl HtmlToText { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::HTML, + to: &formats::TEXT, + fidelity: Fidelity::Lossy(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(HtmlToText.name(), "html-to-text"); - assert_eq!(HtmlToText.from().id, "html"); - assert_eq!(HtmlToText.to().id, "text"); - assert_eq!(HtmlToText.fidelity(), Fidelity::Lossy(FIDELITY_NOTE)); + assert_eq!(HtmlToText.entry().name, "html-to-text"); + assert_eq!(HtmlToText.entry().from.id, "html"); + assert_eq!(HtmlToText.entry().to.id, "text"); + assert_eq!(HtmlToText.entry().fidelity, Fidelity::Lossy(FIDELITY_NOTE)); } } diff --git a/src/converters/hub.rs b/src/converters/hub.rs index 8687250e..9e3ad0fb 100644 --- a/src/converters/hub.rs +++ b/src/converters/hub.rs @@ -5,7 +5,7 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Tier}; use crate::converters::input::read_text_document; use crate::event::Context; use crate::format::Format; @@ -30,26 +30,6 @@ pub struct HubPair { } impl Converter for HubPair { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - self.from - } - - fn to(&self) -> &'static Format { - self.to - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(self.note) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, input: Input<'_>, @@ -61,6 +41,19 @@ impl Converter for HubPair { } } +impl HubPair { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: self.from, + to: self.to, + fidelity: Fidelity::Conditional(self.note), + tier: Tier::Native, + converter: self, + } + } +} + /// Reads the input into a tree; what the reader could not keep is /// reported as losses. fn read( @@ -243,10 +236,10 @@ mod tests { ]; for pair in pairs { assert_eq!( - pair.name(), - format!("{}-to-{}", pair.from().id, pair.to().id) + pair.entry().name, + format!("{}-to-{}", pair.entry().from.id, pair.entry().to.id) ); - assert!(matches!(pair.fidelity(), Fidelity::Conditional(_))); + assert!(matches!(pair.entry().fidelity, Fidelity::Conditional(_))); } } } diff --git a/src/converters/image.rs b/src/converters/image.rs index 642d2d5a..95698477 100644 --- a/src/converters/image.rs +++ b/src/converters/image.rs @@ -6,7 +6,9 @@ use std::io::{Read, Write}; use std::sync::OnceLock; -use crate::converter::{ConvertError, Converter, Fidelity, FidelityKind, Input, Location, Tier}; +use crate::converter::{ + ConvertError, Converter, Entry, Fidelity, FidelityKind, Input, Location, Tier, +}; use crate::event::Context; use crate::format::Format; use crate::format::formats; @@ -47,20 +49,8 @@ pub struct ImagePair { } impl Converter for ImagePair { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - self.from - } - - fn to(&self) -> &'static Format { - self.to - } - /// Images set on a PDF page have no text, so no `--font`. - fn options(&self) -> Vec { + fn options(&self, _from: &'static Format, _to: &'static Format) -> Vec { let mut options = self.from.read_options.to_vec(); options.extend( self.to.write_options.iter().filter(|option| { @@ -70,14 +60,6 @@ impl Converter for ImagePair { options } - fn fidelity(&self) -> Fidelity { - self.fidelity.clone() - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -176,6 +158,19 @@ impl Converter for ImagePair { } } +impl ImagePair { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: self.from, + to: self.to, + fidelity: self.fidelity, + tier: Tier::Native, + converter: self, + } + } +} + fn rows_error(error: RowsError) -> ConvertError { match error { RowsError::Png(error) => error.into(), @@ -658,6 +653,11 @@ pub fn codecs() -> impl Iterator< .map(|codec| (codec.format, codec.read.as_ref(), codec.write.as_ref())) } +/// Whether an image codec reads or writes `format` (PDF among them). +pub fn is_image_format(format: &'static Format) -> bool { + CODECS.iter().any(|codec| codec.format.id == format.id) +} + /// The pair from one format id to another (`pair("png", "bmp")`). pub fn pair(from: &str, to: &str) -> &'static ImagePair { pairs() @@ -718,12 +718,18 @@ mod tests { #[test] fn pairs_declare_their_contracts() { - assert_eq!(pair("png", "bmp").name(), "png-to-bmp"); - assert_eq!(pair("png", "bmp").from().id, "png"); - assert_eq!(pair("bmp", "png").to().id, "png"); - assert_eq!(pair("bmp", "png").fidelity(), Fidelity::Lossless); - assert_eq!(pair("bmp", "jpeg").fidelity(), Fidelity::Lossy(JPEG_LOSS)); - assert_eq!(pair("webp", "jpeg").fidelity().kind(), FidelityKind::Lossy); + assert_eq!(pair("png", "bmp").entry().name, "png-to-bmp"); + assert_eq!(pair("png", "bmp").entry().from.id, "png"); + assert_eq!(pair("bmp", "png").entry().to.id, "png"); + assert_eq!(pair("bmp", "png").entry().fidelity, Fidelity::Lossless); + assert_eq!( + pair("bmp", "jpeg").entry().fidelity, + Fidelity::Lossy(JPEG_LOSS) + ); + assert_eq!( + pair("webp", "jpeg").entry().fidelity.kind(), + FidelityKind::Lossy + ); } #[test] diff --git a/src/converters/json_to_csv.rs b/src/converters/json_to_csv.rs index 71ea5b80..40d778d5 100644 --- a/src/converters/json_to_csv.rs +++ b/src/converters/json_to_csv.rs @@ -5,7 +5,7 @@ use std::collections::{HashMap, HashSet}; use std::io::{Read, Write}; use crate::converter::{ - ConvertError, Converter, Fidelity, Input, MemoryParts, Parts, RewindableRead, Tier, + ConvertError, Converter, Entry, Fidelity, Input, MemoryParts, Parts, RewindableRead, Tier, }; use crate::event::Context; use crate::format::Format; @@ -74,26 +74,6 @@ struct ColumnReports { } impl Converter for JsonToCsv { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - self.from - } - - fn to(&self) -> &'static Format { - self.to - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, input: Input<'_>, @@ -140,6 +120,19 @@ impl Converter for JsonToCsv { } } +impl JsonToCsv { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: self.from, + to: self.to, + fidelity: Fidelity::Conditional(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + /// The first byte that is not white space. fn first_byte(reader: &mut dyn RewindableRead) -> Result, ConvertError> { let mut byte = [0u8; 1]; @@ -210,31 +203,11 @@ pub struct JsonToXlsx; pub static JSON_TO_XLSX: JsonToXlsx = JsonToXlsx; impl Converter for JsonToXlsx { - fn name(&self) -> &'static str { - "json-to-xlsx" - } - /// Sheets are named after the JSON's members, not `--sheet`. - fn options(&self) -> Vec { + fn options(&self, _from: &'static Format, _to: &'static Format) -> Vec { vec![crate::format::Setting::Effort] } - fn from(&self) -> &'static Format { - &formats::JSON - } - - fn to(&self) -> &'static Format { - &formats::XLSX - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(JSON_TO_XLSX_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, input: Input<'_>, @@ -272,6 +245,19 @@ impl Converter for JsonToXlsx { } } +impl JsonToXlsx { + pub const fn entry(&'static self) -> Entry { + Entry { + name: "json-to-xlsx", + from: &formats::JSON, + to: &formats::XLSX, + fidelity: Fidelity::Conditional(JSON_TO_XLSX_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + fn unsupported_root(shape: Shape, found: Token) -> ConvertError { let message = if shape.lines { format!( @@ -737,10 +723,10 @@ mod tests { #[test] fn declares_contract() { let converter = &JSON_TO_CSV; - assert_eq!(converter.name(), "json-to-csv"); - assert_eq!(converter.from().id, "json"); - assert_eq!(converter.to().id, "csv"); - assert_eq!(converter.fidelity().kind(), FidelityKind::Conditional); - assert_eq!(converter.tier(), Tier::Native); + assert_eq!(converter.entry().name, "json-to-csv"); + assert_eq!(converter.entry().from.id, "json"); + assert_eq!(converter.entry().to.id, "csv"); + assert_eq!(converter.entry().fidelity.kind(), FidelityKind::Conditional); + assert_eq!(converter.entry().tier, Tier::Native); } } diff --git a/src/converters/json_to_pages.rs b/src/converters/json_to_pages.rs index 69f2ae43..e73c3b16 100644 --- a/src/converters/json_to_pages.rs +++ b/src/converters/json_to_pages.rs @@ -3,10 +3,9 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Tier}; use crate::converters::pages_to_json::package_error; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::pages::json::{ReadError, read_json}; @@ -15,26 +14,6 @@ const NAME: &str = "json-to-pages"; pub struct JsonToPages; impl Converter for JsonToPages { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::PAGES_JSON - } - - fn to(&self) -> &'static Format { - &formats::PAGES - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossless - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, input: Input<'_>, @@ -54,15 +33,28 @@ impl Converter for JsonToPages { } } +impl JsonToPages { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::PAGES_JSON, + to: &formats::PAGES, + fidelity: Fidelity::Lossless, + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(JsonToPages.name(), "json-to-pages"); - assert_eq!(JsonToPages.from().id, "pages-json"); - assert_eq!(JsonToPages.to().id, "pages"); - assert_eq!(JsonToPages.fidelity(), Fidelity::Lossless); + assert_eq!(JsonToPages.entry().name, "json-to-pages"); + assert_eq!(JsonToPages.entry().from.id, "pages-json"); + assert_eq!(JsonToPages.entry().to.id, "pages"); + assert_eq!(JsonToPages.entry().fidelity, Fidelity::Lossless); } } diff --git a/src/converters/json_to_toml.rs b/src/converters/json_to_toml.rs index cd074962..56d9a79f 100644 --- a/src/converters/json_to_toml.rs +++ b/src/converters/json_to_toml.rs @@ -4,9 +4,8 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Tier}; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::json; use crate::io::toml; @@ -19,26 +18,6 @@ const FIDELITY_NOTE: &str = pub struct JsonToToml; impl Converter for JsonToToml { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::JSON - } - - fn to(&self) -> &'static Format { - &formats::TOML - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, input: Input<'_>, @@ -66,6 +45,19 @@ impl Converter for JsonToToml { } } +impl JsonToToml { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::JSON, + to: &formats::TOML, + fidelity: Fidelity::Conditional(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + fn not_a_table(what: &str) -> ConvertError { ConvertError::Unsupported(format!( "a TOML document is a table; the JSON root is {what}" @@ -78,9 +70,12 @@ mod tests { #[test] fn declares_contract() { - assert_eq!(JsonToToml.name(), "json-to-toml"); - assert_eq!(JsonToToml.from().id, "json"); - assert_eq!(JsonToToml.to().id, "toml"); - assert_eq!(JsonToToml.fidelity(), Fidelity::Conditional(FIDELITY_NOTE)); + assert_eq!(JsonToToml.entry().name, "json-to-toml"); + assert_eq!(JsonToToml.entry().from.id, "json"); + assert_eq!(JsonToToml.entry().to.id, "toml"); + assert_eq!( + JsonToToml.entry().fidelity, + Fidelity::Conditional(FIDELITY_NOTE) + ); } } diff --git a/src/converters/json_to_xml.rs b/src/converters/json_to_xml.rs index cdb054d3..d4a46da7 100644 --- a/src/converters/json_to_xml.rs +++ b/src/converters/json_to_xml.rs @@ -4,9 +4,8 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::json; use crate::io::xml::writer; @@ -18,26 +17,6 @@ const FIDELITY_NOTE: &str = "@-keys become attributes, #text the text, arrays re pub struct JsonToXml; impl Converter for JsonToXml { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::JSON - } - - fn to(&self) -> &'static Format { - &formats::XML - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, input: Input<'_>, @@ -59,15 +38,31 @@ impl Converter for JsonToXml { } } +impl JsonToXml { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::JSON, + to: &formats::XML, + fidelity: Fidelity::Conditional(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(JsonToXml.name(), "json-to-xml"); - assert_eq!(JsonToXml.from().id, "json"); - assert_eq!(JsonToXml.to().id, "xml"); - assert_eq!(JsonToXml.fidelity(), Fidelity::Conditional(FIDELITY_NOTE)); + assert_eq!(JsonToXml.entry().name, "json-to-xml"); + assert_eq!(JsonToXml.entry().from.id, "json"); + assert_eq!(JsonToXml.entry().to.id, "xml"); + assert_eq!( + JsonToXml.entry().fidelity, + Fidelity::Conditional(FIDELITY_NOTE) + ); } } diff --git a/src/converters/json_to_yaml.rs b/src/converters/json_to_yaml.rs index b0f1f210..9832c4b7 100644 --- a/src/converters/json_to_yaml.rs +++ b/src/converters/json_to_yaml.rs @@ -3,9 +3,8 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::json; use crate::io::yaml; @@ -16,26 +15,6 @@ const NAME: &str = "json-to-yaml"; pub struct JsonToYaml; impl Converter for JsonToYaml { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::JSON - } - - fn to(&self) -> &'static Format { - &formats::YAML - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossless - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, input: Input<'_>, @@ -50,15 +29,28 @@ impl Converter for JsonToYaml { } } +impl JsonToYaml { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::JSON, + to: &formats::YAML, + fidelity: Fidelity::Lossless, + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(JsonToYaml.name(), "json-to-yaml"); - assert_eq!(JsonToYaml.from().id, "json"); - assert_eq!(JsonToYaml.to().id, "yaml"); - assert_eq!(JsonToYaml.fidelity(), Fidelity::Lossless); + assert_eq!(JsonToYaml.entry().name, "json-to-yaml"); + assert_eq!(JsonToYaml.entry().from.id, "json"); + assert_eq!(JsonToYaml.entry().to.id, "yaml"); + assert_eq!(JsonToYaml.entry().fidelity, Fidelity::Lossless); } } diff --git a/src/converters/markdown_json_to_markdown.rs b/src/converters/markdown_json_to_markdown.rs index aa1f57d4..c113925e 100644 --- a/src/converters/markdown_json_to_markdown.rs +++ b/src/converters/markdown_json_to_markdown.rs @@ -3,9 +3,8 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::markdown::MarkdownWriter; use crate::io::markdown::events_json::read_events_json; @@ -15,26 +14,6 @@ const NAME: &str = "markdown-json-to-markdown"; pub struct MarkdownJsonToMarkdown; impl Converter for MarkdownJsonToMarkdown { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::MARKDOWN_JSON - } - - fn to(&self) -> &'static Format { - &formats::MARKDOWN - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossless - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, input: Input<'_>, @@ -49,6 +28,19 @@ impl Converter for MarkdownJsonToMarkdown { } } +impl MarkdownJsonToMarkdown { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::MARKDOWN_JSON, + to: &formats::MARKDOWN, + fidelity: Fidelity::Lossless, + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/converters/markdown_to_docx.rs b/src/converters/markdown_to_docx.rs index 48f9df90..c62cddb7 100644 --- a/src/converters/markdown_to_docx.rs +++ b/src/converters/markdown_to_docx.rs @@ -5,11 +5,10 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::converters::input::read_text_document; use crate::document::from_events::DocumentBuilder; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::docx::DocxStream; use crate::io::markdown::{Options, parse_into}; @@ -20,26 +19,6 @@ const FIDELITY_NOTE: &str = "lossless for headings, paragraphs, lists, quotes, c pub struct MarkdownToDocx; impl Converter for MarkdownToDocx { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::MARKDOWN - } - - fn to(&self) -> &'static Format { - &formats::DOCX - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -58,17 +37,30 @@ impl Converter for MarkdownToDocx { } } +impl MarkdownToDocx { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::MARKDOWN, + to: &formats::DOCX, + fidelity: Fidelity::Conditional(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(MarkdownToDocx.name(), "markdown-to-docx"); - assert_eq!(MarkdownToDocx.from().id, "markdown"); - assert_eq!(MarkdownToDocx.to().id, "docx"); + assert_eq!(MarkdownToDocx.entry().name, "markdown-to-docx"); + assert_eq!(MarkdownToDocx.entry().from.id, "markdown"); + assert_eq!(MarkdownToDocx.entry().to.id, "docx"); assert_eq!( - MarkdownToDocx.fidelity(), + MarkdownToDocx.entry().fidelity, Fidelity::Conditional(FIDELITY_NOTE) ); } diff --git a/src/converters/markdown_to_html.rs b/src/converters/markdown_to_html.rs index f4d0c68b..f6fcc7d3 100644 --- a/src/converters/markdown_to_html.rs +++ b/src/converters/markdown_to_html.rs @@ -4,10 +4,9 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::converters::input::read_text_document; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::html::HtmlWriter; use crate::io::markdown::{Options, parse_into}; @@ -17,26 +16,6 @@ const NAME: &str = "markdown-to-html"; pub struct MarkdownToHtml; impl Converter for MarkdownToHtml { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::MARKDOWN - } - - fn to(&self) -> &'static Format { - &formats::HTML - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossless - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -52,6 +31,19 @@ impl Converter for MarkdownToHtml { } } +impl MarkdownToHtml { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::MARKDOWN, + to: &formats::HTML, + fidelity: Fidelity::Lossless, + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -77,11 +69,11 @@ mod tests { #[test] fn declares_contract() { - let converter = MarkdownToHtml; - assert_eq!(converter.name(), "markdown-to-html"); - assert_eq!(converter.from().id, "markdown"); - assert_eq!(converter.to().id, "html"); - assert_eq!(converter.fidelity(), Fidelity::Lossless); - assert_eq!(converter.tier(), Tier::Native); + let converter: &'static MarkdownToHtml = &MarkdownToHtml; + assert_eq!(converter.entry().name, "markdown-to-html"); + assert_eq!(converter.entry().from.id, "markdown"); + assert_eq!(converter.entry().to.id, "html"); + assert_eq!(converter.entry().fidelity, Fidelity::Lossless); + assert_eq!(converter.entry().tier, Tier::Native); } } diff --git a/src/converters/markdown_to_json.rs b/src/converters/markdown_to_json.rs index 8e1c7501..2d1a5d89 100644 --- a/src/converters/markdown_to_json.rs +++ b/src/converters/markdown_to_json.rs @@ -3,10 +3,9 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::converters::input::read_text_document; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::markdown::events_json::JsonEventWriter; use crate::io::markdown::{Options, parse_into}; @@ -16,26 +15,6 @@ const NAME: &str = "markdown-to-json"; pub struct MarkdownToJson; impl Converter for MarkdownToJson { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::MARKDOWN - } - - fn to(&self) -> &'static Format { - &formats::MARKDOWN_JSON - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossless - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -51,6 +30,19 @@ impl Converter for MarkdownToJson { } } +impl MarkdownToJson { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::MARKDOWN, + to: &formats::MARKDOWN_JSON, + fidelity: Fidelity::Lossless, + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -76,10 +68,10 @@ mod tests { #[test] fn declares_contract() { - let converter = MarkdownToJson; - assert_eq!(converter.name(), "markdown-to-json"); - assert_eq!(converter.from().id, "markdown"); - assert_eq!(converter.to().id, "markdown-json"); - assert_eq!(converter.fidelity(), Fidelity::Lossless); + let converter: &'static MarkdownToJson = &MarkdownToJson; + assert_eq!(converter.entry().name, "markdown-to-json"); + assert_eq!(converter.entry().from.id, "markdown"); + assert_eq!(converter.entry().to.id, "markdown-json"); + assert_eq!(converter.entry().fidelity, Fidelity::Lossless); } } diff --git a/src/converters/markdown_to_pages.rs b/src/converters/markdown_to_pages.rs index 3e51fffb..0a67103a 100644 --- a/src/converters/markdown_to_pages.rs +++ b/src/converters/markdown_to_pages.rs @@ -5,12 +5,11 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::converters::input::read_text_document; use crate::converters::pages_to_json::package_error; use crate::document::from_events::DocumentBuilder; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::markdown::{Options, parse_into}; use crate::io::pages::write_package_to; @@ -21,26 +20,6 @@ const FIDELITY_NOTE: &str = "paragraphs, headings, bulleted and numbered lists, pub struct MarkdownToPages; impl Converter for MarkdownToPages { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::MARKDOWN - } - - fn to(&self) -> &'static Format { - &formats::PAGES - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -62,15 +41,31 @@ impl Converter for MarkdownToPages { } } +impl MarkdownToPages { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::MARKDOWN, + to: &formats::PAGES, + fidelity: Fidelity::Lossy(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(MarkdownToPages.name(), "markdown-to-pages"); - assert_eq!(MarkdownToPages.from().id, "markdown"); - assert_eq!(MarkdownToPages.to().id, "pages"); - assert_eq!(MarkdownToPages.fidelity(), Fidelity::Lossy(FIDELITY_NOTE)); + assert_eq!(MarkdownToPages.entry().name, "markdown-to-pages"); + assert_eq!(MarkdownToPages.entry().from.id, "markdown"); + assert_eq!(MarkdownToPages.entry().to.id, "pages"); + assert_eq!( + MarkdownToPages.entry().fidelity, + Fidelity::Lossy(FIDELITY_NOTE) + ); } } diff --git a/src/converters/markdown_to_text.rs b/src/converters/markdown_to_text.rs index c380c9a5..9ac913d7 100644 --- a/src/converters/markdown_to_text.rs +++ b/src/converters/markdown_to_text.rs @@ -3,10 +3,9 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::converters::input::read_text_document; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::markdown::{Options, parse_into}; use crate::io::text::TextWriter; @@ -17,26 +16,6 @@ const FIDELITY_NOTE: &str = "markup, raw HTML, and image sources are dropped"; pub struct MarkdownToText; impl Converter for MarkdownToText { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::MARKDOWN - } - - fn to(&self) -> &'static Format { - &formats::TEXT - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -52,6 +31,19 @@ impl Converter for MarkdownToText { } } +impl MarkdownToText { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::MARKDOWN, + to: &formats::TEXT, + fidelity: Fidelity::Lossy(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -77,11 +69,11 @@ mod tests { #[test] fn declares_contract() { - let converter = MarkdownToText; - assert_eq!(converter.name(), "markdown-to-text"); - assert_eq!(converter.from().id, "markdown"); - assert_eq!(converter.to().id, "text"); - assert_eq!(converter.fidelity(), Fidelity::Lossy(FIDELITY_NOTE)); - assert_eq!(converter.tier(), Tier::Native); + let converter: &'static MarkdownToText = &MarkdownToText; + assert_eq!(converter.entry().name, "markdown-to-text"); + assert_eq!(converter.entry().from.id, "markdown"); + assert_eq!(converter.entry().to.id, "text"); + assert_eq!(converter.entry().fidelity, Fidelity::Lossy(FIDELITY_NOTE)); + assert_eq!(converter.entry().tier, Tier::Native); } } diff --git a/src/converters/n64.rs b/src/converters/n64.rs index c6fa8c3b..52c1a0e2 100644 --- a/src/converters/n64.rs +++ b/src/converters/n64.rs @@ -6,7 +6,7 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Tier}; use crate::event::Context; use crate::format::Format; use crate::format::formats; @@ -59,26 +59,6 @@ pub static N64_TO_V64: Reorder = Reorder { }; impl Converter for Reorder { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - self.from.0 - } - - fn to(&self) -> &'static Format { - self.to.0 - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossless - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -118,6 +98,19 @@ impl Converter for Reorder { } } +impl Reorder { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: self.from.0, + to: self.to.0, + fidelity: Fidelity::Lossless, + tier: Tier::Native, + converter: self, + } + } +} + fn malformed(message: String) -> ConvertError { ConvertError::Malformed { location: Location::default(), @@ -140,9 +133,13 @@ mod tests { &N64_TO_V64, ]; for converter in all { - let expected = format!("{}-to-{}", converter.from().id, converter.to().id); - assert_eq!(converter.name(), expected); - assert_eq!(converter.fidelity(), Fidelity::Lossless); + let expected = format!( + "{}-to-{}", + converter.entry().from.id, + converter.entry().to.id + ); + assert_eq!(converter.entry().name, expected); + assert_eq!(converter.entry().fidelity, Fidelity::Lossless); } } } diff --git a/src/converters/numbers.rs b/src/converters/numbers.rs index 9985496b..bc75266e 100644 --- a/src/converters/numbers.rs +++ b/src/converters/numbers.rs @@ -8,7 +8,7 @@ use std::io::{Read, Write}; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Parts, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Parts, Tier}; use crate::converters::pages_to_json::package_error; use crate::event::Context; use crate::format::Format; @@ -72,26 +72,6 @@ pub static NUMBERS_TO_XLSX: NumbersTo = NumbersTo { }; impl Converter for NumbersTo { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - &formats::NUMBERS - } - - fn to(&self) -> &'static Format { - self.to - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(self.note) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -195,6 +175,19 @@ impl Converter for NumbersTo { } } +impl NumbersTo { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: &formats::NUMBERS, + to: self.to, + fidelity: Fidelity::Conditional(self.note), + tier: Tier::Native, + converter: self, + } + } +} + /// Each cell format's Excel form, worked out once per format. pub(crate) type ExcelCodes = std::collections::HashMap<*const CellFormat, (Excel, Option>)>; @@ -393,9 +386,9 @@ mod tests { #[test] fn declares_contract() { - assert_eq!(NUMBERS_TO_CSV.name(), "numbers-to-csv"); - assert_eq!(NUMBERS_TO_CSV.from().id, "numbers"); - assert_eq!(NUMBERS_TO_XLSX.to().id, "xlsx"); + assert_eq!(NUMBERS_TO_CSV.entry().name, "numbers-to-csv"); + assert_eq!(NUMBERS_TO_CSV.entry().from.id, "numbers"); + assert_eq!(NUMBERS_TO_XLSX.entry().to.id, "xlsx"); assert!(NUMBERS_TO_TSV.splits()); assert!(!NUMBERS_TO_JSON.splits()); } diff --git a/src/converters/ods.rs b/src/converters/ods.rs index 0533d48d..a1fbc090 100644 --- a/src/converters/ods.rs +++ b/src/converters/ods.rs @@ -9,7 +9,7 @@ use std::io::Write; use crate::converter::{ - ConvertError, Converter, Fidelity, Input, Location, MemoryParts, Parts, Tier, + ConvertError, Converter, Entry, Fidelity, Input, Location, MemoryParts, Parts, Tier, }; use crate::converters::numbers::{ExcelCodes, excel_cell, pick_tables, read_all}; use crate::converters::pages_to_json::package_error; @@ -97,26 +97,6 @@ pub static ODS_TO_NUMBERS: OdsTo = OdsTo { }; impl Converter for OdsTo { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - &formats::ODS - } - - fn to(&self) -> &'static Format { - self.to - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(self.note) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -250,6 +230,19 @@ impl Converter for OdsTo { } } +impl OdsTo { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: &formats::ODS, + to: self.to, + fidelity: Fidelity::Conditional(self.note), + tier: Tier::Native, + converter: self, + } + } +} + /// Whether `--sheet` (a name, or a number from 1) picks a sheet; every /// sheet without it. fn picks(selector: Option<&str>, index: usize, name: &str) -> bool { @@ -423,19 +416,7 @@ pub static NUMBERS_TO_ODS: ToOds = ToOds { }; impl Converter for ToOds { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - self.from - } - - fn to(&self) -> &'static Format { - &formats::ODS - } - - fn options(&self) -> Vec { + fn options(&self, _from: &'static Format, _to: &'static Format) -> Vec { match self.source { // Sheets are named after the source's own names. Source::Json | Source::Document => vec![Setting::Effort], @@ -448,17 +429,6 @@ impl Converter for ToOds { } } - fn fidelity(&self) -> Fidelity { - match self.source { - Source::Document => Fidelity::Lossy(self.note), - _ => Fidelity::Conditional(self.note), - } - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -619,6 +589,22 @@ impl Converter for ToOds { } } +impl ToOds { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: self.from, + to: &formats::ODS, + fidelity: match self.source { + Source::Document => Fidelity::Lossy(self.note), + _ => Fidelity::Conditional(self.note), + }, + tier: Tier::Native, + converter: self, + } + } +} + /// An Excel cell (as the workbook reader gives it: numbers as stored, /// dates in ISO 8601, booleans as `TRUE` and `FALSE`) typed by its text, /// with its number format. @@ -703,12 +689,12 @@ mod tests { #[test] fn declares_contracts() { - assert_eq!(ODS_TO_CSV.from().id, "ods"); - assert_eq!(ODS_TO_XLSX.to().id, "xlsx"); + assert_eq!(ODS_TO_CSV.entry().from.id, "ods"); + assert_eq!(ODS_TO_XLSX.entry().to.id, "xlsx"); assert!(ODS_TO_TSV.splits()); assert!(!ODS_TO_JSON.splits()); - assert_eq!(CSV_TO_ODS.to().id, "ods"); - assert_eq!(NUMBERS_TO_ODS.from().id, "numbers"); + assert_eq!(CSV_TO_ODS.entry().to.id, "ods"); + assert_eq!(NUMBERS_TO_ODS.entry().from.id, "numbers"); } #[test] diff --git a/src/converters/odt.rs b/src/converters/odt.rs index f3b8335b..c5495e0b 100644 --- a/src/converters/odt.rs +++ b/src/converters/odt.rs @@ -4,7 +4,7 @@ use std::io::{Read, Write}; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Tier}; use crate::converters::input::read_text_document; use crate::converters::pages_to_json::package_error; use crate::document::Document; @@ -95,26 +95,6 @@ pub fn read_input(input: &mut Input<'_>) -> Result { } impl Converter for FromOdt { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - &formats::ODT - } - - fn to(&self) -> &'static Format { - self.to - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(self.note) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -161,6 +141,19 @@ impl Converter for FromOdt { } } +impl FromOdt { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: &formats::ODT, + to: self.to, + fidelity: Fidelity::Lossy(self.note), + tier: Tier::Native, + converter: self, + } + } +} + const FROM_DOCUMENT_NOTE: &str = "paragraphs and headings, named styles, run and paragraph formatting, lists (restarts and start numbers kept), tables (merged cells, borders, shading, nesting), sections with page setup, columns, headers, and footers, links, page-number fields, footnotes, comments with their replies, tracked changes, pictures, text boxes, and equations (as formula objects) carry over; charts and drawn shapes do not"; const FROM_TEXT_NOTE: &str = "headings, paragraphs, emphasis, lists, tables, links, footnotes, and code carry over as OpenDocument formatting; images are kept only when the source holds their bytes"; @@ -224,26 +217,6 @@ pub static TEXT_TO_ODT: ToOdt = ToOdt { }; impl Converter for ToOdt { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - self.from - } - - fn to(&self) -> &'static Format { - &formats::ODT - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(self.note) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -294,6 +267,19 @@ impl Converter for ToOdt { } } +impl ToOdt { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: self.from, + to: &formats::ODT, + fidelity: Fidelity::Lossy(self.note), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -308,9 +294,9 @@ mod tests { (&ODT_TO_HTML, "html"), (&ODT_TO_TEXT, "text"), ] { - assert_eq!(converter.from().id, "odt"); - assert_eq!(converter.to().id, to); - assert_eq!(converter.name(), format!("odt-to-{to}")); + assert_eq!(converter.entry().from.id, "odt"); + assert_eq!(converter.entry().to.id, to); + assert_eq!(converter.entry().name, format!("odt-to-{to}")); } for (converter, from) in [ (&DOCX_TO_ODT, "docx"), @@ -320,9 +306,9 @@ mod tests { (&HTML_TO_ODT, "html"), (&TEXT_TO_ODT, "text"), ] { - assert_eq!(converter.from().id, from); - assert_eq!(converter.to().id, "odt"); - assert_eq!(converter.name(), format!("{from}-to-odt")); + assert_eq!(converter.entry().from.id, from); + assert_eq!(converter.entry().to.id, "odt"); + assert_eq!(converter.entry().name, format!("{from}-to-odt")); } } } diff --git a/src/converters/pages_to_docx.rs b/src/converters/pages_to_docx.rs index 9fd05aa7..3b0dcdf2 100644 --- a/src/converters/pages_to_docx.rs +++ b/src/converters/pages_to_docx.rs @@ -4,10 +4,9 @@ use std::io::{Read, Write}; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::converters::pages_to_json::package_error; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::docx::write_docx; use crate::io::pages::{Package, Scope, read_document}; @@ -18,26 +17,6 @@ const FIDELITY_NOTE: &str = "text, named styles, lists (with their indents and n pub struct PagesToDocx; impl Converter for PagesToDocx { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::PAGES - } - - fn to(&self) -> &'static Format { - &formats::DOCX - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -58,15 +37,31 @@ impl Converter for PagesToDocx { } } +impl PagesToDocx { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::PAGES, + to: &formats::DOCX, + fidelity: Fidelity::Conditional(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(PagesToDocx.name(), "pages-to-docx"); - assert_eq!(PagesToDocx.from().id, "pages"); - assert_eq!(PagesToDocx.to().id, "docx"); - assert_eq!(PagesToDocx.fidelity(), Fidelity::Conditional(FIDELITY_NOTE)); + assert_eq!(PagesToDocx.entry().name, "pages-to-docx"); + assert_eq!(PagesToDocx.entry().from.id, "pages"); + assert_eq!(PagesToDocx.entry().to.id, "docx"); + assert_eq!( + PagesToDocx.entry().fidelity, + Fidelity::Conditional(FIDELITY_NOTE) + ); } } diff --git a/src/converters/pages_to_html.rs b/src/converters/pages_to_html.rs index 5a9c094c..ea73ba7a 100644 --- a/src/converters/pages_to_html.rs +++ b/src/converters/pages_to_html.rs @@ -3,11 +3,10 @@ use std::io::{Read, Write}; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::converters::pages_to_json::package_error; use crate::document::markdown::emit_events; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::html::HtmlWriter; use crate::io::pages::{Package, Scope, read_document}; @@ -18,26 +17,6 @@ const FIDELITY_NOTE: &str = "page layout, headers and footers, fonts, sizes, and pub struct PagesToHtml; impl Converter for PagesToHtml { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::PAGES - } - - fn to(&self) -> &'static Format { - &formats::HTML - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -56,15 +35,28 @@ impl Converter for PagesToHtml { } } +impl PagesToHtml { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::PAGES, + to: &formats::HTML, + fidelity: Fidelity::Lossy(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(PagesToHtml.name(), "pages-to-html"); - assert_eq!(PagesToHtml.from().id, "pages"); - assert_eq!(PagesToHtml.to().id, "html"); - assert_eq!(PagesToHtml.fidelity(), Fidelity::Lossy(FIDELITY_NOTE)); + assert_eq!(PagesToHtml.entry().name, "pages-to-html"); + assert_eq!(PagesToHtml.entry().from.id, "pages"); + assert_eq!(PagesToHtml.entry().to.id, "html"); + assert_eq!(PagesToHtml.entry().fidelity, Fidelity::Lossy(FIDELITY_NOTE)); } } diff --git a/src/converters/pages_to_json.rs b/src/converters/pages_to_json.rs index 74760a6b..0c12e89d 100644 --- a/src/converters/pages_to_json.rs +++ b/src/converters/pages_to_json.rs @@ -4,9 +4,8 @@ use std::io::{Read, Write}; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Tier}; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::pages::json::write_json; use crate::io::pages::{Package, PackageError}; @@ -16,26 +15,6 @@ const NAME: &str = "pages-to-json"; pub struct PagesToJson; impl Converter for PagesToJson { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::PAGES - } - - fn to(&self) -> &'static Format { - &formats::PAGES_JSON - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossless - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -51,6 +30,19 @@ impl Converter for PagesToJson { } } +impl PagesToJson { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::PAGES, + to: &formats::PAGES_JSON, + fidelity: Fidelity::Lossless, + tier: Tier::Native, + converter: self, + } + } +} + pub(crate) fn package_error(error: PackageError) -> ConvertError { match error { PackageError::Io(error) => ConvertError::Io(error), @@ -67,9 +59,9 @@ mod tests { #[test] fn declares_contract() { - assert_eq!(PagesToJson.name(), "pages-to-json"); - assert_eq!(PagesToJson.from().id, "pages"); - assert_eq!(PagesToJson.to().id, "pages-json"); - assert_eq!(PagesToJson.fidelity(), Fidelity::Lossless); + assert_eq!(PagesToJson.entry().name, "pages-to-json"); + assert_eq!(PagesToJson.entry().from.id, "pages"); + assert_eq!(PagesToJson.entry().to.id, "pages-json"); + assert_eq!(PagesToJson.entry().fidelity, Fidelity::Lossless); } } diff --git a/src/converters/pages_to_markdown.rs b/src/converters/pages_to_markdown.rs index e4d1c56a..e01b702c 100644 --- a/src/converters/pages_to_markdown.rs +++ b/src/converters/pages_to_markdown.rs @@ -3,11 +3,10 @@ use std::io::{Read, Write}; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::converters::pages_to_json::package_error; use crate::document::markdown::emit_events; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::markdown::MarkdownWriter; use crate::io::pages::{Package, Scope, read_document}; @@ -18,26 +17,6 @@ const FIDELITY_NOTE: &str = "page layout, headers and footers, fonts, sizes, and pub struct PagesToMarkdown; impl Converter for PagesToMarkdown { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::PAGES - } - - fn to(&self) -> &'static Format { - &formats::MARKDOWN - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -56,15 +35,31 @@ impl Converter for PagesToMarkdown { } } +impl PagesToMarkdown { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::PAGES, + to: &formats::MARKDOWN, + fidelity: Fidelity::Lossy(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(PagesToMarkdown.name(), "pages-to-markdown"); - assert_eq!(PagesToMarkdown.from().id, "pages"); - assert_eq!(PagesToMarkdown.to().id, "markdown"); - assert_eq!(PagesToMarkdown.fidelity(), Fidelity::Lossy(FIDELITY_NOTE)); + assert_eq!(PagesToMarkdown.entry().name, "pages-to-markdown"); + assert_eq!(PagesToMarkdown.entry().from.id, "pages"); + assert_eq!(PagesToMarkdown.entry().to.id, "markdown"); + assert_eq!( + PagesToMarkdown.entry().fidelity, + Fidelity::Lossy(FIDELITY_NOTE) + ); } } diff --git a/src/converters/pages_to_text.rs b/src/converters/pages_to_text.rs index cc660cf0..4fe9ea36 100644 --- a/src/converters/pages_to_text.rs +++ b/src/converters/pages_to_text.rs @@ -3,11 +3,10 @@ use std::io::{Read, Write}; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::converters::pages_to_json::package_error; use crate::document::markdown::emit_events; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::pages::{Package, Scope, read_document}; use crate::io::text::TextWriter; @@ -19,26 +18,6 @@ const FIDELITY_NOTE: &str = pub struct PagesToText; impl Converter for PagesToText { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::PAGES - } - - fn to(&self) -> &'static Format { - &formats::TEXT - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -57,15 +36,28 @@ impl Converter for PagesToText { } } +impl PagesToText { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::PAGES, + to: &formats::TEXT, + fidelity: Fidelity::Lossy(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(PagesToText.name(), "pages-to-text"); - assert_eq!(PagesToText.from().id, "pages"); - assert_eq!(PagesToText.to().id, "text"); - assert_eq!(PagesToText.fidelity(), Fidelity::Lossy(FIDELITY_NOTE)); + assert_eq!(PagesToText.entry().name, "pages-to-text"); + assert_eq!(PagesToText.entry().from.id, "pages"); + assert_eq!(PagesToText.entry().to.id, "text"); + assert_eq!(PagesToText.entry().fidelity, Fidelity::Lossy(FIDELITY_NOTE)); } } diff --git a/src/converters/pdf_to_document.rs b/src/converters/pdf_to_document.rs index 427c55a9..3b2bd49f 100644 --- a/src/converters/pdf_to_document.rs +++ b/src/converters/pdf_to_document.rs @@ -4,7 +4,7 @@ use std::io::{Read, Write}; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Tier}; use crate::converters::pdf_to_text::report_text_notes; use crate::document::from_events::DocumentBuilder; use crate::event::Context; @@ -64,26 +64,6 @@ pub static PDF_TO_ODT: PdfToDocument = PdfToDocument { }; impl Converter for PdfToDocument { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - &formats::PDF - } - - fn to(&self) -> &'static Format { - self.to - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -152,3 +132,16 @@ impl Converter for PdfToDocument { Ok(()) } } + +impl PdfToDocument { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: &formats::PDF, + to: self.to, + fidelity: Fidelity::Lossy(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} diff --git a/src/converters/pdf_to_text.rs b/src/converters/pdf_to_text.rs index 3ca5a169..fbf2d0c9 100644 --- a/src/converters/pdf_to_text.rs +++ b/src/converters/pdf_to_text.rs @@ -4,9 +4,8 @@ use std::io::{Read, Write}; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Tier}; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::pdf::text::{TextNotes, write_pdf_text}; @@ -16,26 +15,6 @@ const FIDELITY_NOTE: &str = "fonts, layout, images, and drawings are dropped; re pub struct PdfToText; impl Converter for PdfToText { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::PDF - } - - fn to(&self) -> &'static Format { - &formats::TEXT - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -56,6 +35,19 @@ impl Converter for PdfToText { } } +impl PdfToText { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::PDF, + to: &formats::TEXT, + fidelity: Fidelity::Lossy(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + /// Warnings for what the text left out. pub(crate) fn report_text_notes(notes: &TextNotes, context: &mut Context<'_>) { if notes.pages_without_text > 0 { diff --git a/src/converters/rows.rs b/src/converters/rows.rs index 50e0c3f8..4bfd1528 100644 --- a/src/converters/rows.rs +++ b/src/converters/rows.rs @@ -4,7 +4,7 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::event::Context; use crate::format::Format; use crate::format::formats; @@ -41,26 +41,6 @@ pub static TSV_TO_CSV: Delimited = Delimited { }; impl Converter for Delimited { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - self.from - } - - fn to(&self) -> &'static Format { - self.to - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossless - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, input: Input<'_>, @@ -93,30 +73,23 @@ impl Converter for Delimited { } } +impl Delimited { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: self.from, + to: self.to, + fidelity: Fidelity::Lossless, + tier: Tier::Native, + converter: self, + } + } +} + /// JSON Lines -> JSON: the values, in order, as one array. pub struct JsonlToJson; impl Converter for JsonlToJson { - fn name(&self) -> &'static str { - "jsonl-to-json" - } - - fn from(&self) -> &'static Format { - &formats::JSONL - } - - fn to(&self) -> &'static Format { - &formats::JSON - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossless - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, input: Input<'_>, @@ -135,7 +108,7 @@ impl Converter for JsonlToJson { copy_value(&mut tokens, first, &mut writer)?; count += 1; if count % PROGRESS_INTERVAL == 0 { - context.progress(self.name(), tokens.bytes_consumed()); + context.progress("jsonl-to-json", tokens.bytes_consumed()); } } writer.end_array()?; @@ -144,6 +117,19 @@ impl Converter for JsonlToJson { } } +impl JsonlToJson { + pub const fn entry(&'static self) -> Entry { + Entry { + name: "jsonl-to-json", + from: &formats::JSONL, + to: &formats::JSON, + fidelity: Fidelity::Lossless, + tier: Tier::Native, + converter: self, + } + } +} + /// JSON -> JSON Lines: an array's elements one per line; any other root /// on one line. pub struct JsonToJsonl; @@ -152,26 +138,6 @@ const JSON_TO_JSONL_NOTE: &str = "an array becomes one line per element; any other root becomes one line"; impl Converter for JsonToJsonl { - fn name(&self) -> &'static str { - "json-to-jsonl" - } - - fn from(&self) -> &'static Format { - &formats::JSON - } - - fn to(&self) -> &'static Format { - &formats::JSONL - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(JSON_TO_JSONL_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, input: Input<'_>, @@ -205,7 +171,7 @@ impl Converter for JsonToJsonl { expect_value = false; count += 1; if count % PROGRESS_INTERVAL == 0 { - context.progress(self.name(), tokens.bytes_consumed()); + context.progress("json-to-jsonl", tokens.bytes_consumed()); } } other => { @@ -222,6 +188,19 @@ impl Converter for JsonToJsonl { } } +impl JsonToJsonl { + pub const fn entry(&'static self) -> Entry { + Entry { + name: "json-to-jsonl", + from: &formats::JSON, + to: &formats::JSONL, + fidelity: Fidelity::Conditional(JSON_TO_JSONL_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + fn expect_end(tokens: &mut JsonTokenizer) -> Result<(), ConvertError> { match tokens.next_token()? { Token::End => Ok(()), diff --git a/src/converters/rows_document.rs b/src/converters/rows_document.rs index a973df9a..8ccc19f8 100644 --- a/src/converters/rows_document.rs +++ b/src/converters/rows_document.rs @@ -5,7 +5,7 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Parts, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Parts, Tier}; use crate::converters::input::read_text_document; use crate::event::Context; use crate::format::Format; @@ -35,26 +35,6 @@ pub static TSV_TO_MARKDOWN: RowsToMarkdown = RowsToMarkdown { }; impl Converter for RowsToMarkdown { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - self.from - } - - fn to(&self) -> &'static Format { - &formats::MARKDOWN - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(ROWS_TO_MARKDOWN_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, input: Input<'_>, @@ -93,6 +73,19 @@ impl Converter for RowsToMarkdown { } } +impl RowsToMarkdown { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: self.from, + to: &formats::MARKDOWN, + fidelity: Fidelity::Conditional(ROWS_TO_MARKDOWN_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + /// Markdown -> rows: the document's first table. pub struct MarkdownToRows { pub name: &'static str, @@ -115,26 +108,6 @@ pub static MARKDOWN_TO_TSV: MarkdownToRows = MarkdownToRows { }; impl Converter for MarkdownToRows { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - &formats::MARKDOWN - } - - fn to(&self) -> &'static Format { - self.to - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(MARKDOWN_TO_ROWS_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -185,6 +158,19 @@ impl Converter for MarkdownToRows { } } +impl MarkdownToRows { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: &formats::MARKDOWN, + to: self.to, + fidelity: Fidelity::Lossy(MARKDOWN_TO_ROWS_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + /// Markdown -> workbook: every table a sheet, named after the heading /// before it. pub struct MarkdownToXlsx; @@ -194,31 +180,11 @@ pub static MARKDOWN_TO_XLSX: MarkdownToXlsx = MarkdownToXlsx; const MARKDOWN_TO_XLSX_NOTE: &str = "each of the document's tables a sheet, named after the heading before it, inline formatting flattened to text, decimals as numbers, ISO dates as dates, and TRUE and FALSE as booleans; everything else in the document is dropped"; impl Converter for MarkdownToXlsx { - fn name(&self) -> &'static str { - "markdown-to-xlsx" - } - /// Sheets are named after the headings, not `--sheet`. - fn options(&self) -> Vec { + fn options(&self, _from: &'static Format, _to: &'static Format) -> Vec { vec![crate::format::Setting::Effort] } - fn from(&self) -> &'static Format { - &formats::MARKDOWN - } - - fn to(&self) -> &'static Format { - &formats::XLSX - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(MARKDOWN_TO_XLSX_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -254,6 +220,19 @@ impl Converter for MarkdownToXlsx { } } +impl MarkdownToXlsx { + pub const fn entry(&'static self) -> Entry { + Entry { + name: "markdown-to-xlsx", + from: &formats::MARKDOWN, + to: &formats::XLSX, + fidelity: Fidelity::Lossy(MARKDOWN_TO_XLSX_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + /// Tables (each as CSV) as Markdown: one table alone, several each under a /// heading of its name. pub fn tables_to_markdown( @@ -294,26 +273,6 @@ pub static JSON_TO_MARKDOWN: JsonToMarkdown = JsonToMarkdown; const JSON_TO_MARKDOWN_NOTE: &str = "an array of objects as one table, an object of such arrays as a table under a heading per member; keys become the header row, nested values JSON text, and line breaks inside cells spaces"; impl Converter for JsonToMarkdown { - fn name(&self) -> &'static str { - "json-to-markdown" - } - - fn from(&self) -> &'static Format { - &formats::JSON - } - - fn to(&self) -> &'static Format { - &formats::MARKDOWN - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(JSON_TO_MARKDOWN_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, input: Input<'_>, @@ -326,6 +285,19 @@ impl Converter for JsonToMarkdown { } } +impl JsonToMarkdown { + pub const fn entry(&'static self) -> Entry { + Entry { + name: "json-to-markdown", + from: &formats::JSON, + to: &formats::MARKDOWN, + fidelity: Fidelity::Conditional(JSON_TO_MARKDOWN_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + /// Markdown -> JSON of its tables: one table as an array of objects keyed /// by its header, several as an object of such arrays keyed by the heading /// before each. @@ -336,26 +308,6 @@ pub static MARKDOWN_TABLES_TO_JSON: MarkdownTablesToJson = MarkdownTablesToJson; const TABLES_TO_JSON_NOTE: &str = "the document's tables only: one as an array of objects keyed by its header, several as an object of such arrays keyed by the heading before each; inline formatting is flattened to text and everything else in the document is dropped"; impl Converter for MarkdownTablesToJson { - fn name(&self) -> &'static str { - "markdown-tables-to-json" - } - - fn from(&self) -> &'static Format { - &formats::MARKDOWN - } - - fn to(&self) -> &'static Format { - &formats::JSON - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(TABLES_TO_JSON_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -386,17 +338,33 @@ impl Converter for MarkdownTablesToJson { } } +impl MarkdownTablesToJson { + pub const fn entry(&'static self) -> Entry { + Entry { + name: "markdown-tables-to-json", + from: &formats::MARKDOWN, + to: &formats::JSON, + fidelity: Fidelity::Lossy(TABLES_TO_JSON_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contracts() { - assert_eq!(CSV_TO_MARKDOWN.name(), "csv-to-markdown"); - assert_eq!(TSV_TO_MARKDOWN.from().id, "tsv"); - assert_eq!(CSV_TO_MARKDOWN.to().id, "markdown"); - assert_eq!(MARKDOWN_TO_CSV.from().id, "markdown"); - assert_eq!(MARKDOWN_TO_TSV.to().id, "tsv"); - assert!(matches!(MARKDOWN_TO_CSV.fidelity(), Fidelity::Lossy(_))); + assert_eq!(CSV_TO_MARKDOWN.entry().name, "csv-to-markdown"); + assert_eq!(TSV_TO_MARKDOWN.entry().from.id, "tsv"); + assert_eq!(CSV_TO_MARKDOWN.entry().to.id, "markdown"); + assert_eq!(MARKDOWN_TO_CSV.entry().from.id, "markdown"); + assert_eq!(MARKDOWN_TO_TSV.entry().to.id, "tsv"); + assert!(matches!( + MARKDOWN_TO_CSV.entry().fidelity, + Fidelity::Lossy(_) + )); } } diff --git a/src/converters/rtf.rs b/src/converters/rtf.rs index 58d026ff..2dd8b7b6 100644 --- a/src/converters/rtf.rs +++ b/src/converters/rtf.rs @@ -4,7 +4,7 @@ use std::io::{Read, Write}; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Tier}; use crate::converters::input::read_text_document; use crate::converters::pages_to_json::package_error; use crate::document::Document; @@ -87,26 +87,6 @@ pub fn read_input(input: &mut Input<'_>) -> Result { } impl Converter for FromRtf { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - &formats::RTF - } - - fn to(&self) -> &'static Format { - self.to - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(self.note) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -148,6 +128,19 @@ impl Converter for FromRtf { } } +impl FromRtf { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: &formats::RTF, + to: self.to, + fidelity: Fidelity::Lossy(self.note), + tier: Tier::Native, + converter: self, + } + } +} + const FROM_WORD_NOTE: &str = "paragraphs, named styles, run and paragraph formatting, lists (restarts and start numbers kept), tables (merged cells, borders, shading, nesting), sections with page setup, columns, headers, and footers, links, page-number fields, footnotes, comments with their replies, tracked changes, pictures (PNG, JPEG, EMF, WMF, BMP), text boxes, and the page colour carry over; charts, equations (their text is kept), and pictures in other formats do not"; const FROM_PAGES_NOTE: &str = "paragraphs, named styles, run and paragraph formatting, lists, tables, sections, headers and footers, links, footnotes, comments, tracked changes, pictures, text boxes, and the page colour carry over; charts and pictures in formats RTF cannot hold do not"; const FROM_TEXT_NOTE: &str = "headings, paragraphs, emphasis, lists, tables, links, footnotes, and code carry over as RTF formatting; images are kept only when the source holds their bytes"; @@ -204,26 +197,6 @@ pub static TEXT_TO_RTF: ToRtf = ToRtf { }; impl Converter for ToRtf { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - self.from - } - - fn to(&self) -> &'static Format { - &formats::RTF - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(self.note) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -281,6 +254,19 @@ impl Converter for ToRtf { } } +impl ToRtf { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: self.from, + to: &formats::RTF, + fidelity: Fidelity::Lossy(self.note), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -294,9 +280,9 @@ mod tests { (&RTF_TO_HTML, "html"), (&RTF_TO_TEXT, "text"), ] { - assert_eq!(converter.from().id, "rtf"); - assert_eq!(converter.to().id, to); - assert_eq!(converter.name(), format!("rtf-to-{to}")); + assert_eq!(converter.entry().from.id, "rtf"); + assert_eq!(converter.entry().to.id, to); + assert_eq!(converter.entry().name, format!("rtf-to-{to}")); } for (converter, from) in [ (&DOCX_TO_RTF, "docx"), @@ -305,9 +291,9 @@ mod tests { (&HTML_TO_RTF, "html"), (&TEXT_TO_RTF, "text"), ] { - assert_eq!(converter.from().id, from); - assert_eq!(converter.to().id, "rtf"); - assert_eq!(converter.name(), format!("{from}-to-rtf")); + assert_eq!(converter.entry().from.id, from); + assert_eq!(converter.entry().to.id, "rtf"); + assert_eq!(converter.entry().name, format!("{from}-to-rtf")); } } } diff --git a/src/converters/text_to_docx.rs b/src/converters/text_to_docx.rs index f96b52f4..bc54e220 100644 --- a/src/converters/text_to_docx.rs +++ b/src/converters/text_to_docx.rs @@ -4,11 +4,10 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::converters::input::read_text_document; use crate::document::from_events::DocumentBuilder; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::docx::DocxStream; use crate::io::text::reader::parse_into; @@ -19,26 +18,6 @@ const FIDELITY_NOTE: &str = "paragraphs are runs of lines separated by blank lin pub struct TextToDocx; impl Converter for TextToDocx { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::TEXT - } - - fn to(&self) -> &'static Format { - &formats::DOCX - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -57,15 +36,31 @@ impl Converter for TextToDocx { } } +impl TextToDocx { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::TEXT, + to: &formats::DOCX, + fidelity: Fidelity::Conditional(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(TextToDocx.name(), "text-to-docx"); - assert_eq!(TextToDocx.from().id, "text"); - assert_eq!(TextToDocx.to().id, "docx"); - assert_eq!(TextToDocx.fidelity(), Fidelity::Conditional(FIDELITY_NOTE)); + assert_eq!(TextToDocx.entry().name, "text-to-docx"); + assert_eq!(TextToDocx.entry().from.id, "text"); + assert_eq!(TextToDocx.entry().to.id, "docx"); + assert_eq!( + TextToDocx.entry().fidelity, + Fidelity::Conditional(FIDELITY_NOTE) + ); } } diff --git a/src/converters/text_to_html.rs b/src/converters/text_to_html.rs index acbab119..0f8fcd04 100644 --- a/src/converters/text_to_html.rs +++ b/src/converters/text_to_html.rs @@ -3,10 +3,9 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::converters::input::read_text_document; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::html::HtmlWriter; use crate::io::text::reader::parse_into; @@ -18,26 +17,6 @@ const FIDELITY_NOTE: &str = pub struct TextToHtml; impl Converter for TextToHtml { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::TEXT - } - - fn to(&self) -> &'static Format { - &formats::HTML - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -53,15 +32,31 @@ impl Converter for TextToHtml { } } +impl TextToHtml { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::TEXT, + to: &formats::HTML, + fidelity: Fidelity::Conditional(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(TextToHtml.name(), "text-to-html"); - assert_eq!(TextToHtml.from().id, "text"); - assert_eq!(TextToHtml.to().id, "html"); - assert_eq!(TextToHtml.fidelity(), Fidelity::Conditional(FIDELITY_NOTE)); + assert_eq!(TextToHtml.entry().name, "text-to-html"); + assert_eq!(TextToHtml.entry().from.id, "text"); + assert_eq!(TextToHtml.entry().to.id, "html"); + assert_eq!( + TextToHtml.entry().fidelity, + Fidelity::Conditional(FIDELITY_NOTE) + ); } } diff --git a/src/converters/text_to_markdown.rs b/src/converters/text_to_markdown.rs index 0ea43e0e..2525bdfa 100644 --- a/src/converters/text_to_markdown.rs +++ b/src/converters/text_to_markdown.rs @@ -4,10 +4,9 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::converters::input::read_text_document; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::markdown::MarkdownWriter; use crate::io::text::reader::parse_into; @@ -18,26 +17,6 @@ const FIDELITY_NOTE: &str = "paragraphs are runs of lines separated by blank lin pub struct TextToMarkdown; impl Converter for TextToMarkdown { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::TEXT - } - - fn to(&self) -> &'static Format { - &formats::MARKDOWN - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -53,17 +32,30 @@ impl Converter for TextToMarkdown { } } +impl TextToMarkdown { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::TEXT, + to: &formats::MARKDOWN, + fidelity: Fidelity::Conditional(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(TextToMarkdown.name(), "text-to-markdown"); - assert_eq!(TextToMarkdown.from().id, "text"); - assert_eq!(TextToMarkdown.to().id, "markdown"); + assert_eq!(TextToMarkdown.entry().name, "text-to-markdown"); + assert_eq!(TextToMarkdown.entry().from.id, "text"); + assert_eq!(TextToMarkdown.entry().to.id, "markdown"); assert_eq!( - TextToMarkdown.fidelity(), + TextToMarkdown.entry().fidelity, Fidelity::Conditional(FIDELITY_NOTE) ); } diff --git a/src/converters/text_to_pages.rs b/src/converters/text_to_pages.rs index 22ee3080..10d2a25c 100644 --- a/src/converters/text_to_pages.rs +++ b/src/converters/text_to_pages.rs @@ -3,12 +3,11 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::converters::input::read_text_document; use crate::converters::pages_to_json::package_error; use crate::document::from_events::DocumentBuilder; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::pages::write_package_to; use crate::io::text::reader::parse_into; @@ -19,26 +18,6 @@ const FIDELITY_NOTE: &str = "paragraphs reach Pages as body text; text carries n pub struct TextToPages; impl Converter for TextToPages { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::TEXT - } - - fn to(&self) -> &'static Format { - &formats::PAGES - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Lossy(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -60,14 +39,27 @@ impl Converter for TextToPages { } } +impl TextToPages { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::TEXT, + to: &formats::PAGES, + fidelity: Fidelity::Lossy(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(TextToPages.name(), "text-to-pages"); - assert_eq!(TextToPages.from().id, "text"); - assert_eq!(TextToPages.to().id, "pages"); + assert_eq!(TextToPages.entry().name, "text-to-pages"); + assert_eq!(TextToPages.entry().from.id, "text"); + assert_eq!(TextToPages.entry().to.id, "pages"); } } diff --git a/src/converters/to_numbers.rs b/src/converters/to_numbers.rs index c840989b..2c12df51 100644 --- a/src/converters/to_numbers.rs +++ b/src/converters/to_numbers.rs @@ -10,7 +10,7 @@ use std::io::{Read, Write}; use crate::converter::MemoryParts; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::converters::pages_to_json::package_error; use crate::event::Context; use crate::format::Format; @@ -73,29 +73,6 @@ pub static MARKDOWN_TO_NUMBERS: ToNumbers = ToNumbers { }; impl Converter for ToNumbers { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - self.from - } - - fn to(&self) -> &'static Format { - &formats::NUMBERS - } - - fn fidelity(&self) -> Fidelity { - match self.source { - Source::Document => Fidelity::Lossy(DOCUMENT_NOTE), - _ => Fidelity::Conditional(NOTE), - } - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -196,6 +173,22 @@ impl Converter for ToNumbers { } } +impl ToNumbers { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: self.from, + to: &formats::NUMBERS, + fidelity: match self.source { + Source::Document => Fidelity::Lossy(DOCUMENT_NOTE), + _ => Fidelity::Conditional(NOTE), + }, + tier: Tier::Native, + converter: self, + } + } +} + fn one_table(sheet: &str, table: &str, rows: NumbersRows) -> NumbersSheet { NumbersSheet { name: sheet.to_string(), @@ -273,8 +266,8 @@ mod tests { #[test] fn declares_contract() { - assert_eq!(CSV_TO_NUMBERS.name(), "csv-to-numbers"); - assert_eq!(XLSX_TO_NUMBERS.from().id, "xlsx"); - assert_eq!(TSV_TO_NUMBERS.to().id, "numbers"); + assert_eq!(CSV_TO_NUMBERS.entry().name, "csv-to-numbers"); + assert_eq!(XLSX_TO_NUMBERS.entry().from.id, "xlsx"); + assert_eq!(TSV_TO_NUMBERS.entry().to.id, "numbers"); } } diff --git a/src/converters/to_pdf.rs b/src/converters/to_pdf.rs index 87ae06e7..6dcbb29a 100644 --- a/src/converters/to_pdf.rs +++ b/src/converters/to_pdf.rs @@ -5,7 +5,7 @@ use std::io::{Read, Write}; use std::sync::Arc; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Tier}; use crate::converters::input::read_text_document; use crate::converters::pages_to_json::package_error; use crate::document::Document; @@ -90,26 +90,6 @@ pub static RTF_TO_PDF: ToPdf = ToPdf { }; impl Converter for ToPdf { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - self.from - } - - fn to(&self) -> &'static Format { - &formats::PDF - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -187,6 +167,19 @@ impl Converter for ToPdf { } } +impl ToPdf { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: self.from, + to: &formats::PDF, + fidelity: Fidelity::Conditional(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + fn report(notes: &ComposeNotes, name: &'static str, context: &mut Context<'_>) { if notes.unset_characters > 0 { context.loss( diff --git a/src/converters/toml_to_json.rs b/src/converters/toml_to_json.rs index 2e8854ca..f09e38af 100644 --- a/src/converters/toml_to_json.rs +++ b/src/converters/toml_to_json.rs @@ -4,10 +4,9 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Tier}; use crate::converters::input::read_text_document; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::json::{JsonWriter, from_tree}; use crate::io::toml; @@ -18,26 +17,6 @@ const FIDELITY_NOTE: &str = "dates, times, infinities, and NaN become strings"; pub struct TomlToJson; impl Converter for TomlToJson { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::TOML - } - - fn to(&self) -> &'static Format { - &formats::JSON - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -53,15 +32,31 @@ impl Converter for TomlToJson { } } +impl TomlToJson { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::TOML, + to: &formats::JSON, + fidelity: Fidelity::Conditional(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(TomlToJson.name(), "toml-to-json"); - assert_eq!(TomlToJson.from().id, "toml"); - assert_eq!(TomlToJson.to().id, "json"); - assert_eq!(TomlToJson.fidelity(), Fidelity::Conditional(FIDELITY_NOTE)); + assert_eq!(TomlToJson.entry().name, "toml-to-json"); + assert_eq!(TomlToJson.entry().from.id, "toml"); + assert_eq!(TomlToJson.entry().to.id, "json"); + assert_eq!( + TomlToJson.entry().fidelity, + Fidelity::Conditional(FIDELITY_NOTE) + ); } } diff --git a/src/converters/xlsx_to_csv.rs b/src/converters/xlsx_to_csv.rs index dca2f514..282a149d 100644 --- a/src/converters/xlsx_to_csv.rs +++ b/src/converters/xlsx_to_csv.rs @@ -3,7 +3,7 @@ use std::io::{Read, Write}; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Parts, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Parts, Tier}; use crate::event::Context; use crate::format::Format; use crate::format::formats; @@ -32,26 +32,6 @@ pub static XLSX_TO_TSV: XlsxToCsv = XlsxToCsv { }; impl Converter for XlsxToCsv { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - &formats::XLSX - } - - fn to(&self) -> &'static Format { - self.to - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -97,6 +77,19 @@ impl Converter for XlsxToCsv { } } +impl XlsxToCsv { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: &formats::XLSX, + to: self.to, + fidelity: Fidelity::Conditional(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + impl XlsxToCsv { fn write_sheet( &self, @@ -155,26 +148,6 @@ pub static XLSX_TO_MARKDOWN: XlsxWhole = XlsxWhole { }; impl Converter for XlsxWhole { - fn name(&self) -> &'static str { - self.name - } - - fn from(&self) -> &'static Format { - &formats::XLSX - } - - fn to(&self) -> &'static Format { - self.to - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(self.note) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -203,6 +176,19 @@ impl Converter for XlsxWhole { } } +impl XlsxWhole { + pub const fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: &formats::XLSX, + to: self.to, + fidelity: Fidelity::Conditional(self.note), + tier: Tier::Native, + converter: self, + } + } +} + impl From for ConvertError { fn from(error: XlsxError) -> Self { match error { @@ -221,9 +207,12 @@ mod tests { #[test] fn declares_contract() { - assert_eq!(XLSX_TO_CSV.name(), "xlsx-to-csv"); - assert_eq!(XLSX_TO_CSV.from().id, "xlsx"); - assert_eq!(XLSX_TO_TSV.to().id, "tsv"); - assert_eq!(XLSX_TO_CSV.fidelity(), Fidelity::Conditional(FIDELITY_NOTE)); + assert_eq!(XLSX_TO_CSV.entry().name, "xlsx-to-csv"); + assert_eq!(XLSX_TO_CSV.entry().from.id, "xlsx"); + assert_eq!(XLSX_TO_TSV.entry().to.id, "tsv"); + assert_eq!( + XLSX_TO_CSV.entry().fidelity, + Fidelity::Conditional(FIDELITY_NOTE) + ); } } diff --git a/src/converters/xml_to_json.rs b/src/converters/xml_to_json.rs index 123d66bc..824c58ad 100644 --- a/src/converters/xml_to_json.rs +++ b/src/converters/xml_to_json.rs @@ -4,9 +4,8 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Tier}; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::json::{JsonWriter, from_tree}; use crate::io::xml::tree; @@ -17,26 +16,6 @@ const FIDELITY_NOTE: &str = "attributes become @-keys, text #text, repeated elem pub struct XmlToJson; impl Converter for XmlToJson { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::XML - } - - fn to(&self) -> &'static Format { - &formats::JSON - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -54,15 +33,31 @@ impl Converter for XmlToJson { } } +impl XmlToJson { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::XML, + to: &formats::JSON, + fidelity: Fidelity::Conditional(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(XmlToJson.name(), "xml-to-json"); - assert_eq!(XmlToJson.from().id, "xml"); - assert_eq!(XmlToJson.to().id, "json"); - assert_eq!(XmlToJson.fidelity(), Fidelity::Conditional(FIDELITY_NOTE)); + assert_eq!(XmlToJson.entry().name, "xml-to-json"); + assert_eq!(XmlToJson.entry().from.id, "xml"); + assert_eq!(XmlToJson.entry().to.id, "json"); + assert_eq!( + XmlToJson.entry().fidelity, + Fidelity::Conditional(FIDELITY_NOTE) + ); } } diff --git a/src/converters/yaml_to_json.rs b/src/converters/yaml_to_json.rs index 5b72fc28..ce8c2f13 100644 --- a/src/converters/yaml_to_json.rs +++ b/src/converters/yaml_to_json.rs @@ -6,10 +6,9 @@ use std::io::Write; -use crate::converter::{ConvertError, Converter, Fidelity, Input, Location, Tier}; +use crate::converter::{ConvertError, Converter, Entry, Fidelity, Input, Location, Tier}; use crate::converters::input::read_text_document; use crate::event::Context; -use crate::format::Format; use crate::format::formats; use crate::io::json::{JsonWriter, from_tree}; use crate::io::yaml; @@ -21,26 +20,6 @@ const FIDELITY_NOTE: &str = "anchors are expanded, tags outside the core schema pub struct YamlToJson; impl Converter for YamlToJson { - fn name(&self) -> &'static str { - NAME - } - - fn from(&self) -> &'static Format { - &formats::YAML - } - - fn to(&self) -> &'static Format { - &formats::JSON - } - - fn fidelity(&self) -> Fidelity { - Fidelity::Conditional(FIDELITY_NOTE) - } - - fn tier(&self) -> Tier { - Tier::Native - } - fn convert( &self, mut input: Input<'_>, @@ -71,15 +50,31 @@ impl Converter for YamlToJson { } } +impl YamlToJson { + pub const fn entry(&'static self) -> Entry { + Entry { + name: NAME, + from: &formats::YAML, + to: &formats::JSON, + fidelity: Fidelity::Conditional(FIDELITY_NOTE), + tier: Tier::Native, + converter: self, + } + } +} + #[cfg(test)] mod tests { use super::*; #[test] fn declares_contract() { - assert_eq!(YamlToJson.name(), "yaml-to-json"); - assert_eq!(YamlToJson.from().id, "yaml"); - assert_eq!(YamlToJson.to().id, "json"); - assert_eq!(YamlToJson.fidelity(), Fidelity::Conditional(FIDELITY_NOTE)); + assert_eq!(YamlToJson.entry().name, "yaml-to-json"); + assert_eq!(YamlToJson.entry().from.id, "yaml"); + assert_eq!(YamlToJson.entry().to.id, "json"); + assert_eq!( + YamlToJson.entry().fidelity, + Fidelity::Conditional(FIDELITY_NOTE) + ); } } diff --git a/src/planner/chain.rs b/src/planner/chain.rs index a366d25d..68ba2bdb 100644 --- a/src/planner/chain.rs +++ b/src/planner/chain.rs @@ -3,7 +3,7 @@ use std::io::{self, BufWriter, Write}; -use crate::converter::{ConvertError, ConvertOptions, Converter, Input}; +use crate::converter::{ConvertError, ConvertOptions, Entry, Input}; use crate::event::{CollectingSink, Context}; use crate::planner::Plan; use crate::planner::execute::run_hop; @@ -71,7 +71,7 @@ pub(super) fn run_chain( } fn run_hop_into_pipe( - converter: &'static dyn Converter, + converter: &'static Entry, source: HopSource<'_>, pipe_writer: PipeWriter, options: &ConvertOptions, @@ -89,7 +89,7 @@ fn run_hop_into_pipe( } fn run_hop_collecting( - converter: &'static dyn Converter, + converter: &'static Entry, source: HopSource<'_>, output: &mut dyn Write, options: &ConvertOptions, diff --git a/src/planner/execute.rs b/src/planner/execute.rs index 884fb9a7..6d9791db 100644 --- a/src/planner/execute.rs +++ b/src/planner/execute.rs @@ -6,7 +6,7 @@ use std::io::Write; #[cfg(not(target_arch = "wasm32"))] use std::time::Instant; -use crate::converter::{ConvertError, Converter, Input, MemoryParts, Parts}; +use crate::converter::{ConvertError, Entry, Input, MemoryParts, Parts}; use crate::event::{Context, Event}; use crate::planner::Plan; #[cfg(not(target_arch = "wasm32"))] @@ -54,19 +54,19 @@ pub fn execute_in_memory( for converter in first_hops { let mut next = Vec::new(); if is_first { - run_hop(*converter, Input::Stream(&mut input), &mut next, context)?; + run_hop(converter, Input::Stream(&mut input), &mut next, context)?; is_first = false; } else { let mut source: &[u8] = &buffer; - run_hop(*converter, Input::Stream(&mut source), &mut next, context)?; + run_hop(converter, Input::Stream(&mut source), &mut next, context)?; } buffer = next; } if is_first { - run_hop(*last, input, output, context) + run_hop(last, input, output, context) } else { let mut source: &[u8] = &buffer; - run_hop(*last, Input::Stream(&mut source), output, context) + run_hop(last, Input::Stream(&mut source), output, context) } } @@ -83,11 +83,11 @@ pub fn execute_parts( let target_splits = plan .hops .last() - .is_some_and(|last| last.to().holds_one_part()); + .is_some_and(|last| last.to.holds_one_part()); let split_at = plan .hops .iter() - .position(|hop| hop.splits()) + .position(|hop| hop.converter.splits()) .filter(|_| target_splits); let Some(split_at) = split_at else { let output = parts.part("")?; @@ -100,7 +100,7 @@ pub fn execute_parts( )); }; if before.is_empty() { - return split_and_finish(*splitter, after, input, parts, context); + return split_and_finish(splitter, after, input, parts, context); } // The hops before the split make one input for it. let head = Plan { @@ -109,13 +109,13 @@ pub fn execute_parts( let mut prepared = Vec::new(); execute(&head, input, &mut prepared, context)?; let mut source: &[u8] = &prepared; - split_and_finish(*splitter, after, Input::Stream(&mut source), parts, context) + split_and_finish(splitter, after, Input::Stream(&mut source), parts, context) } /// Runs the splitting hop, then the hops after it once per part. fn split_and_finish( - splitter: &'static dyn Converter, - after: &[&'static dyn Converter], + splitter: &'static Entry, + after: &[&'static Entry], input: Input<'_>, parts: &mut dyn Parts, context: &mut Context<'_>, @@ -138,46 +138,46 @@ fn split_and_finish( } fn run_hop_parts( - converter: &'static dyn Converter, + converter: &'static Entry, input: Input<'_>, parts: &mut dyn Parts, context: &mut Context<'_>, ) -> Result<(), ConvertError> { context.emit(Event::StepStarted { - converter: converter.name(), + converter: converter.name, }); #[cfg(not(target_arch = "wasm32"))] let started = Instant::now(); - let result = converter.convert_parts(input, parts, context); + let result = converter.converter.convert_parts(input, parts, context); #[cfg(not(target_arch = "wasm32"))] let elapsed = started.elapsed(); #[cfg(target_arch = "wasm32")] let elapsed = std::time::Duration::ZERO; context.emit(Event::StepFinished { - converter: converter.name(), + converter: converter.name, elapsed, }); result } pub(super) fn run_hop( - converter: &'static dyn Converter, + converter: &'static Entry, input: Input<'_>, output: &mut dyn Write, context: &mut Context<'_>, ) -> Result<(), ConvertError> { context.emit(Event::StepStarted { - converter: converter.name(), + converter: converter.name, }); #[cfg(not(target_arch = "wasm32"))] let started = Instant::now(); - let result = converter.convert(input, output, context); + let result = converter.converter.convert(input, output, context); #[cfg(not(target_arch = "wasm32"))] let elapsed = started.elapsed(); #[cfg(target_arch = "wasm32")] let elapsed = std::time::Duration::ZERO; context.emit(Event::StepFinished { - converter: converter.name(), + converter: converter.name, elapsed, }); result @@ -186,7 +186,7 @@ pub(super) fn run_hop( #[cfg(test)] mod tests { use super::*; - use crate::converter::{ConvertOptions, Converter, Fidelity, Input, Tier}; + use crate::converter::{ConvertOptions, Converter, Entry, Fidelity, Input, Tier}; use crate::event::{CollectingSink, Event}; use crate::format::{Category, Format}; use std::io::{Read, Write}; @@ -235,21 +235,6 @@ mod tests { struct WritesALot; impl Converter for Upper { - fn name(&self) -> &'static str { - "upper" - } - fn from(&self) -> &'static Format { - &A - } - fn to(&self) -> &'static Format { - &B - } - fn fidelity(&self) -> Fidelity { - Fidelity::Lossless - } - fn tier(&self) -> Tier { - Tier::Native - } fn convert( &self, mut input: Input<'_>, @@ -265,21 +250,6 @@ mod tests { } impl Converter for Exclaim { - fn name(&self) -> &'static str { - "exclaim" - } - fn from(&self) -> &'static Format { - &B - } - fn to(&self) -> &'static Format { - &C - } - fn fidelity(&self) -> Fidelity { - Fidelity::Lossless - } - fn tier(&self) -> Tier { - Tier::Native - } fn convert( &self, mut input: Input<'_>, @@ -296,21 +266,6 @@ mod tests { } impl Converter for Fails { - fn name(&self) -> &'static str { - "fails" - } - fn from(&self) -> &'static Format { - &B - } - fn to(&self) -> &'static Format { - &C - } - fn fidelity(&self) -> Fidelity { - Fidelity::Lossless - } - fn tier(&self) -> Tier { - Tier::Native - } fn convert( &self, _input: Input<'_>, @@ -322,21 +277,6 @@ mod tests { } impl Converter for Tail { - fn name(&self) -> &'static str { - "tail" - } - fn from(&self) -> &'static Format { - &C - } - fn to(&self) -> &'static Format { - &D - } - fn fidelity(&self) -> Fidelity { - Fidelity::Lossless - } - fn tier(&self) -> Tier { - Tier::Native - } fn convert( &self, mut input: Input<'_>, @@ -353,21 +293,6 @@ mod tests { } impl Converter for WritesALot { - fn name(&self) -> &'static str { - "writes_a_lot" - } - fn from(&self) -> &'static Format { - &A - } - fn to(&self) -> &'static Format { - &B - } - fn fidelity(&self) -> Fidelity { - Fidelity::Lossless - } - fn tier(&self) -> Tier { - Tier::Native - } fn convert( &self, _input: Input<'_>, @@ -380,11 +305,46 @@ mod tests { } } - static UPPER: Upper = Upper; - static EXCLAIM: Exclaim = Exclaim; - static FAILS: Fails = Fails; - static TAIL: Tail = Tail; - static WRITES_A_LOT: WritesALot = WritesALot; + static UPPER: Entry = Entry { + name: "upper", + from: &A, + to: &B, + fidelity: Fidelity::Lossless, + tier: Tier::Native, + converter: &Upper, + }; + static EXCLAIM: Entry = Entry { + name: "exclaim", + from: &B, + to: &C, + fidelity: Fidelity::Lossless, + tier: Tier::Native, + converter: &Exclaim, + }; + static FAILS: Entry = Entry { + name: "fails", + from: &B, + to: &C, + fidelity: Fidelity::Lossless, + tier: Tier::Native, + converter: &Fails, + }; + static TAIL: Entry = Entry { + name: "tail", + from: &C, + to: &D, + fidelity: Fidelity::Lossless, + tier: Tier::Native, + converter: &Tail, + }; + static WRITES_A_LOT: Entry = Entry { + name: "writes_a_lot", + from: &A, + to: &B, + fidelity: Fidelity::Lossless, + tier: Tier::Native, + converter: &WritesALot, + }; fn run(plan: &Plan, input: &[u8]) -> (Result<(), ConvertError>, Vec, CollectingSink) { let options = ConvertOptions::default(); diff --git a/src/planner/mod.rs b/src/planner/mod.rs index d52ad910..3b565dce 100644 --- a/src/planner/mod.rs +++ b/src/planner/mod.rs @@ -8,19 +8,19 @@ pub mod pipe; use std::fmt; -use crate::converter::{Converter, FidelityKind}; +use crate::converter::{Entry, FidelityKind}; use crate::event::Hop; use crate::format::Format; pub use execute::{execute, execute_in_memory, execute_parts}; pub struct Plan { - pub hops: Vec<&'static dyn Converter>, + pub hops: Vec<&'static Entry>, } impl fmt::Debug for Plan { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - let names: Vec<&'static str> = self.hops.iter().map(|hop| hop.name()).collect(); + let names: Vec<&'static str> = self.hops.iter().map(|hop| hop.name).collect(); formatter .debug_struct("Plan") .field("hops", &names) @@ -30,19 +30,19 @@ impl fmt::Debug for Plan { impl Plan { pub fn from(&self) -> &'static Format { - self.hops[0].from() + self.hops[0].from } pub fn to(&self) -> &'static Format { let last_index = self.hops.len() - 1; - self.hops[last_index].to() + self.hops[last_index].to } /// The options some hop honours (`Converter::options`). pub fn options(&self) -> Vec { let mut options: Vec = Vec::new(); for hop in &self.hops { - for option in hop.options() { + for option in hop.converter.options(hop.from, hop.to) { if !options.contains(&option) { options.push(option); } @@ -54,7 +54,7 @@ impl Plan { pub fn worst_fidelity(&self) -> FidelityKind { let mut worst = FidelityKind::Lossless; for hop in &self.hops { - let kind = hop.fidelity().kind(); + let kind = hop.fidelity.kind(); if kind > worst { worst = kind; } @@ -69,7 +69,7 @@ impl Plan { pub fn loss_descriptions(&self) -> Vec<&'static str> { let mut descriptions = Vec::new(); for hop in &self.hops { - if let Some(text) = hop.fidelity().description() { + if let Some(text) = hop.fidelity.description() { descriptions.push(text); } } @@ -80,11 +80,11 @@ impl Plan { let mut hops = Vec::with_capacity(self.hops.len()); for converter in &self.hops { let hop = Hop { - converter: converter.name(), - from: converter.from().id, - to: converter.to().id, - fidelity: converter.fidelity(), - tier: converter.tier(), + converter: converter.name, + from: converter.from.id, + to: converter.to.id, + fidelity: converter.fidelity, + tier: converter.tier, }; hops.push(hop); } @@ -135,7 +135,7 @@ impl fmt::Display for PlanError { impl std::error::Error for PlanError {} pub fn plan( - converters: &[&'static dyn Converter], + converters: &'static [Entry], from: &'static Format, to: &'static Format, options: &PlanOptions, @@ -159,7 +159,7 @@ pub fn plan( } fn search( - converters: &[&'static dyn Converter], + converters: &'static [Entry], from: &'static Format, to: &'static Format, strict: bool, @@ -182,8 +182,8 @@ fn search( let mut pruned = Vec::new(); if let Some(plan) = unrestricted { for hop in plan.hops { - if !hop.fidelity().is_lossless() { - pruned.push(hop.name()); + if !hop.fidelity.is_lossless() { + pruned.push(hop.name); } } } @@ -209,7 +209,7 @@ impl PathCost { /// Readable Dijkstra over a graph of at most a few hundred formats. fn dijkstra( - converters: &[&'static dyn Converter], + converters: &'static [Entry], from: &'static Format, to: &'static Format, lossless_only: bool, @@ -217,15 +217,15 @@ fn dijkstra( let mut node_ids: Vec<&'static str> = Vec::new(); push_node(&mut node_ids, from.id); push_node(&mut node_ids, to.id); - let mut edges: Vec<&'static dyn Converter> = Vec::new(); + let mut edges: Vec<&'static Entry> = Vec::new(); for converter in converters { - let allowed = !lossless_only || converter.fidelity().is_lossless(); + let allowed = !lossless_only || converter.fidelity.is_lossless(); if !allowed { continue; } - push_node(&mut node_ids, converter.from().id); - push_node(&mut node_ids, converter.to().id); - edges.push(*converter); + push_node(&mut node_ids, converter.from.id); + push_node(&mut node_ids, converter.to.id); + edges.push(converter); } let node_count = node_ids.len(); @@ -250,21 +250,21 @@ fn dijkstra( None => break, }; for (edge_index, converter) in edges.iter().enumerate() { - let edge_source = index_of(&node_ids, converter.from().id); + let edge_source = index_of(&node_ids, converter.from.id); if edge_source != current { continue; } - let neighbor = index_of(&node_ids, converter.to().id); + let neighbor = index_of(&node_ids, converter.to.id); let candidate = PathCost { - cost: current_cost.cost + converter.fidelity().cost(), + cost: current_cost.cost + converter.fidelity.cost(), hops: current_cost.hops + 1, - tier: current_cost.tier + converter.tier().rank(), + tier: current_cost.tier + converter.tier.rank(), }; let is_better = match best[neighbor] { None => true, Some(existing) => { let existing_name = incoming_name(&edges, incoming_edge[neighbor]); - beats(candidate, existing, converter.name(), existing_name) + beats(candidate, existing, converter.name, existing_name) } }; if is_better { @@ -275,13 +275,13 @@ fn dijkstra( } best[target]?; - let mut hops: Vec<&'static dyn Converter> = Vec::new(); + let mut hops: Vec<&'static Entry> = Vec::new(); let mut cursor = target; while cursor != source { let edge_index = incoming_edge[cursor]?; let converter = edges[edge_index]; hops.push(converter); - cursor = index_of(&node_ids, converter.from().id); + cursor = index_of(&node_ids, converter.from.id); } hops.reverse(); Some(Plan { hops }) @@ -327,9 +327,9 @@ fn pick_next(best: &[Option], visited: &[bool]) -> Option { chosen } -fn incoming_name(edges: &[&'static dyn Converter], edge_index: Option) -> &'static str { +fn incoming_name(edges: &[&'static Entry], edge_index: Option) -> &'static str { match edge_index { - Some(index) => edges[index].name(), + Some(index) => edges[index].name, None => "", } } @@ -349,7 +349,7 @@ fn beats( #[cfg(test)] mod tests { use super::*; - use crate::converter::{ConvertError, Fidelity, Input, Tier}; + use crate::converter::{ConvertError, Converter, Fidelity, Input, Tier}; use crate::event::Context; use crate::format::Category; use std::io::Write; @@ -400,21 +400,6 @@ mod tests { } impl Converter for Fake { - fn name(&self) -> &'static str { - self.name - } - fn from(&self) -> &'static Format { - self.from - } - fn to(&self) -> &'static Format { - self.to - } - fn fidelity(&self) -> Fidelity { - self.fidelity.clone() - } - fn tier(&self) -> Tier { - self.tier - } fn convert( &self, _input: Input<'_>, @@ -425,6 +410,24 @@ mod tests { } } + impl Fake { + fn entry(&'static self) -> Entry { + Entry { + name: self.name, + from: self.from, + to: self.to, + fidelity: self.fidelity, + tier: self.tier, + converter: self, + } + } + } + + fn graph(fakes: &[&'static Fake]) -> &'static [Entry] { + let entries: Vec = fakes.iter().map(|fake| fake.entry()).collect(); + Box::leak(entries.into_boxed_slice()) + } + static A_B: Fake = Fake { name: "a-b", from: &A, @@ -469,21 +472,21 @@ mod tests { }; fn names(plan: &Plan) -> Vec<&'static str> { - plan.hops.iter().map(|hop| hop.name()).collect() + plan.hops.iter().map(|hop| hop.name).collect() } #[test] fn prefers_three_lossless_hops_over_one_lossy_hop() { - let converters: Vec<&'static dyn Converter> = vec![&A_B, &B_C, &C_D, &A_D_LOSSY]; - let plan = plan(&converters, &A, &D, &PlanOptions::default()).unwrap(); + let converters = graph(&[&A_B, &B_C, &C_D, &A_D_LOSSY]); + let plan = plan(converters, &A, &D, &PlanOptions::default()).unwrap(); assert_eq!(names(&plan), vec!["a-b", "b-c", "c-d"]); assert!(plan.is_lossless()); } #[test] fn takes_the_lossy_hop_when_it_is_the_only_way() { - let converters: Vec<&'static dyn Converter> = vec![&A_D_LOSSY]; - let plan = plan(&converters, &A, &D, &PlanOptions::default()).unwrap(); + let converters = graph(&[&A_D_LOSSY]); + let plan = plan(converters, &A, &D, &PlanOptions::default()).unwrap(); assert_eq!(names(&plan), vec!["a-d-lossy"]); assert_eq!(plan.worst_fidelity(), FidelityKind::Lossy); assert_eq!(plan.loss_descriptions(), vec!["drops"]); @@ -491,12 +494,12 @@ mod tests { #[test] fn strict_refuses_lossy_and_names_the_pruned_edges() { - let converters: Vec<&'static dyn Converter> = vec![&A_D_LOSSY, &A_C_COND, &C_D]; + let converters = graph(&[&A_D_LOSSY, &A_C_COND, &C_D]); let options = PlanOptions { strict: true, via: None, }; - let error = plan(&converters, &A, &D, &options).unwrap_err(); + let error = plan(converters, &A, &D, &options).unwrap_err(); match error { PlanError::NoPath { pruned, .. } => assert_eq!(pruned, vec!["a-c-cond"]), other => panic!("wrong error {other:?}"), @@ -505,18 +508,18 @@ mod tests { #[test] fn via_forces_a_waypoint() { - let converters: Vec<&'static dyn Converter> = vec![&A_B, &B_C, &C_D, &A_D_LOSSY, &A_C_COND]; + let converters = graph(&[&A_B, &B_C, &C_D, &A_D_LOSSY, &A_C_COND]); let options = PlanOptions { strict: false, via: Some(&C), }; - let plan = plan(&converters, &A, &D, &options).unwrap(); + let plan = plan(converters, &A, &D, &options).unwrap(); assert_eq!(names(&plan), vec!["a-b", "b-c", "c-d"]); } #[test] fn via_equal_to_an_endpoint_is_ignored() { - let converters: Vec<&'static dyn Converter> = vec![&A_B, &B_C]; + let converters = graph(&[&A_B, &B_C]); let via_from = PlanOptions { strict: false, via: Some(&A), @@ -525,38 +528,38 @@ mod tests { strict: false, via: Some(&C), }; - let first = plan(&converters, &A, &C, &via_from).unwrap(); - let second = plan(&converters, &A, &C, &via_to).unwrap(); + let first = plan(converters, &A, &C, &via_from).unwrap(); + let second = plan(converters, &A, &C, &via_to).unwrap(); assert_eq!(names(&first), vec!["a-b", "b-c"]); assert_eq!(names(&second), vec!["a-b", "b-c"]); } #[test] fn no_path_is_an_error() { - let converters: Vec<&'static dyn Converter> = vec![&A_B]; - let error = plan(&converters, &A, &D, &PlanOptions::default()).unwrap_err(); + let converters = graph(&[&A_B]); + let error = plan(converters, &A, &D, &PlanOptions::default()).unwrap_err(); let is_no_path = matches!(error, PlanError::NoPath { .. }); assert!(is_no_path); } #[test] fn same_format_is_an_error() { - let converters: Vec<&'static dyn Converter> = vec![&A_B]; - let error = plan(&converters, &A, &A, &PlanOptions::default()).unwrap_err(); + let converters = graph(&[&A_B]); + let error = plan(converters, &A, &A, &PlanOptions::default()).unwrap_err(); assert_eq!(error, PlanError::SameFormat("a")); } #[test] fn native_beats_external_on_a_tie() { - let converters: Vec<&'static dyn Converter> = vec![&A_B_EXTERNAL, &A_B]; - let plan = plan(&converters, &A, &B, &PlanOptions::default()).unwrap(); + let converters = graph(&[&A_B_EXTERNAL, &A_B]); + let plan = plan(converters, &A, &B, &PlanOptions::default()).unwrap(); assert_eq!(names(&plan), vec!["a-b"]); } #[test] fn describe_lists_hops() { - let converters: Vec<&'static dyn Converter> = vec![&A_B]; - let plan = plan(&converters, &A, &B, &PlanOptions::default()).unwrap(); + let converters = graph(&[&A_B]); + let plan = plan(converters, &A, &B, &PlanOptions::default()).unwrap(); let hops = plan.describe(); assert_eq!(hops.len(), 1); assert_eq!(hops[0].converter, "a-b"); diff --git a/src/registry.rs b/src/registry.rs index 33e49386..20ab1de0 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -3,7 +3,7 @@ use std::sync::OnceLock; -use crate::converter::Converter; +use crate::converter::Entry; use crate::converters::csv_to_json; use crate::converters::csv_to_xlsx; use crate::converters::docx_to_html::DocxToHtml; @@ -54,167 +54,195 @@ use crate::converters::xml_to_json::XmlToJson; use crate::converters::yaml_to_json::YamlToJson; use crate::format::Format; -static CONVERTERS: [&dyn Converter; 140] = [ - &csv_to_json::CSV_TO_JSON, - &json_to_csv::JSON_TO_CSV, - &MarkdownToHtml, - &MarkdownToText, - &MarkdownToJson, - &MarkdownJsonToMarkdown, - &PagesToJson, - &JsonToPages, - &PagesToDocx, - &PagesToMarkdown, - &PagesToHtml, - &PagesToText, - &DocxToMarkdown, - &DocxToHtml, - &DocxToText, - &MarkdownToDocx, - &MarkdownToPages, - &TextToPages, - &HtmlToPages, - &DocxToPages, - &HtmlToMarkdown, - &HtmlToText, - &HtmlToDocx, - &TextToMarkdown, - &TextToHtml, - &TextToDocx, - &PdfToText, - &pdf_to_document::PDF_TO_MARKDOWN, - &pdf_to_document::PDF_TO_HTML, - &pdf_to_document::PDF_TO_DOCX, - &to_pdf::MARKDOWN_TO_PDF, - &to_pdf::HTML_TO_PDF, - &to_pdf::TEXT_TO_PDF, - &to_pdf::DOCX_TO_PDF, - &to_pdf::PAGES_TO_PDF, - &to_pdf::RTF_TO_PDF, - &epub::EPUB_TO_DOCX, - &epub::EPUB_TO_PAGES, - &epub::EPUB_TO_RTF, - &epub::EPUB_TO_ODT, - &epub::EPUB_TO_MARKDOWN, - &epub::EPUB_TO_HTML, - &epub::EPUB_TO_TEXT, - &to_pdf::EPUB_TO_PDF, - &epub::DOCX_TO_EPUB, - &epub::PAGES_TO_EPUB, - &epub::RTF_TO_EPUB, - &epub::ODT_TO_EPUB, - &epub::MARKDOWN_TO_EPUB, - &epub::HTML_TO_EPUB, - &epub::TEXT_TO_EPUB, - &pdf_to_document::PDF_TO_EPUB, - &odt::ODT_TO_DOCX, - &odt::ODT_TO_PAGES, - &odt::ODT_TO_RTF, - &odt::ODT_TO_MARKDOWN, - &odt::ODT_TO_HTML, - &odt::ODT_TO_TEXT, - &to_pdf::ODT_TO_PDF, - &odt::DOCX_TO_ODT, - &odt::PAGES_TO_ODT, - &odt::RTF_TO_ODT, - &odt::MARKDOWN_TO_ODT, - &odt::HTML_TO_ODT, - &odt::TEXT_TO_ODT, - &pdf_to_document::PDF_TO_ODT, - &rtf::RTF_TO_DOCX, - &rtf::RTF_TO_PAGES, - &rtf::RTF_TO_MARKDOWN, - &rtf::RTF_TO_HTML, - &rtf::RTF_TO_TEXT, - &rtf::DOCX_TO_RTF, - &rtf::PAGES_TO_RTF, - &rtf::MARKDOWN_TO_RTF, - &rtf::HTML_TO_RTF, - &rtf::TEXT_TO_RTF, - &TomlToJson, - &JsonToToml, - &YamlToJson, - &JsonToYaml, - &XmlToJson, - &JsonToXml, - &hub::TOML_TO_YAML, - &hub::YAML_TO_TOML, - &hub::TOML_TO_XML, - &hub::XML_TO_TOML, - &hub::YAML_TO_XML, - &hub::XML_TO_YAML, - &csv_to_json::TSV_TO_JSON, - &csv_to_json::CSV_TO_JSONL, - &csv_to_json::TSV_TO_JSONL, - &json_to_csv::JSON_TO_TSV, - &json_to_csv::JSONL_TO_CSV, - &json_to_csv::JSONL_TO_TSV, - &rows::CSV_TO_TSV, - &rows::TSV_TO_CSV, - &rows::JsonlToJson, - &rows::JsonToJsonl, - &xlsx_to_csv::XLSX_TO_CSV, - &xlsx_to_csv::XLSX_TO_TSV, - &xlsx_to_csv::XLSX_TO_JSON, - &xlsx_to_csv::XLSX_TO_MARKDOWN, - &numbers::NUMBERS_TO_CSV, - &numbers::NUMBERS_TO_TSV, - &numbers::NUMBERS_TO_JSON, - &numbers::NUMBERS_TO_MARKDOWN, - &numbers::NUMBERS_TO_XLSX, - &ods::ODS_TO_CSV, - &ods::ODS_TO_TSV, - &ods::ODS_TO_JSON, - &ods::ODS_TO_MARKDOWN, - &ods::ODS_TO_XLSX, - &ods::ODS_TO_NUMBERS, - &ods::CSV_TO_ODS, - &ods::TSV_TO_ODS, - &ods::XLSX_TO_ODS, - &ods::JSON_TO_ODS, - &ods::MARKDOWN_TO_ODS, - &ods::NUMBERS_TO_ODS, - &to_numbers::CSV_TO_NUMBERS, - &to_numbers::TSV_TO_NUMBERS, - &to_numbers::XLSX_TO_NUMBERS, - &to_numbers::JSON_TO_NUMBERS, - &to_numbers::MARKDOWN_TO_NUMBERS, - &rows_document::MARKDOWN_TO_XLSX, - &json_to_csv::JSON_TO_XLSX, - &rows_document::MARKDOWN_TABLES_TO_JSON, - &rows_document::JSON_TO_MARKDOWN, - &csv_to_xlsx::CSV_TO_XLSX, - &csv_to_xlsx::TSV_TO_XLSX, - &rows_document::CSV_TO_MARKDOWN, - &rows_document::TSV_TO_MARKDOWN, - &rows_document::MARKDOWN_TO_CSV, - &rows_document::MARKDOWN_TO_TSV, - &n64::Z64_TO_V64, - &n64::Z64_TO_N64, - &n64::V64_TO_Z64, - &n64::V64_TO_N64, - &n64::N64_TO_Z64, - &n64::N64_TO_V64, +static CONVERTERS: [Entry; 140] = [ + csv_to_json::CSV_TO_JSON.entry(), + json_to_csv::JSON_TO_CSV.entry(), + MarkdownToHtml.entry(), + MarkdownToText.entry(), + MarkdownToJson.entry(), + MarkdownJsonToMarkdown.entry(), + PagesToJson.entry(), + JsonToPages.entry(), + PagesToDocx.entry(), + PagesToMarkdown.entry(), + PagesToHtml.entry(), + PagesToText.entry(), + DocxToMarkdown.entry(), + DocxToHtml.entry(), + DocxToText.entry(), + MarkdownToDocx.entry(), + MarkdownToPages.entry(), + TextToPages.entry(), + HtmlToPages.entry(), + DocxToPages.entry(), + HtmlToMarkdown.entry(), + HtmlToText.entry(), + HtmlToDocx.entry(), + TextToMarkdown.entry(), + TextToHtml.entry(), + TextToDocx.entry(), + PdfToText.entry(), + pdf_to_document::PDF_TO_MARKDOWN.entry(), + pdf_to_document::PDF_TO_HTML.entry(), + pdf_to_document::PDF_TO_DOCX.entry(), + to_pdf::MARKDOWN_TO_PDF.entry(), + to_pdf::HTML_TO_PDF.entry(), + to_pdf::TEXT_TO_PDF.entry(), + to_pdf::DOCX_TO_PDF.entry(), + to_pdf::PAGES_TO_PDF.entry(), + to_pdf::RTF_TO_PDF.entry(), + epub::EPUB_TO_DOCX.entry(), + epub::EPUB_TO_PAGES.entry(), + epub::EPUB_TO_RTF.entry(), + epub::EPUB_TO_ODT.entry(), + epub::EPUB_TO_MARKDOWN.entry(), + epub::EPUB_TO_HTML.entry(), + epub::EPUB_TO_TEXT.entry(), + to_pdf::EPUB_TO_PDF.entry(), + epub::DOCX_TO_EPUB.entry(), + epub::PAGES_TO_EPUB.entry(), + epub::RTF_TO_EPUB.entry(), + epub::ODT_TO_EPUB.entry(), + epub::MARKDOWN_TO_EPUB.entry(), + epub::HTML_TO_EPUB.entry(), + epub::TEXT_TO_EPUB.entry(), + pdf_to_document::PDF_TO_EPUB.entry(), + odt::ODT_TO_DOCX.entry(), + odt::ODT_TO_PAGES.entry(), + odt::ODT_TO_RTF.entry(), + odt::ODT_TO_MARKDOWN.entry(), + odt::ODT_TO_HTML.entry(), + odt::ODT_TO_TEXT.entry(), + to_pdf::ODT_TO_PDF.entry(), + odt::DOCX_TO_ODT.entry(), + odt::PAGES_TO_ODT.entry(), + odt::RTF_TO_ODT.entry(), + odt::MARKDOWN_TO_ODT.entry(), + odt::HTML_TO_ODT.entry(), + odt::TEXT_TO_ODT.entry(), + pdf_to_document::PDF_TO_ODT.entry(), + rtf::RTF_TO_DOCX.entry(), + rtf::RTF_TO_PAGES.entry(), + rtf::RTF_TO_MARKDOWN.entry(), + rtf::RTF_TO_HTML.entry(), + rtf::RTF_TO_TEXT.entry(), + rtf::DOCX_TO_RTF.entry(), + rtf::PAGES_TO_RTF.entry(), + rtf::MARKDOWN_TO_RTF.entry(), + rtf::HTML_TO_RTF.entry(), + rtf::TEXT_TO_RTF.entry(), + TomlToJson.entry(), + JsonToToml.entry(), + YamlToJson.entry(), + JsonToYaml.entry(), + XmlToJson.entry(), + JsonToXml.entry(), + hub::TOML_TO_YAML.entry(), + hub::YAML_TO_TOML.entry(), + hub::TOML_TO_XML.entry(), + hub::XML_TO_TOML.entry(), + hub::YAML_TO_XML.entry(), + hub::XML_TO_YAML.entry(), + csv_to_json::TSV_TO_JSON.entry(), + csv_to_json::CSV_TO_JSONL.entry(), + csv_to_json::TSV_TO_JSONL.entry(), + json_to_csv::JSON_TO_TSV.entry(), + json_to_csv::JSONL_TO_CSV.entry(), + json_to_csv::JSONL_TO_TSV.entry(), + rows::CSV_TO_TSV.entry(), + rows::TSV_TO_CSV.entry(), + rows::JsonlToJson.entry(), + rows::JsonToJsonl.entry(), + xlsx_to_csv::XLSX_TO_CSV.entry(), + xlsx_to_csv::XLSX_TO_TSV.entry(), + xlsx_to_csv::XLSX_TO_JSON.entry(), + xlsx_to_csv::XLSX_TO_MARKDOWN.entry(), + numbers::NUMBERS_TO_CSV.entry(), + numbers::NUMBERS_TO_TSV.entry(), + numbers::NUMBERS_TO_JSON.entry(), + numbers::NUMBERS_TO_MARKDOWN.entry(), + numbers::NUMBERS_TO_XLSX.entry(), + ods::ODS_TO_CSV.entry(), + ods::ODS_TO_TSV.entry(), + ods::ODS_TO_JSON.entry(), + ods::ODS_TO_MARKDOWN.entry(), + ods::ODS_TO_XLSX.entry(), + ods::ODS_TO_NUMBERS.entry(), + ods::CSV_TO_ODS.entry(), + ods::TSV_TO_ODS.entry(), + ods::XLSX_TO_ODS.entry(), + ods::JSON_TO_ODS.entry(), + ods::MARKDOWN_TO_ODS.entry(), + ods::NUMBERS_TO_ODS.entry(), + to_numbers::CSV_TO_NUMBERS.entry(), + to_numbers::TSV_TO_NUMBERS.entry(), + to_numbers::XLSX_TO_NUMBERS.entry(), + to_numbers::JSON_TO_NUMBERS.entry(), + to_numbers::MARKDOWN_TO_NUMBERS.entry(), + rows_document::MARKDOWN_TO_XLSX.entry(), + json_to_csv::JSON_TO_XLSX.entry(), + rows_document::MARKDOWN_TABLES_TO_JSON.entry(), + rows_document::JSON_TO_MARKDOWN.entry(), + csv_to_xlsx::CSV_TO_XLSX.entry(), + csv_to_xlsx::TSV_TO_XLSX.entry(), + rows_document::CSV_TO_MARKDOWN.entry(), + rows_document::TSV_TO_MARKDOWN.entry(), + rows_document::MARKDOWN_TO_CSV.entry(), + rows_document::MARKDOWN_TO_TSV.entry(), + n64::Z64_TO_V64.entry(), + n64::Z64_TO_N64.entry(), + n64::V64_TO_Z64.entry(), + n64::V64_TO_N64.entry(), + n64::N64_TO_Z64.entry(), + n64::N64_TO_V64.entry(), ]; -/// The listed converters and the generated image pairs, gathered once. -pub fn all_converters() -> &'static [&'static dyn Converter] { - static ALL: OnceLock> = OnceLock::new(); +/// The listed converters and the generated image pairs, gathered once: +/// what the planner reads. Nothing here runs a converter's code. +pub fn all_converters() -> &'static [Entry] { + static ALL: OnceLock> = OnceLock::new(); ALL.get_or_init(|| { - let mut all: Vec<&'static dyn Converter> = CONVERTERS.to_vec(); + let mut all: Vec = CONVERTERS.to_vec(); for pair in image::pairs() { - all.push(pair); + all.push(pair.entry()); } all }) } +/// The converters a conversion between these formats (and through `via`) +/// can use. An image pair joins two image formats, and the only listed +/// converters into or out of one are PDF's, so a path between formats that +/// are not images never takes a pair (it would leave PDF and come back): +/// building the pairs, about 400 KB of a conversion's memory, is left to +/// conversions with an image at one end. +pub fn converters_for( + from: &'static Format, + to: &'static Format, + via: Option<&'static Format>, +) -> &'static [Entry] { + let touches_images = [Some(from), Some(to), via] + .into_iter() + .flatten() + .any(image::is_image_format); + if touches_images { + all_converters() + } else { + &CONVERTERS + } +} + /// Every format referenced by a registered converter, unique, sorted by id. pub fn all_formats() -> Vec<&'static Format> { let mut formats: Vec<&'static Format> = Vec::new(); - for converter in all_converters() { - push_unique(&mut formats, converter.from()); - push_unique(&mut formats, converter.to()); + for converter in &CONVERTERS { + push_unique(&mut formats, converter.from); + push_unique(&mut formats, converter.to); + } + // The image pairs join every codec's format, read from the codec table + // rather than the pairs, which are not built for this. + for (format, _, _) in image::codecs() { + push_unique(&mut formats, format); } formats.sort_by(|left, right| left.id.cmp(right.id)); formats @@ -232,12 +260,51 @@ mod tests { use super::*; use std::collections::HashSet; + /// `converters_for` leaves the image pairs out only where no plan uses + /// them: every pair of formats plans the same with and without them. + #[test] + fn plans_without_the_image_pairs_are_unchanged() { + use crate::planner::{PlanOptions, plan}; + let formats = all_formats(); + for strict in [false, true] { + let options = PlanOptions { strict, via: None }; + for from in &formats { + for to in &formats { + if from == to { + continue; + } + let full = plan(all_converters(), from, to, &options); + let chosen = plan(converters_for(from, to, None), from, to, &options); + let names = |result: &Result| match result { + Ok(plan) => format!("{plan:?}"), + Err(error) => format!("{error:?}"), + }; + assert_eq!(names(&full), names(&chosen), "{} -> {}", from.id, to.id); + } + } + } + } + + #[test] + fn every_format_is_an_end_of_some_converter() { + let formats = all_formats(); + for format in &formats { + let used = all_converters() + .iter() + .any(|converter| converter.from == *format || converter.to == *format); + assert!(used, "{} has no converter", format.id); + } + for converter in all_converters() { + assert!(formats.contains(&converter.from) && formats.contains(&converter.to)); + } + } + #[test] fn converter_names_are_unique() { let mut names = HashSet::new(); for converter in all_converters() { - let is_new = names.insert(converter.name()); - assert!(is_new, "duplicate converter name {}", converter.name()); + let is_new = names.insert(converter.name); + assert!(is_new, "duplicate converter name {}", converter.name); } } @@ -245,7 +312,7 @@ mod tests { fn edges_are_unique() { let mut edges = HashSet::new(); for converter in all_converters() { - let edge = (converter.from().id, converter.to().id); + let edge = (converter.from.id, converter.to.id); let is_new = edges.insert(edge); assert!(is_new, "duplicate edge {:?}", edge); } diff --git a/tests/hub_pairs.rs b/tests/hub_pairs.rs index e04d7e89..cca68dde 100644 --- a/tests/hub_pairs.rs +++ b/tests/hub_pairs.rs @@ -6,7 +6,7 @@ use std::fs; use std::path::PathBuf; -use sublime::converter::{ConvertError, ConvertOptions, Converter, Input}; +use sublime::converter::{ConvertError, ConvertOptions, Converter, Entry, Input}; use sublime::converters::hub; use sublime::converters::json_to_toml::JsonToToml; use sublime::converters::json_to_xml::JsonToXml; @@ -53,7 +53,7 @@ fn fixtures(format: &str) -> Vec<(String, Vec)> { /// outputs are read back to JSON with `target_to_json` and compared. fn check( format: &str, - direct: &dyn Converter, + direct: Entry, to_json: &dyn Converter, from_json: &dyn Converter, target_to_json: &dyn Converter, @@ -69,7 +69,7 @@ fn check( } let json = convert(to_json, &source).expect("source reads"); let via_json = convert(from_json, &json); - let straight = convert(direct, &source); + let straight = convert(direct.converter, &source); match (straight, via_json) { (Ok(straight), Ok(via_json)) => { let straight_json = @@ -80,13 +80,13 @@ fn check( String::from_utf8_lossy(&straight_json), String::from_utf8_lossy(&via_json_json), "{}: {name}", - direct.name() + direct.name ); } (Err(ConvertError::Unsupported(_)), Err(ConvertError::Unsupported(_))) => {} (straight, via_json) => panic!( "{}: {name}: direct {:?}, via JSON {:?}", - direct.name(), + direct.name, straight.map(|bytes| bytes.len()), via_json.map(|bytes| bytes.len()) ), @@ -98,14 +98,14 @@ fn check( fn toml_pairs_match_the_json_route() { check( "toml", - &hub::TOML_TO_YAML, + hub::TOML_TO_YAML.entry(), &TomlToJson, &JsonToYaml, &YamlToJson, ); check( "toml", - &hub::TOML_TO_XML, + hub::TOML_TO_XML.entry(), &TomlToJson, &JsonToXml, &XmlToJson, @@ -116,14 +116,14 @@ fn toml_pairs_match_the_json_route() { fn yaml_pairs_match_the_json_route() { check( "yaml", - &hub::YAML_TO_TOML, + hub::YAML_TO_TOML.entry(), &YamlToJson, &JsonToToml, &TomlToJson, ); check( "yaml", - &hub::YAML_TO_XML, + hub::YAML_TO_XML.entry(), &YamlToJson, &JsonToXml, &XmlToJson, @@ -134,14 +134,14 @@ fn yaml_pairs_match_the_json_route() { fn xml_pairs_match_the_json_route() { check( "xml", - &hub::XML_TO_TOML, + hub::XML_TO_TOML.entry(), &XmlToJson, &JsonToToml, &TomlToJson, ); check( "xml", - &hub::XML_TO_YAML, + hub::XML_TO_YAML.entry(), &XmlToJson, &JsonToYaml, &YamlToJson, diff --git a/tests/to_pdf.rs b/tests/to_pdf.rs index 3684a201..8f841ee5 100644 --- a/tests/to_pdf.rs +++ b/tests/to_pdf.rs @@ -113,8 +113,12 @@ fn every_document_format_reaches_pdf() { let mut out = Vec::new(); write_pdf_text(&pdf, None, &mut out).expect("our PDF reads"); let out = String::from_utf8(out).unwrap(); - assert!(out.contains(expect), "{}: {out}", converter.name()); - assert!(!out.trim().is_empty(), "{} wrote no text", converter.name()); + assert!(out.contains(expect), "{}: {out}", converter.entry().name); + assert!( + !out.trim().is_empty(), + "{} wrote no text", + converter.entry().name + ); } } diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs index 20551e55..e562011a 100644 --- a/wasm/src/lib.rs +++ b/wasm/src/lib.rs @@ -214,7 +214,8 @@ fn run(from: &[u8], to: &[u8], input: &[u8], options_text: &[u8]) -> u32 { strict: false, via: None, }; - let plan = match planner::plan(registry::all_converters(), from, to, &plan_options) { + let converters = registry::converters_for(from, to, None); + let plan = match planner::plan(converters, from, to, &plan_options) { Ok(plan) => plan, Err(error @ PlanError::NoPath { .. }) => { set_message(&error.to_string()); From 4943f254e30d6a62562ca724daf7f008efc6479c Mon Sep 17 00:00:00 2001 From: Chimi6 <43803253+Chimi6@users.noreply.github.com> Date: Wed, 21 Oct 2026 04:55:02 -0600 Subject: [PATCH 2/2] docs: v1 wrap-up; every benchmark pair rerun and recorded ROADMAP item 9 done and the audio/video cycle planned. Each pair document gains a 2026-10-21 block on the M1 Max (under load), and binary.md a 1.0.0 block. Lines failing for the first time and the Pages goals move to STATE Tech Debt. The bench harness runs on this Mac again: gtime in csv-json, awk helpers in ico-png, a bash 5 guard in jpeg-pdf, and the pdf-text pair on the registry's entry table. --- DOCS/CHANGELOG.md | 4 ++ DOCS/CONTRIBUTING.md | 11 +++-- DOCS/ROADMAP.md | 57 ++++++++++++++++++++------ DOCS/STATE.md | 68 +++++++++++++------------------ DOCS/benchmarks/binary.md | 31 ++++++++++++++ DOCS/benchmarks/csv-json.md | 31 ++++++++++++-- DOCS/benchmarks/csv-markdown.md | 32 ++++++++++++++- DOCS/benchmarks/docx-html.md | 24 ++++++++++- DOCS/benchmarks/docx-markdown.md | 22 +++++++++- DOCS/benchmarks/docx-text.md | 28 ++++++++++++- DOCS/benchmarks/epub-docx.md | 28 ++++++++++++- DOCS/benchmarks/heic-png.md | 55 ++++++++++++++++++++++--- DOCS/benchmarks/html-docx.md | 24 ++++++++++- DOCS/benchmarks/html-markdown.md | 25 +++++++++++- DOCS/benchmarks/html-text.md | 28 ++++++++++++- DOCS/benchmarks/ico-png.md | 37 ++++++++++++++++- DOCS/benchmarks/jpeg-pdf.md | 31 +++++++++++++- DOCS/benchmarks/jpeg-png.md | 31 +++++++++++++- DOCS/benchmarks/jsonl-json.md | 22 +++++++++- DOCS/benchmarks/markdown-docx.md | 26 +++++++++++- DOCS/benchmarks/markdown-html.md | 27 +++++++++++- DOCS/benchmarks/markdown-json.md | 38 ++++++++++++++++- DOCS/benchmarks/markdown-text.md | 27 +++++++++++- DOCS/benchmarks/numbers-csv.md | 33 ++++++++++++++- DOCS/benchmarks/numbers-xlsx.md | 36 +++++++++++++++- DOCS/benchmarks/ods-csv.md | 24 ++++++++++- DOCS/benchmarks/odt-docx.md | 34 +++++++++++++++- DOCS/benchmarks/pages-docx.md | 32 ++++++++++++++- DOCS/benchmarks/pages-html.md | 24 ++++++++++- DOCS/benchmarks/pages-json.md | 29 ++++++++++++- DOCS/benchmarks/pages-markdown.md | 24 ++++++++++- DOCS/benchmarks/pages-text.md | 24 ++++++++++- DOCS/benchmarks/pages-xlsx.md | 24 ++++++++++- DOCS/benchmarks/pdf-text.md | 39 +++++++++++++++++- DOCS/benchmarks/png-bmp.md | 31 +++++++++++++- DOCS/benchmarks/png-heic.md | 50 ++++++++++++++++++++--- DOCS/benchmarks/png-pdf.md | 31 +++++++++++++- DOCS/benchmarks/ppm-png.md | 30 +++++++++++++- DOCS/benchmarks/qoi-png.md | 41 ++++++++++++++++++- DOCS/benchmarks/rtf-docx.md | 32 ++++++++++++++- DOCS/benchmarks/text-markdown.md | 24 ++++++++++- DOCS/benchmarks/tga-png.md | 30 +++++++++++++- DOCS/benchmarks/tiff-png.md | 27 +++++++++++- DOCS/benchmarks/toml-json.md | 35 +++++++++++++++- DOCS/benchmarks/tsv-json.md | 28 ++++++++++++- DOCS/benchmarks/webp-png.md | 44 +++++++++++++++++++- DOCS/benchmarks/xlsx-csv.md | 28 ++++++++++++- DOCS/benchmarks/xml-json.md | 37 ++++++++++++++++- DOCS/benchmarks/yaml-json.md | 37 ++++++++++++++++- DOCS/benchmarks/z64-v64.md | 59 ++++++++++++++++++++------- DOCS/formats/n64.md | 2 +- bench/pairs/csv-json.sh | 2 +- bench/pairs/ico-png.sh | 12 ++++-- bench/pairs/jpeg-pdf.sh | 2 + bench/src/pairs/pdf_text.rs | 4 +- 55 files changed, 1483 insertions(+), 133 deletions(-) diff --git a/DOCS/CHANGELOG.md b/DOCS/CHANGELOG.md index 69a4a5b3..2edcd7d3 100644 --- a/DOCS/CHANGELOG.md +++ b/DOCS/CHANGELOG.md @@ -6,6 +6,10 @@ section under a version heading. ## [Unreleased] +### Changed + +- Every conversion starts in less memory: the converters' names, formats, and fidelity are a table fixed when the binary is built, so planning runs none of their code, and the 195 image pairs are built only when an image is at one end. `sublime check z64 v64` holds 1.95 MB (was 2.24), a 40 MB N64 ROM converts in 2.42 MB (was 2.66). Sizes barely move: on CI's Linux build the binary is 5,596,192 bytes (was 5,592,448) and the WebAssembly module 2,951,471 (was 2,952,440). + ## [0.33.0] - 2026-10-20 ### Added diff --git a/DOCS/CONTRIBUTING.md b/DOCS/CONTRIBUTING.md index 0706f49a..438f1934 100644 --- a/DOCS/CONTRIBUTING.md +++ b/DOCS/CONTRIBUTING.md @@ -41,13 +41,16 @@ Six places, and the compiler or a test catches anything you miss. Give it an id, display name, extensions, magic bytes if the format has a reliable signature, and a category (data, document, image, audio, video, archive). Categories only group listings and docs; code stays flat. -2. `src/converters/_to_.rs`: a struct implementing `Converter`. - Declare `name`, `from`, `to`, `fidelity`, `tier`, and write `convert`. - Stream through the `Input` and the `Write` you are given. Call +2. `src/converters/_to_.rs`: a struct implementing `Converter` + (write `convert`), and a `const fn entry` on it returning the `Entry` + that declares `name`, `from`, `to`, `fidelity`, and `tier`: data the + planner reads without running your code. Stream through the `Input` + and the `Write` you are given. Call `context.loss(...)` for every loss, `context.warning(...)` for tolerated oddities, `context.progress(...)` every few thousand records. 3. `src/converters/mod.rs`: one `pub mod` line. -4. `src/registry.rs`: one entry in `CONVERTERS` and bump the array length. +4. `src/registry.rs`: your `entry()` in `CONVERTERS`, and bump the array + length. 5. `tests/fixtures//`: real sample files, including nasty ones. 6. `cargo run --release -- paths --markdown > DOCS/FORMATS.md` and `cargo run --release -- paths --conversions > DOCS/CONVERSIONS.md`, and diff --git a/DOCS/ROADMAP.md b/DOCS/ROADMAP.md index 412668ab..3812bd01 100644 --- a/DOCS/ROADMAP.md +++ b/DOCS/ROADMAP.md @@ -31,13 +31,42 @@ and from WebP and PNG) or not at all. SQLite is out of v1. In order: 7. ODS (done). 8. N64 ROMs between `z64`, `v64`, and `n64` (decided 2026-10-09: in v1; done). -9. Wrap-up: the registry held as data, lowering every conversion's - startup memory (STATE, Tech Debt); every benchmark pair re-run, size - budgets settled (HEIC grows the binary and the WebAssembly module), - this document current. - -N64 ROMs between `z64`, `v64`, and `n64` come before v1 (item 8 above); -the other novelty formats wait until after it (Games and consoles below). +9. Wrap-up (done): the registry held as data, and the image pairs built + only for conversions with an image at one end (a conversion's startup + memory 2.24 -> 1.95 MB); every benchmark pair re-run; size budgets + settled; this document current. + +v1 shipped as 1.0.0. The other novelty formats wait until after it +(Games and consoles below). + +## After v1: audio and video + +Decided 2026-10-21: the next cycle is audio and video, with the goal of a +universal converter for them: any file type. ffmpeg itself is mostly the +codecs, and its best encoders are other projects (x264, x265, libvpx, +SVT-AV1, LAME, libopus), so linking it would break zero runtime +dependencies, the size budget, the WebAssembly module, and the release +builds. Instead: + +- **Our own code first**, as HEIC was: each codec written here, judged by + a benchmark pair against ffmpeg and the codec's reference tool. A speed + tie or a slight miss counts as a win when the binary stays much smaller; + an encoder is still judged on quality for its size (a bigger file at + equal quality is a loss, not a tie). +- **ffmpeg as the fallback** in the External tier: an installed `ffmpeg` + does what we have not written yet (the long tail above all), declared + lossy where it re-encodes, with a clear message when it is missing. Our + own route replaces it path by path; the planner prefers Native. A + download that bundles ffmpeg beside the binary (never linked) is a later + option if needing it installed proves a barrier. +- **Order**, each step useful alone: the containers (ISO BMFF for MP4, + MOV, and M4A; EBML for MKV and WebM; RIFF) and remuxing; WAV, AIFF, + FLAC, and ALAC both ways; MP3, AAC, Vorbis, and Opus decoders; video + decoders (H.264, then HEVC video on the HEIC decoder, then VP9 and AV1); + then encoders one at a time (FLAC; Opus or AAC; H.264, HEVC video, AV1). +- **Patents**: H.264, HEVC, and AAC are covered by patent pools, decoders + included, as HEIC's docs already say; VP9, AV1, Opus, FLAC, and Vorbis + are royalty-free. ## How to read this @@ -202,6 +231,8 @@ lossless and universal. The audio hub is PCM samples plus channel layout and rate. Encoders for lossy codecs are the hardest code in this document; decoders are tractable. +The plan for the cycle is under After v1, above; the External entries +below are what ffmpeg covers until our own code replaces it. | Format | Extensions | Status | Tier | Effort | Directions and notes | |---|---|---|---|---|---| @@ -222,8 +253,8 @@ lossy codecs are the hardest code in this document; decoders are tractable. ## Video and subtitles -Codecs are External-tier work (ffmpeg wrapping, see `STATE.md` Future). -Containers can be handled natively. +Our own codecs, with ffmpeg in the External tier for what is not written +yet (After v1, above). Containers are native from the start. | Format | Extensions | Status | Tier | Effort | Directions and notes | |---|---|---|---|---|---| @@ -340,7 +371,7 @@ communities, and have no good cross-platform tool. | Mind maps, outlines | opml, mm | [ ] | C | S | OPML <-> Markdown outlines; FreeMind XML. | | Feeds | rss, atom | [ ] | B | S | XML to JSON/Markdown. | | Log formats | syslog, clf, json | [ ] | C | S | To CSV/JSON; open-ended. | -| Spreadsheets to Markdown tables | — | [ ] | A | S | CSV/XLSX -> Markdown table and back; small and constantly wanted. | +| Spreadsheets to Markdown tables | — | [x] | — | — | Shipped (0.17.0): rows from CSV, TSV, Excel, Numbers, and ODS become Markdown tables, and a document's tables come back as rows. | ## Tier list @@ -349,9 +380,9 @@ listed. | Tier | Formats | |---|---| -| **S** | HEIC both ways (v1) · ODT, EPUB, ODS (v1, done) · Images in documents to PDF (v1) · SQLite read (after v1) · WAV · ZIP and gzip (keystones) | -| **A** | Semicolon and pipe CSV · INI · MessagePack and CBOR · plist · ODS and ODT · Numbers · Parquet read · NumPy · PEM/DER · vCard and iCalendar · RTF · EPUB · reStructuredText · AsciiDoc · Jupyter · Email (EML, MBOX) · Subtitles (SRT, VTT, ASS, TTML) · GIF · BMP · TIFF · Netpbm · ICO/ICNS · Exif and XMP · AIFF · FLAC · MP3 decode · MIDI · TAR · TrueType/OpenType · WOFF · STL · OBJ · glTF · GeoJSON · GPX · KML · FASTA/FASTQ · DICOM · Intel HEX and S-record · N64 ROM byte order · N64 saves · SNES headers · Genesis SMD/BIN · Minecraft NBT · Spreadsheet to Markdown table | -| **B** | Java properties · .env · JSON5 · BSON · dBase · Arrow · Avro · MATLAB · R data · JWK/JWT · BibTeX/RIS · PGN · Keynote · MOBI/AZW3 · FictionBook · Org-mode · LaTeX · man pages · R Markdown · Outlook MSG · WebP decode · farbfeld and QOI · TGA · SVG subset rasterizer · PSD · Aseprite · DDS/KTX · Ogg Vorbis decode · Tracker modules · N64 audio · Video remuxing · Transcoding via External · bzip2 · XZ · Zstandard · ISO 9660 · WOFF2 · Glyphs to SVG · PLY · 3MF/AMF · Quake models · Minecraft schematics · Shapefile · WKT/WKB · GenBank/GFF/BED/VCF · PDB · FITS · NES headers · Game Boy saves · PlayStation images · Doom WAD · Quake PAK · PICO-8 · Base64 and hex · Feeds | +| **S** | SQLite read · WAV · ZIP and gzip (keystones) · Video remuxing (the audio and video cycle) | +| **A** | INI · MessagePack and CBOR · plist · Parquet read · NumPy · PEM/DER · vCard and iCalendar · reStructuredText · AsciiDoc · Jupyter · Email (EML, MBOX) · Subtitles (SRT, VTT, ASS, TTML) · GIF animation · ICNS · Exif and XMP · AIFF · FLAC · MP3 decode · MIDI · TAR · TrueType/OpenType · WOFF · STL · OBJ · glTF · GeoJSON · GPX · KML · FASTA/FASTQ · DICOM · Intel HEX and S-record · N64 saves · SNES headers · Genesis SMD/BIN · Minecraft NBT | +| **B** | Java properties · .env · JSON5 · BSON · dBase · Arrow · Avro · MATLAB · R data · JWK/JWT · BibTeX/RIS · PGN · Keynote · MOBI/AZW3 · FictionBook · Org-mode · LaTeX · man pages · R Markdown · Outlook MSG · Lossy WebP output · farbfeld · SVG subset rasterizer · PSD · Aseprite · DDS/KTX · Ogg Vorbis decode · Tracker modules · N64 audio · Transcoding via External · bzip2 · XZ · Zstandard · ISO 9660 · WOFF2 · Glyphs to SVG · PLY · 3MF/AMF · Quake models · Minecraft schematics · Shapefile · WKT/WKB · GenBank/GFF/BED/VCF · PDB · FITS · NES headers · Game Boy saves · PlayStation images · Doom WAD · Quake PAK · PICO-8 · Base64 and hex · Feeds | | **C** | XLS · HDF5 · NetCDF · SPSS/Stata/SAS · Protobuf generic · Word `.doc` · Textile/MediaWiki · RTFD · PCX · XCF · ORA/Krita · Camera raw · IFF ILBM · legacy rasters · AU/VOC/CAF · Opus, AAC, ALAC · Chiptune (SID, NSF, SPC, GBS, VGM) · Animated GIF via External · 7z · cpio/ar/deb · CAB · Type 1 fonts · Bitmap fonts · OFF/X3D/VRML · COLLADA · Source SMD · TopoJSON · OSM · GeoTIFF · SAM/BAM · NIfTI · EDF · N64 textures · Retro disk images · Hex dumps · OPML · Log formats · Diagram text-to-text | | **D** | UBJSON · ORC · GraphQL/OpenAPI · DjVu · CHM · WordPerfect · AVIF/JXL · PICT · WavPack/APE · Speech codecs · RAR · LHA/ARJ/ZOO · StuffIt · FBX · ELF/PE · Save states · Pokémon saves · N64 models · Diagram rendering | diff --git a/DOCS/STATE.md b/DOCS/STATE.md index 2d4ff381..36cbc6e9 100644 --- a/DOCS/STATE.md +++ b/DOCS/STATE.md @@ -6,6 +6,7 @@ describes. ## Now +- 2026-10-21: v1 wrap-up and 1.0.0 (branch `v1-wrapup`). The converter registry is data: each converter has a `const fn entry()` and the registry is a static table of entries, so planning runs no converter's code, and the 195 image pairs are built only when an image format is an end (`registry::converters_for`). Starting memory 2.24 -> 1.95 MB, a ROM conversion 2.66 -> 2.42 MB, startup 0.15 ms faster; sizes move by a few KB. Every benchmark pair was rerun and recorded (first run on the M1 Max for many pairs, under load); the failing lines are under Tech Debt. The bench harness was fixed for this Mac (`gtime` in `csv-json`, awk helpers in `ico-png`, a bash 5 guard in `jpeg-pdf`). ROADMAP item 9 done; the next cycle is audio and video. - 2026-10-20: N64 ROMs between `z64`, `v64`, and `n64` (`src/io/n64.rs`, branch `n64`): six lossless converters permute each 32-bit word in a 256 KB buffer, vectorized as an array shuffle, reading the input's order from where the header's `0x80` lies whatever the name. Detection on the command line now lets content overrule an extension whose signature it fails when another format's matches (`format::detect`, for every format), with a warning; errors without a position drop `line 0, column 0`. Checked on Paper Mario (USA), a 40 MB No-Intro dump in `~/Downloads` (not committed): round trips return its SHA-1, outputs byte-identical to ucon64, `n64romconvert`, and `dd conv=swab`. Pair `z64-v64`: every speed line passes (1.2 to 1.35 times `dd`); the four memory lines against ucon64 and `dd` fail by 0.1 to 0.6 MB, the binary's startup floor (Tech Debt). The bench harness takes `bench_runs` for millisecond pairs. The wrap-up is next. - 2026-10-17: OpenDocument spreadsheet both ways (`src/io/ods`, branch `ods`): every sheet reads into rows with cells shown as LibreOffice shows them to an English reader, and rows write as sheets typed by the read-back rule; number formats pass through one model between ODS data styles and Excel codes (`numfmt.rs`), so ODS reaches Excel and Numbers with its formats and back. LibreOffice's built-in dates per document language come from its locale data (`builtin.rs`, MPL-2.0, `scripts/gen-ods-dates.py`). `content.xml` streams in four-megabyte pieces through a new `ZipArchive::stream`, read once, each wanted sheet held as its cells' text. Against LibreOffice's CSV export (`scripts/ods-check`): 303 of 463 development sheets byte-identical, 99.65% of non-formula cells; held out (35 government files, 12.2 million cells), 190 of 240 sheets, 99.999%, with five faults the held-out set found fixed before those numbers; every development CSV written as ODS reads back the same in LibreOffice. Pair `ods-csv` passes every line (calamine 0.36, LibreOffice). Budgets: binary 5.62 MB (Linux 5.59), wasm 3.14 MB (3.11). N64 ROMs are next. - 2026-10-14: EPUB both ways (`src/io/epub`, branch `epub`): books are read in spine order through the HTML reader, with their cover, pictures, and notes (EPUB, pandoc, Standard Ebooks, and Gutenberg markup, in the chapter or a notes file), and written as EPUB 3 with an EPUB 2 table of contents, chapters at page breaks and top headings, styles as CSS in `em`, pop-up footnotes, merged cells, pictures, MathML, and embedded fonts. Every book from the 89 Word files and 21 held-out ones passes epubcheck (pandoc's fail on 82 and 21); on 11 held-out books, EPUB -> Word keeps every note and picture in all 11 (pandoc 8 and 8); `scripts/epub-check`. Fixed on the way for every format: Google Docs headings (named, not levelled), HTML spaces before an inline end, merged cells in HTML tables, SVG-framed pictures. Pair `epub-docx` passes every line against pandoc. Budgets: binary 5.44 MB (Linux 5.36), wasm 3.02 MB (2.99). @@ -49,27 +50,9 @@ describes. ## Next -- 2026-10-08, the v1 plan (decided with the user; full list in `ROADMAP.md`, v1): release the Numbers work as 0.29.0; the Pages reader's memory on large tables; CSV delimiters; images in documents to PDF; HEIC read, then HEIC write, native and measured against libheif/x265 and `sips`; ODT, then EPUB; ODS; the wrap-up. SQLite and the niche formats (MessagePack, CBOR, INI, subtitles, notebooks, ICNS, GIF) come after v1, and N64 ROMs (`z64`, `v64`, `n64`) are the first novelty after it. - -The document category's one-way streets are closed (0.8.0 to 0.10.0) and the data category's planned set is in (0.11.0 to 0.16.0). What follows, in order: - -- 2026-09-28, PDF first (a flagship: complete both ways before 1.0). Shipped: images <-> PDF (0.23), PDF to text, Markdown, HTML, and Word (0.24), and documents to PDF with system fonts (0.25). In order: - 1. **Images in documents to PDF** (done 2026-10-08, `formats/pdf.md`): pictures set as blocks at their size. Left: pictures in table cells, inline with text, floating beside it, and crops; and Markdown to Word does not carry images by path (the Word writer has no input folder yet). - 2. **The `markdown-pdf` pair** (bench): a long generated Markdown document to PDF against printpdf's HTML layout (Markdown to HTML by us, HTML to PDF by printpdf 0.8) and pandoc when a PDF engine is installed (none on the Linux box); pass lines speed and memory; output size recorded. - 3. **Release 0.26.0** (minor: new paths): release blocks for the new pair and every pair the inflater or PDF code touched, the binary block (budgets are 2.70 MB and 1.40 MB after the raise for this work), changelog, ROADMAP. - 4. After 0.25: complex scripts (a shaper for Arabic joining and Indic conjuncts, the bidi algorithm for right-to-left order) and CFF font subsetting (most CJK `.otf`), both under Tech Debt; page numbers, headers, footers, and hyphenation; reading a PDF's embedded fonts to get true sizes (the Ghostscript CID case in 0.24). - 5. The Mac session's Pages <-> Word work shipped in 0.25.0, so changes to the Pages or Word readers, writers, or the document model are open again, including setting a Word or Pages document in its own named fonts. - Encrypted PDFs with an empty user password (RC4, AES) remain their own item. -- 2026-09-25, spreadsheets as a family (the XLSX reader and writer are the model; every entry below reads into rows and writes from them so it reaches CSV, TSV, JSON, JSON Lines, and the hub formats through the planner): - - **Apple Numbers** (reader done 2026-10-06, `DOCS/formats/numbers.md`): left are number formats (currency, percentages, custom formats) as Numbers shows them, references from Numbers' own CSV export on Numbers-made fixtures (Mac session), and the writer, which needs the object-graph work the Pages writer did. - - **OpenDocument spreadsheet** (done 2026-10-17, `DOCS/formats/ods.md`). - - **Google Sheets** is not a file format: it exports XLSX, CSV, or ODS, which we read (said in `DOCS/formats/ods.md`). - - **Spreadsheet to document** (done 2026-09-25): rows become a Markdown table and reach every document format; a document's first table comes back as rows. Workbooks whole (2026-10-06): a workbook's sheets and a document's tables travel together into formats that hold several (JSON as an object of sheets, documents as a table per sheet, workbooks as a sheet per table) and become a file each, in a folder, for formats that hold one (CSV, TSV, JSON Lines). Numbers reads into it (2026-10-06). Still open: PDF pages and multi-page TIFFs as an image each through the same split. - - **Excel-made fixtures** for the XLSX reader (Mac session, Excel or Numbers export), and number formats beyond dates. -- 2026-09-26, the image category after PNG, BMP, and JPEG: GIF once there is somewhere for its frames to go (APNG or video); until then its first frame is a small reader on the hub. Every codec lands in the pixel hub and ships with a pair against a real crate. JPEG levers in Tech Debt: subsampling chosen from the chroma's detail, optimized Huffman tables, progressive output, Exif orientation on the image path. -- 2026-09-25, the cheap hub batch: INI, plist (XML and binary), MessagePack, CBOR, each a day on the value tree. -- Then the document category: ODT and EPUB, which reuse the Word and HTML work almost entirely; RTF. -- The Pages writer decision stays with the Mac session. +- 2026-10-21: v1 is done (`ROADMAP.md`, v1) and released as 1.0.0. The next cycle is audio and video (`ROADMAP.md`, After v1): our own codecs first, judged against ffmpeg and each codec's reference tool, with an installed ffmpeg as the External-tier fallback for what is not written yet. In order: the containers (ISO BMFF, EBML, RIFF) and remuxing; WAV, AIFF, FLAC, and ALAC both ways; MP3, AAC, Vorbis, and Opus decoders; video decoders (H.264, HEVC video, VP9, AV1); then encoders one at a time. +- 2026-10-21: Record the v1.0.0 musl asset and gzipped module in `DOCS/benchmarks/binary.md` (this Mac has neither toolchain), and rerun the failing bench lines (Tech Debt) on an idle machine. +- Left from the earlier plans, after the media cycle unless a user needs them sooner: SQLite read; the cheap hub batch (INI, plist, MessagePack, CBOR); the `markdown-pdf` bench pair; PDF pictures in table cells, inline, floating, and cropped, and Markdown to Word carrying images by path; complex scripts and CFF fonts in PDF (Tech Debt); the Pages reader's memory on large tables (1.8 GB on 100,000 rows against the 64 MB goal; Optimization); Excel-made fixtures and number formats beyond dates into CSV, JSON, and documents. ## Future @@ -90,12 +73,21 @@ The document category's one-way streets are closed (0.8.0 to 0.10.0) and the dat ## Blockers -- 2026-09-24: The Pages document paths fail the shared Pages goals of 50 MB/s of input and 64 MB peak (`DOCS/benchmarks/pages-docx.md`, `pages-markdown.md`, `pages-html.md`, `pages-text.md`). After the typed decode of the attribute tables and the interned Word styles (on top of the slim model, the streamed Word body, and the reachable decode): memory passes on every document row (28 to 46 MB), Word passes throughput under the standard's compressed-output measure (150 MB/s of input plus uncompressed output), and the text paths fail throughput at 32 to 40 MB/s against 50. A real resume converts in 3 ms at 4.8 MB, inside both goals. What remains on the dense shape, in-process: package decode 10 ms (7.5 of it Snappy), document build 25 ms, the text writers 11 ms; wall clock adds about 10 ms of process start, file I/O, and first-touch page faults (13,000 pages). The text paths are 5 to 8 ms from the line; levers under Spikes. Causes measured: a 224-byte run struct with cloned font and language strings (48 MB of model for 170,000 runs), the Word body held whole before Deflate (30 MB), and the package decoding 570 objects to use 70. Fix for 0.6.1: intern strings in the style table and slim the run, stream the Word body through the compressor, decode objects on lookup. -- 2026-09-24: `pages -> pages-json` misses the 50 MB/s goal at 28 and 40 MB/s of package bytes (`DOCS/benchmarks/pages-json.md`); the reverse direction passes at 330 MB/s. Levers: the typed decode of the attribute tables written as JSON directly (the Spikes entry below), then `Tree::decode` itself and the JSON writer's per-field work. +- (none) The two Pages throughput entries moved to Tech Debt on 2026-10-21; 1.0.0 ships with them. ## Tech Debt -- 2026-10-20: Startup memory floor. A conversion holds 2.24 MB before it reads a byte (a minimal Rust program 1.36): about 700 KB more of our code is paged in (16 KB pages on Apple Silicon; the command line, detection, planner, error formatting, the image-pair table, and 122 converter metadata methods on 14 pages, 224 KB), 144 KB more of patched static tables, and little heap. It fails `z64-v64`'s four memory lines against ucon64 and `dd` by 0.1 to 0.6 MB. Decided (with the user): hold the registry as data (name, formats, fidelity, tier read from a static table, the image pairs too) so planning runs no converter's code; in the v1 wrap-up. Grouping startup code with a linker order file is set aside as too much for now. +- 2026-09-24 (updated 2026-10-21, moved from Blockers): The Pages document paths' throughput against the shared Pages goal of 50 MB/s of input (`DOCS/benchmarks/pages-markdown.md`, `pages-html.md`, `pages-text.md`). The 1.0.0 run, on an M1 Max under load: memory passes on every document row (26 to 47 MB against 64), Word passes both ways (Pages -> Word 129 and 140 MB/s of input plus uncompressed output, Word -> Pages 67 and 89), and the text paths fail throughput at 26 to 33 MB/s (styled 29 to 33, prose 26 to 28). A real resume converts in 3 ms at 4.8 MB, inside both goals. Where the time goes on the dense shape, in-process (2026-09-24): package decode 10 ms (7.5 of it Snappy), document build 25 ms, the text writers 11 ms; wall clock adds about 10 ms of process start, file I/O, and first-touch page faults. Levers under Spikes. Pages -> Word's peak memory rose on every row since 2026-10-06 (prose 33.8 -> 46.8 MB), still inside the goal. +- 2026-09-24 (updated 2026-10-21, moved from Blockers): `pages -> pages-json` misses the 50 MB/s goal at 24 and 26 MB/s of package bytes (`DOCS/benchmarks/pages-json.md`); the reverse direction passes at 146 and 239. Levers: the typed decode of the attribute tables written as JSON directly (the Spikes entry below), then `Tree::decode` itself and the JSON writer's per-field work. +- 2026-10-21: Benchmark lines failing in the 1.0.0 run that passed in their previous blocks. The run was on the M1 Max with the machine in use (load average 7 to 26 on 10 cores); for several pairs every earlier block was on the Linux i7, so these are not like for like. An alternated A/B of ec68950 against the commit before it showed the registry change caused none of those it covered. Rerun on an idle machine before acting on any of them. + - `csv-markdown`: markdown -> csv 49.6 against pulldown-cmark with the csv crate's 79.4 MB/s (0.17.0: 120.8 against 107.1). A/B: same median. + - `markdown-json`: markdown-json -> markdown 159.1 against 235.4 (each side's own JSON; the gate is wall time). Not A/B'd at this input size. + - `qoi-png`: png -> qoi, photo 281.4 against 336.3 and flat 920.3 against 964.1 (earlier blocks on Linux). A/B on photo: same median. + - `webp-png`: png -> webp lossless, flat 534.8 against 662.2 (earlier on Linux), besides the known photo line. A/B: same median. + - `pdf-text`: peak memory on `gs` 21.6 MB against pdftotext's 12.1 and pdf-extract's 10.0, on `base14` 13.9 against pdftotext's 13.2 (earlier on Linux; the `gs` input is Ghostscript 10.08's rewrite, new to this run). + - `binary`: startup above the spawn floor read 1.11 to 1.26 ms in single loaded runs (FAIL against 1 ms); alternated with the previous commit it is 0.78 (PASS). +- 2026-10-21: The bench harness depends on the machine more than its documents say. Inputs are generated only when missing and several pairs share names in `bench/data` (`big.md`, `prose.md`, `big.csv`), so a file left by an earlier, smaller generation is measured silently; the 1.0.0 run measured 2.2 MB of Markdown where the documents say 73.8 until the files were deleted. Tools that had gone from this Mac (Pillow, PyMuPDF, Ghostscript, Miller, bash 5) failed some pairs and voided `jpeg-pdf`'s timings under bash 3.2 (it now refuses to run without bash 5; `ico-png` no longer needs it). Wanted: inputs named by generator and size, and a preflight that checks every pair's tools before a long run. +- 2026-10-20 (updated 2026-10-21): Startup memory floor. A conversion held 2.24 MB before it read a byte (a minimal Rust program 1.36): code paged in for the command line, detection, planner, error formatting, the image-pair table, and 122 converter metadata methods. The registry is now data (2026-10-21: a const table of entries, the 195 image pairs built only when an image format is an end), and the floor is 1.95 MB (`sublime check z64 v64`); a 40 MB ROM converts in 2.42 MB (was 2.66). It still fails `z64-v64`'s four memory lines against ucon64 (2.0 to 2.1 MB) and `dd` (2.3) by 0.1 to 0.4 MB. Left: image conversions still build the pairs (about 400 KB); grouping startup code with a linker order file is set aside as too much for now. - 2026-10-20: N64 leftovers (`DOCS/formats/n64.md`): copier headers (512 bytes before the ROM), 64DD disk images (`.ndd`), saves (EEPROM, SRAM, FlashRAM, Controller Pak; ROADMAP's next N64 row), and header checksum repair are not handled. - 2026-10-17: ODS leftovers (`DOCS/formats/ods.md`): formulas are carried as their last values only, and nothing of styling (fonts, fills, borders, widths, conditional formatting, validation, comments, charts, pictures). Excel paths read lower where LibreOffice recalculates on load. Known deviations from LibreOffice: text in a styled span inside a link is kept (LibreOffice drops it), and an empty sheet is no rows (LibreOffice writes one blank line). The locale date table (`builtin.rs`) is MPL-2.0 LibreOffice data, kept by decision; regenerate with `scripts/gen-ods-dates.py`. Typed cells end to end (Future) would let ODS pass types without the read-back rule. - 2026-10-14: EPUB leftovers (`DOCS/formats/epub.md`): pages (sizes, margins), headers and footers, page-number fields, comments, deleted text, links to the document's own bookmarks (the model has no bookmarks), charts and drawn shapes, and pictures in formats EPUB does not carry. Reading leaves out style sheets and metadata; fixed-layout books read as their text; MOBI and AZW3 are later. @@ -109,7 +101,6 @@ documents in `DOCS/benchmarks`; measured-best patterns and failed spikes live in `temp/patterns.md`. Size wins over speed where they trade (the WebP color cache stays). -- 2026-10-06: Pages writer tables past 256 rows are now tiles of 256 (shared with the Numbers writer, which Numbers accepts cleanly); open a long Word table converted to Pages in Pages to confirm it loads without repair (not checked in Pages yet). - 2026-10-08, Pages tables at scale (future improvement; `DOCS/benchmarks/pages-xlsx.md`): the reader's levers are in (two-pass document read, data list entries and tile rows kept as bytes, a tile row's legacy copy left out): a Pages document of 100,000 table rows peaks at 1.8 GB (from 2.2), against the 64 MB goal. What remains, in order of return: 1. **Plain cells in the Pages writer** (branch `pages-plain-cells`, written, not checked in Pages): our writer stores every cell of a table as rich text (a payload and a text storage each, about 43 tree fields a cell); Pages keeps a plain cell in the string table with its cell style. With it the same document is 23 MB instead of 60 and reads in 0.9 s at 551 MB. Needs a Pages session: open, no repair, cells and their styles as written, PDF export. 2. **Streaming tables**: half of what is left is the document model (about 200 bytes a cell: `Cell` 104, `Paragraph` 80, its text) and half the tables' own bytes (a Pages row carries a 510-byte offset array). Under 64 MB needs tables read a tile at a time straight to the output, as the Numbers workbook path does, which changes the document model's contract for every document path. @@ -124,7 +115,7 @@ WebP color cache stays). - Lossy WebP output is not written; JPEG covers lossy output for now. - Deflate (PNG, DOCX, XLSX): the default level now matches or beats zlib 6's ratio at about its speed. libdeflate reaches the same ratio at about twice zlib's speed: its hash-table layout and match finder are the model. Photo-shaped data costs about 11 ns a position in the chain walk (dependent loads); noise now skips seven searches in eight. A near-optimal parse would suit a smallest-output level. - PNG writer: filters by smallest residual sum; fixed Sub compressed 1.3% smaller on a decoded lossy photo under zlib. A real-photo corpus would settle a Sub bias. The `png -> bmp` photo line's margin is thin in the harness (ours 102 against 109 ms direct). -- JPEG encoder size: optimized Huffman tables (3 to 5%), trellis quantization (5 to 10% at equal PSNR), and 4:2:0 or 4:4:4 chosen from the chroma planes' edge energy (fixed 4:2:0 costs 24 dB on hard-edged graphics). All but the last hold the coefficients. +- JPEG encoder size: optimized Huffman tables (3 to 5%), trellis quantization (5 to 10% at equal PSNR), and subsampling chosen from the chroma planes' edge energy (today 4:2:0 below quality 90 and 4:4:4 from 90; 4:2:0 costs 24 dB on hard-edged graphics). All but the last hold the coefficients. - Markdown dense parse: level with `pulldown-cmark`, not ahead; the arena-shrinking spike under Spikes is the lever. - `qoi-png` thin margins: the photo decode (65.3 against 64.7 MB/s, the PNG write of a noise photo is most of it) and the stock encode (583.8 against 582.0, a tie). Levers: the deflate noise path (shared with the WebP lossless decode line), and the PNG reader's unfilter on large RGBA images. - `ppm-png` photo encode: a tie (86 to 89 ms against 88 to 90 direct; the harness read 834 against 888 MB/s). The time is the PNG decode, the inflater level with the png crate's fdeflate; the lever is the inflater's decode structure. CRC-32 over the IDAT chunks is 7% of it. @@ -138,18 +129,17 @@ WebP color cache stays). - 2026-09-26: JPEG leftovers: - Not read: CMYK and YCCK (Adobe four-component files, common from print workflows), 12-bit samples, arithmetic coding, lossless and hierarchical modes, DNL-supplied heights. CMYK is the one users will hit. - The reader takes the file whole (its entropy-coded data is one bit stream); a windowed reader would make input memory constant too. The file is a tenth of the pixels, so this is small. - - Exif orientation is reported, not applied; the transform belongs to the image path with the other whole-image operations. - The writer has no progressive output (it holds the coefficients, an image-path feature). - Encoder size levers moved to Optimization above. - Margins (2026-09-26): the photo encode leads jpeg-encoder by 7 to 9% since chroma is computed from summed RGB. Tried and reverted for the decode: libjpeg's color lookup tables (6% slower) and planar color groups (1 to 3% slower); a constant-chroma upsampling skip helps only specially shaped images and was not written. The flat decode's lead rests on the PNG writer. - 2026-09-26: Image leftovers: - - The hub is 8-bit: 16-bit PNG samples lose their low byte (reported as a loss). A 16-bit hub is the change if a lossless 16-bit path is ever wanted. - - No metadata is carried (gamma, ICC, text, physical size); APNG frames are not read; BMP RLE is refused. + - Streaming paths carry 16-bit samples (0.30.0); the in-memory `Image` (the downscale behind icons) and interlaced 16-bit PNGs still drop to 8 bits. + - ICC and Exif are carried from JPEG and HEIC into PNG, JPEG, and HEIC output (0.30.0); the PNG reader skips iCCP, eXIf, gamma, and text; WebP and TIFF writers drop them; APNG frames are not read; BMP RLE is refused. - The PNG writer's filter heuristic (smallest sum of residual magnitudes, the standard) chooses Average on the synthetic photo where fixed Sub compressed 20% smaller under the first generator's mirrored noise; with independent noise both give the same size. An entropy estimate chose the same. A real-photo corpus would settle whether a bias toward Sub is worth it. - Deflate and PNG filter levers moved to Optimization above (lazy matching and priced matches shipped 2026-09-27). - 2026-09-25: Excel leftovers: - The reader inflates the sheet part whole before parsing (415 MB peak on a 39 MB workbook whose sheet inflates to 326 MB); a windowed inflate feeding the XML reader would make it constant. The writer already streams. - - Number formats other than dates are dropped; merged cells read as their top-left value. + - Number formats and merges reach Numbers and ODS; CSV, JSON, and documents get the raw value and the top-left cell. - No Excel-made fixtures yet; the set is hand-built from the specification. - 2026-09-25: Value hub leftovers: - The TOML and YAML readers hold the input text whole beside the tree (60 to 190 MB of their peaks on the benchmark shapes); the XML reader's sliding window is the shape to port if a large config case ever matters. @@ -158,16 +148,15 @@ WebP color cache stays). - 2026-09-24: WebAssembly leftovers: - Multi-hop paths hold each intermediate whole in memory (no threads in the browser); a single-hop path streams as on the command line. Fine for documents, a concern only for large data files through two hops. - The module has no size tooling beyond `opt-level = "z"`; `wasm-opt` would take 10 to 20% more off but is a toolchain dependency the build does not assume. - - No progress or event reporting across the boundary; the host gets a status, the bytes, and one message. + - No progress or event stream across the boundary; the host gets a status, the bytes, an error message, and unused-option warnings. - 2026-09-24: Pages document reader and Word writer leftovers: - Equations are written as MathML text; Office Math (OMML) from MathML is the proper output. - Floating objects anchor to the first paragraph on their page by counting explicit page breaks; pages that begin by overflow are not known without layout. - PDF and other non-picture media are left out of the Word file; converting PDF vector images to PNG needs a rasterizer. - - Text box fills are solid colors only; gradients, image fills, strokes, and shape geometry other than rectangles are dropped. Lines and charts are dropped. - - Comments (`table_highlight`) are not read. + - Gradient and image fills are dropped (strokes, shape geometry, lines, and charts are read since 0.25.0). - Drop caps come out as an ordinary first letter (Apple splits them into a framed paragraph). - Header and footer areas (left, center, right) are joined as paragraphs; Word has no three-area header. - - The release binary grew to 1.36 MB with the document pipeline (budget raised to 1.4 MB); std's backtrace symbolizer (gimli, addr2line, about 100 KB) is the largest non-feature cost and the lever if size matters. + - std's backtrace symbolizer (gimli, addr2line, about 100 KB) is the largest non-feature cost in the binary and the lever if size matters. - 2026-09-23: Pages leftovers from the package reader: - 31 registry types without a schema; they decode raw. Resolve as the document reader needs them. - The schema comes from community protos of two vintages; fields Pages 12 added since decode raw. Coverage is measured by the round-trip test, not by name. @@ -184,15 +173,14 @@ WebP color cache stays). - Execution: a converter thread panic is reported as `Unsupported`; `PipeReader::read` blocks on a zero-length buffer; partial output can reach stdout before a mid-chain error is reported (inherent to streaming, needs a doc note on `execute`). - JSON tokenizer: `skip_value` does not check bracket type agreement (`[1}` skips); a high surrogate followed by a non-escape consumes one extra byte before erroring. - JSON -> CSV: truncated input is reported as `Unsupported` rather than `Malformed`; duplicate keys within one object can mask differing-key detection. - - CSV reader: bare CR accepted as a line ending but undocumented; error paths drop the record buffer's capacity. - - Tests: no coverage for `Progress` events, `\b`/`\f` JSON escapes, `describe_path` with mixed multi-hop fidelity, or `bytes_consumed` with a BOM present. + - CSV reader: error paths drop the record buffer's capacity (bare CR is documented). + - Tests: no coverage for `\b`/`\f` JSON escapes, `describe_path` with mixed multi-hop fidelity, or `bytes_consumed` with a BOM present. - 2026-09-22: JSON -> CSV from stdin buffers the whole input to memory (two passes need a rewind). Files stream in constant memory. See Spikes. - 2026-09-22: Multi-hop chains replay events after completion instead of streaming them live. ## Spikes - 2026-09-24: Pages performance levers past the three phases, in order of expected return (numbers are the dense benchmark shape, in-process, from `DOCS/benchmarks/pages-docx.md`): - - Done 2026-09-24: **typed decode of the attribute tables** (`Tree::deferred`, parsed by the reader): dense-shape memory 70 -> 48 MB (passes), text paths 27 -> 32 MB/s. **Run formatting interned into Word character styles**: XML 19.3 -> 15.6 MB, Word 17 -> 20 MB/s; the gain was smaller than projected because most runs carry only a named style or a toggle, which cannot move into a style. - Tried 2026-09-24 and kept for correctness, without measurable gain: a one-entry cache in front of the style hash lookups (removed), 16-byte table spans, bulk overlapping copies in the Snappy decoder. The text paths sit at 58 ms wall on the dense shape against 50: package decode 10 ms (7.5 Snappy), document build 25 ms, writer 11 ms, process and page faults about 10 ms. What remains is structural: a Snappy decoder inner loop at 1.5 GB/s (about 3 ms), a document without per-paragraph vectors (one run vector per document; fewer pages touched, cheaper drop), and a model-free streaming text path (rejected so far, see below). - **Merging adjacent runs with equal effective formatting.** Pages splits runs at style-object boundaries that often resolve to the same look; fewer runs means less of everything downstream. - **Dropping the object trees once the model is built**, for the document paths only; the media bytes are already copied out. @@ -203,9 +191,9 @@ WebP color cache stays). - 2026-09-23: Word-at-a-time scanning for the inline parser's special characters; the byte loop with a lookup table is its largest remaining cost. - 2026-09-23: Unicode general-category tables for exact delimiter-run classification (see `DOCS/formats/markdown.md`, known deviations). - 2026-09-22: Single-pass JSON -> CSV with header inference from the first N objects, for the stdin case. -- 2026-09-22: `opt-level = "z"` versus `3`: measure size and speed. +- 2026-09-22: `opt-level`: the binary is `3` and the WebAssembly module `"z"`; measure the other setting for each, size and speed. - 2026-09-22: SIMD byte scanning with `std::arch`. The word-at-a-time scanner in `io::scan` closed the CSV -> JSON gap without intrinsics; the CSV reader alone is still slower than the `csv` crate reader (see benchmarks/csv-json.md), so this remains the stretch target. -- 2026-09-22: Content sniffing beyond magic bytes for extensionless input. +- 2026-09-22: Content sniffing for extensionless text input (JSON, CSV, and the like), beyond `format::detect`'s signature check (0.33.0). ## Done @@ -251,7 +239,7 @@ WebP color cache stays). - 2026-09-22: Writers own a 64 KiB buffer and hand sinks whole chunks. Per-field writes through `dyn Write` cost more than parsing did; buffering inside the writer was the single largest win in the first performance spike. - 2026-09-22: Byte scanning is done eight bytes at a time with plain integer bit tricks before reaching for SIMD intrinsics. It is portable, dependency-free, and readable, and it was enough to beat the reference pipeline. - 2026-09-22: Zero runtime dependencies. Every proposed runtime dependency needs an entry in `DEPENDENCIES.md` with measured cost. -- 2026-09-22: Converters register through one explicit list in `src/registry.rs`. No link-time or build-script auto-registration. +- 2026-09-22: Converters register through one explicit list in `src/registry.rs`. No link-time or build-script auto-registration. (2026-10-21: the list is of `Entry` data, each built at compile time by the converter's `const fn entry`, so planning runs no converter's code.) - 2026-09-22: The library never prints; it emits typed events to an explicitly passed sink. No global logger. - 2026-09-22: The planner picks the cheapest path by fidelity and prints it. `--strict` refuses lossy paths. - 2026-09-22: Three tiers: Native (our code), Library (a crate compiled in), External (an installed binary). Only ffmpeg is anticipated in External. diff --git a/DOCS/benchmarks/binary.md b/DOCS/benchmarks/binary.md index 27f763e8..4148fc76 100644 --- a/DOCS/benchmarks/binary.md +++ b/DOCS/benchmarks/binary.md @@ -30,6 +30,37 @@ a comparison. Startup is implemented in `bench/src/startup.rs`. ## Results +### 2026-10-21, 1.0.0: the registry as data + +The gnu size and the module are CI's measurements on ec68950, the code +1.0.0 ships (the release commit changes only the version). The musl +asset and the gzipped module are not recorded here: the musl target and +CI's toolchain are not on this Mac; they are read from the v1.0.0 +release assets in the next block. Startup was measured on a Mac with the +harness's `startup` and `spawn-baseline` commands, five alternated rounds +against the commit before (93a37d4), median. + +| Target | Ours | Reference | Result | +|---|---|---|---| +| Binary size, gnu (bytes) | 5596192 | <= 5620000 (size-budget, what CI checks) | PASS | +| WebAssembly module (bytes) | 2951471 | <= 3140000 (wasm/size-budget, what CI checks) | PASS | +| Startup above spawn floor (ms, 1 KB file) | 0.783 (spawn 2.053, floor 1.270) | < 1 | PASS | +| Startup, the commit before (ms, 1 KB file) [extra] | 0.932 (spawn 2.202, floor 1.270) | recorded | n/a | + +commit: ec68950 (the `v1-wrapup` branch, code as released in v1.0.0) +machine: CI (ubuntu-latest) for the gnu binary and the module; Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max for startup + +From 0.29.0 the binary grew 1,407,600 bytes (+34%) and the module +670,352 over 0.30.0 to 0.33.0 (HEIC both ways, EPUB, the N64 orders and +the rest; those releases have no block here). The registry change itself +moved both by a few KB (binary 5,592,448 -> 5,596,192, module 2,952,440 +-> 2,951,471, CI) and cut startup by 0.15 ms and starting memory from +2.24 to 1.95 MB. The machine was in use (load average 7 to 12): the +harness's own single `bench/run.sh binary` runs read 1.11 to 1.26 ms +above a 1.29 to 1.45 ms floor, FAIL; the alternated rounds above, where +both binaries share the load, are the comparison this block stands on, +and the line should be rerun on an idle machine. + ### 2026-10-08, 0.29.0 with Numbers formats from Excel, pivot and categorised tables The gnu size and the module are CI's measurements on the release commit; diff --git a/DOCS/benchmarks/csv-json.md b/DOCS/benchmarks/csv-json.md index 49478207..b80f09f2 100644 --- a/DOCS/benchmarks/csv-json.md +++ b/DOCS/benchmarks/csv-json.md @@ -1,6 +1,6 @@ # CSV <-> JSON -**Latest** (2026-09-23, 0.3.0: csv -> json 333 MB/s of input, 2.4 MB peak; json -> csv 191 MB/s, 2.4 MB peak; all lines PASS against the csv and serde_json crates) +**Latest** (2026-10-21, 1.0.0 release run: csv -> json 94.8 MB/s of input, 2.9 MB peak; json -> csv 88.5 MB/s, 2.8 MB peak; 2.9 MB peak from stdin; all lines PASS against the csv and serde_json crates, on an M1 Max under load) ## Purpose @@ -36,8 +36,8 @@ results block as printed by the harness. **Inputs.** Generated by `bench/src/pairs/csv_json.rs`. Rows have six fields: `id,name,email,city,score,note`. Every seventh row's `note` is a quoted field containing a comma. Field values are short, mostly under eight -bytes. The full run uses 10,000,000 rows, about 1.18 GB of CSV and a matching -JSON array of objects with string values. The JSON file is generated +bytes. The full run uses 10,000,000 rows, about 600 MB of CSV and a matching +JSON array of objects with string values, about 1.1 GB. The JSON file is generated directly, not converted from the CSV, so the two directions are independent. **Commands.** `bench/run.sh csv-json` builds both crates in release, generates @@ -87,6 +87,31 @@ record and discard it, to separate parser cost from writer cost. ## Results +### 2026-10-21, `v1-wrapup` branch, the 1.0.0 release run + +| Target | Ours | Reference | Result | +|---|---|---|---| +| csv -> json, 10M rows (597.0 MB): throughput (MB/s of input) | 94.8 | 37.1 | PASS | +| json -> csv, 10M rows (1110.6 MB): throughput (MB/s of input) | 88.5 | 48.7 | PASS | +| csv -> json, 10M rows: peak memory (MB) | 2.9 | < 16 | PASS | +| json -> csv, 10M rows: peak memory (MB) | 2.8 | < 16 | PASS | +| csv -> json, 10M rows from stdin: peak memory (MB) | 2.9 | < 16 | PASS | + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +Every line passes. This run is on a different machine from the earlier +blocks (an M1 Max, not the 24-core Linux box), so its absolute MB/s and +memory do not compare with theirs; the comparison that holds is against +the reference in the same session, which ours leads by 2.6 times on +csv -> json and 1.8 times on json -> csv. The machine was in use during +the run (load average 7 to 12 on 10 cores: a browser, a VPN client, a +remote-desktop host), so the wall-clock rows are noisier than earlier +blocks'. The first attempt failed at the stdin-memory step because +`bench/pairs/csv-json.sh` called `/usr/bin/time -f`, which is BSD `time` +on macOS; the script now uses the harness's `$time_bin` (GNU `gtime`), +and this block is the rerun. + ### 2026-09-23, 0.3.0, after the JSON tokenizer change 10,000,000 rows. The tokenizer copies plain string runs a word at a time diff --git a/DOCS/benchmarks/csv-markdown.md b/DOCS/benchmarks/csv-markdown.md index c59204da..f59ea582 100644 --- a/DOCS/benchmarks/csv-markdown.md +++ b/DOCS/benchmarks/csv-markdown.md @@ -1,6 +1,6 @@ # CSV <-> Markdown table -**Latest** (2026-09-25, first release of the bridge: csv -> markdown 202.9 MB/s of input at 3.0 MB peak; markdown -> csv 120.8 MB/s at 215 MB peak; every line PASSES against Miller and pulldown-cmark with the csv crate) +**Latest** (2026-10-21, 1.0.0 release run: csv -> markdown 161.6 MB/s of input at 2.4 MB peak, PASS against Miller; markdown -> csv 49.6 MB/s at 636 MB peak; markdown -> csv throughput FAILS against pulldown-cmark with the csv crate (79.4), its memory passes) ## Purpose @@ -63,6 +63,36 @@ follow `README.md`. ## Results +### 2026-10-21, `v1-wrapup` branch, the 1.0.0 release run + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +| Target | Ours | Reference | Result | +|---|---|---|---| +| csv -> markdown, 1000000 rows (56.8 MB): throughput (MB/s of input) | 161.6 | 117.4 (Miller) | PASS | +| csv -> markdown, 1000000 rows: peak memory (MB) | 2.4 | 405.6 (Miller) | PASS | +| csv -> markdown, 1000000 rows: throughput against a bespoke printer in the harness (MB/s of input) [extra] | 161.6 | 193.0 (csv + table printer) | n/a | +| markdown -> csv, 1000000 rows (70.9 MB): throughput (MB/s of input) | 49.6 | 79.4 (pulldown-cmark + csv) | FAIL | +| markdown -> csv, 1000000 rows: peak memory (MB) | 635.9 | 822.7 (pulldown-cmark + csv) | PASS | + +markdown -> csv throughput FAILS: 49.6 MB/s against pulldown-cmark's +79.4. It passed in the previous block, 120.8 against 107.1. Its peak +memory also rose from 215.1 MB to 635.9 MB, still under the reference's +822.7. The other lines pass. This run is on a different machine from the +previous block (an M1 Max, not the Linux i7-13700K), and the machine was +in use during it (load average 7 to 12 on 10 cores: a browser, a VPN +client, a remote-desktop host), so the wall-clock rows are noisier than +earlier blocks'. Miller was not installed at the first attempt, which +read markdown -> csv at 68.4 against 104.7, also a FAIL; Miller 6.22.0 +was installed from Homebrew and the pair rerun, and this block is the +rerun. A paired A/B of our binary at ec68950 against the commit before +it (93a37d4), 9 alternated runs each, gave the same median on +markdown -> csv (1.01 s against 1.02 s) and on csv -> markdown (0.31 s +both): the registry change in ec68950 did not cause the failure. The drop +against the previous block accrued over the releases since, or is load; +this run does not separate the two. + ### 2026-09-25, first release of the bridge commit: b315c6c diff --git a/DOCS/benchmarks/docx-html.md b/DOCS/benchmarks/docx-html.md index 219dab38..d8f69413 100644 --- a/DOCS/benchmarks/docx-html.md +++ b/DOCS/benchmarks/docx-html.md @@ -1,6 +1,6 @@ # Word -> HTML -**Latest** (2026-09-27, 0.24.0 release: every line PASSES) +**Latest** (2026-10-21, 1.0.0 release run: every line PASSES; 129.4 MB/s of uncompressed input on the styled shape and 162.5 on prose, at 24.3 and 29.1 MB peak, against pandoc's 1.3 on both) ## Purpose @@ -52,6 +52,28 @@ read somewhat slower; repeated text makes the per-file-byte row read low. ## Results +### 2026-10-21, 1.0.0 release run (`v1-wrapup` branch) + +| Target | Ours | Reference | Result | +|---|---|---|---| +| docx -> html, styled (6.0 MB uncompressed): throughput (MB/s of uncompressed input) | 129.4 | 1.3 (pandoc) | PASS | +| docx -> html, styled (0.2 MB file): throughput (MB/s of file bytes) [extra] | 4.4 | recorded | n/a | +| docx -> html, styled: peak memory (MB) | 24.3 | 549.6 (pandoc) | PASS | +| docx -> html, prose (7.5 MB uncompressed): throughput (MB/s of uncompressed input) | 162.5 | 1.3 (pandoc) | PASS | +| docx -> html, prose (0.2 MB file): throughput (MB/s of file bytes) [extra] | 4.7 | recorded | n/a | +| docx -> html, prose: peak memory (MB) | 29.1 | 776.3 (pandoc) | PASS | + +commit: ec68950 (on the `v1-wrapup` branch, before the 1.0.0 release) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +Every line passes. The machine was in use during this run (load average +7 to 12 on 10 cores: a browser, a VPN client, a remote-desktop host), so +the wall-clock rows are noisier than earlier blocks'. This run is on the +Apple M1 Max with the 6.0 MB styled input, as in the 2026-09-24 pandoc +block, not the Linux machine and 14.9 MB input of the block below, so +its numbers read against that one: 129.4 and 162.5 MB/s against 102.6 +and 113.9. + ### 2026-09-27, 0.24.0 release | Target | Ours | Reference | Result | diff --git a/DOCS/benchmarks/docx-markdown.md b/DOCS/benchmarks/docx-markdown.md index 11061bce..ecf734b1 100644 --- a/DOCS/benchmarks/docx-markdown.md +++ b/DOCS/benchmarks/docx-markdown.md @@ -1,6 +1,6 @@ # Word -> Markdown -**Latest** (2026-10-13, the `odt` branch with the Word reader fixes: every line PASSES; 110.2 MB/s on the styled shape and 147.0 on prose, within 1% of `main` measured side by side) +**Latest** (2026-10-21, 1.0.0 release run: every line PASSES; 124.5 MB/s of uncompressed input on the styled shape and 147.3 on prose, at 24.5 and 28.2 MB peak, against pandoc's 1.2 on both) ## Purpose @@ -52,6 +52,26 @@ read somewhat slower; repeated text makes the per-file-byte row read low. ## Results +### 2026-10-21, 1.0.0 release run (`v1-wrapup` branch) + +| Target | Ours | Reference | Result | +|---|---|---|---| +| docx -> markdown, styled (6.0 MB uncompressed): throughput (MB/s of uncompressed input) | 124.5 | 1.2 (pandoc) | PASS | +| docx -> markdown, styled (0.2 MB file): throughput (MB/s of file bytes) [extra] | 4.2 | recorded | n/a | +| docx -> markdown, styled: peak memory (MB) | 24.5 | 821.7 (pandoc) | PASS | +| docx -> markdown, prose (7.5 MB uncompressed): throughput (MB/s of uncompressed input) | 147.3 | 1.2 (pandoc) | PASS | +| docx -> markdown, prose (0.2 MB file): throughput (MB/s of file bytes) [extra] | 4.3 | recorded | n/a | +| docx -> markdown, prose: peak memory (MB) | 28.2 | 1331.5 (pandoc) | PASS | + +commit: ec68950 (on the `v1-wrapup` branch, before the 1.0.0 release) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +Every line passes. The machine was in use during this run (load average +7 to 12 on 10 cores: a browser, a VPN client, a remote-desktop host), so +the wall-clock rows are noisier than earlier blocks'. Against the block +below, on the same machine and inputs, styled is 124.5 MB/s against +110.2 and prose 147.3 against 147.0; memory is level. + ### 2026-10-13, the `odt` branch (Word reader fixes) The reader now reads every style after an empty `` and scales diff --git a/DOCS/benchmarks/docx-text.md b/DOCS/benchmarks/docx-text.md index 3882904d..f618de16 100644 --- a/DOCS/benchmarks/docx-text.md +++ b/DOCS/benchmarks/docx-text.md @@ -1,6 +1,6 @@ # Word -> text -**Latest** (2026-09-27, 0.24.0 release: every line PASSES) +**Latest** (2026-10-21, 1.0.0 release run: every line PASSES; 110.0 MB/s of uncompressed input on the styled shape and 164.6 on prose, at 25.1 and 27.6 MB peak, against docx-rs at 38.1 and 59.9) ## Purpose @@ -67,6 +67,32 @@ the per-file-byte rate as an extra row. Rows and units follow ## Results +### 2026-10-21, 1.0.0 release run (`v1-wrapup` branch) + +| Target | Ours | Reference | Result | +|---|---|---|---| +| docx -> text, styled (6.0 MB uncompressed): throughput (MB/s of uncompressed input) | 110.0 | goal: 50 | PASS | +| docx -> text, styled (0.2 MB file): throughput (MB/s of file bytes) [extra] | 3.7 | recorded | n/a | +| docx -> text, styled: peak memory (MB) | 25.1 | goal: <= 64.0 | PASS | +| docx -> text, styled: throughput (MB/s of uncompressed input) [extra] | 110.0 | 38.1 (docx-rs, paragraph text only) | PASS | +| docx -> text, styled: peak memory (MB) [extra] | 25.1 | 146.8 (docx-rs) | PASS | +| docx -> text, prose (7.5 MB uncompressed): throughput (MB/s of uncompressed input) | 164.6 | goal: 50 | PASS | +| docx -> text, prose (0.2 MB file): throughput (MB/s of file bytes) [extra] | 4.8 | recorded | n/a | +| docx -> text, prose: peak memory (MB) | 27.6 | goal: <= 64.0 | PASS | +| docx -> text, prose: throughput (MB/s of uncompressed input) [extra] | 164.6 | 59.9 (docx-rs, paragraph text only) | PASS | +| docx -> text, prose: peak memory (MB) [extra] | 27.6 | 137.9 (docx-rs) | PASS | + +commit: ec68950 (on the `v1-wrapup` branch, before the 1.0.0 release) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +Every line passes. The machine was in use during this run (load average +7 to 12 on 10 cores: a browser, a VPN client, a remote-desktop host), so +the wall-clock rows are noisier than earlier blocks'. It ran on the +Apple M1 Max with a 6.0 MB styled input, not the Linux machine and +14.9 MB input of the block below, so the two blocks do not compare row +for row; the ratio to `docx-rs` in the same session (2.9 times on styled, +2.7 on prose) is the comparable figure, against 3.7 and 2.7 below. + ### 2026-09-27, 0.24.0 release | Target | Ours | Reference | Result | diff --git a/DOCS/benchmarks/epub-docx.md b/DOCS/benchmarks/epub-docx.md index aa267ff9..5caed179 100644 --- a/DOCS/benchmarks/epub-docx.md +++ b/DOCS/benchmarks/epub-docx.md @@ -1,6 +1,6 @@ # EPUB <-> Word -**Latest** (2026-10-14, the `epub` branch: docx -> epub at 60.6 MB/s of input plus output on the styled shape and 83.6 on prose, epub -> docx at 62.0 and 82.6; every line PASSES against pandoc at 47 to 413 times its speed and a tenth to a twenty-third of its memory) +**Latest** (2026-10-21, 1.0.0 release run: docx -> epub at 62.4 MB/s of input plus output on the styled shape and 87.6 on prose, epub -> docx at 62.3 and 84.1; every line PASSES against pandoc at 45 to 420 times its speed and a tenth to a twenty-fourth of its memory) ## Purpose @@ -50,6 +50,32 @@ and real Word files carry larger style sheets. ## Results +### 2026-10-21, 1.0.0 release run (`v1-wrapup` branch) + +| Target | Ours | Reference | Result | +|---|---|---|---| +| docx -> epub, styled (6.0 MB in + 6.1 MB out, uncompressed): throughput (MB/s of input plus output) | 62.4 | 1.4 (pandoc) | PASS | +| docx -> epub, styled: throughput (MB/s of uncompressed input) [extra] | 30.8 | 0.9 (pandoc) | n/a | +| docx -> epub, styled: peak memory (MB) | 33.5 | 568.6 (pandoc) | PASS | +| epub -> docx, styled (6.1 MB in + 3.6 MB out, uncompressed): throughput (MB/s of input plus output) | 62.3 | 0.5 (pandoc) | PASS | +| epub -> docx, styled: throughput (MB/s of uncompressed input) [extra] | 39.2 | 0.2 (pandoc) | n/a | +| epub -> docx, styled: peak memory (MB) | 38.5 | 400.1 (pandoc) | PASS | +| docx -> epub, prose (7.5 MB in + 7.8 MB out, uncompressed): throughput (MB/s of input plus output) | 87.6 | 1.4 (pandoc) | PASS | +| docx -> epub, prose: throughput (MB/s of uncompressed input) [extra] | 43.1 | 0.9 (pandoc) | n/a | +| docx -> epub, prose: peak memory (MB) | 33.4 | 812.1 (pandoc) | PASS | +| epub -> docx, prose (7.8 MB in + 5.8 MB out, uncompressed): throughput (MB/s of input plus output) | 84.1 | 0.2 (pandoc) | PASS | +| epub -> docx, prose: throughput (MB/s of uncompressed input) [extra] | 48.4 | 0.1 (pandoc) | n/a | +| epub -> docx, prose: peak memory (MB) | 42.9 | 459.6 (pandoc) | PASS | + +commit: ec68950 (on the `v1-wrapup` branch, before the 1.0.0 release) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +Every line passes. The machine was in use during this run (load average +7 to 12 on 10 cores: a browser, a VPN client, a remote-desktop host), so +the wall-clock rows are noisier than earlier blocks'. On the same machine +and inputs as the block below, every throughput row is level or a few +percent higher and memory moves by at most 2.5 MB either way. + ### 2026-10-14, the `epub` branch | Target | Ours | Reference | Result | diff --git a/DOCS/benchmarks/heic-png.md b/DOCS/benchmarks/heic-png.md index 85a61889..185fc11c 100644 --- a/DOCS/benchmarks/heic-png.md +++ b/DOCS/benchmarks/heic-png.md @@ -1,10 +1,12 @@ # HEIC -> PNG and JPEG -**Latest** (2026-10-08, `heic-read` branch, pictures streamed by bands and -files read by item: every line PASSES against libheif; ahead of Apple's -hardware decoder (`sips`, context) in speed and memory on every line but -the 36-megapixel stock grid to JPEG (430 against 529 MB/s, 66 against -42 MB)) +**Latest** (2026-10-21, 1.0.0 release run: the quality-50 photo to PNG +at 112.3 MB/s against libheif's 10.5 and to JPEG at 521.7 against 53.2, +the flat image 769.1 and 1153.6 against 94.9 and 271.2, at 16 to 57% of +its memory; every line PASSES against libheif; ahead of Apple's hardware +decoder (`sips`, context) in speed and memory on every line but the +36-megapixel stock grid to JPEG (435.5 against 524.8 MB/s, 65.9 against +40.0 MB)) ## Purpose @@ -58,6 +60,49 @@ clock of the whole process, peak resident memory from GNU `time`. ## Results +### 2026-10-21, 1.0.0 release + +| Target | Ours | Reference (heif-dec) | Context (sips) | Result | +|---|---|---|---|---| +| heic -> png, hphoto-x265-q50 (45.8 MB of pixels, 2.1 MB on disk): throughput (MB/s of decoded pixels) | 112.3 | 10.5 | 46.4 | PASS | +| heic -> png, hphoto-x265-q50: peak memory (MB) | 23.3 | 120.2 | 149.3 | PASS | +| heic -> jpeg, hphoto-x265-q50 (45.8 MB of pixels, 2.1 MB on disk): throughput (MB/s of decoded pixels) | 521.7 | 53.2 | 276.6 | PASS | +| heic -> jpeg, hphoto-x265-q50: peak memory (MB) | 19.7 | 74.6 | 31.7 | PASS | +| heic -> png, hphoto-x265-q90 (45.8 MB of pixels, 11.9 MB on disk): throughput (MB/s of decoded pixels) | 71.8 | 8.6 | 40.7 | PASS | +| heic -> png, hphoto-x265-q90: peak memory (MB) | 43.2 | 142.2 | 189.3 | PASS | +| heic -> jpeg, hphoto-x265-q90 (45.8 MB of pixels, 11.9 MB on disk): throughput (MB/s of decoded pixels) | 251.5 | 32.2 | 199.9 | PASS | +| heic -> jpeg, hphoto-x265-q90: peak memory (MB) | 41.7 | 104.2 | 76.4 | PASS | +| heic -> png, hphoto-x265-10bit (45.8 MB of pixels, 8.7 MB on disk): throughput (MB/s of decoded pixels) | 52.8 | 1.9 | 15.9 | PASS | +| heic -> png, hphoto-x265-10bit: peak memory (MB) | 46.2 | 239.5 | 181.8 | PASS | +| heic -> jpeg, hphoto-x265-10bit (45.8 MB of pixels, 8.7 MB on disk): throughput (MB/s of decoded pixels) | 235.1 | 32.4 | 196.5 | PASS | +| heic -> jpeg, hphoto-x265-10bit: peak memory (MB) | 42.2 | 167.1 | 60.3 | PASS | +| heic -> png, hflat-x265-q75 (45.8 MB of pixels, 0.1 MB on disk): throughput (MB/s of decoded pixels) | 769.1 | 94.9 | 82.6 | PASS | +| heic -> png, hflat-x265-q75: peak memory (MB) | 17.8 | 114.3 | 142.1 | PASS | +| heic -> jpeg, hflat-x265-q75 (45.8 MB of pixels, 0.1 MB on disk): throughput (MB/s of decoded pixels) | 1153.6 | 271.2 | 392.3 | PASS | +| heic -> jpeg, hflat-x265-q75: peak memory (MB) | 17.5 | 68.4 | 19.0 | PASS | +| heic -> png, hphoto-apple (45.8 MB of pixels, 2.3 MB on disk): throughput (MB/s of decoded pixels) | 90.4 | 10.6 | 40.0 | PASS | +| heic -> png, hphoto-apple: peak memory (MB) | 29.5 | 93.6 | 148.7 | PASS | +| heic -> jpeg, hphoto-apple (45.8 MB of pixels, 2.3 MB on disk): throughput (MB/s of decoded pixels) | 435.4 | 171.5 | 282.0 | PASS | +| heic -> jpeg, hphoto-apple: peak memory (MB) | 26.1 | 45.8 | 28.8 | PASS | +| heic -> png, stock (103.5 MB of pixels, 19.4 MB on disk): throughput (MB/s of decoded pixels) [stock] | 93.7 | 3.6 | 45.2 | PASS | +| heic -> png, stock: peak memory (MB) [stock] | 74.5 | 229.2 | 315.5 | PASS | +| heic -> jpeg, stock (103.5 MB of pixels, 19.4 MB on disk): throughput (MB/s of decoded pixels) [stock] | 435.5 | 74.0 | 524.8 | PASS | +| heic -> jpeg, stock: peak memory (MB) [stock] | 65.9 | 127.4 | 40.0 | PASS | + +commit: ec68950 (the `v1-wrapup` branch, before the version bump) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +Every line passes against libheif, as in the previous block, and `sips` +(context) still leads only on the 36-megapixel stock grid to JPEG +(435.5 against 524.8 MB/s, 65.9 against 40.0 MB). Our throughput reads +2 to 18% lower than the previous block's on ten of twelve lines (the +10-bit photo to PNG most, 52.8 against 64.7 MB/s) and 1 to 5% higher on +two (the quality-90 photo and the stock grid to JPEG); the machine was in use +during this run (load average 7 to 12 on 10 cores: a browser, a VPN +client, a remote-desktop host), so wall-clock rows are noisier than +earlier blocks', and the decoder, which uses every core, loses more to +that load than `heif-dec` does. + ### 2026-10-08, pictures streamed by bands, files read by item | Target | Ours | Reference (heif-dec) | Context (sips) | Result | diff --git a/DOCS/benchmarks/html-docx.md b/DOCS/benchmarks/html-docx.md index 18f6d978..f364d5e3 100644 --- a/DOCS/benchmarks/html-docx.md +++ b/DOCS/benchmarks/html-docx.md @@ -1,6 +1,6 @@ # HTML -> Word -**Latest** (2026-09-27, the lazy deflate and the faster bit writer: 332.4 MB/s on the markup-dense shape (from 217.8) and 423.7 on prose (from 288.9); every line PASSES) +**Latest** (2026-10-21, 1.0.0 release run on the Apple M1 Max at 4,000 units: 214.1 MB/s of input plus uncompressed output on the markup-dense shape and 198.8 on prose, at 19.0 and 10.5 MB peak, against pandoc's 2.1 and 2.3; every line PASSES) ## Purpose @@ -53,6 +53,28 @@ bytes (`unzip -l`), with the rate per input byte as an extra row. ## Results +### 2026-10-21, 1.0.0 release run (`v1-wrapup` branch) + +commit: ec68950 (on the `v1-wrapup` branch, before the 1.0.0 release) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +| Target | Ours | Reference | Result | +|---|---|---|---| +| html -> docx, markup-dense (5.3 MB in + 18.1 MB out): throughput (MB/s of input plus uncompressed output) | 214.1 | 2.1 (pandoc) | PASS | +| html -> docx, markup-dense: throughput (MB/s of input) [extra] | 48.4 | recorded | n/a | +| html -> docx, markup-dense: peak memory (MB) | 19.0 | 2782.8 (pandoc) | PASS | +| html -> docx, prose (2.4 MB in + 4.0 MB out): throughput (MB/s of input plus uncompressed output) | 198.8 | 2.3 (pandoc) | PASS | +| html -> docx, prose: throughput (MB/s of input) [extra] | 74.4 | recorded | n/a | +| html -> docx, prose: peak memory (MB) | 10.5 | 327.9 (pandoc) | PASS | + +Every line passes. The machine was in use during this run (load average +7 to 12 on 10 cores: a browser, a VPN client, a remote-desktop host), so +the wall-clock rows are noisier than earlier blocks'. The harness printed +20,000 units, but the inputs were the 4,000-unit files of the 2026-09-24 +pandoc block (5.3 and 2.4 MB), on the same machine, so this block reads +against that one rather than the Linux block below: 214.1 and 198.8 MB/s +against 153.0 and 150.1, at 19.0 and 10.5 MB against 24.2 and 12.8. + ### 2026-09-27, the lazy deflate and the faster bit writer commit: a5c702a (on the `webp-spikes` branch, before its merge) diff --git a/DOCS/benchmarks/html-markdown.md b/DOCS/benchmarks/html-markdown.md index 6b15ebfc..8830517f 100644 --- a/DOCS/benchmarks/html-markdown.md +++ b/DOCS/benchmarks/html-markdown.md @@ -1,6 +1,6 @@ # HTML -> Markdown -**Latest** (2026-09-25, 0.17.0 Markdown speedups: html -> markdown 167.4 MB/s of input on the markup-dense shape and 234.9 on prose, at 29.6 and 15.0 MB peak; every line PASSES against htmd, whose throughput rounds to zero at one decimal) +**Latest** (2026-10-21, 1.0.0 release run on the Apple M1 Max with 5.3 and 2.4 MB inputs: html -> markdown 106.6 MB/s of input on the markup-dense shape and 135.0 on prose, at 9.0 and 5.0 MB peak; every line PASSES against htmd at 1.0 and 5.1 MB/s) ## Purpose @@ -56,6 +56,29 @@ follow `README.md`. ## Results +### 2026-10-21, 1.0.0 release run (`v1-wrapup` branch) + +commit: ec68950 (on the `v1-wrapup` branch, before the 1.0.0 release) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +| Target | Ours | Reference | Result | +|---|---|---|---| +| html -> markdown, markup-dense (5.3 MB): throughput (MB/s of input) | 106.6 | 1.0 (htmd) | PASS | +| html -> markdown, markup-dense: peak memory (MB) | 9.0 | 187.3 (htmd) | PASS | +| html -> markdown, prose (2.4 MB): throughput (MB/s of input) | 135.0 | 5.1 (htmd) | PASS | +| html -> markdown, prose: peak memory (MB) | 5.0 | 76.9 (htmd) | PASS | + +Every line passes. The machine was in use during this run (load average +7 to 12 on 10 cores: a browser, a VPN client, a remote-desktop host), so +the wall-clock rows are noisier than earlier blocks'. The harness printed +20,000 units, but the HTML on this machine was the 4,000-unit files (5.3 +and 2.4 MB, not the 26.6 and 12.1 MB of the blocks below), and the +machine is the Apple M1 Max, not the Linux one. The rows therefore do +not compare with the block below: the smaller inputs run in tens of +milliseconds, where process start weighs more, and htmd, whose cost grows +faster than its input, reads far faster on them (1.0 and 5.1 MB/s +against 0.1 and 0.4). + ### 2026-09-25, 0.17.0 Markdown speedups commit: 0b33385 (the merge of the rows-to-document bridge, before the version bump) diff --git a/DOCS/benchmarks/html-text.md b/DOCS/benchmarks/html-text.md index fcbbb7f5..fe31dc87 100644 --- a/DOCS/benchmarks/html-text.md +++ b/DOCS/benchmarks/html-text.md @@ -1,6 +1,6 @@ # HTML -> text -**Latest** (2026-09-24, references wired on an Apple M1 Max: html -> text 148.8 MB/s of input on the markup-dense shape and 271.3 on prose, at 29.5 and 14.2 MB peak, faster and far leaner than html2text and orders ahead of pandoc; every line PASSES) +**Latest** (2026-10-21, 1.0.0 release run on an Apple M1 Max with 5.3 and 2.4 MB inputs: html -> text 117.7 MB/s of input on the markup-dense shape and 150.0 on prose, at 8.9 and 5.0 MB peak, against html2text's 16.1 and 28.0 and pandoc's 0.7 and 0.8; every line PASSES) ## Purpose @@ -37,6 +37,28 @@ As `html-markdown.md`: our own writer's HTML is cleaner than a real page. ## Results +### 2026-10-21, 1.0.0 release run (`v1-wrapup` branch) + +commit: ec68950 (on the `v1-wrapup` branch, before the 1.0.0 release) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +| Target | Ours | html2text | pandoc | Result | +|---|---|---|---|---| +| html -> text, markup-dense (5.3 MB): throughput (MB/s of input) | 117.7 | 16.1 | 0.7 | PASS | +| html -> text, markup-dense: peak memory (MB) | 8.9 | 202.0 | 1823.8 | PASS | +| html -> text, prose (2.4 MB): throughput (MB/s of input) | 150.0 | 28.0 | 0.8 | PASS | +| html -> text, prose: peak memory (MB) | 5.0 | 42.9 | 552.7 | PASS | + +Every line passes. The machine was in use during this run (load average +7 to 12 on 10 cores: a browser, a VPN client, a remote-desktop host), so +the wall-clock rows are noisier than earlier blocks'. The harness printed +20,000 units, but the HTML on this machine was the 4,000-unit files (5.3 +and 2.4 MB, not the 26.6 and 12.1 MB of the block below), so the rows do +not compare with that block's: the smaller inputs run in tens of +milliseconds, where process start weighs more. Against html2text in the +same session we are 7.3 times faster on the dense shape and 5.4 on +prose, against 9.4 and 8.2 below. + ### 2026-09-24, references wired: html2text and pandoc commit: 64b08e7 (the reference wiring's working tree, before its commit) @@ -69,3 +91,7 @@ machine: Linux 7.1.5-ogc5.1.fc44.x86_64 x86_64, 24 cpus About nine times html2text's throughput at a few percent of its memory, and hundreds of times pandoc's. The text writer costs almost nothing over the reader. + +2026-10-21: on the 5.3 and 2.4 MB inputs of the 1.0.0 release run the +lead over html2text is 7.3 and 5.4 times, not nine, at 4 and 12% of its +memory. diff --git a/DOCS/benchmarks/ico-png.md b/DOCS/benchmarks/ico-png.md index 48f629b5..9538cd3c 100644 --- a/DOCS/benchmarks/ico-png.md +++ b/DOCS/benchmarks/ico-png.md @@ -1,6 +1,6 @@ # ICO <-> PNG -**Latest** (2026-09-27, 0.24.0 release: 1 line FAILS, the 256-pixel icon (0.22.0 decision)) +**Latest** (2026-10-21, 1.0.0 release run: ico to png of the 256 icon 19.6 ms against 23.3, png to ico 18.0 ms against 14.8 for the 256 picture, 130.7 against 223.3 for the photo and 58.0 against 163.7 for the flat, peak memory 4.6 to 6.6 MB against 6.1 to 53.0; 1 line FAILS, the 256-pixel icon (0.22.0 decision)) ## Purpose @@ -44,6 +44,41 @@ wall clock of the whole process, peak resident memory from GNU `time`. ## Results +### 2026-10-21, 1.0.0 release + +| Target | Ours | Reference | Result | +|---|---|---|---| +| ico -> png, 256 icon (0.1 MB): wall time (ms) | 19.6 | 23.3 (image + png) | PASS | +| ico -> png, 256 icon: peak memory (MB) | 4.6 | 6.3 (image + png) | PASS | +| png -> ico, icon256 (0.0 MB in): wall time (ms) | 18.0 | 14.8 (png + image) | FAIL | +| png -> ico, icon256: peak memory (MB) | 5.1 | 6.1 (png + image) | PASS | +| png -> ico, icon256: output size (KB) [extra] | 56.8 | 101.3 (image) | n/a | +| png -> ico, jphoto (29.0 MB in): wall time (ms) | 130.7 | 223.3 (png + image) | PASS | +| png -> ico, jphoto: peak memory (MB) | 6.2 | 52.9 (png + image) | PASS | +| png -> ico, jphoto: output size (KB) [extra] | 56.4 | 103.0 (image) | n/a | +| png -> ico, jflat (1.6 MB in): wall time (ms) | 58.0 | 163.7 (png + image) | PASS | +| png -> ico, jflat: peak memory (MB) | 6.6 | 53.0 (png + image) | PASS | +| png -> ico, jflat: output size (KB) [extra] | 108.0 | 153.3 (image) | n/a | + +commit: ec68950 (the `v1-wrapup` branch, before the version bump) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +This is the first block on this machine; the earlier blocks are the +Linux i7, so the times are not a like-for-like comparison with them. +The machine was in use during this run (load average 7 to 12 on 10 +cores: a browser, a VPN client, a remote-desktop host), so the wall +times are noisier than earlier blocks', and the 256-pixel lines most of +all. The first attempt could not make the inputs (Pillow was no longer +installed); they were made with Pillow 13.0.0 in a fresh Python 3.13 +environment and the pair rerun, and these are the rerun's rows. The +pair script's awk programs, quoted inside `"$(...)"`, are misread by +macOS's bash 3.2; they are now the `ms` and `kb` helper functions, with +no change to what is measured. The run ended with an `rm` error on a +leftover directory after every row printed, so the harness did not print +its commit and machine lines; the two above are the session's, which +every other pair of the run printed. The one FAIL is the same line as +the previous block's, the 256-pixel icon (18.0 against 14.8 ms). + ### 2026-09-27, 0.24.0 release | Target | Ours | Reference | Result | diff --git a/DOCS/benchmarks/jpeg-pdf.md b/DOCS/benchmarks/jpeg-pdf.md index f12b0111..164dbd79 100644 --- a/DOCS/benchmarks/jpeg-pdf.md +++ b/DOCS/benchmarks/jpeg-pdf.md @@ -1,6 +1,6 @@ # JPEG <-> PDF -**Latest** (2026-09-27, 0.23.1 release: every line PASSES) +**Latest** (2026-10-21, 1.0.0 release run: jpeg to pdf 383.7 MB/s against lopdf's 207.8 on the photo and 81.8 against 41.5 on the flat, pdf to jpeg 383.7 against 204.9 and 79.5 against 41.0, peak memory 2.7 to 4.2 MB against 5.6 to 8.9; every line PASSES) ## Purpose @@ -47,6 +47,35 @@ runs from GNU `time`. ## Results +### 2026-10-21, 1.0.0 release + +| Target | Ours | Reference | Result | +|---|---|---|---| +| jpeg -> pdf, photo (1.6 MB): throughput (MB/s of JPEG, median of 101 alternated runs) | 383.7 | 207.8 (lopdf) | PASS | +| jpeg -> pdf, photo: peak memory (MB) | 4.1 | 8.7 (lopdf) | PASS | +| pdf -> jpeg, photo (1.6 MB): throughput (MB/s of JPEG, median of 101 alternated runs) | 383.7 | 204.9 (lopdf) | PASS | +| pdf -> jpeg, photo: peak memory (MB) | 4.2 | 8.9 (lopdf) | PASS | +| jpeg -> pdf, flat (0.3 MB): throughput (MB/s of JPEG, median of 101 alternated runs) | 81.8 | 41.5 (lopdf) | PASS | +| jpeg -> pdf, flat: peak memory (MB) | 2.7 | 5.6 (lopdf) | PASS | +| pdf -> jpeg, flat (0.3 MB): throughput (MB/s of JPEG, median of 101 alternated runs) | 79.5 | 41.0 (lopdf) | PASS | +| pdf -> jpeg, flat: peak memory (MB) | 2.8 | 6.2 (lopdf) | PASS | + +commit: ec68950 (the `v1-wrapup` branch, before the version bump) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +Every line passes. The first attempt at this run timed nothing: the +alternating timer reads bash's microsecond clock (`EPOCHREALTIME`, +bash 5.0 and later), and it ran under macOS's bash 3.2, which does not +have it, so every run timed as zero and the throughput lines printed +PASS on empty cells. The script now stops with "bash 5 is needed" when +that clock is missing, and these rows are the rerun under bash 5.3. +This is the first block on this machine; the earlier blocks are the +Linux i7, so the rows do not compare like for like with them. The +machine was in use during the rerun (load average about 16 on 10 +cores: a browser, a VPN client, a remote-desktop host), so these +millisecond-scale lines are noisier than earlier blocks'. `[stock]` +rows are absent: `bench/data/stock.*` is not on this machine. + ### 2026-09-27, 0.23.1 release | Target | Ours | Reference | Result | diff --git a/DOCS/benchmarks/jpeg-png.md b/DOCS/benchmarks/jpeg-png.md index c9fae8d4..6eed4a71 100644 --- a/DOCS/benchmarks/jpeg-png.md +++ b/DOCS/benchmarks/jpeg-png.md @@ -1,6 +1,6 @@ # JPEG <-> PNG -**Latest** (2026-09-27, 0.24.0 release: every line PASSES) +**Latest** (2026-10-21, 1.0.0 release run: jpeg to png 88.0 MB/s against 38.8 on the photo and 728.7 against 490.9 on the flat, png to jpeg 553.5 against jpeg-encoder's 363.6 and 797.4 against 401.1, peak memory 7.1 to 13.7 MB against about 51 to 75; every line PASSES) ## Purpose @@ -79,6 +79,35 @@ median wall clock of the whole process, peak resident memory from GNU ## Results +### 2026-10-21, 1.0.0 release + +| Target | Ours | Reference | Result | +|---|---|---|---| +| jpeg -> png, photo (45.8 MB of pixels, 1.6 MB on disk): throughput (MB/s of decoded pixels) | 88.0 | 38.8 (image + png) | PASS | +| jpeg -> png, photo: peak memory (MB) | 13.7 | 75.2 (image + png) | PASS | +| png -> jpeg, photo (29.0 MB in + 45.8 MB of pixels): throughput (MB/s of input plus pixels) | 553.5 | 363.6 (png + jpeg-encoder); 294.6 (png + image) | PASS | +| png -> jpeg, photo: peak memory (MB) | 7.2 | 51.4 (png + jpeg-encoder); 51.3 (png + image) | PASS | +| png -> jpeg, photo: output size (MB) at quality 85 [extra] | 1.5 | 1.6 (jpeg-encoder); 2.0 (image) | n/a | +| png -> jpeg, photo: PSNR against the source (dB) at quality 85 [extra] | 36.19 | 35.90 (jpeg-encoder); 36.47 (image) | n/a | +| jpeg -> png, flat (45.8 MB of pixels, 0.3 MB on disk): throughput (MB/s of decoded pixels) | 728.7 | 490.9 (image + png) | PASS | +| jpeg -> png, flat: peak memory (MB) | 9.2 | 52.6 (image + png) | PASS | +| png -> jpeg, flat (1.6 MB in + 45.8 MB of pixels): throughput (MB/s of input plus pixels) | 797.4 | 401.1 (png + jpeg-encoder); 312.5 (png + image) | PASS | +| png -> jpeg, flat: peak memory (MB) | 7.1 | 51.3 (png + jpeg-encoder); 51.7 (png + image) | PASS | +| png -> jpeg, flat: output size (MB) at quality 85 [extra] | 0.3 | 0.3 (jpeg-encoder); 0.5 (image) | n/a | +| png -> jpeg, flat: PSNR against the source (dB) at quality 85 [extra] | 32.39 | 32.39 (jpeg-encoder); 56.88 (image) | n/a | + +commit: ec68950 (the `v1-wrapup` branch, before the version bump) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +Every line passes. This is the first block on this machine; the earlier +blocks are the Linux i7, so neither the throughput nor the memory rows +compare like for like with them (our memory rows read higher here, 13.7 +MB on the photo decode against 6.8 in the previous block, while the +crates' read about the same). The machine was in use during this run +(load average 7 to 12 on 10 cores: a browser, a VPN client, a +remote-desktop host), so wall-clock rows are noisier than earlier +blocks'. Sizes and PSNR are unchanged. + ### 2026-09-27, 0.24.0 release | Target | Ours | Reference | Result | diff --git a/DOCS/benchmarks/jsonl-json.md b/DOCS/benchmarks/jsonl-json.md index 554abbe9..3f8071e7 100644 --- a/DOCS/benchmarks/jsonl-json.md +++ b/DOCS/benchmarks/jsonl-json.md @@ -1,6 +1,6 @@ # JSON Lines <-> JSON -**Latest** (2026-09-25, first release of JSON Lines: jsonl -> json 301.7 MB/s of input at 2.8 MB peak; json -> jsonl 293.2 MB/s at 2.7 MB peak; every line PASSES against serde_json, at 1.8 to 3 times its throughput) +**Latest** (2026-10-21, 1.0.0 release run: jsonl -> json 187.6 MB/s of input at 2.7 MB peak; json -> jsonl 193.6 MB/s at 2.7 MB peak; every line PASSES against serde_json, at 1.9 to 3.1 times its throughput) ## Purpose @@ -52,6 +52,26 @@ follow `README.md`. ## Results +### 2026-10-21, `v1-wrapup` branch, the 1.0.0 release run + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +| Target | Ours | Reference | Result | +|---|---|---|---| +| jsonl -> json, 10M rows (1234.6 MB): throughput (MB/s of input) | 187.6 | 99.3 (serde_json) | PASS | +| jsonl -> json, 10M rows: peak memory (MB) | 2.7 | 5.1 (serde_json) | PASS | +| json -> jsonl, 10M rows (1234.6 MB): throughput (MB/s of input) | 193.6 | 61.6 (serde_json) | PASS | +| json -> jsonl, 10M rows: peak memory (MB) | 2.7 | 7884.5 (serde_json) | PASS | + +Every line passes, at 1.9 and 3.1 times serde_json's throughput. This +run is on a different machine from the previous block (an M1 Max, not +the Linux i7-13700K), so its absolute MB/s do not compare with that +block's; the ratios against the reference in the same session do. The +machine was in use during the run (load average 7 to 12 on 10 cores: a +browser, a VPN client, a remote-desktop host), so the wall-clock rows +are noisier than earlier blocks'. + ### 2026-09-25, first release of JSON Lines commit: 2f7e30d diff --git a/DOCS/benchmarks/markdown-docx.md b/DOCS/benchmarks/markdown-docx.md index 3036c0a0..2a66674d 100644 --- a/DOCS/benchmarks/markdown-docx.md +++ b/DOCS/benchmarks/markdown-docx.md @@ -1,6 +1,6 @@ # Markdown -> Word -**Latest** (2026-09-27, the lazy deflate and the faster bit writer: 307.4 MB/s of input plus uncompressed output on the markup-dense shape (from 206.7) and 397.0 on prose (from 288.8); every line PASSES) +**Latest** (2026-10-21, 1.0.0 release run on the Apple M1 Max: 229.7 MB/s of input plus uncompressed output on the markup-dense shape and 306.7 on prose, at 104.5 and 36.1 MB peak, against the reference's 40.2 and 87.1; every line PASSES) ## Purpose @@ -70,6 +70,30 @@ row. Rows and units follow `README.md`. ## Results +### 2026-10-21, 1.0.0 release run (`v1-wrapup` branch) + +commit: ec68950 (on the `v1-wrapup` branch, before the 1.0.0 release) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +| Target | Ours | Reference | Result | +|---|---|---|---| +| markdown -> docx, markup-dense (14.7 MB in + 90.7 MB out): throughput (MB/s of input plus uncompressed output) | 229.7 | 40.2 (pulldown-cmark + docx-rs, paragraphs and runs only) | PASS | +| markdown -> docx, markup-dense: throughput (MB/s of input) [extra] | 32.0 | 7.4 (reference) | n/a | +| markdown -> docx, markup-dense: peak memory (MB) | 104.5 | 1660.8 (reference) | PASS | +| markdown -> docx, prose (11.3 MB in + 20.2 MB out): throughput (MB/s of input plus uncompressed output) | 306.7 | 87.1 (pulldown-cmark + docx-rs, paragraphs and runs only) | PASS | +| markdown -> docx, prose: throughput (MB/s of input) [extra] | 110.1 | 26.0 (reference) | n/a | +| markdown -> docx, prose: peak memory (MB) | 36.1 | 466.7 (reference) | PASS | + +Every line passes. The machine was in use during this run (load average +7 to 12 on 10 cores: a browser, a VPN client, a remote-desktop host), so +the wall-clock rows are noisier than earlier blocks'. The inputs are the +same as the block below, but the machine is the Apple M1 Max, not the +Linux one, so the absolute rows do not compare. Against the reference in +the same session we are 5.7 times faster on the dense shape and 3.5 on +prose, against 6.7 and 4.6 below. Dense peak memory is 104.5 MB against +91.0 below; the reference's grew on this machine too (1660.8 against +1580.9). + ### 2026-09-27, the lazy deflate and the faster bit writer commit: a5c702a (on the `webp-spikes` branch, before its merge) diff --git a/DOCS/benchmarks/markdown-html.md b/DOCS/benchmarks/markdown-html.md index 3d27b2a6..e1ee12b3 100644 --- a/DOCS/benchmarks/markdown-html.md +++ b/DOCS/benchmarks/markdown-html.md @@ -1,6 +1,6 @@ # Markdown -> HTML -**Latest** (2026-09-25, 0.17.0 Markdown speedups: markdown -> html 112.9 MB/s of input on the markup-dense shape and 336.6 on prose, at 306 and 9.8 MB peak; all lines PASS against pulldown-cmark; the prose input is 4.7 MB and its median swings by tens of MB/s between runs) +**Latest** (2026-10-21, 1.0.0 release run on the Apple M1 Max: markdown -> html 99.0 MB/s of input on the markup-dense shape (73.8 MB) and 298.3 on prose (56.6 MB), at 352.2 and 83.9 MB peak, against pulldown-cmark's 4.7 and 251.2; all lines PASS) ## Purpose @@ -97,6 +97,31 @@ differ between renderers. ## Results +### 2026-10-21, 1.0.0 release run (`v1-wrapup` branch) + +commit: ec68950 (on the `v1-wrapup` branch, before the 1.0.0 release) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +| Target | Ours | Reference | Result | +|---|---|---|---| +| markdown -> html, markup-dense (73.8 MB): throughput (MB/s of input) | 99.0 | 4.7 | PASS | +| markdown -> html, prose (56.6 MB): throughput (MB/s of input) | 298.3 | 251.2 | PASS | +| markdown -> html, markup-dense: peak memory (MB) | 352.2 | 672.3 (reference) | PASS | +| markdown -> html, prose: peak memory (MB) | 83.9 | 263.9 (reference) | PASS | + +Every line passes. The first attempt at this run read stale, smaller +inputs left in `bench/data` (2.2 MB dense, 1.7 MB prose: the scripts +generate only when the file is missing, and several pairs share the +names); they were deleted, regenerated at the documented sizes, and the +pair rerun. The machine was in use during the rerun, more heavily than +during the rest of this run (load average 16 to 26 on 10 cores: a +browser, a VPN client, a remote-desktop host), so the wall-clock rows +are noisier than earlier blocks'. The machine is the Apple M1 Max, not +the Linux one of the block below, and the prose input is the full +56.6 MB rather than that block's 4.7 MB, so the rows do not compare +with it; against the reference in the same session we are 21 times +faster on dense and 19% faster on prose, against 28 times and 5% below. + ### 2026-09-25, 0.17.0 Markdown speedups commit: 0b33385 (the merge of the rows-to-document bridge, before the version bump) diff --git a/DOCS/benchmarks/markdown-json.md b/DOCS/benchmarks/markdown-json.md index 337cd58c..27ab60e7 100644 --- a/DOCS/benchmarks/markdown-json.md +++ b/DOCS/benchmarks/markdown-json.md @@ -1,6 +1,6 @@ # Markdown <-> events as JSON -**Latest** (2026-09-25, 0.17.0 Markdown speedups: markdown -> markdown-json 100.3 MB/s on markup-dense and 225.1 on prose at 305 MB peak; markdown-json -> markdown 310.5 MB/s at 2.8 MB peak; all lines PASS against pulldown-cmark with serde) +**Latest** (2026-10-21, 1.0.0 release run on the Apple M1 Max: markdown -> markdown-json 61.1 MB/s on markup-dense (73.8 MB) and 276.1 on prose (56.6 MB) at 351.8 MB peak; markdown-json -> markdown 159.1 MB/s at 3.0 MB peak; markdown-json -> markdown throughput FAILS (159.1 against 235.4), every other line PASSES against pulldown-cmark with serde) ## Purpose @@ -64,6 +64,39 @@ JSON reader-writer pair has its own round-trip test. ## Results +### 2026-10-21, 1.0.0 release run (`v1-wrapup` branch) + +commit: ec68950 (on the `v1-wrapup` branch, before the 1.0.0 release) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +| Target | Ours | Reference | Result | +|---|---|---|---| +| markdown -> markdown-json, markup-dense (73.8 MB): throughput (MB/s of input) | 61.1 | 5.1 | PASS | +| markdown -> markdown-json, prose (56.6 MB): throughput (MB/s of input) | 276.1 | 210.8 | PASS | +| markdown -> markdown-json, markup-dense: peak memory (MB) | 351.8 | 680.3 (reference) | PASS | +| markdown-json -> markdown, markup-dense (317.2 MB): throughput (MB/s of input, each side's own JSON) | 159.1 | 235.4 | FAIL | +| markdown-json -> markdown, markup-dense: peak memory (MB) | 3.0 | 1545.8 (reference) | PASS | + +The first attempt at this run read stale, smaller inputs left in +`bench/data` (2.2 MB dense, 1.7 MB prose: the scripts generate only when +the file is missing, and several pairs share the names); they were +deleted, regenerated at the documented sizes, and the pair rerun. The +machine was in use during the rerun, more heavily than during the rest +of this run (load average 16 to 26 on 10 cores: a browser, a VPN +client, a remote-desktop host), so the wall-clock rows are noisier than +earlier blocks'. The machine is the Apple M1 Max, not the Linux one of +the block below, and the prose input is the full 56.6 MB rather than +that block's 4.7 MB, so the rows do not compare with it directly. + +markdown-json -> markdown FAILS, where it passed in the block below +(310.5 against 239.4): 159.1 against 235.4 MB/s now. The harness gates +the row on wall time (`bench/pairs/markdown-json.sh`: ours in seconds +<= the reference's), and here our rate is lower as well. A paired A/B +of our binary at ec68950 against the commit before it (93a37d4) gave +the same median on both, but it was run on the stale 2.2 MB input, so +it does not apply at this input size; the cause of this failure is +unmeasured. + ### 2026-09-25, 0.17.0 Markdown speedups commit: 0b33385 (the merge of the rows-to-document bridge, before the version bump) @@ -108,5 +141,8 @@ for every specification example. - Back, we stream: the reader hands each event to the Markdown writer borrowing its own buffers, so memory stays at 2.4 MB against 1.2 GB for deserializing the array whole, and throughput is a third higher. + 2026-10-21: in the 1.0.0 release run on the Apple M1 Max the back + direction is slower than the reference (159.1 against 235.4 MB/s) and + FAILS; memory still streams (3.0 MB against 1545.8). - The JSON tokenizer now copies plain string runs a word at a time. That is shared code: JSON -> CSV went from 171 to 191 MB/s in the same run. diff --git a/DOCS/benchmarks/markdown-text.md b/DOCS/benchmarks/markdown-text.md index 2dc5b47f..c39f66a1 100644 --- a/DOCS/benchmarks/markdown-text.md +++ b/DOCS/benchmarks/markdown-text.md @@ -1,6 +1,6 @@ # Markdown -> plain text -**Latest** (2026-09-25, 0.17.0 Markdown speedups: markdown -> text 118.9 MB/s of input on the markup-dense shape and 230.4 on prose, at 306 and 10.0 MB peak; all lines PASS against a pulldown-cmark text dump; the prose input is 4.7 MB and its median swings between runs) +**Latest** (2026-10-21, 1.0.0 release run on the Apple M1 Max: markdown -> text 99.3 MB/s of input on the markup-dense shape (73.8 MB) and 272.9 on prose (56.6 MB), at 352.2 and 83.8 MB peak; all lines PASS against a pulldown-cmark text dump at 5.3 and 253.0) ## Purpose @@ -51,6 +51,31 @@ units, three runs each, median wall clock, peak RSS from GNU `time`. ## Results +### 2026-10-21, 1.0.0 release run (`v1-wrapup` branch) + +commit: ec68950 (on the `v1-wrapup` branch, before the 1.0.0 release) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +| Target | Ours | Reference | Result | +|---|---|---|---| +| markdown -> text, markup-dense (73.8 MB): throughput (MB/s of input) | 99.3 | 5.3 | PASS | +| markdown -> text, prose (56.6 MB): throughput (MB/s of input) | 272.9 | 253.0 | PASS | +| markdown -> text, markup-dense: peak memory (MB) | 352.2 | 733.2 (reference) | PASS | +| markdown -> text, prose: peak memory (MB) | 83.8 | 316.6 (reference) | PASS | + +Every line passes. The first attempt at this run read stale, smaller +inputs left in `bench/data` (2.2 MB dense, 1.7 MB prose: the scripts +generate only when the file is missing, and several pairs share the +names) and failed prose throughput on them (88.2 against 98.7); they +were deleted, regenerated at the documented sizes, and the pair rerun. +The machine was in use during the rerun, more heavily than during the +rest of this run (load average 16 to 26 on 10 cores: a browser, a VPN +client, a remote-desktop host), so the wall-clock rows are noisier than +earlier blocks'. The machine is the Apple M1 Max, not the Linux one of +the block below, and the prose input is the full 56.6 MB rather than +that block's 4.7 MB, so the rows do not compare with it; on prose we +are 8% ahead of the reference, against 2% below. + ### 2026-09-25, 0.17.0 Markdown speedups commit: 0b33385 (the merge of the rows-to-document bridge, before the version bump) diff --git a/DOCS/benchmarks/numbers-csv.md b/DOCS/benchmarks/numbers-csv.md index fcd8302b..062eca9e 100644 --- a/DOCS/benchmarks/numbers-csv.md +++ b/DOCS/benchmarks/numbers-csv.md @@ -1,6 +1,6 @@ # Numbers -> CSV -**Latest** (2026-10-06, 0.27.0: 36 MB/s of input on 200,000-row documents at 125 to 141 MB, against numbers-parser's 0.5 MB/s at 4.2 GB; on a real 65,000-row document 64 MB/s at 53 MB, against numbers-parser's 2.2 at 371 MB and LibreOffice's 0.9 at 1.4 GB; every line PASSES. Numbers -> Excel is its own pair, `numbers-xlsx.md`. Earlier, the `workbooks` branch: 37 MB/s of input on 200,000-row documents at 125 to 145 MB, against numbers-parser's 0.5 MB/s at 4.3 to 4.5 GB; on a real 65,000-row document 68 MB/s at 54 MB, against numbers-parser's 2.4 at 353 MB and LibreOffice's 0.9 at 1.4 GB; every line PASSES) +**Latest** (2026-10-21, 1.0.0 release run: 30 to 31 MB/s of input on 200,000-row documents at 105 to 128 MB, against numbers-parser's 0.5 MB/s at 4.5 GB; on a real 65,000-row document 63 MB/s at 41 MB, against numbers-parser's 2.3 at 379 MB and LibreOffice's 0.9 at 1.4 GB; every line PASSES. Numbers -> Excel is its own pair, `numbers-xlsx.md`. Earlier, 0.27.0: 36 MB/s of input on 200,000-row documents at 125 to 141 MB, against numbers-parser's 0.5 MB/s at 4.2 GB; on a real 65,000-row document 64 MB/s at 53 MB, against numbers-parser's 2.2 at 371 MB and LibreOffice's 0.9 at 1.4 GB; every line PASSES) ## Purpose @@ -64,6 +64,32 @@ two seconds) are inside their times, which matters most on the 4 MB file. ## Results +### 2026-10-21, `v1-wrapup` branch, the 1.0.0 release run + +| Target | Ours | numbers-parser | LibreOffice | Result | +|---|---|---|---|---| +| numbers -> csv, real (4.1 MB): throughput (MB/s of input) | 63.0 | 2.3 | 0.9 | PASS | +| numbers -> csv, real: peak memory (MB) | 41.4 | 379.2 | 1408.7 | PASS | +| numbers -> csv, dense (14.3 MB): throughput (MB/s of input) | 29.9 | 0.5 | n/a | PASS | +| numbers -> csv, dense: peak memory (MB) | 127.9 | 4552.5 | n/a | PASS | +| numbers -> csv, sheets (15.1 MB): throughput (MB/s of input) | 30.8 | 0.5 | n/a | PASS | +| numbers -> csv, sheets: peak memory (MB) | 105.1 | 4465.2 | n/a | PASS | + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +Every line passes. The machine was in use during this run (load average +7 to 12 on 10 cores: a browser, a VPN client, a remote-desktop host), so +the wall-clock rows are noisier than earlier blocks'. Against 0.27.0 on +the same machine, throughput is lower on the 200,000-row documents +(dense 36.5 -> 29.9 MB/s, sheets 36.1 -> 30.8) and about level on the +real document (64.1 -> 63.0), and peak memory is lower on every row +(real 53.2 -> 41.4 MB, dense 141.2 -> 127.9, sheets 125.3 -> 105.1). +The commit, ec68950, holds the registry as a const table (converter +metadata as data, the image pairs built only when an image format is an +end), which lowered every conversion's starting memory by about 0.3 MB; +the rest of the drop came with the releases since 0.27.0. + ### 2026-10-06, 0.27.0 | Target | Ours | numbers-parser | LibreOffice | Result | @@ -103,3 +129,8 @@ keeping tile rows as bytes brought it to 145 MB. What remains is the decoded string table and the tiles' cell buffers, held for the whole document; reading tiles from the stream bytes in place is the next lever. Re-run on the Linux bench machine for the record beside the other pairs. + +On 2026-10-21, under load, the margins were 27 to 62 times +numbers-parser's speed at a ninth to a forty-second of its memory, and +70 times LibreOffice's speed at a thirty-fourth of its memory; the real +document's 27 times is below the 28 stated above. diff --git a/DOCS/benchmarks/numbers-xlsx.md b/DOCS/benchmarks/numbers-xlsx.md index 8e80e881..0c1fd5ce 100644 --- a/DOCS/benchmarks/numbers-xlsx.md +++ b/DOCS/benchmarks/numbers-xlsx.md @@ -1,6 +1,6 @@ # Numbers -> Excel -**Latest** (2026-10-06, 0.27.1: 144 to 146 MB/s of input plus uncompressed output on 200,000-row documents at 127 to 147 MB, against numbers-parser with openpyxl's 2.3 to 2.4 at 4.2 GB; on a real 65,000-row document 139 MB/s at 56 MB, against numbers-parser's 4.2 at 365 MB and LibreOffice's 1.4 at 1.4 GB; every line PASSES) +**Latest** (2026-10-21, 1.0.0 release run: 115 to 116 MB/s of input plus uncompressed output on 200,000-row documents at 109 to 129 MB, against numbers-parser with openpyxl's 2.4 at 4.5 GB; on a real 65,000-row document 117 MB/s at 45 MB, against numbers-parser's 4.1 at 371 MB and LibreOffice's 1.4 at 1.4 GB; every line PASSES) ## Purpose @@ -59,6 +59,35 @@ checked by reading both back with openpyxl). ## Results +### 2026-10-21, `v1-wrapup` branch, the 1.0.0 release run + +| Target | Ours | numbers-parser | LibreOffice | Result | +|---|---|---|---|---| +| numbers -> xlsx, real (4.1 MB in + 6.4 MB out): throughput (MB/s of input plus uncompressed output) | 116.8 | 4.1 | 1.4 | PASS | +| numbers -> xlsx, real: peak memory (MB) | 45.1 | 370.6 | 1407.8 | PASS | +| numbers -> xlsx, dense (14.3 MB in + 83.1 MB out): throughput (MB/s of input plus uncompressed output) | 115.5 | 2.4 | n/a | PASS | +| numbers -> xlsx, dense: peak memory (MB) | 128.8 | 4568.4 | n/a | PASS | +| numbers -> xlsx, sheets (15.1 MB in + 80.1 MB out): throughput (MB/s of input plus uncompressed output) | 114.6 | 2.4 | n/a | PASS | +| numbers -> xlsx, sheets: peak memory (MB) | 108.8 | 4475.9 | n/a | PASS | + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +Every line passes. The reference column is numbers-parser with openpyxl, +as before; the harness now heads it `numbers-parser`. The machine was in +use during this run (load average 7 to 12 on 10 cores: a browser, a VPN +client, a remote-desktop host), so the wall-clock rows are noisier than +earlier blocks'. Against 0.27.1 on the same machine, throughput is lower +on every row (real 139.1 -> 116.8, dense 146.1 -> 115.5, sheets +144.3 -> 114.6) and the output is about 3 MB smaller on the generated +documents (86.2 -> 83.1 MB dense, 83.2 -> 80.1 sheets). Peak memory is +lower on every row (real 56.1 -> 45.1 MB, dense 146.6 -> 128.8, sheets +126.8 -> 108.8). The commit, ec68950, holds the registry as a const +table (converter metadata as data, the image pairs built only when an +image format is an end), which lowered every conversion's starting +memory by about 0.3 MB; the rest of the drop came with the releases +since 0.27.1. + ### 2026-10-06, 0.27.1 With typed dates and booleans: the output is smaller than 0.27.0's (86.2 @@ -101,3 +130,8 @@ twenty-ninth of numbers-parser's memory and a twenty-fifth of LibreOffice's. Memory is the reader's, the same as `numbers -> csv`: the workbook writer streams each sheet. The levers left are the reader's (`numbers-csv.md`, Conclusions). + +On 2026-10-21, under load, the margins were 28 to 48 times +numbers-parser with openpyxl and 83 times LibreOffice, at an eighth to a +forty-first of numbers-parser's memory and a thirty-first of +LibreOffice's. diff --git a/DOCS/benchmarks/ods-csv.md b/DOCS/benchmarks/ods-csv.md index abb3967f..de7fbc42 100644 --- a/DOCS/benchmarks/ods-csv.md +++ b/DOCS/benchmarks/ods-csv.md @@ -1,6 +1,6 @@ # ODS <-> CSV -**Latest** (2026-10-17, branch `ods`: every line PASSES) +**Latest** (2026-10-21, 1.0.0 release run: ods -> csv 322.3 MB/s of uncompressed input at 206 MB on the generated sheet and 940.5 MB/s at 51 MB on the real one, against calamine's 277.6 and 310.1 and LibreOffice's 75.7 and 39.9; csv -> ods 531.1 MB/s at 6.4 MB against LibreOffice's 49.2; every line PASSES) ## Purpose @@ -61,6 +61,28 @@ wall clock of the whole process, peak resident memory from GNU `time`. ## Results +2026-10-21, `v1-wrapup` branch, the 1.0.0 release run: + +| Target | Ours | calamine | LibreOffice | Result | +|---|---|---|---|---| +| ods -> csv, generated (764.0 MB uncompressed): throughput (MB/s of uncompressed input) | 322.3 | 277.6 | 75.7 | PASS | +| ods -> csv, generated: peak memory (MB) | 206.4 | 990.5 | 850.2 | PASS | +| ods -> csv, real (710.1 MB uncompressed): throughput (MB/s of uncompressed input) | 940.5 | 310.1 | 39.9 | PASS | +| ods -> csv, real: peak memory (MB) | 51.3 | 898.5 | 396.1 | PASS | +| csv -> ods, generated (56.8 MB in + 583.7 MB out): throughput (MB/s of input plus uncompressed output) | 531.1 | n/a | 49.2 | PASS | +| csv -> ods, generated: peak memory (MB) | 6.4 | n/a | 786.1 | PASS | + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +Every line passes, and the throughput rows are within 1 percent of the +`ods` branch's final run. The machine was in use during this run (load +average 7 to 12 on 10 cores: a browser, a VPN client, a remote-desktop +host), so the wall-clock rows are noisier than earlier blocks'. Peak +memory on the real document rose from 40.1 to 51.3 MB, still under a +seventh of LibreOffice's; csv -> ods fell from 7.4 to 6.4 MB, and the +generated read is level (204.8 -> 206.4 MB). + 2026-10-17, `ods` branch, final code: | Target | Ours | calamine | LibreOffice | Result | diff --git a/DOCS/benchmarks/odt-docx.md b/DOCS/benchmarks/odt-docx.md index 700b9804..ce30f52b 100644 --- a/DOCS/benchmarks/odt-docx.md +++ b/DOCS/benchmarks/odt-docx.md @@ -1,6 +1,6 @@ # ODT <-> Word -**Latest** (2026-10-13, the `odt` branch: docx -> odt at 161.4 MB/s of input plus output on the styled shape and 168.6 on prose, odt -> docx at 192.7 and 243.4; every line PASSES against pandoc at 77 to 122 times its speed and a fifteenth to a thirty-first of its memory) +**Latest** (2026-10-21, the 1.0.0 release run: docx -> odt at 176.4 MB/s of input plus output on the styled shape and 171.4 on prose, odt -> docx at 181.1 and 231.5; every line PASSES against pandoc at 88 to 122 times its speed and a sixteenth to a thirtieth of its memory) ## Purpose @@ -53,6 +53,34 @@ larger style sheets, so both directions will run somewhat slower on them. ## Results +### 2026-10-21, the `v1-wrapup` branch, the 1.0.0 release run + +| Target | Ours | Reference | Result | +|---|---|---|---| +| docx -> odt, styled (6.0 MB in + 3.8 MB out, uncompressed): throughput (MB/s of input plus output) | 176.4 | 2.0 (pandoc) | PASS | +| docx -> odt, styled: throughput (MB/s of uncompressed input) [extra] | 108.1 | 1.2 (pandoc) | n/a | +| docx -> odt, styled: peak memory (MB) | 33.0 | 619.1 (pandoc) | PASS | +| odt -> docx, styled (3.8 MB in + 5.9 MB out, uncompressed): throughput (MB/s of input plus output) | 181.1 | 1.5 (pandoc) | PASS | +| odt -> docx, styled: throughput (MB/s of uncompressed input) [extra] | 71.0 | 0.6 (pandoc) | n/a | +| odt -> docx, styled: peak memory (MB) | 21.8 | 371.5 (pandoc) | PASS | +| docx -> odt, prose (7.5 MB in + 5.7 MB out, uncompressed): throughput (MB/s of input plus output) | 171.4 | 1.6 (pandoc) | PASS | +| docx -> odt, prose: throughput (MB/s of uncompressed input) [extra] | 97.5 | 0.9 (pandoc) | n/a | +| docx -> odt, prose: peak memory (MB) | 49.0 | 1503.4 (pandoc) | PASS | +| odt -> docx, prose (5.7 MB in + 7.2 MB out, uncompressed): throughput (MB/s of input plus output) | 231.5 | 1.9 (pandoc) | PASS | +| odt -> docx, prose: throughput (MB/s of uncompressed input) [extra] | 102.9 | 0.8 (pandoc) | n/a | +| odt -> docx, prose: peak memory (MB) | 34.3 | 561.1 (pandoc) | PASS | + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +The machine was in use during this run (load average 7 to 12 on 10 +cores: a browser, a VPN client, a remote-desktop host), so the +wall-clock rows are noisier than the previous block's. Every line still +passes. The rates moved by up to 10% either way against 2026-10-13 +(`docx -> odt` styled 161.4 to 176.4, `odt -> docx` styled 192.7 to +181.1), and peak memory by up to 3.1 MB either way, the most still `docx -> odt` on +prose at 49.0 MB. + ### 2026-10-13, the `odt` branch | Target | Ours | Reference | Result | @@ -81,3 +109,7 @@ prose shape. The ODT writer builds the body before the styles part (which names every automatic style the body used), so it holds the body until the end; a streaming writer would need the styles first. Re-run on the Linux bench machine for the record beside the other pairs. + +2026-10-21, the 1.0.0 release run: `docx -> odt` on prose peaked at +49.0 MB, at the 49 MB mark rather than under it; every line still +passes, inside the 64 MB. diff --git a/DOCS/benchmarks/pages-docx.md b/DOCS/benchmarks/pages-docx.md index c6ae3a7d..9e8a5a71 100644 --- a/DOCS/benchmarks/pages-docx.md +++ b/DOCS/benchmarks/pages-docx.md @@ -1,6 +1,6 @@ # Pages <-> Word -**Latest** (2026-10-06, 0.26.0: Word -> Pages recorded for the first time, at 66.8 MB/s of uncompressed input on the styled shape and 91.8 on prose, peaking at 39.6 and 51.1 MB; Pages -> Word at 142.8 and 160.3 MB/s on this machine; every goal PASSES) +**Latest** (2026-10-21, the 1.0.0 release run: Pages -> Word at 129.3 MB/s of input plus uncompressed output on the styled shape and 140.0 on prose, peaking at 30.0 and 46.8 MB; Word -> Pages at 66.5 and 89.2 MB/s of uncompressed input, peaking at 42.8 and 54.1 MB; every goal PASSES) ## Purpose @@ -82,6 +82,36 @@ in-process runs; inputs come from the `pages-json` pair's generator. ## Results +### 2026-10-21, the `v1-wrapup` branch, the 1.0.0 release run + +| Target | Ours | Reference | Result | +|---|---|---|---| +| pages -> docx, styled (1.4 MB in + 6.0 MB out): throughput (MB/s of input plus uncompressed output) | 129.3 | goal: 50 | PASS | +| pages -> docx, styled: throughput (MB/s of input) [extra] | 24.6 | recorded | n/a | +| pages -> docx, styled: peak memory (MB) | 30.0 | goal: <= 64.0 | PASS | +| pages -> docx, prose (1.5 MB in + 7.5 MB out): throughput (MB/s of input plus uncompressed output) | 140.0 | goal: 50 | PASS | +| pages -> docx, prose: throughput (MB/s of input) [extra] | 23.5 | recorded | n/a | +| pages -> docx, prose: peak memory (MB) | 46.8 | goal: <= 64.0 | PASS | +| docx -> pages, styled (6.0 MB uncompressed): throughput (MB/s of uncompressed input) | 66.5 | goal: 50 | PASS | +| docx -> pages, styled: throughput (MB/s of input) [extra] | 2.3 | recorded | n/a | +| docx -> pages, styled: peak memory (MB) | 42.8 | goal: <= 64.0 | PASS | +| docx -> pages, prose (7.5 MB uncompressed): throughput (MB/s of uncompressed input) | 89.2 | goal: 50 | PASS | +| docx -> pages, prose: throughput (MB/s of input) [extra] | 2.6 | recorded | n/a | +| docx -> pages, prose: peak memory (MB) | 54.1 | goal: <= 64.0 | PASS | + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +The machine was in use during this run (load average 7 to 12 on 10 +cores: a browser, a VPN client, a remote-desktop host), so the +wall-clock rows are noisier than the previous block's. Every goal still +passes. Against 2026-10-06 on the same machine, Pages -> Word ran at +129.3 and 140.0 MB/s where it ran at 142.8 and 160.3, and Word -> Pages +held at 66.5 and 89.2 (66.8 and 91.8). Peak memory rose on every row: +Pages -> Word 26.3 to 30.0 MB on styled and 33.8 to 46.8 MB on prose, +Word -> Pages 39.6 to 42.8 and 51.1 to 54.1 MB. The prose rise is the +largest and leaves 17 MB of headroom under the goal. + ### 2026-10-06, 0.26.0: Word -> Pages added Word -> Pages (since 0.25.0) reads the Word files the Pages -> Word rows diff --git a/DOCS/benchmarks/pages-html.md b/DOCS/benchmarks/pages-html.md index f975aaa1..f634a5c1 100644 --- a/DOCS/benchmarks/pages-html.md +++ b/DOCS/benchmarks/pages-html.md @@ -1,6 +1,6 @@ # Pages -> HTML -**Latest** (2026-09-24, reader cursors: pages -> html 39.7 MB/s of input on the dense shape and 38.5 on prose, at 45.5 and 27.9 MB peak; memory PASSES on both shapes, throughput FAILS on both, see Conclusions and `STATE.md`) +**Latest** (2026-10-21, the 1.0.0 release run: pages -> html 32.7 MB/s of input on the styled shape and 27.0 on prose, at 27.2 and 33.1 MB peak; memory PASSES on both shapes, throughput FAILS on both, see Conclusions and `STATE.md`) ## Purpose @@ -72,6 +72,28 @@ in-process runs; inputs come from the `pages-json` pair's generator. ## Results +### 2026-10-21, the `v1-wrapup` branch, the 1.0.0 release run + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +| Target | Ours | Reference | Result | +|---|---|---|---| +| pages -> html, styled (1.4 MB): throughput (MB/s of input) | 32.7 | goal: 50 | FAIL | +| pages -> html, styled: peak memory (MB) | 27.2 | goal: <= 64.0 | PASS | +| pages -> html, prose (1.5 MB): throughput (MB/s of input) | 27.0 | goal: 50 | FAIL | +| pages -> html, prose: peak memory (MB) | 33.1 | goal: <= 64.0 | PASS | + +The machine was in use during this run (load average 7 to 12 on 10 +cores: a browser, a VPN client, a remote-desktop host), so the +wall-clock rows are noisier than they would be on an idle machine. This +block is from the M1 Max laptop and the previous one from the Linux +bench machine, and the styled package the generator makes is now +1.4 MB where it was 2.5 MB, so the styled rows are not a like-for-like +comparison. The standing is unchanged: throughput still FAILS on both +shapes (32.7 and 27.0 MB/s against 50) and memory still passes (27.2 +and 33.1 MB against 64). + ### 2026-09-24, reader cursors and one arena copy commit: 1ed4458 diff --git a/DOCS/benchmarks/pages-json.md b/DOCS/benchmarks/pages-json.md index cd965f1a..b1ee5aef 100644 --- a/DOCS/benchmarks/pages-json.md +++ b/DOCS/benchmarks/pages-json.md @@ -1,6 +1,6 @@ # Pages <-> pages-json -**Latest** (2026-09-24: pages -> pages-json 30.0 MB/s of input on the dense shape and 40.6 on prose, at 44.1 and 27.3 MB peak, below the 50 MB/s goal; pages-json -> pages 325.3 and 338.2 MB/s of input at 46 and 31 MB peak, passing both goals) +**Latest** (2026-10-21, the 1.0.0 release run: pages -> pages-json 24.4 MB/s of input on the styled shape and 25.9 on prose, at 27.6 and 35.6 MB peak, below the 50 MB/s goal, so throughput FAILS on both shapes; pages-json -> pages 145.9 and 238.6 MB/s of input at 27.8 and 45.2 MB peak, passing both goals) ## Purpose @@ -90,6 +90,33 @@ and `ours` / `ours-back` modes for in-process runs. ## Results +### 2026-10-21, the `v1-wrapup` branch, the 1.0.0 release run + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +| Target | Ours | Reference | Result | +|---|---|---|---| +| pages -> pages-json, styled (1.4 MB): throughput (MB/s of input) | 24.4 | goal: 50 | FAIL | +| pages -> pages-json, styled: peak memory (MB) | 27.6 | goal: <= 64.0 | PASS | +| pages-json -> pages, styled (15.1 MB): throughput (MB/s of input) | 145.9 | goal: 50 | PASS | +| pages-json -> pages, styled: peak memory (MB) | 27.8 | goal: <= 64.0 | PASS | +| pages -> pages-json, prose (1.5 MB): throughput (MB/s of input) | 25.9 | goal: 50 | FAIL | +| pages -> pages-json, prose: peak memory (MB) | 35.6 | goal: <= 64.0 | PASS | +| pages-json -> pages, prose (16.1 MB): throughput (MB/s of input) | 238.6 | goal: 50 | PASS | +| pages-json -> pages, prose: peak memory (MB) | 45.2 | goal: <= 64.0 | PASS | + +The machine was in use during this run (load average 7 to 12 on 10 +cores: a browser, a VPN client, a remote-desktop host), so the +wall-clock rows are noisier than they would be on an idle machine. This +block is from the M1 Max laptop and the previous one from the Linux +bench machine, and the generator's styled shape is now a 1.4 MB package +and 15.1 MB of JSON where it was 2.5 and 33.7 MB, so the styled rows +are not a like-for-like comparison. The standing is unchanged: the +forward direction's throughput still FAILS on both shapes (24.4 and +25.9 MB/s against 50), and the reverse direction and every memory row +pass. + ### 2026-09-24, reader cursors and one arena copy commit: 1ed4458 diff --git a/DOCS/benchmarks/pages-markdown.md b/DOCS/benchmarks/pages-markdown.md index d28b06f2..22c6565a 100644 --- a/DOCS/benchmarks/pages-markdown.md +++ b/DOCS/benchmarks/pages-markdown.md @@ -1,6 +1,6 @@ # Pages -> Markdown -**Latest** (2026-09-24, reader cursors: pages -> markdown 38.7 MB/s of input on the dense shape and 33.4 on prose, at 45.4 and 27.9 MB peak; memory PASSES on both shapes, throughput FAILS on both, see Conclusions and `STATE.md`) +**Latest** (2026-10-21, the 1.0.0 release run: pages -> markdown 31.5 MB/s of input on the styled shape and 26.1 on prose, at 26.2 and 44.2 MB peak; memory PASSES on both shapes, throughput FAILS on both, see Conclusions and `STATE.md`) ## Purpose @@ -75,6 +75,28 @@ in-process runs; inputs come from the `pages-json` pair's generator. ## Results +### 2026-10-21, the `v1-wrapup` branch, the 1.0.0 release run + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +| Target | Ours | Reference | Result | +|---|---|---|---| +| pages -> markdown, styled (1.4 MB): throughput (MB/s of input) | 31.5 | goal: 50 | FAIL | +| pages -> markdown, styled: peak memory (MB) | 26.2 | goal: <= 64.0 | PASS | +| pages -> markdown, prose (1.5 MB): throughput (MB/s of input) | 26.1 | goal: 50 | FAIL | +| pages -> markdown, prose: peak memory (MB) | 44.2 | goal: <= 64.0 | PASS | + +The machine was in use during this run (load average 7 to 12 on 10 +cores: a browser, a VPN client, a remote-desktop host), so the +wall-clock rows are noisier than they would be on an idle machine. This +block is from the M1 Max laptop and the previous one from the Linux +bench machine, and the styled package the generator makes is now +1.4 MB where it was 2.5 MB, so the styled rows are not a like-for-like +comparison. The standing is unchanged: throughput still FAILS on both +shapes (31.5 and 26.1 MB/s against 50) and memory still passes (26.2 +and 44.2 MB against 64). + ### 2026-09-24, reader cursors and one arena copy commit: 1ed4458 diff --git a/DOCS/benchmarks/pages-text.md b/DOCS/benchmarks/pages-text.md index 350e0691..b85e1c45 100644 --- a/DOCS/benchmarks/pages-text.md +++ b/DOCS/benchmarks/pages-text.md @@ -1,6 +1,6 @@ # Pages -> plain text -**Latest** (2026-09-24, reader cursors: pages -> text 40.1 MB/s of input on the dense shape and 32.3 on prose, at 45.3 and 28.1 MB peak; memory PASSES on both shapes, throughput FAILS on both, see Conclusions and `STATE.md`) +**Latest** (2026-10-21, the 1.0.0 release run: pages -> text 29.3 MB/s of input on the styled shape and 28.0 on prose, at 29.9 and 43.4 MB peak; memory PASSES on both shapes, throughput FAILS on both, see Conclusions and `STATE.md`) ## Purpose @@ -74,6 +74,28 @@ in-process runs; inputs come from the `pages-json` pair's generator. ## Results +### 2026-10-21, the `v1-wrapup` branch, the 1.0.0 release run + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +| Target | Ours | Reference | Result | +|---|---|---|---| +| pages -> text, styled (1.4 MB): throughput (MB/s of input) | 29.3 | goal: 50 | FAIL | +| pages -> text, styled: peak memory (MB) | 29.9 | goal: <= 64.0 | PASS | +| pages -> text, prose (1.5 MB): throughput (MB/s of input) | 28.0 | goal: 50 | FAIL | +| pages -> text, prose: peak memory (MB) | 43.4 | goal: <= 64.0 | PASS | + +The machine was in use during this run (load average 7 to 12 on 10 +cores: a browser, a VPN client, a remote-desktop host), so the +wall-clock rows are noisier than they would be on an idle machine. This +block is from the M1 Max laptop and the previous one from the Linux +bench machine, and the styled package the generator makes is now +1.4 MB where it was 2.5 MB, so the styled rows are not a like-for-like +comparison. The standing is unchanged: throughput still FAILS on both +shapes (29.3 and 28.0 MB/s against 50) and memory still passes (29.9 +and 43.4 MB against 64). + ### 2026-09-24, reader cursors and one arena copy commit: 1ed4458 diff --git a/DOCS/benchmarks/pages-xlsx.md b/DOCS/benchmarks/pages-xlsx.md index 0dd2ac24..f8fe4293 100644 --- a/DOCS/benchmarks/pages-xlsx.md +++ b/DOCS/benchmarks/pages-xlsx.md @@ -1,6 +1,6 @@ # Pages -> Excel -**Latest** (2026-10-08: 14.8 and 18.0 MB/s of input against the 50 MB/s goal, and 1,777 and 618 MB peak against 64 MB, from 2,231 and 736; every line still FAILS; what remains is listed in Conclusions and in `STATE.md` as tech debt) +**Latest** (2026-10-21, the 1.0.0 release run: 13.4 and 16.3 MB/s of input on `tables` and `report` against the 50 MB/s goal, and 1,746 and 597 MB peak against 64 MB; every line still FAILS; what remains is listed in Conclusions and in `STATE.md` as tech debt) ## Purpose @@ -54,6 +54,28 @@ document of the same tables will read lighter; the per-table cost remains. ## Results +### 2026-10-21, the `v1-wrapup` branch, the 1.0.0 release run + +| Target | Ours | Reference | Result | +|---|---|---|---| +| pages -> xlsx, tables (57.6 MB): throughput (MB/s of input) | 13.4 | goal: 50 | FAIL | +| pages -> xlsx, tables: peak memory (MB) | 1745.6 | goal: <= 64.0 | FAIL | +| pages -> xlsx, tables (57.6 MB in + 27.8 MB out): throughput (MB/s of input plus uncompressed output) [extra] | 19.8 | n/a | n/a | +| pages -> xlsx, report (21.6 MB): throughput (MB/s of input) | 16.3 | goal: 50 | FAIL | +| pages -> xlsx, report: peak memory (MB) | 596.7 | goal: <= 64.0 | FAIL | +| pages -> xlsx, report (21.6 MB in + 7.6 MB out): throughput (MB/s of input plus uncompressed output) [extra] | 22.1 | n/a | n/a | + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +The machine was in use during this run (load average 7 to 12 on 10 +cores: a browser, a VPN client, a remote-desktop host), so the +wall-clock rows are noisier than the 2026-10-08 block's. Every line +still FAILS, as before. Against 2026-10-08, throughput read 13.4 and +16.3 MB/s where it read 14.8 and 18.0, and peak memory 1,745.6 and +596.7 MB where it was 1,777.2 and 618.0. The plain-cell writer and +streaming tables are still the tech debt in `STATE.md`. + ### 2026-10-06, 0.27.1 | Target | Ours | Reference | Result | diff --git a/DOCS/benchmarks/pdf-text.md b/DOCS/benchmarks/pdf-text.md index 6ed74098..18435bab 100644 --- a/DOCS/benchmarks/pdf-text.md +++ b/DOCS/benchmarks/pdf-text.md @@ -1,6 +1,6 @@ # PDF -> Text -**Latest** (2026-09-27, 0.24.0 release: every line PASSES) +**Latest** (2026-10-21, the 1.0.0 release run: pdf -> text at 36.3 MB/s of PDF on `story`, 2.4 on `gs`, and 16.3 on `base14`, at 10.5, 21.6, and 13.9 MB peak; every throughput line PASSES, and the `gs` and `base14` memory lines FAIL, against pdf-extract's 10.0 MB on `gs` and pdftotext's 13.2 MB on `base14`) ## Purpose @@ -47,6 +47,43 @@ outputs are removed before each run. ## Results +### 2026-10-21, the `v1-wrapup` branch, the 1.0.0 release run + +| Target | Ours | Reference | Result | +|---|---|---|---| +| pdf -> text, story (2.9 MB, 300 pages): throughput (MB/s of PDF) | 36.3 | 2.8 (pdf-extract); 13.9 (pdftotext) | PASS | +| pdf -> text, story: peak memory (MB) | 10.5 | 17.6 (pdf-extract); 11.8 (pdftotext) | PASS | +| pdf -> text, story: words out [extra] | 116700 | 116700 (pdf-extract); 116700 (pdftotext) | n/a | +| pdf -> text, gs (0.2 MB, 300 pages): throughput (MB/s of PDF) | 2.4 | 1.6 (pdf-extract); 1.4 (pdftotext) | PASS | +| pdf -> text, gs: peak memory (MB) | 21.6 | 10.0 (pdf-extract); 12.1 (pdftotext) | FAIL | +| pdf -> text, gs: words out [extra] | 116700 | 5978 (pdf-extract); 116700 (pdftotext) | n/a | +| pdf -> text, base14 (3.1 MB, 300 pages): throughput (MB/s of PDF) | 16.3 | 2.3 (pdf-extract); 8.6 (pdftotext) | PASS | +| pdf -> text, base14: peak memory (MB) | 13.9 | 30.3 (pdf-extract); 13.2 (pdftotext) | FAIL | +| pdf -> text, base14: words out [extra] | 172800 | 172800 (pdf-extract); 172800 (pdftotext) | n/a | + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +The machine was in use during this run (load average 7 to 12 on 10 +cores: a browser, a VPN client, a remote-desktop host), so the +wall-clock rows are noisier than the earlier blocks'. This block is from +the M1 Max laptop and the earlier ones from the Linux bench machine. + +The inputs were regenerated for this run: PyMuPDF and Ghostscript were +no longer on this machine, so `bench/pairs/pdf-text.sh` ran with +PyMuPDF (pymupdf.VersionBind 2.0) in a fresh Python 3.13 venv and +Ghostscript 10.08.0 from Homebrew. The `gs` input is Ghostscript's +rewrite of the report, so a different Ghostscript makes a different +file: the `gs` rows' input changed with the tool version. The sizes, +page counts, and words out match the earlier blocks. + +Two lines that passed in the 0.24.0 release block FAIL now, both on +memory. `gs`: 21.6 MB against 10.0 (pdf-extract) and 12.1 (pdftotext), +where it was 5.8 against 8.5 and 17.4. `base14`: 13.9 MB against 13.2 +(pdftotext), where it was 9.2 against 25.9 (pdf-extract) and 17.7 +(pdftotext). Every throughput line and the `story` memory line still +pass. + ### 2026-09-27, 0.24.0 release | Target | Ours | Reference | Result | diff --git a/DOCS/benchmarks/png-bmp.md b/DOCS/benchmarks/png-bmp.md index 2170d55b..0a53e5be 100644 --- a/DOCS/benchmarks/png-bmp.md +++ b/DOCS/benchmarks/png-bmp.md @@ -1,6 +1,6 @@ # PNG <-> BMP -**Latest** (2026-09-27, 0.24.0 release: every line PASSES) +**Latest** (2026-10-21, 1.0.0 release run: png to bmp 453.5 MB/s against 276.8 on the photo and 684.2 against 366.6 on the flat, bmp to png 130.2 against 110.0 and 2246.0 against 439.7, peak memory 4.9 to 10.6 MB against 66.5 to 165.9; every line PASSES, no `[stock]` rows on this machine) ## Purpose @@ -86,6 +86,35 @@ median wall clock of the whole process, peak resident memory from GNU ## Results +### 2026-10-21, 1.0.0 release + +| Target | Ours | Reference | Result | +|---|---|---|---| +| png -> bmp, photo (61.0 MB of pixels): throughput (MB/s of decoded pixels) | 453.5 | 276.8 (png + image) | PASS | +| png -> bmp, photo (32.8 MB on disk): throughput (MB/s of file bytes) [extra] | 243.7 | 148.8 (png + image) | n/a | +| png -> bmp, photo: peak memory (MB) | 4.9 | 66.5 (png + image) | PASS | +| bmp -> png, photo (61.0 MB in + 61.0 MB of pixels): throughput (MB/s of input plus pixels) | 130.2 | 110.0 (image + png) | PASS | +| bmp -> png, photo: peak memory (MB) | 10.6 | 165.9 (image + png) | PASS | +| bmp -> png, photo: output size (MB) [extra] | 31.6 | 32.2 (png) | n/a | +| bmp -> png, photo: the png crate's fast level, throughput and size [extra] | - | 439.1 MB/s, 32.8 MB (png fast) | n/a | +| png -> bmp, flat (61.0 MB of pixels): throughput (MB/s of decoded pixels) | 684.2 | 366.6 (png + image) | PASS | +| png -> bmp, flat (1.7 MB on disk): throughput (MB/s of file bytes) [extra] | 19.0 | 10.2 (png + image) | n/a | +| png -> bmp, flat: peak memory (MB) | 4.9 | 66.7 (png + image) | PASS | +| bmp -> png, flat (61.0 MB in + 61.0 MB of pixels): throughput (MB/s of input plus pixels) | 2246.0 | 439.7 (image + png) | PASS | +| bmp -> png, flat: peak memory (MB) | 5.5 | 127.9 (image + png) | PASS | +| bmp -> png, flat: output size (MB) [extra] | 0.1 | 0.4 (png) | n/a | +| bmp -> png, flat: the png crate's fast level, throughput and size [extra] | - | 592.7 MB/s, 1.7 MB (png fast) | n/a | + +commit: ec68950 (the `v1-wrapup` branch, before the version bump) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +Every line passes. This is the first block on this machine; the earlier +blocks are the Linux i7, so the rows do not compare like for like with +them. The machine was in use during this run (load average 7 to 12 on +10 cores: a browser, a VPN client, a remote-desktop host), so +wall-clock rows are noisier than earlier blocks'. `[stock]` rows are +absent: `bench/data/stock.*` is not on this machine. + ### 2026-09-27, 0.24.0 release | Target | Ours | Reference | Result | diff --git a/DOCS/benchmarks/png-heic.md b/DOCS/benchmarks/png-heic.md index f0f1ea3f..4c7186c0 100644 --- a/DOCS/benchmarks/png-heic.md +++ b/DOCS/benchmarks/png-heic.md @@ -1,12 +1,14 @@ # PNG -> HEIC -**Latest** (2026-10-12, `heic-gaps` branch: the default effort codes +**Latest** (2026-10-21, 1.0.0 release run: the default effort codes the 12 photographs of 4032 pixels faster than libheif's `heif-enc` (x265, -19.3 against 18.7 MB/s) and still in fewer bytes at equal quality (PSNR -of luma and of RGB, SSIMULACRA2; a tie at SSIM), `--effort fast` 21% -faster; Kodak's SSIMULACRA2 (+1.0%) is the one failing line; a fifth to -a quarter of x265's memory; 12 to 17% smaller than Apple's hardware -encoder (`sips`, context), which is far faster) +19.1 against 18.5 MB/s) and still in fewer bytes at equal quality (PSNR +of luma -1.4% and of RGB -1.8%, SSIMULACRA2 -0.5%; a tie at SSIM), +`--effort fast` 22.2 MB/s; the synthetic photo 39.3 against 13.6 MB/s +and the flat 194.5 against 58.7; Kodak's SSIMULACRA2 (+1.0%) is the one +failing line; a fifth to a quarter of x265's memory (71.4 against 325.2 +MB on the photo); 12 to 17% smaller than Apple's hardware encoder +(`sips`, context), which is far faster) ## Purpose @@ -108,6 +110,42 @@ by up to 10% at equal RGB PSNR. ## Results +### 2026-10-21, 1.0.0 release + +| Target | Ours | Reference (heif-enc) | Context (sips) | Result | +|---|---|---|---|---| +| png -> heic, jphoto (45.8 MB of pixels, quality 50): throughput (MB/s of pixels) | 39.3 | 13.6 | 126.5 | PASS | +| png -> heic, jphoto: peak memory (MB) | 71.4 | 325.2 | 204.5 | PASS | +| png -> heic, jflat (45.8 MB of pixels, quality 50): throughput (MB/s of pixels) | 194.5 | 58.7 | 183.6 | PASS | +| png -> heic, jflat: peak memory (MB) | 71.9 | 301.5 | 174.0 | PASS | +| png -> heic, hstock (103.5 MB of pixels, quality 50): throughput (MB/s of pixels) [stock] | 114.2 | 28.2 | 182.7 | PASS | +| png -> heic, hstock: peak memory (MB) [stock] | 143.3 | 633.9 | 390.1 | PASS | +| png -> heic, Kodak (24 photographs): BD-rate at equal PSNR of luma (negative: ours smaller) | -1.4% | 0% (itself) | ours -12.2% | PASS | +| png -> heic, Kodak (24 photographs): BD-rate at equal SSIM (negative: ours smaller) | -0.3% | 0% (itself) | ours -13.3% | PASS | +| png -> heic, Kodak (24 photographs): BD-rate at equal PSNR of RGB (negative: ours smaller) | -0.3% | 0% (itself) | ours -12.0% | PASS | +| png -> heic, Kodak (24 photographs): BD-rate at equal SSIMULACRA2 (negative: ours smaller) | +1.0% | 0% (itself) | ours -13.2% | FAIL | +| png -> heic, Kodak (24 photographs of 0.4 megapixels, quality 50): throughput (MB/s of pixels) | 8.3 | 6.5 | 11.7 | PASS | +| png -> heic, 12 photographs of 4032 pixels: BD-rate at equal PSNR of luma (negative: ours smaller) | -1.4% | 0% (itself) | ours -11.5% | PASS | +| png -> heic, 12 photographs of 4032 pixels: BD-rate at equal SSIM (negative: ours smaller) | -0.0% | 0% (itself) | ours -13.3% | PASS | +| png -> heic, 12 photographs of 4032 pixels: BD-rate at equal PSNR of RGB (negative: ours smaller) | -1.8% | 0% (itself) | ours -14.6% | PASS | +| png -> heic, 12 photographs of 4032 pixels: BD-rate at equal SSIMULACRA2 (negative: ours smaller) | -0.5% | 0% (itself) | ours -16.8% | PASS | +| png -> heic, 12 photographs of 4032 pixels (quality 50, --effort fast): throughput (MB/s of pixels) | 22.2 | 18.5 | n/a | PASS | +| png -> heic, 12 photographs of 4032 pixels (quality 50, --effort balanced): throughput (MB/s of pixels) | 19.1 | 18.5 | n/a | PASS | +| png -> heic, 12 photographs of 4032 pixels (quality 50, --effort max): throughput (MB/s of pixels) | 10.0 | 18.5 | n/a | context | + +commit: ec68950 (the `v1-wrapup` branch, before the version bump) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +The pair ran in full (23 minutes). The BD-rate rows are identical to the +previous block's, so the encoder writes the same files, and the one +FAIL is the same line, Kodak's SSIMULACRA2 (+1.0%). The default effort +still codes the 12 photographs faster than `heif-enc` (19.1 against +18.5 MB/s, 19.3 against 18.7 before), the thinnest speed margin here. +The machine was in use during this run (load average 7 to 12 on 10 +cores: a browser, a VPN client, a remote-desktop host), so wall-clock +rows are noisier than earlier blocks', and our encoder, which uses +every core, loses more to that load than `heif-enc` does. + ### 2026-10-12, the default effort faster than x265 on photographs | Target | Ours | Reference (heif-enc) | Context (sips) | Result | diff --git a/DOCS/benchmarks/png-pdf.md b/DOCS/benchmarks/png-pdf.md index 98e173c3..32d9e921 100644 --- a/DOCS/benchmarks/png-pdf.md +++ b/DOCS/benchmarks/png-pdf.md @@ -1,6 +1,6 @@ # PNG <-> PDF -**Latest** (2026-09-27, 0.24.0 release: every line PASSES) +**Latest** (2026-10-21, 1.0.0 release run: pdf to png 92.5 MB/s against 48.2 on the photo and 1012.1 against 484.0 on the flat, png to pdf 192.7 against 63.5 and 736.8 against 202.0, peak memory 5.9 to 53.4 MB against 61.2 to 273.6; every line PASSES, no `[stock]` rows on this machine) ## Purpose @@ -51,6 +51,35 @@ outputs are removed before each run. ## Results +### 2026-10-21, 1.0.0 release + +| Target | Ours | Reference | Result | +|---|---|---|---| +| pdf -> png, photo (45.8 MB of pixels, 39.7 MB on disk): throughput (MB/s of decoded pixels) | 92.5 | 48.2 (lopdf + png) | PASS | +| pdf -> png, photo: peak memory (MB) | 53.4 | 163.3 (lopdf + png) | PASS | +| pdf -> png, photo: output size (MB) [extra] | 25.0 | 27.1 (png) | n/a | +| png -> pdf, photo (29.0 MB in + 45.8 MB of pixels): throughput (MB/s of input plus pixels) | 192.7 | 63.5 (png + printpdf) | PASS | +| png -> pdf, photo: peak memory (MB) | 12.2 | 273.6 (png + printpdf) | PASS | +| png -> pdf, photo: output size (MB) [extra] | 25.0 | 37.9 (printpdf) | n/a | +| pdf -> png, flat (45.8 MB of pixels, 0.4 MB on disk): throughput (MB/s of decoded pixels) | 1012.1 | 484.0 (lopdf + png) | PASS | +| pdf -> png, flat: peak memory (MB) | 7.5 | 61.2 (lopdf + png) | PASS | +| pdf -> png, flat: output size (MB) [extra] | 0.1 | 0.3 (png) | n/a | +| png -> pdf, flat (1.6 MB in + 45.8 MB of pixels): throughput (MB/s of input plus pixels) | 736.8 | 202.0 (png + printpdf) | PASS | +| png -> pdf, flat: peak memory (MB) | 5.9 | 190.9 (png + printpdf) | PASS | +| png -> pdf, flat: output size (MB) [extra] | 0.1 | 0.2 (printpdf) | n/a | + +commit: ec68950 (the `v1-wrapup` branch, before the version bump) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +Every line passes. This is the first block on this machine; the earlier +blocks are the Linux i7, so the rows do not compare like for like with +them. The photo PDF is 39.7 MB here against 40.7 in earlier blocks, +since this machine's ImageMagick made it. The machine was in use during +this run (load average 7 to 12 on 10 cores: a browser, a VPN client, a +remote-desktop host), so wall-clock rows are noisier than earlier +blocks'. `[stock]` rows are absent: `bench/data/stock.*` is not on this +machine. + ### 2026-09-27, 0.24.0 release | Target | Ours | Reference | Result | diff --git a/DOCS/benchmarks/ppm-png.md b/DOCS/benchmarks/ppm-png.md index 82374535..1c826077 100644 --- a/DOCS/benchmarks/ppm-png.md +++ b/DOCS/benchmarks/ppm-png.md @@ -1,6 +1,6 @@ # Netpbm <-> PNG -**Latest** (2026-09-27, 0.24.0 release: 1 line FAILS, the photo encode tie (0.23.1 decision)) +**Latest** (2026-10-21, 1.0.0 release run: ppm to png 152.4 MB/s against 58.7 on the photo and 1021.7 against 479.8 on the flat, png to ppm 436.7 against 205.6 and 837.0 against 744.0, peak memory 4.1 to 9.6 MB against 51.3 to 128.2; every line PASSES on this machine, the photo encode tie (0.23.1 decision) included) ## Purpose @@ -44,6 +44,34 @@ repeated runs 10 to 20% behind on 48 MB outputs. ## Results +### 2026-10-21, 1.0.0 release + +| Target | Ours | Reference | Result | +|---|---|---|---| +| ppm -> png, photo (45.8 MB in): throughput (MB/s of input) | 152.4 | 58.7 (image + png) | PASS | +| ppm -> png, photo: peak memory (MB) | 9.6 | 128.2 (image + png) | PASS | +| ppm -> png, photo: output size (MB) [extra] | 25.0 | 27.1 (png) | n/a | +| png -> ppm, photo (29.0 MB in + 45.8 MB out): throughput (MB/s of input plus output) | 436.7 | 205.6 (png + image) | PASS | +| png -> ppm, photo: peak memory (MB) | 4.1 | 51.3 (png + image) | PASS | +| ppm -> png, flat (45.8 MB in): throughput (MB/s of input) | 1021.7 | 479.8 (image + png) | PASS | +| ppm -> png, flat: peak memory (MB) | 6.0 | 97.0 (image + png) | PASS | +| ppm -> png, flat: output size (MB) [extra] | 0.1 | 0.3 (png) | n/a | +| png -> ppm, flat (1.6 MB in + 45.8 MB out): throughput (MB/s of input plus output) | 837.0 | 744.0 (png + image) | PASS | +| png -> ppm, flat: peak memory (MB) | 4.1 | 51.4 (png + image) | PASS | + +commit: ec68950 (the `v1-wrapup` branch, before the version bump) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +Every line passes, the photo encode included (436.7 against 205.6 +MB/s), which failed as a tie in the previous block (853.3 against +860.7). This is the first block on this machine; the earlier blocks are +the Linux i7, so the rows do not compare like for like with them, and +the photo encode's margin here is not evidence the tie is gone there. +The machine was in use during this run (load average 7 to 12 on 10 +cores: a browser, a VPN client, a remote-desktop host), so wall-clock +rows are noisier than earlier blocks'. `[stock]` rows are absent: +`bench/data/stock.*` is not on this machine. + ### 2026-09-27, 0.24.0 release | Target | Ours | Reference | Result | diff --git a/DOCS/benchmarks/qoi-png.md b/DOCS/benchmarks/qoi-png.md index 417656ed..9c8c8656 100644 --- a/DOCS/benchmarks/qoi-png.md +++ b/DOCS/benchmarks/qoi-png.md @@ -1,6 +1,6 @@ # QOI <-> PNG -**Latest** (2026-09-27, 0.24.0 release: every line PASSES) +**Latest** (2026-10-21, 1.0.0 release run: qoi to png 111.2 MB/s against 51.8 on the photo and 977.4 against 493.6 on the flat, png to qoi 281.4 against 336.3 and 920.3 against 964.1, peak memory 4.1 to 10.7 MB against 51.9 to 131.3; 2 lines FAIL, png -> qoi on the photo and on the flat) ## Purpose @@ -48,6 +48,40 @@ wall clock of the whole process, peak resident memory from GNU `time`. ## Results +### 2026-10-21, 1.0.0 release + +| Target | Ours | Reference | Result | +|---|---|---|---| +| qoi -> png, photo (45.8 MB of pixels, 45.1 MB on disk): throughput (MB/s of decoded pixels) | 111.2 | 51.8 (qoi + png) | PASS | +| qoi -> png, photo: peak memory (MB) | 10.7 | 131.3 (qoi + png) | PASS | +| qoi -> png, photo: output size (MB) [extra] | 25.0 | 27.1 (png) | n/a | +| png -> qoi, photo (29.0 MB in + 45.8 MB of pixels): throughput (MB/s of input plus pixels) | 281.4 | 336.3 (png + qoi) | FAIL | +| png -> qoi, photo: peak memory (MB) | 4.1 | 96.5 (png + qoi) | PASS | +| qoi -> png, flat (45.8 MB of pixels, 0.7 MB on disk): throughput (MB/s of decoded pixels) | 977.4 | 493.6 (qoi + png) | PASS | +| qoi -> png, flat: peak memory (MB) | 5.0 | 51.9 (qoi + png) | PASS | +| qoi -> png, flat: output size (MB) [extra] | 0.1 | 0.3 (png) | n/a | +| png -> qoi, flat (1.6 MB in + 45.8 MB of pixels): throughput (MB/s of input plus pixels) | 920.3 | 964.1 (png + qoi) | FAIL | +| png -> qoi, flat: peak memory (MB) | 4.1 | 52.3 (png + qoi) | PASS | + +commit: ec68950 (the `v1-wrapup` branch, before the version bump) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +Both png -> qoi lines FAIL, and both passed in the previous block: the +photo at 281.4 against 336.3 MB/s (439.6 against 411.4 before), the +flat at 920.3 against 964.1 (1138.4 against 1070.9 before). A paired +A/B of our binary at ec68950 against the commit before it (93a37d4), 9 +alternated runs each, gave the same medians on the photo encode (0.22 s +both): the registry change did not cause it. The flat encode had no +A/B. This is the first block on +this machine; the earlier blocks are the Linux i7, so the rows do not +compare like for like with them. The machine was in use during this run +(load average 7 to 12 on 10 cores: a browser, a VPN client, a +remote-desktop host), so wall-clock rows are noisier than earlier +blocks'. The first attempt could not make the inputs (Pillow was no +longer installed); they were made with Pillow 13.0.0 in a fresh Python +3.13 environment and the pair rerun, and these are the rerun's rows. +`[stock]` rows are absent: `bench/data/stock.*` is not on this machine. + ### 2026-09-27, 0.24.0 release | Target | Ours | Reference | Result | @@ -152,3 +186,8 @@ encoder keeps its state in locals for each row, writes into a scratch slice sized once, hashes in one multiply (the qoi crate's form), and closes a final run once per row. Memory is the streaming win: a row, not the image. + +**2026-10-21, 1.0.0 release run.** The encoder's margin did not hold on +the M1 Max: png -> qoi failed on both shapes (photo 281.4 against +336.3 MB/s, flat 920.3 against 964.1), so the encode lines no longer +show a lead over the qoi crate. diff --git a/DOCS/benchmarks/rtf-docx.md b/DOCS/benchmarks/rtf-docx.md index 6e36cdf6..60d274bb 100644 --- a/DOCS/benchmarks/rtf-docx.md +++ b/DOCS/benchmarks/rtf-docx.md @@ -1,6 +1,6 @@ # RTF <-> Word -**Latest** (2026-10-06, 0.26.0 with RTF: docx -> rtf at 91.8 MB/s on the styled shape and 130.6 on prose, rtf -> docx at 184.7 and 251.4 MB/s of input plus uncompressed output, every line PASSES against pandoc at 25 to 60 times pandoc's speed and a twentieth of its memory) +**Latest** (2026-10-21, the 1.0.0 release run: docx -> rtf at 90.4 MB/s on the styled shape and 133.1 on prose, rtf -> docx at 159.5 and 245.4 MB/s of input plus uncompressed output, every line PASSES against pandoc at 36 to 83 times pandoc's speed and a thirteenth to a twenty-fifth of its memory) ## Purpose @@ -55,6 +55,32 @@ is the RTF our writer makes, which pandoc reads in full. ## Results +### 2026-10-21, the `v1-wrapup` branch, the 1.0.0 release run + +| Target | Ours | Reference | Result | +|---|---|---|---| +| docx -> rtf, styled (6.0 MB uncompressed): throughput (MB/s of uncompressed input) | 90.4 | 1.5 (pandoc) | PASS | +| docx -> rtf, styled: peak memory (MB) | 25.2 | 566.8 (pandoc) | PASS | +| rtf -> docx, styled (4.6 MB in + 9.7 MB out): throughput (MB/s of input plus uncompressed output) | 159.5 | 4.4 (pandoc) | PASS | +| rtf -> docx, styled: throughput (MB/s of input) [extra] | 51.5 | 1.8 (pandoc) | n/a | +| rtf -> docx, styled: peak memory (MB) | 22.8 | 383.7 (pandoc) | PASS | +| docx -> rtf, prose (7.5 MB uncompressed): throughput (MB/s of uncompressed input) | 133.1 | 1.6 (pandoc) | PASS | +| docx -> rtf, prose: peak memory (MB) | 30.7 | 775.8 (pandoc) | PASS | +| rtf -> docx, prose (7.7 MB in + 11.9 MB out): throughput (MB/s of input plus uncompressed output) | 245.4 | 4.9 (pandoc) | PASS | +| rtf -> docx, prose: throughput (MB/s of input) [extra] | 96.2 | 2.5 (pandoc) | n/a | +| rtf -> docx, prose: peak memory (MB) | 32.0 | 435.4 (pandoc) | PASS | + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +The machine was in use during this run (load average 7 to 12 on 10 +cores: a browser, a VPN client, a remote-desktop host), so the +wall-clock rows are noisier than the previous block's. Every line still +passes. `rtf -> docx` on the styled shape read 159.5 MB/s against 184.7 +on 2026-10-06; the other rates are within 3%. Peak memory on +`rtf -> docx` prose rose from 27.2 to 32.0 MB, still under a +thirteenth of pandoc's. + ### 2026-10-06, 0.26.0 with RTF | Target | Ours | Reference | Result | @@ -80,3 +106,7 @@ shapes, inside the 64 MB the Pages and Word pairs keep to. The writer is the slower side: `docx -> rtf` escapes every character outside ASCII and restates each run's formatting, which is what RTF readers expect. Re-run on the Linux bench machine for the record beside the other pairs. + +2026-10-21, the 1.0.0 release run: `rtf -> docx` on prose peaked at +32.0 MB, so the paths no longer all hold under 31 MB; every line still +passes, inside the 64 MB. diff --git a/DOCS/benchmarks/text-markdown.md b/DOCS/benchmarks/text-markdown.md index de0a563d..35f88bd7 100644 --- a/DOCS/benchmarks/text-markdown.md +++ b/DOCS/benchmarks/text-markdown.md @@ -1,6 +1,6 @@ # Text -> Markdown -**Latest** (2026-09-24, first release of the text reader: text -> markdown 256.0 MB/s of input on the dense shape and 442.1 on prose, at 16.7 and 58.8 MB peak; every line PASSES) +**Latest** (2026-10-21, the 1.0.0 release run: text -> markdown 282.1 MB/s of input on the dense shape and 612.5 on prose, at 16.2 and 58.5 MB peak; every line PASSES) ## Purpose @@ -49,6 +49,25 @@ text would. ## Results +### 2026-10-21, the `v1-wrapup` branch, the 1.0.0 release run + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +| Target | Ours | Reference | Result | +|---|---|---|---| +| text -> markdown, dense (13.8 MB): throughput (MB/s of input) | 282.1 | goal: 50 | PASS | +| text -> markdown, dense: peak memory (MB) | 16.2 | goal: <= 64.0 | PASS | +| text -> markdown, prose (56.1 MB): throughput (MB/s of input) | 612.5 | goal: 50 | PASS | +| text -> markdown, prose: peak memory (MB) | 58.5 | goal: <= 64.0 | PASS | + +The machine was in use during this run (load average 7 to 12 on 10 +cores: a browser, a VPN client, a remote-desktop host), so the +wall-clock rows are noisier than they would be on an idle machine. This +block is from the M1 Max laptop and the previous one from the Linux +bench machine, so the rates are not a like-for-like comparison. Every +line still passes; prose memory is still the input held whole. + ### 2026-09-24, first release of the text reader commit: 34a9e90 (the reader's working tree, before its commit) @@ -66,3 +85,6 @@ machine: Linux 7.1.5-ogc5.1.fc44.x86_64 x86_64, 24 cpus Five to nine times the goal. Memory on prose is the input held whole (56 MB) since the writer borrows every line from it; a streaming converter would read line by line, which nothing here needs yet. + +2026-10-21, the 1.0.0 release run on the M1 Max laptop: 282.1 and +612.5 MB/s, five to twelve times the goal. diff --git a/DOCS/benchmarks/tga-png.md b/DOCS/benchmarks/tga-png.md index f6d0bd99..b3c20da1 100644 --- a/DOCS/benchmarks/tga-png.md +++ b/DOCS/benchmarks/tga-png.md @@ -1,6 +1,6 @@ # TGA <-> PNG -**Latest** (2026-09-27, 0.24.0 release: every line PASSES) +**Latest** (2026-10-21, 1.0.0 release run: tga to png 124.6 MB/s against 56.2 on the photo and 439.2 against 283.4 on the flat, png to tga 266.9 against 228.9 and 275.3 against 231.2, peak memory 4.1 to 57.3 MB against 52.3 to 128.3; every line PASSES, no `[stock]` rows on this machine) ## Purpose @@ -40,6 +40,34 @@ outputs are removed before each run. ## Results +### 2026-10-21, 1.0.0 release + +| Target | Ours | Reference | Result | +|---|---|---|---| +| tga -> png, photo (45.8 MB of pixels, 45.9 MB on disk): throughput (MB/s of decoded pixels) | 124.6 | 56.2 (image + png) | PASS | +| tga -> png, photo: peak memory (MB) | 57.3 | 128.3 (image + png) | PASS | +| tga -> png, photo: output size (MB) [extra] | 25.0 | 27.1 (png) | n/a | +| png -> tga, photo (29.0 MB in + 45.8 MB of pixels): throughput (MB/s of input plus pixels) | 266.9 | 228.9 (png + image) | PASS | +| png -> tga, photo: peak memory (MB) | 4.1 | 96.9 (png + image) | PASS | +| tga -> png, flat (45.8 MB of pixels, 1.0 MB on disk): throughput (MB/s of decoded pixels) | 439.2 | 283.4 (image + png) | PASS | +| tga -> png, flat: peak memory (MB) | 7.5 | 52.3 (image + png) | PASS | +| tga -> png, flat: output size (MB) [extra] | 0.1 | 0.3 (png) | n/a | +| png -> tga, flat (1.6 MB in + 45.8 MB of pixels): throughput (MB/s of input plus pixels) | 275.3 | 231.2 (png + image) | PASS | +| png -> tga, flat: peak memory (MB) | 4.2 | 97.3 (png + image) | PASS | + +commit: ec68950 (the `v1-wrapup` branch, before the version bump) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +Every line passes. This is the first block on this machine; the earlier +blocks are the Linux i7, so the rows do not compare like for like with +them. The machine was in use during this run (load average 7 to 12 on +10 cores: a browser, a VPN client, a remote-desktop host), so +wall-clock rows are noisier than earlier blocks'. The first attempt +could not make the inputs (Pillow was no longer installed); they were +made with Pillow 13.0.0 in a fresh Python 3.13 environment and the pair +rerun, and these are the rerun's rows. `[stock]` rows are absent: +`bench/data/stock.*` is not on this machine. + ### 2026-09-27, 0.24.0 release | Target | Ours | Reference | Result | diff --git a/DOCS/benchmarks/tiff-png.md b/DOCS/benchmarks/tiff-png.md index f581a4f4..42b20b3d 100644 --- a/DOCS/benchmarks/tiff-png.md +++ b/DOCS/benchmarks/tiff-png.md @@ -1,6 +1,6 @@ # TIFF <-> PNG -**Latest** (2026-09-27, 0.24.0 release: every line PASSES) +**Latest** (2026-10-21, 1.0.0 release run: tiff to png 73.7 MB/s against 42.4 on the photo and 217.0 against 165.3 on the flat, png to tiff 211.1 against 86.4 and 568.5 against 472.8, peak memory 7.0 to 51.5 MB against 60.9 to 161.7; every line PASSES, no `[stock]` rows on this machine) ## Purpose @@ -43,6 +43,31 @@ outputs are removed before each run. ## Results +### 2026-10-21, 1.0.0 release + +| Target | Ours | Reference | Result | +|---|---|---|---| +| tiff -> png, photo (45.8 MB of pixels, 32.8 MB on disk): throughput (MB/s of decoded pixels) | 73.7 | 42.4 (image + png) | PASS | +| tiff -> png, photo: peak memory (MB) | 51.5 | 161.7 (image + png) | PASS | +| tiff -> png, photo: output size (MB) [extra] | 25.0 | 27.1 (png) | n/a | +| png -> tiff, photo (29.0 MB in + 45.8 MB of pixels): throughput (MB/s of input plus pixels) | 211.1 | 86.4 (png + image) | PASS | +| png -> tiff, photo: peak memory (MB) | 39.4 | 63.5 (png + image) | PASS | +| tiff -> png, flat (45.8 MB of pixels, 0.7 MB on disk): throughput (MB/s of decoded pixels) | 217.0 | 165.3 (image + png) | PASS | +| tiff -> png, flat: peak memory (MB) | 15.6 | 98.8 (image + png) | PASS | +| tiff -> png, flat: output size (MB) [extra] | 0.1 | 0.3 (png) | n/a | +| png -> tiff, flat (1.6 MB in + 45.8 MB of pixels): throughput (MB/s of input plus pixels) | 568.5 | 472.8 (png + image) | PASS | +| png -> tiff, flat: peak memory (MB) | 7.0 | 60.9 (png + image) | PASS | + +commit: ec68950 (the `v1-wrapup` branch, before the version bump) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +Every line passes. This is the first block on this machine; the earlier +blocks are the Linux i7, so the rows do not compare like for like with +them. The machine was in use during this run (load average 7 to 12 on +10 cores: a browser, a VPN client, a remote-desktop host), so +wall-clock rows are noisier than earlier blocks'. `[stock]` rows are +absent: `bench/data/stock.*` is not on this machine. + ### 2026-09-27, 0.24.0 release | Target | Ours | Reference | Result | diff --git a/DOCS/benchmarks/toml-json.md b/DOCS/benchmarks/toml-json.md index 3d7c2cce..476a5203 100644 --- a/DOCS/benchmarks/toml-json.md +++ b/DOCS/benchmarks/toml-json.md @@ -1,6 +1,6 @@ # TOML <-> JSON -**Latest** (2026-09-25, arena tree: toml -> json 148.2 MB/s of input on the dense shape and 323.5 on prose, at 237 and 421 MB peak; json -> toml 156.7 and 309.2 MB/s at 172 and 212 MB peak; every line PASSES against the `toml` crate with `serde_json`, at 1.9 to 3.7 times its throughput and 12 to 47 percent of its memory) +**Latest** (2026-10-21, 1.0.0 release run: toml -> json 88.0 MB/s of input on the dense shape and 200.1 on prose, at 354 and 524 MB peak; json -> toml 107.0 and 199.6 MB/s at 197 and 230 MB peak; every line PASSES against the `toml` crate with `serde_json`, at 1.6 to 2.4 times its throughput and 14 to 50 percent of its memory) ## Purpose @@ -67,6 +67,35 @@ throughput in MB/s over the input file's bytes. Rows and units follow ## Results +### 2026-10-21, `v1-wrapup` branch, the 1.0.0 release run + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +| Target | Ours | Reference | Result | +|---|---|---|---| +| toml -> json, dense (59.9 MB): throughput (MB/s of input) | 88.0 | 36.3 (toml + serde_json) | PASS | +| toml -> json, dense: peak memory (MB) | 354.2 | 2543.0 (toml + serde_json) | PASS | +| toml -> json, prose (182.0 MB): throughput (MB/s of input) | 200.1 | 126.9 (toml + serde_json) | PASS | +| toml -> json, prose: peak memory (MB) | 523.7 | 1039.4 (toml + serde_json) | PASS | +| json -> toml, dense (56.2 MB): throughput (MB/s of input) | 107.0 | 48.7 (serde_json + toml) | PASS | +| json -> toml, dense: peak memory (MB) | 196.9 | 788.2 (serde_json + toml) | PASS | +| json -> toml, prose (178.9 MB): throughput (MB/s of input) | 199.6 | 94.7 (serde_json + toml) | PASS | +| json -> toml, prose: peak memory (MB) | 230.3 | 1040.8 (serde_json + toml) | PASS | + +Every line passes, at 1.6 to 2.4 times the reference's throughput and +14 to 50 percent of its memory. This run is on a different machine from +the previous blocks (an M1 Max, not the Linux i7-13700K), so its absolute +MB/s do not compare with theirs. The machine was in use during the run +(load average 7 to 12 on 10 cores: a browser, a VPN client, a +remote-desktop host), so the wall-clock rows are noisier than earlier +blocks'. Our peak memory is higher than in the previous block on every +row: toml -> json 236.5 -> 354.2 MB dense and 420.7 -> 523.7 MB prose, +json -> toml 172.4 -> 196.9 MB dense and 211.5 -> 230.3 MB prose. The +reference's memory moved as well (1926.9 -> 2543.0 MB on the dense +toml -> json row); this run does not say how much of the rise is the +machine's allocator and how much is a change since. + ### 2026-09-25, the value hub as an arena tree commit: bd7a134 (the working tree of this commit) @@ -138,3 +167,7 @@ the 421). A windowed reader like XML's would take the input out of the peak if a large TOML case ever matters. The numbers do not justify claiming anything about deeply nested documents or pathological key sets, which the inputs do not have. + +On 2026-10-21, on an M1 Max, the dense toml -> json row peaked at 354 MB +for 60 MB of input, about six bytes per input byte rather than the +three-plus-input above, and prose at 524 MB rather than 421. diff --git a/DOCS/benchmarks/tsv-json.md b/DOCS/benchmarks/tsv-json.md index 0e6795ee..377d8924 100644 --- a/DOCS/benchmarks/tsv-json.md +++ b/DOCS/benchmarks/tsv-json.md @@ -1,6 +1,6 @@ # TSV <-> JSON -**Latest** (2026-09-25, first release of TSV: tsv -> json 325.1 MB/s of input at 2.7 MB peak; json -> tsv 191.8 MB/s at 2.6 MB peak; every line PASSES against the csv crate with serde_json, at 1.5 to 2 times its throughput) +**Latest** (2026-10-21, 1.0.0 release run: tsv -> json 174.5 MB/s of input at 2.7 MB peak; json -> tsv 124.5 MB/s at 2.8 MB peak; every line PASSES against the csv crate with serde_json, at 1.5 to 2.1 times its throughput) ## Purpose @@ -52,6 +52,26 @@ throughput in MB/s over the input file's bytes. Rows and units follow ## Results +### 2026-10-21, `v1-wrapup` branch, the 1.0.0 release run + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +| Target | Ours | Reference | Result | +|---|---|---|---| +| tsv -> json, 10M rows (594.3 MB): throughput (MB/s of input) | 174.5 | 114.2 (csv + serde_json) | PASS | +| tsv -> json, 10M rows: peak memory (MB) | 2.7 | 4.7 (csv + serde_json) | PASS | +| json -> tsv, 10M rows (1110.6 MB): throughput (MB/s of input) | 124.5 | 59.1 (serde_json + csv) | PASS | +| json -> tsv, 10M rows: peak memory (MB) | 2.8 | 8012.1 (serde_json + csv) | PASS | + +Every line passes, at 1.5 and 2.1 times the reference's throughput. This +run is on a different machine from the previous block (an M1 Max, not +the Linux i7-13700K), so its absolute MB/s do not compare with that +block's; the ratios against the reference in the same session do. The +machine was in use during the run (load average 7 to 12 on 10 cores: a +browser, a VPN client, a remote-desktop host), so the wall-clock rows +are noisier than earlier blocks'. + ### 2026-09-25, first release of TSV commit: 2f7e30d @@ -69,3 +89,9 @@ machine: Linux 7.1.5-ogc5.1.fc44.x86_64 x86_64, 24 cpus, 13th Gen Intel(R) Core( The separator parameter cost nothing: the rows match `csv-json.md`'s within noise, and the reader's tab scanner is the same word-at-a-time scan with a different byte. Nothing here is close to a limit. + +On 2026-10-21 the rows did not match `csv-json.md`'s: tsv -> json ran at +174.5 MB/s against csv -> json's 94.8, and json -> tsv at 124.5 against +json -> csv's 88.5, on the same loaded machine in separate sessions. The +references fell with them (114.2 here, 37.1 in the csv-json session), so +the two runs saw different load; this run does not test the separator. diff --git a/DOCS/benchmarks/webp-png.md b/DOCS/benchmarks/webp-png.md index 702bb07f..e20c76f1 100644 --- a/DOCS/benchmarks/webp-png.md +++ b/DOCS/benchmarks/webp-png.md @@ -1,6 +1,6 @@ # WebP <-> PNG -**Latest** (2026-09-27, 0.24.0 release: 1 line FAILS, the lossless photo encode (0.21.0 decision)) +**Latest** (2026-10-21, 1.0.0 release run: lossless webp to png 67.9 MB/s against 39.4 on the photo and 612.9 against 431.5 on the flat, lossy 69.1 against 56.1 and 227.0 against 210.4, png to lossless webp 146.4 against 302.9 and 534.8 against 662.2, peak memory 6.9 to 109.2 MB against 77.2 to 190.8; 2 lines FAIL, the lossless photo encode (0.21.0 decision) and, new in this run, the lossless flat encode) ## Purpose @@ -59,6 +59,44 @@ median wall clock of the whole process, peak resident memory from GNU ## Results +### 2026-10-21, 1.0.0 release + +| Target | Ours | Reference | Result | +|---|---|---|---| +| webp (lossless) -> png, photo (45.8 MB of pixels, 25.8 MB on disk): throughput (MB/s of decoded pixels) | 67.9 | 39.4 (image + png) | PASS | +| webp (lossless) -> png, photo: peak memory (MB) | 109.2 | 190.8 (image + png) | PASS | +| webp (lossy) -> png, photo (45.8 MB of pixels, 0.3 MB on disk): throughput (MB/s of decoded pixels) | 69.1 | 56.1 (image + png) | PASS | +| webp (lossy) -> png, photo: peak memory (MB) | 8.5 | 84.7 (image + png) | PASS | +| png -> webp (lossless), photo (29.0 MB in + 45.8 MB of pixels): throughput (MB/s of input plus pixels) | 146.4 | 302.9 (png + image) | FAIL | +| png -> webp (lossless), photo: peak memory (MB) | 97.6 | 148.9 (png + image) | PASS | +| png -> webp (lossless), photo: output size (MB) [extra] | 27.4 | 28.5 (image); 25.8 (libwebp, default effort) | n/a | +| webp (lossless) -> png, flat (45.8 MB of pixels, 0.0 MB on disk): throughput (MB/s of decoded pixels) | 612.9 | 431.5 (image + png) | PASS | +| webp (lossless) -> png, flat: peak memory (MB) | 96.6 | 112.8 (image + png) | PASS | +| webp (lossy) -> png, flat (45.8 MB of pixels, 0.1 MB on disk): throughput (MB/s of decoded pixels) | 227.0 | 210.4 (image + png) | PASS | +| webp (lossy) -> png, flat: peak memory (MB) | 6.9 | 77.2 (image + png) | PASS | +| png -> webp (lossless), flat (1.6 MB in + 45.8 MB of pixels): throughput (MB/s of input plus pixels) | 534.8 | 662.2 (png + image) | FAIL | +| png -> webp (lossless), flat: peak memory (MB) | 51.7 | 112.7 (png + image) | PASS | +| png -> webp (lossless), flat: output size (MB) [extra] | 0.0 | 0.0 (image); 0.0 (libwebp, default effort) | n/a | + +commit: ec68950 (the `v1-wrapup` branch, before the version bump) +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +Two lines FAIL. The lossless photo encode is the known miss (146.4 +against 302.9 MB/s; 253.1 against 435.5 in the previous block). The +lossless flat encode is new: 534.8 against 662.2 MB/s, where it passed +in the previous block at 772.5 against 612.2. A paired A/B of our +binary at ec68950 against the commit before it (93a37d4), 9 alternated +runs each, gave the same medians on the flat encode (0.08 s both): the +registry change did not cause it. This is the first block on this +machine; the earlier blocks are the Linux i7, so the rows do not +compare like for like with them. The machine was in use during this run +(load average 7 to 12 on 10 cores: a browser, a VPN client, a +remote-desktop host), so wall-clock rows are noisier than earlier +blocks'. The first attempt could not make the inputs (Pillow was no +longer installed); they were made with Pillow 13.0.0 in a fresh Python +3.13 environment and the pair rerun, and these are the rerun's rows. +Output sizes are unchanged. + ### 2026-09-27, 0.24.0 release | Target | Ours | Reference | Result | @@ -267,3 +305,7 @@ up chooses predictors by an entropy estimate on every row (1 to 1.5% smaller than the default, 40% slower). No level wins the photo encode line while the color cache is on, and the cache is what makes real photos 8 to 37% smaller; the line stays a known miss by design. + +**2026-10-21, 1.0.0 release run.** The flat encode no longer passes on +the M1 Max: 534.8 against 662.2 MB/s, where the previous block had it +at 772.5 against 612.2. diff --git a/DOCS/benchmarks/xlsx-csv.md b/DOCS/benchmarks/xlsx-csv.md index 33ba929d..cdb79cd1 100644 --- a/DOCS/benchmarks/xlsx-csv.md +++ b/DOCS/benchmarks/xlsx-csv.md @@ -1,6 +1,6 @@ # XLSX <-> CSV -**Latest** (2026-09-27, 0.24.0 release: every line PASSES) +**Latest** (2026-10-21, 1.0.0 release run: xlsx -> csv 136.6 MB/s of uncompressed input at 469 MB peak, against calamine's 86.7 at 756 MB; csv -> xlsx 415.6 MB/s of input plus uncompressed output at 6.7 MB, against rust_xlsxwriter's 96.5 at 1313 MB; every line PASSES) ## Purpose @@ -65,6 +65,29 @@ median wall clock of the whole process, peak resident memory from GNU ## Results +### 2026-10-21, `v1-wrapup` branch, the 1.0.0 release run + +| Target | Ours | Reference | Result | +|---|---|---|---| +| xlsx -> csv, 1000000 rows (326.4 MB uncompressed): throughput (MB/s of uncompressed input) | 136.6 | 86.7 (calamine + csv) | PASS | +| xlsx -> csv, 1000000 rows (39.5 MB on disk): throughput (MB/s of file bytes) [extra] | 16.5 | 10.5 (calamine + csv) | n/a | +| xlsx -> csv, 1000000 rows: peak memory (MB) | 468.8 | 756.1 (calamine + csv) | PASS | +| csv -> xlsx, 1000000 rows (56.8 MB in + 387.7 MB out): throughput (MB/s of input plus uncompressed output) | 415.6 | 96.5 (csv + rust_xlsxwriter) | PASS | +| csv -> xlsx, 1000000 rows: peak memory (MB) | 6.7 | 1313.3 (csv + rust_xlsxwriter) | PASS | + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +Every line passes, at 1.6 times calamine's throughput reading and 4.3 +times rust_xlsxwriter's writing. This run is on a different machine from +the previous blocks (an M1 Max, not the Linux i7-13700K), so its absolute +MB/s and memory do not compare with theirs. The machine was in use during +the run (load average 7 to 12 on 10 cores: a browser, a VPN client, a +remote-desktop host), so the wall-clock rows are noisier than earlier +blocks'. Peak memory is higher than in the previous block on both rows +(xlsx -> csv 415.9 -> 468.8 MB, csv -> xlsx 4.6 -> 6.7 MB), and the +reference's read rose with it (593.9 -> 756.1 MB). + ### 2026-09-27, 0.24.0 release | Target | Ours | Reference | Result | @@ -131,3 +154,6 @@ is the lever, recorded in `STATE.md`, and would make the read memory constant the way the CSV and JSON readers are. The numbers do not justify claims about workbooks heavy in shared strings, styles, or merged cells. + +On 2026-10-21, on an M1 Max, the read peaked at 469 MB rather than 415, +and the writer at 6.7 MB rather than under 5. diff --git a/DOCS/benchmarks/xml-json.md b/DOCS/benchmarks/xml-json.md index 60312091..7910e50c 100644 --- a/DOCS/benchmarks/xml-json.md +++ b/DOCS/benchmarks/xml-json.md @@ -1,6 +1,6 @@ # XML <-> JSON -**Latest** (2026-09-25, arena tree: xml -> json 145.4 MB/s of input on the dense shape and 443.3 on prose, at 171 and 223 MB peak; json -> xml 140.4 and 373.9 MB/s at 168 and 222 MB peak; every line PASSES against quick-xml with serde_json, at 1.3 to 2 times its throughput and 27 to 52 percent of its memory) +**Latest** (2026-10-21, 1.0.0 release run: xml -> json 88.0 MB/s of input on the dense shape and 239.4 on prose, at 260 and 251 MB peak; json -> xml 105.4 and 169.4 MB/s at 213 and 242 MB peak; every line PASSES against quick-xml with serde_json, at 1.06 to 1.7 times its throughput and 39 to 60 percent of its memory) ## Purpose @@ -66,6 +66,36 @@ throughput in MB/s over the input file's bytes. Rows and units follow ## Results +### 2026-10-21, `v1-wrapup` branch, the 1.0.0 release run + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +| Target | Ours | Reference | Result | +|---|---|---|---| +| xml -> json, dense (80.0 MB): throughput (MB/s of input) | 88.0 | 66.0 (quick-xml + serde_json) | PASS | +| xml -> json, dense: peak memory (MB) | 259.5 | 523.7 (quick-xml + serde_json) | PASS | +| xml -> json, prose (194.4 MB): throughput (MB/s of input) | 239.4 | 225.3 (quick-xml + serde_json) | PASS | +| xml -> json, prose: peak memory (MB) | 251.2 | 424.2 (quick-xml + serde_json) | PASS | +| json -> xml, dense (58.2 MB): throughput (MB/s of input) | 105.4 | 60.6 (serde_json + quick-xml) | PASS | +| json -> xml, dense: peak memory (MB) | 212.6 | 542.1 (serde_json + quick-xml) | PASS | +| json -> xml, prose (180.6 MB): throughput (MB/s of input) | 169.4 | 137.1 (serde_json + quick-xml) | PASS | +| json -> xml, prose: peak memory (MB) | 242.2 | 406.6 (serde_json + quick-xml) | PASS | + +Every line passes, at 1.06 to 1.7 times the reference's throughput and +39 to 60 percent of its memory. The narrowest line is xml -> json prose, +239.4 against 225.3 MB/s, 6 percent ahead where the previous block was +34 percent ahead. This run is on a different machine from the previous +blocks (an M1 Max, not the Linux i7-13700K), so its absolute MB/s do not +compare with theirs. The machine was in use during the run (load +average 7 to 12 on 10 cores: a browser, a VPN client, a remote-desktop +host), so the wall-clock rows are noisier than earlier blocks'. Our peak +memory is higher than in the previous block on every row: xml -> json +171.4 -> 259.5 MB dense and 223.3 -> 251.2 MB prose, json -> xml +168.4 -> 212.6 MB dense and 221.5 -> 242.2 MB prose, while the +reference's fell slightly; this run does not say how much of the rise is +the machine's allocator and how much is a change since. + ### 2026-09-25, the value hub as an arena tree commit: bd7a134 (the working tree of this commit) @@ -123,3 +153,8 @@ string per attribute and per text node before escaping; escaping straight into the chunked sink is the next small lever. The numbers do not justify claims about namespace-heavy, mixed-content, or deeply nested documents. + +On 2026-10-21, on an M1 Max under load, xml -> json read at 1.06 to +1.3 times the reference rather than 1.3 to 1.6 (prose 239 against +225 MB/s), and the dense row peaked at 260 MB for 80 MB of input, about +three bytes per input byte rather than two. diff --git a/DOCS/benchmarks/yaml-json.md b/DOCS/benchmarks/yaml-json.md index ee07139c..7513e402 100644 --- a/DOCS/benchmarks/yaml-json.md +++ b/DOCS/benchmarks/yaml-json.md @@ -1,6 +1,6 @@ # YAML <-> JSON -**Latest** (2026-09-25, arena tree: yaml -> json 128.9 MB/s of input on the dense shape and 322.6 on prose, at 235 and 427 MB peak; json -> yaml 98.5 and 356.0 MB/s at 172 and 211 MB peak; every line PASSES against serde_yaml with serde_json, at 2.3 to 5 times its throughput and 11 to 49 percent of its memory) +**Latest** (2026-10-21, 1.0.0 release run: yaml -> json 87.8 MB/s of input on the dense shape and 214.8 on prose, at 359 and 536 MB peak; json -> yaml 70.7 and 212.9 MB/s at 198 and 234 MB peak; every line PASSES against serde_yaml with serde_json, at 2.1 to 4.4 times its throughput and 15 to 57 percent of its memory) ## Purpose @@ -66,6 +66,35 @@ throughput in MB/s over the input file's bytes. Rows and units follow ## Results +### 2026-10-21, `v1-wrapup` branch, the 1.0.0 release run + +commit: ec68950 +machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +| Target | Ours | Reference | Result | +|---|---|---|---| +| yaml -> json, dense (62.3 MB): throughput (MB/s of input) | 87.8 | 19.8 (serde_yaml + serde_json) | PASS | +| yaml -> json, dense: peak memory (MB) | 359.1 | 2317.1 (serde_yaml + serde_json) | PASS | +| yaml -> json, prose (190.1 MB): throughput (MB/s of input) | 214.8 | 71.4 (serde_yaml + serde_json) | PASS | +| yaml -> json, prose: peak memory (MB) | 535.7 | 1081.2 (serde_yaml + serde_json) | PASS | +| json -> yaml, dense (56.2 MB): throughput (MB/s of input) | 70.7 | 34.1 (serde_json + serde_yaml) | PASS | +| json -> yaml, dense: peak memory (MB) | 197.5 | 523.7 (serde_json + serde_yaml) | PASS | +| json -> yaml, prose (180.0 MB): throughput (MB/s of input) | 212.9 | 65.5 (serde_json + serde_yaml) | PASS | +| json -> yaml, prose: peak memory (MB) | 234.0 | 409.4 (serde_json + serde_yaml) | PASS | + +Every line passes, at 2.1 to 4.4 times the reference's throughput and +15 to 57 percent of its memory. This run is on a different machine from +the previous blocks (an M1 Max, not the Linux i7-13700K), so its absolute +MB/s do not compare with theirs. The machine was in use during the run +(load average 7 to 12 on 10 cores: a browser, a VPN client, a +remote-desktop host), so the wall-clock rows are noisier than earlier +blocks'. Our peak memory is higher than in the previous block on every +row: yaml -> json 234.7 -> 359.1 MB dense and 427.2 -> 535.7 MB prose, +json -> yaml 172.0 -> 197.5 MB dense and 210.6 -> 234.0 MB prose. The +reference's dense yaml -> json row rose too (2161.2 -> 2317.1 MB); this +run does not say how much of the rise is the machine's allocator and how +much is a change since. + ### 2026-09-25, the value hub as an arena tree commit: bd7a134 (the working tree of this commit) @@ -118,3 +147,9 @@ yaml -> json rows hold the input text beside the tree (62 and 190 MB of their peaks); a windowed reader like XML's is the lever if a large YAML case ever matters. The numbers do not justify claims about alias-heavy or deeply nested documents. + +On 2026-10-21, on an M1 Max under load, the reader was 4.4 times the +reference on the dense shape rather than five, and the dense yaml -> json +row peaked at 359 MB for 62 MB of input, about six bytes per input byte +rather than three-plus-input; json -> yaml dense (70.7 MB/s) is still the +lowest row. diff --git a/DOCS/benchmarks/z64-v64.md b/DOCS/benchmarks/z64-v64.md index 60841fe9..2b23f95e 100644 --- a/DOCS/benchmarks/z64-v64.md +++ b/DOCS/benchmarks/z64-v64.md @@ -1,11 +1,12 @@ # N64 ROM byte orders (z64, v64, n64) -**Latest** (2026-10-20, branch `n64`): every throughput line PASSES, at -1,450 to 2,090 MB/s against `dd conv=swab`'s 1,366 to 1,551, -`n64romconvert`'s 658 to 734, and ucon64's 460 to 588. The four memory -lines judged against ucon64 and `dd` (z64 <-> v64) FAIL by 0.1 to 0.6 MB: -2.6 MB against 2.0 to 2.1 and 2.3 to 2.5, from the binary's startup floor -(Threats to validity). Memory passes against `n64romconvert` everywhere. +**Latest** (2026-10-21, the 1.0.0 release run): every throughput line +PASSES, at 1,234 to 1,542 MB/s against `dd conv=swab`'s 827 to 1,188, +`n64romconvert`'s 493 to 659, and ucon64's 387 to 469, on a loaded +machine. Memory fell from 2.6 to 2.4 MB with the registry held as data, +and the four lines judged against ucon64 (2.0 to 2.1) and `dd` (2.3) +still FAIL by 0.1 to 0.4 MB. Memory passes against `n64romconvert` +everywhere. ## Purpose @@ -55,6 +56,37 @@ median wall clock of the whole process, peak resident memory from GNU ## Results +2026-10-21, `v1-wrapup` branch, the 1.0.0 release run: + +| Target | Ours | n64romconvert | ucon64 | dd conv=swab | Result | +|---|---|---|---|---|---| +| z64 -> v64, generated (64.0 MB): throughput (MB/s) | 1259.1 | 628.2 | 426.8 | 1187.9 | PASS | +| z64 -> v64, generated: peak memory (MB) | 2.4 | 4.2 | 2.1 | 2.3 | FAIL | +| v64 -> z64, generated (64.0 MB): throughput (MB/s) | 1466.0 | 659.3 | 469.2 | 826.9 | PASS | +| v64 -> z64, generated: peak memory (MB) | 2.4 | 4.3 | 2.1 | 2.3 | FAIL | +| z64 -> n64, generated (64.0 MB): throughput (MB/s) | 1488.3 | 532.7 | n/a | n/a | PASS | +| z64 -> n64, generated: peak memory (MB) | 2.4 | 4.6 | n/a | n/a | PASS | +| v64 -> n64, generated (64.0 MB): throughput (MB/s) | 1270.6 | 492.8 | n/a | n/a | PASS | +| v64 -> n64, generated: peak memory (MB) | 2.4 | 4.1 | n/a | n/a | PASS | +| z64 -> v64, real (40.0 MB): throughput (MB/s) [real] | 1500.4 | 549.3 | 386.5 | 888.4 | PASS | +| z64 -> v64, real: peak memory (MB) [real] | 2.4 | 4.1 | 2.0 | 2.3 | FAIL | +| v64 -> z64, real (40.0 MB): throughput (MB/s) [real] | 1541.7 | 600.1 | 426.7 | 884.0 | PASS | +| v64 -> z64, real: peak memory (MB) [real] | 2.4 | 4.0 | 2.1 | 2.3 | FAIL | +| z64 -> n64, real (40.0 MB): throughput (MB/s) [real] | 1472.4 | 605.6 | n/a | n/a | PASS | +| z64 -> n64, real: peak memory (MB) [real] | 2.5 | 4.2 | n/a | n/a | PASS | +| v64 -> n64, real (40.0 MB): throughput (MB/s) [real] | 1233.9 | 604.8 | n/a | n/a | PASS | +| v64 -> n64, real: peak memory (MB) [real] | 2.4 | 4.2 | n/a | n/a | PASS | + +commit: ec68950, machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max + +The machine was in use during this run (load average 7 to 12 on 10 +cores: a browser, a VPN client, a remote-desktop host), so every tool's +throughput reads lower than the 2026-10-20 block's; the order between +them holds. The registry is now a table fixed at build time, so a +conversion starts at 1.95 MB rather than 2.24 and every memory row reads +2.4 (one 2.5) where it read 2.6. That closes most of the gap to `dd` but +not ucon64's 2.0 to 2.1. + 2026-10-20, `n64` branch, final code: | Target | Ours | n64romconvert | ucon64 | dd conv=swab | Result | @@ -95,14 +127,13 @@ commit: a54a353, machine: Darwin 24.5.0 arm64, 10 cpus, Apple M1 Max ## Threats to validity -- **The memory lines.** The binary holds 2.24 MB before it reads a byte - (`sublime check z64 v64`, which plans and exits), against ucon64's 2.0 - for its whole conversion: the code pages that loading the converter - registry and planning touch (about 800 KB of the program's code is - resident; `--version` alone is 1.68 MB). No buffer size passes these - lines; a registry held as data rather than reached through each - converter's methods would lower every conversion's floor, and is left - for the binary-wide work. +- **The memory lines.** The binary holds 1.95 MB before it reads a byte + (`sublime check z64 v64`, which plans and exits; 2.24 before the + registry became a table fixed at build time, 2026-10-21), against + ucon64's 2.0 for its whole conversion: the code pages that the command + line, detection, and planning touch (`--version` alone is 1.68 MB). + With the 256 KB buffer the conversion peaks at 2.4 MB; no buffer size + passes the ucon64 lines from that floor. - ucon64 does more than convert: it reports the header and checks the checksums, as it does on every run. - `dd conv=swab` only swaps pairs; it is a floor for the z64 <-> v64 copy, diff --git a/DOCS/formats/n64.md b/DOCS/formats/n64.md index bc2a71d7..438f0082 100644 --- a/DOCS/formats/n64.md +++ b/DOCS/formats/n64.md @@ -68,7 +68,7 @@ Streamed through a 256 KB buffer reordered in place, with the shuffle written so the compiler vectorizes it. About 1,750 MB/s on an M1 Max: 1.2 to 1.35 times `dd conv=swab`, 2.2 to 2.9 times `n64romconvert`, and 3 to 4.2 times ucon64. Memory is the binary's startup floor plus the buffer -(2.6 MB), a little over ucon64's 2.0 and `dd`'s 2.3. +(2.4 MB), a little over ucon64's 2.0 and `dd`'s 2.3. `DOCS/benchmarks/z64-v64.md`. ## Not covered diff --git a/bench/pairs/csv-json.sh b/bench/pairs/csv-json.sh index b1aa53cd..138246ed 100755 --- a/bench/pairs/csv-json.sh +++ b/bench/pairs/csv-json.sh @@ -19,7 +19,7 @@ run_pair() { crates_j2c="$(time_cmd crates-json-csv "$bench" csv-json crates-json-csv "$data/big.json" "$data/out4.csv")" echo "== stdin memory" >&2 - /usr/bin/time -f "%M" -o "$data/rss.txt" sh -c "$sublime -q convert - --from csv --to json < $data/big.csv > $data/out5.json" + "$time_bin" -f "%M" -o "$data/rss.txt" sh -c "$sublime -q convert - --from csv --to json < $data/big.csv > $data/out5.json" local stdin_rss stdin_rss="$(cat "$data/rss.txt")" diff --git a/bench/pairs/ico-png.sh b/bench/pairs/ico-png.sh index cffc94d5..33e2d9bc 100755 --- a/bench/pairs/ico-png.sh +++ b/bench/pairs/ico-png.sh @@ -7,6 +7,12 @@ # References: the image crate (its ICO decoder; for icons, `thumbnail` # at each size and its multi-image ICO encoder) with the png crate. +# Milliseconds from seconds and KB from bytes, as functions: macOS's bash +# 3.2 misreads an awk program quoted inside "$(...)" inside a row's +# arguments. +ms() { awk "BEGIN{printf \"%.1f\", $1 * 1000}"; } +kb() { awk "BEGIN{printf \"%.1f\", $1 / 1024}"; } + run_pair() { local side=4000 echo "== generating" >&2 @@ -25,16 +31,16 @@ image.save('$data/icon.ico') bytes="$(wc -c < "$data/icon.ico" | tr -d ' ')" ours="$(time_cmd ours "$sublime" -q convert "$data/icon.ico" "$data/out-icon-ours.png")" crates="$(time_cmd crates "$bench" ico-png crates-ico-png "$data/icon.ico" "$data/out-icon-crates.png")" - row "ico -> png, 256 icon ($(mb "$bytes") MB): wall time (ms)" "$(awk "BEGIN{printf \"%.1f\", $(seconds_of "$ours") * 1000}")" "$(awk "BEGIN{printf \"%.1f\", $(seconds_of "$crates") * 1000}") (image + png)" "$(pass "$(echo "$(seconds_of "$ours") <= $(seconds_of "$crates")" | bc -l)")" + row "ico -> png, 256 icon ($(mb "$bytes") MB): wall time (ms)" "$(ms "$(seconds_of "$ours")")" "$(ms "$(seconds_of "$crates")") (image + png)" "$(pass "$(echo "$(seconds_of "$ours") <= $(seconds_of "$crates")" | bc -l)")" row "ico -> png, 256 icon: peak memory (MB)" "$(rss_mb "$(rss_of "$ours")")" "$(rss_mb "$(rss_of "$crates")") (image + png)" "$(pass "$(echo "$(rss_of "$ours") <= $(rss_of "$crates")" | bc -l)")" for shape in icon256 jphoto jflat; do local png="$data/$shape.png" png_bytes png_bytes="$(wc -c < "$png" | tr -d ' ')" ours="$(time_cmd ours "$sublime" -q convert "$png" "$data/out-$shape-ours.ico")" crates="$(time_cmd crates "$bench" ico-png crates-png-ico "$png" "$data/out-$shape-crates.ico")" - row "png -> ico, $shape ($(mb "$png_bytes") MB in): wall time (ms)" "$(awk "BEGIN{printf \"%.1f\", $(seconds_of "$ours") * 1000}")" "$(awk "BEGIN{printf \"%.1f\", $(seconds_of "$crates") * 1000}") (png + image)" "$(pass "$(echo "$(seconds_of "$ours") <= $(seconds_of "$crates")" | bc -l)")" + row "png -> ico, $shape ($(mb "$png_bytes") MB in): wall time (ms)" "$(ms "$(seconds_of "$ours")")" "$(ms "$(seconds_of "$crates")") (png + image)" "$(pass "$(echo "$(seconds_of "$ours") <= $(seconds_of "$crates")" | bc -l)")" row "png -> ico, $shape: peak memory (MB)" "$(rss_mb "$(rss_of "$ours")")" "$(rss_mb "$(rss_of "$crates")") (png + image)" "$(pass "$(echo "$(rss_of "$ours") <= $(rss_of "$crates")" | bc -l)")" - row "png -> ico, $shape: output size (KB) [extra]" "$(awk "BEGIN{printf \"%.1f\", $(wc -c < "$data/out-$shape-ours.ico") / 1024}")" "$(awk "BEGIN{printf \"%.1f\", $(wc -c < "$data/out-$shape-crates.ico") / 1024}") (image)" "n/a" + row "png -> ico, $shape: output size (KB) [extra]" "$(kb "$(wc -c < "$data/out-$shape-ours.ico")")" "$(kb "$(wc -c < "$data/out-$shape-crates.ico")") (image)" "n/a" done rm -f "$data/out-"* } diff --git a/bench/pairs/jpeg-pdf.sh b/bench/pairs/jpeg-pdf.sh index ab540bc1..a1b5cd29 100755 --- a/bench/pairs/jpeg-pdf.sh +++ b/bench/pairs/jpeg-pdf.sh @@ -27,6 +27,8 @@ interleave() { crates=("$@") local ours_times=() crates_times=() start end # Microseconds from bash's clock (5.0 and later), no fork per reading. + # macOS's bash 3.2 has no EPOCHREALTIME: every time would read zero. + [ -n "${EPOCHREALTIME:-}" ] || { echo "bash 5 is needed (brew install bash)" >&2; return 1; } for _ in $(seq "$rounds"); do rm -f -- "$ours_out" start="${EPOCHREALTIME/[.,]/}"; "${ours[@]}" || return 1; end="${EPOCHREALTIME/[.,]/}" diff --git a/bench/src/pairs/pdf_text.rs b/bench/src/pairs/pdf_text.rs index 279c5bca..56332c9f 100644 --- a/bench/src/pairs/pdf_text.rs +++ b/bench/src/pairs/pdf_text.rs @@ -12,9 +12,9 @@ pub fn run(mode: &str, args: &[String]) -> Result<(), String> { ("ours-pdf-text", [input, output]) => { let converter = registry::all_converters() .iter() - .find(|converter| converter.name() == "pdf-to-text") + .find(|converter| converter.name == "pdf-to-text") .ok_or("no pdf-to-text converter")?; - run_ours(*converter, input, output) + run_ours(converter.converter, input, output) } ("crates-pdf-text", [input, output]) => crates_pdf_text(input, output), _ => Err("pdf-text modes: ours-pdf-text, crates-pdf-text ".to_string()),