From 57b07f45d5ecd5d45b6d623aff7f82d3b1ff00d7 Mon Sep 17 00:00:00 2001 From: swackhamer Date: Sat, 1 Aug 2026 22:47:22 -0500 Subject: [PATCH] feat(html): extract HTML meta, Dublin Core and Office tags HTML was detected but had no parser and no FileFormat variant, so every .html file fell through to the plain-text fallback and was parsed as TXT: five TEXT:* statistics ExifTool never reports for an HTML file, and none of the 57 tags it does. combined-samples/HTML.html scored MATCHED 0, MISSING 57, EXTRA 5. This adds src/parsers/text/html.rs, a transcription of ExifTool 13.55's HTML.pm (ProcessHTML), plus FileFormat::HTML, its dispatch arm and a detection gate. The sample now scores MATCHED 57, MISSING 0, EXTRA 0. Across the 207-file corpus it is the only file whose score changed, and no file lost a matched tag. One file yields five family-1 groups, because a meta name carries its own namespace and ExifTool routes each to a different tag table: HTML:Title, HTTP-equiv:ContentType, HTML-dc:*, HTML-ncc:*, HTML-prod:* and HTML-office:* (the MS-Office XML island inside the mso conditional comment). The details that make values match rather than merely appear: - dc:creator is a Seq, so two META elements collapse into one list, while dc:language and dc:subject are also List but hold one value each and stay scalars -- which is what ExifTool emits. - iso-8859-1 maps to ExifTool's Latin charset, which is cp1252 and not bare ISO 8859-1. Values seen before the content-type element are not recoded; HTML.pm says so explicitly and this reproduces it rather than fixing it. - The meta path unescapes with the full 253-entity HTML table, the XML island with only the five XML entities plus numeric references, so HTML-office:Description keeps its as a literal CR. - Conversions are per tag, not per kind: HTML-office:CreateDate goes through ConvertXMPDate to 2010:06:28 23:52:00Z while HTML-dc:Date has no ValueConv and stays raw as the malformed 2007-30-01. - A tag the table does not declare is named by the module's own mangling and then by AddTagToTable's normalisation. Without that second step an unknown namespace reported unknownNsWeird_thing where ExifTool reports UnknownNsWeird_thing. - With no declared charset ExifTool does not recode, so a stray high byte reaches its writer raw and FixUTF8 turns each bad byte into one '?'. Rust's lossy conversion collapses a whole invalid run into a single U+FFFD, which is a different string. The tables are not hand-invented. All seven (112 tags) were diffed key by key against Image::ExifTool::GetTagTable on tag ID, Name and List flag; the 253-entry entity table was diffed against a Perl eval of %entityNum; and ConvertTimeSpan was compared with the Perl for every integer TotalTime from 1 to 200000. ConvertTimeSpan's sub-minute branch would need Perl's %.15g number stringification for a fractional value, so that one case omits the tag rather than print an approximation under a real tag name. Detection uses ProcessHTML's own gate -- an XML declaration alone is not enough, an HTML element must appear in the first 256 bytes -- and runs after the existing SVG and XML-plist root checks, so all four --- src/core/file_format.rs | 5 + src/core/format_dispatch.rs | 2 + src/parsers/detection/mod.rs | 63 ++ src/parsers/font/ttf.rs | 2 +- src/parsers/text/html.rs | 1752 ++++++++++++++++++++++++++++++++++ src/parsers/text/mod.rs | 2 + 6 files changed, 1825 insertions(+), 1 deletion(-) create mode 100644 src/parsers/text/html.rs diff --git a/src/core/file_format.rs b/src/core/file_format.rs index 7cc292eec..23418e79a 100644 --- a/src/core/file_format.rs +++ b/src/core/file_format.rs @@ -249,6 +249,9 @@ pub enum FileFormat { /// Plain text format (.txt) TXT, + /// HTML / XHTML document (.htm, .html, .xhtml) + HTML, + /// Windows shortcut (.lnk) LNK, @@ -382,6 +385,7 @@ impl FileFormat { FileFormat::EML => "EML", FileFormat::TXT => "TXT", FileFormat::LFP => "LFP", + FileFormat::HTML => "HTML", FileFormat::LNK => "Windows Shortcut", FileFormat::SQLite => "SQLite", FileFormat::Prefetch => "Windows Prefetch", @@ -483,6 +487,7 @@ impl FileFormat { FileFormat::EML => &["eml", "email"], FileFormat::TXT => &["txt", "text"], FileFormat::LFP => &["lfp", "lfr"], + FileFormat::HTML => &["htm", "html", "xhtml"], FileFormat::LNK => &["lnk"], FileFormat::SQLite => &["db", "sqlite", "sqlite3"], FileFormat::Prefetch => &["pf"], diff --git a/src/core/format_dispatch.rs b/src/core/format_dispatch.rs index 8b6b2a610..7498a6709 100644 --- a/src/core/format_dispatch.rs +++ b/src/core/format_dispatch.rs @@ -69,6 +69,7 @@ use crate::parsers::specialized::sqlite::parse_sqlite_metadata; use crate::parsers::specialized::stl::parse_stl_metadata; use crate::parsers::specialized::x509::parse_x509_metadata; use crate::parsers::text::eps::parse_eps_metadata; +use crate::parsers::text::html::parse_html_metadata; use crate::parsers::text::txt::parse_txt_metadata; use crate::parsers::text::vcf::parse_vcf_metadata; use crate::parsers::video::asf::parse_asf_metadata; @@ -171,6 +172,7 @@ pub fn dispatch_format_parser(reader: &dyn FileReader, format: FileFormat) -> Re FileFormat::HDF5 => convert_string_error(parse_hdf5_metadata(reader), "HDF5"), FileFormat::VCF => convert_string_error(parse_vcf_metadata(reader), "VCF"), FileFormat::TXT => convert_string_error(parse_txt_metadata(reader), "TXT"), + FileFormat::HTML => convert_string_error(parse_html_metadata(reader), "HTML"), FileFormat::LNK => convert_string_error(parse_lnk_metadata(reader), "LNK"), FileFormat::LFP => convert_string_error(parse_lytro_metadata(reader), "LFP"), FileFormat::SQLite => convert_string_error(parse_sqlite_metadata(reader), "SQLite"), diff --git a/src/parsers/detection/mod.rs b/src/parsers/detection/mod.rs index b5946145e..286ff4b48 100644 --- a/src/parsers/detection/mod.rs +++ b/src/parsers/detection/mod.rs @@ -266,6 +266,18 @@ pub fn detect_format(reader: &dyn FileReader) -> io::Result { return Ok(FileFormat::Plist); } + // HTML and XHTML, using ExifTool's own gate from `HTML.pm`'s ProcessHTML. + // It runs after the SVG and plist roots because those three share the + // `\n\ + \n\ + t\n", + FileFormat::HTML, + ), + ( + b"\n\n\n", + FileFormat::HTML, + ), + ( + b"\n\n", + FileFormat::SVG, + ), + ( + b"\n\ + \n\ + \n", + FileFormat::Plist, + ), + ( + b"\n\ + \n", + FileFormat::XMP, + ), + ( + // A plain XML document names no HTML element, so it must stay + // on the plain-text path rather than being claimed as HTML. + b"\n\ + \n\ + \n", + FileFormat::TXT, + ), + ]; + for (data, expected) in cases { + let reader = TestReader::new(data.to_vec()); + assert_eq!( + detect_format(&reader).unwrap(), + *expected, + "misdetected: {}", + String::from_utf8_lossy(&data[..data.len().min(60)]) + ); + } + } + #[test] fn test_detect_svg_with_multibyte_char_straddling_probe_boundary() { // An SVG whose 1 KiB probe cut splits a multibyte character must still diff --git a/src/parsers/font/ttf.rs b/src/parsers/font/ttf.rs index e48b185d0..6240cec98 100644 --- a/src/parsers/font/ttf.rs +++ b/src/parsers/font/ttf.rs @@ -195,7 +195,7 @@ impl TTFParser { } /// Decodes a string using the Macintosh Roman encoding. - fn decode_mac_roman(data: &[u8]) -> String { + pub(crate) fn decode_mac_roman(data: &[u8]) -> String { const MAC_ROMAN_HIGH: [char; 128] = [ 'Ä', 'Å', 'Ç', 'É', 'Ñ', 'Ö', 'Ü', 'á', 'à', 'â', 'ä', 'ã', 'å', 'ç', 'é', 'è', 'ê', 'ë', 'í', 'ì', 'î', 'ï', 'ñ', 'ó', 'ò', 'ô', 'ö', 'õ', 'ú', 'ù', 'û', 'ü', '†', '°', diff --git a/src/parsers/text/html.rs b/src/parsers/text/html.rs new file mode 100644 index 000000000..67ebd0ed2 --- /dev/null +++ b/src/parsers/text/html.rs @@ -0,0 +1,1752 @@ +//! HTML / XHTML `` meta information parser. +//! +//! A transcription of ExifTool's `HTML.pm` (`ProcessHTML`, ExifTool 13.55). +//! Everything ExifTool reports for an HTML file comes out of the document +//! head: `META` elements, the `TITLE` element, and the MS-Office `XML` island +//! that Word and Excel write inside a `\n\ + \n", + ); + + let metadata = HTMLParser::parse_bytes(&doc); + + // One file, five family-1 groups. + assert_eq!( + value(&metadata, "HTML:Title").as_deref(), + Some("ExifTool HTML Test") + ); + assert_eq!( + value(&metadata, "HTTP-equiv:ContentType").as_deref(), + Some("text/html; charset=\"iso-8859-1\"") + ); + assert_eq!( + value(&metadata, "HTML-ncc:Duration").as_deref(), + Some("91:27:21") + ); + assert_eq!( + value(&metadata, "HTML-prod:RecEngineer").as_deref(), + Some("P Harvey") + ); + + // dc:creator is a Seq, so two META elements collapse into one list. + assert_eq!( + metadata.get("HTML-dc:Creator"), + Some(&TagValue::Array(vec![ + TagValue::String("Phil Harvey".into()), + TagValue::String("Another Creator".into()), + ])) + ); + // dc:date carries only ConvertDateTime, which is the identity here -- + // the malformed date is reported exactly as written. + assert_eq!( + value(&metadata, "HTML-dc:Date").as_deref(), + Some("2007-30-01") + ); + assert_eq!( + value(&metadata, "HTML-dc:Subject").as_deref(), + Some("Greek: α β γ") + ); + + // The Office island: charset-decoded, XML-unescaped, converted. + assert_eq!( + value(&metadata, "HTML-office:Category").as_deref(), + Some("a catégory") + ); + assert_eq!( + value(&metadata, "HTML-office:Description").as_deref(), + Some("a comments\ra new line") + ); + assert_eq!( + value(&metadata, "HTML-office:TotalEditTime").as_deref(), + Some("1 minute") + ); + assert_eq!( + value(&metadata, "HTML-office:CreateDate").as_deref(), + Some("2010:06:28 23:52:00Z") + ); + // o:Revision and o:Version are both RevisionNumber and neither is a + // list, so the later element wins. + assert_eq!( + value(&metadata, "HTML-office:RevisionNumber").as_deref(), + Some("12.0") + ); + // A custom property not in the table gets its name derived. + assert_eq!( + value(&metadata, "HTML-office:CheckedBy").as_deref(), + Some("Phil") + ); + } + + #[test] + fn a_document_without_a_head_yields_nothing() { + let metadata = HTMLParser::parse_bytes(b"\nno head here\n\n"); + assert_eq!(metadata.len(), 0); + } +} diff --git a/src/parsers/text/mod.rs b/src/parsers/text/mod.rs index 2c1811330..555e83d96 100644 --- a/src/parsers/text/mod.rs +++ b/src/parsers/text/mod.rs @@ -3,9 +3,11 @@ #![allow(dead_code)] pub mod eps; +pub mod html; pub mod txt; pub mod vcf; pub use eps::{EPSParser, parse_eps_metadata}; +pub use html::{HTMLParser, parse_html_metadata}; pub use txt::{TXTParser, parse_txt_metadata}; pub use vcf::VCFParser;