diff --git a/crates/html/src/conformance/execution.rs b/crates/html/src/conformance/execution.rs index 4d080acf..fa7cc3ba 100644 --- a/crates/html/src/conformance/execution.rs +++ b/crates/html/src/conformance/execution.rs @@ -10,8 +10,9 @@ use super::{CanonicalParserResult, IncompleteObservationReason, ObservationState use crate::html5::PatchHistoryObservationConfig; use crate::html5::shared::{ CapturedSurface, DocumentParseContext, ErrorPolicy, ObservationOccurrenceSequence, - ObservationSurface, ParserObservationCapture, ParserObservationConfig, - ParserObservationInvariant, SurfaceCaptureRequest, + ObservationSurface, ParserObservationCapture, ParserObservationCaptureFailure, + ParserObservationConfig, ParserObservationFailure, ParserObservationInvariant, + SurfaceCaptureRequest, UnsupportedFeatureObservationFailure, }; use crate::html5::{ByteStreamDecoder, Html5Tokenizer, Input, TokenizeResult, TokenizerConfig}; use crate::{HtmlParseOptions, HtmlParser}; @@ -53,6 +54,10 @@ pub struct ParserObservationRequest<'a> { pub tokens: ObservationRequest, pub parse_errors: ObservationRequest, pub implementation_diagnostics: ObservationRequest, + /// Maximum retained central tree-builder dispatch attempts. This is an + /// event-count capacity, not a retained-string byte budget. + pub transitions: ObservationRequest, + pub unsupported_features: ObservationRequest, pub document_mode: ScalarObservationRequest, /// Maximum canonical structural units: document, document type, element or /// HTML template host, text, comment, processing instruction, and typed @@ -69,6 +74,8 @@ pub enum ParserObservationExecutionError { ParserInvariant, TokenizerInvariant(ParserTokenizerInvariantError), TokenCanonicalizationInvariant, + TreeTransitionTokenCanonicalizationInvariant, + UnsupportedFeatureObservationInvariant(UnsupportedFeatureObservationInvariantError), ObservationRecorderMissing, PatchHistoryCaptureMissing, ObservationInvariant(ParserObservationInvariantError), @@ -148,9 +155,13 @@ pub enum ParserTokenizerInvariantError { pub enum ParserObservationInvariantError { ParseErrorOccurrenceOverflow, ImplementationDiagnosticOccurrenceOverflow, + TreeTransitionOccurrenceOverflow, + UnsupportedFeatureOccurrenceOverflow, TokenDroppedCountOverflow, ParseErrorDroppedCountOverflow, ImplementationDiagnosticDroppedCountOverflow, + TreeTransitionDroppedCountOverflow, + UnsupportedFeatureDroppedCountOverflow, NormalizedPositionOverflow, NormalizedPositionIndexDiscontinuity, NormalizedPositionIndexMissing, @@ -169,6 +180,14 @@ pub enum ParserObservationInvariantError { SnapshotLabelSequenceOverflow, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum UnsupportedFeatureObservationInvariantError { + TokenAttributeNameUnavailable, + ExistingHtmlElementSemanticsUnavailable, + ExistingBodyElementSemanticsUnavailable, + ExistingElementIdentityContradiction, +} + impl std::fmt::Display for ParserObservationExecutionError { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -183,6 +202,13 @@ impl std::fmt::Display for ParserObservationExecutionError { Self::TokenCanonicalizationInvariant => { formatter.write_str("production token could not be resolved at its drain boundary") } + Self::TreeTransitionTokenCanonicalizationInvariant => formatter.write_str( + "tree transition token summary could not be resolved at its dispatch boundary", + ), + Self::UnsupportedFeatureObservationInvariant(invariant) => write!( + formatter, + "unsupported-feature observation invariant failed: {invariant:?}" + ), Self::ObservationRecorderMissing => formatter.write_str( "parser observation was requested but the production recorder was missing", ), @@ -217,6 +243,11 @@ pub fn execute_parser_observation( tokens: internal_request(request.tokens), parse_errors: internal_request(request.parse_errors), implementation_diagnostics: internal_request(request.implementation_diagnostics), + tree_transitions: match request.target { + ParserObservationTarget::StandaloneTokenizer => SurfaceCaptureRequest::NotRequested, + ParserObservationTarget::DocumentParser => internal_request(request.transitions), + }, + unsupported_features: internal_request(request.unsupported_features), }; let patch_config = match request.patches { ObservationRequest::NotRequested => PatchHistoryObservationConfig::default(), @@ -252,7 +283,14 @@ pub fn execute_parser_observation( (capture, mode, tree, patches) } }; - canonical_result(capture, document_mode, tree, patches) + canonical_result( + capture, + document_mode, + tree, + patches, + request.target, + request.transitions, + ) } fn not_applicable_or_not_requested(request: ObservationRequest) -> ObservationState { @@ -689,8 +727,17 @@ fn empty_capture() -> ParserObservationCapture { items: Vec::new(), dropped: 0, }, - token_capture_failed: false, - invariant: None, + tree_transitions: CapturedSurface { + requested: false, + items: Vec::new(), + dropped: 0, + }, + unsupported_features: CapturedSurface { + requested: false, + items: Vec::new(), + dropped: 0, + }, + failure: None, } } @@ -728,8 +775,16 @@ fn canonical_result( document_mode: ObservationState, tree: ObservationState, patches: ObservationState, + target: ParserObservationTarget, + transitions_request: ObservationRequest, ) -> Result { validate_capture(&capture)?; + let transitions = match target { + ParserObservationTarget::StandaloneTokenizer => { + not_applicable_or_not_requested(transitions_request) + } + ParserObservationTarget::DocumentParser => finish_surface(capture.tree_transitions), + }; Ok(CanonicalParserResult { tokens: finish_surface(capture.tokens), parse_errors: finish_surface(capture.parse_errors), @@ -737,8 +792,8 @@ fn canonical_result( document_mode, tree, patches, - transitions: ObservationState::NotRequested, - unsupported_features: ObservationState::NotRequested, + transitions, + unsupported_features: finish_surface(capture.unsupported_features), final_invariants: ObservationState::NotRequested, }) } @@ -746,17 +801,54 @@ fn canonical_result( fn validate_capture( capture: &ParserObservationCapture, ) -> Result<(), ParserObservationExecutionError> { - if capture.token_capture_failed { - return Err(ParserObservationExecutionError::TokenCanonicalizationInvariant); - } - if let Some(invariant) = capture.invariant { - return Err(ParserObservationExecutionError::ObservationInvariant( - public_observation_invariant(invariant), - )); + if let Some(failure) = capture.failure { + return Err(public_observation_failure(failure)); } Ok(()) } +fn public_observation_failure( + failure: ParserObservationFailure, +) -> ParserObservationExecutionError { + match failure { + ParserObservationFailure::Capture( + ParserObservationCaptureFailure::TokenCanonicalization, + ) => ParserObservationExecutionError::TokenCanonicalizationInvariant, + ParserObservationFailure::Capture( + ParserObservationCaptureFailure::TreeTransitionTokenCanonicalization, + ) => ParserObservationExecutionError::TreeTransitionTokenCanonicalizationInvariant, + ParserObservationFailure::Capture( + ParserObservationCaptureFailure::UnsupportedFeatureEligibility(failure), + ) => ParserObservationExecutionError::UnsupportedFeatureObservationInvariant( + public_unsupported_feature_observation_failure(failure), + ), + ParserObservationFailure::Invariant(invariant) => { + ParserObservationExecutionError::ObservationInvariant(public_observation_invariant( + invariant, + )) + } + } +} + +fn public_unsupported_feature_observation_failure( + failure: UnsupportedFeatureObservationFailure, +) -> UnsupportedFeatureObservationInvariantError { + match failure { + UnsupportedFeatureObservationFailure::TokenAttributeNameUnavailable => { + UnsupportedFeatureObservationInvariantError::TokenAttributeNameUnavailable + } + UnsupportedFeatureObservationFailure::ExistingHtmlElementSemanticsUnavailable => { + UnsupportedFeatureObservationInvariantError::ExistingHtmlElementSemanticsUnavailable + } + UnsupportedFeatureObservationFailure::ExistingBodyElementSemanticsUnavailable => { + UnsupportedFeatureObservationInvariantError::ExistingBodyElementSemanticsUnavailable + } + UnsupportedFeatureObservationFailure::ExistingElementIdentityContradiction => { + UnsupportedFeatureObservationInvariantError::ExistingElementIdentityContradiction + } + } +} + fn public_observation_invariant( invariant: ParserObservationInvariant, ) -> ParserObservationInvariantError { @@ -767,6 +859,12 @@ fn public_observation_invariant( ParserObservationInvariant::OccurrenceSequenceOverflow( ObservationOccurrenceSequence::ImplementationDiagnostics, ) => ParserObservationInvariantError::ImplementationDiagnosticOccurrenceOverflow, + ParserObservationInvariant::OccurrenceSequenceOverflow( + ObservationOccurrenceSequence::TreeTransitions, + ) => ParserObservationInvariantError::TreeTransitionOccurrenceOverflow, + ParserObservationInvariant::OccurrenceSequenceOverflow( + ObservationOccurrenceSequence::UnsupportedFeatures, + ) => ParserObservationInvariantError::UnsupportedFeatureOccurrenceOverflow, ParserObservationInvariant::DroppedCountOverflow(ObservationSurface::Tokens) => { ParserObservationInvariantError::TokenDroppedCountOverflow } @@ -776,6 +874,12 @@ fn public_observation_invariant( ParserObservationInvariant::DroppedCountOverflow( ObservationSurface::ImplementationDiagnostics, ) => ParserObservationInvariantError::ImplementationDiagnosticDroppedCountOverflow, + ParserObservationInvariant::DroppedCountOverflow(ObservationSurface::TreeTransitions) => { + ParserObservationInvariantError::TreeTransitionDroppedCountOverflow + } + ParserObservationInvariant::DroppedCountOverflow( + ObservationSurface::UnsupportedFeatures, + ) => ParserObservationInvariantError::UnsupportedFeatureDroppedCountOverflow, ParserObservationInvariant::NormalizedPositionOverflow => { ParserObservationInvariantError::NormalizedPositionOverflow } @@ -856,6 +960,8 @@ mod tests { implementation_diagnostics: ObservationRequest::Capture { capacity: DIAGNOSTIC_CAPACITY, }, + transitions: ObservationRequest::NotRequested, + unsupported_features: ObservationRequest::NotRequested, document_mode: ScalarObservationRequest::NotRequested, tree: ObservationRequest::NotRequested, patches: ObservationRequest::NotRequested, @@ -870,6 +976,66 @@ mod tests { } } + fn observe_ae13b4( + source: &str, + transitions: ObservationRequest, + unsupported_features: ObservationRequest, + ) -> CanonicalParserResult { + observe_ae13b4_input( + ParserObservationInput::Utf8(source), + transitions, + unsupported_features, + ObservationRequest::NotRequested, + ) + } + + fn observe_ae13b4_input( + input: ParserObservationInput<'_>, + transitions: ObservationRequest, + unsupported_features: ObservationRequest, + tree: ObservationRequest, + ) -> CanonicalParserResult { + execute_parser_observation(ParserObservationRequest { + target: ParserObservationTarget::DocumentParser, + input, + tokens: ObservationRequest::NotRequested, + parse_errors: ObservationRequest::Capture { capacity: 256 }, + implementation_diagnostics: ObservationRequest::Capture { capacity: 256 }, + transitions, + unsupported_features, + document_mode: ScalarObservationRequest::NotRequested, + tree, + patches: ObservationRequest::NotRequested, + }) + .expect("AE13b4 production observation") + } + + fn unsupported_identities( + result: &CanonicalParserResult, + ) -> Vec { + captured(&result.unsupported_features) + .iter() + .map(|event| match event { + crate::html5::shared::UnsupportedFeatureEvent::TreeConstruction { + feature, .. + } => *feature, + }) + .collect() + } + + fn canonical_result_with_unrequested_projections( + capture: ParserObservationCapture, + ) -> Result { + canonical_result( + capture, + ObservationState::NotRequested, + ObservationState::NotRequested, + ObservationState::NotRequested, + ParserObservationTarget::DocumentParser, + ObservationRequest::NotRequested, + ) + } + fn observed_scalars(result: &CanonicalParserResult) -> String { captured(&result.tokens) .iter() @@ -960,6 +1126,8 @@ mod tests { tokens: ObservationRequest::NotRequested, parse_errors: ObservationRequest::NotRequested, implementation_diagnostics: ObservationRequest::NotRequested, + transitions: ObservationRequest::NotRequested, + unsupported_features: ObservationRequest::NotRequested, document_mode: ScalarObservationRequest::NotRequested, tree: ObservationRequest::NotRequested, patches: ObservationRequest::NotRequested, @@ -976,6 +1144,662 @@ mod tests { )); } + #[test] + fn ae13b4_target_applicability_is_explicit_and_surface_local() { + let standalone = execute_parser_observation(ParserObservationRequest { + target: ParserObservationTarget::StandaloneTokenizer, + input: ParserObservationInput::Utf8("

x"), + tokens: ObservationRequest::NotRequested, + parse_errors: ObservationRequest::NotRequested, + implementation_diagnostics: ObservationRequest::NotRequested, + transitions: ObservationRequest::Capture { capacity: 8 }, + unsupported_features: ObservationRequest::Capture { capacity: 8 }, + document_mode: ScalarObservationRequest::NotRequested, + tree: ObservationRequest::NotRequested, + patches: ObservationRequest::NotRequested, + }) + .unwrap(); + assert_eq!( + standalone.transitions, + ObservationState::NotApplicable { + reason: NotApplicableReason::StandaloneTokenizerRun + } + ); + assert_eq!( + standalone.unsupported_features, + ObservationState::Captured(Vec::new()) + ); + + let document = observe_ae13b4( + "

x", + ObservationRequest::Capture { capacity: 32 }, + ObservationRequest::NotRequested, + ); + assert!(matches!( + document.transitions, + ObservationState::Captured(_) + )); + assert!(matches!( + document.unsupported_features, + ObservationState::NotRequested + )); + } + + #[test] + fn eof_reprocessing_records_each_central_attempt_with_committed_modes() { + use crate::html5::shared::{ObservedInsertionMode as Mode, TreeDispatchPath}; + + let result = observe_ae13b4( + "", + ObservationRequest::Capture { capacity: 16 }, + ObservationRequest::NotRequested, + ); + let transitions = captured(&result.transitions); + let modes = [ + (Mode::Initial, Mode::BeforeHtml), + (Mode::BeforeHtml, Mode::BeforeHead), + (Mode::BeforeHead, Mode::InHead), + (Mode::InHead, Mode::AfterHead), + (Mode::AfterHead, Mode::InBody), + (Mode::InBody, Mode::InBody), + ]; + assert_eq!(transitions.len(), modes.len()); + for (index, (event, (before, after))) in transitions.iter().zip(modes).enumerate() { + assert_eq!(event.occurrence, index as u64 + 1); + assert_eq!(event.insertion_mode_before, before); + assert_eq!(event.insertion_mode_after, after); + assert_eq!( + event.dispatch_path, + TreeDispatchPath::HtmlInsertionMode(before) + ); + assert_eq!(event.reprocessed, index != 0); + assert!(matches!( + event.token.as_ref(), + crate::html5::shared::TransitionTokenSummary::Eof + )); + } + } + + #[test] + fn dispatch_paths_distinguish_template_text_foreign_and_internal_delegation() { + use crate::html5::shared::{ + ObservedInsertionMode as Mode, TransitionTokenSummary, TreeDispatchPath, + }; + + let result = observe_ae13b4( + "", + ObservationRequest::Capture { capacity: 128 }, + ObservationRequest::NotRequested, + ); + let transitions = captured(&result.transitions); + assert!(transitions.iter().any(|event| { + event.dispatch_path == TreeDispatchPath::SharedTemplateRules + && matches!( + event.token.as_ref(), + TransitionTokenSummary::StartTag { name, .. } if name == "template" + ) + })); + assert!(transitions.iter().any(|event| { + event.dispatch_path == TreeDispatchPath::TextMode + && event.insertion_mode_before == Mode::Text + && matches!( + event.token.as_ref(), + TransitionTokenSummary::Character { data } if data == "x" + ) + })); + assert!(transitions.iter().any(|event| { + event.insertion_mode_before == Mode::InTemplate + && event.insertion_mode_after == Mode::Text + && !event.reprocessed + && matches!( + event.token.as_ref(), + TransitionTokenSummary::StartTag { name, .. } if name == "title" + ) + })); + assert!(transitions.iter().any(|event| { + event.dispatch_path == TreeDispatchPath::ForeignContent + && matches!( + event.token.as_ref(), + TransitionTokenSummary::StartTag { name, .. } if name == "g" + ) + })); + + let delegated = observe_ae13b4( + "
", + ObservationRequest::Capture { capacity: 64 }, + ObservationRequest::NotRequested, + ); + let html_attempts = captured(&delegated.transitions) + .iter() + .filter(|event| { + matches!( + event.token.as_ref(), + TransitionTokenSummary::StartTag { name, .. } if name == "html" + ) + }) + .collect::>(); + assert_eq!(html_attempts.len(), 1); + assert_eq!( + html_attempts[0].dispatch_path, + TreeDispatchPath::HtmlInsertionMode(Mode::InCell) + ); + } + + #[test] + fn template_and_table_mode_changes_redispatch_only_through_the_driver() { + use crate::html5::shared::{ + ObservedInsertionMode as Mode, TransitionTokenSummary, TreeDispatchPath, + }; + + for (source, name, expected_paths) in [ + ( + "