diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2b33037..c5af41a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,4 +29,4 @@ jobs: git diff --exit-code || (echo "::error::Run 'pixi run fmt' — sources are not mojo-format clean" && exit 1) - name: Tests - run: .venv/bin/mojo run -I src test/test_tokenizer.mojo && .venv/bin/mojo run -I src test/test_extract.mojo + run: .venv/bin/mojo run -I src test/test_tokenizer.mojo && .venv/bin/mojo run -I src test/test_extract.mojo && .venv/bin/mojo run -I src test/test_errors.mojo diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bf1087..b7de25c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## Unreleased + +- New `html.errors` module (exported from the package), following the + shared mojo-* suite pattern: `line_col(source, offset)` maps a byte + offset to a 1-based (line, column) pair — the column is the 1-based + BYTE offset within the line, no UTF-8 decoding — and + `parse_error(msg, source, offset)` builds an `Error` reading + ` at line , column : ''`, where the snippet is up + to ~30 bytes of the offending line centered on the column, + whitespace-trimmed, with `...` where truncated, and never multi-line. +- Strict-mode tokenizer errors now carry that position + snippet + (previously a bare `(line L, column C)` suffix with no snippet), with + more useful offsets: unclosed-element-at-EOF errors point at the + unclosed start tag instead of the end of input, and entity errors + (unknown entity, bare `&`, malformed numeric reference) point at the + offending `&` — in text, attribute values, and escapable raw text — + instead of the tokenizer's position after the run. +- No mechanism change: the tokenizer still `raise`s a plain `Error`, the + `mojo-html [strict]: ` prefix is unchanged, and existing + `contains=`-style message checks keep matching. The default liberal + mode still never raises on malformed markup. + ## 0.1.0 — 2026-07-05 Initial release. Liberal HTML tokenizer (void elements, raw-text diff --git a/pixi.toml b/pixi.toml index 40d2efe..61d634d 100644 --- a/pixi.toml +++ b/pixi.toml @@ -7,7 +7,7 @@ platforms = ["linux-64", "osx-arm64"] version = "0.1.0" [tasks] -test = "mojo run -I src test/test_tokenizer.mojo && mojo run -I src test/test_extract.mojo" +test = "mojo run -I src test/test_tokenizer.mojo && mojo run -I src test/test_extract.mojo && mojo run -I src test/test_errors.mojo" demo = "mojo run -I src examples/extract_article.mojo test/data/substack_article.html" bench = "mojo build -I src bench/bench_parse.mojo -o .bench_parse && ./.bench_parse" fmt = "mojo format src/ test/ examples/ bench/" diff --git a/src/html/__init__.mojo b/src/html/__init__.mojo index e5c5fe6..63b8136 100644 --- a/src/html/__init__.mojo +++ b/src/html/__init__.mojo @@ -1,5 +1,6 @@ """Liberal HTML parsing and readable-text extraction for Mojo (mojo-html).""" +from html.errors import line_col, parse_error from html.extract import ( extract, main_text, diff --git a/src/html/errors.mojo b/src/html/errors.mojo new file mode 100644 index 0000000..a9c0039 --- /dev/null +++ b/src/html/errors.mojo @@ -0,0 +1,135 @@ +"""Position-aware parse errors. + +`line_col` maps a byte offset in a source buffer to a 1-based +(line, column) pair, and `parse_error` builds an `Error` whose message +carries that position plus a short snippet of the offending line: + + at line , column : '' + +Positions are byte-based: the column is the 1-based BYTE offset within +the line, not a codepoint or display column. That keeps the computation +deterministic and free of UTF-8 decode cost; for ASCII-heavy markup the +byte column and the visual column coincide. + +This module is the error-reporting pattern shared across the mojo-* +parser suite. +""" + +comptime _LF = UInt8(0x0A) +comptime _CR = UInt8(0x0D) + +# Snippet size budget, in bytes, before the `...` truncation markers are +# added. Wide enough to show meaningful context, narrow enough that error +# messages stay one readable line. +comptime _SNIPPET_BUDGET = 30 + + +def _is_ws(b: UInt8) -> Bool: + return b == 0x20 or b == 0x09 or b == _CR or b == _LF + + +def line_col(source: Span[UInt8, _], offset: Int) -> Tuple[Int, Int]: + """1-based (line, column) of byte `offset` in `source`. + + The column is the 1-based BYTE offset within the line — codepoints + are never decoded, so the result is cheap and deterministic even on + invalid UTF-8. Only LF (0x0A) terminates a line: after a CRLF + sequence the next byte is column 1 of the next line, with no phantom + column contributed by the CR. An offset pointing AT an LF reports + the line that newline terminates (column = line length + 1). Offsets + past the end of `source` clamp to the end; an empty source yields + (1, 1). + """ + var limit = offset + if limit > len(source): + limit = len(source) + if limit < 0: + limit = 0 + var line = 1 + var last_nl = -1 + for i in range(limit): + if source[i] == _LF: + line += 1 + last_nl = i + return (line, limit - last_nl) + + +def _snippet(source: Span[UInt8, _], offset: Int) -> String: + """Up to ~`_SNIPPET_BUDGET` bytes of the line containing `offset`. + + The line is trimmed of leading/trailing whitespace (which also drops + the CR of a CRLF line ending), then windowed around the offset with + `...` marking each side that was cut. Window edges are nudged off + UTF-8 continuation bytes so the result is always valid UTF-8. The + result never contains a newline. + """ + var n = len(source) + var anchor = offset + if anchor > n: + anchor = n + if anchor < 0: + anchor = 0 + # Line bounds around the anchor; an anchor sitting AT an LF belongs + # to the line that newline terminates. + var line_start = anchor + while line_start > 0 and source[line_start - 1] != _LF: + line_start -= 1 + var line_end = anchor + while line_end < n and source[line_end] != _LF: + line_end += 1 + # Trim surrounding whitespace. + while line_start < line_end and _is_ws(source[line_start]): + line_start += 1 + while line_end > line_start and _is_ws(source[line_end - 1]): + line_end -= 1 + var win_start = line_start + var win_end = line_end + var cut_left = False + var cut_right = False + if line_end - line_start > _SNIPPET_BUDGET: + win_start = anchor - _SNIPPET_BUDGET // 2 + if win_start > line_end - _SNIPPET_BUDGET: + win_start = line_end - _SNIPPET_BUDGET + if win_start < line_start: + win_start = line_start + win_end = win_start + _SNIPPET_BUDGET + # Never split a multi-byte UTF-8 sequence at a window edge. + while win_start < win_end and (source[win_start] & 0xC0) == 0x80: + win_start += 1 + while win_end < line_end and (source[win_end] & 0xC0) == 0x80: + win_end += 1 + cut_left = win_start > line_start + cut_right = win_end < line_end + var out = String() + if cut_left: + out += "..." + out += String(StringSlice(unsafe_from_utf8=source[win_start:win_end])) + if cut_right: + out += "..." + return out^ + + +def parse_error(msg: String, source: Span[UInt8, _], offset: Int) -> Error: + """An `Error` locating `msg` at byte `offset` of `source`. + + The message is exactly: + + at line , column : '' + + where line/column come from `line_col` (1-based; column is a byte + offset within the line) and the snippet is the offending line, + whitespace-trimmed and truncated to ~30 bytes centered on the + column, with `...` where truncated. The message never contains a + newline, so it renders on one line in test output and logs. + """ + var lc = line_col(source, offset) + return Error( + msg + + " at line " + + String(lc[0]) + + ", column " + + String(lc[1]) + + ": '" + + _snippet(source, offset) + + "'" + ) diff --git a/src/html/tokenizer.mojo b/src/html/tokenizer.mojo index cddf826..b4b416c 100644 --- a/src/html/tokenizer.mojo +++ b/src/html/tokenizer.mojo @@ -13,6 +13,8 @@ markup in its default liberal mode. Structural recovery (unclosed

,

  • , crossed tags) is the mapping layer's job — see extract.mojo. """ +from html.errors import parse_error + comptime EVENT_START = 0 comptime EVENT_END = 1 comptime EVENT_TEXT = 2 @@ -486,9 +488,10 @@ struct HtmlTokenizer(Copyable, Movable): With `strict=True` the tokenizer reports well-formedness problems — mismatched or stray end tags, elements left open at EOF, malformed - or unknown entities — as errors with a line/column location instead - of recovering liberally. Useful for linting HTML you produce; leave - it off for pages you merely consume. + or unknown entities — as errors with a line/column location and a + snippet of the offending line instead of recovering liberally. + Useful for linting HTML you produce; leave it off for pages you + merely consume. """ var src: String @@ -498,6 +501,10 @@ struct HtmlTokenizer(Copyable, Movable): var _has_pending_end: Bool var _rawtext: String var _open: List[String] + # Byte offset of each open element's start tag, parallel to `_open`, + # so an unclosed-element error can point at the construct start + # rather than the (useless) EOF position. + var _open_pos: List[Int] def __init__(out self, var source: String, *, strict: Bool = False) raises: self.src = normalize_encoding(source^) @@ -507,31 +514,17 @@ struct HtmlTokenizer(Copyable, Movable): self._has_pending_end = False self._rawtext = String() self._open = List[String]() + self._open_pos = List[Int]() - def _location(self, p: Int) -> String: - """Human-readable "line L, column C" for byte offset `p`. + def _strict_error(self, msg: String, p: Int) -> Error: + """Strict-mode error locating `msg` at byte offset `p`. - Computed lazily (only on error paths), so the happy path pays - nothing for location tracking. + Position and snippet come from `html.errors.parse_error`, so the + message reads `mojo-html [strict]: at line , column : + ''`. Computed lazily (only on error paths), so the happy + path pays nothing for location tracking. """ - var bytes = self.src.as_bytes() - var line = 1 - var col = 1 - var limit = p - if limit > len(bytes): - limit = len(bytes) - for i in range(limit): - if bytes[i] == 0x0A: - line += 1 - col = 1 - else: - col += 1 - return String("line ") + String(line) + ", column " + String(col) - - def _strict_error(self, msg: String, p: Int) -> Error: - return Error( - "mojo-html [strict]: " + msg + " (" + self._location(p) + ")" - ) + return parse_error("mojo-html [strict]: " + msg, self.src.as_bytes(), p) def _len(self) -> Int: return self.src.byte_length() @@ -577,7 +570,10 @@ struct HtmlTokenizer(Copyable, Movable): var b = self._at(i + 1) return _is_alpha(b) or b == _SLASH or b == _BANG or b == _QUESTION - def _decode_entities(self, var raw: String) raises -> String: + def _decode_entities(self, var raw: String, base: Int) raises -> String: + # `base` is the byte offset of `raw`'s first byte within + # `self.src` (every caller passes a direct slice of the source), + # so strict-mode entity errors can point at the offending '&'. # Zero-copy fast path: most text and attribute values contain no # entities at all — hand the string back untouched. var has_amp = False @@ -610,18 +606,21 @@ struct HtmlTokenizer(Copyable, Movable): if semi == -1: if self.strict: raise self._strict_error( - "bare '&' without a terminated entity", self.pos + "bare '&' without a terminated entity", base + i ) # Malformed bare '&' — pass it through (liberal parsing). out += String(StringSlice(unsafe_from_utf8=bytes[i : i + 1])) i += 1 continue - var entity = self._entity_body(raw, i + 1, semi) + var entity = self._entity_body(raw, i + 1, semi, base + i) out += entity i = semi + 1 return out^ - def _entity_body(self, raw: String, start: Int, end: Int) raises -> String: + def _entity_body( + self, raw: String, start: Int, end: Int, amp_pos: Int + ) raises -> String: + # `amp_pos` is the byte offset of the entity's '&' in `self.src`. var bytes = raw.as_bytes() var out = String() if start < end and bytes[start] == _HASH: @@ -658,7 +657,7 @@ struct HtmlTokenizer(Copyable, Movable): if not valid: if self.strict: raise self._strict_error( - "malformed numeric character reference", self.pos + "malformed numeric character reference", amp_pos ) out += String("&") out += String(StringSlice(unsafe_from_utf8=bytes[start:end])) @@ -678,7 +677,7 @@ struct HtmlTokenizer(Copyable, Movable): return out^ # Unknown named entity — preserve it verbatim (liberal parsing). if self.strict: - raise self._strict_error("unknown entity &" + name + ";", self.pos) + raise self._strict_error("unknown entity &" + name + ";", amp_pos) return String("&") + name + String(";") def _read_name(mut self) -> String: @@ -715,6 +714,7 @@ struct HtmlTokenizer(Copyable, Movable): continue self._skip_space() var raw = String() + var raw_base = 0 var has_value = False if self.pos < self._len() and self._at(self.pos) == _EQUALS: has_value = True @@ -730,6 +730,7 @@ struct HtmlTokenizer(Copyable, Movable): ): self.pos += 1 raw = self._slice_to_string(vstart, self.pos) + raw_base = vstart if self.pos < self._len(): self.pos += 1 # closing quote else: @@ -741,10 +742,11 @@ struct HtmlTokenizer(Copyable, Movable): break self.pos += 1 raw = self._slice_to_string(vstart, self.pos) + raw_base = vstart # First occurrence wins for duplicate attributes (HTML rule). if name not in attrs: if has_value: - attrs[name] = self._decode_entities(raw^) + attrs[name] = self._decode_entities(raw^, raw_base) else: attrs[name] = String() @@ -791,7 +793,7 @@ struct HtmlTokenizer(Copyable, Movable): else: self.pos = n if _rawtext_decodes_entities(name): - text = self._decode_entities(text^) + text = self._decode_entities(text^, start) if text.byte_length() == 0: return HtmlEvent.end(name^) self._pending_end = name^ @@ -807,11 +809,12 @@ struct HtmlTokenizer(Copyable, Movable): while True: if self.pos >= self._len(): if self.strict and len(self._open) > 0: + # Point at the unclosed start tag, not the EOF. raise self._strict_error( "unclosed element <" + self._open[len(self._open) - 1] + "> at end of input", - self.pos, + self._open_pos[len(self._open_pos) - 1], ) return HtmlEvent.eof() if self._at(self.pos) != _LT or not self._tag_open_at(self.pos): @@ -825,7 +828,7 @@ struct HtmlTokenizer(Copyable, Movable): break self.pos += 1 var raw = self._slice_to_string(start, self.pos) - return HtmlEvent.text_event(self._decode_entities(raw^)) + return HtmlEvent.text_event(self._decode_entities(raw^, start)) # self.pos is at '<'. Dispatch on the next byte first so the # overwhelmingly common plain tags skip the literal probes. var next_b = self._at(self.pos + 1) @@ -908,8 +911,10 @@ struct HtmlTokenizer(Copyable, Movable): tag_start, ) _ = self._open.pop() + _ = self._open_pos.pop() return HtmlEvent.end(name^) # Start tag. + var start_tag_pos = self.pos self.pos += 1 var name = self._read_name() var attrs = self._read_attrs() @@ -930,4 +935,5 @@ struct HtmlTokenizer(Copyable, Movable): self._rawtext = name.copy() elif self.strict: self._open.append(name.copy()) + self._open_pos.append(start_tag_pos) return HtmlEvent.start(name^, attrs^) diff --git a/test/test_errors.mojo b/test/test_errors.mojo new file mode 100644 index 0000000..e45202e --- /dev/null +++ b/test/test_errors.mojo @@ -0,0 +1,296 @@ +from std.testing import assert_equal, assert_true, assert_raises, TestSuite + +from html.errors import line_col, parse_error +from html.tokenizer import HtmlTokenizer, EVENT_EOF + + +def _assert_lc(source: String, offset: Int, line: Int, col: Int) raises: + var lc = line_col(source.as_bytes(), offset) + assert_equal(lc[0], line) + assert_equal(lc[1], col) + + +def _msg(e: Error) -> String: + return String.write(e) + + +def _strict_drain(var source: String) raises: + var tok = HtmlTokenizer(source^, strict=True) + while True: + var event = tok.next_event() + if event.kind == EVENT_EOF: + break + + +# -------------------------------------------------------------------------- +# line_col unit tests — every documented edge case. +# -------------------------------------------------------------------------- + + +def test_line_col_offset_zero() raises: + _assert_lc("abc", 0, 1, 1) + + +def test_line_col_empty_source() raises: + _assert_lc("", 0, 1, 1) + _assert_lc("", 7, 1, 1) + + +def test_line_col_negative_offset_clamps() raises: + _assert_lc("abc", -5, 1, 1) + + +def test_line_col_middle_of_lines() raises: + # "ab\ncd": a=0 b=1 \n=2 c=3 d=4 + _assert_lc("ab\ncd", 1, 1, 2) + _assert_lc("ab\ncd", 3, 2, 1) + _assert_lc("ab\ncd", 4, 2, 2) + + +def test_line_col_offset_at_newline() raises: + # An offset AT a '\n' reports the line that newline terminates. + _assert_lc("ab\ncd", 2, 1, 3) + + +def test_line_col_offset_past_end_clamps() raises: + _assert_lc("ab\ncd", 5, 2, 3) # == len + _assert_lc("ab\ncd", 99, 2, 3) # > len + + +def test_line_col_crlf_no_phantom_column() raises: + # "ab\r\ncd": a=0 b=1 \r=2 \n=3 c=4 d=5. The byte after a CRLF is + # column 1 of the next line — the '\r' contributes no phantom column. + _assert_lc("ab\r\ncd", 4, 2, 1) + _assert_lc("ab\r\ncd", 5, 2, 2) + _assert_lc("ab\r\ncd", 2, 1, 3) # at the '\r' + _assert_lc("ab\r\ncd", 3, 1, 4) # at the '\n' of the CRLF + + +def test_line_col_consecutive_newlines() raises: + # "a\n\nb": a=0 \n=1 \n=2 b=3 + _assert_lc("a\n\nb", 2, 2, 1) + _assert_lc("a\n\nb", 3, 3, 1) + + +def test_line_col_trailing_newline() raises: + _assert_lc("ab\n", 3, 2, 1) + + +# -------------------------------------------------------------------------- +# parse_error unit tests — exact message format + snippet behavior. +# -------------------------------------------------------------------------- + + +def test_parse_error_exact_format() raises: + var e = parse_error("boom", "hello".as_bytes(), 2) + assert_equal(_msg(e), "boom at line 1, column 3: 'hello'") + + +def test_parse_error_multiline_source_single_line_message() raises: + # Offset 8 is the 'n' of "line2"; snippet is that line only — the + # message never embeds a newline. + var e = parse_error("bad", "line1\nline2\nline3".as_bytes(), 8) + assert_equal(_msg(e), "bad at line 2, column 3: 'line2'") + + +def test_parse_error_snippet_trims_whitespace() raises: + # " pad ": offset 4 is the 'a'. Column counts the raw bytes, the + # snippet is the trimmed line content. + var e = parse_error("boom", " pad ".as_bytes(), 4) + assert_equal(_msg(e), "boom at line 1, column 5: 'pad'") + + +def test_parse_error_offset_at_newline_snippet_is_ended_line() raises: + var e = parse_error("boom", "ab\ncd".as_bytes(), 2) + assert_equal(_msg(e), "boom at line 1, column 3: 'ab'") + + +def test_parse_error_crlf_line_has_no_stray_cr() raises: + # The snippet for a CRLF-terminated line drops the '\r' (whitespace + # trim), so the quoted snippet is clean. + var e = parse_error("boom", "ab\r\ncd".as_bytes(), 1) + assert_equal(_msg(e), "boom at line 1, column 2: 'ab'") + + +def test_parse_error_offset_past_end_clamps() raises: + var e = parse_error("eof", "ab\ncd".as_bytes(), 99) + assert_equal(_msg(e), "eof at line 2, column 3: 'cd'") + + +def test_parse_error_empty_source() raises: + var e = parse_error("boom", "".as_bytes(), 0) + assert_equal(_msg(e), "boom at line 1, column 1: ''") + + +def test_parse_error_truncates_both_sides() raises: + # An 80-byte line with the offending '!' at offset 40: the snippet is + # a 30-byte window centered on it, with '...' on both cut sides. + var source = String() + for _ in range(40): + source += "x" + source += "!" + for _ in range(39): + source += "x" + var e = parse_error("bang", source.as_bytes(), 40) + assert_equal( + _msg(e), + "bang at line 1, column 41: '...xxxxxxxxxxxxxxx!xxxxxxxxxxxxxx...'", + ) + + +def test_parse_error_truncates_right_only() raises: + var source = String("abcde") + for _ in range(75): + source += "x" + var e = parse_error("bang", source.as_bytes(), 0) + assert_equal( + _msg(e), + "bang at line 1, column 1: 'abcdexxxxxxxxxxxxxxxxxxxxxxxxx...'", + ) + + +def test_parse_error_truncates_left_only() raises: + var source = String() + for _ in range(79): + source += "x" + source += "!" + var e = parse_error("bang", source.as_bytes(), 79) + assert_equal( + _msg(e), + "bang at line 1, column 80: '...xxxxxxxxxxxxxxxxxxxxxxxxxxxxx!'", + ) + + +def test_parse_error_snippet_never_splits_utf8() raises: + # 40 'é' (2 bytes each, 80 bytes total); an offset landing mid-sequence + # still yields a valid-UTF-8 snippet: window edges are nudged off + # continuation bytes, so the snippet is whole codepoints only. + var source = String() + for _ in range(40): + source += "é" + var e = parse_error("bang", source.as_bytes(), 40) + var rendered = _msg(e) + assert_true(rendered.startswith("bang at line 1, column 41: '...")) + # 30-byte budget at a 2-byte-char boundary nudge = 15 whole 'é'. + var expected_snippet = String("'...") + for _ in range(15): + expected_snippet += "é" + expected_snippet += "...'" + assert_true(expected_snippet in rendered) + + +# -------------------------------------------------------------------------- +# Integration — strict-mode tokenizer errors report hand-verified positions. +# -------------------------------------------------------------------------- + + +def test_integration_mismatched_end_tag_position() raises: + # "
    \nx
    \n": the offending "" starts at + # byte 13 — line 2 (newline at 5), column 13-5 = 8. + with assert_raises( + contains=( + "mismatched end tag , expected " + " at line 2, column 8: 'x'" + ) + ): + _strict_drain("
    \nx
    \n    ") + + +def test_integration_stray_end_tag_position() raises: + # "

    x

    \n": the stray "" starts at byte 9 — line 2, + # column 1. + with assert_raises( + contains="stray end tag at line 2, column 1: ''" + ): + _strict_drain("

    x

    \n") + + +def test_integration_unclosed_element_points_at_start_tag() raises: + # "\n

    hello": the unclosed "

    " starts at byte 7 — line 2, + # column 1. The error points at the construct start, not the EOF. + with assert_raises( + contains=( + "unclosed element

    at end of input" + " at line 2, column 1: '

    hello'" + ) + ): + _strict_drain("\n

    hello") + + +def test_integration_unterminated_comment_points_at_open() raises: + # "

    \n