Parse errors carry line/column + context snippet

.github/workflows/test.yml

@@ -29,4 +29,4 @@ jobs:
           git diff --exit-code || (echo "::error::Run 'pixi run fmt' — sources are not mojo-format clean" && exit 1)
 
       - name: Tests
-        run: .venv/bin/mojo run -I src test/test_tokenizer.mojo && .venv/bin/mojo run -I src test/test_extract.mojo
+        run: .venv/bin/mojo run -I src test/test_tokenizer.mojo && .venv/bin/mojo run -I src test/test_extract.mojo && .venv/bin/mojo run -I src test/test_errors.mojo

CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## Unreleased
+
+- New `html.errors` module (exported from the package), following the
+  shared mojo-* suite pattern: `line_col(source, offset)` maps a byte
+  offset to a 1-based (line, column) pair — the column is the 1-based
+  BYTE offset within the line, no UTF-8 decoding — and
+  `parse_error(msg, source, offset)` builds an `Error` reading
+  `<msg> at line <L>, column <C>: '<snippet>'`, where the snippet is up
+  to ~30 bytes of the offending line centered on the column,
+  whitespace-trimmed, with `...` where truncated, and never multi-line.
+- Strict-mode tokenizer errors now carry that position + snippet
+  (previously a bare `(line L, column C)` suffix with no snippet), with
+  more useful offsets: unclosed-element-at-EOF errors point at the
+  unclosed start tag instead of the end of input, and entity errors
+  (unknown entity, bare `&`, malformed numeric reference) point at the
+  offending `&` — in text, attribute values, and escapable raw text —
+  instead of the tokenizer's position after the run.
+- No mechanism change: the tokenizer still `raise`s a plain `Error`, the
+  `mojo-html [strict]: ` prefix is unchanged, and existing
+  `contains=`-style message checks keep matching. The default liberal
+  mode still never raises on malformed markup.
+
 ## 0.1.0 — 2026-07-05
 
 Initial release. Liberal HTML tokenizer (void elements, raw-text

pixi.toml

@@ -7,7 +7,7 @@ platforms = ["linux-64", "osx-arm64"]
 version = "0.1.0"
 
 [tasks]
-test = "mojo run -I src test/test_tokenizer.mojo && mojo run -I src test/test_extract.mojo"
+test = "mojo run -I src test/test_tokenizer.mojo && mojo run -I src test/test_extract.mojo && mojo run -I src test/test_errors.mojo"
 demo = "mojo run -I src examples/extract_article.mojo test/data/substack_article.html"
 bench = "mojo build -I src bench/bench_parse.mojo -o .bench_parse && ./.bench_parse"
 fmt = "mojo format src/ test/ examples/ bench/"

src/html/__init__.mojo

@@ -1,5 +1,6 @@
 """Liberal HTML parsing and readable-text extraction for Mojo (mojo-html)."""
 
+from html.errors import line_col, parse_error
 from html.extract import (
     extract,
     main_text,

src/html/errors.mojo

@@ -0,0 +1,135 @@
+"""Position-aware parse errors.
+
+`line_col` maps a byte offset in a source buffer to a 1-based
+(line, column) pair, and `parse_error` builds an `Error` whose message
+carries that position plus a short snippet of the offending line:
+
+    <msg> at line <L>, column <C>: '<snippet>'
+
+Positions are byte-based: the column is the 1-based BYTE offset within
+the line, not a codepoint or display column. That keeps the computation
+deterministic and free of UTF-8 decode cost; for ASCII-heavy markup the
+byte column and the visual column coincide.
+
+This module is the error-reporting pattern shared across the mojo-*
+parser suite.
+"""
+
+comptime _LF = UInt8(0x0A)
+comptime _CR = UInt8(0x0D)
+
+# Snippet size budget, in bytes, before the `...` truncation markers are
+# added. Wide enough to show meaningful context, narrow enough that error
+# messages stay one readable line.
+comptime _SNIPPET_BUDGET = 30
+
+
+def _is_ws(b: UInt8) -> Bool:
+    return b == 0x20 or b == 0x09 or b == _CR or b == _LF
+
+
+def line_col(source: Span[UInt8, _], offset: Int) -> Tuple[Int, Int]:
+    """1-based (line, column) of byte `offset` in `source`.
+
+    The column is the 1-based BYTE offset within the line — codepoints
+    are never decoded, so the result is cheap and deterministic even on
+    invalid UTF-8. Only LF (0x0A) terminates a line: after a CRLF
+    sequence the next byte is column 1 of the next line, with no phantom
+    column contributed by the CR. An offset pointing AT an LF reports
+    the line that newline terminates (column = line length + 1). Offsets
+    past the end of `source` clamp to the end; an empty source yields
+    (1, 1).
+    """
+    var limit = offset
+    if limit > len(source):
+        limit = len(source)
+    if limit < 0:
+        limit = 0
+    var line = 1
+    var last_nl = -1
+    for i in range(limit):
+        if source[i] == _LF:
+            line += 1
+            last_nl = i
+    return (line, limit - last_nl)
+
+
+def _snippet(source: Span[UInt8, _], offset: Int) -> String:
+    """Up to ~`_SNIPPET_BUDGET` bytes of the line containing `offset`.
+
+    The line is trimmed of leading/trailing whitespace (which also drops
+    the CR of a CRLF line ending), then windowed around the offset with
+    `...` marking each side that was cut. Window edges are nudged off
+    UTF-8 continuation bytes so the result is always valid UTF-8. The
+    result never contains a newline.
+    """
+    var n = len(source)
+    var anchor = offset
+    if anchor > n:
+        anchor = n
+    if anchor < 0:
+        anchor = 0
+    # Line bounds around the anchor; an anchor sitting AT an LF belongs
+    # to the line that newline terminates.
+    var line_start = anchor
+    while line_start > 0 and source[line_start - 1] != _LF:
+        line_start -= 1
+    var line_end = anchor
+    while line_end < n and source[line_end] != _LF:
+        line_end += 1
+    # Trim surrounding whitespace.
+    while line_start < line_end and _is_ws(source[line_start]):
+        line_start += 1
+    while line_end > line_start and _is_ws(source[line_end - 1]):
+        line_end -= 1
+    var win_start = line_start
+    var win_end = line_end
+    var cut_left = False
+    var cut_right = False
+    if line_end - line_start > _SNIPPET_BUDGET:
+        win_start = anchor - _SNIPPET_BUDGET // 2
+        if win_start > line_end - _SNIPPET_BUDGET:
+            win_start = line_end - _SNIPPET_BUDGET
+        if win_start < line_start:
+            win_start = line_start
+        win_end = win_start + _SNIPPET_BUDGET
+        # Never split a multi-byte UTF-8 sequence at a window edge.
+        while win_start < win_end and (source[win_start] & 0xC0) == 0x80:
+            win_start += 1
+        while win_end < line_end and (source[win_end] & 0xC0) == 0x80:
+            win_end += 1
+        cut_left = win_start > line_start
+        cut_right = win_end < line_end
+    var out = String()
+    if cut_left:
+        out += "..."
+    out += String(StringSlice(unsafe_from_utf8=source[win_start:win_end]))
+    if cut_right:
+        out += "..."
+    return out^
+
+
+def parse_error(msg: String, source: Span[UInt8, _], offset: Int) -> Error:
+    """An `Error` locating `msg` at byte `offset` of `source`.
+
+    The message is exactly:
+
+        <msg> at line <L>, column <C>: '<snippet>'
+
+    where line/column come from `line_col` (1-based; column is a byte
+    offset within the line) and the snippet is the offending line,
+    whitespace-trimmed and truncated to ~30 bytes centered on the
+    column, with `...` where truncated. The message never contains a
+    newline, so it renders on one line in test output and logs.
+    """
+    var lc = line_col(source, offset)
+    return Error(
+        msg
+        + " at line "
+        + String(lc[0])
+        + ", column "
+        + String(lc[1])
+        + ": '"
+        + _snippet(source, offset)
+        + "'"
+    )

src/html/tokenizer.mojo

@@ -13,6 +13,8 @@ markup in its default liberal mode. Structural recovery (unclosed <p>,
 <li>, crossed tags) is the mapping layer's job — see extract.mojo.
 """
 
+from html.errors import parse_error
+
 comptime EVENT_START = 0
 comptime EVENT_END = 1
 comptime EVENT_TEXT = 2
@@ -486,9 +488,10 @@ struct HtmlTokenizer(Copyable, Movable):
 
     With `strict=True` the tokenizer reports well-formedness problems —
     mismatched or stray end tags, elements left open at EOF, malformed
-    or unknown entities — as errors with a line/column location instead
-    of recovering liberally. Useful for linting HTML you produce; leave
-    it off for pages you merely consume.
+    or unknown entities — as errors with a line/column location and a
+    snippet of the offending line instead of recovering liberally.
+    Useful for linting HTML you produce; leave it off for pages you
+    merely consume.
     """
 
     var src: String
@@ -498,6 +501,10 @@ struct HtmlTokenizer(Copyable, Movable):
     var _has_pending_end: Bool
     var _rawtext: String
     var _open: List[String]
+    # Byte offset of each open element's start tag, parallel to `_open`,
+    # so an unclosed-element error can point at the construct start
+    # rather than the (useless) EOF position.
+    var _open_pos: List[Int]
 
     def __init__(out self, var source: String, *, strict: Bool = False) raises:
         self.src = normalize_encoding(source^)
@@ -507,31 +514,17 @@ struct HtmlTokenizer(Copyable, Movable):
         self._has_pending_end = False
         self._rawtext = String()
         self._open = List[String]()
+        self._open_pos = List[Int]()
 
-    def _location(self, p: Int) -> String:
-        """Human-readable "line L, column C" for byte offset `p`.
+    def _strict_error(self, msg: String, p: Int) -> Error:
+        """Strict-mode error locating `msg` at byte offset `p`.
 
-        Computed lazily (only on error paths), so the happy path pays
-        nothing for location tracking.
+        Position and snippet come from `html.errors.parse_error`, so the
+        message reads `mojo-html [strict]: <msg> at line <L>, column <C>:
+        '<snippet>'`. Computed lazily (only on error paths), so the happy
+        path pays nothing for location tracking.
         """
-        var bytes = self.src.as_bytes()
-        var line = 1
-        var col = 1
-        var limit = p
-        if limit > len(bytes):
-            limit = len(bytes)
-        for i in range(limit):
-            if bytes[i] == 0x0A:
-                line += 1
-                col = 1
-            else:
-                col += 1
-        return String("line ") + String(line) + ", column " + String(col)
-
-    def _strict_error(self, msg: String, p: Int) -> Error:
-        return Error(
-            "mojo-html [strict]: " + msg + " (" + self._location(p) + ")"
-        )
+        return parse_error("mojo-html [strict]: " + msg, self.src.as_bytes(), p)
 
     def _len(self) -> Int:
         return self.src.byte_length()
@@ -577,7 +570,10 @@ struct HtmlTokenizer(Copyable, Movable):
         var b = self._at(i + 1)
         return _is_alpha(b) or b == _SLASH or b == _BANG or b == _QUESTION
 
-    def _decode_entities(self, var raw: String) raises -> String:
+    def _decode_entities(self, var raw: String, base: Int) raises -> String:
+        # `base` is the byte offset of `raw`'s first byte within
+        # `self.src` (every caller passes a direct slice of the source),
+        # so strict-mode entity errors can point at the offending '&'.
         # Zero-copy fast path: most text and attribute values contain no
         # entities at all — hand the string back untouched.
         var has_amp = False
@@ -610,18 +606,21 @@ struct HtmlTokenizer(Copyable, Movable):
             if semi == -1:
                 if self.strict:
                     raise self._strict_error(
-                        "bare '&' without a terminated entity", self.pos
+                        "bare '&' without a terminated entity", base + i
                     )
                 # Malformed bare '&' — pass it through (liberal parsing).
                 out += String(StringSlice(unsafe_from_utf8=bytes[i : i + 1]))
                 i += 1
                 continue
-            var entity = self._entity_body(raw, i + 1, semi)
+            var entity = self._entity_body(raw, i + 1, semi, base + i)
             out += entity
             i = semi + 1
         return out^
 
-    def _entity_body(self, raw: String, start: Int, end: Int) raises -> String:
+    def _entity_body(
+        self, raw: String, start: Int, end: Int, amp_pos: Int
+    ) raises -> String:
+        # `amp_pos` is the byte offset of the entity's '&' in `self.src`.
         var bytes = raw.as_bytes()
         var out = String()
         if start < end and bytes[start] == _HASH:
@@ -658,7 +657,7 @@ struct HtmlTokenizer(Copyable, Movable):
             if not valid:
                 if self.strict:
                     raise self._strict_error(
-                        "malformed numeric character reference", self.pos
+                        "malformed numeric character reference", amp_pos
                     )
                 out += String("&")
                 out += String(StringSlice(unsafe_from_utf8=bytes[start:end]))
@@ -678,7 +677,7 @@ struct HtmlTokenizer(Copyable, Movable):
             return out^
         # Unknown named entity — preserve it verbatim (liberal parsing).
         if self.strict:
-            raise self._strict_error("unknown entity &" + name + ";", self.pos)
+            raise self._strict_error("unknown entity &" + name + ";", amp_pos)
         return String("&") + name + String(";")
 
     def _read_name(mut self) -> String:
@@ -715,6 +714,7 @@ struct HtmlTokenizer(Copyable, Movable):
                 continue
             self._skip_space()
             var raw = String()
+            var raw_base = 0
             var has_value = False
             if self.pos < self._len() and self._at(self.pos) == _EQUALS:
                 has_value = True
@@ -730,6 +730,7 @@ struct HtmlTokenizer(Copyable, Movable):
                         ):
                             self.pos += 1
                         raw = self._slice_to_string(vstart, self.pos)
+                        raw_base = vstart
                         if self.pos < self._len():
                             self.pos += 1  # closing quote
                     else:
@@ -741,10 +742,11 @@ struct HtmlTokenizer(Copyable, Movable):
                                 break
                             self.pos += 1
                         raw = self._slice_to_string(vstart, self.pos)
+                        raw_base = vstart
             # First occurrence wins for duplicate attributes (HTML rule).
             if name not in attrs:
                 if has_value:
-                    attrs[name] = self._decode_entities(raw^)
+                    attrs[name] = self._decode_entities(raw^, raw_base)
                 else:
                     attrs[name] = String()
 
@@ -791,7 +793,7 @@ struct HtmlTokenizer(Copyable, Movable):
             else:
                 self.pos = n
         if _rawtext_decodes_entities(name):
-            text = self._decode_entities(text^)
+            text = self._decode_entities(text^, start)
         if text.byte_length() == 0:
             return HtmlEvent.end(name^)
         self._pending_end = name^
@@ -807,11 +809,12 @@ struct HtmlTokenizer(Copyable, Movable):
         while True:
             if self.pos >= self._len():
                 if self.strict and len(self._open) > 0:
+                    # Point at the unclosed start tag, not the EOF.
                     raise self._strict_error(
                         "unclosed element <"
                         + self._open[len(self._open) - 1]
                         + "> at end of input",
-                        self.pos,
+                        self._open_pos[len(self._open_pos) - 1],
                     )
                 return HtmlEvent.eof()
             if self._at(self.pos) != _LT or not self._tag_open_at(self.pos):
@@ -825,7 +828,7 @@ struct HtmlTokenizer(Copyable, Movable):
                         break
                     self.pos += 1
                 var raw = self._slice_to_string(start, self.pos)
-                return HtmlEvent.text_event(self._decode_entities(raw^))
+                return HtmlEvent.text_event(self._decode_entities(raw^, start))
             # self.pos is at '<'. Dispatch on the next byte first so the
             # overwhelmingly common plain tags skip the literal probes.
             var next_b = self._at(self.pos + 1)
@@ -908,8 +911,10 @@ struct HtmlTokenizer(Copyable, Movable):
                             tag_start,
                         )
                     _ = self._open.pop()
+                    _ = self._open_pos.pop()
                 return HtmlEvent.end(name^)
             # Start tag.
+            var start_tag_pos = self.pos
             self.pos += 1
             var name = self._read_name()
             var attrs = self._read_attrs()
@@ -930,4 +935,5 @@ struct HtmlTokenizer(Copyable, Movable):
                 self._rawtext = name.copy()
             elif self.strict:
                 self._open.append(name.copy())
+                self._open_pos.append(start_tag_pos)
             return HtmlEvent.start(name^, attrs^)