fix(ast): keep source positions through rawHtml reparse

.sampo/changesets/raw-html-mdx-component-overrides.md

@@ -0,0 +1,6 @@
+---
+cargo/satteri-ast: patch
+npm/satteri: patch
+---
+
+Fixes `export const components` being ignored in MDX when `rawHtml` and `optimizeStatic` are both enabled.

.sampo/changesets/raw-html-positions.md

@@ -0,0 +1,6 @@
+---
+cargo/satteri-ast: patch
+npm/satteri: patch
+---
+
+Fixes `rawHtml` dropping node positions, so nodes that came from Markdown keep their source positions like `rehype-raw` does.

crates/satteri-ast/src/hast/render.rs

@@ -39,66 +39,43 @@ pub fn render_node(
     in_raw_text: bool,
     in_svg: bool,
 ) {
-    render_node_inner(node_id, view, out, in_raw_text, in_svg, None, 0);
+    render_node_inner(node_id, view, out, in_raw_text, in_svg, 0);
 }
 
-/// Raw-HTML reparse hook: receives the output buffer and the MDX node's id.
-pub(crate) type OnMdx<'a> = dyn FnMut(&mut String, u32) + 'a;
-
-/// MDX nodes have no HTML representation: `on_mdx` decides what to emit for
-/// them; `None` skips them.
-pub(crate) fn render_node_inner<'cb>(
+fn render_node_inner(
     node_id: u32,
     view: &Arena<Hast>,
     out: &mut String,
     in_raw_text: bool,
     in_svg: bool,
-    on_mdx: Option<&mut OnMdx<'cb>>,
     depth: u32,
 ) {
     crate::stack::with_headroom(depth, || {
-        render_node_at(node_id, view, out, in_raw_text, in_svg, on_mdx, depth);
+        render_node_at(node_id, view, out, in_raw_text, in_svg, depth);
     });
 }
 
-fn render_node_at<'cb>(
+fn render_node_at(
     node_id: u32,
     view: &Arena<Hast>,
     out: &mut String,
     in_raw_text: bool,
     in_svg: bool,
-    mut on_mdx: Option<&mut OnMdx<'cb>>,
     depth: u32,
 ) {
     let node = view.get_node(node_id);
 
     let Some(node_type) = HastNodeType::from_u8(node.node_type) else {
         for &child_id in view.get_children(node_id) {
-            render_node_inner(
-                child_id,
-                view,
-                out,
-                in_raw_text,
-                in_svg,
-                on_mdx.as_deref_mut(),
-                depth + 1,
-            );
+            render_node_inner(child_id, view, out, in_raw_text, in_svg, depth + 1);
         }
         return;
     };
 
     match node_type {
         HastNodeType::Root => {
             for &child_id in view.get_children(node_id) {
-                render_node_inner(
-                    child_id,
-                    view,
-                    out,
-                    in_raw_text,
-                    in_svg,
-                    on_mdx.as_deref_mut(),
-                    depth + 1,
-                );
+                render_node_inner(child_id, view, out, in_raw_text, in_svg, depth + 1);
             }
         }
 
@@ -153,7 +130,6 @@ fn render_node_at<'cb>(
                         out,
                         child_in_raw_text,
                         element_in_svg,
-                        on_mdx.as_deref_mut(),
                         depth + 1,
                     );
                 }
@@ -200,15 +176,12 @@ fn render_node_at<'cb>(
             }
         }
 
+        // MDX nodes have no HTML form.
         HastNodeType::MdxJsxElement
         | HastNodeType::MdxJsxTextElement
         | HastNodeType::MdxFlowExpression
         | HastNodeType::MdxTextExpression
-        | HastNodeType::MdxEsm => {
-            if let Some(cb) = on_mdx.as_mut() {
-                cb(out, node_id);
-            }
-        }
+        | HastNodeType::MdxEsm => {}
     }
 }
 

packages/satteri/src/compile.ts

@@ -591,7 +591,9 @@ export interface Features {
    * (re-emitted verbatim on stringify). With `rawHtml: true`, the tree is
    * reparsed so raw HTML becomes structured `element`/`text`/`comment` nodes
    * with normalized properties, including tags that open in one raw block
-   * and close in another. Positions are not preserved through the reparse.
+   * and close in another. Nodes that came from Markdown keep their positions,
+   * as does a raw block that is exactly one element; nodes nested inside raw
+   * HTML have none.
    */
   rawHtml?: boolean;
 }

packages/satteri/test/raw-html-conformance.test.ts

@@ -159,3 +159,54 @@ describe("rawHtml conformance vs rehype-raw", () => {
     });
   }
 });
+
+/** Flatten to `path → span`, so two trees compare node position by node position. */
+function spans(node: HastNode, path = "", out: Array<[string, string | null]> = []) {
+  const p = node.position;
+  out.push([path, p ? `${p.start.offset}..${p.end ? p.end.offset : "?"}` : null]);
+  if ("children" in node && node.children) {
+    (node.children as HastNode[]).forEach((child, i) => spans(child, `${path}/${i}`, out));
+  }
+  return out;
+}
+
+// The root is excluded: rehype-raw collapses it to a zero-width span at 1:1.
+describe("rawHtml position conformance vs rehype-raw", () => {
+  for (const { name, md } of cases) {
+    test(`every position kept matches the reference: ${name}`, () => {
+      const reference = new Map(spans(referenceTree(md)));
+      for (const [path, span] of spans(markdownToHast(md, { features: { rawHtml: true } }))) {
+        if (path === "" || span === null) continue;
+        expect([path, span]).toEqual([path, reference.get(path)]);
+      }
+    });
+  }
+
+  test("nodes that came from Markdown keep their positions", () => {
+    const md = "# Hi\n\n<div>raw</div>\n\nA [link](x) here.\n";
+    const reference = new Map(spans(referenceTree(md)));
+    const kept = spans(markdownToHast(md, { features: { rawHtml: true } })).filter(
+      ([path, span]) => path !== "" && span !== null,
+    );
+    expect(kept.length).toBeGreaterThan(4);
+    for (const [path, span] of kept) expect([path, span]).toEqual([path, reference.get(path)]);
+  });
+
+  test("a raw block that is one element gets that element's span", () => {
+    const md = "text\n\n<div><em>x</em></div>\n";
+    const reference = new Map(spans(referenceTree(md)));
+    const ours = new Map(spans(markdownToHast(md, { features: { rawHtml: true } })));
+    expect(ours.get("/2")).toBe(reference.get("/2"));
+    expect(ours.get("/2")).toBe("6..27");
+  });
+
+  // Spans within a raw block need per-token offsets, which html5ever does not expose.
+  test("nodes nested inside raw HTML carry no position", () => {
+    const tree = markdownToHast("text\n\n<div><em>x</em></div>\n", {
+      features: { rawHtml: true },
+    });
+    const inner = spans(tree).filter(([path]) => path.startsWith("/2/"));
+    expect(inner.length).toBeGreaterThan(0);
+    for (const [, span] of inner) expect(span).toBeNull();
+  });
+});

packages/satteri/test/raw-html-pipelines.test.ts

@@ -1,5 +1,12 @@
 import { test, expect } from "vitest";
-import { markdownToHtml, markdownToJs, defineMdastPlugin, defineHastPlugin } from "../src/index.js";
+import {
+  markdownToHtml,
+  markdownToHast,
+  markdownToJs,
+  mdxToJs,
+  defineMdastPlugin,
+  defineHastPlugin,
+} from "../src/index.js";
 
 // `rawHtml` is applied during MDAST→HAST conversion, so every pipeline (the
 // no-plugin fast path, the MDAST-plugin fused tail, and the full
@@ -131,6 +138,27 @@ test("plugin-spliced raw HTML is reparsed too", () => {
   expect(html).toContain('<aside class="n m">hi</aside>');
 });
 
+test("rawHtml keeps the document span on the root", () => {
+  const md = "# Hi\n\n<div>hello</div>\n\nBye\n";
+  expect(markdownToHast(md, { features: { rawHtml: true } }).position).toEqual(
+    markdownToHast(md).position,
+  );
+});
+
+// Offsets are UTF-16 code units, so only a multibyte document exposes a lost source.
+test("rawHtml root offsets stay in UTF-16 code units", () => {
+  const md = "# 👋 héllo\n\n<div>x</div>\n\nBye\n";
+  expect(markdownToHast(md, { features: { rawHtml: true } }).position?.end.offset).toBe(md.length);
+});
+
+test("rawHtml keeps component overrides visible to optimizeStatic", () => {
+  const mdx = 'export const components = { h1: "h2" };\n\n# Hello\n\nSome text.\n';
+  const optimizeStatic = { component: "Fragment", prop: "set:html" };
+  expect(mdxToJs(mdx, { optimizeStatic, features: { rawHtml: true } }).code).toBe(
+    mdxToJs(mdx, { optimizeStatic }).code,
+  );
+});
+
 test("rawHtml keeps namespaced SVG attributes through the round trip", () => {
   const sprite =
     '<svg xmlns:xlink="http://www.w3.org/1999/xlink"><use xlink:href="#icon"/></svg>\n';

website/content/docs/entry-points.md

@@ -148,4 +148,4 @@ const tree = markdownToHast(`<div class="note">\n\n**hi**\n\n</div>`, {
 // <div> is now a real element wrapping the parsed <p><strong>hi</strong></p>
 ```
 
-The whole tree is reparsed through the HTML parser, so a tag opened in one raw block and closed in another is resolved against the surrounding Markdown. Positions are not preserved through the reparse.
+The whole tree is reparsed through the HTML parser, so a tag opened in one raw block and closed in another is resolved against the surrounding Markdown. Nodes that came from Markdown keep their positions through the reparse, as does a raw block that is exactly one element; nodes nested inside raw HTML have none.

website/content/docs/features.md

@@ -286,7 +286,7 @@ rawHtml?: boolean
 
 By default, raw HTML embedded in Markdown is kept as opaque `raw` nodes and re-emitted verbatim. `rawHtml: true` reparses it into real HAST element, text, and comment nodes. The reparse runs during the mdast→hast conversion, so `markdownToHast`, `markdownToHtml`, and the plugin pipelines all reparse identically, and HAST plugins always see the reparsed elements.
 
-The whole tree goes through the HTML parser, so a tag opened in one raw block and closed in another is resolved against the surrounding Markdown. Attributes are normalised into typed hast properties (`class` → `className: ["…"]`, `disabled` → `true`, `tabindex` → a number, `data-foo-bar` → `dataFooBar`). In MDX, JSX elements and expressions are preserved in place while the raw HTML around them is still resolved. Positions are not preserved through the reparse.
+The whole tree goes through the HTML parser, so a tag opened in one raw block and closed in another is resolved against the surrounding Markdown. Attributes are normalised into typed hast properties (`class` → `className: ["…"]`, `disabled` → `true`, `tabindex` → a number, `data-foo-bar` → `dataFooBar`). In MDX, JSX elements and expressions are preserved in place while the raw HTML around them is still resolved. Nodes that came from Markdown keep their positions through the reparse, as does a raw block that is exactly one element; nodes nested inside raw HTML have none.
 
 ```js
 import { markdownToHast } from "satteri";