[v3.1.0] List rendering robustness fixes (itemize / enumerate)

.npmignore

@@ -6,3 +6,4 @@ webpack.config.js
 webpack.dev.js
 .gitignore
 .yarnignore
+/pr-specs

README.md

@@ -970,10 +970,10 @@ The `MathpixMarkdown` React element accepts the following props:
 | `include_linearmath`     | boolean&nbsp;*`false`*       | outputs linearmath `<linearmath style="display: none">...</linearmath>`                                            |
 | `include_latex`          | boolean&nbsp;*`true`*        | outputs latex `<latex style="display: none">...</latex>`                                                          |
 | `include_svg`            | boolean&nbsp;*`true`*        | outputs svg `<svg>...</svg>`                                                                                      |
-| `include_tsv`            | boolean&nbsp;*`false`*       | outputs tsv `<tsv style="display: none">...</tsv>`                                                                |
-| `include_csv`            | boolean&nbsp;*`false`*       | outputs csv `<csv style="display: none">...</csv>`                                                                |
+| `include_tsv`            | boolean&nbsp;*`false`*       | outputs tsv `<tsv style="display: none">...</tsv>`. A link contributes its `href` and an image its `src` in place of the visible text — these formats hold no markup; use `include_table_markdown` for the label. Whatever else the cell holds is still exported beside it, space-separated, so a cell is not an address even when it starts with one (`[a](http://u) tail` → `http://u tail`) |
+| `include_csv`            | boolean&nbsp;*`false`*       | outputs csv `<csv style="display: none">...</csv>`. Same link/image rule as `include_tsv`                          |
 | `include_table_html`     | boolean&nbsp;*`true`*        | outputs html table `<table>...</table>`                                                                           |
-| `include_table_markdown` | boolean&nbsp;*`false`*       | outputs markdown table `<table-markdown>...</table-markdown>`                                                     |
+| `include_table_markdown` | boolean&nbsp;*`false`*       | outputs markdown table `<table-markdown>...</table-markdown>`. A link keeps its address as written; see [Security notes on the Markdown exports](#security-notes-on-the-markdown-exports) |
 | `include_smiles`         | boolean&nbsp;*`false`*       | outputs smiles `<smiles>...</smiles>`                                                                             |
 | `tsv_separators`         | `{column: '\t', row: '\n'}`  | Separators for tsv tables                                                                                         |
 | `csv_separators`         | `{column: ',', row: '\n', toQuoteAllFields: false}` | Separators for csv tables. If `toQuoteAllFields=true` - all fields will be enclosed in double quotes|
@@ -984,6 +984,32 @@ The `MathpixMarkdown` React element accepts the following props:
 | `table_markdown`         | `{math_as_ascii: false, math_inline_delimiters: ['$','$']}`| By default, math goes into Markdown tables as latex and is enclosed in `$...$` delimiters. If `math_as_ascii` is set to `true`, then math will be represented as asciimath | 
 | `skipMathToHtml`         | boolean&nbsp;*`false`*       | When `true`, skips SVG serialization and `token.mathEquation` storage. Overrides `include_svg`; other MathJax outputs (`mathml`, `asciimath`, `linearmath`, etc.) still respect their own `include_*` flags. Intended for callers that walk the token tree directly and never read the serialized math HTML. |
 
+### Security notes on the Markdown exports
+
+`include_table_markdown`, `include_tsv` and `include_csv` return text, so the reader re-parses it.
+
+- A rejected link (`javascript:`, `data:`, or anything a custom `validateLink` refuses) produces no `<a>` and stays as the literal `[label](address)` — in the HTML output and in the export alike. A reader with a permissive validator will make it a link again, so **validating the scheme is the reader's job**.
+- An accepted link is exported with its address as written: escaping guards the Markdown syntax, not the URL scheme. `tsv`/`csv` carry the address without the label.
+- A cell's inline markup is exported as written, raw HTML included — `<b onclick="…">x</b>` in a cell, or in a link label, reaches the export verbatim. Escaping is narrow: `\`, `[` and `]` inside a link label in the `table-markdown` export, nothing at all in a plain cell or in `tsv`/`csv`. A reader that parses the result with HTML enabled will render it, so **sanitising HTML is the reader's job** as well. This branch widened only how much of a link label is exported, not what is escaped.
+
+Both are pinned by tests. If the reader cannot validate, treat the exported text as untrusted input or render from the HTML output.
+
+### Diagnostics
+
+Malformed LaTeX makes the list rules report to the global `console.warn`, prefixed `[list]`, `[list-state]`, `[env]` or `[mmd]`. There is **no option to silence or redirect them** — a server that renders untrusted input at volume should expect them in its log.
+
+- They are informational. `[list] list rule failed` means one list was left as literal text while the rest of the document rendered. A marker asking for more room than the `20em` cap allows is **not** reported: the reserve is generous enough that the chain usually still fits, so the line said nothing a consumer could act on.
+- Each distinct case is reported once per render, capped at 40 per family and 200 in total; past either cap one line says the rest are silent.
+- A failure never throws out of `markdownToHTML` — the rule declines and the document renders — so a warning is the only signal that a list degraded.
+- Deduplication counts per render, which is per call to `markdownToHTML`. A consumer driving `md.block.parse` or `md.inline.parse` directly never reaches the reset, so its keys accumulate until the per-family cap (40) reports and stops that family; the other families keep speaking.
+- MathJax reports invalid math on its own channel, prefixed `[TexConvert]`, not deduplicated and not covered by the caps above. Malformed math written inside a list wrapper env now reaches it, where `3.0.1` never built the wrapper and so never sent it.
+
+### Three notes for consumers who reach past the public API
+
+- **Toggle a rule, do not swap its `fn`.** The footnote rules cache which block rules terminate them, and that cache is dropped when markdown-it invalidates its own rule cache — which `enable`, `disable` and `ruler.at` do. Assigning `rule.fn` inside `md.block.ruler.__rules__` does not, so the cache would keep pointing at your old function. Use `md.block.ruler.at(name, fn)`.
+- **Keep `env` small.** A list parse snapshots every own string key of the `env` you pass, so that it can be put back exactly, and it does so once per line that opens a list. Measured on 200 list units: 62 ms with a plain `env`, 164 ms with a thousand extra keys on it.
+- **Your `env` comes back as the parse found it, with one shape difference.** A key the parse added comes back as an own key holding `undefined` rather than removed, so `'key' in env` answers `true` where `env.key` is `undefined` — `delete` drops the object into dictionary mode, and this runs per line that opens a list. Read values, not key presence. The parse's own caches live under symbols, invisible to `Object.keys` and `JSON.stringify`.
+
 ### TOutputMathJax
 
 |                      | type&nbsp;*`default`*        |  description                                                                                                       |
@@ -1151,6 +1177,11 @@ Build the es5 file for node.
 $ npm run build
 ```
 
+`lib/` and `es5/` are committed, and only `npm run build` produces what is committed. `npm run compile`
+is for iterating: it skips `es5/`, and it prints tuple types in `.d.ts` on one line where the build's
+`ts-loader` prints them across several — so its output shows up as a diff in files no source change
+touched.
+
 # Testing
 
 ```shell

doc/changelog.md

@@ -1,4 +1,39 @@
-# July 2026
+# August 2026
+
+## [3.1.0] - List rendering robustness fixes
+
+Malformed LaTeX lists rendered incorrectly in several unrelated ways. Details and measurements: `pr-specs/2026-07-list-rendering-robustness.md`, `pr-specs/2026-07-code-block-font-scaling.md`.
+
+A minor by decision: strict semver would call it a major, `tsv`/`csv` cells and `data-padding-inline-start` being documented output that changed shape. Pin the exact version if you need to review the diff before taking it.
+
+### Breaking changes
+
+- **`data-padding-inline-start` carries its unit** (`"56"` → `"4.23em"`) and appears on nested lists too. Parse it as an em value; a value we cannot parse is ignored with a warning.
+- **`tsv`/`csv` cells are not addresses.** An image exports its src alone (`![the alt](i.png)` → `i.png`); a link exports its href and then the rest of the cell (`[a](http://u) tail` → `http://u tail`, was `http://u`).
+- **`table-markdown`/`tsv`/`csv` for a cell holding a one-line list**: a sublist's item bodies now appear, and a flat list exports what sits before its first `\item`.
+- **A `ul`/`ol` without our `.itemize`/`.enumerate` class indents `2.5em` inside `#preview-content`/`#setText`**, where the UA `40px` applied — core-markdown lists, and lists passed as HTML under `htmlTags: true`. The rule outranks a consumer rule at class specificity. A TOC keeps `padding: 0` under `#toc_container` or `.table-of-contents`; copied elsewhere, give your container `padding: 0`.
+- **A failing list rule warns and returns degraded HTML** instead of throwing, so a `catch` around `markdownToHTML` for that case will not fire.
+- **New `console.warn` diagnostics**, deduplicated per render, with no option to silence them.
+- **Your `env` comes back as the list rule found it**, with one shape difference: a key the parse added comes back holding `undefined` rather than removed, so read values, not key presence. The rule's own caches sit under symbols, invisible to `Object.keys` and `JSON.stringify`.
+- **Deep imports.** `ClearTableNumbers`/`ClearFigureNumbers` → `clearTableNumbers`/`clearFigureNumbers` in `lib/markdown/common/caption-counters`; `getListLevelState` and `ListItemsBlock` are gone (read the depth through `getListDepth`); `ListItemsResult`/`ListInlineContext` dropped `padding` and require `openTokens`/`allListTokens`; `skipBackticks` answers past an unmatched run of backticks, not at the end of the string; `LATEX_ITEM_COMMAND_INLINE_RE` needs a real `\item`; `OpaqueEnvType` gains `table`, `figure`, `center`, `left`, `right`; `lib/styles/styles-code` no longer sizes code text.
+
+### Fixes that change existing output
+
+Nothing to do — these replace output that was broken or invalid.
+
+- **Markers no longer clip their item text**: the reserve is per glyph class in `em`, per level, capped at `20em`. Default indent and gap are pixel-identical at a 16px base.
+- **A `table`, `figure`, `center`, `left`, `right` or `tabular` in a list body gains its wrapper and caption wherever it sits**, and a list written inside one keeps its nesting depth.
+- **Content sitting directly in a list is wrapped in an `<li>`** — except a chunk holding only `\setcounter` or `\renewcommand`, which print nothing.
+- **A closer or an `\item` in the wrong place no longer breaks the tags.** Crossed names, collapsed closers, a mid-line closer opening a sibling, a list opened mid-paragraph, a list in a `tabular` cell, an empty `\item` and a closer written in `\item[…]`, in a code span or in a `\renewcommand` body all parse.
+- **Text is no longer dropped** beside a `\renewcommand`, after the outermost `\end{itemize}`, between a backslash command and a `\diagbox`, or by a `\\` ending a caption, heading or footnote argument. Two of those move numbering: that argument now renders, shifting `Table N`, `Figure N` and the section numbers below it, and a leftover after the closer is read as a block, so `\end{itemize} # H` becomes a heading.
+- **Every list reads its own copy of the marker tokens**, so a marker no longer leaks into a later list. Within one list the accumulation remains, as on 3.0.1.
+- **`table-markdown` exports what the cell holds** — a link's whole label, `sub`/`sup`/`ins` markup, math under `math_inline_delimiters`/`math_as_ascii`. Escaping guards Markdown syntax, not the URL scheme or HTML: see **Security notes on the Markdown exports** in the README.
+- **`prentLevel` on an inline-opened `*_list_open` is the real nesting depth**, was always `0`, and the depth is reset per parse rather than healed mid-render.
+- **Also fixed:** empty `<>` item bodies, `\itemsep`/`\itemindent` read as `\item`, a continuation line containing the word `item` losing its break, a list swallowed by a `\footnote`/`\footnotetext` scan, an orphan `<br>` after a `\renewcommand` line, whitespace-only text nodes between `<ul>` and `<li>`, edge whitespace in a custom marker, nested lists missing line numbers under `lineNumbering`, and a marker-less `<li>` exporting a marker looked up by `NaN`.
+- **Malformed input costs less**, and well-formed lists parse faster than 3.0.1. Unclosed `\begin{tabular}`, and unclosed envs with no blank line between them, stay super-linear.
+- **Code blocks scale with their container**: `pre`/`pre code` sizes are relative. Pixel-identical at a 16px base except code padding (16px → 15px) and a raw-HTML `<pre>` with no `<code>` child (13.6px → 15px).
+
+What malformed input still costs is in the spec under **Boundaries**, and every shape of it is pinned in `tests/_data/_lists/_data_known_quirks.js`.
 
 ## [3.0.1] - Preserve code indentation inside list/table/align env wrappers
 

lib/markdown/common.d.ts

@@ -1,6 +1,9 @@
 export declare const tocRegexp: RegExp;
+export declare const isAsciiLetter: (code: number) => boolean;
 export declare const isSpace: (code: any) => boolean;
 export declare const isWhitespace: (s: string) => boolean;
+/** Position of the next `endMarker` at or after `i` that is not escaped by a `\`, or -1. */
+export declare const findEndMarkerPos: (str: string, endMarker: string, i: number) => number;
 export declare const slugify: (s: string) => string;
 export declare const uniqueSlug: (slug: string, slugs: any) => string;
 export interface InlineCodeItem {
@@ -28,9 +31,9 @@ export declare const getInlineCodeListFromString: (str: any) => Array<InlineCode
  * The function returns an object containing the information:
  *     res: boolean, - Contains false if the end marker could not be found
  *     content?: string, - Contains content between start and end markers
- *     nextPos?: number - Contains the position of the end marker in the string
+ *     nextPos?: number - Contains the position just past the end marker (`endPos` is the marker itself)
  * */
-export declare const findEndMarker: (str: string, startPos?: number, beginMarker?: string, endMarker?: string, onlyEnd?: boolean, openBracketsBefore?: number) => {
+export declare const findEndMarker: (str: string, startPos?: number, beginMarker?: string, endMarker?: string, onlyEnd?: boolean, openBracketsBefore?: number, inlineCodePositions?: Set<number>) => {
     res: boolean;
     content?: undefined;
     openBrackets?: undefined;
@@ -49,6 +52,15 @@ export declare const findEndMarker: (str: string, startPos?: number, beginMarker
     endPos: number;
     openBrackets?: undefined;
 };
+/** Offset past a `[...]` option at `at`, `at` itself when there is none, -1 when it does not close.
+ *  One reader for all three places that skip one: their own versions disagreed on `]` in a code span,
+ *  on `\]`, on `[[m]]` and on a `]` one line down — the last of which `sameLine` still decides. */
+export declare const skipOptionalArg: (text: string, at: number, sameLine: boolean, codePositions?: Set<number>) => number;
+/** Offset past `\renewcommand{\name}{body}` — also the starred form, `[n]` and `[n][default]`, and a
+ *  bare `\name` as the first argument — or -1 when the arguments do not close in `text`. Braces are
+ *  paired, so a closer or an `\item` in the body is part of the command, not structure.
+ *  Answers at `from`, absolute: a slice per command made reading a line of them quadratic. */
+export declare const renewCommandSpanEnd: (text: string, from?: number, codeIndex?: Set<number>) => number;
 export declare const getTerminatedRules: (rule: string) => any[];
 export declare const removeCaptionsFromTableAndFigure: (content: string) => {
     content: string;