From 5c56c42db4facb022300502c3f09ca692c2f9095 Mon Sep 17 00:00:00 2001 From: koding88 Date: Tue, 25 Aug 2026 00:32:56 +0700 Subject: [PATCH 1/3] perf: fast-fail link tokenizing when no ')' remains ahead MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The inline link rule backtracks quadratically over unterminated link sequences. '[a](b'.repeat(n) parses in O(n^2) — 97KB took ~2.5s — because at every '[' the regex's greedy unquoted-href run re-partitions the whole remainder while hunting for a closing ')' that never comes. Lexer.inlineTokens now anchors one paren scan per inline run (lastIndexOf(')') over the paragraph), and Tokenizer.link consults it: since every src it receives during the run is a suffix of that anchor, 'is there a ) ahead?' is O(1). When none remains the regex cannot match anyway, so skipping is semantics-preserving; anchors are saved/restored around nested runs (e.g. link labels) and live in a WeakMap so they stay off the public tokenizer API. Measured on Node 24: '[a](b'.repeat(20000) ~2500ms -> ~24ms '[a](b'.repeat(80000) (OOM-adjacent) -> ~60ms Differential check: 1506 generated documents (links with titles, balanced parens, angle destinations, images, autolinks, code spans, reflinks + defs, nested brackets, unterminated sequences) x gfm on/ off produce byte-identical output before and after the change. Adds test/specs/redos/quadratic_unterminated_links.cjs following the existing guard convention; it exceeds the redos budget on master and passes with this change. --- src/Lexer.ts | 24 +++++++++++++- src/Tokenizer.ts | 33 +++++++++++++++++++ .../redos/quadratic_unterminated_links.cjs | 9 +++++ 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 test/specs/redos/quadratic_unterminated_links.cjs diff --git a/src/Lexer.ts b/src/Lexer.ts index bc12e9f3be..11bd5f7d9f 100644 --- a/src/Lexer.ts +++ b/src/Lexer.ts @@ -1,4 +1,4 @@ -import { _Tokenizer } from './Tokenizer.ts'; +import { _Tokenizer, linkParenHints } from './Tokenizer.ts'; import { _defaults } from './defaults.ts'; import { other, block, inline } from './rules.ts'; import type { Token, TokensList, Tokens } from './Tokens.ts'; @@ -338,7 +338,29 @@ export class _Lexer { * Lexing/Compiling */ inlineTokens(src: string, tokens: Token[] = []): Token[] { + const prevHint = linkParenHints.get(this.tokenizer); + linkParenHints.set(this.tokenizer, { + anchorLen: src.length, + lastParenFromStart: src.lastIndexOf(')'), + }); + try { + return this.inlineTokensInner(src, tokens); + } finally { + // Nested runs anchored their own label substring; restoring keeps the + // outer run's anchor valid for the remainder of its loop. + if (prevHint) { + linkParenHints.set(this.tokenizer, prevHint); + } else { + linkParenHints.delete(this.tokenizer); + } + } + } + + private inlineTokensInner(src: string, tokens: Token[] = []): Token[] { this.tokenizer.lexer = this; + // Every src seen in the loop below is a suffix of the string anchored by + // inlineTokens(), which lets Tokenizer.link answer "is there a ')' ahead?" + // in O(1) instead of backtracking quadratically over such input. // String with links masked to avoid interference with em and strong let maskedSrc = src; diff --git a/src/Tokenizer.ts b/src/Tokenizer.ts index 5f7d878599..ea2f467d31 100644 --- a/src/Tokenizer.ts +++ b/src/Tokenizer.ts @@ -11,6 +11,21 @@ import type { _Lexer } from './Lexer.ts'; import type { Links, Tokens, Token } from './Tokens.ts'; import type { MarkedOptions } from './MarkedOptions.ts'; +/** + * Fast-fail anchor for {@link _Tokenizer.link}: `Lexer.inlineTokens` stores + * one entry per inline run (keyed by the tokenizer instance), and restores + * the previous entry when a nested run finishes. Every src passed to link() + * during that run is a suffix of the anchored string, so "is there a ')' + * ahead?" is answerable in O(1) instead of letting the link regex backtrack + * quadratically over unterminated-link inputs like `'[a](b'.repeat(n)`. + * Kept in a WeakMap so it never shows up on the public tokenizer API surface. + */ +export interface LinkParenHint { + anchorLen: number; + lastParenFromStart: number; +} +export const linkParenHints = new WeakMap(); + function outputLink(cap: string[], link: Pick, raw: string, lexer: _Lexer, rules: Rules): Tokens.Link | Tokens.Image | undefined { const href = link.href; const title = link.title || null; @@ -683,6 +698,24 @@ export class _Tokenizer { } link(src: string): Tokens.Link | Tokens.Image | undefined { + // Fast fail when no ')' can appear ahead (see LinkParenHint). Skipping + // here is semantics-preserving: the link regex requires a ')' to match, + // so with no paren left in the remainder it would fail anyway — the + // skip only removes its O(n^2) backtracking over such input. + const hint = linkParenHints.get(this); + if (hint && src.length <= hint.anchorLen) { + if (hint.lastParenFromStart < hint.anchorLen - src.length) { + return undefined; + } + } else { + // No anchor yet (direct call outside a lexer run) or the string grew + // beyond it: anchor lazily. Every later call in this run is a suffix + // of this string, so anchoring here stays sound. + linkParenHints.set(this, { + anchorLen: src.length, + lastParenFromStart: src.lastIndexOf(')'), + }); + } const cap = this.rules.inline.link.exec(src); if (cap) { const trimmedUrl = cap[2].trim(); diff --git a/test/specs/redos/quadratic_unterminated_links.cjs b/test/specs/redos/quadratic_unterminated_links.cjs new file mode 100644 index 0000000000..dc40e3adf8 --- /dev/null +++ b/test/specs/redos/quadratic_unterminated_links.cjs @@ -0,0 +1,9 @@ +module.exports = [ + { + // Unterminated inline links: each '[' starts a link candidate whose + // href backtracking scanned the whole remainder before the paren-hint + // fast fail in Tokenizer.link landed. + markdown: '[a](b'.repeat(50000), + html: `

${'[a](b'.repeat(50000)}

`, + }, +]; From d21faeb503390d6d0835569757e8289bbdb2786c Mon Sep 17 00:00:00 2001 From: koding88 Date: Tue, 25 Aug 2026 11:52:07 +0700 Subject: [PATCH 2/3] refactor: one paren scan per inline run via lexer state Per review feedback: drop the WeakMap anchor machinery in favor of a single state flag. Lexer.inlineTokens scans for ')' once per run and stores it in state.linkParenPossible (same pattern as linkEmitted), restoring the previous value after nested runs. Tokenizer.link bails out only when the flag is false, which is exactly when its regex cannot match. This keeps the fast-fail while avoiding a repeated full-text includes() on reflink-heavy paragraphs, where attempting link() at every token would otherwise re-scan the remainder each time (quadratic_inline_masking[2]: ~1.03s vs ~144ms here, upstream ~140ms). --- src/Lexer.ts | 37 +++++++++++++++++++------------------ src/Tokenizer.ts | 40 ++++++++-------------------------------- 2 files changed, 27 insertions(+), 50 deletions(-) diff --git a/src/Lexer.ts b/src/Lexer.ts index 11bd5f7d9f..5ec610f3b3 100644 --- a/src/Lexer.ts +++ b/src/Lexer.ts @@ -1,4 +1,4 @@ -import { _Tokenizer, linkParenHints } from './Tokenizer.ts'; +import { _Tokenizer } from './Tokenizer.ts'; import { _defaults } from './defaults.ts'; import { other, block, inline } from './rules.ts'; import type { Token, TokensList, Tokens } from './Tokens.ts'; @@ -15,6 +15,12 @@ export class _Lexer { inRawBlock: boolean; /** a link was produced in the inline run currently being scanned */ linkEmitted: boolean; + /** + * false while scanning an inline run whose source contains no ')', + * letting Tokenizer.link bail out before its quadratic backtracking. + * Anchored once per run by inlineTokens. + */ + linkParenPossible: boolean; top: boolean; }; @@ -36,6 +42,7 @@ export class _Lexer { inLink: false, inRawBlock: false, linkEmitted: false, + linkParenPossible: true, top: true, }; @@ -338,29 +345,23 @@ export class _Lexer { * Lexing/Compiling */ inlineTokens(src: string, tokens: Token[] = []): Token[] { - const prevHint = linkParenHints.get(this.tokenizer); - linkParenHints.set(this.tokenizer, { - anchorLen: src.length, - lastParenFromStart: src.lastIndexOf(')'), - }); + this.tokenizer.lexer = this; + // One paren scan per inline run, shared with Tokenizer.link: without a + // ')' ahead the link regex cannot match, and skipping it avoids its + // quadratic backtracking over unterminated-link inputs like + // `'[a](b'.repeat(n)`. + const prevLinkParenPossible = this.state.linkParenPossible; + this.state.linkParenPossible = src.includes(')'); try { return this.inlineTokensInner(src, tokens); } finally { - // Nested runs anchored their own label substring; restoring keeps the - // outer run's anchor valid for the remainder of its loop. - if (prevHint) { - linkParenHints.set(this.tokenizer, prevHint); - } else { - linkParenHints.delete(this.tokenizer); - } + // A nested run (e.g. a link label) anchored on its own substring; + // restore the outer run's flag for the rest of its loop. + this.state.linkParenPossible = prevLinkParenPossible; } } - private inlineTokensInner(src: string, tokens: Token[] = []): Token[] { - this.tokenizer.lexer = this; - // Every src seen in the loop below is a suffix of the string anchored by - // inlineTokens(), which lets Tokenizer.link answer "is there a ')' ahead?" - // in O(1) instead of backtracking quadratically over such input. + private inlineTokensInner(src: string, tokens: Token[]): Token[] { // String with links masked to avoid interference with em and strong let maskedSrc = src; diff --git a/src/Tokenizer.ts b/src/Tokenizer.ts index ea2f467d31..129260d21f 100644 --- a/src/Tokenizer.ts +++ b/src/Tokenizer.ts @@ -11,21 +11,6 @@ import type { _Lexer } from './Lexer.ts'; import type { Links, Tokens, Token } from './Tokens.ts'; import type { MarkedOptions } from './MarkedOptions.ts'; -/** - * Fast-fail anchor for {@link _Tokenizer.link}: `Lexer.inlineTokens` stores - * one entry per inline run (keyed by the tokenizer instance), and restores - * the previous entry when a nested run finishes. Every src passed to link() - * during that run is a suffix of the anchored string, so "is there a ')' - * ahead?" is answerable in O(1) instead of letting the link regex backtrack - * quadratically over unterminated-link inputs like `'[a](b'.repeat(n)`. - * Kept in a WeakMap so it never shows up on the public tokenizer API surface. - */ -export interface LinkParenHint { - anchorLen: number; - lastParenFromStart: number; -} -export const linkParenHints = new WeakMap(); - function outputLink(cap: string[], link: Pick, raw: string, lexer: _Lexer, rules: Rules): Tokens.Link | Tokens.Image | undefined { const href = link.href; const title = link.title || null; @@ -698,23 +683,14 @@ export class _Tokenizer { } link(src: string): Tokens.Link | Tokens.Image | undefined { - // Fast fail when no ')' can appear ahead (see LinkParenHint). Skipping - // here is semantics-preserving: the link regex requires a ')' to match, - // so with no paren left in the remainder it would fail anyway — the - // skip only removes its O(n^2) backtracking over such input. - const hint = linkParenHints.get(this); - if (hint && src.length <= hint.anchorLen) { - if (hint.lastParenFromStart < hint.anchorLen - src.length) { - return undefined; - } - } else { - // No anchor yet (direct call outside a lexer run) or the string grew - // beyond it: anchor lazily. Every later call in this run is a suffix - // of this string, so anchoring here stays sound. - linkParenHints.set(this, { - anchorLen: src.length, - lastParenFromStart: src.lastIndexOf(')'), - }); + // The link regex backtracks quadratically over unterminated-link inputs + // like '[a](b'.repeat(n): at every '[' its greedy unquoted-href run + // re-partitions the whole remainder while hunting for a ')' that never + // comes. inlineTokens anchors one paren scan per run in + // state.linkParenPossible; when it is false the regex cannot match, so + // bail out before running it. + if (this.lexer.state.linkParenPossible === false) { + return undefined; } const cap = this.rules.inline.link.exec(src); if (cap) { From 908ad325c6cc52de42c182cbd7f950ccf14b21d0 Mon Sep 17 00:00:00 2001 From: koding88 Date: Tue, 25 Aug 2026 16:24:18 +0700 Subject: [PATCH 3/3] refactor: skip paren scan for nested runs when outer had no ')' When the outer inline run contains no ')' (linkParenPossible=false), a nested run's substring can't contain one either, so skip the redundant src.includes(')') scan. Addresses UziTech's observation on line 354: 'This also only needs to be checked if prevLinkParenPossible is true.' --- src/Lexer.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Lexer.ts b/src/Lexer.ts index 5ec610f3b3..0db4645ffd 100644 --- a/src/Lexer.ts +++ b/src/Lexer.ts @@ -349,9 +349,11 @@ export class _Lexer { // One paren scan per inline run, shared with Tokenizer.link: without a // ')' ahead the link regex cannot match, and skipping it avoids its // quadratic backtracking over unterminated-link inputs like - // `'[a](b'.repeat(n)`. + // `'[a](b'.repeat(n)`. A nested run receives a substring of the outer + // run, so if the outer had no ')' neither does this substring — skip + // the scan in that case. const prevLinkParenPossible = this.state.linkParenPossible; - this.state.linkParenPossible = src.includes(')'); + this.state.linkParenPossible = prevLinkParenPossible && src.includes(')'); try { return this.inlineTokensInner(src, tokens); } finally {