perf: fast-fail link tokenizing when no ')' remains ahead

src/Lexer.ts

@@ -15,6 +15,12 @@ export class _Lexer<ParserOutput = string, RendererOutput = string> {
     inRawBlock: boolean;
     /** a link was produced in the inline run currently being scanned */
     linkEmitted: boolean;
+    /**
+     * false while scanning an inline run whose source contains no ')',
+     * letting Tokenizer.link bail out before its quadratic backtracking.
+     * Anchored once per run by inlineTokens.
+     */
+    linkParenPossible: boolean;
     top: boolean;
   };
 
@@ -36,6 +42,7 @@ export class _Lexer<ParserOutput = string, RendererOutput = string> {
       inLink: false,
       inRawBlock: false,
       linkEmitted: false,
+      linkParenPossible: true,
       top: true,
     };
 
@@ -339,6 +346,24 @@ export class _Lexer<ParserOutput = string, RendererOutput = string> {
    */
   inlineTokens(src: string, tokens: Token[] = []): Token[] {
     this.tokenizer.lexer = this;
+    // One paren scan per inline run, shared with Tokenizer.link: without a
+    // ')' ahead the link regex cannot match, and skipping it avoids its
+    // quadratic backtracking over unterminated-link inputs like
+    // `'[a](b'.repeat(n)`. A nested run receives a substring of the outer
+    // run, so if the outer had no ')' neither does this substring — skip
+    // the scan in that case.
+    const prevLinkParenPossible = this.state.linkParenPossible;
+    this.state.linkParenPossible = prevLinkParenPossible && src.includes(')');
+    try {
+      return this.inlineTokensInner(src, tokens);
+    } finally {
+      // A nested run (e.g. a link label) anchored on its own substring;
+      // restore the outer run's flag for the rest of its loop.
+      this.state.linkParenPossible = prevLinkParenPossible;
+    }
+  }
+
+  private inlineTokensInner(src: string, tokens: Token[]): Token[] {
     // String with links masked to avoid interference with em and strong
     let maskedSrc = src;
 

src/Tokenizer.ts

@@ -683,6 +683,15 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
   }
 
   link(src: string): Tokens.Link | Tokens.Image | undefined {
+    // The link regex backtracks quadratically over unterminated-link inputs
+    // like '[a](b'.repeat(n): at every '[' its greedy unquoted-href run
+    // re-partitions the whole remainder while hunting for a ')' that never
+    // comes. inlineTokens anchors one paren scan per run in
+    // state.linkParenPossible; when it is false the regex cannot match, so
+    // bail out before running it.
+    if (this.lexer.state.linkParenPossible === false) {
+      return undefined;
+    }
     const cap = this.rules.inline.link.exec(src);
     if (cap) {
       const trimmedUrl = cap[2].trim();

test/specs/redos/quadratic_unterminated_links.cjs

@@ -0,0 +1,9 @@
+module.exports = [
+  {
+    // Unterminated inline links: each '[' starts a link candidate whose
+    // href backtracking scanned the whole remainder before the paren-hint
+    // fast fail in Tokenizer.link landed.
+    markdown: '[a](b'.repeat(50000),
+    html: `<p>${'[a](b'.repeat(50000)}</p>`,
+  },
+];