diff --git a/package-lock.json b/package-lock.json index f839fda358..0d06d301cb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "marked": "bin/marked.js" }, "devDependencies": { - "@arethetypeswrong/cli": "^0.18.4", + "@arethetypeswrong/cli": "^0.18.5", "@markedjs/eslint-config": "^1.0.14", "@markedjs/testutils": "18.0.0-1", "@semantic-release/commit-analyzer": "^13.0.1", @@ -26,14 +26,14 @@ "dts-bundle-generator": "^9.5.1", "esbuild": "^0.28.1", "esbuild-plugin-umd-wrapper": "^3.0.0", - "eslint": "^10.6.0", + "eslint": "^10.7.0", "highlight.js": "^11.11.1", "markdown-it": "14.3.0", "marked-highlight": "^2.2.4", "marked-man": "^2.1.1", "recheck": "^4.5.0", "rimraf": "^6.1.3", - "semantic-release": "^25.0.5", + "semantic-release": "^25.0.7", "titleize": "^4.0.0", "tslib": "^2.8.1", "typescript": "6.0.3" @@ -98,13 +98,13 @@ "dev": true }, "node_modules/@arethetypeswrong/cli": { - "version": "0.18.4", - "resolved": "https://registry.npmjs.org/@arethetypeswrong/cli/-/cli-0.18.4.tgz", - "integrity": "sha512-kNWo6LTzGAuLYPpJ7Sgo63whSUeeSuKMlYx6IBgzs4ONEG807gW4hSSENvpeCHzO2H2wIzG5EFl0OKBbqGBAyA==", + "version": "0.18.5", + "resolved": "https://registry.npmjs.org/@arethetypeswrong/cli/-/cli-0.18.5.tgz", + "integrity": "sha512-gM+8vRsQOD/Uc7EnBedUhkG5OCsDWE4uoak5QvomGpMpaky0Eh41p04nIMgrWb8EOmqZUJGc6zz9hsP6E56R7g==", "dev": true, "license": "MIT", "dependencies": { - "@arethetypeswrong/core": "0.18.4", + "@arethetypeswrong/core": "0.18.5", "chalk": "^4.1.2", "cli-table3": "^0.6.3", "commander": "^10.0.1", @@ -133,9 +133,9 @@ } }, "node_modules/@arethetypeswrong/core": { - "version": "0.18.4", - "resolved": "https://registry.npmjs.org/@arethetypeswrong/core/-/core-0.18.4.tgz", - "integrity": "sha512-M5F0ePyN6h2Z6XxRiyIPqjGbltotXLjR0CKA0uKspsDu0QmgTNYvRb4RSQPMUs2ZXZHCCYpbaZbFbYOXLxCjUA==", + "version": "0.18.5", + "resolved": "https://registry.npmjs.org/@arethetypeswrong/core/-/core-0.18.5.tgz", + "integrity": "sha512-9ytjzGwxjm9Uz7I9avfbt5vlQt6uk9uRRESzJjqrznl6WKvI6dwYTo+vJ3U02Wrq/mR3iql/PzhvHhKdJIAjDQ==", "dev": true, "license": "MIT", "dependencies": { @@ -3254,9 +3254,9 @@ } }, "node_modules/eslint": { - "version": "10.6.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.6.0.tgz", - "integrity": "sha512-6lVbcqSodALYo+4ELD0heG6lFiFxnLMuLkiMi2qV8LMp54N8tE8FT1GMH+ev4Ti00nFjNze2+Su6DsV5OQW3Dg==", + "version": "10.7.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.7.0.tgz", + "integrity": "sha512-GVTD7s1vdIl6UYvAfriOPeY1Df8LIZjfofLvHwde+erDHGGuHyuM6xoxRxmHiebhYuD2p1vN4wWh0XzPARSGDQ==", "dev": true, "license": "MIT", "workspaces": [ @@ -7713,9 +7713,9 @@ "license": "MIT" }, "node_modules/semantic-release": { - "version": "25.0.5", - "resolved": "https://registry.npmjs.org/semantic-release/-/semantic-release-25.0.5.tgz", - "integrity": "sha512-mn61SUJwtM8ThrWn2WmgLVpwVJeG/hPSupua1psdMoufmwRIPyvRLkRkL0JDXkP67OntlLWUYnBnfVc8EDO3/g==", + "version": "25.0.7", + "resolved": "https://registry.npmjs.org/semantic-release/-/semantic-release-25.0.7.tgz", + "integrity": "sha512-fFiUD7LNFI0TOGkc49WDHUX4GYKrOMDKct5ReSB1EJCZiPsPdbIPIJGys1xb2ILpK1v9JMsRkjJQgTRpbr7DgQ==", "dev": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 24c972f395..f38c10d283 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "html" ], "devDependencies": { - "@arethetypeswrong/cli": "^0.18.4", + "@arethetypeswrong/cli": "^0.18.5", "@markedjs/eslint-config": "^1.0.14", "@markedjs/testutils": "18.0.0-1", "@semantic-release/commit-analyzer": "^13.0.1", @@ -62,14 +62,14 @@ "dts-bundle-generator": "^9.5.1", "esbuild": "^0.28.1", "esbuild-plugin-umd-wrapper": "^3.0.0", - "eslint": "^10.6.0", + "eslint": "^10.7.0", "highlight.js": "^11.11.1", "markdown-it": "14.3.0", "marked-highlight": "^2.2.4", "marked-man": "^2.1.1", "recheck": "^4.5.0", "rimraf": "^6.1.3", - "semantic-release": "^25.0.5", + "semantic-release": "^25.0.7", "titleize": "^4.0.0", "tslib": "^2.8.1", "typescript": "6.0.3" diff --git a/src/Lexer.ts b/src/Lexer.ts index 9efbdd38c8..89e291f19c 100644 --- a/src/Lexer.ts +++ b/src/Lexer.ts @@ -304,33 +304,26 @@ export class _Lexer { this.tokenizer.lexer = this; // String with links masked to avoid interference with em and strong let maskedSrc = src; - let match: RegExpExecArray | null = null; // Mask out reflinks if (this.tokens.links) { const links = Object.keys(this.tokens.links); if (links.length > 0) { - while ((match = this.tokenizer.rules.inline.reflinkSearch.exec(maskedSrc)) !== null) { - if (links.includes(match[0].slice(match[0].lastIndexOf('[') + 1, -1))) { - maskedSrc = maskedSrc.slice(0, match.index) - + '[' + 'a'.repeat(match[0].length - 2) + ']' - + maskedSrc.slice(this.tokenizer.rules.inline.reflinkSearch.lastIndex); - } - } + maskedSrc = maskedSrc.replace(this.tokenizer.rules.inline.reflinkSearch, match0 => + links.includes(match0.slice(match0.lastIndexOf('[') + 1, -1)) + ? '[' + 'a'.repeat(match0.length - 2) + ']' + : match0); } } // Mask out escaped characters - while ((match = this.tokenizer.rules.inline.anyPunctuation.exec(maskedSrc)) !== null) { - maskedSrc = maskedSrc.slice(0, match.index) + '++' + maskedSrc.slice(this.tokenizer.rules.inline.anyPunctuation.lastIndex); - } + maskedSrc = maskedSrc.replace(this.tokenizer.rules.inline.anyPunctuation, '++'); // Mask out other blocks - let offset; - while ((match = this.tokenizer.rules.inline.blockSkip.exec(maskedSrc)) !== null) { - offset = match[2] ? match[2].length : 0; - maskedSrc = maskedSrc.slice(0, match.index + offset) + '[' + 'a'.repeat(match[0].length - offset - 2) + ']' + maskedSrc.slice(this.tokenizer.rules.inline.blockSkip.lastIndex); - } + maskedSrc = maskedSrc.replace(this.tokenizer.rules.inline.blockSkip, (match0, _link, context) => { + const offset = context ? context.length : 0; + return match0.slice(0, offset) + '[' + 'a'.repeat(match0.length - offset - 2) + ']'; + }); // Mask out blocks from extensions maskedSrc = this.options.hooks?.emStrongMask?.call({ lexer: this }, maskedSrc) ?? maskedSrc; diff --git a/src/rules.ts b/src/rules.ts index cdbbeadb93..eb1b8d367d 100644 --- a/src/rules.ts +++ b/src/rules.ts @@ -115,7 +115,7 @@ const lheading = edit(lheadingCore) .replace(/blockCode/g, /(?: {4}| {0,3}\t)/) // indented code blocks can interrupt .replace(/fences/g, / {0,3}(?:`{3,}|~{3,})/) // fenced code blocks can interrupt .replace(/blockquote/g, / {0,3}>/) // blockquote can interrupt - .replace(/heading/g, / {0,3}#{1,6}/) // ATX heading can interrupt + .replace(/heading/g, / {0,3}#{1,6}(?:\s|$)/) // ATX heading can interrupt .replace(/html/g, / {0,3}<[^\n>]+>\n/) // block html can interrupt .replace(/\|table/g, '') // table not in commonmark .getRegex(); @@ -124,11 +124,11 @@ const lheadingGfm = edit(lheadingCore) .replace(/blockCode/g, /(?: {4}| {0,3}\t)/) // indented code blocks can interrupt .replace(/fences/g, / {0,3}(?:`{3,}|~{3,})/) // fenced code blocks can interrupt .replace(/blockquote/g, / {0,3}>/) // blockquote can interrupt - .replace(/heading/g, / {0,3}#{1,6}/) // ATX heading can interrupt + .replace(/heading/g, / {0,3}#{1,6}(?:\s|$)/) // ATX heading can interrupt .replace(/html/g, / {0,3}<[^\n>]+>\n/) // block html can interrupt .replace(/table/g, / {0,3}\|?(?:[:\- ]*\|)+[\:\- ]*\n/) // table can interrupt .getRegex(); -const _paragraph = /^([^\n]+(?:\n(?!hr|heading|lheading|blockquote|fences|list|html|table| +\n)[^\n]+)*)/; +const _paragraph = /^([^\n]+(?:\n(?!hr|heading|lheading|blockquote|fences|list|html|table|[ \t]+\n)[^\n]+)*)/; const blockText = /^[^\n]+/; const _blockLabel = /(?!\s*\])(?:\\[\s\S]|[^\[\]\\])+/; const def = edit(/^ {0,3}\[(label)\]: *(?:\n[ \t]*)?([^<\s][^\s]*|<.*?>)(?:(?: +(?:\n[ \t]*)?| *\n[ \t]*)(title))? *(?:\n+|$)/) @@ -149,11 +149,11 @@ const _tag = 'address|article|aside|base|basefont|blockquote|body|caption' const _comment = /|$))/; const html = edit( '^ {0,3}(?:' // optional indentation -+ '<(script|pre|style|textarea)[\\s>][\\s\\S]*?(?:[^\\n]*\\n+|$)' // (1) ++ '<(script|pre|style|textarea)[\\s>][\\s\\S]*?(?:[^\\n]*\\n*|$)' // (1) + '|comment[^\\n]*(\\n+|$)' // (2) -+ '|<\\?[\\s\\S]*?(?:\\?>[^\\n]*\\n+|$)' // (3) -+ '|[^\\n]*\\n+|$)' // (4) -+ '|[^\\n]*\\n+|$)' // (5) ++ '|<\\?[\\s\\S]*?(?:\\?>[^\\n]*\\n*|$)' // (3) ++ '|[^\\n]*\\n*|$)' // (4) ++ '|[^\\n]*\\n*|$)' // (5) + '|)[\\s\\S]*?(?:(?:\\n[ \t]*)+\\n|$)' // (6) + '|<(?!script|pre|style|textarea)([a-z][\\w-]*)(?:attribute)*? */?>(?=[ \\t]*(?:\\n|$))[\\s\\S]*?(?:(?:\\n[ \t]*)+\\n|$)' // (7) open tag + '|(?=[ \\t]*(?:\\n|$))[\\s\\S]*?(?:(?:\\n[ \t]*)+\\n|$)' // (7) closing tag @@ -169,7 +169,7 @@ const createParagraph = (listInterrupt: RegExp) => edit(_paragraph) .replace('|lheading', '') // setext headings don't interrupt commonmark paragraphs .replace('|table', '') .replace('blockquote', ' {0,3}>') - .replace('fences', ' {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n') + .replace('fences', ' {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~~~)[^\\n]*\\n') .replace('list', listInterrupt) .replace('html', ')|<(?:script|pre|style|textarea|!--)') .replace('tag', _tag) // pars can be interrupted by type (6) html blocks @@ -177,8 +177,10 @@ const createParagraph = (listInterrupt: RegExp) => edit(_paragraph) // only non-empty lists starting from 1 can interrupt paragraphs const paragraph = createParagraph(/ {0,3}(?:[*+-]|1[.)])[ \t]+[^ \t\n]/); -// blockquotes can be interrupted by lists starting from any number -const blockquoteParagraph = createParagraph(/ {0,3}(?:[*+-]|\d{1,9}[.)])[ \t]+[^ \t\n]/); +// inside a blockquote a bare list marker (any number) starts a sibling list, +// so it must not be lazily continued as paragraph text (unlike a top level +// paragraph, where an empty list cannot interrupt) +const blockquoteParagraph = createParagraph(/ {0,3}(?:[*+-]|\d{1,9}[.)])(?:[ \t]|\n|$)/); const blockquote = edit(/^( {0,3}> ?(paragraph|[^\n]*)(?:\n|$))+/) .replace('paragraph', blockquoteParagraph) @@ -218,7 +220,7 @@ const gfmTable = edit( .replace('heading', ' {0,3}#{1,6}(?:\\s|$)') .replace('blockquote', ' {0,3}>') .replace('code', '(?: {4}| {0,3}\t)[^\\n]') - .replace('fences', ' {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n') + .replace('fences', ' {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~~~)[^\\n]*\\n') .replace('list', ' {0,3}(?:[*+-]|1[.)])[ \\t]') // any bullet ends the table rows .replace('html', ')|<(?:script|pre|style|textarea|!--)') .replace('tag', _tag) // tables can be interrupted by type (6) html blocks @@ -234,7 +236,7 @@ const blockGfm: Record = { .replace('|lheading', '') // setext headings don't interrupt commonmark paragraphs .replace('table', gfmTable) // interrupt paragraphs with table .replace('blockquote', ' {0,3}>') - .replace('fences', ' {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n') + .replace('fences', ' {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~~~)[^\\n]*\\n') .replace('list', ' {0,3}(?:[*+-]|1[.)])[ \\t]+[^ \\t\\n]') // only non-empty lists starting from 1 can interrupt .replace('html', ')|<(?:script|pre|style|textarea|!--)') .replace('tag', _tag) // pars can be interrupted by type (6) html blocks @@ -471,11 +473,11 @@ const inlineGfm: Record = { delRDelim, url: edit(/^((?:protocol):\/\/|www\.)(?:[a-zA-Z0-9\-]+\.?)+[^\s<]*|^email/) .replace('protocol', _caseInsensitiveProtocol) - .replace('email', /[A-Za-z0-9._+-]+(@)[a-zA-Z0-9-_]+(?:\.[a-zA-Z0-9-_]*[a-zA-Z0-9])+(?![-_])/) + .replace('email', /[A-Za-z0-9._+-]{1,64}(@)[a-zA-Z0-9-_]+(?:\.[a-zA-Z0-9-_]*[a-zA-Z0-9])+(?![-_])/) .getRegex(), _backpedal: /(?:[^?!.,:;*_'"~()&]+|\([^)]*\)|&(?![a-zA-Z0-9]+;$)|[?!.,:;*_'"~)]+(?!$))+/, del: /^(~~?)(?=[^\s~])((?:\\[\s\S]|[^\\])*?(?:\\[\s\S]|[^\s~\\]))\1(?=[^~]|$)/, - text: edit(/^([`~]+|[^`~])(?:(?= {2,}\n)|(?=[a-zA-Z0-9.!#$%&'*+\/=?_`{\|}~-]+@)|[\s\S]*?(?:(?=[\\foo

+

bar

+

baz

+

qux

diff --git a/test/specs/new/blank_line_with_tabs.md b/test/specs/new/blank_line_with_tabs.md new file mode 100644 index 0000000000..a79bd0f899 --- /dev/null +++ b/test/specs/new/blank_line_with_tabs.md @@ -0,0 +1,10 @@ +--- +gfm: false +--- +foo + +bar + +baz + +qux diff --git a/test/specs/new/codespan_adjacent_tilde.html b/test/specs/new/codespan_adjacent_tilde.html new file mode 100644 index 0000000000..e49041d9cc --- /dev/null +++ b/test/specs/new/codespan_adjacent_tilde.html @@ -0,0 +1,5 @@ +

~code

+

before~after

+

`strike

+

~code ` tick

+

deleted code

diff --git a/test/specs/new/codespan_adjacent_tilde.md b/test/specs/new/codespan_adjacent_tilde.md new file mode 100644 index 0000000000..c7506a61f1 --- /dev/null +++ b/test/specs/new/codespan_adjacent_tilde.md @@ -0,0 +1,9 @@ +~`code` + +`before`~`after` + +`~strike~ + +~``code ` tick`` + +~~`deleted code`~~ diff --git a/test/specs/new/empty_list_after_blockquote.html b/test/specs/new/empty_list_after_blockquote.html new file mode 100644 index 0000000000..0ea2962633 --- /dev/null +++ b/test/specs/new/empty_list_after_blockquote.html @@ -0,0 +1,13 @@ +
+

foo

+
+
    +
  • +
+
+

foo +bar

+
+
    +
  1. +
diff --git a/test/specs/new/empty_list_after_blockquote.md b/test/specs/new/empty_list_after_blockquote.md new file mode 100644 index 0000000000..33ee472888 --- /dev/null +++ b/test/specs/new/empty_list_after_blockquote.md @@ -0,0 +1,6 @@ +> foo +- + +> foo +> bar +1. diff --git a/test/specs/new/lheading_hash_prefix.html b/test/specs/new/lheading_hash_prefix.html new file mode 100644 index 0000000000..b5c5ec34aa --- /dev/null +++ b/test/specs/new/lheading_hash_prefix.html @@ -0,0 +1,3 @@ +

#1 Goals

+

#hashtag

+

####### seven

diff --git a/test/specs/new/lheading_hash_prefix.md b/test/specs/new/lheading_hash_prefix.md new file mode 100644 index 0000000000..4d88c47bf3 --- /dev/null +++ b/test/specs/new/lheading_hash_prefix.md @@ -0,0 +1,8 @@ +#1 Goals +--------- + +#hashtag +=== + +####### seven +=== \ No newline at end of file diff --git a/test/specs/redos/quadratic_email_autolink.cjs b/test/specs/redos/quadratic_email_autolink.cjs new file mode 100644 index 0000000000..83e28f290e --- /dev/null +++ b/test/specs/redos/quadratic_email_autolink.cjs @@ -0,0 +1,10 @@ +module.exports = [ + { + markdown: 'a_'.repeat(50000), + html: `

${'a_'.repeat(50000)}

`, + }, + { + markdown: 'a*'.repeat(50000), + html: `

${'aa'.repeat(25000)}

`, + }, +]; diff --git a/test/specs/redos/quadratic_html_block_close.cjs b/test/specs/redos/quadratic_html_block_close.cjs new file mode 100644 index 0000000000..b5c753f6df --- /dev/null +++ b/test/specs/redos/quadratic_html_block_close.cjs @@ -0,0 +1,4 @@ +module.exports = { + markdown: ''.repeat(50000), + html: ''.repeat(50000), +}; diff --git a/test/specs/redos/quadratic_inline_masking.cjs b/test/specs/redos/quadratic_inline_masking.cjs new file mode 100644 index 0000000000..6418a770cd --- /dev/null +++ b/test/specs/redos/quadratic_inline_masking.cjs @@ -0,0 +1,14 @@ +module.exports = [ + { + markdown: '\\.'.repeat(100000), + html: `

${'.'.repeat(100000)}

`, + }, + { + markdown: '`x` '.repeat(60000), + html: `

${'x '.repeat(60000)}

`, + }, + { + markdown: '[a]: x\n\n' + '[a] '.repeat(100000), + html: `

${'a '.repeat(100000)}

`, + }, +]; diff --git a/test/specs/redos/quadratic_tilde_paragraph_interrupt.cjs b/test/specs/redos/quadratic_tilde_paragraph_interrupt.cjs new file mode 100644 index 0000000000..b9e9c4c096 --- /dev/null +++ b/test/specs/redos/quadratic_tilde_paragraph_interrupt.cjs @@ -0,0 +1,4 @@ +module.exports = { + markdown: 'intro\n' + '~'.repeat(50000), + html: `

intro\n${'~'.repeat(50000)}

\n`, +};