From d9fdc7cd13ddc321ed96ab64d7aa96226c42c439 Mon Sep 17 00:00:00 2001 From: Tony Brix Date: Tue, 22 Jul 2025 18:36:33 -0600 Subject: [PATCH 1/5] fix: fix multiline list item adds extra newline to raw --- src/Lexer.ts | 4 ++-- test/unit/Lexer.test.js | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/Lexer.ts b/src/Lexer.ts index 6502ea7271..aa4ccd03e3 100644 --- a/src/Lexer.ts +++ b/src/Lexer.ts @@ -244,7 +244,7 @@ export class _Lexer { if (this.state.top && (token = this.tokenizer.paragraph(cutSrc))) { const lastToken = tokens.at(-1); if (lastParagraphClipped && lastToken?.type === 'paragraph') { - lastToken.raw += '\n' + token.raw; + lastToken.raw += (lastToken.raw.endsWith('\n') ? '' : '\n') + token.raw; lastToken.text += '\n' + token.text; this.inlineQueue.pop(); this.inlineQueue.at(-1)!.src = lastToken.text; @@ -261,7 +261,7 @@ export class _Lexer { src = src.substring(token.raw.length); const lastToken = tokens.at(-1); if (lastToken?.type === 'text') { - lastToken.raw += '\n' + token.raw; + lastToken.raw += (lastToken.raw.endsWith('\n') ? '' : '\n') + token.raw; lastToken.text += '\n' + token.text; this.inlineQueue.pop(); this.inlineQueue.at(-1)!.src = lastToken.text; diff --git a/test/unit/Lexer.test.js b/test/unit/Lexer.test.js index 59c90640f0..6f50cba541 100644 --- a/test/unit/Lexer.test.js +++ b/test/unit/Lexer.test.js @@ -1173,6 +1173,43 @@ paragraph ], }); }); + + it.only('multiline', () => { + expectTokens({ + md: ` +- line 1 + line 2 +`, + tokens: [ + { + type: 'space', + raw: '\n', + }, { + type: 'list', + raw: '- line 1\n line 2\n', + ordered: false, + start: '', + loose: false, + items: [ + { + type: 'list_item', + raw: '- line 1\n line 2', + task: false, + checked: undefined, + loose: false, + text: 'line 1\nline 2', + tokens: [{ + type: 'text', + raw: 'line 1\nline 2', + text: 'line 1\nline 2', + tokens: [{ type: 'text', raw: 'line 1\nline 2', text: 'line 1\nline 2', escaped: false }], + }], + }, + ], + }, + ], + }); + }); }); describe('html', () => { From a50cac80bf20aa2b4dd5e546a598d6da5710b463 Mon Sep 17 00:00:00 2001 From: Tony Brix Date: Tue, 22 Jul 2025 22:34:09 -0600 Subject: [PATCH 2/5] update other raw --- src/Lexer.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Lexer.ts b/src/Lexer.ts index aa4ccd03e3..4382aba963 100644 --- a/src/Lexer.ts +++ b/src/Lexer.ts @@ -142,7 +142,7 @@ export class _Lexer { const lastToken = tokens.at(-1); // An indented code block cannot interrupt a paragraph. if (lastToken?.type === 'paragraph' || lastToken?.type === 'text') { - lastToken.raw += '\n' + token.raw; + lastToken.raw += (lastToken.raw.endsWith('\n') ? '' : '\n') + token.raw; lastToken.text += '\n' + token.text; this.inlineQueue.at(-1)!.src = lastToken.text; } else { @@ -198,7 +198,7 @@ export class _Lexer { src = src.substring(token.raw.length); const lastToken = tokens.at(-1); if (lastToken?.type === 'paragraph' || lastToken?.type === 'text') { - lastToken.raw += '\n' + token.raw; + lastToken.raw += (lastToken.raw.endsWith('\n') ? '' : '\n') + token.raw; lastToken.text += '\n' + token.raw; this.inlineQueue.at(-1)!.src = lastToken.text; } else if (!this.tokens.links[token.tag]) { From 25a3a98bb3d8c90020d8e63e597ac00b0db81ef6 Mon Sep 17 00:00:00 2001 From: Tony Brix Date: Sun, 3 Aug 2025 10:44:55 -0600 Subject: [PATCH 3/5] remove only --- test/unit/Lexer.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/Lexer.test.js b/test/unit/Lexer.test.js index 6f50cba541..12fc08eec2 100644 --- a/test/unit/Lexer.test.js +++ b/test/unit/Lexer.test.js @@ -1174,7 +1174,7 @@ paragraph }); }); - it.only('multiline', () => { + it('multiline', () => { expectTokens({ md: ` - line 1 From e52af889c5fbb3de650a7eacd646748cd767aacc Mon Sep 17 00:00:00 2001 From: Tony Brix Date: Sun, 3 Aug 2025 11:10:45 -0600 Subject: [PATCH 4/5] write test for code --- test/unit/Lexer.test.js | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/test/unit/Lexer.test.js b/test/unit/Lexer.test.js index 12fc08eec2..d47501f75d 100644 --- a/test/unit/Lexer.test.js +++ b/test/unit/Lexer.test.js @@ -1209,6 +1209,47 @@ paragraph }, ], }); + + it('indented code after text', () => { + expectTokens({ + md: ` +- a + - b +`, + tokens: [{ + type: 'space', + raw: '\n', + }, + { + type: 'list', + raw: '- a\n - b\n', + ordered: false, + start: '', + loose: false, + items: [ + { + type: 'list_item', + raw: '- a\n - b', + task: false, + checked: undefined, + loose: false, + text: 'a\n - b', + tokens: [ + { + type: 'text', + raw: 'a\n - b', + text: 'a\n- b', + tokens: [ + { type: 'text', raw: 'a\n- b', text: 'a\n- b', escaped: false }, + ], + }, + ], + }, + ], + }, + ], + }); + }); }); }); From 4a21874b9b314b488896c7f3388fee922858912c Mon Sep 17 00:00:00 2001 From: Tony Brix Date: Sun, 3 Aug 2025 11:18:50 -0600 Subject: [PATCH 5/5] add def test --- test/unit/Lexer.test.js | 69 ++++++++++++++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 15 deletions(-) diff --git a/test/unit/Lexer.test.js b/test/unit/Lexer.test.js index d47501f75d..a8676660a5 100644 --- a/test/unit/Lexer.test.js +++ b/test/unit/Lexer.test.js @@ -1209,46 +1209,85 @@ paragraph }, ], }); + }); - it('indented code after text', () => { - expectTokens({ - md: ` + it('indented code after paragraph', () => { + expectTokens({ + md: ` - a - b `, - tokens: [{ - type: 'space', - raw: '\n', - }, + tokens: [{ + type: 'space', + raw: '\n', + }, + { + type: 'list', + raw: '- a\n - b\n', + ordered: false, + start: '', + loose: false, + items: [ + { + type: 'list_item', + raw: '- a\n - b', + task: false, + checked: undefined, + loose: false, + text: 'a\n - b', + tokens: [ + { + type: 'text', + raw: 'a\n - b', + text: 'a\n- b', + tokens: [ + { type: 'text', raw: 'a\n- b', text: 'a\n- b', escaped: false }, + ], + }, + ], + }, + ], + }, + ], + }); + }); + + it('def after paragraph', () => { + expectTokens({ + md: ` +- hello +[1]: hello +`, + tokens: [ + { type: 'space', raw: '\n' }, { type: 'list', - raw: '- a\n - b\n', + raw: '- hello\n[1]: hello\n', ordered: false, start: '', loose: false, items: [ { type: 'list_item', - raw: '- a\n - b', + raw: '- hello\n[1]: hello', task: false, checked: undefined, loose: false, - text: 'a\n - b', + text: 'hello\n[1]: hello', tokens: [ { type: 'text', - raw: 'a\n - b', - text: 'a\n- b', + raw: 'hello\n[1]: hello', + text: 'hello\n[1]: hello', tokens: [ - { type: 'text', raw: 'a\n- b', text: 'a\n- b', escaped: false }, + { type: 'text', raw: 'hello\n[1]: hello', text: 'hello\n[1]: hello', escaped: false }, ], }, ], }, ], }, - ], - }); + ], }); }); });