From c44e633004448a3135b60cb725c9cbb40ad7c8f7 Mon Sep 17 00:00:00 2001 From: Mimi <1119186082@qq.com> Date: Fri, 24 Jul 2026 12:15:20 +0800 Subject: [PATCH 1/4] fix(post): parse nested tags with context-aware lexer --- lib/hexo/post.ts | 345 +------------- lib/hexo/post_render_lexer.ts | 431 ++++++++++++++++++ .../before_post_render/backtick_code_block.ts | 17 +- test/scripts/hexo/post.ts | 69 +++ test/scripts/hexo/post_render_lexer.ts | 85 ++++ 5 files changed, 596 insertions(+), 351 deletions(-) create mode 100644 lib/hexo/post_render_lexer.ts create mode 100644 test/scripts/hexo/post_render_lexer.ts diff --git a/lib/hexo/post.ts b/lib/hexo/post.ts index 9418039e4..d382e1dc1 100644 --- a/lib/hexo/post.ts +++ b/lib/hexo/post.ts @@ -1,4 +1,3 @@ -import assert from 'assert'; import moment from 'moment'; import Promise from 'bluebird'; import { join, extname, basename } from 'path'; @@ -8,337 +7,11 @@ import { slugize, escapeRegExp, deepMerge} from 'hexo-util'; import { copyDir, exists, listDir, mkdirs, readFile, rmdir, unlink, writeFile } from 'hexo-fs'; import { parse as yfmParse, split as yfmSplit, stringify as yfmStringify } from 'hexo-front-matter'; import type Hexo from './index'; +import PostRenderEscape from './post_render_lexer'; import type { NodeJSLikeCallback, RenderData } from '../types'; const preservedKeys = ['title', 'slug', 'path', 'layout', 'date', 'content']; -const rHexoPostRenderEscape = /([\s\S]+?)<\/hexoPostRenderCodeBlock>/g; -const rSwigTag = /(\{\{.+?\}\})|(\{#.+?#\})|(\{%.+?%\})/s; - -const rSwigPlaceHolder = /(?:<|<)!--swig\uFFFC(\d+)--(?:>|>)/g; -const rCodeBlockPlaceHolder = /(?:<|<)!--code\uFFFC(\d+)--(?:>|>)/g; -const rCommentHolder = /(?:<|<)!--comment\uFFFC(\d+)--(?:>|>)/g; - -const STATE_PLAINTEXT = 0; -const STATE_SWIG_VAR = 1; -const STATE_SWIG_COMMENT = 2; -const STATE_SWIG_TAG = 3; -const STATE_SWIG_FULL_TAG = 4; -const STATE_PLAINTEXT_COMMENT = 5; -const STATE_INLINE_CODE = 6; - -const isNonWhiteSpaceChar = (char: string) => char !== '\r' - && char !== '\n' - && char !== '\t' - && char !== '\f' - && char !== '\v' - && char !== ' '; - -class PostRenderEscape { - public stored: string[]; - public length: number; - - constructor() { - this.stored = []; - } - - static escapeContent(cache: string[], flag: string, str: string) { - return ``; - } - - static restoreContent(cache: string[]) { - return (_: string, index: number) => { - assert(cache[index]); - const value = cache[index]; - cache[index] = null; - return value; - }; - } - - restoreAllSwigTags(str: string) { - const restored = str.replace(rSwigPlaceHolder, PostRenderEscape.restoreContent(this.stored)); - return restored; - } - - restoreCodeBlocks(str: string) { - return str.replace(rCodeBlockPlaceHolder, PostRenderEscape.restoreContent(this.stored)); - } - - restoreComments(str: string) { - return str.replace(rCommentHolder, PostRenderEscape.restoreContent(this.stored)); - } - - escapeCodeBlocks(str: string) { - return str.replace(rHexoPostRenderEscape, (_, content) => PostRenderEscape.escapeContent(this.stored, 'code', content)); - } - - escapeAllSwigTags(str: string) { - let state = STATE_PLAINTEXT; - let buffer_start = -1; - let plaintext_comment_start = -1; - let inline_code_backtick_count = 0; - let plain_text_start = 0; - let output = ''; - - let swig_tag_name_begin = false; - let swig_tag_name_end = false; - let swig_tag_name = ''; - - let swig_full_tag_start_start = -1; - let swig_full_tag_start_end = -1; - // current we just consider one level of string quote - let swig_string_quote = ''; - - const { length } = str; - - let idx = 0; - - // for backtracking - const swig_start_idx = [0, 0, 0, 0, 0, 0, 0]; - - const flushPlainText = (end: number) => { - if (plain_text_start !== -1 && end > plain_text_start) { - output += str.slice(plain_text_start, end); - } - plain_text_start = -1; - }; - - const ensurePlainTextStart = (position: number) => { - if (plain_text_start === -1) { - plain_text_start = position; - } - }; - - const pushAndReset = (value: string) => { - output += value; - plain_text_start = -1; - }; - - while (idx < length) { - while (idx < length) { - const char = str[idx]; - const prev_char = idx > 0 ? str[idx - 1] : ''; - const next_char = str[idx + 1]; - - if (state === STATE_PLAINTEXT) { // From plain text to swig or inline code - ensurePlainTextStart(idx); - // Check for inline code block (backticks) - if (char === '`' && prev_char !== '\\') { - // Count consecutive backticks - let backtick_count = 1; - while (str[idx + backtick_count] === '`') { - backtick_count++; - } - - flushPlainText(idx); - state = STATE_INLINE_CODE; - inline_code_backtick_count = backtick_count; - swig_start_idx[state] = idx; - idx += backtick_count - 1; // Skip the counted backticks - } else if (char === '{') { - // check if it is a complete tag {{ }} - if (next_char === '{') { - flushPlainText(idx); - state = STATE_SWIG_VAR; - idx++; - buffer_start = idx + 1; - swig_start_idx[state] = idx; - } else if (next_char === '#') { - flushPlainText(idx); - state = STATE_SWIG_COMMENT; - idx++; - buffer_start = idx + 1; - swig_start_idx[state] = idx; - } else if (next_char === '%') { - flushPlainText(idx); - state = STATE_SWIG_TAG; - idx++; - buffer_start = idx + 1; - swig_full_tag_start_start = idx + 1; - swig_full_tag_start_end = idx + 1; - swig_tag_name = ''; - swig_tag_name_begin = false; // Mark if it is the first non white space char in the swig tag - swig_tag_name_end = false; - swig_start_idx[state] = idx; - } - } else if (char === '<' && next_char === '!' && str[idx + 2] === '-' && str[idx + 3] === '-') { - flushPlainText(idx); - state = STATE_PLAINTEXT_COMMENT; - plaintext_comment_start = idx; - idx += 3; - } - } else if (state === STATE_INLINE_CODE) { - const inline_code_start = swig_start_idx[state]; - // Check for newline - inline code cannot span multiple lines - if ((char === '\n' && next_char === '\n') - || (char === '\r' && next_char === '\n' && str[idx + 2] === '\r' && str[idx + 3] === '\n') - || (char === '\r' && next_char === '\n' && str[idx + 2] === '\n') - || (char === '\n' && next_char === '\r' && str[idx + 2] === '\n') - ) { - // Backtrack: treat the opening backticks as plain text and retry from after them - pushAndReset(str.slice(inline_code_start, inline_code_start + inline_code_backtick_count)); - // Reset idx to position right after the opening backticks - idx = inline_code_start + inline_code_backtick_count - 1; - state = STATE_PLAINTEXT; - } else if (char === '{' && next_char === '%' && str.slice(idx).match(/^\{% *raw *%\}/)) { - // we may have raw tag in inline code - const raw_tag_end_match = str.slice(idx).match(/\{% *endraw *%\}/); - if (raw_tag_end_match) { - pushAndReset(str.slice(inline_code_start, idx)); - // escape the raw tag content - pushAndReset(PostRenderEscape.escapeContent(this.stored, 'swig', str.slice(idx, idx + raw_tag_end_match.index! + raw_tag_end_match[0].length))); - idx = idx + raw_tag_end_match.index! + raw_tag_end_match[0].length - 1; - swig_start_idx[state] = idx + 1; - } - } else if (char === '`') { - // Count consecutive backticks - let backtick_count = 1; - while (str[idx + backtick_count] === '`') { - backtick_count++; - } - - // If the count matches, we found the closing backticks - if (backtick_count === inline_code_backtick_count) { - pushAndReset(str.slice(inline_code_start, idx + backtick_count)); - idx += backtick_count - 1; // Skip the counted backticks - state = STATE_PLAINTEXT; - } - } - } else if (state === STATE_SWIG_TAG) { - if (char === '"' || char === '\'') { - if (swig_string_quote === '') { - swig_string_quote = char; - } else if (swig_string_quote === char) { - swig_string_quote = ''; - } - } - if (char === '%' && next_char === '}' && swig_string_quote === '') { // From swig back to plain text - idx++; - if (swig_tag_name !== '' && str.includes(`end${swig_tag_name}`)) { - state = STATE_SWIG_FULL_TAG; - buffer_start = idx + 1; - // since we have already move idx to next char of '}', so here is idx -1 - swig_full_tag_start_end = idx - 1; - swig_start_idx[state] = idx; - } else { - swig_tag_name = ''; - state = STATE_PLAINTEXT; - // since we have already move idx to next char of '}', so here is idx -1 - pushAndReset(PostRenderEscape.escapeContent(this.stored, 'swig', `{%${str.slice(buffer_start, idx - 1)}%}`)); - } - - } else { - if (isNonWhiteSpaceChar(char)) { - if (!swig_tag_name_begin && !swig_tag_name_end) { - swig_tag_name_begin = true; - } - - if (swig_tag_name_begin) { - swig_tag_name += char; - } - } else { - if (swig_tag_name_begin === true) { - swig_tag_name_begin = false; - swig_tag_name_end = true; - } - } - } - } else if (state === STATE_SWIG_VAR) { - if (char === '"' || char === '\'') { - if (swig_string_quote === '') { - swig_string_quote = char; - } else if (swig_string_quote === char) { - swig_string_quote = ''; - } - } - // {{ } - if (char === '}' && next_char !== '}' && swig_string_quote === '') { - // From swig back to plain text - state = STATE_PLAINTEXT; - pushAndReset(`{{${str.slice(buffer_start, idx)}${char}`); - } else if (char === '}' && next_char === '}' && swig_string_quote === '') { - pushAndReset(PostRenderEscape.escapeContent(this.stored, 'swig', `{{${str.slice(buffer_start, idx)}}}`)); - idx++; - state = STATE_PLAINTEXT; - } - } else if (state === STATE_SWIG_COMMENT) { // From swig back to plain text - if (char === '#' && next_char === '}') { - idx++; - state = STATE_PLAINTEXT; - plain_text_start = -1; - } - } else if (state === STATE_SWIG_FULL_TAG) { - if (char === '{' && next_char === '%') { - let swig_full_tag_end_buffer = ''; - let swig_full_tag_found = false; - - let _idx = idx + 2; - for (; _idx < length; _idx++) { - const _char = str[_idx]; - const _next_char = str[_idx + 1]; - - if (_char === '%' && _next_char === '}') { - _idx++; - swig_full_tag_found = true; - break; - } - - swig_full_tag_end_buffer = swig_full_tag_end_buffer + _char; - } - - if (swig_full_tag_found && swig_full_tag_end_buffer.includes(`end${swig_tag_name}`)) { - state = STATE_PLAINTEXT; - pushAndReset(PostRenderEscape.escapeContent(this.stored, 'swig', `{%${str.slice(swig_full_tag_start_start, swig_full_tag_start_end)}%}${str.slice(buffer_start, idx)}{%${swig_full_tag_end_buffer}%}`)); - idx = _idx; - swig_full_tag_end_buffer = ''; - } - } - } else if (state === STATE_PLAINTEXT_COMMENT) { - if (char === '-' && next_char === '-' && str[idx + 2] === '>') { - state = STATE_PLAINTEXT; - const comment = str.slice(plaintext_comment_start, idx + 3); - pushAndReset(PostRenderEscape.escapeContent(this.stored, 'comment', comment)); - idx += 2; - } - } - idx++; - } - if (state === STATE_PLAINTEXT) { - break; - } - if (state === STATE_PLAINTEXT_COMMENT) { - // Unterminated comment, just push the rest as comment - const comment = str.slice(plaintext_comment_start, length); - pushAndReset(PostRenderEscape.escapeContent(this.stored, 'comment', comment)); - break; - } - if (state === STATE_INLINE_CODE) { - const inline_code_start = swig_start_idx[state]; - pushAndReset(str.slice(inline_code_start, inline_code_start + inline_code_backtick_count)); - // Reset idx to position right after the opening backticks - idx = inline_code_start + inline_code_backtick_count; - state = STATE_PLAINTEXT; - continue; - } - // If the swig tag is not closed, then it is a plain text, we need to backtrack - if (state === STATE_SWIG_FULL_TAG) { - pushAndReset(`{%${str.slice(swig_full_tag_start_start, swig_full_tag_start_end)}%`); - } else { - pushAndReset('{'); - } - idx = swig_start_idx[state]; - swig_string_quote = ''; - state = STATE_PLAINTEXT; - } - - if (plain_text_start !== -1 && plain_text_start < length) { - output += str.slice(plain_text_start); - } - - return output; - } -} - const prepareFrontMatter = (data: any, jsonMode: boolean): Record => { for (const [key, item] of Object.entries(data)) { if (moment.isMoment(item)) { @@ -584,15 +257,11 @@ class Post { // Run "before_post_render" filters return ctx.execFilter('before_post_render', data, { context: ctx }); }).then(() => { - // Escape all comments to avoid conflict with Nunjucks and code block data.content = cacheObj.escapeCodeBlocks(data.content); - // Escape all Nunjucks/Swig tags - let hasSwigTag = true; - if (disableNunjucks === false) { - hasSwigTag = rSwigTag.test(data.content); - if (hasSwigTag) { - data.content = cacheObj.escapeAllSwigTags(data.content); - } + let hasSwigTag = false; + if (!disableNunjucks) { + data.content = cacheObj.escapeAllSwigTags(data.content); + hasSwigTag = cacheObj.hasNunjucks; } const options: { highlight?: boolean; } = data.markdown || {}; @@ -608,6 +277,7 @@ class Post { onRenderEnd(content) { // Replace cache data with real contents data.content = cacheObj.restoreAllSwigTags(content); + data.content = cacheObj.restoreComments(data.content); // Return content after replace the placeholders if (disableNunjucks || !hasSwigTag) return data.content; @@ -617,8 +287,7 @@ class Post { } }, options); }).then(content => { - data.content = cacheObj.restoreComments(content); - data.content = cacheObj.restoreCodeBlocks(data.content); + data.content = cacheObj.restoreCodeBlocks(content); // Run "after_post_render" filters return ctx.execFilter('after_post_render', data, { context: ctx }); diff --git a/lib/hexo/post_render_lexer.ts b/lib/hexo/post_render_lexer.ts new file mode 100644 index 000000000..8b56beb15 --- /dev/null +++ b/lib/hexo/post_render_lexer.ts @@ -0,0 +1,431 @@ +import assert from 'assert'; + +/* + * Nunjucks delimiter handling in this file is adapted from its lexer. + * + * Copyright (c) 2012-2015, James Long + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DAMAGES ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +const rSwigPlaceHolder = /(?:<|<)!--swig\uFFFC(\d+)--(?:>|>)/g; +const rCodeBlockPlaceHolder = /(?:<|<)!--code\uFFFC(\d+)--(?:>|>)/g; +const rCommentHolder = /(?:<|<)!--comment\uFFFC(\d+)--(?:>|>)/g; +const rContextHolder = /hexoPostRenderContext\uFFFC(\d+)\uFFFC/g; + +const BLOCK_START = '{%'; +const BLOCK_END = '%}'; +const VARIABLE_START = '{{'; +const VARIABLE_END = '}}'; +const COMMENT_START = '{#'; +const COMMENT_END = '#}'; +const NUNJUCKS_TOKEN_BOUNDARIES = ' \n\t\r\u00A0()[]{}%*-+~/#,:|.<>=!'; + +type SegmentType = 'text' | 'fenced-code' | 'inline-code' | 'html-comment'; + +interface Segment { + end: number; + start: number; + type: SegmentType; +} + +type NunjucksTokenType = 'block' | 'variable' | 'comment'; + +interface NunjucksToken { + end: number; + name?: string; + start: number; + type: NunjucksTokenType; +} + +const countBackticks = (str: string, start: number) => { + let end = start; + while (str[end] === '`') end++; + return end - start; +}; + +const isBlankLineBoundary = (str: string, index: number) => str.startsWith('\n\n', index) + || str.startsWith('\r\n\r\n', index) + || str.startsWith('\r\n\n', index) + || str.startsWith('\n\r\n', index); + +const rRawStart = /^\{%-?\s*raw\s*-?%\}/; +const rRawEnd = /\{%-?\s*endraw\s*-?%\}/; + +const findRawBlockEnd = (str: string, start: number) => { + const opening = rRawStart.exec(str.slice(start)); + if (!opening) return -1; + + const contentStart = start + opening[0].length; + const closing = rRawEnd.exec(str.slice(contentStart)); + return closing ? contentStart + closing.index + closing[0].length : -1; +}; + +const findInlineCodeEnd = (str: string, start: number, size: number) => { + let index = start + size; + while (index < str.length) { + if (isBlankLineBoundary(str, index)) return -1; + if (str.startsWith(BLOCK_START, index)) { + const rawEnd = findRawBlockEnd(str, index); + if (rawEnd !== -1) { + index = rawEnd; + continue; + } + } + if (str[index] === '`') { + const currentSize = countBackticks(str, index); + if (currentSize === size) return index + size; + index += currentSize; + } else { + index++; + } + } + return -1; +}; + +interface Fence { + char: '`' | '~'; + end: number; + size: number; +} + +const findFence = (str: string, start: number): Fence | undefined => { + if (start > 0 && str[start - 1] !== '\n') return; + + const lineEnd = str.indexOf('\n', start); + if (lineEnd === -1) return; + + const line = str.slice(start, lineEnd).replace(/\r$/, ''); + const match = /^((?:(?:[^\S\r\n]*>){0,3}|[-*+]|[0-9]+\.)[^\S\r\n]*)(`{3,}|~{3,})[^\r\n]*$/.exec(line); + if (!match) return; + + return { + char: match[2][0] as '`' | '~', + end: lineEnd + 1, + size: match[2].length + }; +}; + +const findFenceEnd = (str: string, fence: Fence) => { + let lineStart = fence.end; + while (lineStart < str.length) { + const lineEnd = str.indexOf('\n', lineStart); + const end = lineEnd === -1 ? str.length : lineEnd; + const line = str.slice(lineStart, end).replace(/\r$/, ''); + const match = /^(?:(?:[^\S\r\n]*>){0,3}[^\S\r\n]*)(`+|~+)\s*$/.exec(line); + if (match && match[1][0] === fence.char && match[1].length >= fence.size) { + return lineEnd === -1 ? str.length : lineEnd + 1; + } + if (lineEnd === -1) break; + lineStart = lineEnd + 1; + } + return str.length; +}; + +export const scanPostSegments = (str: string): Segment[] => { + const segments: Segment[] = []; + let textStart = 0; + let index = 0; + + const pushSegment = (type: SegmentType, start: number, end: number) => { + if (textStart < start) segments.push({ type: 'text', start: textStart, end: start }); + segments.push({ type, start, end }); + textStart = end; + index = end; + }; + + while (index < str.length) { + const fence = findFence(str, index); + if (fence) { + pushSegment('fenced-code', index, findFenceEnd(str, fence)); + continue; + } + + if (str[index] === '`' && str[index - 1] !== '\\') { + const size = countBackticks(str, index); + const end = findInlineCodeEnd(str, index, size); + if (end !== -1) { + pushSegment('inline-code', index, end); + continue; + } + index += size; + continue; + } + + if (str.startsWith('', index + 4); + pushSegment('html-comment', index, commentEnd === -1 ? str.length : commentEnd + 3); + continue; + } + + index++; + } + + if (textStart < str.length) segments.push({ type: 'text', start: textStart, end: str.length }); + return segments; +}; + +export const getHtmlCommentRanges = (str: string) => scanPostSegments(str) + .filter(segment => segment.type === 'html-comment') + .map(({ start, end }) => ({ start, end })); + +const findDelimiterEnd = (str: string, start: number, delimiter: string) => { + let quote = ''; + let regex = false; + let index = start; + + while (index < str.length) { + const char = str[index]; + if (quote) { + if (char === '\\') { + index += 2; + continue; + } + if (char === quote) quote = ''; + index++; + continue; + } + if (regex) { + if (char === '\\') { + index += 2; + continue; + } + if (char === '/') regex = false; + index++; + continue; + } + if (char === '"' || char === '\'') { + quote = char; + index++; + continue; + } + const previous = str[index - 1]; + if (char === 'r' && str[index + 1] === '/' + && (index === start || NUNJUCKS_TOKEN_BOUNDARIES.includes(previous))) { + regex = true; + index += 2; + continue; + } + if (str.startsWith(delimiter, index) || str.startsWith(`-${delimiter}`, index)) { + return index + delimiter.length + (str[index] === '-' ? 1 : 0); + } + index++; + } + return -1; +}; + +const getBlockName = (raw: string) => { + let content = raw.slice(BLOCK_START.length, -BLOCK_END.length).trim(); + if (content.startsWith('-')) content = content.slice(1).trimStart(); + if (content.endsWith('-')) content = content.slice(0, -1).trimEnd(); + return /^([^\s]+)/.exec(content)?.[1] || ''; +}; + +const scanNunjucks = (str: string): NunjucksToken[] => { + const tokens: NunjucksToken[] = []; + let index = 0; + + while (index < str.length) { + if (str.startsWith(COMMENT_START, index)) { + const close = str.indexOf(COMMENT_END, index + COMMENT_START.length); + if (close === -1) { + index += COMMENT_START.length; + continue; + } + const end = close + COMMENT_END.length; + tokens.push({ type: 'comment', start: index, end }); + index = end; + continue; + } + + if (str.startsWith(BLOCK_START, index)) { + const end = findDelimiterEnd(str, index + BLOCK_START.length, BLOCK_END); + if (end === -1) { + index += BLOCK_START.length; + continue; + } + tokens.push({ + type: 'block', + name: getBlockName(str.slice(index, end)), + start: index, + end + }); + index = end; + continue; + } + + if (str.startsWith(VARIABLE_START, index)) { + const end = findDelimiterEnd(str, index + VARIABLE_START.length, VARIABLE_END); + if (end === -1) { + index += VARIABLE_START.length; + continue; + } + tokens.push({ type: 'variable', start: index, end }); + index = end; + continue; + } + + index++; + } + + return tokens; +}; + +const pairNunjucksBlocks = (tokens: NunjucksToken[]) => { + const endNames = new Set(tokens + .filter(token => token.type === 'block' && token.name?.startsWith('end')) + .map(token => token.name!.slice(3))); + const stack: Array<{ index: number, name: string }> = []; + const pairs = new Map(); + + tokens.forEach((token, index) => { + if (token.type !== 'block' || !token.name) return; + if (token.name.startsWith('end')) { + const name = token.name.slice(3); + const opening = stack[stack.length - 1]; + if (opening?.name === name) { + pairs.set(opening.index, index); + stack.pop(); + } + } else if (endNames.has(token.name)) { + stack.push({ index, name: token.name }); + } + }); + + return pairs; +}; + +class PostRenderEscape { + private readonly codeBlocks: Array = []; + private readonly comments: Array = []; + private readonly swig: Array = []; + public hasNunjucks = false; + + private static escapeContent(cache: Array, flag: string, str: string) { + return ``; + } + + private static restoreContent(cache: Array) { + return (_: string, index: string) => { + const value = cache[Number(index)]; + assert(value != null); + cache[Number(index)] = null; + return value; + }; + } + + restoreAllSwigTags(str: string) { + return str.replace(rSwigPlaceHolder, PostRenderEscape.restoreContent(this.swig)); + } + + restoreCodeBlocks(str: string) { + return str.replace(rCodeBlockPlaceHolder, PostRenderEscape.restoreContent(this.codeBlocks)); + } + + restoreComments(str: string) { + return str.replace(rCommentHolder, PostRenderEscape.restoreContent(this.comments)); + } + + escapeCodeBlocks(str: string) { + return str.replace(/([\s\S]+?)<\/hexoPostRenderCodeBlock>/g, + (_, content) => PostRenderEscape.escapeContent(this.codeBlocks, 'code', content)); + } + + private escapeComment(str: string) { + // Nunjucks does not reparse expression output, so this recreates the first + // brace after rendering without evaluating the original comment contents. + const protectedComment = str.replace(/\{(?=[{%#])/g, () => { + this.hasNunjucks = true; + return '{{ "{" }}'; + }); + return PostRenderEscape.escapeContent(this.comments, 'comment', protectedComment); + } + + private escapeSwig(str: string) { + return PostRenderEscape.escapeContent(this.swig, 'swig', str); + } + + private escapeInlineCode(str: string) { + const tokens = scanNunjucks(str); + if (tokens.length === 0) return str; + + this.hasNunjucks = true; + const pairs = pairNunjucksBlocks(tokens); + let output = ''; + let cursor = 0; + + tokens.forEach((token, index) => { + if (token.start < cursor || token.type !== 'block' || token.name !== 'raw') return; + const closingIndex = pairs.get(index); + if (closingIndex == null) return; + + const closing = tokens[closingIndex]; + output += str.slice(cursor, token.start); + output += this.escapeSwig(str.slice(token.start, closing.end)); + cursor = closing.end; + }); + + return output + str.slice(cursor); + } + + escapeAllSwigTags(str: string) { + if (!str.includes(VARIABLE_START) && !str.includes(BLOCK_START) && !str.includes(COMMENT_START)) return str; + + const contexts: string[] = []; + const contextPlaceholder = (value: string) => `hexoPostRenderContext\uFFFC${contexts.push(value) - 1}\uFFFC`; + const restoreContexts = (value: string) => value.replace(rContextHolder, (_, index) => contexts[Number(index)]); + + const contextual = scanPostSegments(str).map(segment => { + const value = str.slice(segment.start, segment.end); + if (segment.type === 'html-comment') return this.escapeComment(value); + if (segment.type === 'inline-code') return contextPlaceholder(this.escapeInlineCode(value)); + if (segment.type === 'fenced-code') return contextPlaceholder(value); + return value; + }).join(''); + + const tokens = scanNunjucks(contextual); + if (tokens.length === 0) return restoreContexts(contextual); + + this.hasNunjucks = true; + const pairs = pairNunjucksBlocks(tokens); + let output = ''; + let cursor = 0; + + for (let index = 0; index < tokens.length; index++) { + const token = tokens[index]; + if (token.start < cursor) continue; + + output += contextual.slice(cursor, token.start); + const closingIndex = pairs.get(index); + if (closingIndex == null) { + output += this.escapeSwig(contextual.slice(token.start, token.end)); + cursor = token.end; + continue; + } + + const closing = tokens[closingIndex]; + output += this.escapeSwig(restoreContexts(contextual.slice(token.start, closing.end))); + cursor = closing.end; + index = closingIndex; + } + + output += contextual.slice(cursor); + return restoreContexts(output); + } +} + +export default PostRenderEscape; diff --git a/lib/plugins/filter/before_post_render/backtick_code_block.ts b/lib/plugins/filter/before_post_render/backtick_code_block.ts index 2097b6296..bc4954f7c 100644 --- a/lib/plugins/filter/before_post_render/backtick_code_block.ts +++ b/lib/plugins/filter/before_post_render/backtick_code_block.ts @@ -1,11 +1,11 @@ import type { HighlightOptions } from '../../../extend/syntax_highlight'; import type Hexo from '../../../hexo'; +import { getHtmlCommentRanges } from '../../../hexo/post_render_lexer'; import type { RenderData } from '../../../types'; const rBacktick = /^((?:(?:[^\S\r\n]*>){0,3}|[-*+]|[0-9]+\.)[^\S\r\n]*)(`{3,}|~{3,})[^\S\r\n]*((?:.*?[^`\s])?)[^\S\r\n]*\n((?:[\s\S]*?\n)?)(?:(?:[^\S\r\n]*>){0,3}[^\S\r\n]*)\2[^\S\r\n]?(\n+|$)/gm; const rAllOptions = /([^\s]+)\s+(.+?)\s+(https?:\/\/\S+|\/\S+)\s*(.+)?/; const rLangCaption = /([^\s]+)\s*(.+)?/; -const rCommentEscape = /()/g; const rAdditionalOptions = /\s((?:line_number|line_threshold|first_line|wrap|mark|language_attr|highlight):\S+)/g; const escapeSwigTag = (str: string) => str.replace(/{/g, '{').replace(/}/g, '}'); @@ -86,26 +86,17 @@ export = (ctx: Hexo): (data: RenderData) => void => { const dataContent = data.content; if ((!dataContent.includes('```') && !dataContent.includes('~~~')) || !ctx.extend.highlight.query(ctx.config.syntax_highlighter)) return; - // get all comment starts and ends - const commentStarts = []; - const commentEnds = []; - let match: RegExpExecArray | null; - rCommentEscape.lastIndex = 0; - while ((match = rCommentEscape.exec(dataContent)) !== null) { - commentStarts.push(match.index); - commentEnds.push(match.index + match[0].length); - } - // notice that commentStarts and commentEnds are sorted, and commentStarts[i] < commentEnds[i], commentEnds[i] <= commentStarts[i+1] + const comments = getHtmlCommentRanges(dataContent); let commentIndex = 0; data.content = data.content.replace(rBacktick, ($0, start, $2, _args, _content, end, matchIndex) => { // get the start and end of the code block const codeBlockStart = matchIndex; const codeBlockEnd = matchIndex + $0.length; // check if the code block is nested in a comment - while (commentIndex < commentStarts.length && commentEnds[commentIndex] <= codeBlockStart) { + while (commentIndex < comments.length && comments[commentIndex].end <= codeBlockStart) { commentIndex++; } - if (commentIndex < commentStarts.length && commentStarts[commentIndex] < codeBlockStart && commentEnds[commentIndex] > codeBlockEnd) { + if (commentIndex < comments.length && comments[commentIndex].start < codeBlockStart && comments[commentIndex].end > codeBlockEnd) { // the code block is nested in a comment, return escaped content directly return escapeSwigTag($0); } diff --git a/test/scripts/hexo/post.ts b/test/scripts/hexo/post.ts index 3e43ad01e..4f3dd2f38 100644 --- a/test/scripts/hexo/post.ts +++ b/test/scripts/hexo/post.ts @@ -1829,6 +1829,75 @@ describe('Post', () => { hexo.extend.tag.unregister('testTag'); }); + // https://github.com/hexojs/hexo/issues/5799 + it('render() - comment protocol in same-name nested tags', async () => { + hexo.extend.tag.register('folding', (_args, content) => `
${content}
`, { + ends: true + }); + hexo.extend.tag.register('tabs', (_args, content) => { + const match = /\n([\s\S]*?)/.exec(content); + if (!match) return '
'; + return `
${match[2]}
`; + }, { + ends: true + }); + + const content = [ + '{% folding outer %}', + '{% tabs install %}', + '', + '{% folding inner %}', + 'visible content', + '{% endfolding %}', + '', + '{% endtabs %}', + '{% endfolding %}' + ].join('\n'); + + try { + const data = await post.render('', { + content, + engine: 'markdown' + }); + + data.content.should.include('data-caption="`JavaScript`"'); + data.content.should.include('visible content'); + } finally { + hexo.extend.tag.unregister('tabs'); + hexo.extend.tag.unregister('folding'); + } + }); + + it('render() - nunjucks remains literal in comment protocol', async () => { + const tagSpy = spy(); + hexo.extend.tag.register('commentProtocol', (_args, content) => { + tagSpy(content); + return ''; + }, { + ends: true + }); + + const content = [ + '{% commentProtocol %}', + '', + 'content', + '', + '{% endcommentProtocol %}' + ].join('\n'); + + try { + await post.render('', { + content, + engine: 'markdown' + }); + + tagSpy.calledOnce.should.be.true; + tagSpy.firstCall.args[0].should.include(''); + } finally { + hexo.extend.tag.unregister('commentProtocol'); + } + }); + // https://github.com/hexojs/hexo/issues/5433 it('render() - code fence nesting in comments', async () => { const code = 'alert("Hello world")'; diff --git a/test/scripts/hexo/post_render_lexer.ts b/test/scripts/hexo/post_render_lexer.ts new file mode 100644 index 000000000..7ef9d5f17 --- /dev/null +++ b/test/scripts/hexo/post_render_lexer.ts @@ -0,0 +1,85 @@ +import chai from 'chai'; +import nunjucks from 'nunjucks'; +import PostRenderEscape, { getHtmlCommentRanges } from '../../../lib/hexo/post_render_lexer'; + +chai.should(); + +describe('PostRenderEscape', () => { + it('finds HTML comments only outside Markdown code', () => { + const content = [ + '```html', + '', + '```', + '> ~~~html', + '> ', + '> ~~~', + '``', + '' + ].join('\n'); + + const comments = getHtmlCommentRanges(content) + .map(({ start, end }) => content.slice(start, end)); + + comments.should.eql(['']); + }); + + it('pairs same-name nested Nunjucks blocks', () => { + const content = [ + '{% folding outer %}', + '{% tabs install %}', + '{% folding inner %}', + '{% endfolding %}', + '{% endtabs %}', + '{% endfolding %}' + ].join('\n'); + const escape = new PostRenderEscape(); + + const escaped = escape.escapeAllSwigTags(content); + + (escaped.match(/'; + const escape = new PostRenderEscape(); + + const escaped = escape.escapeAllSwigTags(content); + const restored = escape.restoreComments(escape.restoreAllSwigTags(escaped)); + + nunjucks.renderString(restored, {}).should.eql(content); + }); + + it('preserves raw blocks containing backticks in inline code', () => { + const content = '`{% raw %}test`111`{{ value }}{% endraw %}`'; + const escape = new PostRenderEscape(); + + const escaped = escape.escapeAllSwigTags(content); + + escape.hasNunjucks.should.be.true; + escape.restoreAllSwigTags(escaped).should.eql(content); + }); + + it('does not close a Nunjucks variable on a delimiter inside a string', () => { + const content = 'before {{ "}}" }} after'; + const escape = new PostRenderEscape(); + + const escaped = escape.escapeAllSwigTags(content); + + (escaped.match(/', index + 4); - pushSegment('html-comment', index, commentEnd === -1 ? str.length : commentEnd + 3); + pushSegment({ type: 'html-comment', start: index, end: commentEnd === -1 ? str.length : commentEnd + 3 }); continue; } @@ -179,10 +234,6 @@ export const scanPostSegments = (str: string): Segment[] => { return segments; }; -export const getHtmlCommentRanges = (str: string) => scanPostSegments(str) - .filter(segment => segment.type === 'html-comment') - .map(({ start, end }) => ({ start, end })); - const findDelimiterEnd = (str: string, start: number, delimiter: string) => { let quote = ''; let regex = false; diff --git a/lib/plugins/filter/before_post_render/backtick_code_block.ts b/lib/plugins/filter/before_post_render/backtick_code_block.ts index bc4954f7c..3cfdfaa7e 100644 --- a/lib/plugins/filter/before_post_render/backtick_code_block.ts +++ b/lib/plugins/filter/before_post_render/backtick_code_block.ts @@ -1,9 +1,8 @@ import type { HighlightOptions } from '../../../extend/syntax_highlight'; import type Hexo from '../../../hexo'; -import { getHtmlCommentRanges } from '../../../hexo/post_render_lexer'; +import { scanPostSegments } from '../../../hexo/post_render_lexer'; import type { RenderData } from '../../../types'; -const rBacktick = /^((?:(?:[^\S\r\n]*>){0,3}|[-*+]|[0-9]+\.)[^\S\r\n]*)(`{3,}|~{3,})[^\S\r\n]*((?:.*?[^`\s])?)[^\S\r\n]*\n((?:[\s\S]*?\n)?)(?:(?:[^\S\r\n]*>){0,3}[^\S\r\n]*)\2[^\S\r\n]?(\n+|$)/gm; const rAllOptions = /([^\s]+)\s+(.+?)\s+(https?:\/\/\S+|\/\S+)\s*(.+)?/; const rLangCaption = /([^\s]+)\s*(.+)?/; const rAdditionalOptions = /\s((?:line_number|line_threshold|first_line|wrap|mark|language_attr|highlight):\S+)/g; @@ -86,35 +85,22 @@ export = (ctx: Hexo): (data: RenderData) => void => { const dataContent = data.content; if ((!dataContent.includes('```') && !dataContent.includes('~~~')) || !ctx.extend.highlight.query(ctx.config.syntax_highlighter)) return; - const comments = getHtmlCommentRanges(dataContent); - let commentIndex = 0; - data.content = data.content.replace(rBacktick, ($0, start, $2, _args, _content, end, matchIndex) => { - // get the start and end of the code block - const codeBlockStart = matchIndex; - const codeBlockEnd = matchIndex + $0.length; - // check if the code block is nested in a comment - while (commentIndex < comments.length && comments[commentIndex].end <= codeBlockStart) { - commentIndex++; - } - if (commentIndex < comments.length && comments[commentIndex].start < codeBlockStart && comments[commentIndex].end > codeBlockEnd) { - // the code block is nested in a comment, return escaped content directly - return escapeSwigTag($0); - } - let content = _content.replace(/\n$/, ''); - - // neither highlight or prismjs is enabled, return escaped content directly. - if (!ctx.extend.highlight.query(ctx.config.syntax_highlighter)) return escapeSwigTag($0); + data.content = scanPostSegments(dataContent).map(segment => { + const source = dataContent.slice(segment.start, segment.end); + if (segment.type !== 'fenced-code' || !segment.closed) return source; - const parsedArgs = parseArgs(_args); - if (!parsedArgs.enableHighlight) return escapeSwigTag($0); - _args = parsedArgs._args; + let content = dataContent.slice(segment.contentStart, segment.contentEnd).replace(/\r?\n$/, ''); + let args = segment.info; + const parsedArgs = parseArgs(args); + if (!parsedArgs.enableHighlight) return escapeSwigTag(source); + args = parsedArgs._args; // Extract language and caption of code blocks - const args = _args.split('=').shift(); + const langArgs = args.split('=').shift(); let lang: string, caption: string; - if (args) { - const match = rAllOptions.exec(args) || rLangCaption.exec(args); + if (langArgs) { + const match = rAllOptions.exec(langArgs) || rLangCaption.exec(langArgs); if (match) { lang = match[1]; @@ -130,9 +116,8 @@ export = (ctx: Hexo): (data: RenderData) => void => { } // PR #3765 - if (start.includes('>')) { - // heading of last line is already removed by the top RegExp "rBacktick" - const depth = start.split('>').length - 1; + if (segment.prefix.includes('>')) { + const depth = segment.prefix.split('>').length - 1; const regexp = new RegExp(`^([^\\S\\r\\n]*>){0,${depth}}([^\\S\\r\\n]|$)`, 'mg'); content = content.replace(regexp, ''); } @@ -144,22 +129,22 @@ export = (ctx: Hexo): (data: RenderData) => void => { ...parsedArgs.options }; // setup line number by inline - _args = _args.replace('=+', '='); + args = args.replace('=+', '='); // setup firstLineNumber; - if (_args.includes('=')) { - options.firstLineNumber = _args.split('=')[1] || 1; + if (args.includes('=')) { + options.firstLineNumber = args.split('=')[1] || 1; } content = ctx.extend.highlight.exec(ctx.config.syntax_highlighter, { context: ctx, args: [content, options] }); - return start + return segment.prefix + '' + escapeSwigTag(content) + '' - + end; - }); + + dataContent.slice(segment.closingEnd, segment.end); + }).join(''); }; }; diff --git a/test/scripts/filters/backtick_code_block.ts b/test/scripts/filters/backtick_code_block.ts index 83d2a11ea..e2efee3ea 100644 --- a/test/scripts/filters/backtick_code_block.ts +++ b/test/scripts/filters/backtick_code_block.ts @@ -464,6 +464,62 @@ describe('Backtick code block', () => { data.content.should.eql('```foo```\n\n' + highlight(code, {}) + ''); }); + it('does not process a code fence inside an HTML comment', () => { + const content = [ + '' + ].join('\n'); + const data = { content }; + + codeBlock(data); + + data.content.should.eql(content); + }); + + it('processes an HTML comment inside a code fence', () => { + const source = ''; + const data = { + content: [ + '```html', + source, + '```' + ].join('\n') + }; + + codeBlock(data); + + data.content.should.eql('' + highlight(source, { lang: 'html' }) + ''); + }); + + it('allows a closing fence longer than the opening fence', () => { + const data = { + content: [ + '```js', + code, + '````' + ].join('\n') + }; + + codeBlock(data); + + data.content.should.eql('' + highlight(code, { lang: 'js' }) + ''); + }); + + it('leaves an unclosed code fence unchanged', () => { + const content = [ + '```js', + code + ].join('\n'); + const data = { content }; + + codeBlock(data); + + data.content.should.eql(content); + }); + // test for Issue #4190 it('ignore triple backticks at the line which is started by extra characters', () => { const data = { diff --git a/test/scripts/hexo/post.ts b/test/scripts/hexo/post.ts index 4f3dd2f38..49a015a8c 100644 --- a/test/scripts/hexo/post.ts +++ b/test/scripts/hexo/post.ts @@ -1900,7 +1900,7 @@ describe('Post', () => { // https://github.com/hexojs/hexo/issues/5433 it('render() - code fence nesting in comments', async () => { - const code = 'alert("Hello world")'; + const code = 'const value = {{ value }};'; const content = [ 'foo', '' ].join('\n'); - const comments = getHtmlCommentRanges(content) + const comments = scanPostSegments(content) + .filter(segment => segment.type === 'html-comment') .map(({ start, end }) => content.slice(start, end)); comments.should.eql(['']); }); + it('returns fenced code metadata for downstream filters', () => { + const content = [ + '> ```js', + '> const value = 1;', + '> ````', + 'after' + ].join('\n'); + + const fence = scanPostSegments(content).find(segment => segment.type === 'fenced-code'); + + chai.expect(fence).to.exist; + if (!fence || fence.type !== 'fenced-code') return; + fence.closed.should.be.true; + fence.prefix.should.eql('> '); + fence.marker.should.eql('```'); + fence.info.should.eql('js'); + content.slice(fence.contentStart, fence.contentEnd).should.eql('> const value = 1;\n'); + content.slice(fence.closingEnd, fence.end).should.eql('\n'); + }); + it('pairs same-name nested Nunjucks blocks', () => { const content = [ '{% folding outer %}', From d7a3047970e4888729dea33526d76c500a72df06 Mon Sep 17 00:00:00 2001 From: Mimi <1119186082@qq.com> Date: Fri, 24 Jul 2026 13:22:44 +0800 Subject: [PATCH 3/4] refactor(post): move fenced code processing into renderer --- lib/extend/syntax_highlight.ts | 2 +- lib/hexo/post.ts | 19 +- lib/hexo/post_render_lexer.ts | 148 +-------- lib/hexo/post_render_processor.ts | 282 ++++++++++++++++++ .../before_post_render/backtick_code_block.ts | 150 ---------- .../filter/before_post_render/index.ts | 1 - test/scripts/hexo/post.ts | 42 +++ test/scripts/hexo/post_render_lexer.ts | 42 +-- .../post_render_processor.ts} | 151 +++++----- 9 files changed, 441 insertions(+), 396 deletions(-) create mode 100644 lib/hexo/post_render_processor.ts delete mode 100644 lib/plugins/filter/before_post_render/backtick_code_block.ts rename test/scripts/{filters/backtick_code_block.ts => hexo/post_render_processor.ts} (75%) diff --git a/lib/extend/syntax_highlight.ts b/lib/extend/syntax_highlight.ts index d5cde7e09..f2cc97ad5 100644 --- a/lib/extend/syntax_highlight.ts +++ b/lib/extend/syntax_highlight.ts @@ -5,7 +5,7 @@ export interface HighlightOptions { caption: string | undefined, lines_length?: number | undefined, - // plugins/filter/before_post_render/backtick_code_block + // hexo/post_render_processor firstLineNumber?: string | number // plugins/tag/code.ts diff --git a/lib/hexo/post.ts b/lib/hexo/post.ts index d382e1dc1..972f6c181 100644 --- a/lib/hexo/post.ts +++ b/lib/hexo/post.ts @@ -7,7 +7,7 @@ import { slugize, escapeRegExp, deepMerge} from 'hexo-util'; import { copyDir, exists, listDir, mkdirs, readFile, rmdir, unlink, writeFile } from 'hexo-fs'; import { parse as yfmParse, split as yfmSplit, stringify as yfmStringify } from 'hexo-front-matter'; import type Hexo from './index'; -import PostRenderEscape from './post_render_lexer'; +import PostRenderProcessor from './post_render_processor'; import type { NodeJSLikeCallback, RenderData } from '../types'; const preservedKeys = ['title', 'slug', 'path', 'layout', 'date', 'content']; @@ -250,19 +250,14 @@ class Post { // front-matter overrides renderer's option if (typeof data.disableNunjucks === 'boolean') disableNunjucks = data.disableNunjucks; - const cacheObj = new PostRenderEscape(); + const processor = new PostRenderProcessor(ctx); return promise.then(content => { data.content = content; // Run "before_post_render" filters return ctx.execFilter('before_post_render', data, { context: ctx }); }).then(() => { - data.content = cacheObj.escapeCodeBlocks(data.content); - let hasSwigTag = false; - if (!disableNunjucks) { - data.content = cacheObj.escapeAllSwigTags(data.content); - hasSwigTag = cacheObj.hasNunjucks; - } + data.content = processor.prepare(data.content, !disableNunjucks); const options: { highlight?: boolean; } = data.markdown || {}; if (!config.syntax_highlighter) options.highlight = null; @@ -276,18 +271,18 @@ class Post { toString: true, onRenderEnd(content) { // Replace cache data with real contents - data.content = cacheObj.restoreAllSwigTags(content); - data.content = cacheObj.restoreComments(data.content); + data.content = processor.restoreAllSwigTags(content); + data.content = processor.restoreComments(data.content); // Return content after replace the placeholders - if (disableNunjucks || !hasSwigTag) return data.content; + if (disableNunjucks || !processor.hasNunjucks) return data.content; // Render with Nunjucks if there are Swig tags return tag.render(data.content, data); } }, options); }).then(content => { - data.content = cacheObj.restoreCodeBlocks(content); + data.content = processor.restoreCodeBlocks(content); // Run "after_post_render" filters return ctx.execFilter('after_post_render', data, { context: ctx }); diff --git a/lib/hexo/post_render_lexer.ts b/lib/hexo/post_render_lexer.ts index 7c9615139..495d52fa3 100644 --- a/lib/hexo/post_render_lexer.ts +++ b/lib/hexo/post_render_lexer.ts @@ -1,5 +1,3 @@ -import assert from 'assert'; - /* * Nunjucks delimiter handling in this file is adapted from its lexer. * @@ -22,17 +20,12 @@ import assert from 'assert'; * SUCH DAMAGE. */ -const rSwigPlaceHolder = /(?:<|<)!--swig\uFFFC(\d+)--(?:>|>)/g; -const rCodeBlockPlaceHolder = /(?:<|<)!--code\uFFFC(\d+)--(?:>|>)/g; -const rCommentHolder = /(?:<|<)!--comment\uFFFC(\d+)--(?:>|>)/g; -const rContextHolder = /hexoPostRenderContext\uFFFC(\d+)\uFFFC/g; - -const BLOCK_START = '{%'; -const BLOCK_END = '%}'; -const VARIABLE_START = '{{'; -const VARIABLE_END = '}}'; -const COMMENT_START = '{#'; -const COMMENT_END = '#}'; +export const BLOCK_START = '{%'; +export const BLOCK_END = '%}'; +export const VARIABLE_START = '{{'; +export const VARIABLE_END = '}}'; +export const COMMENT_START = '{#'; +export const COMMENT_END = '#}'; const NUNJUCKS_TOKEN_BOUNDARIES = ' \n\t\r\u00A0()[]{}%*-+~/#,:|.<>=!'; type SimpleSegmentType = 'text' | 'inline-code' | 'html-comment'; @@ -58,9 +51,9 @@ export interface FencedCodeSegment { export type PostSegment = SimpleSegment | FencedCodeSegment; -type NunjucksTokenType = 'block' | 'variable' | 'comment'; +export type NunjucksTokenType = 'block' | 'variable' | 'comment'; -interface NunjucksToken { +export interface NunjucksToken { end: number; name?: string; start: number; @@ -286,7 +279,7 @@ const getBlockName = (raw: string) => { return /^([^\s]+)/.exec(content)?.[1] || ''; }; -const scanNunjucks = (str: string): NunjucksToken[] => { +export const scanNunjucks = (str: string): NunjucksToken[] => { const tokens: NunjucksToken[] = []; let index = 0; @@ -336,7 +329,7 @@ const scanNunjucks = (str: string): NunjucksToken[] => { return tokens; }; -const pairNunjucksBlocks = (tokens: NunjucksToken[]) => { +export const pairNunjucksBlocks = (tokens: NunjucksToken[]) => { const endNames = new Set(tokens .filter(token => token.type === 'block' && token.name?.startsWith('end')) .map(token => token.name!.slice(3))); @@ -359,124 +352,3 @@ const pairNunjucksBlocks = (tokens: NunjucksToken[]) => { return pairs; }; - -class PostRenderEscape { - private readonly codeBlocks: Array = []; - private readonly comments: Array = []; - private readonly swig: Array = []; - public hasNunjucks = false; - - private static escapeContent(cache: Array, flag: string, str: string) { - return ``; - } - - private static restoreContent(cache: Array) { - return (_: string, index: string) => { - const value = cache[Number(index)]; - assert(value != null); - cache[Number(index)] = null; - return value; - }; - } - - restoreAllSwigTags(str: string) { - return str.replace(rSwigPlaceHolder, PostRenderEscape.restoreContent(this.swig)); - } - - restoreCodeBlocks(str: string) { - return str.replace(rCodeBlockPlaceHolder, PostRenderEscape.restoreContent(this.codeBlocks)); - } - - restoreComments(str: string) { - return str.replace(rCommentHolder, PostRenderEscape.restoreContent(this.comments)); - } - - escapeCodeBlocks(str: string) { - return str.replace(/([\s\S]+?)<\/hexoPostRenderCodeBlock>/g, - (_, content) => PostRenderEscape.escapeContent(this.codeBlocks, 'code', content)); - } - - private escapeComment(str: string) { - // Nunjucks does not reparse expression output, so this recreates the first - // brace after rendering without evaluating the original comment contents. - const protectedComment = str.replace(/\{(?=[{%#])/g, () => { - this.hasNunjucks = true; - return '{{ "{" }}'; - }); - return PostRenderEscape.escapeContent(this.comments, 'comment', protectedComment); - } - - private escapeSwig(str: string) { - return PostRenderEscape.escapeContent(this.swig, 'swig', str); - } - - private escapeInlineCode(str: string) { - const tokens = scanNunjucks(str); - if (tokens.length === 0) return str; - - this.hasNunjucks = true; - const pairs = pairNunjucksBlocks(tokens); - let output = ''; - let cursor = 0; - - tokens.forEach((token, index) => { - if (token.start < cursor || token.type !== 'block' || token.name !== 'raw') return; - const closingIndex = pairs.get(index); - if (closingIndex == null) return; - - const closing = tokens[closingIndex]; - output += str.slice(cursor, token.start); - output += this.escapeSwig(str.slice(token.start, closing.end)); - cursor = closing.end; - }); - - return output + str.slice(cursor); - } - - escapeAllSwigTags(str: string) { - if (!str.includes(VARIABLE_START) && !str.includes(BLOCK_START) && !str.includes(COMMENT_START)) return str; - - const contexts: string[] = []; - const contextPlaceholder = (value: string) => `hexoPostRenderContext\uFFFC${contexts.push(value) - 1}\uFFFC`; - const restoreContexts = (value: string) => value.replace(rContextHolder, (_, index) => contexts[Number(index)]); - - const contextual = scanPostSegments(str).map(segment => { - const value = str.slice(segment.start, segment.end); - if (segment.type === 'html-comment') return this.escapeComment(value); - if (segment.type === 'inline-code') return contextPlaceholder(this.escapeInlineCode(value)); - if (segment.type === 'fenced-code') return contextPlaceholder(value); - return value; - }).join(''); - - const tokens = scanNunjucks(contextual); - if (tokens.length === 0) return restoreContexts(contextual); - - this.hasNunjucks = true; - const pairs = pairNunjucksBlocks(tokens); - let output = ''; - let cursor = 0; - - for (let index = 0; index < tokens.length; index++) { - const token = tokens[index]; - if (token.start < cursor) continue; - - output += contextual.slice(cursor, token.start); - const closingIndex = pairs.get(index); - if (closingIndex == null) { - output += this.escapeSwig(contextual.slice(token.start, token.end)); - cursor = token.end; - continue; - } - - const closing = tokens[closingIndex]; - output += this.escapeSwig(restoreContexts(contextual.slice(token.start, closing.end))); - cursor = closing.end; - index = closingIndex; - } - - output += contextual.slice(cursor); - return restoreContexts(output); - } -} - -export default PostRenderEscape; diff --git a/lib/hexo/post_render_processor.ts b/lib/hexo/post_render_processor.ts new file mode 100644 index 000000000..059814841 --- /dev/null +++ b/lib/hexo/post_render_processor.ts @@ -0,0 +1,282 @@ +import assert from 'assert'; +import type { HighlightOptions } from '../extend/syntax_highlight'; +import type Hexo from './index'; +import { + BLOCK_START, + COMMENT_START, + type FencedCodeSegment, + VARIABLE_START, + pairNunjucksBlocks, + scanNunjucks, + scanPostSegments +} from './post_render_lexer'; + +const rSwigPlaceHolder = /(?:<|<)!--swig\uFFFC(\d+)--(?:>|>)/g; +const rCodeBlockPlaceHolder = /(?:<|<)!--code\uFFFC(\d+)--(?:>|>)/g; +const rCommentHolder = /(?:<|<)!--comment\uFFFC(\d+)--(?:>|>)/g; +const rContextHolder = /hexoPostRenderContext\uFFFC(\d+)\uFFFC/g; + +const rAllOptions = /([^\s]+)\s+(.+?)\s+(https?:\/\/\S+|\/\S+)\s*(.+)?/; +const rLangCaption = /([^\s]+)\s*(.+)?/; +const rAdditionalOptions = /\s((?:line_number|line_threshold|first_line|wrap|mark|language_attr|highlight):\S+)/g; + +const escapeCodeBraces = (str: string) => str.replace(/{/g, '{').replace(/}/g, '}'); +const hasNunjucksSyntax = (str: string) => str.includes(VARIABLE_START) + || str.includes(BLOCK_START) + || str.includes(COMMENT_START); + +function parseArgs(args: string) { + const matches = []; + + let match: RegExpExecArray | null, language_attr: boolean, + line_number: boolean, line_threshold: number, wrap: boolean; + let enableHighlight = true; + while ((match = rAdditionalOptions.exec(args)) !== null) { + matches.push(match[1]); + } + + const mark: number[] = []; + let firstLine = 1; + for (const option of matches) { + const [key, value] = option.split(':'); + + switch (key) { + case 'highlight': + enableHighlight = value === 'true'; + break; + case 'line_number': + line_number = value === 'true'; + break; + case 'line_threshold': + if (!isNaN(Number(value))) line_threshold = +value; + break; + case 'first_line': + if (!isNaN(Number(value))) firstLine = +value; + break; + case 'wrap': + wrap = value === 'true'; + break; + case 'mark': { + for (const cur of value.split(',')) { + const hyphen = cur.indexOf('-'); + if (hyphen !== -1) { + let a = +cur.slice(0, hyphen); + let b = +cur.slice(hyphen + 1); + if (Number.isNaN(a) || Number.isNaN(b)) continue; + if (b < a) [a, b] = [b, a]; + + for (; a <= b; a++) mark.push(a); + } + if (!isNaN(Number(cur))) mark.push(+cur); + } + break; + } + case 'language_attr': + language_attr = value === 'true'; + break; + } + } + + return { + options: { + language_attr, + firstLine, + line_number, + line_threshold, + mark, + wrap + }, + enableHighlight, + args: args.replace(rAdditionalOptions, '') + }; +} + +class PostRenderProcessor { + private readonly codeBlocks: Array = []; + private readonly comments: Array = []; + private readonly swig: Array = []; + public hasNunjucks = false; + + constructor(private readonly ctx: Hexo) {} + + private static escapeContent(cache: Array, flag: string, str: string) { + return ``; + } + + private static restoreContent(cache: Array) { + return (_: string, index: string) => { + const value = cache[Number(index)]; + assert(value != null); + cache[Number(index)] = null; + return value; + }; + } + + restoreAllSwigTags(str: string) { + return str.replace(rSwigPlaceHolder, PostRenderProcessor.restoreContent(this.swig)); + } + + restoreCodeBlocks(str: string) { + return str.replace(rCodeBlockPlaceHolder, PostRenderProcessor.restoreContent(this.codeBlocks)); + } + + restoreComments(str: string) { + return str.replace(rCommentHolder, PostRenderProcessor.restoreContent(this.comments)); + } + + private escapeCodeBlock(str: string) { + // Keep the historical HTML output while code remains hidden from Nunjucks. + return PostRenderProcessor.escapeContent(this.codeBlocks, 'code', escapeCodeBraces(str)); + } + + private protectMarkdownCode(str: string) { + if (!hasNunjucksSyntax(str)) return str; + + return str.replace(/[{}]/g, brace => PostRenderProcessor.escapeContent( + this.codeBlocks, + 'code', + brace === '{' ? '{' : '}' + )); + } + + private prepareInlineCode(str: string) { + const tokens = scanNunjucks(str); + if (tokens.length === 0) return str; + + this.hasNunjucks = true; + const pairs = pairNunjucksBlocks(tokens); + let output = ''; + let cursor = 0; + + tokens.forEach((token, index) => { + if (token.start < cursor || token.type !== 'block' || token.name !== 'raw') return; + const closingIndex = pairs.get(index); + if (closingIndex == null) return; + + const closing = tokens[closingIndex]; + output += str.slice(cursor, token.start); + output += this.escapeSwig(str.slice(token.start, closing.end)); + cursor = closing.end; + }); + + return output + str.slice(cursor); + } + + private escapeComment(str: string) { + // Nunjucks does not reparse expression output, so this recreates the first + // brace after rendering without evaluating the original comment contents. + const protectedComment = str.replace(/\{(?=[{%#])/g, () => { + this.hasNunjucks = true; + return '{{ "{" }}'; + }); + return PostRenderProcessor.escapeContent(this.comments, 'comment', protectedComment); + } + + private escapeSwig(str: string) { + return PostRenderProcessor.escapeContent(this.swig, 'swig', str); + } + + private renderFencedCode(str: string, segment: FencedCodeSegment, canHighlight: boolean) { + if (!segment.closed || !canHighlight) return; + + const parsedArgs = parseArgs(segment.info); + if (!parsedArgs.enableHighlight) return; + + let content = str.slice(segment.contentStart, segment.contentEnd).replace(/\r?\n$/, ''); + let args = parsedArgs.args; + const langArgs = args.split('=').shift(); + let lang: string, caption: string; + + if (langArgs) { + const match = rAllOptions.exec(langArgs) || rLangCaption.exec(langArgs); + if (match) { + lang = match[1]; + if (match[2]) { + caption = `${match[2]}`; + if (match[3]) caption += `${match[4] || 'link'}`; + } + } + } + + if (segment.prefix.includes('>')) { + const depth = segment.prefix.split('>').length - 1; + const regexp = new RegExp(`^([^\\S\\r\\n]*>){0,${depth}}([^\\S\\r\\n]|$)`, 'mg'); + content = content.replace(regexp, ''); + } + + const options: HighlightOptions = { + lang, + caption, + lines_length: content.split('\n').length, + ...parsedArgs.options + }; + args = args.replace('=+', '='); + if (args.includes('=')) options.firstLineNumber = args.split('=')[1] || 1; + + content = this.ctx.extend.highlight.exec(this.ctx.config.syntax_highlighter, { + context: this.ctx, + args: [content, options] + }); + + return segment.prefix + + this.escapeCodeBlock(content) + + str.slice(segment.closingEnd, segment.end); + } + + prepare(str: string, enableNunjucks = true) { + const containsNunjucks = enableNunjucks && hasNunjucksSyntax(str); + const canHighlight = (str.includes('```') || str.includes('~~~')) + && !!this.ctx.extend.highlight.query(this.ctx.config.syntax_highlighter); + if (!containsNunjucks && !canHighlight) return str; + + const contexts: string[] = []; + const contextPlaceholder = (value: string) => `hexoPostRenderContext\uFFFC${contexts.push(value) - 1}\uFFFC`; + const restoreContexts = (value: string) => value.replace(rContextHolder, (_, index) => contexts[Number(index)]); + + const contextual = scanPostSegments(str).map(segment => { + const value = str.slice(segment.start, segment.end); + if (segment.type === 'fenced-code') { + const rendered = this.renderFencedCode(str, segment, canHighlight); + if (rendered != null) return rendered; + return containsNunjucks ? contextPlaceholder(this.protectMarkdownCode(value)) : value; + } + if (!containsNunjucks) return value; + if (segment.type === 'html-comment') return this.escapeComment(value); + if (segment.type === 'inline-code') return contextPlaceholder(this.prepareInlineCode(value)); + return value; + }).join(''); + + if (!containsNunjucks) return contextual; + + const tokens = scanNunjucks(contextual); + if (tokens.length === 0) return restoreContexts(contextual); + + this.hasNunjucks = true; + const pairs = pairNunjucksBlocks(tokens); + let output = ''; + let cursor = 0; + + for (let index = 0; index < tokens.length; index++) { + const token = tokens[index]; + if (token.start < cursor) continue; + + output += contextual.slice(cursor, token.start); + const closingIndex = pairs.get(index); + if (closingIndex == null) { + output += this.escapeSwig(contextual.slice(token.start, token.end)); + cursor = token.end; + continue; + } + + const closing = tokens[closingIndex]; + output += this.escapeSwig(restoreContexts(contextual.slice(token.start, closing.end))); + cursor = closing.end; + index = closingIndex; + } + + output += contextual.slice(cursor); + return restoreContexts(output); + } +} + +export default PostRenderProcessor; diff --git a/lib/plugins/filter/before_post_render/backtick_code_block.ts b/lib/plugins/filter/before_post_render/backtick_code_block.ts deleted file mode 100644 index 3cfdfaa7e..000000000 --- a/lib/plugins/filter/before_post_render/backtick_code_block.ts +++ /dev/null @@ -1,150 +0,0 @@ -import type { HighlightOptions } from '../../../extend/syntax_highlight'; -import type Hexo from '../../../hexo'; -import { scanPostSegments } from '../../../hexo/post_render_lexer'; -import type { RenderData } from '../../../types'; - -const rAllOptions = /([^\s]+)\s+(.+?)\s+(https?:\/\/\S+|\/\S+)\s*(.+)?/; -const rLangCaption = /([^\s]+)\s*(.+)?/; -const rAdditionalOptions = /\s((?:line_number|line_threshold|first_line|wrap|mark|language_attr|highlight):\S+)/g; - -const escapeSwigTag = (str: string) => str.replace(/{/g, '{').replace(/}/g, '}'); - -function parseArgs(args: string) { - const matches = []; - - let match: RegExpExecArray | null, language_attr: boolean, - line_number: boolean, line_threshold: number, wrap: boolean; - let enableHighlight = true; - while ((match = rAdditionalOptions.exec(args)) !== null) { - matches.push(match[1]); - } - - const len = matches.length; - const mark: number[] = []; - let firstLine = 1; - for (let i = 0; i < len; i++) { - const [key, value] = matches[i].split(':'); - - switch (key) { - case 'highlight': - enableHighlight = value === 'true'; - break; - case 'line_number': - line_number = value === 'true'; - break; - case 'line_threshold': - if (!isNaN(Number(value))) line_threshold = +value; - break; - case 'first_line': - if (!isNaN(Number(value))) firstLine = +value; - break; - case 'wrap': - wrap = value === 'true'; - break; - case 'mark': { - for (const cur of value.split(',')) { - const hyphen = cur.indexOf('-'); - if (hyphen !== -1) { - let a = +cur.slice(0, hyphen); - let b = +cur.slice(hyphen + 1); - if (Number.isNaN(a) || Number.isNaN(b)) continue; - if (b < a) { // switch a & b - [a, b] = [b, a]; - } - - for (; a <= b; a++) { - mark.push(a); - } - } - if (!isNaN(Number(cur))) mark.push(+cur); - } - break; - } - case 'language_attr': { - language_attr = value === 'true'; - break; - } - } - } - return { - options: { - language_attr, - firstLine, - line_number, - line_threshold, - mark, - wrap - }, - enableHighlight, - _args: args.replace(rAdditionalOptions, '') - }; -} - -export = (ctx: Hexo): (data: RenderData) => void => { - return function backtickCodeBlock(data: RenderData): void { - const dataContent = data.content; - - if ((!dataContent.includes('```') && !dataContent.includes('~~~')) || !ctx.extend.highlight.query(ctx.config.syntax_highlighter)) return; - data.content = scanPostSegments(dataContent).map(segment => { - const source = dataContent.slice(segment.start, segment.end); - if (segment.type !== 'fenced-code' || !segment.closed) return source; - - let content = dataContent.slice(segment.contentStart, segment.contentEnd).replace(/\r?\n$/, ''); - let args = segment.info; - const parsedArgs = parseArgs(args); - if (!parsedArgs.enableHighlight) return escapeSwigTag(source); - args = parsedArgs._args; - - // Extract language and caption of code blocks - const langArgs = args.split('=').shift(); - let lang: string, caption: string; - - if (langArgs) { - const match = rAllOptions.exec(langArgs) || rLangCaption.exec(langArgs); - - if (match) { - lang = match[1]; - - if (match[2]) { - caption = `${match[2]}`; - - if (match[3]) { - caption += `${match[4] ? match[4] : 'link'}`; - } - } - } - } - - // PR #3765 - if (segment.prefix.includes('>')) { - const depth = segment.prefix.split('>').length - 1; - const regexp = new RegExp(`^([^\\S\\r\\n]*>){0,${depth}}([^\\S\\r\\n]|$)`, 'mg'); - content = content.replace(regexp, ''); - } - - const options: HighlightOptions = { - lang, - caption, - lines_length: content.split('\n').length, - ...parsedArgs.options - }; - // setup line number by inline - args = args.replace('=+', '='); - - // setup firstLineNumber; - if (args.includes('=')) { - options.firstLineNumber = args.split('=')[1] || 1; - } - content = ctx.extend.highlight.exec(ctx.config.syntax_highlighter, { - context: ctx, - args: [content, options] - }); - - return segment.prefix - + '' - + escapeSwigTag(content) - + '' - + dataContent.slice(segment.closingEnd, segment.end); - }).join(''); - }; -}; diff --git a/lib/plugins/filter/before_post_render/index.ts b/lib/plugins/filter/before_post_render/index.ts index 501b483df..dcac82f3d 100644 --- a/lib/plugins/filter/before_post_render/index.ts +++ b/lib/plugins/filter/before_post_render/index.ts @@ -3,6 +3,5 @@ import type Hexo from '../../../hexo'; export = (ctx: Hexo) => { const { filter } = ctx.extend; - filter.register('before_post_render', require('./backtick_code_block')(ctx)); filter.register('before_post_render', require('./titlecase')); }; diff --git a/test/scripts/hexo/post.ts b/test/scripts/hexo/post.ts index 49a015a8c..f137020fd 100644 --- a/test/scripts/hexo/post.ts +++ b/test/scripts/hexo/post.ts @@ -705,6 +705,30 @@ describe('Post', () => { afterHook.calledOnce.should.be.true; }); + it('render() - before_post_render receives the original Markdown', async () => { + const source = [ + '```js', + 'const value = {{ value }};', + '```' + ].join('\n'); + const filter = spy(data => { + data.content.should.eql(source); + }); + + hexo.extend.filter.register('before_post_render', filter); + + try { + await post.render('', { + content: source, + engine: 'markdown' + }); + + filter.calledOnce.should.be.true; + } finally { + hexo.extend.filter.unregister('before_post_render', filter); + } + }); + it('render() - callback', done => { post.render('', { content, @@ -1928,6 +1952,24 @@ describe('Post', () => { ].join('\n')); }); + it('render() - nunjucks remains literal in an unhighlighted code fence', async () => { + const content = [ + '```js highlight:false', + 'const value = {{ value }};', + '```', + '', + '{{ 1 + 1 }}' + ].join('\n'); + + const data = await post.render('', { + content, + engine: 'markdown' + }); + + data.content.should.include('const value = {{ value }};'); + data.content.trim().slice(-1).should.eql('2'); + }); + // https://github.com/hexojs/hexo/issues/5715 it('render() - comment nesting in code fence', async () => { const code = 'alert("Hello world")'; diff --git a/test/scripts/hexo/post_render_lexer.ts b/test/scripts/hexo/post_render_lexer.ts index 961d51180..934eeea1e 100644 --- a/test/scripts/hexo/post_render_lexer.ts +++ b/test/scripts/hexo/post_render_lexer.ts @@ -1,10 +1,14 @@ import chai from 'chai'; import nunjucks from 'nunjucks'; -import PostRenderEscape, { scanPostSegments } from '../../../lib/hexo/post_render_lexer'; +import Hexo from '../../../lib/hexo'; +import { scanPostSegments } from '../../../lib/hexo/post_render_lexer'; +import PostRenderProcessor from '../../../lib/hexo/post_render_processor'; chai.should(); -describe('PostRenderEscape', () => { +describe('Post render lexer', () => { + const hexo = new Hexo(); + it('finds HTML comments only outside Markdown code', () => { const content = [ '```html', @@ -24,7 +28,7 @@ describe('PostRenderEscape', () => { comments.should.eql(['']); }); - it('returns fenced code metadata for downstream filters', () => { + it('returns fenced code metadata for the post renderer', () => { const content = [ '> ```js', '> const value = 1;', @@ -53,42 +57,42 @@ describe('PostRenderEscape', () => { '{% endtabs %}', '{% endfolding %}' ].join('\n'); - const escape = new PostRenderEscape(); + const processor = new PostRenderProcessor(hexo); - const escaped = escape.escapeAllSwigTags(content); + const escaped = processor.prepare(content); (escaped.match(/'; - const escape = new PostRenderEscape(); + const processor = new PostRenderProcessor(hexo); - const escaped = escape.escapeAllSwigTags(content); - const restored = escape.restoreComments(escape.restoreAllSwigTags(escaped)); + const escaped = processor.prepare(content); + const restored = processor.restoreComments(processor.restoreAllSwigTags(escaped)); nunjucks.renderString(restored, {}).should.eql(content); }); it('preserves raw blocks containing backticks in inline code', () => { const content = '`{% raw %}test`111`{{ value }}{% endraw %}`'; - const escape = new PostRenderEscape(); + const processor = new PostRenderProcessor(hexo); - const escaped = escape.escapeAllSwigTags(content); + const escaped = processor.prepare(content); - escape.hasNunjucks.should.be.true; - escape.restoreAllSwigTags(escaped).should.eql(content); + processor.hasNunjucks.should.be.true; + processor.restoreAllSwigTags(escaped).should.eql(content); }); it('does not close a Nunjucks variable on a delimiter inside a string', () => { const content = 'before {{ "}}" }} after'; - const escape = new PostRenderEscape(); + const processor = new PostRenderProcessor(hexo); - const escaped = escape.escapeAllSwigTags(content); + const escaped = processor.prepare(content); (escaped.match(/' ].join('\n'); - const comments = scanPostSegments(content) + const comments = lexPost(content).segments .filter(segment => segment.type === 'html-comment') .map(({ start, end }) => content.slice(start, end)); @@ -36,7 +36,7 @@ describe('Post render lexer', () => { 'after' ].join('\n'); - const fence = scanPostSegments(content).find(segment => segment.type === 'fenced-code'); + const fence = lexPost(content).segments.find(segment => segment.type === 'fenced-code'); chai.expect(fence).to.exist; if (!fence || fence.type !== 'fenced-code') return; @@ -48,6 +48,37 @@ describe('Post render lexer', () => { content.slice(fence.closingEnd, fence.end).should.eql('\n'); }); + it('treats Nunjucks tokens as opaque to Markdown and HTML delimiters', () => { + const content = [ + '{{ "`code` ', + '```njk', + '{{ "