From 0ea061cd64f14db3e132ba1be58538f006a4a61f Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Sat, 13 Jun 2026 17:38:26 +0000 Subject: [PATCH 01/13] Add txt2tags-to-Markdown converter for legacy entries --- rednotebook/util/t2t_to_markdown.py | 117 ++++++++++++++++++++++++++++ tests/test_t2t_to_markdown.py | 100 ++++++++++++++++++++++++ 2 files changed, 217 insertions(+) create mode 100644 rednotebook/util/t2t_to_markdown.py create mode 100644 tests/test_t2t_to_markdown.py diff --git a/rednotebook/util/t2t_to_markdown.py b/rednotebook/util/t2t_to_markdown.py new file mode 100644 index 00000000..3efc38e1 --- /dev/null +++ b/rednotebook/util/t2t_to_markdown.py @@ -0,0 +1,117 @@ +# ----------------------------------------------------------------------- +# Copyright (c) 2008-2024 Jendrik Seipp +# +# RedNotebook is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# RedNotebook is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# ----------------------------------------------------------------------- + +"""Translate legacy txt2tags markup to its Markdown equivalent. + +RedNotebook used txt2tags markup for many years. To keep old journals +readable after the switch to Markdown, every entry is run through this +converter before it reaches the Markdown parser. Constructs that Markdown +shares with txt2tags (``**bold**``, ``- lists``, fenced ```` ``` ```` code +blocks, ...) are left untouched, so text that is already Markdown passes +through essentially unchanged. +""" + +import re + + +IMG_EXT = r"png|jpe?g|gif|eps|bmp|svg" + +# Inline conversions applied to a single line of text (outside code fences). +# Order matters: links/images are handled before the emphasis markers so that +# their contents are not mangled. + +# [alt ""/path/to/pic"".png?50] and [""/path/to/pic"".png?50] +REGEX_IMAGE = re.compile(rf'\[(?:(.*?) )?""(\S.*?\S|\S)""\.({IMG_EXT})(\?\d+)?\]', flags=re.I) +# [text ""url""] +REGEX_QUOTED_LINK = re.compile(r'\[(.*?)\s""(\S.*?\S)""\]') +# [text http://url] (only genuine URLs, to avoid swallowing entry references) +REGEX_NAMED_LINK = re.compile( + r"\[([^\]]+?)\s+((?:https?|ftp|news|telnet|gopher|wais)://|www[23]?\.|ftp\.)([^\]]+)\]" +) + +# Emphasis. Boundaries follow txt2tags: markers hug non-space characters. +REGEX_ITALIC = re.compile(r"(?\1", line) + line = REGEX_STRIKE.sub(r"~~\1~~", line) + line = REGEX_LINEBREAK.sub(" ", line) + return line + + +def _image_repl(match): + alt = match.group(1) or "" + width = match.group(4) or "" + return f"![{alt}]({match.group(2)}.{match.group(3)}{width})" + + +def _convert_block(line): + """Convert line-level constructs that have no Markdown counterpart.""" + if REGEX_HRULE.match(line): + return "---" + + heading = REGEX_HEADING.match(line) + if heading: + level = len(heading.group(1)) + return "#" * level + " " + heading.group(2) + + numbered = REGEX_NUMBERED.match(line) + if numbered: + return f"{numbered.group(1)}1. {numbered.group(2)}" + + return None + + +def convert_to_markdown(text): + lines = text.split("\n") + result = [] + in_fence = False + for line in lines: + if FENCE.match(line): + in_fence = not in_fence + result.append(line) + continue + if in_fence: + result.append(line) + continue + + block = _convert_block(line) + if block is not None: + result.append(block) + else: + result.append(_convert_inline(line)) + return "\n".join(result) diff --git a/tests/test_t2t_to_markdown.py b/tests/test_t2t_to_markdown.py new file mode 100644 index 00000000..085ee0ef --- /dev/null +++ b/tests/test_t2t_to_markdown.py @@ -0,0 +1,100 @@ +from rednotebook.util.t2t_to_markdown import convert_to_markdown as c + + +class TestHeadings: + def test_levels(self): + assert c("= Title =") == "# Title" + assert c("== Title ==") == "## Title" + assert c("===== Title =====") == "##### Title" + + def test_heading_with_anchor(self): + assert c("== Clouds ==[clouds]") == "## Clouds" + + def test_markdown_heading_untouched(self): + assert c("# Already Markdown") == "# Already Markdown" + + +class TestInlineFormatting: + def test_italic(self): + assert c("//italic//") == "*italic*" + + def test_bold_unchanged(self): + assert c("**bold**") == "**bold**" + + def test_underline(self): + assert c("__underlined__") == "underlined" + + def test_strikethrough(self): + assert c("--struck--") == "~~struck~~" + + def test_monospace(self): + assert c("``code``") == "`code`" + + def test_combination(self): + assert c("a //b// and **c** and --d--") == "a *b* and **c** and ~~d~~" + + def test_italic_does_not_touch_urls(self): + assert c("see http://example.com now") == "see http://example.com now" + + +class TestHorizontalRule: + def test_long_equals(self): + assert c("====================") == "---" + + def test_long_dashes(self): + assert c("--------------------") == "---" + + def test_short_dashes_untouched(self): + # Three dashes is already a Markdown rule and must survive. + assert c("---") == "---" + + +class TestLists: + def test_bullet_unchanged(self): + assert c("- item") == "- item" + + def test_numbered(self): + assert c("+ item") == "1. item" + + def test_indented_numbered(self): + assert c(" + item") == " 1. item" + + +class TestLinks: + def test_named_web_link(self): + assert c("[heise http://heise.de]") == "[heise](http://heise.de)" + + def test_quoted_link(self): + assert c('[my file ""file:///home/me/f.txt""]') == "[my file](file:///home/me/f.txt)" + + def test_bare_url_unchanged(self): + assert c("http://example.com") == "http://example.com" + + +class TestImages: + def test_simple_image(self): + assert c('[""/home/pic"".png]') == "![](/home/pic.png)" + + def test_image_with_width(self): + assert c('[""/home/pic"".png?50]') == "![](/home/pic.png?50)" + + def test_named_image(self): + assert c('[alt ""/home/pic"".jpg]') == "![alt](/home/pic.jpg)" + + +class TestLineBreak: + def test_trailing_backslashes(self): + assert c("First\\\\\nSecond") == "First \nSecond" + + def test_backslashes_midline_unchanged(self): + assert c("First\\\\Second") == "First\\\\Second" + + +class TestFencedCodeIsPreserved: + def test_no_inline_conversion_in_fence(self): + text = "```\n//not italic//\n+ not numbered\n```" + assert c(text) == text + + def test_entry_reference_passthrough(self): + # Entry references are handled later in the pipeline, not here. + assert c("[2019-08-01]") == "[2019-08-01]" From 0f72fe61802c1fa79a74a324b0d0f8202de38157 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Sat, 13 Jun 2026 17:47:03 +0000 Subject: [PATCH 02/13] Render entries with markdown-it-py instead of txt2tags Route the live preview and the HTML, LaTeX and plain-text exporters through markdown-it-py. Legacy txt2tags entries are converted to Markdown first, so existing journals keep working. RedNotebook-specific constructs (#hashtags, {text|color:...}, image widths, entry references and math) are handled by inline plugins and pre-/post-processing. --- rednotebook/util/markdownmarkup.py | 473 +++++++++++++++++++++++++++++ rednotebook/util/markup.py | 386 +++++------------------ rednotebook/util/pango_markup.py | 136 ++++----- tests/test_markdown_render.py | 143 +++++++++ tests/test_markup.py | 403 +++++------------------- 5 files changed, 825 insertions(+), 716 deletions(-) create mode 100644 rednotebook/util/markdownmarkup.py create mode 100644 tests/test_markdown_render.py diff --git a/rednotebook/util/markdownmarkup.py b/rednotebook/util/markdownmarkup.py new file mode 100644 index 00000000..3edd19bb --- /dev/null +++ b/rednotebook/util/markdownmarkup.py @@ -0,0 +1,473 @@ +# ----------------------------------------------------------------------- +# Copyright (c) 2008-2024 Jendrik Seipp +# +# RedNotebook is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# RedNotebook is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# ----------------------------------------------------------------------- + +"""Render Markdown to HTML, LaTeX and plain text. + +This module wraps `markdown-it-py `_ and adds the RedNotebook specific constructs that have no +standard Markdown equivalent: + +* ``#hashtags`` are coloured (and indexed in LaTeX), +* ``{text|color:value}`` colours arbitrary text, +* image links may carry a ``?width`` suffix, and +* ``$...$`` / ``$$...$$`` math is rendered for MathJax/LaTeX. +""" + +import re + +from markdown_it import MarkdownIt +from markdown_it.common.utils import escapeHtml +from markdown_it.renderer import RendererHTML +from mdit_py_plugins.dollarmath import dollarmath_plugin + + +CSS = """\ + +""" + +MATHJAX_FILE = "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-svg.js" +MATHJAX = f"""\ + + +""" + +# A #hashtag must contain at least one letter and must not be a hex colour or +# a C preprocessor directive. This mirrors rednotebook.data.HASHTAG. +HASHTAG_BODY = re.compile( + r"(?![0-9a-fA-F]{6}\b|include\b|define\b|ifdef\b|ifndef\b|endif\b)(\w*[^\W\d_]+\w*)" +) +COLOR = re.compile(r"\{([^{}|]+)\|color:([^{}]+)\}") +IMAGE_WIDTH = re.compile(r"\?(\d+)$") + + +# -------------------------------------------------------------------------- +# Inline plugins +# -------------------------------------------------------------------------- + + +def _hashtag_rule(state, silent): + pos = state.pos + if state.src[pos] not in "#＃": + return False + if pos > 0: + prev = state.src[pos - 1] + if prev.isalnum() or prev == "_" or prev in "&#": + return False + match = HASHTAG_BODY.match(state.src, pos + 1) + if not match: + return False + if not silent: + token = state.push("hashtag", "", 0) + token.content = state.src[pos : match.end()] + token.meta = {"tag": match.group(1)} + state.pos = match.end() + return True + + +def _color_rule(state, silent): + if state.src[state.pos] != "{": + return False + match = COLOR.match(state.src, state.pos) + if not match: + return False + if not silent: + token = state.push("rn_color", "", 0) + token.meta = {"text": match.group(1), "color": match.group(2)} + state.pos = match.end() + return True + + +def _rednotebook_plugin(md): + md.inline.ruler.before("emphasis", "hashtag", _hashtag_rule) + md.inline.ruler.before("emphasis", "rn_color", _color_rule) + + +# -------------------------------------------------------------------------- +# HTML renderer +# -------------------------------------------------------------------------- + + +class HtmlRenderer(RendererHTML): + def hashtag(self, tokens, idx, options, env): + return f'{escapeHtml(tokens[idx].content)}' + + def rn_color(self, tokens, idx, options, env): + meta = tokens[idx].meta + return f'{escapeHtml(meta["text"])}' + + def image(self, tokens, idx, options, env): + token = tokens[idx] + src = token.attrs.get("src", "") + width = "" + match = IMAGE_WIDTH.search(src) + if match: + width = f' width="{match.group(1)}"' + src = src[: match.start()] + alt = escapeHtml(token.content) + return f'{alt}' + + def math_inline(self, tokens, idx, options, env): + return f"\\({tokens[idx].content}\\)" + + def math_block(self, tokens, idx, options, env): + return f"$$\n{tokens[idx].content}\n$$\n" + + +# -------------------------------------------------------------------------- +# Token-walking renderers for LaTeX and plain text +# -------------------------------------------------------------------------- + +_HEADING_TO_TEX = { + "h1": "section", + "h2": "subsection", + "h3": "subsubsection", + "h4": "paragraph", + "h5": "subparagraph", + "h6": "subparagraph", +} + +_TEX_ESCAPES = { + "\\": r"\textbackslash{}", + "&": r"\&", + "%": r"\%", + "$": r"\$", + "#": r"\#", + "_": r"\_", + "{": r"\{", + "}": r"\}", + "~": r"\textasciitilde{}", + "^": r"\textasciicircum{}", +} +_TEX_ESCAPE_RE = re.compile("|".join(re.escape(key) for key in _TEX_ESCAPES)) + + +def tex_escape(text): + return _TEX_ESCAPE_RE.sub(lambda m: _TEX_ESCAPES[m.group()], text) + + +class _TokenRenderer: + """Walk the markdown-it token stream and dispatch by token type.""" + + def __init__(self, parser=None): + self.parser = parser + + def render(self, tokens, options, env): + out = [] + for token in tokens: + if token.type == "inline": + out.append(self.render(token.children or [], options, env)) + else: + method = getattr(self, token.type, None) + if method is not None: + out.append(method(token, env)) + return "".join(out) + + # Fallbacks for the markup we do not specially handle. + def text(self, token, env): + return token.content + + def softbreak(self, token, env): + return "\n" + + def html_inline(self, token, env): + return "" + + def html_block(self, token, env): + return "" + + +class LatexRenderer(_TokenRenderer): + def text(self, token, env): + return tex_escape(token.content) + + def softbreak(self, token, env): + return "\n" + + def hardbreak(self, token, env): + return "\\\\\n" + + def paragraph_open(self, token, env): + return "" + + def paragraph_close(self, token, env): + return "\n\n" + + def heading_open(self, token, env): + return "\\" + _HEADING_TO_TEX.get(token.tag, "section") + "{" + + def heading_close(self, token, env): + return "}\n\n" + + def strong_open(self, token, env): + return "\\textbf{" + + def strong_close(self, token, env): + return "}" + + def em_open(self, token, env): + return "\\textit{" + + def em_close(self, token, env): + return "}" + + def s_open(self, token, env): + return "\\sout{" + + def s_close(self, token, env): + return "}" + + def code_inline(self, token, env): + return "\\texttt{" + tex_escape(token.content) + "}" + + def fence(self, token, env): + return "\\begin{verbatim}\n" + token.content + "\\end{verbatim}\n\n" + + code_block = fence + + def link_open(self, token, env): + return "\\href{" + token.attrs.get("href", "") + "}{" + + def link_close(self, token, env): + return "}" + + def image(self, token, env): + src = token.attrs.get("src", "") + match = IMAGE_WIDTH.search(src) + options = "" + if match: + options = f"[width={match.group(1)}px]" + src = src[: match.start()] + return f'\\includegraphics{options}{{"{src}"}}' + + def bullet_list_open(self, token, env): + return "\\begin{itemize}\n" + + def bullet_list_close(self, token, env): + return "\\end{itemize}\n" + + def ordered_list_open(self, token, env): + return "\\begin{enumerate}\n" + + def ordered_list_close(self, token, env): + return "\\end{enumerate}\n" + + def list_item_open(self, token, env): + return "\\item " + + def list_item_close(self, token, env): + return "\n" + + def hr(self, token, env): + return "\\par\\noindent\\rule{\\linewidth}{0.4pt}\n\n" + + def blockquote_open(self, token, env): + return "\\begin{quote}\n" + + def blockquote_close(self, token, env): + return "\\end{quote}\n" + + def hashtag(self, token, env): + display = tex_escape(token.content.lstrip("#＃")) + index = token.meta["tag"] + return f"\\textcolor{{red}}{{\\#{display}\\index{{{index}}}}}" + + def rn_color(self, token, env): + meta = token.meta + return f"\\textcolor{{{tex_escape(meta['color'])}}}{{{tex_escape(meta['text'])}}}" + + def math_inline(self, token, env): + return f"${token.content}$" + + def math_block(self, token, env): + return f"$${token.content}$$\n" + + # Minimal table support: render cells separated by " & " and rows by "\\". + def tr_close(self, token, env): + return "\\\\\n" + + def td_close(self, token, env): + return " & " + + th_close = td_close + + +class PlainRenderer(_TokenRenderer): + def hardbreak(self, token, env): + return "\n" + + def paragraph_close(self, token, env): + return "\n\n" + + def heading_close(self, token, env): + return "\n\n" + + def code_inline(self, token, env): + return token.content + + def fence(self, token, env): + return token.content + "\n" + + code_block = fence + + def image(self, token, env): + return f"[{token.attrs.get('src', '')}]" + + def list_item_open(self, token, env): + return "- " + + def list_item_close(self, token, env): + return "\n" + + def hr(self, token, env): + return "\n" + "=" * 20 + "\n\n" + + def hashtag(self, token, env): + return token.content + + def rn_color(self, token, env): + return token.meta["text"] + + def math_inline(self, token, env): + return token.content + + def math_block(self, token, env): + return token.content + "\n" + + +_RENDERERS = {"html": HtmlRenderer, "tex": LatexRenderer, "txt": PlainRenderer} + + +def _get_parser(target): + md = MarkdownIt("commonmark", {"html": True, "linkify": True, "breaks": False}) + md.enable(["table", "strikethrough", "linkify"]) + md.use(dollarmath_plugin, double_inline=True) + md.use(_rednotebook_plugin) + md.renderer = _RENDERERS[target](md) + return md + + +def _walk(tokens): + for token in tokens: + yield token + if token.children: + yield from _walk(token.children) + + +# -------------------------------------------------------------------------- +# Document templates +# -------------------------------------------------------------------------- + +LATEX_PREAMBLE = r"""\documentclass[a4paper]{article} +\usepackage[utf8]{inputenc} +\usepackage[T1]{fontenc} +\usepackage{graphicx} +\usepackage{xcolor} +\usepackage[normalem]{ulem} +\usepackage{hyperref} +\usepackage{makeidx} +\makeindex +\title{%(title)s} +\begin{document} +\maketitle +""" + +LATEX_FOOTER = r""" +\printindex +\end{document} +""" + + +def _html_document(body, options, has_math): + css = CSS % { + "font": options.get("font", "sans-serif"), + "bgcolor": options.get("bgcolor", "white"), + "fgcolor": options.get("fgcolor", "black"), + } + mathjax = MATHJAX if has_math else "" + return ( + "\n\n\n" + '\n' + f"{css}{mathjax}" + "\n\n" + f"{body}" + "\n\n" + ) + + +def _latex_document(body, options): + title = options.get("title", "RedNotebook") + return LATEX_PREAMBLE % {"title": title} + body + LATEX_FOOTER + + +def render(text, target, options=None): + """Render Markdown ``text`` to ``target`` (``html``, ``tex`` or ``txt``).""" + options = options or {} + md = _get_parser(target) + env = {} + tokens = md.parse(text, env) + body = md.renderer.render(tokens, md.options, env) + + if target == "html": + has_math = options.get("add_mathjax") + if has_math is None: + has_math = any(token.type.startswith("math") for token in _walk(tokens)) + return _html_document(body, options, has_math) + if target == "tex": + return _latex_document(body, options) + return body.strip() + "\n" diff --git a/rednotebook/util/markup.py b/rednotebook/util/markup.py index 794db153..e4b78299 100644 --- a/rednotebook/util/markup.py +++ b/rednotebook/util/markup.py @@ -19,107 +19,35 @@ import os import re -from rednotebook.data import HASHTAG -from rednotebook.external import txt2tags -from rednotebook.util import filesystem, urls +from rednotebook.util import filesystem, markdownmarkup, t2t_to_markdown, urls -# Linebreaks are only allowed at line ends -REGEX_LINEBREAK = r"\\\\[\s]*$" +# A trailing "text" link, used by pango_markup to strip links. REGEX_HTML_LINK = r"(.*?)" +# A Markdown hard line break (two trailing spaces), used by pango_markup. +REGEX_LINEBREAK = r" $" -# pic [""/home/user/Desktop/RedNotebook pic"".png] -PIC_NAME = r"\S.*?\S|\S" -PIC_EXT = r"(?:png|jpe?g|gif|eps|bmp|svg)" -REGEX_PIC = re.compile(rf'(\["")({PIC_NAME})("")(\.{PIC_EXT})(\?\d+)?(\])', flags=re.I) - -# named local link [my file.txt ""file:///home/user/my file.txt""] -# named link in web [heise ""http://heise.de""] -REGEX_NAMED_LINK = re.compile(r'(\[)(.*?)(\s"")(\S.*?\S)(""\])', flags=re.I) - -ESCAPE_COLOR = r"XBEGINCOLORX\1XSEPARATORX\2XENDCOLORX" -COLOR_ESCAPED = r"XBEGINCOLORX(.*?)XSEPARATORX(.*?)XENDCOLORX" - -CSS = """\ - -""" - -# MathJax -FORMULAS_SUPPORTED = True -MATHJAX_FILE = "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-svg.js" - -# Explicitly setting inlineMath: [ ['\\(','\\)'] ] doesn't work. -# Using defaults: -# displayMath: [ ['$$','$$'], ['\[','\]'] ] -# inlineMath: [['\(','\)']] -MATHJAX_DELIMITERS = ["$$", "\\(", "\\)", r"\\[", "\\]"] -MATHJAX = f"""\ - - -""" +# Markdown image/link target: "![alt](url)" or "[text](url)". +REGEX_MD_LINK = re.compile(r"(!?\[[^\]]*\]\()([^)\s]+)(\))") +# Optional image width suffix. +REGEX_IMAGE_WIDTH = re.compile(r"\?(\d+)$") + +# Entry references such as "[2019-08-01]" or "[my day 2019-08-01]". +REGEX_NAMED_REFERENCE = re.compile(r"\[(?P.+?)\s+(?P\d{4}-\d{2}-\d{2})\s*\]") +REGEX_DATE_REFERENCE = re.compile(r"\[(?P\d{4}-\d{2}-\d{2})\]") + +# Math delimiters that MathJax understands besides "$"/"$$". +REGEX_MATH_DISPLAY = re.compile(r"\\\[(.+?)\\\]", flags=re.DOTALL) +REGEX_MATH_INLINE = re.compile(r"\\\((.+?)\\\)", flags=re.DOTALL) def convert_categories_to_markup(categories, with_category_title=True): - # Only add Category title if the text is displayed - markup = "== {} ==\n".format(_("Tags")) if with_category_title else "" + # Only add the "Tags" title if the text is displayed. + markup = "## {}\n".format(_("Tags")) if with_category_title else "" for category, entry_list in categories.items(): - markup += f"- {category}" + "\n" + markup += f"- {category}\n" for entry in entry_list: - markup += f" - {entry}" + "\n" + markup += f" - {entry}\n" markup += "\n\n" return markup @@ -133,11 +61,10 @@ def get_markup_for_day(day, target, with_text=True, with_tags=True, categories=N # Add date if it is not None and not the empty string if date: if target == "html": - # Following anchor will be used as a target for every entry reference mentioning - # this entry's date. - export_string += f"''''\n" + # The anchor is the target for entry references mentioning this date. + export_string += f'\n\n' - export_string += f"= {date} =\n\n" + export_string += f"# {date}\n\n" # Add text if with_text: @@ -174,244 +101,79 @@ def get_markup_for_day(day, target, with_text=True, with_tags=True, categories=N return "" -def _get_config(target, options): - # Set the configuration on the 'config' dict. - config = txt2tags.ConfigMaster()._get_defaults() - - config["outfile"] = txt2tags.MODULEOUT # results as list - config["target"] = target - - # The Pre (and Post) processing config is a list of lists: - # [ [this, that], [foo, bar], [patt, replace] ] - config["postproc"] = [] - config["preproc"] = [] - config["style"] = [] - - # Allow line breaks, r'\\\\' are 2 \ for regexes - config["preproc"].append([REGEX_LINEBREAK, "LINEBREAK"]) - - # Highlight hashtags. - if target == "tex": - config["preproc"].append([HASHTAG.pattern, r"\1{\2\3BEGININDEX\3ENDINDEX|color:red}"]) - else: - config["preproc"].append([HASHTAG.pattern, r"\1{\2\3|color:red}"]) - - # Escape color markup. - config["preproc"].append([r"\{(.*?)\|color:(.+?)\}", ESCAPE_COLOR]) - - if target == "html": - config["encoding"] = "UTF-8" # document encoding - config["toc"] = 0 - config["css-sugar"] = 1 - - # Line breaks - config["postproc"].append([r"LINEBREAK", "
"]) - - # Apply image resizing - config["postproc"].append([r"src=\"WIDTH(\d+)-", r'width="\1" src="']) - - # Flow paragraph from right to left or left to right depending on the language. - config["postproc"].append(["

", '

']) - - # {{red text|color:red}} -> red text - config["postproc"].append([COLOR_ESCAPED, r'\1']) - - # Custom css - font = options.pop("font", "sans-serif") - css = CSS % { - "font": font, - "bgcolor": options.get("bgcolor", "white"), - "fgcolor": options.get("fgcolor", "black"), - } - config["postproc"].append([r"", f"{css}"]) - - # MathJax - if options.pop("add_mathjax"): - config["postproc"].append([r"", f"{MATHJAX}"]) - - elif target == "tex": - config["encoding"] = "utf8" - config["preproc"].append(["€", "Euro"]) - - # Latex only allows whitespace and underscores in filenames if - # the filename is surrounded by "...". This is in turn only possible - # if the extension is omitted. - config["preproc"].append([r'\[""', r'["""']) - config["preproc"].append([r'""\.', r'""".']) - - scheme = filesystem.LOCAL_FILE_PEFIX - - # For images we have to omit the file:// prefix - config["postproc"].append([rf'includegraphics\{{(.*)"{scheme}', r'includegraphics{"\1']) - - # Special handling for LOCAL file links (Omit scheme, add run:) - # \htmladdnormallink{file.txt}{file:///home/user/file.txt} - # --> - # \htmladdnormallink{file.txt}{run:/home/user/file.txt} - config["postproc"].append( - [ - rf"htmladdnormallink\{{(.*)\}}\{{{scheme}(.*)\}}", - r"htmladdnormallink{\1}{run:\2}", - ] - ) - - # Line breaks - config["postproc"].append([r"LINEBREAK", r"\\\\"]) - - # Apply image resizing - config["postproc"].append( - [r'includegraphics\{("?)WIDTH(\d+)-', r"includegraphics[width=\2px]{\1"] - ) +def _convert_uri(uri, data_dir): + path = uri[len("file://") :] if uri.startswith("file://") else uri + # Check if relative file exists and convert it if it does. + if not any( + uri.startswith(proto) for proto in filesystem.REMOTE_PROTOCOLS + ) and not os.path.isabs(path): + path = os.path.join(data_dir, path) + assert os.path.isabs(path), path + if os.path.exists(path): + uri = urls.get_local_url(path) + return uri - # We want the plain latex formulas unescaped. - # Allowed formulas: $$...$$, \[...\], \(...\) - config["preproc"].append([r"\\\[\s*(.+?)\s*\\\]", r"BEGINEQUATION''\1''ENDEQUATION"]) - config["preproc"].append([r"\$\$\s*(.+?)\s*\$\$", r"BEGINEQUATION''\1''ENDEQUATION"]) - config["postproc"].append([r"BEGINEQUATION(.+)ENDEQUATION", r"$$\1$$"]) - config["preproc"].append([r"\\\(\s*(.+?)\s*\\\)", r"BEGINMATH''\1''ENDMATH"]) - config["postproc"].append([r"BEGINMATH(.+)ENDMATH", r"$\1$"]) - - # Fix utf8 quotations - „, “ and ” cause problems compiling the latex document. - config["postproc"].extend([["„", '"'], ["”", '"'], ["“", '"']]) - - # Enable index. - config["style"].append("makeidx") - config["postproc"].append([r"BEGININDEX(.+?)ENDINDEX", r"\\index{\1}"]) - config["postproc"].append(["begin{document}", "makeindex\n\\\\begin{document}"]) - config["postproc"].append(["end{document}", "printindex\n\n\\\\end{document}"]) +def _convert_paths(txt, data_dir): + """Turn relative paths in Markdown links and images into absolute URLs.""" + data_dir = str(data_dir) - config["postproc"].append([COLOR_ESCAPED, r"\\textcolor{\2}{\1}"]) + def repl(match): + prefix, url, suffix = match.groups() + width = "" + is_image = prefix.startswith("!") + width_match = REGEX_IMAGE_WIDTH.search(url) + if is_image and width_match: + width = width_match.group(0) + url = url[: width_match.start()] + # Leave fragment-only references (entry references) untouched. + if url.startswith("#"): + return match.group(0) + return prefix + _convert_uri(url, data_dir) + width + suffix - elif target == "txt": - # Line breaks - config["postproc"].append([r"LINEBREAK", "\n"]) + return REGEX_MD_LINK.sub(repl, txt) - # Apply image resizing ([WIDTH400-file:///pathtoimage.jpg]) - config["postproc"].append([r"\[WIDTH(\d+)-(.+)\]", r"[\2?\1]"]) - # Entry references +def _convert_entry_references(txt, target): + """Turn date references into links (HTML) or plain text (other targets).""" if target == "html": - # txt2tags will generate links to the named entry references because they share - # common bracket notation used by the URIs. We just need to add our internal - # schema to make it a proper URI. - config["preproc"].append( - [ - r"\[(?P.+)\s+(?P\d{4}-\d{2}-\d{2})\s*\]", - r"[\g #\g]", - ] - ) - - # Convert bracketed dates into named references where the date itself is being - # used as a name. For example: - # "Today is [2019-10-20]" will be converted into "Today is [2019-10-20 #2019-10-20]" - config["preproc"].append([r"\[(?P\d{4}-\d{2}-\d{2})\]", r"[\g #\g]"]) + txt = REGEX_NAMED_REFERENCE.sub(r"[\g](#\g)", txt) + txt = REGEX_DATE_REFERENCE.sub(r"[\g](#\g)", txt) else: - # Links to entry references are not supported for targets other than HTML - config["preproc"].append( - [r"\[(?P.+)\s+(?P\d{4}-\d{2}-\d{2})\]", r"\g (\g)"] - ) - - # Allow resizing images by changing - # [filename.png?width] to [WIDTHwidth-filename.png] - img_ext = r"png|jpe?g|gif|eps|bmp|svg" - img_name = r"\S.*\S|\S" - - # Apply this prepoc only after the latex image quotes have been added - config["preproc"].append([rf"\[({img_name}\.({img_ext}))\?(\d+)\]", r"[WIDTH\3-\1]"]) - - # Disable colors for all other targets. - config["postproc"].append([COLOR_ESCAPED, r"\1"]) - - config.update(options) - - return config + txt = REGEX_NAMED_REFERENCE.sub(r"\g (\g)", txt) + txt = REGEX_DATE_REFERENCE.sub(r"\g", txt) + return txt -def _convert_paths(txt, data_dir): - data_dir = str(data_dir) - - def _convert_uri(uri): - path = uri[len("file://") :] if uri.startswith("file://") else uri - # Check if relative file exists and convert it if it does. - if not any( - uri.startswith(proto) for proto in filesystem.REMOTE_PROTOCOLS - ) and not os.path.isabs(path): - path = os.path.join(data_dir, path) - assert os.path.isabs(path), path - if os.path.exists(path): - uri = urls.get_local_url(path) - return uri - - def _convert_pic_path(match): - uri = _convert_uri(match.group(2) + match.group(4)) - # Reassemble picture markup. - name, ext = os.path.splitext(uri) - parts = [match.group(1), name, match.group(3), ext] - if match.group(5) is not None: - parts.append(match.group(5)) - parts.append(match.group(6)) - return "".join(parts) - - def _convert_file_path(match): - uri = _convert_uri(match.group(4)) - # Reassemble link markup - parts = [match.group(i) for i in range(1, 6)] - parts[3] = uri - return "".join(parts) - - txt = REGEX_PIC.sub(_convert_pic_path, txt) - txt = REGEX_NAMED_LINK.sub(_convert_file_path, txt) +def _normalize_math(txt): + """Rewrite "\\(...\\)" and "\\[...\\]" to the "$" delimiters MathJax uses.""" + txt = REGEX_MATH_DISPLAY.sub(r"$$\1$$", txt) + txt = REGEX_MATH_INLINE.sub(r"$\1$", txt) return txt def convert(txt, target, data_dir, headers=None, options=None): - """ - Code partly taken from txt2tags tarball - """ + """Convert journal text (Markdown, with txt2tags fallback) to ``target``.""" data_dir = str(data_dir) options = options or {} - # Only add MathJax code if there is a formula. - options["add_mathjax"] = ( - FORMULAS_SUPPORTED and "html" in target and any(x in txt for x in MATHJAX_DELIMITERS) - ) - logging.debug(f"Add mathjax code: {options['add_mathjax']}") + # Translate any legacy txt2tags markup to Markdown first. + txt = t2t_to_markdown.convert_to_markdown(txt) # Turn relative paths into absolute paths. txt = _convert_paths(txt, data_dir) - # The body text must be a list. - txt = txt.split("\n") - - # Set the three header fields - if headers is None: - # LaTeX requires a title if \maketitle is used. - headers = ["RedNotebook", "", ""] if target == "tex" else ["", "", ""] - config = _get_config(target, options) + # Handle RedNotebook-specific constructs. + txt = _convert_entry_references(txt, target) + txt = _normalize_math(txt) - # Let's do the conversion try: - headers = txt2tags.doHeader(headers, config) - body, toc = txt2tags.convert(txt, config) - footer = txt2tags.doFooter(config) - toc = txt2tags.toc_tagger(toc, config) - toc = txt2tags.toc_formatter(toc, config) - full_doc = headers + toc + body + footer - finished = txt2tags.finish_him(full_doc, config) - result = "\n".join(finished) - # Txt2tags error, show the message to the user - except txt2tags.error as msg: - logging.error(msg) - result = msg - # Unknown error, show the traceback to the user + return markdownmarkup.render(txt, target, options) except Exception: - result = ( - "Error: This day contains invalid " - 'txt2tags markup. ' - "You can help us fix this by submitting a bugreport in the " - '' - "txt2tags bugtracker. Please append the day's text to the issue." + logging.exception("Markdown conversion failed") + return ( + "Error: This day contains markup that RedNotebook could not " + "convert. Please report this at " + '' + "the RedNotebook bugtracker and append the day's text to the issue." ) - logging.error(f"Invalid markup:\n{txt2tags.getUnknownErrorMessage()}") - return result diff --git a/rednotebook/util/pango_markup.py b/rednotebook/util/pango_markup.py index 83b58288..14790f97 100644 --- a/rednotebook/util/pango_markup.py +++ b/rednotebook/util/pango_markup.py @@ -1,118 +1,96 @@ +# ----------------------------------------------------------------------- +# Copyright (c) 2008-2024 Jendrik Seipp +# +# RedNotebook is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# RedNotebook is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# ----------------------------------------------------------------------- + import logging import re import gi +from markdown_it import MarkdownIt gi.require_version("Pango", "1.0") -from gi.repository import GObject, Pango - -from rednotebook.external import txt2tags -from rednotebook.util.markup import REGEX_HTML_LINK, REGEX_LINEBREAK - - -def convert_to_pango(txt, headers=None, options=None): - """ - Code partly taken from txt2tags tarball - """ - original_txt = txt - - # Here is the marked body text, it must be a list. - txt = txt.split("\n") - - # Set the three header fields - if headers is None: - headers = ["", "", ""] +from gi.repository import GObject, Pango # noqa: E402 - config = txt2tags.ConfigMaster()._get_defaults() +from rednotebook.util import t2t_to_markdown # noqa: E402 +from rednotebook.util.markup import REGEX_HTML_LINK # noqa: E402 - config["outfile"] = txt2tags.MODULEOUT # results as list - config["target"] = "html" - config["preproc"] = [] - # We need to escape the ampersand here, otherwise "&" would become - # "&" - config["preproc"].append([r"&", "&"]) +# Categories are short, single-line strings, so inline rendering is enough. +_PARSER = MarkdownIt("commonmark").enable("strikethrough") - # Allow line breaks - config["postproc"] = [] - config["postproc"].append([REGEX_LINEBREAK, "\n"]) +# Map the (limited) set of HTML tags that markdown-it emits to the Pango tags +# the category tree understands. +_HTML_TO_PANGO = [ + ("", ""), + ("", ""), + ("", ""), + ("", ""), + ("", ""), + ("", ""), + ("", ""), + ("", ""), +] - if options is not None: - config.update(options) - # Let's do the conversion - try: - body, toc = txt2tags.convert(txt, config) - full_doc = body - finished = txt2tags.finish_him(full_doc, config) - result = "".join(finished) - - # Txt2tags error, show the message to the user - except txt2tags.error as msg: - logging.error(msg) - result = msg - - # Unknown error, show the traceback to the user - except Exception: - result = txt2tags.getUnknownErrorMessage() - logging.error(result) - - print(result) - - # remove unwanted paragraphs - result = result.replace('

', "").replace("

", "") +def convert_to_pango(txt, headers=None, options=None): + """Convert (Markdown) category markup to Pango markup for display.""" + original_txt = txt - logging.log( - 5, - f'Converted "{repr(original_txt)}" text to "{repr(result)}" txt2tags markup', - ) + txt = t2t_to_markdown.convert_to_markdown(txt) + result = _PARSER.renderInline(txt) - # Remove unknown tags () - def replace_links(match): - """Return the link name.""" - return match.group(1) + for html_tag, pango_tag in _HTML_TO_PANGO: + result = result.replace(html_tag, pango_tag) - result = re.sub(REGEX_HTML_LINK, replace_links, result) - print(result) + # Pango has no anchor element, so reduce links to their text. + result = re.sub(REGEX_HTML_LINK, r"\1", result) - for new_tag, old_tag in [("del", "s"), ("em", "i"), ("strong", "b")]: - result = result.replace(f"<{new_tag}>", f"<{old_tag}>") - result = result.replace(f"", f"") - print(result) + logging.log(5, f'Converted "{original_txt!r}" to Pango "{result!r}"') try: Pango.parse_markup(result, -1, "0") - # result is valid pango markup, return the markup. - return result except GObject.GError: - # There are unknown tags in the markup, return the original text + # There are unknown tags in the markup, return the original text. logging.debug(f"There are unknown tags in the markup: {result}") return original_txt + return result def convert_from_pango(pango_markup): + """Convert Pango markup back to the Markdown stored in the journal.""" original_txt = pango_markup replacements = { "": "**", "": "**", - "": "//", - "": "//", - "": "--", - "": "--", - "": "__", - "": "__", + "": "*", + "": "*", + "": "~~", + "": "~~", + "": "", + "": "", + "": "`", + "": "`", "&": "&", "<": "<", ">": ">", - "\n": r"\\", } for orig, repl in replacements.items(): pango_markup = pango_markup.replace(orig, repl) - logging.log( - 5, - f'Converted "{repr(original_txt)}" pango to "{repr(pango_markup)}" txt2tags', - ) + logging.log(5, f'Converted Pango "{original_txt!r}" to Markdown "{pango_markup!r}"') return pango_markup diff --git a/tests/test_markdown_render.py b/tests/test_markdown_render.py new file mode 100644 index 00000000..a27538b1 --- /dev/null +++ b/tests/test_markdown_render.py @@ -0,0 +1,143 @@ +"""Tests for the Markdown rendering pipeline (HTML, LaTeX and plain text).""" + +import pytest + +from rednotebook.util.markdownmarkup import render + + +def html(markup, **options): + return render(markup, "html", options) + + +def tex(markup, **options): + return render(markup, "tex", options) + + +def txt(markup, **options): + return render(markup, "txt", options) + + +class TestHtmlBasics: + def test_document_skeleton(self): + doc = html("Content") + assert "" in doc + assert '' in doc + assert "

Content

" in doc + + def test_emphasis(self): + doc = html("**b** and *i* and ~~s~~ and `c`") + assert "b" in doc + assert "i" in doc + assert "s" in doc + assert "c" in doc + + def test_heading(self): + assert "

Title

" in html("## Title") + + def test_bullet_list(self): + doc = html("- a\n- b") + assert "
    " in doc and "
  • a
    • " in doc + + def test_link(self): + assert '    name' in html("[name](http://x.com)") + + def test_autolink_bare_url(self): + assert 'http://x.com' in html("see http://x.com") + + +class TestHtmlRedNotebookFeatures: + @pytest.mark.parametrize( + "markup,expected", + [ + ("#TAG", '#TAG'), + ("Numeric #3tag", '#3tag'), + ("Just #34 numbers", "#34"), + ], + ) + def test_hashtags(self, markup, expected): + assert expected in html(markup) + + def test_hashtag_not_in_code(self): + assert '' not in html("`#TAG`") + + @pytest.mark.parametrize( + "markup,expected", + [ + ("{Sky|color:blue}", 'Sky'), + ("{Red|color:#FF0000} truck", 'Red truck'), + ], + ) + def test_colors(self, markup, expected): + assert expected in html(markup) + + def test_image_width(self): + doc = html("![](/image.png?50)") + assert 'src="/image.png"' in doc + assert 'width="50"' in doc + + def test_image_no_width(self): + doc = html("![](/image.jpg)") + assert 'src="/image.jpg"' in doc + assert "width=" not in doc + + @pytest.mark.parametrize( + "markup,expected", + [ + ("[named ref](#2019-08-01)", 'named ref'), + ], + ) + def test_entry_reference_links(self, markup, expected): + assert expected in html(markup) + + def test_mathjax_added_when_formula_present(self): + doc = html("$$x^3$$") + assert "MathJax" in doc + + def test_mathjax_absent_without_formula(self): + assert "MathJax" not in html("no math here") + + def test_linebreak(self): + assert "stricken"), - (r"//italic//", "italic"), - (r"--www.test.com--", "www.test.com"), - # Linebreaks only on line ends - (r"First\\Second", r"First\\Second"), - (r"First\\", "First\n"), - (r"a&b", "a&b"), - (r"a&b", "a&b"), - (r"http://site/s.php?q&c", "http://site/s.php?q&c"), - (r"http://site/s.php?q&c", "http://site/s.php?q&c"), + ("**bold**", "bold"), + ("*italic*", "italic"), + ("~~struck~~", "struck"), + ("`code`", "code"), + ("underlined", "underlined"), ], ) -def test_pango(t2t_markup, expected): - pango = convert_to_pango(t2t_markup) - assert pango == expected - # Ampersand escaping is only needed in sourcecode, so we do not try to - # preserve the encoding - if "&" not in t2t_markup: - assert convert_from_pango(pango) == t2t_markup +def test_pango(markdown, pango): + assert convert_to_pango(markdown) == pango + assert convert_from_pango(pango) == markdown + + +def test_pango_strips_links(): + assert convert_to_pango("[name](http://x.com)") == "name" def test_relative_path_conversion(tmp_path): - for path in [tmp_path / f for f in ("rel.jpg", "rel.pdf")]: - path.write_text("") # Create empty file. - tmp_path_uri = filesystem.LOCAL_FILE_PEFIX + str(tmp_path) + os.sep + "rel" + for name in ("rel.jpg", "rel.pdf", "rel.png"): + (tmp_path / name).write_text("") # Create empty file. + + def url(name): + return urls.get_local_url(str(tmp_path / name)) rel_paths = [ - ('[""file://rel"".jpg]', f'[""{tmp_path_uri}"".jpg]'), - ('[""rel"".jpg]', f'[""{tmp_path_uri}"".jpg]'), - ( - '[rel.pdf ""file://rel.pdf""]', - f'[rel.pdf ""{tmp_path_uri}.pdf""]', - ), - ('[rel.pdf ""rel.pdf""]', f'[rel.pdf ""{tmp_path_uri}.pdf""]'), + ("![](rel.jpg)", f"![]({url('rel.jpg')})"), + ("[doc](rel.pdf)", f"[doc]({url('rel.pdf')})"), + ("![](rel.png?50)", f"![]({url('rel.png')}?50)"), ] - for markup, expected in rel_paths: - assert expected == _convert_paths(markup, tmp_path) + assert _convert_paths(markup, tmp_path) == expected def test_absolute_path_conversion(tmp_path): abs_paths = [ - '[""file:///abs"".jpg]', - f'[""{tmp_path}/aha 1"".jpg]', - '[abs.pdf ""file:///abs.pdf""]', - f'[abs.pdf ""{tmp_path}/abs.pdf""]', - "www.google.com", - "www.google.com/page.php", + "![](file:///abs.jpg)", + f"![]({tmp_path}/aha.jpg)", + "[doc](file:///abs.pdf)", + "[site](http://www.google.com)", ] - for path in abs_paths: assert path == _convert_paths(path, tmp_path) -class TestGetHtmlExportConfig: +def test_entry_reference_fragment_untouched(tmp_path): + assert _convert_paths("[2019-08-01](#2019-08-01)", tmp_path) == "[2019-08-01](#2019-08-01)" + + +class TestHtml: @staticmethod @pytest.fixture def process(tmp_path): def process(markup): - html_document = convert(markup, "html", tmp_path) - return html_document.split("\n") + return convert(markup, "html", tmp_path) return process def test_encoding(self, process): - document = process("Content") - assert '' in document - - def test_toc(self, process): - document = process("Content") - assert '
    ' not in document + assert '' in process("Content") - def test_css_sugar(self, process): - document = process("Content") - assert '
    ' in document + def test_legacy_txt2tags_input(self, process): + document = process("//italic// and --struck--") + assert "italic" in document + assert "struck" in document @pytest.mark.parametrize( "markup,expected", [ - ("content \\\\ ", "content
    "), - ("content\\\\", "content
    "), - ("content\\\\ ", "content
    "), - ], - ) - def test_line_break_escaping(self, markup, expected, process): - document = process(markup) - assert expected in document - - @pytest.mark.parametrize( - "markup,expected", - [ - ("#TAG", r'#TAG'), - ("Numeric #3tag", r'Numeric #3tag'), - ("Just #34 numbers", r"Just #34 numbers"), + ("#TAG", '#TAG'), + ("Numeric #3tag", '#3tag'), + ("Just #34 numbers", "#34"), ], ) def test_hashtags(self, markup, expected, process): - document = process(markup) - assert expected in document + assert expected in process(markup) @pytest.mark.parametrize( "markup,expected", [ - ("{Sky|color:blue}", r'Sky'), - ( - "{Red|color:#FF0000} firetruck", - r'Red firetruck', - ), + ("{Sky|color:blue}", 'Sky'), + ("{Red|color:#FF0000} truck", 'Red truck'), ], ) def test_colors(self, markup, expected, process): - document = process(markup) - assert expected in document + assert expected in process(markup) - @pytest.mark.parametrize( - "markup,expected", - [ - ( - '[""/image"".png?50]', - '', - ), - ( - '[""/image"".jpg?50]', - '', - ), - ( - '[""/image"".jpeg?50]', - '', - ), - ( - '[""/image"".gif?50]', - '', - ), - ( - '[""/image"".eps?50]', - '', - ), - ( - '[""/image"".bmp?50]', - '', - ), - ( - '[""/image"".svg?50]', - '', - ), - ], - ) - def test_images_resize_allowed_extensions(self, markup, expected, process): - document = process(markup) - assert expected in document + def test_image_resize(self, process): + document = process("![](/image.png?50)") + assert 'src="/image.png"' in document + assert 'width="50"' in document @pytest.mark.parametrize( "markup,expected", [ - ( - '[""/image"".png?50]', - '', - ), - ( - '[""/image"".jpg]', - '', - ), - ( - '[""file:///image"".png?10]', - '', - ), - ( - '[""file:///image"".jpg]', - '', - ), - ], - ) - def test_images_width_resize(self, markup, expected, process): - document = process(markup) - assert expected in document - - @pytest.mark.parametrize( - "markup,expected", - [ - ( - "Simple [named reference 2019-08-01]", - 'Simple named reference', - ), - ( - "An inline [2019-08-01] date", - 'An inline 2019-08-01 date', - ), - ("[2019-10-20] is first", '2019-10-20 is first'), + ("Simple [named reference 2019-08-01]", 'named reference'), + ("An inline [2019-08-01] date", '2019-08-01'), ], ) def test_entry_reference_links(self, markup, expected, process): - document = process(markup) - assert expected in document + assert expected in process(markup) def test_day_fragment_anchor_element(self, process): date = datetime.date(2019, 10, 21) day = Day(Month(date.year, date.month), date.day) - markup = get_markup_for_day(day, "html", date=date.strftime("%d-%m-%Y")) - document = process(markup) + assert f'' in process(markup) - assert rf'' in document + def test_mathjax_added(self, process): + assert "MathJax" in process("$$x^3$$") - def test_mathjax(self, process): - document = process("$$x^3$$") - assert r"" in document + def test_mathjax_absent(self, process): + assert "MathJax" not in process("no formula") -class TestGetTexExportConfig: +class TestLatex: @staticmethod @pytest.fixture def process(tmp_path): def process(markup): - html_document = convert(markup, "tex", tmp_path) - return html_document.split("\n") + return convert(markup, "tex", tmp_path) return process - def test_encoding(self, process): + def test_preamble(self, process): document = process("Content") - assert r"\usepackage[utf8]{inputenc} % char encoding" in document - - def test_euro_replacement(self, process): - document = process("€") - assert "Euro" in document - - @pytest.mark.parametrize( - "markup,expected", - [(r'[""file/path"".png]', r'\includegraphics{"file/path".png}')], - ) - def test_path_escape(self, markup, expected, process): - document = process(markup) - assert expected in document - - @pytest.mark.skipif(sys.platform != "win32", reason="Windows-specific") - def test_image_scheme_fix_win32(self, process): - document = process('[""file:///image"".png]') - assert r'\includegraphics{"image".png}' in document - - @pytest.mark.skipif(sys.platform == "win32", reason="POSIX-specific") - def test_image_scheme_fix_posix(self, process): - document = process('[""file://image"".png]') - assert r'\includegraphics{"image".png}' in document - - @pytest.mark.skipif(sys.platform != "win32", reason="Windows-specific") - def test_file_scheme_fix_win32(self, process): - document = process("[file.txt file:///path/to/text/file.txt]") - assert r"\htmladdnormallink{file.txt}{run:path/to/text/file.txt}" in document - - @pytest.mark.skipif(sys.platform == "win32", reason="POSIX-specific") - def test_file_scheme_fix_posix(self, process): - document = process("[file.txt file://path/to/text/file.txt]") - assert r"\htmladdnormallink{file.txt}{run:path/to/text/file.txt}" in document + assert r"\documentclass" in document + assert r"\begin{document}" in document - @pytest.mark.parametrize( - "markup,expected", - [ - ("content \\\\ ", "content \\\\"), - ("content\\\\", "content\\\\"), - ("content\\\\ ", "content\\\\"), - ], - ) - def test_line_break_escaping(self, markup, expected, process): - document = process(markup) - assert expected in document + def test_legacy_heading(self, process): + assert r"\section{Title}" in process("= Title =") @pytest.mark.parametrize( "markup,expected", [ ("#TAG", r"\textcolor{red}{\#TAG\index{TAG}}"), - ("Numeric #3tag", r"Numeric \textcolor{red}{\#3tag\index{3tag}}"), - ("Just #34 numbers", r"Just \#34 numbers"), - ("#include ", r"\#include $<$iostream$>$"), - # TODO: ('#define FOO BAR', r'\#define FOO BAR'), - ("Blue: #0000FF", r"Blue: \#0000FF"), + ("Numeric #3tag", r"\textcolor{red}{\#3tag\index{3tag}}"), ], ) def test_hashtags(self, markup, expected, process): - document = process(markup) - assert expected in document - - @pytest.mark.parametrize( - "markup,expected", - [ - ('[""image"".png?50]', '\\includegraphics[width=50px]{"image".png}'), - ('[""image"".jpg]', '\\includegraphics{"image".jpg}'), - ], - ) - def test_images_width_resize(self, markup, expected, process): - document = process(markup) - assert expected in document - - @pytest.mark.parametrize( - "markup,expected", - [(r"\[f(x) = x^2\]", "$$f(x) = x^2$$"), (r"$$f(x) = x^2$$", "$$f(x) = x^2$$")], - ) - def test_latex_equation_escape_display_mode(self, markup, expected, process): - document = process(markup) - assert expected in document - - @pytest.mark.parametrize("markup,expected", [(r"\(f(x) = x^2\)", "$f(x) = x^2$")]) - def test_latex_equation_escape_inline_mode(self, markup, expected, process): - document = process(markup) - assert expected in document + assert expected in process(markup) - @pytest.mark.parametrize( - "markup,expected", [(r"„content”", '"content"'), (r"”content“", '"content"')] - ) - def test_quotation_mark_replacement(self, markup, expected, process): - document = process(markup) - assert expected in document + def test_color(self, process): + assert r"\textcolor{blue}{Sky}" in process("{Sky|color:blue}") - @pytest.mark.parametrize( - "markup,expected", - [ - ("{Sky|color:blue}", r"\textcolor{blue}{Sky}"), - ("{Red|color:#FF0000} firetruck", r"\textcolor{\#FF0000}{Red} firetruck"), - ], - ) - def test_colors(self, markup, expected, process): - document = process(markup) - assert expected in document - - def test_index_generation(self, process): - document = process("content") - assert r"\usepackage{makeidx} % user defined" in document - assert r"\makeindex" in document - assert r"\printindex" in document - - def test_tags_are_collected_for_index_generation(self, process): - document = process("#tag") - assert r"\index{tag}" in "".join(document) - - @pytest.mark.parametrize( - "markup,expected", - [ - ( - "This is a [named reference 2019-08-01]", - "This is a named reference (2019-08-01)", - ), - ( - "Today is 2019-08-01 - a wonderful day", - "Today is 2019-08-01 - a wonderful day", - ), - ], - ) - def test_entry_reference_links(self, markup, expected, process): - document = process(markup) - assert expected in document + def test_entry_reference(self, process): + assert "named reference (2019-08-01)" in process("A [named reference 2019-08-01]") -class TestGetPlainTextExportConfig: +class TestPlainText: @staticmethod @pytest.fixture def process(tmp_path): def process(markup): - html_document = convert(markup, "txt", tmp_path) - return html_document.split("\n") + return convert(markup, "txt", tmp_path) return process - @pytest.mark.parametrize( - "markup,expected", - [ - ("content \\\\ ", "content "), - ("content\\\\", "content"), - ("content\\\\ ", "content"), - ], - ) - def test_line_break_escaping(self, markup, expected, process): - document = process(markup) - assert expected in document - - @pytest.mark.parametrize( - "markup,expected", - [ - ("{Sky|color:blue}", r"Sky"), - ("{Red|color:#FF0000} firetruck", r"Red firetruck"), - ], - ) - def test_colors(self, markup, expected, process): - document = process(markup) - assert expected in document - - @pytest.mark.parametrize( - "markup,expected", - [("[image.png?50]", "[image.png?50]"), ("[image.jpg]", "[image.jpg]")], - ) - def test_images_width_resize(self, markup, expected, process): - document = process(markup) - assert expected in document + def test_color_stripped(self, process): + document = process("{Sky|color:blue}") + assert "Sky" in document + assert "color" not in document - @pytest.mark.parametrize( - "markup,expected", - [ - ( - "This is a [named reference 2019-08-01]", - "This is a named reference (2019-08-01)", - ), - ( - "Today is 2019-08-01 - a wonderful day", - "Today is 2019-08-01 - a wonderful day", - ), - ], - ) - def test_entry_reference_links(self, markup, expected, process): - document = process(markup) - assert expected in document + def test_entry_reference(self, process): + assert "named reference (2019-08-01)" in process("A [named reference 2019-08-01]") From 11e0323aec4f410af1352a0f34ebb64a55577337 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Sat, 13 Jun 2026 17:49:22 +0000 Subject: [PATCH 03/13] Add Markdown syntax highlighting in the editor Ship a customized copy of GtkSourceView's markdown.lang that also highlights RedNotebook's #hashtags, {text|color:value} markup, entry references and ~~strikethrough~~, and load it instead of the txt2tags language. --- rednotebook/files/markdown.lang | 462 ++++++++++++++++++ .../files/rednotebook-highlight-style.xml | 4 + rednotebook/gui/main_window.py | 14 +- 3 files changed, 473 insertions(+), 7 deletions(-) create mode 100644 rednotebook/files/markdown.lang diff --git a/rednotebook/files/markdown.lang b/rednotebook/files/markdown.lang new file mode 100644 index 00000000..9f9810d0 --- /dev/null +++ b/rednotebook/files/markdown.lang @@ -0,0 +1,462 @@ + + + + + + text/x-markdown + *.markdown;*.md;*.mkd + <!-- + --> + + + + - - -
    -
    -

    %(HEADER1)s

    -

    %(HEADER2)s

    -

    %(HEADER3)s

    -
    -
    - -""", - "dbk": """\ - - -
    - - %(HEADER1)s - - %(HEADER2)s - - %(HEADER3)s - -""", - "man": """\ -.TH "%(HEADER1)s" 1 "%(HEADER3)s" "%(HEADER2)s" -""", - "mgp": """\ -#!/usr/X11R6/bin/mgp -t 90 -%%deffont "normal" xfont "utopia-medium-r", charset "iso8859-1" -%%deffont "normal-i" xfont "utopia-medium-i", charset "iso8859-1" -%%deffont "normal-b" xfont "utopia-bold-r" , charset "iso8859-1" -%%deffont "normal-bi" xfont "utopia-bold-i" , charset "iso8859-1" -%%deffont "mono" xfont "courier-medium-r", charset "iso8859-1" -%%default 1 size 5 -%%default 2 size 8, fore "yellow", font "normal-b", center -%%default 3 size 5, fore "white", font "normal", left, prefix " " -%%tab 1 size 4, vgap 30, prefix " ", icon arc "red" 40, leftfill -%%tab 2 prefix " ", icon arc "orange" 40, leftfill -%%tab 3 prefix " ", icon arc "brown" 40, leftfill -%%tab 4 prefix " ", icon arc "darkmagenta" 40, leftfill -%%tab 5 prefix " ", icon arc "magenta" 40, leftfill -%%%%------------------------- end of headers ----------------------------- -%%page - - - - - -%%size 10, center, fore "yellow" -%(HEADER1)s - -%%font "normal-i", size 6, fore "white", center -%(HEADER2)s - -%%font "mono", size 7, center -%(HEADER3)s -""", - "moin": """\ -'''%(HEADER1)s''' - -''%(HEADER2)s'' - -%(HEADER3)s -""", - "gwiki": """\ -*%(HEADER1)s* - -%(HEADER2)s - -_%(HEADER3)s_ -""", - "adoc": """\ -= %(HEADER1)s -%(HEADER2)s -%(HEADER3)s -""", - "doku": """\ -===== %(HEADER1)s ===== - -**//%(HEADER2)s//** - -//%(HEADER3)s// -""", - "pmw": """\ -(:Title %(HEADER1)s:) - -(:Description %(HEADER2)s:) - -(:Summary %(HEADER3)s:) -""", - "wiki": """\ -'''%(HEADER1)s''' - -%(HEADER2)s - -''%(HEADER3)s'' -""", - "tex": r"""\documentclass{article} -\usepackage{booktabs} %% needed for tables -\usepackage{graphicx} -\usepackage{paralist} %% needed for compact lists -\usepackage[normalem]{ulem} %% needed by strike -\usepackage[urlcolor=blue,colorlinks=true]{hyperref} -\usepackage[%(ENCODING)s]{inputenc} %% char encoding -\usepackage{%(STYLE)s} %% user defined - -\title{%(HEADER1)s} -\author{%(HEADER2)s} -\begin{document} -\date{%(HEADER3)s} -\maketitle -\clearpage -""", - "lout": """\ -@SysInclude { doc } -@Document - @InitialFont { Times Base 12p } # Times, Courier, Helvetica, ... - @PageOrientation { Portrait } # Portrait, Landscape - @ColumnNumber { 1 } # Number of columns (2, 3, ...) - @PageHeaders { Simple } # None, Simple, Titles, NoTitles - @InitialLanguage { English } # German, French, Portuguese, ... - @OptimizePages { Yes } # Yes/No smart page break feature -// -@Text @Begin -@Display @Heading { %(HEADER1)s } -@Display @I { %(HEADER2)s } -@Display { %(HEADER3)s } -#@NP # Break page after Headers -""", - "creole": """\ -%(HEADER1)s -%(HEADER2)s -%(HEADER3)s -""", - "md": """\ -%(HEADER1)s -%(HEADER2)s -%(HEADER3)s -""", - "ctx": r"""\mainlanguage[en] -\definecolor[linkcolor][h=0007F0] -\setupcolors[state=start] -\setupinteraction[state=start, - title={%(HEADER1)s}, - author={%(HEADER2)s}, - contrastcolor=linkcolor, - color=linkcolor, - ] -\placebookmarks[section,subsection,subsubsection] -\definehead[myheaderone][title] -\setuphead - [myheaderone] - [textstyle=cap, - align=middle, - after=\nowhitespace - ] -\definehead[myheadertwo][subject] -\setuphead - [myheadertwo] - [ before=\nowhitespace, - align=middle, - after=\nowhitespace - ] -\definehead[myheaderthree][subsubject] -\setuphead - [myheaderthree] - [before=\nowhitespace, - align=middle, - ] -\definedescription - [compdesc] - [alternative=serried, - headstyle=bold, - width=broad, - ] -\setupTABLE[frame=off] -\setupexternalfigures[maxwidth=0.7\textwidth] -\setupheadertexts[] -\setupfootertexts[pagenumber] -\setupwhitespace[medium] -\setupheads[number=no] -\usemodule[%(STYLE)s] -\starttext - -\myheaderone{%(HEADER1)s} -\myheadertwo{%(HEADER2)s} -\myheaderthree{%(HEADER3)s} - -""", - # @SysInclude { tbl } # Tables support - # setup: @MakeContents { Yes } # show TOC - # setup: @SectionGap # break page at each section -} -assert set(HEADER_TEMPLATE) == set(TARGETS) - - -############################################################################## - - -def getTags(config): - "Returns all the known tags for the specified target" - - keys = """ - title1 numtitle1 - title2 numtitle2 - title3 numtitle3 - title4 numtitle4 - title5 numtitle5 - title1Open title1Close - title2Open title2Close - title3Open title3Close - title4Open title4Close - title5Open title5Close - blockTitle1Open blockTitle1Close - blockTitle2Open blockTitle2Close - blockTitle3Open blockTitle3Close - - paragraphOpen paragraphClose - blockVerbOpen blockVerbClose blockVerbLine - blockQuoteOpen blockQuoteClose blockQuoteLine - blockCommentOpen blockCommentClose - - fontMonoOpen fontMonoClose - fontBoldOpen fontBoldClose - fontItalicOpen fontItalicClose - fontUnderlineOpen fontUnderlineClose - fontStrikeOpen fontStrikeClose - - listOpen listClose - listOpenCompact listCloseCompact - listItemOpen listItemClose listItemLine - numlistOpen numlistClose - numlistOpenCompact numlistCloseCompact - numlistItemOpen numlistItemClose numlistItemLine - deflistOpen deflistClose - deflistOpenCompact deflistCloseCompact - deflistItem1Open deflistItem1Close - deflistItem2Open deflistItem2Close deflistItem2LinePrefix - - bar1 bar2 - url urlMark - email emailMark - img imgAlignLeft imgAlignRight imgAlignCenter - _imgAlignLeft _imgAlignRight _imgAlignCenter - - tableOpen tableClose - _tableBorder _tableAlignLeft _tableAlignCenter - tableRowOpen tableRowClose tableRowSep - tableTitleRowOpen tableTitleRowClose - tableCellOpen tableCellClose tableCellSep - tableTitleCellOpen tableTitleCellClose tableTitleCellSep - _tableColAlignLeft _tableColAlignRight _tableColAlignCenter - _tableCellAlignLeft _tableCellAlignRight _tableCellAlignCenter - _tableCellColSpan tableColAlignSep - _tableCellMulticolOpen - _tableCellMulticolClose - - bodyOpen bodyClose - cssOpen cssClose - tocOpen tocClose TOC - anchor - comment - pageBreak - EOD - """.split() - - # TIP: \a represents the current text inside the mark - # TIP: ~A~, ~B~ and ~C~ are expanded to other tags parts - alltags = { - "txt": { - "title1": " \a", - "title2": "\t\a", - "title3": "\t\t\a", - "title4": "\t\t\t\a", - "title5": "\t\t\t\t\a", - "blockQuoteLine": "\t", - "listItemOpen": "- ", - "numlistItemOpen": "\a. ", - "bar1": "\a", - "url": "\a", - "urlMark": "\a (\a)", - "email": "\a", - "emailMark": "\a (\a)", - "img": "[\a]", - }, - "html": { - "anchor": ' id="\a"', - "bar1": '
    ', - "bar2": '
    ', - "blockQuoteClose": "", - "blockQuoteOpen": "
    ", - "blockVerbClose": "", - "blockVerbOpen": "
    ",
-            "bodyClose": "
    ", - "bodyOpen": '
    ', - "comment": "", - "cssClose": "", - "cssOpen": "