diff --git a/CHANGELOG.md b/CHANGELOG.md index 97b15fc4..0ca62348 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,4 @@ +* Format entries with Markdown, rendered by markdown-it-py. Legacy txt2tags entries are converted to Markdown automatically, the editor highlights Markdown syntax, and the format/insert buttons now insert Markdown (@jendrikseipp). * Fix segfault on Wayland when setting the window icon (#806, @sjg20) # 2.42 (2025-12-28) diff --git a/README.md b/README.md index a7114de6..9bf729c0 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ Needed for running RedNotebook: * GtkSourceView (3.0+): https://wiki.gnome.org/Projects/GtkSourceView * Python (3.8+): https://www.python.org * PyYAML (3.10+): https://pyyaml.org + * markdown-it-py, mdit-py-plugins and linkify-it-py: https://github.com/executablebooks/markdown-it-py * WebKitGTK (2.16+): https://webkitgtk.org (only on Linux and macOS) * PyEnchant for spell checking (1.6+): https://pypi.org/project/pyenchant/ (optional) diff --git a/debian/control b/debian/control index 201a3b76..d3600b58 100644 --- a/debian/control +++ b/debian/control @@ -27,6 +27,9 @@ Depends: ${python3:Depends}, gir1.2-pango-1.0, gir1.2-webkit2-4.1 | gir1.2-webkit2-4.0, python3-gi, + python3-linkify-it, + python3-markdown-it, + python3-mdit-py-plugins, python3-yaml Recommends: python3-enchant Description: Modern desktop diary and personal journaling tool diff --git a/dev/whitelist.py b/dev/whitelist.py index 078cf54f..4de44106 100644 --- a/dev/whitelist.py +++ b/dev/whitelist.py @@ -51,3 +51,37 @@ def __getattr__(self, _): Dummy().error_par Dummy().goodbye_par Dummy().example_entry + +# markdownmarkup renderer methods are dispatched dynamically by token type. +Dummy().blockquote_close +Dummy().blockquote_open +Dummy().bullet_list_close +Dummy().bullet_list_open +Dummy().code_block +Dummy().code_inline +Dummy().em_close +Dummy().em_open +Dummy().hardbreak +Dummy().heading_close +Dummy().heading_open +Dummy().hr +Dummy().html_block +Dummy().html_inline +Dummy().link_close +Dummy().link_open +Dummy().list_item_close +Dummy().list_item_open +Dummy().math_block +Dummy().math_inline +Dummy().ordered_list_close +Dummy().ordered_list_open +Dummy().paragraph_close +Dummy().paragraph_open +Dummy().rn_color +Dummy().s_close +Dummy().s_open +Dummy().softbreak +Dummy().strong_close +Dummy().strong_open +Dummy().th_close +Dummy().tr_close diff --git a/rednotebook/data.py b/rednotebook/data.py index 685e5b3b..8221436d 100644 --- a/rednotebook/data.py +++ b/rednotebook/data.py @@ -41,11 +41,11 @@ and add them to the 'Tags' section on the left panel. This pattern DOES NOT control the styling of hashtags in the text. -To control this behaviour refer to rednotebook/files/t2t.lang +To control this behaviour refer to rednotebook/files/markdown.lang (regexes) and rednotebook/files/rednotebook-highlight-style.xml (styles). If you make changes to this pattern, is very likely you will have to -make changes to /rednotebook/files/t2t.lang +make changes to /rednotebook/files/markdown.lang """ HASHTAG = re.compile(HASHTAG_PATTERN, flags=re.IGNORECASE) diff --git a/rednotebook/external/txt2tags.py b/rednotebook/external/txt2tags.py deleted file mode 100644 index 9b6feceb..00000000 --- a/rednotebook/external/txt2tags.py +++ /dev/null @@ -1,5045 +0,0 @@ -#!/usr/bin/env python -# txt2tags - generic text conversion tool -# https://txt2tags.org/ -# https://github.com/jendrikseipp/txt2tags -# -# Copyright 2001-2010 Aurelio Jargas -# Copyright 2010-2019 Jendrik Seipp -# -# License: GPL2+ (http://www.gnu.org/licenses/gpl-2.0.txt) -# -######################################################################## -# -# The code that [1] parses the marked text is separated from the -# code that [2] insert the target tags. -# -# [1] made by: def convert() -# [2] made by: class BlockMaster -# -# The structures of the marked text are identified and its contents are -# extracted into a data holder (Python lists and dictionaries). -# -# When parsing the source file, the blocks (para, lists, quote, table) -# are opened with BlockMaster, right when found. Then its contents, -# which spans on several lines, are feeded into a special holder on the -# BlockMaster instance. Just when the block is closed, the target tags -# are inserted for the full block as a whole, in one pass. This way, we -# have a better control on blocks. Much better than the previous line by -# line approach. -# -# In other words, whenever inside a block, the parser *holds* the tag -# insertion process, waiting until the full block is read. That was -# needed primary to close paragraphs for the XHTML target, but -# proved to be a very good adding, improving many other processing. -# -# ------------------------------------------------------------------- -# -# These important classes are all documented: -# CommandLine, SourceDocument, ConfigMaster, ConfigLines. -# -# There is a RAW Config format and all kind of configuration is first -# converted to this format. Then a generic method parses it. -# -# These functions get information about the input file(s) and take -# care of the init processing: -# process_source_file() and convert_file() -# -######################################################################## - -# XXX Smart Image Align don't work if the image is a link -# Can't fix that because the image is expanded together with the -# link, at the linkbank filling moment. Only the image is passed -# to parse_images(), not the full line, so it is always 'middle'. - -# XXX Paragraph separation not valid inside Quote -# Quote will not have
inside, instead will close and open -# again the. This really sux in CSS, when defining a -# different background color. Still don't know how to fix it. - -# XXX TODO (maybe) -# New mark which expands to an anchor full title. -# It is necessary to parse the full document in this order: -# DONE 1st scan: HEAD: get all settings, including %!includeconf -# DONE 2nd scan: BODY: expand includes & apply %!preproc -# 3rd scan: BODY: read titles and compose TOC info -# 4th scan: BODY: full parsing, expanding [#anchor] 1st -# Steps 2 and 3 can be made together, with no tag adding. -# Two complete body scans will be *slow*, don't know if it worths. -# One solution may be add the titles as postproc rules - - -import collections -import getopt -import os -import re -import sys - -############################################################################## - -# Program information -my_url = "https://txt2tags.org" -my_name = "txt2tags" -my_email = "jendrikseipp@gmail.com" -__version__ = "3.9" - -# FLAGS : the conversion related flags , may be used in %!options -# OPTIONS : the conversion related options, may be used in %!options -# ACTIONS : the other behavior modifiers, valid on command line only -# NO_TARGET: actions that don't require a target specification -# NO_MULTI_INPUT: actions that don't accept more than one input file -# CONFIG_KEYWORDS: the valid %!key:val keywords -# -# FLAGS and OPTIONS are configs that affect the converted document. -# They usually have also a --no-", - "blockQuoteOpen": "
", - "blockVerbClose": "", - "blockVerbOpen": "", - "bodyClose": "", - "bodyOpen": '', - "comment": "", - "cssClose": "", - "cssOpen": " +""" + +MATHJAX_FILE = "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-svg.js" +MATHJAX = f"""\ + + +""" + +# A #hashtag must contain at least one letter and must not be a hex colour or +# a C preprocessor directive. This mirrors rednotebook.data.HASHTAG. +HASHTAG_BODY = re.compile( + r"(?![0-9a-fA-F]{6}\b|include\b|define\b|ifdef\b|ifndef\b|endif\b)(\w*[^\W\d_]+\w*)" +) +COLOR = re.compile(r"\{([^{}|]+)\|color:([^{}]+)\}") +IMAGE_WIDTH = re.compile(r"\?(\d+)$") + + +# -------------------------------------------------------------------------- +# Inline plugins +# -------------------------------------------------------------------------- + + +def _hashtag_rule(state, silent): + pos = state.pos + if state.src[pos] not in "##": + return False + if pos > 0: + prev = state.src[pos - 1] + if prev.isalnum() or prev == "_" or prev in "": + return False + match = HASHTAG_BODY.match(state.src, pos + 1) + if not match: + return False + if not silent: + token = state.push("hashtag", "", 0) + token.content = state.src[pos : match.end()] + token.meta = {"tag": match.group(1)} + state.pos = match.end() + return True + + +def _color_rule(state, silent): + if state.src[state.pos] != "{": + return False + match = COLOR.match(state.src, state.pos) + if not match: + return False + if not silent: + token = state.push("rn_color", "", 0) + token.meta = {"text": match.group(1), "color": match.group(2)} + state.pos = match.end() + return True + + +def _rednotebook_plugin(md): + md.inline.ruler.before("emphasis", "hashtag", _hashtag_rule) + md.inline.ruler.before("emphasis", "rn_color", _color_rule) + + +# -------------------------------------------------------------------------- +# HTML renderer +# -------------------------------------------------------------------------- + + +class HtmlRenderer(RendererHTML): + def hashtag(self, tokens, idx, options, env): + return f'{escapeHtml(tokens[idx].content)}' + + def rn_color(self, tokens, idx, options, env): + meta = tokens[idx].meta + return f'{escapeHtml(meta["text"])}' + + def image(self, tokens, idx, options, env): + token = tokens[idx] + src = token.attrs.get("src", "") + width = "" + match = IMAGE_WIDTH.search(src) + if match: + width = f' width="{match.group(1)}"' + src = src[: match.start()] + alt = escapeHtml(token.content) + return f'' + + def math_inline(self, tokens, idx, options, env): + return f"\\({tokens[idx].content}\\)" + + def math_block(self, tokens, idx, options, env): + return f"$$\n{tokens[idx].content}\n$$\n" + + +# -------------------------------------------------------------------------- +# Token-walking renderers for LaTeX and plain text +# -------------------------------------------------------------------------- + +_HEADING_TO_TEX = { + "h1": "section", + "h2": "subsection", + "h3": "subsubsection", + "h4": "paragraph", + "h5": "subparagraph", + "h6": "subparagraph", +} + +_TEX_ESCAPES = { + "\\": r"\textbackslash{}", + "&": r"\&", + "%": r"\%", + "$": r"\$", + "#": r"\#", + "_": r"\_", + "{": r"\{", + "}": r"\}", + "~": r"\textasciitilde{}", + "^": r"\textasciicircum{}", +} +_TEX_ESCAPE_RE = re.compile("|".join(re.escape(key) for key in _TEX_ESCAPES)) + + +def tex_escape(text): + return _TEX_ESCAPE_RE.sub(lambda m: _TEX_ESCAPES[m.group()], text) + + +class _TokenRenderer: + """Walk the markdown-it token stream and dispatch by token type.""" + + def __init__(self, parser=None): + self.parser = parser + + def render(self, tokens, options, env): + out = [] + for token in tokens: + if token.type == "inline": + out.append(self.render(token.children or [], options, env)) + else: + method = getattr(self, token.type, None) + if method is not None: + out.append(method(token, env)) + return "".join(out) + + # Fallbacks for the markup we do not specially handle. + def text(self, token, env): + return token.content + + def softbreak(self, token, env): + return "\n" + + def html_inline(self, token, env): + return "" + + def html_block(self, token, env): + return "" + + +class LatexRenderer(_TokenRenderer): + def text(self, token, env): + return tex_escape(token.content) + + def softbreak(self, token, env): + return "\n" + + def hardbreak(self, token, env): + return "\\\\\n" + + def paragraph_open(self, token, env): + return "" + + def paragraph_close(self, token, env): + # Tight list items wrap their text in hidden paragraphs. + return "" if token.hidden else "\n\n" + + def heading_open(self, token, env): + return "\\" + _HEADING_TO_TEX.get(token.tag, "section") + "{" + + def heading_close(self, token, env): + return "}\n\n" + + def strong_open(self, token, env): + return "\\textbf{" + + def strong_close(self, token, env): + return "}" + + def em_open(self, token, env): + return "\\textit{" + + def em_close(self, token, env): + return "}" + + def s_open(self, token, env): + return "\\sout{" + + def s_close(self, token, env): + return "}" + + def code_inline(self, token, env): + return "\\texttt{" + tex_escape(token.content) + "}" + + def fence(self, token, env): + return "\\begin{verbatim}\n" + token.content + "\\end{verbatim}\n\n" + + code_block = fence + + def link_open(self, token, env): + return "\\href{" + token.attrs.get("href", "") + "}{" + + def link_close(self, token, env): + return "}" + + def image(self, token, env): + src = token.attrs.get("src", "") + match = IMAGE_WIDTH.search(src) + options = "" + if match: + options = f"[width={match.group(1)}px]" + src = src[: match.start()] + return f'\\includegraphics{options}{{"{src}"}}' + + def bullet_list_open(self, token, env): + return "\\begin{itemize}\n" + + def bullet_list_close(self, token, env): + return "\\end{itemize}\n" + + def ordered_list_open(self, token, env): + return "\\begin{enumerate}\n" + + def ordered_list_close(self, token, env): + return "\\end{enumerate}\n" + + def list_item_open(self, token, env): + return "\\item " + + def list_item_close(self, token, env): + return "\n" + + def hr(self, token, env): + return "\\par\\noindent\\rule{\\linewidth}{0.4pt}\n\n" + + def blockquote_open(self, token, env): + return "\\begin{quote}\n" + + def blockquote_close(self, token, env): + return "\\end{quote}\n" + + def hashtag(self, token, env): + display = tex_escape(token.content.lstrip("##")) + index = token.meta["tag"] + return f"\\textcolor{{red}}{{\\#{display}\\index{{{index}}}}}" + + def rn_color(self, token, env): + meta = token.meta + return f"\\textcolor{{{tex_escape(meta['color'])}}}{{{tex_escape(meta['text'])}}}" + + def math_inline(self, token, env): + return f"${token.content}$" + + def math_block(self, token, env): + return f"$${token.content}$$\n" + + # Minimal table support: render cells separated by " & " and rows by "\\". + def tr_close(self, token, env): + return "\\\\\n" + + def td_close(self, token, env): + return " & " + + th_close = td_close + + +class PlainRenderer(_TokenRenderer): + def hardbreak(self, token, env): + return "\n" + + def paragraph_close(self, token, env): + # Tight list items wrap their text in hidden paragraphs. + return "" if token.hidden else "\n\n" + + def heading_close(self, token, env): + return "\n\n" + + def code_inline(self, token, env): + return token.content + + def fence(self, token, env): + return token.content + "\n" + + code_block = fence + + def image(self, token, env): + return f"[{token.attrs.get('src', '')}]" + + def bullet_list_open(self, token, env): + # Start nested lists on their own line. + return "\n" if token.level else "" + + ordered_list_open = bullet_list_open + + def list_item_open(self, token, env): + return "- " + + def list_item_close(self, token, env): + return "\n" + + def hr(self, token, env): + return "\n" + "=" * 20 + "\n\n" + + def hashtag(self, token, env): + return token.content + + def rn_color(self, token, env): + return token.meta["text"] + + def math_inline(self, token, env): + return token.content + + def math_block(self, token, env): + return token.content + "\n" + + +_RENDERERS = {"html": HtmlRenderer, "tex": LatexRenderer, "txt": PlainRenderer} + + +def _get_parser(target): + md = MarkdownIt("commonmark", {"html": True, "linkify": True, "breaks": False}) + md.enable(["table", "strikethrough", "linkify"]) + md.use(dollarmath_plugin, double_inline=True) + md.use(_rednotebook_plugin) + md.renderer = _RENDERERS[target](md) + return md + + +def _walk(tokens): + for token in tokens: + yield token + if token.children: + yield from _walk(token.children) + + +# -------------------------------------------------------------------------- +# Document templates +# -------------------------------------------------------------------------- + +LATEX_PREAMBLE = r"""\documentclass[a4paper]{article} +\usepackage[utf8]{inputenc} +\usepackage[T1]{fontenc} +\usepackage{graphicx} +\usepackage{xcolor} +\usepackage[normalem]{ulem} +\usepackage{hyperref} +\usepackage{makeidx} +\makeindex +\title{%(title)s} +\begin{document} +\maketitle +""" + +LATEX_FOOTER = r""" +\printindex +\end{document} +""" + + +def _html_document(body, options, has_math): + css = CSS % { + "font": options.get("font", "sans-serif"), + "bgcolor": options.get("bgcolor", "white"), + "fgcolor": options.get("fgcolor", "black"), + } + mathjax = MATHJAX if has_math else "" + return ( + "\n\n\n" + '\n' + f"{css}{mathjax}" + "\n\n" + f"{body}" + "\n\n" + ) + + +def _latex_document(body, options): + title = options.get("title", "RedNotebook") + return LATEX_PREAMBLE % {"title": title} + body + LATEX_FOOTER + + +def render(text, target, options=None): + """Render Markdown ``text`` to ``target`` (``html``, ``tex`` or ``txt``).""" + options = options or {} + md = _get_parser(target) + env = {} + tokens = md.parse(text, env) + body = md.renderer.render(tokens, md.options, env) + + if target == "html": + has_math = options.get("add_mathjax") + if has_math is None: + has_math = any(token.type.startswith("math") for token in _walk(tokens)) + return _html_document(body, options, has_math) + # Collapse runs of blank lines that arise between block elements. + body = re.sub(r"\n{3,}", "\n\n", body) + if target == "tex": + return _latex_document(body, options) + return body.strip() + "\n" diff --git a/rednotebook/util/markup.py b/rednotebook/util/markup.py index 794db153..814bb7b5 100644 --- a/rednotebook/util/markup.py +++ b/rednotebook/util/markup.py @@ -19,107 +19,33 @@ import os import re -from rednotebook.data import HASHTAG -from rednotebook.external import txt2tags -from rednotebook.util import filesystem, urls +from rednotebook.util import filesystem, markdownmarkup, t2t_to_markdown, urls -# Linebreaks are only allowed at line ends -REGEX_LINEBREAK = r"\\\\[\s]*$" +# A trailing "text" link, used by pango_markup to strip links. REGEX_HTML_LINK = r"
(.*?)" -# pic [""/home/user/Desktop/RedNotebook pic"".png] -PIC_NAME = r"\S.*?\S|\S" -PIC_EXT = r"(?:png|jpe?g|gif|eps|bmp|svg)" -REGEX_PIC = re.compile(rf'(\["")({PIC_NAME})("")(\.{PIC_EXT})(\?\d+)?(\])', flags=re.I) - -# named local link [my file.txt ""file:///home/user/my file.txt""] -# named link in web [heise ""http://heise.de""] -REGEX_NAMED_LINK = re.compile(r'(\[)(.*?)(\s"")(\S.*?\S)(""\])', flags=re.I) - -ESCAPE_COLOR = r"XBEGINCOLORX\1XSEPARATORX\2XENDCOLORX" -COLOR_ESCAPED = r"XBEGINCOLORX(.*?)XSEPARATORX(.*?)XENDCOLORX" - -CSS = """\ - -""" - -# MathJax -FORMULAS_SUPPORTED = True -MATHJAX_FILE = "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-svg.js" - -# Explicitly setting inlineMath: [ ['\\(','\\)'] ] doesn't work. -# Using defaults: -# displayMath: [ ['$$','$$'], ['\[','\]'] ] -# inlineMath: [['\(','\)']] -MATHJAX_DELIMITERS = ["$$", "\\(", "\\)", r"\\[", "\\]"] -MATHJAX = f"""\ - - -""" +# Markdown image/link target: "" or "[text](url)". +REGEX_MD_LINK = re.compile(r"(!?\[[^\]]*\]\()([^)\s]+)(\))") +# Optional image width suffix. +REGEX_IMAGE_WIDTH = re.compile(r"\?(\d+)$") + +# Entry references such as "[2019-08-01]" or "[my day 2019-08-01]". +REGEX_NAMED_REFERENCE = re.compile(r"\[(?P .+?)\s+(?P \d{4}-\d{2}-\d{2})\s*\]") +REGEX_DATE_REFERENCE = re.compile(r"\[(?P \d{4}-\d{2}-\d{2})\]") + +# Math delimiters that MathJax understands besides "$"/"$$". +REGEX_MATH_DISPLAY = re.compile(r"\\\[(.+?)\\\]", flags=re.DOTALL) +REGEX_MATH_INLINE = re.compile(r"\\\((.+?)\\\)", flags=re.DOTALL) def convert_categories_to_markup(categories, with_category_title=True): - # Only add Category title if the text is displayed - markup = "== {} ==\n".format(_("Tags")) if with_category_title else "" + # Only add the "Tags" title if the text is displayed. + markup = "## {}\n".format(_("Tags")) if with_category_title else "" for category, entry_list in categories.items(): - markup += f"- {category}" + "\n" + markup += f"- {category}\n" for entry in entry_list: - markup += f" - {entry}" + "\n" + markup += f" - {entry}\n" markup += "\n\n" return markup @@ -133,11 +59,10 @@ def get_markup_for_day(day, target, with_text=True, with_tags=True, categories=N # Add date if it is not None and not the empty string if date: if target == "html": - # Following anchor will be used as a target for every entry reference mentioning - # this entry's date. - export_string += f"''''\n" + # The anchor is the target for entry references mentioning this date. + export_string += f'\n\n' - export_string += f"= {date} =\n\n" + export_string += f"# {date}\n\n" # Add text if with_text: @@ -174,244 +99,79 @@ def get_markup_for_day(day, target, with_text=True, with_tags=True, categories=N return "" -def _get_config(target, options): - # Set the configuration on the 'config' dict. - config = txt2tags.ConfigMaster()._get_defaults() - - config["outfile"] = txt2tags.MODULEOUT # results as list - config["target"] = target - - # The Pre (and Post) processing config is a list of lists: - # [ [this, that], [foo, bar], [patt, replace] ] - config["postproc"] = [] - config["preproc"] = [] - config["style"] = [] - - # Allow line breaks, r'\\\\' are 2 \ for regexes - config["preproc"].append([REGEX_LINEBREAK, "LINEBREAK"]) - - # Highlight hashtags. - if target == "tex": - config["preproc"].append([HASHTAG.pattern, r"\1{\2\3BEGININDEX\3ENDINDEX|color:red}"]) - else: - config["preproc"].append([HASHTAG.pattern, r"\1{\2\3|color:red}"]) - - # Escape color markup. - config["preproc"].append([r"\{(.*?)\|color:(.+?)\}", ESCAPE_COLOR]) - - if target == "html": - config["encoding"] = "UTF-8" # document encoding - config["toc"] = 0 - config["css-sugar"] = 1 - - # Line breaks - config["postproc"].append([r"LINEBREAK", "
"]) - - # Apply image resizing - config["postproc"].append([r"src=\"WIDTH(\d+)-", r'width="\1" src="']) - - # Flow paragraph from right to left or left to right depending on the language. - config["postproc"].append(["", '
']) - - # {{red text|color:red}} -> red text - config["postproc"].append([COLOR_ESCAPED, r'\1']) - - # Custom css - font = options.pop("font", "sans-serif") - css = CSS % { - "font": font, - "bgcolor": options.get("bgcolor", "white"), - "fgcolor": options.get("fgcolor", "black"), - } - config["postproc"].append([r"", f"{css}"]) - - # MathJax - if options.pop("add_mathjax"): - config["postproc"].append([r"