diff --git a/CHANGELOG.md b/CHANGELOG.md index 97b15fc4..0ca62348 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,4 @@ +* Format entries with Markdown, rendered by markdown-it-py. Legacy txt2tags entries are converted to Markdown automatically, the editor highlights Markdown syntax, and the format/insert buttons now insert Markdown (@jendrikseipp). * Fix segfault on Wayland when setting the window icon (#806, @sjg20) # 2.42 (2025-12-28) diff --git a/README.md b/README.md index a7114de6..9bf729c0 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ Needed for running RedNotebook: * GtkSourceView (3.0+): https://wiki.gnome.org/Projects/GtkSourceView * Python (3.8+): https://www.python.org * PyYAML (3.10+): https://pyyaml.org + * markdown-it-py, mdit-py-plugins and linkify-it-py: https://github.com/executablebooks/markdown-it-py * WebKitGTK (2.16+): https://webkitgtk.org (only on Linux and macOS) * PyEnchant for spell checking (1.6+): https://pypi.org/project/pyenchant/ (optional) diff --git a/debian/control b/debian/control index 201a3b76..d3600b58 100644 --- a/debian/control +++ b/debian/control @@ -27,6 +27,9 @@ Depends: ${python3:Depends}, gir1.2-pango-1.0, gir1.2-webkit2-4.1 | gir1.2-webkit2-4.0, python3-gi, + python3-linkify-it, + python3-markdown-it, + python3-mdit-py-plugins, python3-yaml Recommends: python3-enchant Description: Modern desktop diary and personal journaling tool diff --git a/dev/whitelist.py b/dev/whitelist.py index 078cf54f..4de44106 100644 --- a/dev/whitelist.py +++ b/dev/whitelist.py @@ -51,3 +51,37 @@ def __getattr__(self, _): Dummy().error_par Dummy().goodbye_par Dummy().example_entry + +# markdownmarkup renderer methods are dispatched dynamically by token type. +Dummy().blockquote_close +Dummy().blockquote_open +Dummy().bullet_list_close +Dummy().bullet_list_open +Dummy().code_block +Dummy().code_inline +Dummy().em_close +Dummy().em_open +Dummy().hardbreak +Dummy().heading_close +Dummy().heading_open +Dummy().hr +Dummy().html_block +Dummy().html_inline +Dummy().link_close +Dummy().link_open +Dummy().list_item_close +Dummy().list_item_open +Dummy().math_block +Dummy().math_inline +Dummy().ordered_list_close +Dummy().ordered_list_open +Dummy().paragraph_close +Dummy().paragraph_open +Dummy().rn_color +Dummy().s_close +Dummy().s_open +Dummy().softbreak +Dummy().strong_close +Dummy().strong_open +Dummy().th_close +Dummy().tr_close diff --git a/rednotebook/data.py b/rednotebook/data.py index 685e5b3b..8221436d 100644 --- a/rednotebook/data.py +++ b/rednotebook/data.py @@ -41,11 +41,11 @@ and add them to the 'Tags' section on the left panel. This pattern DOES NOT control the styling of hashtags in the text. -To control this behaviour refer to rednotebook/files/t2t.lang +To control this behaviour refer to rednotebook/files/markdown.lang (regexes) and rednotebook/files/rednotebook-highlight-style.xml (styles). If you make changes to this pattern, is very likely you will have to -make changes to /rednotebook/files/t2t.lang +make changes to /rednotebook/files/markdown.lang """ HASHTAG = re.compile(HASHTAG_PATTERN, flags=re.IGNORECASE) diff --git a/rednotebook/external/txt2tags.py b/rednotebook/external/txt2tags.py deleted file mode 100644 index 9b6feceb..00000000 --- a/rednotebook/external/txt2tags.py +++ /dev/null @@ -1,5045 +0,0 @@ -#!/usr/bin/env python -# txt2tags - generic text conversion tool -# https://txt2tags.org/ -# https://github.com/jendrikseipp/txt2tags -# -# Copyright 2001-2010 Aurelio Jargas -# Copyright 2010-2019 Jendrik Seipp -# -# License: GPL2+ (http://www.gnu.org/licenses/gpl-2.0.txt) -# -######################################################################## -# -# The code that [1] parses the marked text is separated from the -# code that [2] insert the target tags. -# -# [1] made by: def convert() -# [2] made by: class BlockMaster -# -# The structures of the marked text are identified and its contents are -# extracted into a data holder (Python lists and dictionaries). -# -# When parsing the source file, the blocks (para, lists, quote, table) -# are opened with BlockMaster, right when found. Then its contents, -# which spans on several lines, are feeded into a special holder on the -# BlockMaster instance. Just when the block is closed, the target tags -# are inserted for the full block as a whole, in one pass. This way, we -# have a better control on blocks. Much better than the previous line by -# line approach. -# -# In other words, whenever inside a block, the parser *holds* the tag -# insertion process, waiting until the full block is read. That was -# needed primary to close paragraphs for the XHTML target, but -# proved to be a very good adding, improving many other processing. -# -# ------------------------------------------------------------------- -# -# These important classes are all documented: -# CommandLine, SourceDocument, ConfigMaster, ConfigLines. -# -# There is a RAW Config format and all kind of configuration is first -# converted to this format. Then a generic method parses it. -# -# These functions get information about the input file(s) and take -# care of the init processing: -# process_source_file() and convert_file() -# -######################################################################## - -# XXX Smart Image Align don't work if the image is a link -# Can't fix that because the image is expanded together with the -# link, at the linkbank filling moment. Only the image is passed -# to parse_images(), not the full line, so it is always 'middle'. - -# XXX Paragraph separation not valid inside Quote -# Quote will not have

inside, instead will close and open -# again the
. This really sux in CSS, when defining a -# different background color. Still don't know how to fix it. - -# XXX TODO (maybe) -# New mark which expands to an anchor full title. -# It is necessary to parse the full document in this order: -# DONE 1st scan: HEAD: get all settings, including %!includeconf -# DONE 2nd scan: BODY: expand includes & apply %!preproc -# 3rd scan: BODY: read titles and compose TOC info -# 4th scan: BODY: full parsing, expanding [#anchor] 1st -# Steps 2 and 3 can be made together, with no tag adding. -# Two complete body scans will be *slow*, don't know if it worths. -# One solution may be add the titles as postproc rules - - -import collections -import getopt -import os -import re -import sys - -############################################################################## - -# Program information -my_url = "https://txt2tags.org" -my_name = "txt2tags" -my_email = "jendrikseipp@gmail.com" -__version__ = "3.9" - -# FLAGS : the conversion related flags , may be used in %!options -# OPTIONS : the conversion related options, may be used in %!options -# ACTIONS : the other behavior modifiers, valid on command line only -# NO_TARGET: actions that don't require a target specification -# NO_MULTI_INPUT: actions that don't accept more than one input file -# CONFIG_KEYWORDS: the valid %!key:val keywords -# -# FLAGS and OPTIONS are configs that affect the converted document. -# They usually have also a --no-
", - "blockQuoteOpen": "
", - "blockVerbClose": "", - "blockVerbOpen": "
",
-            "bodyClose": "",
-            "bodyOpen": '
', - "comment": "", - "cssClose": "", - "cssOpen": " +""" + +MATHJAX_FILE = "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-svg.js" +MATHJAX = f"""\ + + +""" + +# A #hashtag must contain at least one letter and must not be a hex colour or +# a C preprocessor directive. This mirrors rednotebook.data.HASHTAG. +HASHTAG_BODY = re.compile( + r"(?![0-9a-fA-F]{6}\b|include\b|define\b|ifdef\b|ifndef\b|endif\b)(\w*[^\W\d_]+\w*)" +) +COLOR = re.compile(r"\{([^{}|]+)\|color:([^{}]+)\}") +IMAGE_WIDTH = re.compile(r"\?(\d+)$") + + +# -------------------------------------------------------------------------- +# Inline plugins +# -------------------------------------------------------------------------- + + +def _hashtag_rule(state, silent): + pos = state.pos + if state.src[pos] not in "##": + return False + if pos > 0: + prev = state.src[pos - 1] + if prev.isalnum() or prev == "_" or prev in "&#": + return False + match = HASHTAG_BODY.match(state.src, pos + 1) + if not match: + return False + if not silent: + token = state.push("hashtag", "", 0) + token.content = state.src[pos : match.end()] + token.meta = {"tag": match.group(1)} + state.pos = match.end() + return True + + +def _color_rule(state, silent): + if state.src[state.pos] != "{": + return False + match = COLOR.match(state.src, state.pos) + if not match: + return False + if not silent: + token = state.push("rn_color", "", 0) + token.meta = {"text": match.group(1), "color": match.group(2)} + state.pos = match.end() + return True + + +def _rednotebook_plugin(md): + md.inline.ruler.before("emphasis", "hashtag", _hashtag_rule) + md.inline.ruler.before("emphasis", "rn_color", _color_rule) + + +# -------------------------------------------------------------------------- +# HTML renderer +# -------------------------------------------------------------------------- + + +class HtmlRenderer(RendererHTML): + def hashtag(self, tokens, idx, options, env): + return f'{escapeHtml(tokens[idx].content)}' + + def rn_color(self, tokens, idx, options, env): + meta = tokens[idx].meta + return f'{escapeHtml(meta["text"])}' + + def image(self, tokens, idx, options, env): + token = tokens[idx] + src = token.attrs.get("src", "") + width = "" + match = IMAGE_WIDTH.search(src) + if match: + width = f' width="{match.group(1)}"' + src = src[: match.start()] + alt = escapeHtml(token.content) + return f'{alt}' + + def math_inline(self, tokens, idx, options, env): + return f"\\({tokens[idx].content}\\)" + + def math_block(self, tokens, idx, options, env): + return f"$$\n{tokens[idx].content}\n$$\n" + + +# -------------------------------------------------------------------------- +# Token-walking renderers for LaTeX and plain text +# -------------------------------------------------------------------------- + +_HEADING_TO_TEX = { + "h1": "section", + "h2": "subsection", + "h3": "subsubsection", + "h4": "paragraph", + "h5": "subparagraph", + "h6": "subparagraph", +} + +_TEX_ESCAPES = { + "\\": r"\textbackslash{}", + "&": r"\&", + "%": r"\%", + "$": r"\$", + "#": r"\#", + "_": r"\_", + "{": r"\{", + "}": r"\}", + "~": r"\textasciitilde{}", + "^": r"\textasciicircum{}", +} +_TEX_ESCAPE_RE = re.compile("|".join(re.escape(key) for key in _TEX_ESCAPES)) + + +def tex_escape(text): + return _TEX_ESCAPE_RE.sub(lambda m: _TEX_ESCAPES[m.group()], text) + + +class _TokenRenderer: + """Walk the markdown-it token stream and dispatch by token type.""" + + def __init__(self, parser=None): + self.parser = parser + + def render(self, tokens, options, env): + out = [] + for token in tokens: + if token.type == "inline": + out.append(self.render(token.children or [], options, env)) + else: + method = getattr(self, token.type, None) + if method is not None: + out.append(method(token, env)) + return "".join(out) + + # Fallbacks for the markup we do not specially handle. + def text(self, token, env): + return token.content + + def softbreak(self, token, env): + return "\n" + + def html_inline(self, token, env): + return "" + + def html_block(self, token, env): + return "" + + +class LatexRenderer(_TokenRenderer): + def text(self, token, env): + return tex_escape(token.content) + + def softbreak(self, token, env): + return "\n" + + def hardbreak(self, token, env): + return "\\\\\n" + + def paragraph_open(self, token, env): + return "" + + def paragraph_close(self, token, env): + # Tight list items wrap their text in hidden paragraphs. + return "" if token.hidden else "\n\n" + + def heading_open(self, token, env): + return "\\" + _HEADING_TO_TEX.get(token.tag, "section") + "{" + + def heading_close(self, token, env): + return "}\n\n" + + def strong_open(self, token, env): + return "\\textbf{" + + def strong_close(self, token, env): + return "}" + + def em_open(self, token, env): + return "\\textit{" + + def em_close(self, token, env): + return "}" + + def s_open(self, token, env): + return "\\sout{" + + def s_close(self, token, env): + return "}" + + def code_inline(self, token, env): + return "\\texttt{" + tex_escape(token.content) + "}" + + def fence(self, token, env): + return "\\begin{verbatim}\n" + token.content + "\\end{verbatim}\n\n" + + code_block = fence + + def link_open(self, token, env): + return "\\href{" + token.attrs.get("href", "") + "}{" + + def link_close(self, token, env): + return "}" + + def image(self, token, env): + src = token.attrs.get("src", "") + match = IMAGE_WIDTH.search(src) + options = "" + if match: + options = f"[width={match.group(1)}px]" + src = src[: match.start()] + return f'\\includegraphics{options}{{"{src}"}}' + + def bullet_list_open(self, token, env): + return "\\begin{itemize}\n" + + def bullet_list_close(self, token, env): + return "\\end{itemize}\n" + + def ordered_list_open(self, token, env): + return "\\begin{enumerate}\n" + + def ordered_list_close(self, token, env): + return "\\end{enumerate}\n" + + def list_item_open(self, token, env): + return "\\item " + + def list_item_close(self, token, env): + return "\n" + + def hr(self, token, env): + return "\\par\\noindent\\rule{\\linewidth}{0.4pt}\n\n" + + def blockquote_open(self, token, env): + return "\\begin{quote}\n" + + def blockquote_close(self, token, env): + return "\\end{quote}\n" + + def hashtag(self, token, env): + display = tex_escape(token.content.lstrip("##")) + index = token.meta["tag"] + return f"\\textcolor{{red}}{{\\#{display}\\index{{{index}}}}}" + + def rn_color(self, token, env): + meta = token.meta + return f"\\textcolor{{{tex_escape(meta['color'])}}}{{{tex_escape(meta['text'])}}}" + + def math_inline(self, token, env): + return f"${token.content}$" + + def math_block(self, token, env): + return f"$${token.content}$$\n" + + # Minimal table support: render cells separated by " & " and rows by "\\". + def tr_close(self, token, env): + return "\\\\\n" + + def td_close(self, token, env): + return " & " + + th_close = td_close + + +class PlainRenderer(_TokenRenderer): + def hardbreak(self, token, env): + return "\n" + + def paragraph_close(self, token, env): + # Tight list items wrap their text in hidden paragraphs. + return "" if token.hidden else "\n\n" + + def heading_close(self, token, env): + return "\n\n" + + def code_inline(self, token, env): + return token.content + + def fence(self, token, env): + return token.content + "\n" + + code_block = fence + + def image(self, token, env): + return f"[{token.attrs.get('src', '')}]" + + def bullet_list_open(self, token, env): + # Start nested lists on their own line. + return "\n" if token.level else "" + + ordered_list_open = bullet_list_open + + def list_item_open(self, token, env): + return "- " + + def list_item_close(self, token, env): + return "\n" + + def hr(self, token, env): + return "\n" + "=" * 20 + "\n\n" + + def hashtag(self, token, env): + return token.content + + def rn_color(self, token, env): + return token.meta["text"] + + def math_inline(self, token, env): + return token.content + + def math_block(self, token, env): + return token.content + "\n" + + +_RENDERERS = {"html": HtmlRenderer, "tex": LatexRenderer, "txt": PlainRenderer} + + +def _get_parser(target): + md = MarkdownIt("commonmark", {"html": True, "linkify": True, "breaks": False}) + md.enable(["table", "strikethrough", "linkify"]) + md.use(dollarmath_plugin, double_inline=True) + md.use(_rednotebook_plugin) + md.renderer = _RENDERERS[target](md) + return md + + +def _walk(tokens): + for token in tokens: + yield token + if token.children: + yield from _walk(token.children) + + +# -------------------------------------------------------------------------- +# Document templates +# -------------------------------------------------------------------------- + +LATEX_PREAMBLE = r"""\documentclass[a4paper]{article} +\usepackage[utf8]{inputenc} +\usepackage[T1]{fontenc} +\usepackage{graphicx} +\usepackage{xcolor} +\usepackage[normalem]{ulem} +\usepackage{hyperref} +\usepackage{makeidx} +\makeindex +\title{%(title)s} +\begin{document} +\maketitle +""" + +LATEX_FOOTER = r""" +\printindex +\end{document} +""" + + +def _html_document(body, options, has_math): + css = CSS % { + "font": options.get("font", "sans-serif"), + "bgcolor": options.get("bgcolor", "white"), + "fgcolor": options.get("fgcolor", "black"), + } + mathjax = MATHJAX if has_math else "" + return ( + "\n\n\n" + '\n' + f"{css}{mathjax}" + "\n\n" + f"{body}" + "\n\n" + ) + + +def _latex_document(body, options): + title = options.get("title", "RedNotebook") + return LATEX_PREAMBLE % {"title": title} + body + LATEX_FOOTER + + +def render(text, target, options=None): + """Render Markdown ``text`` to ``target`` (``html``, ``tex`` or ``txt``).""" + options = options or {} + md = _get_parser(target) + env = {} + tokens = md.parse(text, env) + body = md.renderer.render(tokens, md.options, env) + + if target == "html": + has_math = options.get("add_mathjax") + if has_math is None: + has_math = any(token.type.startswith("math") for token in _walk(tokens)) + return _html_document(body, options, has_math) + # Collapse runs of blank lines that arise between block elements. + body = re.sub(r"\n{3,}", "\n\n", body) + if target == "tex": + return _latex_document(body, options) + return body.strip() + "\n" diff --git a/rednotebook/util/markup.py b/rednotebook/util/markup.py index 794db153..814bb7b5 100644 --- a/rednotebook/util/markup.py +++ b/rednotebook/util/markup.py @@ -19,107 +19,33 @@ import os import re -from rednotebook.data import HASHTAG -from rednotebook.external import txt2tags -from rednotebook.util import filesystem, urls +from rednotebook.util import filesystem, markdownmarkup, t2t_to_markdown, urls -# Linebreaks are only allowed at line ends -REGEX_LINEBREAK = r"\\\\[\s]*$" +# A trailing "text" link, used by pango_markup to strip links. REGEX_HTML_LINK = r"(.*?)" -# pic [""/home/user/Desktop/RedNotebook pic"".png] -PIC_NAME = r"\S.*?\S|\S" -PIC_EXT = r"(?:png|jpe?g|gif|eps|bmp|svg)" -REGEX_PIC = re.compile(rf'(\["")({PIC_NAME})("")(\.{PIC_EXT})(\?\d+)?(\])', flags=re.I) - -# named local link [my file.txt ""file:///home/user/my file.txt""] -# named link in web [heise ""http://heise.de""] -REGEX_NAMED_LINK = re.compile(r'(\[)(.*?)(\s"")(\S.*?\S)(""\])', flags=re.I) - -ESCAPE_COLOR = r"XBEGINCOLORX\1XSEPARATORX\2XENDCOLORX" -COLOR_ESCAPED = r"XBEGINCOLORX(.*?)XSEPARATORX(.*?)XENDCOLORX" - -CSS = """\ - -""" - -# MathJax -FORMULAS_SUPPORTED = True -MATHJAX_FILE = "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-svg.js" - -# Explicitly setting inlineMath: [ ['\\(','\\)'] ] doesn't work. -# Using defaults: -# displayMath: [ ['$$','$$'], ['\[','\]'] ] -# inlineMath: [['\(','\)']] -MATHJAX_DELIMITERS = ["$$", "\\(", "\\)", r"\\[", "\\]"] -MATHJAX = f"""\ - - -""" +# Markdown image/link target: "![alt](url)" or "[text](url)". +REGEX_MD_LINK = re.compile(r"(!?\[[^\]]*\]\()([^)\s]+)(\))") +# Optional image width suffix. +REGEX_IMAGE_WIDTH = re.compile(r"\?(\d+)$") + +# Entry references such as "[2019-08-01]" or "[my day 2019-08-01]". +REGEX_NAMED_REFERENCE = re.compile(r"\[(?P.+?)\s+(?P\d{4}-\d{2}-\d{2})\s*\]") +REGEX_DATE_REFERENCE = re.compile(r"\[(?P\d{4}-\d{2}-\d{2})\]") + +# Math delimiters that MathJax understands besides "$"/"$$". +REGEX_MATH_DISPLAY = re.compile(r"\\\[(.+?)\\\]", flags=re.DOTALL) +REGEX_MATH_INLINE = re.compile(r"\\\((.+?)\\\)", flags=re.DOTALL) def convert_categories_to_markup(categories, with_category_title=True): - # Only add Category title if the text is displayed - markup = "== {} ==\n".format(_("Tags")) if with_category_title else "" + # Only add the "Tags" title if the text is displayed. + markup = "## {}\n".format(_("Tags")) if with_category_title else "" for category, entry_list in categories.items(): - markup += f"- {category}" + "\n" + markup += f"- {category}\n" for entry in entry_list: - markup += f" - {entry}" + "\n" + markup += f" - {entry}\n" markup += "\n\n" return markup @@ -133,11 +59,10 @@ def get_markup_for_day(day, target, with_text=True, with_tags=True, categories=N # Add date if it is not None and not the empty string if date: if target == "html": - # Following anchor will be used as a target for every entry reference mentioning - # this entry's date. - export_string += f"''''\n" + # The anchor is the target for entry references mentioning this date. + export_string += f'\n\n' - export_string += f"= {date} =\n\n" + export_string += f"# {date}\n\n" # Add text if with_text: @@ -174,244 +99,79 @@ def get_markup_for_day(day, target, with_text=True, with_tags=True, categories=N return "" -def _get_config(target, options): - # Set the configuration on the 'config' dict. - config = txt2tags.ConfigMaster()._get_defaults() - - config["outfile"] = txt2tags.MODULEOUT # results as list - config["target"] = target - - # The Pre (and Post) processing config is a list of lists: - # [ [this, that], [foo, bar], [patt, replace] ] - config["postproc"] = [] - config["preproc"] = [] - config["style"] = [] - - # Allow line breaks, r'\\\\' are 2 \ for regexes - config["preproc"].append([REGEX_LINEBREAK, "LINEBREAK"]) - - # Highlight hashtags. - if target == "tex": - config["preproc"].append([HASHTAG.pattern, r"\1{\2\3BEGININDEX\3ENDINDEX|color:red}"]) - else: - config["preproc"].append([HASHTAG.pattern, r"\1{\2\3|color:red}"]) - - # Escape color markup. - config["preproc"].append([r"\{(.*?)\|color:(.+?)\}", ESCAPE_COLOR]) - - if target == "html": - config["encoding"] = "UTF-8" # document encoding - config["toc"] = 0 - config["css-sugar"] = 1 - - # Line breaks - config["postproc"].append([r"LINEBREAK", "
"]) - - # Apply image resizing - config["postproc"].append([r"src=\"WIDTH(\d+)-", r'width="\1" src="']) - - # Flow paragraph from right to left or left to right depending on the language. - config["postproc"].append(["

", '

']) - - # {{red text|color:red}} -> red text - config["postproc"].append([COLOR_ESCAPED, r'\1']) - - # Custom css - font = options.pop("font", "sans-serif") - css = CSS % { - "font": font, - "bgcolor": options.get("bgcolor", "white"), - "fgcolor": options.get("fgcolor", "black"), - } - config["postproc"].append([r"", f"{css}"]) - - # MathJax - if options.pop("add_mathjax"): - config["postproc"].append([r"", f"{MATHJAX}"]) - - elif target == "tex": - config["encoding"] = "utf8" - config["preproc"].append(["€", "Euro"]) - - # Latex only allows whitespace and underscores in filenames if - # the filename is surrounded by "...". This is in turn only possible - # if the extension is omitted. - config["preproc"].append([r'\[""', r'["""']) - config["preproc"].append([r'""\.', r'""".']) - - scheme = filesystem.LOCAL_FILE_PEFIX - - # For images we have to omit the file:// prefix - config["postproc"].append([rf'includegraphics\{{(.*)"{scheme}', r'includegraphics{"\1']) - - # Special handling for LOCAL file links (Omit scheme, add run:) - # \htmladdnormallink{file.txt}{file:///home/user/file.txt} - # --> - # \htmladdnormallink{file.txt}{run:/home/user/file.txt} - config["postproc"].append( - [ - rf"htmladdnormallink\{{(.*)\}}\{{{scheme}(.*)\}}", - r"htmladdnormallink{\1}{run:\2}", - ] - ) - - # Line breaks - config["postproc"].append([r"LINEBREAK", r"\\\\"]) - - # Apply image resizing - config["postproc"].append( - [r'includegraphics\{("?)WIDTH(\d+)-', r"includegraphics[width=\2px]{\1"] - ) +def _convert_uri(uri, data_dir): + path = uri[len("file://") :] if uri.startswith("file://") else uri + # Check if relative file exists and convert it if it does. + if not any( + uri.startswith(proto) for proto in filesystem.REMOTE_PROTOCOLS + ) and not os.path.isabs(path): + path = os.path.join(data_dir, path) + assert os.path.isabs(path), path + if os.path.exists(path): + uri = urls.get_local_url(path) + return uri - # We want the plain latex formulas unescaped. - # Allowed formulas: $$...$$, \[...\], \(...\) - config["preproc"].append([r"\\\[\s*(.+?)\s*\\\]", r"BEGINEQUATION''\1''ENDEQUATION"]) - config["preproc"].append([r"\$\$\s*(.+?)\s*\$\$", r"BEGINEQUATION''\1''ENDEQUATION"]) - config["postproc"].append([r"BEGINEQUATION(.+)ENDEQUATION", r"$$\1$$"]) - config["preproc"].append([r"\\\(\s*(.+?)\s*\\\)", r"BEGINMATH''\1''ENDMATH"]) - config["postproc"].append([r"BEGINMATH(.+)ENDMATH", r"$\1$"]) - - # Fix utf8 quotations - „, “ and ” cause problems compiling the latex document. - config["postproc"].extend([["„", '"'], ["”", '"'], ["“", '"']]) - - # Enable index. - config["style"].append("makeidx") - config["postproc"].append([r"BEGININDEX(.+?)ENDINDEX", r"\\index{\1}"]) - config["postproc"].append(["begin{document}", "makeindex\n\\\\begin{document}"]) - config["postproc"].append(["end{document}", "printindex\n\n\\\\end{document}"]) +def _convert_paths(txt, data_dir): + """Turn relative paths in Markdown links and images into absolute URLs.""" + data_dir = str(data_dir) - config["postproc"].append([COLOR_ESCAPED, r"\\textcolor{\2}{\1}"]) + def repl(match): + prefix, url, suffix = match.groups() + width = "" + is_image = prefix.startswith("!") + width_match = REGEX_IMAGE_WIDTH.search(url) + if is_image and width_match: + width = width_match.group(0) + url = url[: width_match.start()] + # Leave fragment-only references (entry references) untouched. + if url.startswith("#"): + return match.group(0) + return prefix + _convert_uri(url, data_dir) + width + suffix - elif target == "txt": - # Line breaks - config["postproc"].append([r"LINEBREAK", "\n"]) + return REGEX_MD_LINK.sub(repl, txt) - # Apply image resizing ([WIDTH400-file:///pathtoimage.jpg]) - config["postproc"].append([r"\[WIDTH(\d+)-(.+)\]", r"[\2?\1]"]) - # Entry references +def _convert_entry_references(txt, target): + """Turn date references into links (HTML) or plain text (other targets).""" if target == "html": - # txt2tags will generate links to the named entry references because they share - # common bracket notation used by the URIs. We just need to add our internal - # schema to make it a proper URI. - config["preproc"].append( - [ - r"\[(?P.+)\s+(?P\d{4}-\d{2}-\d{2})\s*\]", - r"[\g #\g]", - ] - ) - - # Convert bracketed dates into named references where the date itself is being - # used as a name. For example: - # "Today is [2019-10-20]" will be converted into "Today is [2019-10-20 #2019-10-20]" - config["preproc"].append([r"\[(?P\d{4}-\d{2}-\d{2})\]", r"[\g #\g]"]) + txt = REGEX_NAMED_REFERENCE.sub(r"[\g](#\g)", txt) + txt = REGEX_DATE_REFERENCE.sub(r"[\g](#\g)", txt) else: - # Links to entry references are not supported for targets other than HTML - config["preproc"].append( - [r"\[(?P.+)\s+(?P\d{4}-\d{2}-\d{2})\]", r"\g (\g)"] - ) - - # Allow resizing images by changing - # [filename.png?width] to [WIDTHwidth-filename.png] - img_ext = r"png|jpe?g|gif|eps|bmp|svg" - img_name = r"\S.*\S|\S" - - # Apply this prepoc only after the latex image quotes have been added - config["preproc"].append([rf"\[({img_name}\.({img_ext}))\?(\d+)\]", r"[WIDTH\3-\1]"]) - - # Disable colors for all other targets. - config["postproc"].append([COLOR_ESCAPED, r"\1"]) - - config.update(options) - - return config + txt = REGEX_NAMED_REFERENCE.sub(r"\g (\g)", txt) + txt = REGEX_DATE_REFERENCE.sub(r"\g", txt) + return txt -def _convert_paths(txt, data_dir): - data_dir = str(data_dir) - - def _convert_uri(uri): - path = uri[len("file://") :] if uri.startswith("file://") else uri - # Check if relative file exists and convert it if it does. - if not any( - uri.startswith(proto) for proto in filesystem.REMOTE_PROTOCOLS - ) and not os.path.isabs(path): - path = os.path.join(data_dir, path) - assert os.path.isabs(path), path - if os.path.exists(path): - uri = urls.get_local_url(path) - return uri - - def _convert_pic_path(match): - uri = _convert_uri(match.group(2) + match.group(4)) - # Reassemble picture markup. - name, ext = os.path.splitext(uri) - parts = [match.group(1), name, match.group(3), ext] - if match.group(5) is not None: - parts.append(match.group(5)) - parts.append(match.group(6)) - return "".join(parts) - - def _convert_file_path(match): - uri = _convert_uri(match.group(4)) - # Reassemble link markup - parts = [match.group(i) for i in range(1, 6)] - parts[3] = uri - return "".join(parts) - - txt = REGEX_PIC.sub(_convert_pic_path, txt) - txt = REGEX_NAMED_LINK.sub(_convert_file_path, txt) +def _normalize_math(txt): + """Rewrite "\\(...\\)" and "\\[...\\]" to the "$" delimiters MathJax uses.""" + txt = REGEX_MATH_DISPLAY.sub(r"$$\1$$", txt) + txt = REGEX_MATH_INLINE.sub(r"$\1$", txt) return txt -def convert(txt, target, data_dir, headers=None, options=None): - """ - Code partly taken from txt2tags tarball - """ +def convert(txt, target, data_dir, options=None): + """Convert journal text (Markdown, with txt2tags fallback) to ``target``.""" data_dir = str(data_dir) options = options or {} - # Only add MathJax code if there is a formula. - options["add_mathjax"] = ( - FORMULAS_SUPPORTED and "html" in target and any(x in txt for x in MATHJAX_DELIMITERS) - ) - logging.debug(f"Add mathjax code: {options['add_mathjax']}") + # Translate any legacy txt2tags markup to Markdown first. + txt = t2t_to_markdown.convert_to_markdown(txt) # Turn relative paths into absolute paths. txt = _convert_paths(txt, data_dir) - # The body text must be a list. - txt = txt.split("\n") - - # Set the three header fields - if headers is None: - # LaTeX requires a title if \maketitle is used. - headers = ["RedNotebook", "", ""] if target == "tex" else ["", "", ""] - config = _get_config(target, options) + # Handle RedNotebook-specific constructs. + txt = _convert_entry_references(txt, target) + txt = _normalize_math(txt) - # Let's do the conversion try: - headers = txt2tags.doHeader(headers, config) - body, toc = txt2tags.convert(txt, config) - footer = txt2tags.doFooter(config) - toc = txt2tags.toc_tagger(toc, config) - toc = txt2tags.toc_formatter(toc, config) - full_doc = headers + toc + body + footer - finished = txt2tags.finish_him(full_doc, config) - result = "\n".join(finished) - # Txt2tags error, show the message to the user - except txt2tags.error as msg: - logging.error(msg) - result = msg - # Unknown error, show the traceback to the user + return markdownmarkup.render(txt, target, options) except Exception: - result = ( - "Error: This day contains invalid " - 'txt2tags markup. ' - "You can help us fix this by submitting a bugreport in the " - '' - "txt2tags bugtracker. Please append the day's text to the issue." + logging.exception("Markdown conversion failed") + return ( + "Error: This day contains markup that RedNotebook could not " + "convert. Please report this at " + '' + "the RedNotebook bugtracker and append the day's text to the issue." ) - logging.error(f"Invalid markup:\n{txt2tags.getUnknownErrorMessage()}") - return result diff --git a/rednotebook/util/pango_markup.py b/rednotebook/util/pango_markup.py index 83b58288..0ccd6e1b 100644 --- a/rednotebook/util/pango_markup.py +++ b/rednotebook/util/pango_markup.py @@ -1,118 +1,96 @@ +# ----------------------------------------------------------------------- +# Copyright (c) 2008-2024 Jendrik Seipp +# +# RedNotebook is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# RedNotebook is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# ----------------------------------------------------------------------- + import logging import re import gi +from markdown_it import MarkdownIt gi.require_version("Pango", "1.0") -from gi.repository import GObject, Pango - -from rednotebook.external import txt2tags -from rednotebook.util.markup import REGEX_HTML_LINK, REGEX_LINEBREAK - - -def convert_to_pango(txt, headers=None, options=None): - """ - Code partly taken from txt2tags tarball - """ - original_txt = txt - - # Here is the marked body text, it must be a list. - txt = txt.split("\n") - - # Set the three header fields - if headers is None: - headers = ["", "", ""] +from gi.repository import GObject, Pango # noqa: E402 - config = txt2tags.ConfigMaster()._get_defaults() +from rednotebook.util import t2t_to_markdown # noqa: E402 +from rednotebook.util.markup import REGEX_HTML_LINK # noqa: E402 - config["outfile"] = txt2tags.MODULEOUT # results as list - config["target"] = "html" - config["preproc"] = [] - # We need to escape the ampersand here, otherwise "&" would become - # "&" - config["preproc"].append([r"&", "&"]) +# Categories are short, single-line strings, so inline rendering is enough. +_PARSER = MarkdownIt("commonmark").enable("strikethrough") - # Allow line breaks - config["postproc"] = [] - config["postproc"].append([REGEX_LINEBREAK, "\n"]) +# Map the (limited) set of HTML tags that markdown-it emits to the Pango tags +# the category tree understands. +_HTML_TO_PANGO = [ + ("", ""), + ("", ""), + ("", ""), + ("", ""), + ("", ""), + ("", ""), + ("", ""), + ("", ""), +] - if options is not None: - config.update(options) - # Let's do the conversion - try: - body, toc = txt2tags.convert(txt, config) - full_doc = body - finished = txt2tags.finish_him(full_doc, config) - result = "".join(finished) - - # Txt2tags error, show the message to the user - except txt2tags.error as msg: - logging.error(msg) - result = msg - - # Unknown error, show the traceback to the user - except Exception: - result = txt2tags.getUnknownErrorMessage() - logging.error(result) - - print(result) - - # remove unwanted paragraphs - result = result.replace('

', "").replace("

", "") +def convert_to_pango(txt): + """Convert (Markdown) category markup to Pango markup for display.""" + original_txt = txt - logging.log( - 5, - f'Converted "{repr(original_txt)}" text to "{repr(result)}" txt2tags markup', - ) + txt = t2t_to_markdown.convert_to_markdown(txt) + result = _PARSER.renderInline(txt) - # Remove unknown tags () - def replace_links(match): - """Return the link name.""" - return match.group(1) + for html_tag, pango_tag in _HTML_TO_PANGO: + result = result.replace(html_tag, pango_tag) - result = re.sub(REGEX_HTML_LINK, replace_links, result) - print(result) + # Pango has no anchor element, so reduce links to their text. + result = re.sub(REGEX_HTML_LINK, r"\1", result) - for new_tag, old_tag in [("del", "s"), ("em", "i"), ("strong", "b")]: - result = result.replace(f"<{new_tag}>", f"<{old_tag}>") - result = result.replace(f"", f"") - print(result) + logging.log(5, f'Converted "{original_txt!r}" to Pango "{result!r}"') try: Pango.parse_markup(result, -1, "0") - # result is valid pango markup, return the markup. - return result except GObject.GError: - # There are unknown tags in the markup, return the original text + # There are unknown tags in the markup, return the original text. logging.debug(f"There are unknown tags in the markup: {result}") return original_txt + return result def convert_from_pango(pango_markup): + """Convert Pango markup back to the Markdown stored in the journal.""" original_txt = pango_markup replacements = { "": "**", "": "**", - "": "//", - "": "//", - "": "--", - "": "--", - "": "__", - "": "__", + "": "*", + "": "*", + "": "~~", + "": "~~", + "": "", + "": "", + "": "`", + "": "`", "&": "&", "<": "<", ">": ">", - "\n": r"\\", } for orig, repl in replacements.items(): pango_markup = pango_markup.replace(orig, repl) - logging.log( - 5, - f'Converted "{repr(original_txt)}" pango to "{repr(pango_markup)}" txt2tags', - ) + logging.log(5, f'Converted Pango "{original_txt!r}" to Markdown "{pango_markup!r}"') return pango_markup diff --git a/rednotebook/util/t2t_to_markdown.py b/rednotebook/util/t2t_to_markdown.py new file mode 100644 index 00000000..3efc38e1 --- /dev/null +++ b/rednotebook/util/t2t_to_markdown.py @@ -0,0 +1,117 @@ +# ----------------------------------------------------------------------- +# Copyright (c) 2008-2024 Jendrik Seipp +# +# RedNotebook is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# RedNotebook is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# ----------------------------------------------------------------------- + +"""Translate legacy txt2tags markup to its Markdown equivalent. + +RedNotebook used txt2tags markup for many years. To keep old journals +readable after the switch to Markdown, every entry is run through this +converter before it reaches the Markdown parser. Constructs that Markdown +shares with txt2tags (``**bold**``, ``- lists``, fenced ```` ``` ```` code +blocks, ...) are left untouched, so text that is already Markdown passes +through essentially unchanged. +""" + +import re + + +IMG_EXT = r"png|jpe?g|gif|eps|bmp|svg" + +# Inline conversions applied to a single line of text (outside code fences). +# Order matters: links/images are handled before the emphasis markers so that +# their contents are not mangled. + +# [alt ""/path/to/pic"".png?50] and [""/path/to/pic"".png?50] +REGEX_IMAGE = re.compile(rf'\[(?:(.*?) )?""(\S.*?\S|\S)""\.({IMG_EXT})(\?\d+)?\]', flags=re.I) +# [text ""url""] +REGEX_QUOTED_LINK = re.compile(r'\[(.*?)\s""(\S.*?\S)""\]') +# [text http://url] (only genuine URLs, to avoid swallowing entry references) +REGEX_NAMED_LINK = re.compile( + r"\[([^\]]+?)\s+((?:https?|ftp|news|telnet|gopher|wais)://|www[23]?\.|ftp\.)([^\]]+)\]" +) + +# Emphasis. Boundaries follow txt2tags: markers hug non-space characters. +REGEX_ITALIC = re.compile(r"(?\1", line) + line = REGEX_STRIKE.sub(r"~~\1~~", line) + line = REGEX_LINEBREAK.sub(" ", line) + return line + + +def _image_repl(match): + alt = match.group(1) or "" + width = match.group(4) or "" + return f"![{alt}]({match.group(2)}.{match.group(3)}{width})" + + +def _convert_block(line): + """Convert line-level constructs that have no Markdown counterpart.""" + if REGEX_HRULE.match(line): + return "---" + + heading = REGEX_HEADING.match(line) + if heading: + level = len(heading.group(1)) + return "#" * level + " " + heading.group(2) + + numbered = REGEX_NUMBERED.match(line) + if numbered: + return f"{numbered.group(1)}1. {numbered.group(2)}" + + return None + + +def convert_to_markdown(text): + lines = text.split("\n") + result = [] + in_fence = False + for line in lines: + if FENCE.match(line): + in_fence = not in_fence + result.append(line) + continue + if in_fence: + result.append(line) + continue + + block = _convert_block(line) + if block is not None: + result.append(block) + else: + result.append(_convert_inline(line)) + return "\n".join(result) diff --git a/setup.py b/setup.py index 813ad817..5b9486c0 100644 --- a/setup.py +++ b/setup.py @@ -87,7 +87,13 @@ def run(self): "license": "GPL", "keywords": "journal, diary", "cmdclass": {"build_py": build_py, "install": install}, - "install_requires": ["PyGObject", "PyYAML"], + "install_requires": [ + "PyGObject", + "PyYAML", + "markdown-it-py", + "mdit-py-plugins", + "linkify-it-py", + ], "extras_require": {"spellcheck": ["pyenchant"]}, "entry_points": { "gui_scripts": [ diff --git a/tests/test_highlighting.py b/tests/test_highlighting.py new file mode 100644 index 00000000..574adcea --- /dev/null +++ b/tests/test_highlighting.py @@ -0,0 +1,116 @@ +"""Verify that the editor's syntax-highlighting patterns match both Markdown +and legacy txt2tags constructs. + +GtkSourceView applies these regexes (in GRegex/PCRE syntax) line by line. We +cannot easily exercise the highlighter headlessly, but we can read the patterns +straight from ``markdown.lang`` and check that they match the constructs they +are meant to highlight (and reject look-alikes). This guards against the +patterns silently breaking. +""" + +import re +import xml.etree.ElementTree as ET +from pathlib import Path + +import pytest + + +LANG_FILE = Path(__file__).parent.parent / "rednotebook" / "files" / "markdown.lang" + + +def _patterns(): + """Map each context id to the regex string of its direct element. + + Patterns are kept as strings (not compiled) because some Markdown contexts + use variable-width look-behinds that GRegex accepts but Python's ``re`` + rejects. Only the patterns we actually test are compiled, on demand. + """ + tree = ET.parse(LANG_FILE) + patterns = {} + for context in tree.iter("context"): + cid = context.get("id") + match = context.find("match") + if cid and match is not None and match.text: + # ElementTree already decodes < etc. for us. + flags = re.VERBOSE if match.get("extended") == "true" else 0 + patterns[cid] = (match.text.strip(), flags) + return patterns + + +PATTERNS = _patterns() + + +def _search(context_id, text): + pattern, flags = PATTERNS[context_id] + return re.search(pattern, text, flags) + + +@pytest.mark.parametrize( + "context_id,text", + [ + # Markdown. + ("atx-header", "# Heading"), + ("atx-header", "### Smaller heading"), + ("asterisks-strong-emphasis", "a **bold** b"), + ("asterisks-emphasis", "a *italic* b"), + ("underscores-strong-emphasis", "a __bold__ b"), + ("strikethrough", "a ~~gone~~ b"), + ("inline-link", "see [text](http://x.com)"), + ("inline-image", "![alt](pic.png)"), + ("list", "- item"), + ("list", "1. item"), + # RedNotebook constructs. + ("hashtag", "a #work tag"), + ("color", "{important|color:red}"), + ("entry-reference", "[2019-02-14]"), + ("named-entry-reference", "[my day 2019-02-14]"), + # txt2tags constructs. + ("t2t-italic", "a //italic// b"), + ("t2t-strikethrough", "a --gone-- b"), + ("t2t-heading", "= Title ="), + ("t2t-heading", "=== Sub ==="), + ("t2t-heading", "== Clouds ==[anchor]"), + ("t2t-image", "[foo.png]"), + ("t2t-image-quoted", '[""/path to/foo"".png?50]'), + ("t2t-link", "[heise http://heise.de]"), + ("t2t-link-quoted", '[my file ""file:///home/me/f.txt""]'), + ], +) +def test_pattern_matches(context_id, text): + assert _search(context_id, text), f"{context_id} should match {text!r}" + + +@pytest.mark.parametrize( + "context_id,text", + [ + # A bare URL must not be mistaken for //italic//. + ("t2t-italic", "visit http://example.com today"), + # A horizontal rule must not be mistaken for --strikethrough--. + ("t2t-strikethrough", "--------------------"), + # A plain hashtag-less number is not a hashtag. + ("hashtag", "issue 1234 done"), + # A Markdown heading is not a txt2tags heading. + ("t2t-heading", "# Markdown heading"), + ], +) +def test_pattern_rejects(context_id, text): + assert not _search(context_id, text), f"{context_id} should not match {text!r}" + + +def test_expected_contexts_present(): + # Both Markdown and txt2tags contexts must exist. + for cid in [ + "atx-header", + "strikethrough", + "hashtag", + "color", + "entry-reference", + "t2t-italic", + "t2t-strikethrough", + "t2t-heading", + "t2t-link", + "t2t-link-quoted", + "t2t-image", + "t2t-image-quoted", + ]: + assert cid in PATTERNS diff --git a/tests/test_markdown_render.py b/tests/test_markdown_render.py new file mode 100644 index 00000000..a27538b1 --- /dev/null +++ b/tests/test_markdown_render.py @@ -0,0 +1,143 @@ +"""Tests for the Markdown rendering pipeline (HTML, LaTeX and plain text).""" + +import pytest + +from rednotebook.util.markdownmarkup import render + + +def html(markup, **options): + return render(markup, "html", options) + + +def tex(markup, **options): + return render(markup, "tex", options) + + +def txt(markup, **options): + return render(markup, "txt", options) + + +class TestHtmlBasics: + def test_document_skeleton(self): + doc = html("Content") + assert "" in doc + assert '' in doc + assert "

Content

" in doc + + def test_emphasis(self): + doc = html("**b** and *i* and ~~s~~ and `c`") + assert "b" in doc + assert "i" in doc + assert "s" in doc + assert "c" in doc + + def test_heading(self): + assert "

Title

" in html("## Title") + + def test_bullet_list(self): + doc = html("- a\n- b") + assert "
    " in doc and "
  • a
  • " in doc + + def test_link(self): + assert '
    name' in html("[name](http://x.com)") + + def test_autolink_bare_url(self): + assert 'http://x.com' in html("see http://x.com") + + +class TestHtmlRedNotebookFeatures: + @pytest.mark.parametrize( + "markup,expected", + [ + ("#TAG", '#TAG'), + ("Numeric #3tag", '#3tag'), + ("Just #34 numbers", "#34"), + ], + ) + def test_hashtags(self, markup, expected): + assert expected in html(markup) + + def test_hashtag_not_in_code(self): + assert '' not in html("`#TAG`") + + @pytest.mark.parametrize( + "markup,expected", + [ + ("{Sky|color:blue}", 'Sky'), + ("{Red|color:#FF0000} truck", 'Red truck'), + ], + ) + def test_colors(self, markup, expected): + assert expected in html(markup) + + def test_image_width(self): + doc = html("![](/image.png?50)") + assert 'src="/image.png"' in doc + assert 'width="50"' in doc + + def test_image_no_width(self): + doc = html("![](/image.jpg)") + assert 'src="/image.jpg"' in doc + assert "width=" not in doc + + @pytest.mark.parametrize( + "markup,expected", + [ + ("[named ref](#2019-08-01)", 'named ref'), + ], + ) + def test_entry_reference_links(self, markup, expected): + assert expected in html(markup) + + def test_mathjax_added_when_formula_present(self): + doc = html("$$x^3$$") + assert "MathJax" in doc + + def test_mathjax_absent_without_formula(self): + assert "MathJax" not in html("no math here") + + def test_linebreak(self): + assert "stricken"), - (r"//italic//", "italic"), - (r"--www.test.com--", "www.test.com"), - # Linebreaks only on line ends - (r"First\\Second", r"First\\Second"), - (r"First\\", "First\n"), - (r"a&b", "a&b"), - (r"a&b", "a&b"), - (r"http://site/s.php?q&c", "http://site/s.php?q&c"), - (r"http://site/s.php?q&c", "http://site/s.php?q&c"), + ("**bold**", "bold"), + ("*italic*", "italic"), + ("~~struck~~", "struck"), + ("`code`", "code"), + ("underlined", "underlined"), ], ) -def test_pango(t2t_markup, expected): - pango = convert_to_pango(t2t_markup) - assert pango == expected - # Ampersand escaping is only needed in sourcecode, so we do not try to - # preserve the encoding - if "&" not in t2t_markup: - assert convert_from_pango(pango) == t2t_markup +def test_pango(markdown, pango): + assert convert_to_pango(markdown) == pango + assert convert_from_pango(pango) == markdown + + +def test_pango_strips_links(): + assert convert_to_pango("[name](http://x.com)") == "name" def test_relative_path_conversion(tmp_path): - for path in [tmp_path / f for f in ("rel.jpg", "rel.pdf")]: - path.write_text("") # Create empty file. - tmp_path_uri = filesystem.LOCAL_FILE_PEFIX + str(tmp_path) + os.sep + "rel" + for name in ("rel.jpg", "rel.pdf", "rel.png"): + (tmp_path / name).write_text("") # Create empty file. + + def url(name): + return urls.get_local_url(str(tmp_path / name)) rel_paths = [ - ('[""file://rel"".jpg]', f'[""{tmp_path_uri}"".jpg]'), - ('[""rel"".jpg]', f'[""{tmp_path_uri}"".jpg]'), - ( - '[rel.pdf ""file://rel.pdf""]', - f'[rel.pdf ""{tmp_path_uri}.pdf""]', - ), - ('[rel.pdf ""rel.pdf""]', f'[rel.pdf ""{tmp_path_uri}.pdf""]'), + ("![](rel.jpg)", f"![]({url('rel.jpg')})"), + ("[doc](rel.pdf)", f"[doc]({url('rel.pdf')})"), + ("![](rel.png?50)", f"![]({url('rel.png')}?50)"), ] - for markup, expected in rel_paths: - assert expected == _convert_paths(markup, tmp_path) + assert _convert_paths(markup, tmp_path) == expected def test_absolute_path_conversion(tmp_path): abs_paths = [ - '[""file:///abs"".jpg]', - f'[""{tmp_path}/aha 1"".jpg]', - '[abs.pdf ""file:///abs.pdf""]', - f'[abs.pdf ""{tmp_path}/abs.pdf""]', - "www.google.com", - "www.google.com/page.php", + "![](file:///abs.jpg)", + f"![]({tmp_path}/aha.jpg)", + "[doc](file:///abs.pdf)", + "[site](http://www.google.com)", ] - for path in abs_paths: assert path == _convert_paths(path, tmp_path) -class TestGetHtmlExportConfig: +def test_entry_reference_fragment_untouched(tmp_path): + assert _convert_paths("[2019-08-01](#2019-08-01)", tmp_path) == "[2019-08-01](#2019-08-01)" + + +class TestHtml: @staticmethod @pytest.fixture def process(tmp_path): def process(markup): - html_document = convert(markup, "html", tmp_path) - return html_document.split("\n") + return convert(markup, "html", tmp_path) return process def test_encoding(self, process): - document = process("Content") - assert '' in document - - def test_toc(self, process): - document = process("Content") - assert '
    ' not in document + assert '' in process("Content") - def test_css_sugar(self, process): - document = process("Content") - assert '
    ' in document + def test_legacy_txt2tags_input(self, process): + document = process("//italic// and --struck--") + assert "italic" in document + assert "struck" in document @pytest.mark.parametrize( "markup,expected", [ - ("content \\\\ ", "content
    "), - ("content\\\\", "content
    "), - ("content\\\\ ", "content
    "), - ], - ) - def test_line_break_escaping(self, markup, expected, process): - document = process(markup) - assert expected in document - - @pytest.mark.parametrize( - "markup,expected", - [ - ("#TAG", r'#TAG'), - ("Numeric #3tag", r'Numeric #3tag'), - ("Just #34 numbers", r"Just #34 numbers"), + ("#TAG", '#TAG'), + ("Numeric #3tag", '#3tag'), + ("Just #34 numbers", "#34"), ], ) def test_hashtags(self, markup, expected, process): - document = process(markup) - assert expected in document + assert expected in process(markup) @pytest.mark.parametrize( "markup,expected", [ - ("{Sky|color:blue}", r'Sky'), - ( - "{Red|color:#FF0000} firetruck", - r'Red firetruck', - ), + ("{Sky|color:blue}", 'Sky'), + ("{Red|color:#FF0000} truck", 'Red truck'), ], ) def test_colors(self, markup, expected, process): - document = process(markup) - assert expected in document + assert expected in process(markup) - @pytest.mark.parametrize( - "markup,expected", - [ - ( - '[""/image"".png?50]', - '', - ), - ( - '[""/image"".jpg?50]', - '', - ), - ( - '[""/image"".jpeg?50]', - '', - ), - ( - '[""/image"".gif?50]', - '', - ), - ( - '[""/image"".eps?50]', - '', - ), - ( - '[""/image"".bmp?50]', - '', - ), - ( - '[""/image"".svg?50]', - '', - ), - ], - ) - def test_images_resize_allowed_extensions(self, markup, expected, process): - document = process(markup) - assert expected in document + def test_image_resize(self, process): + document = process("![](/image.png?50)") + assert 'src="/image.png"' in document + assert 'width="50"' in document @pytest.mark.parametrize( "markup,expected", [ - ( - '[""/image"".png?50]', - '', - ), - ( - '[""/image"".jpg]', - '', - ), - ( - '[""file:///image"".png?10]', - '', - ), - ( - '[""file:///image"".jpg]', - '', - ), - ], - ) - def test_images_width_resize(self, markup, expected, process): - document = process(markup) - assert expected in document - - @pytest.mark.parametrize( - "markup,expected", - [ - ( - "Simple [named reference 2019-08-01]", - 'Simple named reference', - ), - ( - "An inline [2019-08-01] date", - 'An inline 2019-08-01 date', - ), - ("[2019-10-20] is first", '2019-10-20 is first'), + ("Simple [named reference 2019-08-01]", 'named reference'), + ("An inline [2019-08-01] date", '2019-08-01'), ], ) def test_entry_reference_links(self, markup, expected, process): - document = process(markup) - assert expected in document + assert expected in process(markup) def test_day_fragment_anchor_element(self, process): date = datetime.date(2019, 10, 21) day = Day(Month(date.year, date.month), date.day) - markup = get_markup_for_day(day, "html", date=date.strftime("%d-%m-%Y")) - document = process(markup) + assert f'' in process(markup) - assert rf'' in document + def test_mathjax_added(self, process): + assert "MathJax" in process("$$x^3$$") - def test_mathjax(self, process): - document = process("$$x^3$$") - assert r"" in document + def test_mathjax_absent(self, process): + assert "MathJax" not in process("no formula") -class TestGetTexExportConfig: +class TestLatex: @staticmethod @pytest.fixture def process(tmp_path): def process(markup): - html_document = convert(markup, "tex", tmp_path) - return html_document.split("\n") + return convert(markup, "tex", tmp_path) return process - def test_encoding(self, process): + def test_preamble(self, process): document = process("Content") - assert r"\usepackage[utf8]{inputenc} % char encoding" in document - - def test_euro_replacement(self, process): - document = process("€") - assert "Euro" in document - - @pytest.mark.parametrize( - "markup,expected", - [(r'[""file/path"".png]', r'\includegraphics{"file/path".png}')], - ) - def test_path_escape(self, markup, expected, process): - document = process(markup) - assert expected in document - - @pytest.mark.skipif(sys.platform != "win32", reason="Windows-specific") - def test_image_scheme_fix_win32(self, process): - document = process('[""file:///image"".png]') - assert r'\includegraphics{"image".png}' in document - - @pytest.mark.skipif(sys.platform == "win32", reason="POSIX-specific") - def test_image_scheme_fix_posix(self, process): - document = process('[""file://image"".png]') - assert r'\includegraphics{"image".png}' in document - - @pytest.mark.skipif(sys.platform != "win32", reason="Windows-specific") - def test_file_scheme_fix_win32(self, process): - document = process("[file.txt file:///path/to/text/file.txt]") - assert r"\htmladdnormallink{file.txt}{run:path/to/text/file.txt}" in document - - @pytest.mark.skipif(sys.platform == "win32", reason="POSIX-specific") - def test_file_scheme_fix_posix(self, process): - document = process("[file.txt file://path/to/text/file.txt]") - assert r"\htmladdnormallink{file.txt}{run:path/to/text/file.txt}" in document + assert r"\documentclass" in document + assert r"\begin{document}" in document - @pytest.mark.parametrize( - "markup,expected", - [ - ("content \\\\ ", "content \\\\"), - ("content\\\\", "content\\\\"), - ("content\\\\ ", "content\\\\"), - ], - ) - def test_line_break_escaping(self, markup, expected, process): - document = process(markup) - assert expected in document + def test_legacy_heading(self, process): + assert r"\section{Title}" in process("= Title =") @pytest.mark.parametrize( "markup,expected", [ ("#TAG", r"\textcolor{red}{\#TAG\index{TAG}}"), - ("Numeric #3tag", r"Numeric \textcolor{red}{\#3tag\index{3tag}}"), - ("Just #34 numbers", r"Just \#34 numbers"), - ("#include ", r"\#include $<$iostream$>$"), - # TODO: ('#define FOO BAR', r'\#define FOO BAR'), - ("Blue: #0000FF", r"Blue: \#0000FF"), + ("Numeric #3tag", r"\textcolor{red}{\#3tag\index{3tag}}"), ], ) def test_hashtags(self, markup, expected, process): - document = process(markup) - assert expected in document - - @pytest.mark.parametrize( - "markup,expected", - [ - ('[""image"".png?50]', '\\includegraphics[width=50px]{"image".png}'), - ('[""image"".jpg]', '\\includegraphics{"image".jpg}'), - ], - ) - def test_images_width_resize(self, markup, expected, process): - document = process(markup) - assert expected in document - - @pytest.mark.parametrize( - "markup,expected", - [(r"\[f(x) = x^2\]", "$$f(x) = x^2$$"), (r"$$f(x) = x^2$$", "$$f(x) = x^2$$")], - ) - def test_latex_equation_escape_display_mode(self, markup, expected, process): - document = process(markup) - assert expected in document - - @pytest.mark.parametrize("markup,expected", [(r"\(f(x) = x^2\)", "$f(x) = x^2$")]) - def test_latex_equation_escape_inline_mode(self, markup, expected, process): - document = process(markup) - assert expected in document + assert expected in process(markup) - @pytest.mark.parametrize( - "markup,expected", [(r"„content”", '"content"'), (r"”content“", '"content"')] - ) - def test_quotation_mark_replacement(self, markup, expected, process): - document = process(markup) - assert expected in document + def test_color(self, process): + assert r"\textcolor{blue}{Sky}" in process("{Sky|color:blue}") - @pytest.mark.parametrize( - "markup,expected", - [ - ("{Sky|color:blue}", r"\textcolor{blue}{Sky}"), - ("{Red|color:#FF0000} firetruck", r"\textcolor{\#FF0000}{Red} firetruck"), - ], - ) - def test_colors(self, markup, expected, process): - document = process(markup) - assert expected in document - - def test_index_generation(self, process): - document = process("content") - assert r"\usepackage{makeidx} % user defined" in document - assert r"\makeindex" in document - assert r"\printindex" in document - - def test_tags_are_collected_for_index_generation(self, process): - document = process("#tag") - assert r"\index{tag}" in "".join(document) - - @pytest.mark.parametrize( - "markup,expected", - [ - ( - "This is a [named reference 2019-08-01]", - "This is a named reference (2019-08-01)", - ), - ( - "Today is 2019-08-01 - a wonderful day", - "Today is 2019-08-01 - a wonderful day", - ), - ], - ) - def test_entry_reference_links(self, markup, expected, process): - document = process(markup) - assert expected in document + def test_entry_reference(self, process): + assert "named reference (2019-08-01)" in process("A [named reference 2019-08-01]") -class TestGetPlainTextExportConfig: +class TestPlainText: @staticmethod @pytest.fixture def process(tmp_path): def process(markup): - html_document = convert(markup, "txt", tmp_path) - return html_document.split("\n") + return convert(markup, "txt", tmp_path) return process - @pytest.mark.parametrize( - "markup,expected", - [ - ("content \\\\ ", "content "), - ("content\\\\", "content"), - ("content\\\\ ", "content"), - ], - ) - def test_line_break_escaping(self, markup, expected, process): - document = process(markup) - assert expected in document - - @pytest.mark.parametrize( - "markup,expected", - [ - ("{Sky|color:blue}", r"Sky"), - ("{Red|color:#FF0000} firetruck", r"Red firetruck"), - ], - ) - def test_colors(self, markup, expected, process): - document = process(markup) - assert expected in document - - @pytest.mark.parametrize( - "markup,expected", - [("[image.png?50]", "[image.png?50]"), ("[image.jpg]", "[image.jpg]")], - ) - def test_images_width_resize(self, markup, expected, process): - document = process(markup) - assert expected in document + def test_color_stripped(self, process): + document = process("{Sky|color:blue}") + assert "Sky" in document + assert "color" not in document - @pytest.mark.parametrize( - "markup,expected", - [ - ( - "This is a [named reference 2019-08-01]", - "This is a named reference (2019-08-01)", - ), - ( - "Today is 2019-08-01 - a wonderful day", - "Today is 2019-08-01 - a wonderful day", - ), - ], - ) - def test_entry_reference_links(self, markup, expected, process): - document = process(markup) - assert expected in document + def test_entry_reference(self, process): + assert "named reference (2019-08-01)" in process("A [named reference 2019-08-01]") diff --git a/tests/test_t2t_to_markdown.py b/tests/test_t2t_to_markdown.py new file mode 100644 index 00000000..085ee0ef --- /dev/null +++ b/tests/test_t2t_to_markdown.py @@ -0,0 +1,100 @@ +from rednotebook.util.t2t_to_markdown import convert_to_markdown as c + + +class TestHeadings: + def test_levels(self): + assert c("= Title =") == "# Title" + assert c("== Title ==") == "## Title" + assert c("===== Title =====") == "##### Title" + + def test_heading_with_anchor(self): + assert c("== Clouds ==[clouds]") == "## Clouds" + + def test_markdown_heading_untouched(self): + assert c("# Already Markdown") == "# Already Markdown" + + +class TestInlineFormatting: + def test_italic(self): + assert c("//italic//") == "*italic*" + + def test_bold_unchanged(self): + assert c("**bold**") == "**bold**" + + def test_underline(self): + assert c("__underlined__") == "underlined" + + def test_strikethrough(self): + assert c("--struck--") == "~~struck~~" + + def test_monospace(self): + assert c("``code``") == "`code`" + + def test_combination(self): + assert c("a //b// and **c** and --d--") == "a *b* and **c** and ~~d~~" + + def test_italic_does_not_touch_urls(self): + assert c("see http://example.com now") == "see http://example.com now" + + +class TestHorizontalRule: + def test_long_equals(self): + assert c("====================") == "---" + + def test_long_dashes(self): + assert c("--------------------") == "---" + + def test_short_dashes_untouched(self): + # Three dashes is already a Markdown rule and must survive. + assert c("---") == "---" + + +class TestLists: + def test_bullet_unchanged(self): + assert c("- item") == "- item" + + def test_numbered(self): + assert c("+ item") == "1. item" + + def test_indented_numbered(self): + assert c(" + item") == " 1. item" + + +class TestLinks: + def test_named_web_link(self): + assert c("[heise http://heise.de]") == "[heise](http://heise.de)" + + def test_quoted_link(self): + assert c('[my file ""file:///home/me/f.txt""]') == "[my file](file:///home/me/f.txt)" + + def test_bare_url_unchanged(self): + assert c("http://example.com") == "http://example.com" + + +class TestImages: + def test_simple_image(self): + assert c('[""/home/pic"".png]') == "![](/home/pic.png)" + + def test_image_with_width(self): + assert c('[""/home/pic"".png?50]') == "![](/home/pic.png?50)" + + def test_named_image(self): + assert c('[alt ""/home/pic"".jpg]') == "![alt](/home/pic.jpg)" + + +class TestLineBreak: + def test_trailing_backslashes(self): + assert c("First\\\\\nSecond") == "First \nSecond" + + def test_backslashes_midline_unchanged(self): + assert c("First\\\\Second") == "First\\\\Second" + + +class TestFencedCodeIsPreserved: + def test_no_inline_conversion_in_fence(self): + text = "```\n//not italic//\n+ not numbered\n```" + assert c(text) == text + + def test_entry_reference_passthrough(self): + # Entry references are handled later in the pipeline, not here. + assert c("[2019-08-01]") == "[2019-08-01]" diff --git a/tox.ini b/tox.ini index 9c9b1556..707768a5 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,9 @@ basepython = python3 [testenv] deps = pytest + markdown-it-py + mdit-py-plugins + linkify-it-py commands_pre = python -m pip install --no-deps -e . commands = diff --git a/win/rednotebook.spec b/win/rednotebook.spec index 5f702ed3..51bc28b0 100644 --- a/win/rednotebook.spec +++ b/win/rednotebook.spec @@ -48,7 +48,7 @@ a = Analysis( pathex=[repo], binaries=MISSED_BINARIES, datas=ENCHANT_DICT_FILES, - hiddenimports=[], + hiddenimports=["markdown_it", "mdit_py_plugins", "linkify_it"], hookspath=["."], # To find custom hooks. runtime_hooks=[], excludes=[], diff --git a/win/requirements.txt b/win/requirements.txt index 78f5ce6a..a5754c38 100644 --- a/win/requirements.txt +++ b/win/requirements.txt @@ -2,6 +2,9 @@ altgraph==0.17.5 distlib==0.3.4 filelock==3.6.0 future==0.18.2 +linkify-it-py==2.0.3 +markdown-it-py==3.0.0 +mdit-py-plugins==0.4.2 pefile==2022.5.30 platformdirs==2.5.1 pyinstaller==6.16.0