diff --git a/README.md b/README.md index 9196b8c..f8e407d 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,40 @@ Optional Sphinx config: pyrepl_js = "../pyrepl.js" # default; path to the pyrepl-web loader script ``` +## Autodoc integration + +When `sphinx.ext.autodoc` is enabled, you can automatically convert docstring +``Example:`` / ``Examples:`` doctest blocks into interactive REPLs at build time. +Python source docstrings stay unchanged (full doctests with expected output are +preserved for IDE hover docs and optional ``doctest`` runs). + +Add to `conf.py`: + +```python +extensions = [ + "sphinx.ext.autodoc", + "sphinx_pyrepl_web", +] + +pyrepl_autodoc = True +pyrepl_autodoc_packages = "my_package" # required for Pyodide import +pyrepl_autodoc_sections = ["Example"] +pyrepl_autodoc_options = {"no-banner": True} +``` + +| Config | Default | Description | +|--------|---------|-------------| +| `pyrepl_autodoc` | `False` | Enable autodoc docstring → REPL conversion | +| `pyrepl_autodoc_packages` | `None` | Value for `:packages:` on generated directives (auto-derived from object name when unset) | +| `pyrepl_autodoc_sections` | `["Example", "Examples"]` | Section titles to convert | +| `pyrepl_autodoc_options` | `{}` | Default directive flags, e.g. `{"no-banner": True}` | + +**Limitations (v0.2.0):** + +- Targets raw Google-style / plain-text section headers (without `sphinx.ext.napoleon`). Napoleon-converted sections are not detected yet. +- `:packages:` (or auto-derivation) is required so Pyodide can import your library. +- Live REPL output may differ from doctest expected output (e.g. `PosixPath` vs `WindowsPath`, traceback formatting). + ## Updating pyrepl-web Since [chrizzFTD/pyrepl-web](https://github.com/chrizzFTD/pyrepl-web) is a fork, this sphinx extension vendors the JavaScript assets for easier distribution. To update them, run: diff --git a/sphinx_pyrepl_web/__init__.py b/sphinx_pyrepl_web/__init__.py index 24ea698..a15b59c 100644 --- a/sphinx_pyrepl_web/__init__.py +++ b/sphinx_pyrepl_web/__init__.py @@ -1,6 +1,6 @@ """A Sphinx extension for embedding pyrepl-web Python REPLs in documentation.""" -__version__ = "0.1.1" +__version__ = "0.2.0" import json from pathlib import Path @@ -23,6 +23,11 @@ def setup(app: Sphinx): app.connect("doctree-read", doctree_read) app.connect("html-page-context", add_html_context) app.connect("env-updated", copy_asset_files) + + from sphinx_pyrepl_web.autodoc import register as register_autodoc + + register_autodoc(app) + return {"version": __version__, "parallel_read_safe": True} @@ -51,6 +56,20 @@ def strip_doctest_prompts(lines: list[str]) -> list[str]: return result +def register_replay_script(env, docname: str, body_lines: list[str]) -> str: + """Strip doctest prompts, store replay script in metadata, return replay-src path.""" + body_text = "\n".join(strip_doctest_prompts(body_lines)) + "\n" + + replay_files = json.loads( + env.metadata[docname].setdefault(REPLAY_FILES_KEY, "{}") + ) + counter = len(replay_files) + 1 + script_name = f"{docname.replace('/', '-')}-{counter}.py" + replay_files[script_name] = body_text + env.metadata[docname][REPLAY_FILES_KEY] = json.dumps(replay_files) + return f"_static/pyrepl/{script_name}" + + class PyRepl(SphinxDirective): """Embed a pyrepl-web ```` element.""" @@ -116,19 +135,7 @@ def run(self): attrs.append(f'src="{rel_src}"') if has_body: - body_lines = strip_doctest_prompts(list(self.content)) - body_text = "\n".join(body_lines) + "\n" - - replay_files = json.loads( - self.env.metadata[self.env.docname].setdefault(REPLAY_FILES_KEY, "{}") - ) - counter = len(replay_files) + 1 - script_name = f"{env.docname.replace('/', '-')}-{counter}.py" - replay_files[script_name] = body_text - self.env.metadata[self.env.docname][REPLAY_FILES_KEY] = json.dumps( - replay_files - ) - replay_src = f"_static/pyrepl/{script_name}" + replay_src = register_replay_script(env, env.docname, list(self.content)) attrs.append(f'replay-src="{replay_src}"') self.env.metadata[self.env.docname]["pyrepl"] = True diff --git a/sphinx_pyrepl_web/autodoc.py b/sphinx_pyrepl_web/autodoc.py new file mode 100644 index 0000000..34dd173 --- /dev/null +++ b/sphinx_pyrepl_web/autodoc.py @@ -0,0 +1,214 @@ +"""Autodoc integration: convert docstring Example blocks to py-repl directives.""" + +from __future__ import annotations + +import re +from typing import Any + +from sphinx.application import Sphinx + +from sphinx_pyrepl_web.doctest import _is_doctest_input_line, extract_doctest_inputs + +# Common docstring section headers that end an Example doctest block. +_OTHER_SECTION_HEADERS = frozenset( + { + "Args", + "Arguments", + "Attributes", + "Caution", + "Danger", + "Error", + "Hint", + "Important", + "Keyword Args", + "Keyword Arguments", + "Methods", + "Note", + "Notes", + "Parameters", + "Raises", + "Return", + "Returns", + "See Also", + "Tip", + "Todo", + "Warning", + "Warnings", + "Warns", + "Yield", + "Yields", + } +) + +_DEFAULT_WHAT = frozenset({"class", "function", "method"}) + + +def _section_header_pattern(sections: list[str]) -> re.Pattern[str]: + names = "|".join(re.escape(section) for section in sections) + return re.compile(rf"^\s*({names})\s*:?\s*$") + + +def _matches_configured_section(line: str, pattern: re.Pattern[str]) -> bool: + return bool(pattern.match(line)) + + +def _matches_other_section_header(line: str) -> bool: + stripped = line.lstrip() + if not stripped.endswith(":"): + return False + title = stripped[:-1].strip() + return title in _OTHER_SECTION_HEADERS + + +def _collect_doctest_block( + lines: list[str], start: int, section_pattern: re.Pattern[str] +) -> tuple[list[str], int]: + """Return doctest block lines and index after the block.""" + block: list[str] = [] + block_started = False + i = start + + while i < len(lines): + line = lines[i] + stripped = line.lstrip() + + if stripped == "": + if block_started: + block.append(line) + i += 1 + continue + + if _matches_configured_section(line, section_pattern): + break + + if _matches_other_section_header(stripped): + break + + if _is_doctest_input_line(line): + block_started = True + block.append(line) + i += 1 + continue + + if block_started and (line.startswith(" ") or line.startswith("\t")): + block.append(line) + i += 1 + continue + + break + + return block, i + + +def _has_primary_prompt(lines: list[str]) -> bool: + return any(line.lstrip().startswith(">>>") for line in lines) + + +def format_pyrepl_directive( + input_lines: list[str], options: dict[str, str | bool] +) -> list[str]: + """Format a ``.. py-repl::`` directive block for injection into a docstring.""" + result = [".. py-repl::"] + + for key in sorted(options): + value = options[key] + if value is True: + result.append(f" :{key}:") + elif value: + result.append(f" :{key}: {value}") + + if input_lines: + result.append("") + for line in input_lines: + normalized = line.lstrip() + result.append(f" {normalized}" if normalized else "") + + return result + + +def transform_docstring_lines( + lines: list[str], + sections: list[str], + options: dict[str, str | bool], +) -> None: + """Replace configured Example sections with ``.. py-repl::`` directives in place.""" + section_pattern = _section_header_pattern(sections) + i = 0 + + while i < len(lines): + line = lines[i] + if not _matches_configured_section(line, section_pattern): + i += 1 + continue + + header_line = line + block, next_i = _collect_doctest_block(lines, i + 1, section_pattern) + + if not block or not _has_primary_prompt(block): + i = next_i + continue + + input_lines = extract_doctest_inputs(block) + if not input_lines: + i = next_i + continue + + replacement = [header_line, ""] + format_pyrepl_directive(input_lines, options) + lines[i:next_i] = replacement + i += len(replacement) + + +def derive_packages(qualified_name: str) -> str: + """Derive a PyPI/import package name from a documented object's qualified name.""" + if "." in qualified_name: + return qualified_name.rsplit(".", 1)[0] + return qualified_name + + +def process_autodoc_docstring( + app: Sphinx, + what: str, + name: str, + obj: Any, + options: Any, + lines: list[str], +) -> None: + """Convert matching docstring Example blocks into ``.. py-repl::`` directives.""" + if what not in _DEFAULT_WHAT: + return + + sections: list[str] = app.config.pyrepl_autodoc_sections + if not sections: + return + + directive_options: dict[str, str | bool] = dict(app.config.pyrepl_autodoc_options) + packages = app.config.pyrepl_autodoc_packages + if packages is None: + packages = derive_packages(name) + if packages: + directive_options["packages"] = packages + + transform_docstring_lines(lines, sections, directive_options) + + +def register(app: Sphinx) -> None: + """Register autodoc config values and hook when enabled.""" + app.add_config_value("pyrepl_autodoc", False, "env") + app.add_config_value("pyrepl_autodoc_packages", None, "env") + app.add_config_value("pyrepl_autodoc_sections", ["Example", "Examples"], "env") + app.add_config_value("pyrepl_autodoc_options", {}, "env") + + if not app.config.pyrepl_autodoc: + return + + if "sphinx.ext.autodoc" not in app.config.extensions: + from sphinx.util import logging + + logger = logging.getLogger(__name__) + logger.warning( + "pyrepl_autodoc is enabled but sphinx.ext.autodoc is not loaded; " + "autodoc integration will have no effect." + ) + return + + app.connect("autodoc-process-docstring", process_autodoc_docstring) diff --git a/sphinx_pyrepl_web/doctest.py b/sphinx_pyrepl_web/doctest.py new file mode 100644 index 0000000..8e6becb --- /dev/null +++ b/sphinx_pyrepl_web/doctest.py @@ -0,0 +1,50 @@ +"""Utilities for parsing doctest blocks in docstrings.""" + + +def _is_doctest_input_line(line: str) -> bool: + """Return True if *line* is a doctest input or continuation prompt.""" + stripped = line.lstrip() + if stripped.startswith(">>> "): + return True + if stripped == ">>>": + return True + if stripped.startswith("..."): + return True + return False + + +def extract_doctest_inputs(lines: list[str]) -> list[str]: + """Return input-only doctest lines, dropping expected output and tracebacks. + + Keeps lines starting with ``>>>`` or ``...`` (including bare ``...`` block + terminators). Drops expected output lines and traceback blocks (from + ``Traceback`` through the error message). + """ + result: list[str] = [] + in_traceback = False + in_block = False + + for line in lines: + stripped = line.lstrip() + + if stripped.startswith("Traceback"): + in_traceback = True + continue + + if in_traceback: + # Only resume on a new primary prompt; traceback ellipsis uses ``...``. + if stripped.startswith(">>> "): + in_traceback = False + else: + continue + + if _is_doctest_input_line(line): + in_block = True + result.append(line) + elif in_block and stripped == "": + result.append(line) + else: + # Expected output or trailing non-doctest content. + pass + + return result diff --git a/tests/test_autodoc.py b/tests/test_autodoc.py new file mode 100644 index 0000000..2a9ccd8 --- /dev/null +++ b/tests/test_autodoc.py @@ -0,0 +1,92 @@ +from sphinx_pyrepl_web.autodoc import ( + derive_packages, + format_pyrepl_directive, + transform_docstring_lines, +) + + +def test_format_pyrepl_directive_options_and_body(): + result = format_pyrepl_directive( + [" >>> x = 1", " >>> x + 1"], + {"no-banner": True, "packages": "naming"}, + ) + assert result == [ + ".. py-repl::", + " :no-banner:", + " :packages: naming", + "", + " >>> x = 1", + " >>> x + 1", + ] + + +def test_transform_docstring_injects_pyrepl(): + lines = [ + "Base class for name objects.", + "", + "Example:", + " >>> from naming import Name", + " >>> n = MyName()", + " >>> n.get()", + " '{base}'", + ] + transform_docstring_lines( + lines, + sections=["Example"], + options={"no-banner": True, "packages": "naming"}, + ) + assert lines[2] == "Example:" + assert lines[3] == "" + assert lines[4] == ".. py-repl::" + assert " :packages: naming" in lines + assert " >>> from naming import Name" in lines + assert "'{base}'" not in lines + + +def test_transform_docstring_leaves_non_example_unchanged(): + lines = [ + "A helper function.", + "", + "Returns:", + " A string value.", + ] + original = list(lines) + transform_docstring_lines(lines, sections=["Example"], options={}) + assert lines == original + + +def test_transform_docstring_skips_example_without_doctest(): + lines = [ + "Example:", + "Use this class for naming things.", + ] + original = list(lines) + transform_docstring_lines(lines, sections=["Example"], options={}) + assert lines == original + + +def test_transform_docstring_pipefile_traceback(): + lines = [ + "Example:", + " >>> pf.year = 'nondigits'", + " Traceback (most recent call last):", + " ...", + " ValueError: invalid", + " >>> pf.year = 1907", + ] + transform_docstring_lines( + lines, + sections=["Example"], + options={"packages": "naming", "no-banner": True}, + ) + body = "\n".join(lines) + assert ".. py-repl::" in body + assert "Traceback" not in body + assert "ValueError" not in body + assert " >>> pf.year = 'nondigits'" in lines + assert " >>> pf.year = 1907" in lines + + +def test_derive_packages(): + assert derive_packages("naming.Name") == "naming" + assert derive_packages("naming") == "naming" diff --git a/tests/test_build_autodoc_replay.py b/tests/test_build_autodoc_replay.py new file mode 100644 index 0000000..707721b --- /dev/null +++ b/tests/test_build_autodoc_replay.py @@ -0,0 +1,157 @@ +import json +import py_compile +import sys +from pathlib import Path + +import pytest +from sphinx.application import Sphinx + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + + +@pytest.fixture +def autodoc_project(tmp_path): + srcdir = tmp_path / "docs" + pkgdir = tmp_path / "mockpkg" + srcdir.mkdir() + pkgdir.mkdir() + (pkgdir / "__init__.py").write_text( + r''' +class Name: + r"""Base class for name objects. + + Example: + >>> from naming import Name + >>> class MyName(Name): + ... config = dict(base=r'\w+') + ... + >>> n = MyName() + >>> n.get() + '{base}' + >>> n.name = 'hello_world' + >>> n + Name("hello_world") + """ + +class PipeFile: + """Pipeline file example. + + Example: + >>> from naming import PipeFile + >>> pf = PipeFile('wipfile.7.ext') + >>> pf.values + {'base': 'wipfile', 'version': '7'} + >>> pf.year = 'nondigits' + Traceback (most recent call last): + ... + ValueError: invalid year + >>> pf.year = 1907 + """ +'''.strip() + + "\n", + encoding="utf-8", + ) + + outdir = tmp_path / "_build" + doctreedir = tmp_path / "_doctree" + (srcdir / "conf.py").write_text( + f"import sys\n" + f"sys.path.insert(0, {str(tmp_path)!r})\n" + "extensions = ['sphinx.ext.autodoc', 'sphinx_pyrepl_web']\n" + "pyrepl_js = 'pyrepl.js'\n" + "master_doc = 'index'\n" + "pyrepl_autodoc = True\n" + "pyrepl_autodoc_packages = 'mockpkg'\n" + "pyrepl_autodoc_sections = ['Example']\n" + "pyrepl_autodoc_options = {'no-banner': True}\n", + encoding="utf-8", + ) + (srcdir / "index.rst").write_text( + """ +API +=== + +.. autoclass:: mockpkg.Name + :members: + +.. autoclass:: mockpkg.PipeFile + :members: +""".strip() + + "\n", + encoding="utf-8", + ) + return srcdir, outdir, doctreedir + + +def _build(srcdir, outdir, doctreedir, parallel=0): + outdir.mkdir(parents=True, exist_ok=True) + doctreedir.mkdir(parents=True, exist_ok=True) + with open(outdir / "warnings.txt", "w", encoding="utf-8") as warning_file: + app = Sphinx( + srcdir=str(srcdir), + confdir=str(srcdir), + outdir=str(outdir), + doctreedir=str(doctreedir), + buildername="html", + warning=warning_file, + freshenv=True, + parallel=parallel, + ) + app.build() + return app + + +def test_build_autodoc_injects_pyrepl_and_replay_scripts(autodoc_project): + srcdir, outdir, doctreedir = autodoc_project + app = _build(srcdir, outdir, doctreedir) + + html = (outdir / "index.html").read_text(encoding="utf-8") + assert html.count("= 2 + assert 'packages="mockpkg"' in html + assert "no-banner" in html + + replay_dir = outdir / "_static" / "pyrepl" + scripts = sorted(replay_dir.glob("index-*.py")) + assert len(scripts) >= 2 + + for script in scripts: + content = script.read_text(encoding="utf-8") + assert ">>>" not in content + assert "'{base}'" not in content + assert "Traceback" not in content + py_compile.compile(str(script), doraise=True) + + replay_files = json.loads( + app.env.metadata["index"].get("pyrepl-replay-files", "{}") + ) + assert len(replay_files) >= 2 + + +def test_build_autodoc_disabled_by_default(tmp_path): + srcdir = tmp_path / "docs" + pkgdir = tmp_path / "mockpkg" + srcdir.mkdir() + pkgdir.mkdir() + (pkgdir / "__init__.py").write_text( + 'class Name:\n """Example:\n >>> x = 1\n 1\n """\n', + encoding="utf-8", + ) + outdir = tmp_path / "_build" + doctreedir = tmp_path / "_doctree" + (srcdir / "conf.py").write_text( + f"import sys\nsys.path.insert(0, {str(tmp_path)!r})\n" + "extensions = ['sphinx.ext.autodoc', 'sphinx_pyrepl_web']\n" + "pyrepl_js = 'pyrepl.js'\n" + "master_doc = 'index'\n", + encoding="utf-8", + ) + (srcdir / "index.rst").write_text( + ".. autoclass:: mockpkg.Name\n :members:\n", + encoding="utf-8", + ) + app = _build(srcdir, outdir, doctreedir) + + html = (outdir / "index.html").read_text(encoding="utf-8") + assert ">> from naming import Name", + " >>> class MyName(Name):", + " ... config = dict(base=r'\\w+')", + " ...", + " >>> n = MyName()", + " >>> n.get()", + " '{base}'", + " >>> n.values", + " {}", + " >>> n.name = 'hello_world'", + " >>> n", + ' Name("hello_world")', + ] + assert extract_doctest_inputs(lines) == [ + " >>> from naming import Name", + " >>> class MyName(Name):", + " ... config = dict(base=r'\\w+')", + " ...", + " >>> n = MyName()", + " >>> n.get()", + " >>> n.values", + " >>> n.name = 'hello_world'", + " >>> n", + ] + + +def test_pipefile_list_output(): + lines = [ + " >>> [p.get(index=x, output='render') for x in range(10)]", + " ['wipfile.render.7.0.ext',", + " 'wipfile.render.7.1.ext',", + " 'wipfile.render.7.2.ext',", + " 'wipfile.render.7.3.ext',", + " 'wipfile.render.7.4.ext',", + " 'wipfile.render.7.5.ext',", + " 'wipfile.render.7.6.ext',", + " 'wipfile.render.7.7.ext',", + " 'wipfile.render.7.8.ext',", + " 'wipfile.render.7.9.ext']", + ] + assert extract_doctest_inputs(lines) == [ + " >>> [p.get(index=x, output='render') for x in range(10)]", + ] + + +def test_pipefile_traceback_block(): + lines = [ + " >>> pf.year = 'nondigits'", + " Traceback (most recent call last):", + " ...", + " ValueError: Can't set field 'year' with invalid value 'nondigits'", + " >>> pf.year = 1907", + " >>> pf", + ' ProjectFile("project_data_name_1907_christianl_constant_iamlast.data.17.abc")', + ] + assert extract_doctest_inputs(lines) == [ + " >>> pf.year = 'nondigits'", + " >>> pf.year = 1907", + " >>> pf", + ] + + +def test_bare_terminator_preserved(): + lines = [ + ">>> class Foo:", + "... x = 1", + "...", + ">>> Foo()", + ] + assert extract_doctest_inputs(lines) == lines + + +def test_comment_input_line(): + lines = [ + " >>> n.name = 'hello_world'", + " >>> n", + ' Name("hello_world")', + " >>> # modify name and get values from field names", + " >>> n.base = 'through_field_name'", + ] + assert extract_doctest_inputs(lines) == [ + " >>> n.name = 'hello_world'", + " >>> n", + " >>> # modify name and get values from field names", + " >>> n.base = 'through_field_name'", + ] + + +def test_input_only_passthrough(): + lines = [ + ">>> x = 1", + ">>> x + 1", + ] + assert extract_doctest_inputs(lines) == lines + + +def test_empty_block(): + assert extract_doctest_inputs([]) == []