From bfb17f2ab874a67e501c01e464b83c44564cf7b8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 27 Jun 2026 12:59:15 +0000 Subject: [PATCH 1/3] Add opt-in autodoc integration for Example doctest blocks (v0.2.0) Implement issue #9: convert autodoc docstring Example/Examples sections into interactive py-repl directives at Sphinx build time. - Add extract_doctest_inputs() to strip expected output and tracebacks - Add autodoc-process-docstring hook with pyrepl_autodoc config values - Extract register_replay_script() helper from PyRepl.run() - Add unit and integration tests for autodoc conversion - Document autodoc integration and limitations in README Co-authored-by: chrizzftd --- README.md | 34 +++++ sphinx_pyrepl_web/__init__.py | 35 +++-- sphinx_pyrepl_web/autodoc.py | 214 +++++++++++++++++++++++++++ sphinx_pyrepl_web/doctest.py | 50 +++++++ tests/test_autodoc.py | 92 ++++++++++++ tests/test_build_autodoc_replay.py | 157 ++++++++++++++++++++ tests/test_extract_doctest_inputs.py | 103 +++++++++++++ 7 files changed, 671 insertions(+), 14 deletions(-) create mode 100644 sphinx_pyrepl_web/autodoc.py create mode 100644 sphinx_pyrepl_web/doctest.py create mode 100644 tests/test_autodoc.py create mode 100644 tests/test_build_autodoc_replay.py create mode 100644 tests/test_extract_doctest_inputs.py diff --git a/README.md b/README.md index 9196b8c..f8e407d 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,40 @@ Optional Sphinx config: pyrepl_js = "../pyrepl.js" # default; path to the pyrepl-web loader script ``` +## Autodoc integration + +When `sphinx.ext.autodoc` is enabled, you can automatically convert docstring +``Example:`` / ``Examples:`` doctest blocks into interactive REPLs at build time. +Python source docstrings stay unchanged (full doctests with expected output are +preserved for IDE hover docs and optional ``doctest`` runs). + +Add to `conf.py`: + +```python +extensions = [ + "sphinx.ext.autodoc", + "sphinx_pyrepl_web", +] + +pyrepl_autodoc = True +pyrepl_autodoc_packages = "my_package" # required for Pyodide import +pyrepl_autodoc_sections = ["Example"] +pyrepl_autodoc_options = {"no-banner": True} +``` + +| Config | Default | Description | +|--------|---------|-------------| +| `pyrepl_autodoc` | `False` | Enable autodoc docstring → REPL conversion | +| `pyrepl_autodoc_packages` | `None` | Value for `:packages:` on generated directives (auto-derived from object name when unset) | +| `pyrepl_autodoc_sections` | `["Example", "Examples"]` | Section titles to convert | +| `pyrepl_autodoc_options` | `{}` | Default directive flags, e.g. `{"no-banner": True}` | + +**Limitations (v0.2.0):** + +- Targets raw Google-style / plain-text section headers (without `sphinx.ext.napoleon`). Napoleon-converted sections are not detected yet. +- `:packages:` (or auto-derivation) is required so Pyodide can import your library. +- Live REPL output may differ from doctest expected output (e.g. `PosixPath` vs `WindowsPath`, traceback formatting). + ## Updating pyrepl-web Since [chrizzFTD/pyrepl-web](https://github.com/chrizzFTD/pyrepl-web) is a fork, this sphinx extension vendors the JavaScript assets for easier distribution. To update them, run: diff --git a/sphinx_pyrepl_web/__init__.py b/sphinx_pyrepl_web/__init__.py index 24ea698..a15b59c 100644 --- a/sphinx_pyrepl_web/__init__.py +++ b/sphinx_pyrepl_web/__init__.py @@ -1,6 +1,6 @@ """A Sphinx extension for embedding pyrepl-web Python REPLs in documentation.""" -__version__ = "0.1.1" +__version__ = "0.2.0" import json from pathlib import Path @@ -23,6 +23,11 @@ def setup(app: Sphinx): app.connect("doctree-read", doctree_read) app.connect("html-page-context", add_html_context) app.connect("env-updated", copy_asset_files) + + from sphinx_pyrepl_web.autodoc import register as register_autodoc + + register_autodoc(app) + return {"version": __version__, "parallel_read_safe": True} @@ -51,6 +56,20 @@ def strip_doctest_prompts(lines: list[str]) -> list[str]: return result +def register_replay_script(env, docname: str, body_lines: list[str]) -> str: + """Strip doctest prompts, store replay script in metadata, return replay-src path.""" + body_text = "\n".join(strip_doctest_prompts(body_lines)) + "\n" + + replay_files = json.loads( + env.metadata[docname].setdefault(REPLAY_FILES_KEY, "{}") + ) + counter = len(replay_files) + 1 + script_name = f"{docname.replace('/', '-')}-{counter}.py" + replay_files[script_name] = body_text + env.metadata[docname][REPLAY_FILES_KEY] = json.dumps(replay_files) + return f"_static/pyrepl/{script_name}" + + class PyRepl(SphinxDirective): """Embed a pyrepl-web ```` element.""" @@ -116,19 +135,7 @@ def run(self): attrs.append(f'src="{rel_src}"') if has_body: - body_lines = strip_doctest_prompts(list(self.content)) - body_text = "\n".join(body_lines) + "\n" - - replay_files = json.loads( - self.env.metadata[self.env.docname].setdefault(REPLAY_FILES_KEY, "{}") - ) - counter = len(replay_files) + 1 - script_name = f"{env.docname.replace('/', '-')}-{counter}.py" - replay_files[script_name] = body_text - self.env.metadata[self.env.docname][REPLAY_FILES_KEY] = json.dumps( - replay_files - ) - replay_src = f"_static/pyrepl/{script_name}" + replay_src = register_replay_script(env, env.docname, list(self.content)) attrs.append(f'replay-src="{replay_src}"') self.env.metadata[self.env.docname]["pyrepl"] = True diff --git a/sphinx_pyrepl_web/autodoc.py b/sphinx_pyrepl_web/autodoc.py new file mode 100644 index 0000000..34dd173 --- /dev/null +++ b/sphinx_pyrepl_web/autodoc.py @@ -0,0 +1,214 @@ +"""Autodoc integration: convert docstring Example blocks to py-repl directives.""" + +from __future__ import annotations + +import re +from typing import Any + +from sphinx.application import Sphinx + +from sphinx_pyrepl_web.doctest import _is_doctest_input_line, extract_doctest_inputs + +# Common docstring section headers that end an Example doctest block. +_OTHER_SECTION_HEADERS = frozenset( + { + "Args", + "Arguments", + "Attributes", + "Caution", + "Danger", + "Error", + "Hint", + "Important", + "Keyword Args", + "Keyword Arguments", + "Methods", + "Note", + "Notes", + "Parameters", + "Raises", + "Return", + "Returns", + "See Also", + "Tip", + "Todo", + "Warning", + "Warnings", + "Warns", + "Yield", + "Yields", + } +) + +_DEFAULT_WHAT = frozenset({"class", "function", "method"}) + + +def _section_header_pattern(sections: list[str]) -> re.Pattern[str]: + names = "|".join(re.escape(section) for section in sections) + return re.compile(rf"^\s*({names})\s*:?\s*$") + + +def _matches_configured_section(line: str, pattern: re.Pattern[str]) -> bool: + return bool(pattern.match(line)) + + +def _matches_other_section_header(line: str) -> bool: + stripped = line.lstrip() + if not stripped.endswith(":"): + return False + title = stripped[:-1].strip() + return title in _OTHER_SECTION_HEADERS + + +def _collect_doctest_block( + lines: list[str], start: int, section_pattern: re.Pattern[str] +) -> tuple[list[str], int]: + """Return doctest block lines and index after the block.""" + block: list[str] = [] + block_started = False + i = start + + while i < len(lines): + line = lines[i] + stripped = line.lstrip() + + if stripped == "": + if block_started: + block.append(line) + i += 1 + continue + + if _matches_configured_section(line, section_pattern): + break + + if _matches_other_section_header(stripped): + break + + if _is_doctest_input_line(line): + block_started = True + block.append(line) + i += 1 + continue + + if block_started and (line.startswith(" ") or line.startswith("\t")): + block.append(line) + i += 1 + continue + + break + + return block, i + + +def _has_primary_prompt(lines: list[str]) -> bool: + return any(line.lstrip().startswith(">>>") for line in lines) + + +def format_pyrepl_directive( + input_lines: list[str], options: dict[str, str | bool] +) -> list[str]: + """Format a ``.. py-repl::`` directive block for injection into a docstring.""" + result = [".. py-repl::"] + + for key in sorted(options): + value = options[key] + if value is True: + result.append(f" :{key}:") + elif value: + result.append(f" :{key}: {value}") + + if input_lines: + result.append("") + for line in input_lines: + normalized = line.lstrip() + result.append(f" {normalized}" if normalized else "") + + return result + + +def transform_docstring_lines( + lines: list[str], + sections: list[str], + options: dict[str, str | bool], +) -> None: + """Replace configured Example sections with ``.. py-repl::`` directives in place.""" + section_pattern = _section_header_pattern(sections) + i = 0 + + while i < len(lines): + line = lines[i] + if not _matches_configured_section(line, section_pattern): + i += 1 + continue + + header_line = line + block, next_i = _collect_doctest_block(lines, i + 1, section_pattern) + + if not block or not _has_primary_prompt(block): + i = next_i + continue + + input_lines = extract_doctest_inputs(block) + if not input_lines: + i = next_i + continue + + replacement = [header_line, ""] + format_pyrepl_directive(input_lines, options) + lines[i:next_i] = replacement + i += len(replacement) + + +def derive_packages(qualified_name: str) -> str: + """Derive a PyPI/import package name from a documented object's qualified name.""" + if "." in qualified_name: + return qualified_name.rsplit(".", 1)[0] + return qualified_name + + +def process_autodoc_docstring( + app: Sphinx, + what: str, + name: str, + obj: Any, + options: Any, + lines: list[str], +) -> None: + """Convert matching docstring Example blocks into ``.. py-repl::`` directives.""" + if what not in _DEFAULT_WHAT: + return + + sections: list[str] = app.config.pyrepl_autodoc_sections + if not sections: + return + + directive_options: dict[str, str | bool] = dict(app.config.pyrepl_autodoc_options) + packages = app.config.pyrepl_autodoc_packages + if packages is None: + packages = derive_packages(name) + if packages: + directive_options["packages"] = packages + + transform_docstring_lines(lines, sections, directive_options) + + +def register(app: Sphinx) -> None: + """Register autodoc config values and hook when enabled.""" + app.add_config_value("pyrepl_autodoc", False, "env") + app.add_config_value("pyrepl_autodoc_packages", None, "env") + app.add_config_value("pyrepl_autodoc_sections", ["Example", "Examples"], "env") + app.add_config_value("pyrepl_autodoc_options", {}, "env") + + if not app.config.pyrepl_autodoc: + return + + if "sphinx.ext.autodoc" not in app.config.extensions: + from sphinx.util import logging + + logger = logging.getLogger(__name__) + logger.warning( + "pyrepl_autodoc is enabled but sphinx.ext.autodoc is not loaded; " + "autodoc integration will have no effect." + ) + return + + app.connect("autodoc-process-docstring", process_autodoc_docstring) diff --git a/sphinx_pyrepl_web/doctest.py b/sphinx_pyrepl_web/doctest.py new file mode 100644 index 0000000..8e6becb --- /dev/null +++ b/sphinx_pyrepl_web/doctest.py @@ -0,0 +1,50 @@ +"""Utilities for parsing doctest blocks in docstrings.""" + + +def _is_doctest_input_line(line: str) -> bool: + """Return True if *line* is a doctest input or continuation prompt.""" + stripped = line.lstrip() + if stripped.startswith(">>> "): + return True + if stripped == ">>>": + return True + if stripped.startswith("..."): + return True + return False + + +def extract_doctest_inputs(lines: list[str]) -> list[str]: + """Return input-only doctest lines, dropping expected output and tracebacks. + + Keeps lines starting with ``>>>`` or ``...`` (including bare ``...`` block + terminators). Drops expected output lines and traceback blocks (from + ``Traceback`` through the error message). + """ + result: list[str] = [] + in_traceback = False + in_block = False + + for line in lines: + stripped = line.lstrip() + + if stripped.startswith("Traceback"): + in_traceback = True + continue + + if in_traceback: + # Only resume on a new primary prompt; traceback ellipsis uses ``...``. + if stripped.startswith(">>> "): + in_traceback = False + else: + continue + + if _is_doctest_input_line(line): + in_block = True + result.append(line) + elif in_block and stripped == "": + result.append(line) + else: + # Expected output or trailing non-doctest content. + pass + + return result diff --git a/tests/test_autodoc.py b/tests/test_autodoc.py new file mode 100644 index 0000000..2a9ccd8 --- /dev/null +++ b/tests/test_autodoc.py @@ -0,0 +1,92 @@ +from sphinx_pyrepl_web.autodoc import ( + derive_packages, + format_pyrepl_directive, + transform_docstring_lines, +) + + +def test_format_pyrepl_directive_options_and_body(): + result = format_pyrepl_directive( + [" >>> x = 1", " >>> x + 1"], + {"no-banner": True, "packages": "naming"}, + ) + assert result == [ + ".. py-repl::", + " :no-banner:", + " :packages: naming", + "", + " >>> x = 1", + " >>> x + 1", + ] + + +def test_transform_docstring_injects_pyrepl(): + lines = [ + "Base class for name objects.", + "", + "Example:", + " >>> from naming import Name", + " >>> n = MyName()", + " >>> n.get()", + " '{base}'", + ] + transform_docstring_lines( + lines, + sections=["Example"], + options={"no-banner": True, "packages": "naming"}, + ) + assert lines[2] == "Example:" + assert lines[3] == "" + assert lines[4] == ".. py-repl::" + assert " :packages: naming" in lines + assert " >>> from naming import Name" in lines + assert "'{base}'" not in lines + + +def test_transform_docstring_leaves_non_example_unchanged(): + lines = [ + "A helper function.", + "", + "Returns:", + " A string value.", + ] + original = list(lines) + transform_docstring_lines(lines, sections=["Example"], options={}) + assert lines == original + + +def test_transform_docstring_skips_example_without_doctest(): + lines = [ + "Example:", + "Use this class for naming things.", + ] + original = list(lines) + transform_docstring_lines(lines, sections=["Example"], options={}) + assert lines == original + + +def test_transform_docstring_pipefile_traceback(): + lines = [ + "Example:", + " >>> pf.year = 'nondigits'", + " Traceback (most recent call last):", + " ...", + " ValueError: invalid", + " >>> pf.year = 1907", + ] + transform_docstring_lines( + lines, + sections=["Example"], + options={"packages": "naming", "no-banner": True}, + ) + body = "\n".join(lines) + assert ".. py-repl::" in body + assert "Traceback" not in body + assert "ValueError" not in body + assert " >>> pf.year = 'nondigits'" in lines + assert " >>> pf.year = 1907" in lines + + +def test_derive_packages(): + assert derive_packages("naming.Name") == "naming" + assert derive_packages("naming") == "naming" diff --git a/tests/test_build_autodoc_replay.py b/tests/test_build_autodoc_replay.py new file mode 100644 index 0000000..707721b --- /dev/null +++ b/tests/test_build_autodoc_replay.py @@ -0,0 +1,157 @@ +import json +import py_compile +import sys +from pathlib import Path + +import pytest +from sphinx.application import Sphinx + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + + +@pytest.fixture +def autodoc_project(tmp_path): + srcdir = tmp_path / "docs" + pkgdir = tmp_path / "mockpkg" + srcdir.mkdir() + pkgdir.mkdir() + (pkgdir / "__init__.py").write_text( + r''' +class Name: + r"""Base class for name objects. + + Example: + >>> from naming import Name + >>> class MyName(Name): + ... config = dict(base=r'\w+') + ... + >>> n = MyName() + >>> n.get() + '{base}' + >>> n.name = 'hello_world' + >>> n + Name("hello_world") + """ + +class PipeFile: + """Pipeline file example. + + Example: + >>> from naming import PipeFile + >>> pf = PipeFile('wipfile.7.ext') + >>> pf.values + {'base': 'wipfile', 'version': '7'} + >>> pf.year = 'nondigits' + Traceback (most recent call last): + ... + ValueError: invalid year + >>> pf.year = 1907 + """ +'''.strip() + + "\n", + encoding="utf-8", + ) + + outdir = tmp_path / "_build" + doctreedir = tmp_path / "_doctree" + (srcdir / "conf.py").write_text( + f"import sys\n" + f"sys.path.insert(0, {str(tmp_path)!r})\n" + "extensions = ['sphinx.ext.autodoc', 'sphinx_pyrepl_web']\n" + "pyrepl_js = 'pyrepl.js'\n" + "master_doc = 'index'\n" + "pyrepl_autodoc = True\n" + "pyrepl_autodoc_packages = 'mockpkg'\n" + "pyrepl_autodoc_sections = ['Example']\n" + "pyrepl_autodoc_options = {'no-banner': True}\n", + encoding="utf-8", + ) + (srcdir / "index.rst").write_text( + """ +API +=== + +.. autoclass:: mockpkg.Name + :members: + +.. autoclass:: mockpkg.PipeFile + :members: +""".strip() + + "\n", + encoding="utf-8", + ) + return srcdir, outdir, doctreedir + + +def _build(srcdir, outdir, doctreedir, parallel=0): + outdir.mkdir(parents=True, exist_ok=True) + doctreedir.mkdir(parents=True, exist_ok=True) + with open(outdir / "warnings.txt", "w", encoding="utf-8") as warning_file: + app = Sphinx( + srcdir=str(srcdir), + confdir=str(srcdir), + outdir=str(outdir), + doctreedir=str(doctreedir), + buildername="html", + warning=warning_file, + freshenv=True, + parallel=parallel, + ) + app.build() + return app + + +def test_build_autodoc_injects_pyrepl_and_replay_scripts(autodoc_project): + srcdir, outdir, doctreedir = autodoc_project + app = _build(srcdir, outdir, doctreedir) + + html = (outdir / "index.html").read_text(encoding="utf-8") + assert html.count("= 2 + assert 'packages="mockpkg"' in html + assert "no-banner" in html + + replay_dir = outdir / "_static" / "pyrepl" + scripts = sorted(replay_dir.glob("index-*.py")) + assert len(scripts) >= 2 + + for script in scripts: + content = script.read_text(encoding="utf-8") + assert ">>>" not in content + assert "'{base}'" not in content + assert "Traceback" not in content + py_compile.compile(str(script), doraise=True) + + replay_files = json.loads( + app.env.metadata["index"].get("pyrepl-replay-files", "{}") + ) + assert len(replay_files) >= 2 + + +def test_build_autodoc_disabled_by_default(tmp_path): + srcdir = tmp_path / "docs" + pkgdir = tmp_path / "mockpkg" + srcdir.mkdir() + pkgdir.mkdir() + (pkgdir / "__init__.py").write_text( + 'class Name:\n """Example:\n >>> x = 1\n 1\n """\n', + encoding="utf-8", + ) + outdir = tmp_path / "_build" + doctreedir = tmp_path / "_doctree" + (srcdir / "conf.py").write_text( + f"import sys\nsys.path.insert(0, {str(tmp_path)!r})\n" + "extensions = ['sphinx.ext.autodoc', 'sphinx_pyrepl_web']\n" + "pyrepl_js = 'pyrepl.js'\n" + "master_doc = 'index'\n", + encoding="utf-8", + ) + (srcdir / "index.rst").write_text( + ".. autoclass:: mockpkg.Name\n :members:\n", + encoding="utf-8", + ) + app = _build(srcdir, outdir, doctreedir) + + html = (outdir / "index.html").read_text(encoding="utf-8") + assert ">> from naming import Name", + " >>> class MyName(Name):", + " ... config = dict(base=r'\\w+')", + " ...", + " >>> n = MyName()", + " >>> n.get()", + " '{base}'", + " >>> n.values", + " {}", + " >>> n.name = 'hello_world'", + " >>> n", + ' Name("hello_world")', + ] + assert extract_doctest_inputs(lines) == [ + " >>> from naming import Name", + " >>> class MyName(Name):", + " ... config = dict(base=r'\\w+')", + " ...", + " >>> n = MyName()", + " >>> n.get()", + " >>> n.values", + " >>> n.name = 'hello_world'", + " >>> n", + ] + + +def test_pipefile_list_output(): + lines = [ + " >>> [p.get(index=x, output='render') for x in range(10)]", + " ['wipfile.render.7.0.ext',", + " 'wipfile.render.7.1.ext',", + " 'wipfile.render.7.2.ext',", + " 'wipfile.render.7.3.ext',", + " 'wipfile.render.7.4.ext',", + " 'wipfile.render.7.5.ext',", + " 'wipfile.render.7.6.ext',", + " 'wipfile.render.7.7.ext',", + " 'wipfile.render.7.8.ext',", + " 'wipfile.render.7.9.ext']", + ] + assert extract_doctest_inputs(lines) == [ + " >>> [p.get(index=x, output='render') for x in range(10)]", + ] + + +def test_pipefile_traceback_block(): + lines = [ + " >>> pf.year = 'nondigits'", + " Traceback (most recent call last):", + " ...", + " ValueError: Can't set field 'year' with invalid value 'nondigits'", + " >>> pf.year = 1907", + " >>> pf", + ' ProjectFile("project_data_name_1907_christianl_constant_iamlast.data.17.abc")', + ] + assert extract_doctest_inputs(lines) == [ + " >>> pf.year = 'nondigits'", + " >>> pf.year = 1907", + " >>> pf", + ] + + +def test_bare_terminator_preserved(): + lines = [ + ">>> class Foo:", + "... x = 1", + "...", + ">>> Foo()", + ] + assert extract_doctest_inputs(lines) == lines + + +def test_comment_input_line(): + lines = [ + " >>> n.name = 'hello_world'", + " >>> n", + ' Name("hello_world")', + " >>> # modify name and get values from field names", + " >>> n.base = 'through_field_name'", + ] + assert extract_doctest_inputs(lines) == [ + " >>> n.name = 'hello_world'", + " >>> n", + " >>> # modify name and get values from field names", + " >>> n.base = 'through_field_name'", + ] + + +def test_input_only_passthrough(): + lines = [ + ">>> x = 1", + ">>> x + 1", + ] + assert extract_doctest_inputs(lines) == lines + + +def test_empty_block(): + assert extract_doctest_inputs([]) == [] From bbb8ec85de4cd6fe95eab71cf2ff222e7a04f29a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 27 Jun 2026 13:01:34 +0000 Subject: [PATCH 2/3] Revert "Add opt-in autodoc integration for Example doctest blocks (v0.2.0)" This reverts commit bfb17f2ab874a67e501c01e464b83c44564cf7b8. --- README.md | 34 ----- sphinx_pyrepl_web/__init__.py | 35 ++--- sphinx_pyrepl_web/autodoc.py | 214 --------------------------- sphinx_pyrepl_web/doctest.py | 50 ------- tests/test_autodoc.py | 92 ------------ tests/test_build_autodoc_replay.py | 157 -------------------- tests/test_extract_doctest_inputs.py | 103 ------------- 7 files changed, 14 insertions(+), 671 deletions(-) delete mode 100644 sphinx_pyrepl_web/autodoc.py delete mode 100644 sphinx_pyrepl_web/doctest.py delete mode 100644 tests/test_autodoc.py delete mode 100644 tests/test_build_autodoc_replay.py delete mode 100644 tests/test_extract_doctest_inputs.py diff --git a/README.md b/README.md index f8e407d..9196b8c 100644 --- a/README.md +++ b/README.md @@ -70,40 +70,6 @@ Optional Sphinx config: pyrepl_js = "../pyrepl.js" # default; path to the pyrepl-web loader script ``` -## Autodoc integration - -When `sphinx.ext.autodoc` is enabled, you can automatically convert docstring -``Example:`` / ``Examples:`` doctest blocks into interactive REPLs at build time. -Python source docstrings stay unchanged (full doctests with expected output are -preserved for IDE hover docs and optional ``doctest`` runs). - -Add to `conf.py`: - -```python -extensions = [ - "sphinx.ext.autodoc", - "sphinx_pyrepl_web", -] - -pyrepl_autodoc = True -pyrepl_autodoc_packages = "my_package" # required for Pyodide import -pyrepl_autodoc_sections = ["Example"] -pyrepl_autodoc_options = {"no-banner": True} -``` - -| Config | Default | Description | -|--------|---------|-------------| -| `pyrepl_autodoc` | `False` | Enable autodoc docstring → REPL conversion | -| `pyrepl_autodoc_packages` | `None` | Value for `:packages:` on generated directives (auto-derived from object name when unset) | -| `pyrepl_autodoc_sections` | `["Example", "Examples"]` | Section titles to convert | -| `pyrepl_autodoc_options` | `{}` | Default directive flags, e.g. `{"no-banner": True}` | - -**Limitations (v0.2.0):** - -- Targets raw Google-style / plain-text section headers (without `sphinx.ext.napoleon`). Napoleon-converted sections are not detected yet. -- `:packages:` (or auto-derivation) is required so Pyodide can import your library. -- Live REPL output may differ from doctest expected output (e.g. `PosixPath` vs `WindowsPath`, traceback formatting). - ## Updating pyrepl-web Since [chrizzFTD/pyrepl-web](https://github.com/chrizzFTD/pyrepl-web) is a fork, this sphinx extension vendors the JavaScript assets for easier distribution. To update them, run: diff --git a/sphinx_pyrepl_web/__init__.py b/sphinx_pyrepl_web/__init__.py index a15b59c..24ea698 100644 --- a/sphinx_pyrepl_web/__init__.py +++ b/sphinx_pyrepl_web/__init__.py @@ -1,6 +1,6 @@ """A Sphinx extension for embedding pyrepl-web Python REPLs in documentation.""" -__version__ = "0.2.0" +__version__ = "0.1.1" import json from pathlib import Path @@ -23,11 +23,6 @@ def setup(app: Sphinx): app.connect("doctree-read", doctree_read) app.connect("html-page-context", add_html_context) app.connect("env-updated", copy_asset_files) - - from sphinx_pyrepl_web.autodoc import register as register_autodoc - - register_autodoc(app) - return {"version": __version__, "parallel_read_safe": True} @@ -56,20 +51,6 @@ def strip_doctest_prompts(lines: list[str]) -> list[str]: return result -def register_replay_script(env, docname: str, body_lines: list[str]) -> str: - """Strip doctest prompts, store replay script in metadata, return replay-src path.""" - body_text = "\n".join(strip_doctest_prompts(body_lines)) + "\n" - - replay_files = json.loads( - env.metadata[docname].setdefault(REPLAY_FILES_KEY, "{}") - ) - counter = len(replay_files) + 1 - script_name = f"{docname.replace('/', '-')}-{counter}.py" - replay_files[script_name] = body_text - env.metadata[docname][REPLAY_FILES_KEY] = json.dumps(replay_files) - return f"_static/pyrepl/{script_name}" - - class PyRepl(SphinxDirective): """Embed a pyrepl-web ```` element.""" @@ -135,7 +116,19 @@ def run(self): attrs.append(f'src="{rel_src}"') if has_body: - replay_src = register_replay_script(env, env.docname, list(self.content)) + body_lines = strip_doctest_prompts(list(self.content)) + body_text = "\n".join(body_lines) + "\n" + + replay_files = json.loads( + self.env.metadata[self.env.docname].setdefault(REPLAY_FILES_KEY, "{}") + ) + counter = len(replay_files) + 1 + script_name = f"{env.docname.replace('/', '-')}-{counter}.py" + replay_files[script_name] = body_text + self.env.metadata[self.env.docname][REPLAY_FILES_KEY] = json.dumps( + replay_files + ) + replay_src = f"_static/pyrepl/{script_name}" attrs.append(f'replay-src="{replay_src}"') self.env.metadata[self.env.docname]["pyrepl"] = True diff --git a/sphinx_pyrepl_web/autodoc.py b/sphinx_pyrepl_web/autodoc.py deleted file mode 100644 index 34dd173..0000000 --- a/sphinx_pyrepl_web/autodoc.py +++ /dev/null @@ -1,214 +0,0 @@ -"""Autodoc integration: convert docstring Example blocks to py-repl directives.""" - -from __future__ import annotations - -import re -from typing import Any - -from sphinx.application import Sphinx - -from sphinx_pyrepl_web.doctest import _is_doctest_input_line, extract_doctest_inputs - -# Common docstring section headers that end an Example doctest block. -_OTHER_SECTION_HEADERS = frozenset( - { - "Args", - "Arguments", - "Attributes", - "Caution", - "Danger", - "Error", - "Hint", - "Important", - "Keyword Args", - "Keyword Arguments", - "Methods", - "Note", - "Notes", - "Parameters", - "Raises", - "Return", - "Returns", - "See Also", - "Tip", - "Todo", - "Warning", - "Warnings", - "Warns", - "Yield", - "Yields", - } -) - -_DEFAULT_WHAT = frozenset({"class", "function", "method"}) - - -def _section_header_pattern(sections: list[str]) -> re.Pattern[str]: - names = "|".join(re.escape(section) for section in sections) - return re.compile(rf"^\s*({names})\s*:?\s*$") - - -def _matches_configured_section(line: str, pattern: re.Pattern[str]) -> bool: - return bool(pattern.match(line)) - - -def _matches_other_section_header(line: str) -> bool: - stripped = line.lstrip() - if not stripped.endswith(":"): - return False - title = stripped[:-1].strip() - return title in _OTHER_SECTION_HEADERS - - -def _collect_doctest_block( - lines: list[str], start: int, section_pattern: re.Pattern[str] -) -> tuple[list[str], int]: - """Return doctest block lines and index after the block.""" - block: list[str] = [] - block_started = False - i = start - - while i < len(lines): - line = lines[i] - stripped = line.lstrip() - - if stripped == "": - if block_started: - block.append(line) - i += 1 - continue - - if _matches_configured_section(line, section_pattern): - break - - if _matches_other_section_header(stripped): - break - - if _is_doctest_input_line(line): - block_started = True - block.append(line) - i += 1 - continue - - if block_started and (line.startswith(" ") or line.startswith("\t")): - block.append(line) - i += 1 - continue - - break - - return block, i - - -def _has_primary_prompt(lines: list[str]) -> bool: - return any(line.lstrip().startswith(">>>") for line in lines) - - -def format_pyrepl_directive( - input_lines: list[str], options: dict[str, str | bool] -) -> list[str]: - """Format a ``.. py-repl::`` directive block for injection into a docstring.""" - result = [".. py-repl::"] - - for key in sorted(options): - value = options[key] - if value is True: - result.append(f" :{key}:") - elif value: - result.append(f" :{key}: {value}") - - if input_lines: - result.append("") - for line in input_lines: - normalized = line.lstrip() - result.append(f" {normalized}" if normalized else "") - - return result - - -def transform_docstring_lines( - lines: list[str], - sections: list[str], - options: dict[str, str | bool], -) -> None: - """Replace configured Example sections with ``.. py-repl::`` directives in place.""" - section_pattern = _section_header_pattern(sections) - i = 0 - - while i < len(lines): - line = lines[i] - if not _matches_configured_section(line, section_pattern): - i += 1 - continue - - header_line = line - block, next_i = _collect_doctest_block(lines, i + 1, section_pattern) - - if not block or not _has_primary_prompt(block): - i = next_i - continue - - input_lines = extract_doctest_inputs(block) - if not input_lines: - i = next_i - continue - - replacement = [header_line, ""] + format_pyrepl_directive(input_lines, options) - lines[i:next_i] = replacement - i += len(replacement) - - -def derive_packages(qualified_name: str) -> str: - """Derive a PyPI/import package name from a documented object's qualified name.""" - if "." in qualified_name: - return qualified_name.rsplit(".", 1)[0] - return qualified_name - - -def process_autodoc_docstring( - app: Sphinx, - what: str, - name: str, - obj: Any, - options: Any, - lines: list[str], -) -> None: - """Convert matching docstring Example blocks into ``.. py-repl::`` directives.""" - if what not in _DEFAULT_WHAT: - return - - sections: list[str] = app.config.pyrepl_autodoc_sections - if not sections: - return - - directive_options: dict[str, str | bool] = dict(app.config.pyrepl_autodoc_options) - packages = app.config.pyrepl_autodoc_packages - if packages is None: - packages = derive_packages(name) - if packages: - directive_options["packages"] = packages - - transform_docstring_lines(lines, sections, directive_options) - - -def register(app: Sphinx) -> None: - """Register autodoc config values and hook when enabled.""" - app.add_config_value("pyrepl_autodoc", False, "env") - app.add_config_value("pyrepl_autodoc_packages", None, "env") - app.add_config_value("pyrepl_autodoc_sections", ["Example", "Examples"], "env") - app.add_config_value("pyrepl_autodoc_options", {}, "env") - - if not app.config.pyrepl_autodoc: - return - - if "sphinx.ext.autodoc" not in app.config.extensions: - from sphinx.util import logging - - logger = logging.getLogger(__name__) - logger.warning( - "pyrepl_autodoc is enabled but sphinx.ext.autodoc is not loaded; " - "autodoc integration will have no effect." - ) - return - - app.connect("autodoc-process-docstring", process_autodoc_docstring) diff --git a/sphinx_pyrepl_web/doctest.py b/sphinx_pyrepl_web/doctest.py deleted file mode 100644 index 8e6becb..0000000 --- a/sphinx_pyrepl_web/doctest.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Utilities for parsing doctest blocks in docstrings.""" - - -def _is_doctest_input_line(line: str) -> bool: - """Return True if *line* is a doctest input or continuation prompt.""" - stripped = line.lstrip() - if stripped.startswith(">>> "): - return True - if stripped == ">>>": - return True - if stripped.startswith("..."): - return True - return False - - -def extract_doctest_inputs(lines: list[str]) -> list[str]: - """Return input-only doctest lines, dropping expected output and tracebacks. - - Keeps lines starting with ``>>>`` or ``...`` (including bare ``...`` block - terminators). Drops expected output lines and traceback blocks (from - ``Traceback`` through the error message). - """ - result: list[str] = [] - in_traceback = False - in_block = False - - for line in lines: - stripped = line.lstrip() - - if stripped.startswith("Traceback"): - in_traceback = True - continue - - if in_traceback: - # Only resume on a new primary prompt; traceback ellipsis uses ``...``. - if stripped.startswith(">>> "): - in_traceback = False - else: - continue - - if _is_doctest_input_line(line): - in_block = True - result.append(line) - elif in_block and stripped == "": - result.append(line) - else: - # Expected output or trailing non-doctest content. - pass - - return result diff --git a/tests/test_autodoc.py b/tests/test_autodoc.py deleted file mode 100644 index 2a9ccd8..0000000 --- a/tests/test_autodoc.py +++ /dev/null @@ -1,92 +0,0 @@ -from sphinx_pyrepl_web.autodoc import ( - derive_packages, - format_pyrepl_directive, - transform_docstring_lines, -) - - -def test_format_pyrepl_directive_options_and_body(): - result = format_pyrepl_directive( - [" >>> x = 1", " >>> x + 1"], - {"no-banner": True, "packages": "naming"}, - ) - assert result == [ - ".. py-repl::", - " :no-banner:", - " :packages: naming", - "", - " >>> x = 1", - " >>> x + 1", - ] - - -def test_transform_docstring_injects_pyrepl(): - lines = [ - "Base class for name objects.", - "", - "Example:", - " >>> from naming import Name", - " >>> n = MyName()", - " >>> n.get()", - " '{base}'", - ] - transform_docstring_lines( - lines, - sections=["Example"], - options={"no-banner": True, "packages": "naming"}, - ) - assert lines[2] == "Example:" - assert lines[3] == "" - assert lines[4] == ".. py-repl::" - assert " :packages: naming" in lines - assert " >>> from naming import Name" in lines - assert "'{base}'" not in lines - - -def test_transform_docstring_leaves_non_example_unchanged(): - lines = [ - "A helper function.", - "", - "Returns:", - " A string value.", - ] - original = list(lines) - transform_docstring_lines(lines, sections=["Example"], options={}) - assert lines == original - - -def test_transform_docstring_skips_example_without_doctest(): - lines = [ - "Example:", - "Use this class for naming things.", - ] - original = list(lines) - transform_docstring_lines(lines, sections=["Example"], options={}) - assert lines == original - - -def test_transform_docstring_pipefile_traceback(): - lines = [ - "Example:", - " >>> pf.year = 'nondigits'", - " Traceback (most recent call last):", - " ...", - " ValueError: invalid", - " >>> pf.year = 1907", - ] - transform_docstring_lines( - lines, - sections=["Example"], - options={"packages": "naming", "no-banner": True}, - ) - body = "\n".join(lines) - assert ".. py-repl::" in body - assert "Traceback" not in body - assert "ValueError" not in body - assert " >>> pf.year = 'nondigits'" in lines - assert " >>> pf.year = 1907" in lines - - -def test_derive_packages(): - assert derive_packages("naming.Name") == "naming" - assert derive_packages("naming") == "naming" diff --git a/tests/test_build_autodoc_replay.py b/tests/test_build_autodoc_replay.py deleted file mode 100644 index 707721b..0000000 --- a/tests/test_build_autodoc_replay.py +++ /dev/null @@ -1,157 +0,0 @@ -import json -import py_compile -import sys -from pathlib import Path - -import pytest -from sphinx.application import Sphinx - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT)) - - -@pytest.fixture -def autodoc_project(tmp_path): - srcdir = tmp_path / "docs" - pkgdir = tmp_path / "mockpkg" - srcdir.mkdir() - pkgdir.mkdir() - (pkgdir / "__init__.py").write_text( - r''' -class Name: - r"""Base class for name objects. - - Example: - >>> from naming import Name - >>> class MyName(Name): - ... config = dict(base=r'\w+') - ... - >>> n = MyName() - >>> n.get() - '{base}' - >>> n.name = 'hello_world' - >>> n - Name("hello_world") - """ - -class PipeFile: - """Pipeline file example. - - Example: - >>> from naming import PipeFile - >>> pf = PipeFile('wipfile.7.ext') - >>> pf.values - {'base': 'wipfile', 'version': '7'} - >>> pf.year = 'nondigits' - Traceback (most recent call last): - ... - ValueError: invalid year - >>> pf.year = 1907 - """ -'''.strip() - + "\n", - encoding="utf-8", - ) - - outdir = tmp_path / "_build" - doctreedir = tmp_path / "_doctree" - (srcdir / "conf.py").write_text( - f"import sys\n" - f"sys.path.insert(0, {str(tmp_path)!r})\n" - "extensions = ['sphinx.ext.autodoc', 'sphinx_pyrepl_web']\n" - "pyrepl_js = 'pyrepl.js'\n" - "master_doc = 'index'\n" - "pyrepl_autodoc = True\n" - "pyrepl_autodoc_packages = 'mockpkg'\n" - "pyrepl_autodoc_sections = ['Example']\n" - "pyrepl_autodoc_options = {'no-banner': True}\n", - encoding="utf-8", - ) - (srcdir / "index.rst").write_text( - """ -API -=== - -.. autoclass:: mockpkg.Name - :members: - -.. autoclass:: mockpkg.PipeFile - :members: -""".strip() - + "\n", - encoding="utf-8", - ) - return srcdir, outdir, doctreedir - - -def _build(srcdir, outdir, doctreedir, parallel=0): - outdir.mkdir(parents=True, exist_ok=True) - doctreedir.mkdir(parents=True, exist_ok=True) - with open(outdir / "warnings.txt", "w", encoding="utf-8") as warning_file: - app = Sphinx( - srcdir=str(srcdir), - confdir=str(srcdir), - outdir=str(outdir), - doctreedir=str(doctreedir), - buildername="html", - warning=warning_file, - freshenv=True, - parallel=parallel, - ) - app.build() - return app - - -def test_build_autodoc_injects_pyrepl_and_replay_scripts(autodoc_project): - srcdir, outdir, doctreedir = autodoc_project - app = _build(srcdir, outdir, doctreedir) - - html = (outdir / "index.html").read_text(encoding="utf-8") - assert html.count("= 2 - assert 'packages="mockpkg"' in html - assert "no-banner" in html - - replay_dir = outdir / "_static" / "pyrepl" - scripts = sorted(replay_dir.glob("index-*.py")) - assert len(scripts) >= 2 - - for script in scripts: - content = script.read_text(encoding="utf-8") - assert ">>>" not in content - assert "'{base}'" not in content - assert "Traceback" not in content - py_compile.compile(str(script), doraise=True) - - replay_files = json.loads( - app.env.metadata["index"].get("pyrepl-replay-files", "{}") - ) - assert len(replay_files) >= 2 - - -def test_build_autodoc_disabled_by_default(tmp_path): - srcdir = tmp_path / "docs" - pkgdir = tmp_path / "mockpkg" - srcdir.mkdir() - pkgdir.mkdir() - (pkgdir / "__init__.py").write_text( - 'class Name:\n """Example:\n >>> x = 1\n 1\n """\n', - encoding="utf-8", - ) - outdir = tmp_path / "_build" - doctreedir = tmp_path / "_doctree" - (srcdir / "conf.py").write_text( - f"import sys\nsys.path.insert(0, {str(tmp_path)!r})\n" - "extensions = ['sphinx.ext.autodoc', 'sphinx_pyrepl_web']\n" - "pyrepl_js = 'pyrepl.js'\n" - "master_doc = 'index'\n", - encoding="utf-8", - ) - (srcdir / "index.rst").write_text( - ".. autoclass:: mockpkg.Name\n :members:\n", - encoding="utf-8", - ) - app = _build(srcdir, outdir, doctreedir) - - html = (outdir / "index.html").read_text(encoding="utf-8") - assert ">> from naming import Name", - " >>> class MyName(Name):", - " ... config = dict(base=r'\\w+')", - " ...", - " >>> n = MyName()", - " >>> n.get()", - " '{base}'", - " >>> n.values", - " {}", - " >>> n.name = 'hello_world'", - " >>> n", - ' Name("hello_world")', - ] - assert extract_doctest_inputs(lines) == [ - " >>> from naming import Name", - " >>> class MyName(Name):", - " ... config = dict(base=r'\\w+')", - " ...", - " >>> n = MyName()", - " >>> n.get()", - " >>> n.values", - " >>> n.name = 'hello_world'", - " >>> n", - ] - - -def test_pipefile_list_output(): - lines = [ - " >>> [p.get(index=x, output='render') for x in range(10)]", - " ['wipfile.render.7.0.ext',", - " 'wipfile.render.7.1.ext',", - " 'wipfile.render.7.2.ext',", - " 'wipfile.render.7.3.ext',", - " 'wipfile.render.7.4.ext',", - " 'wipfile.render.7.5.ext',", - " 'wipfile.render.7.6.ext',", - " 'wipfile.render.7.7.ext',", - " 'wipfile.render.7.8.ext',", - " 'wipfile.render.7.9.ext']", - ] - assert extract_doctest_inputs(lines) == [ - " >>> [p.get(index=x, output='render') for x in range(10)]", - ] - - -def test_pipefile_traceback_block(): - lines = [ - " >>> pf.year = 'nondigits'", - " Traceback (most recent call last):", - " ...", - " ValueError: Can't set field 'year' with invalid value 'nondigits'", - " >>> pf.year = 1907", - " >>> pf", - ' ProjectFile("project_data_name_1907_christianl_constant_iamlast.data.17.abc")', - ] - assert extract_doctest_inputs(lines) == [ - " >>> pf.year = 'nondigits'", - " >>> pf.year = 1907", - " >>> pf", - ] - - -def test_bare_terminator_preserved(): - lines = [ - ">>> class Foo:", - "... x = 1", - "...", - ">>> Foo()", - ] - assert extract_doctest_inputs(lines) == lines - - -def test_comment_input_line(): - lines = [ - " >>> n.name = 'hello_world'", - " >>> n", - ' Name("hello_world")', - " >>> # modify name and get values from field names", - " >>> n.base = 'through_field_name'", - ] - assert extract_doctest_inputs(lines) == [ - " >>> n.name = 'hello_world'", - " >>> n", - " >>> # modify name and get values from field names", - " >>> n.base = 'through_field_name'", - ] - - -def test_input_only_passthrough(): - lines = [ - ">>> x = 1", - ">>> x + 1", - ] - assert extract_doctest_inputs(lines) == lines - - -def test_empty_block(): - assert extract_doctest_inputs([]) == [] From 08dd88ff3b453dd1bfcb81a7acfb50a15512bfba Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 27 Jun 2026 13:01:56 +0000 Subject: [PATCH 3/3] Reapply "Add opt-in autodoc integration for Example doctest blocks (v0.2.0)" This reverts commit bbb8ec85de4cd6fe95eab71cf2ff222e7a04f29a. --- README.md | 34 +++++ sphinx_pyrepl_web/__init__.py | 35 +++-- sphinx_pyrepl_web/autodoc.py | 214 +++++++++++++++++++++++++++ sphinx_pyrepl_web/doctest.py | 50 +++++++ tests/test_autodoc.py | 92 ++++++++++++ tests/test_build_autodoc_replay.py | 157 ++++++++++++++++++++ tests/test_extract_doctest_inputs.py | 103 +++++++++++++ 7 files changed, 671 insertions(+), 14 deletions(-) create mode 100644 sphinx_pyrepl_web/autodoc.py create mode 100644 sphinx_pyrepl_web/doctest.py create mode 100644 tests/test_autodoc.py create mode 100644 tests/test_build_autodoc_replay.py create mode 100644 tests/test_extract_doctest_inputs.py diff --git a/README.md b/README.md index 9196b8c..f8e407d 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,40 @@ Optional Sphinx config: pyrepl_js = "../pyrepl.js" # default; path to the pyrepl-web loader script ``` +## Autodoc integration + +When `sphinx.ext.autodoc` is enabled, you can automatically convert docstring +``Example:`` / ``Examples:`` doctest blocks into interactive REPLs at build time. +Python source docstrings stay unchanged (full doctests with expected output are +preserved for IDE hover docs and optional ``doctest`` runs). + +Add to `conf.py`: + +```python +extensions = [ + "sphinx.ext.autodoc", + "sphinx_pyrepl_web", +] + +pyrepl_autodoc = True +pyrepl_autodoc_packages = "my_package" # required for Pyodide import +pyrepl_autodoc_sections = ["Example"] +pyrepl_autodoc_options = {"no-banner": True} +``` + +| Config | Default | Description | +|--------|---------|-------------| +| `pyrepl_autodoc` | `False` | Enable autodoc docstring → REPL conversion | +| `pyrepl_autodoc_packages` | `None` | Value for `:packages:` on generated directives (auto-derived from object name when unset) | +| `pyrepl_autodoc_sections` | `["Example", "Examples"]` | Section titles to convert | +| `pyrepl_autodoc_options` | `{}` | Default directive flags, e.g. `{"no-banner": True}` | + +**Limitations (v0.2.0):** + +- Targets raw Google-style / plain-text section headers (without `sphinx.ext.napoleon`). Napoleon-converted sections are not detected yet. +- `:packages:` (or auto-derivation) is required so Pyodide can import your library. +- Live REPL output may differ from doctest expected output (e.g. `PosixPath` vs `WindowsPath`, traceback formatting). + ## Updating pyrepl-web Since [chrizzFTD/pyrepl-web](https://github.com/chrizzFTD/pyrepl-web) is a fork, this sphinx extension vendors the JavaScript assets for easier distribution. To update them, run: diff --git a/sphinx_pyrepl_web/__init__.py b/sphinx_pyrepl_web/__init__.py index 24ea698..a15b59c 100644 --- a/sphinx_pyrepl_web/__init__.py +++ b/sphinx_pyrepl_web/__init__.py @@ -1,6 +1,6 @@ """A Sphinx extension for embedding pyrepl-web Python REPLs in documentation.""" -__version__ = "0.1.1" +__version__ = "0.2.0" import json from pathlib import Path @@ -23,6 +23,11 @@ def setup(app: Sphinx): app.connect("doctree-read", doctree_read) app.connect("html-page-context", add_html_context) app.connect("env-updated", copy_asset_files) + + from sphinx_pyrepl_web.autodoc import register as register_autodoc + + register_autodoc(app) + return {"version": __version__, "parallel_read_safe": True} @@ -51,6 +56,20 @@ def strip_doctest_prompts(lines: list[str]) -> list[str]: return result +def register_replay_script(env, docname: str, body_lines: list[str]) -> str: + """Strip doctest prompts, store replay script in metadata, return replay-src path.""" + body_text = "\n".join(strip_doctest_prompts(body_lines)) + "\n" + + replay_files = json.loads( + env.metadata[docname].setdefault(REPLAY_FILES_KEY, "{}") + ) + counter = len(replay_files) + 1 + script_name = f"{docname.replace('/', '-')}-{counter}.py" + replay_files[script_name] = body_text + env.metadata[docname][REPLAY_FILES_KEY] = json.dumps(replay_files) + return f"_static/pyrepl/{script_name}" + + class PyRepl(SphinxDirective): """Embed a pyrepl-web ```` element.""" @@ -116,19 +135,7 @@ def run(self): attrs.append(f'src="{rel_src}"') if has_body: - body_lines = strip_doctest_prompts(list(self.content)) - body_text = "\n".join(body_lines) + "\n" - - replay_files = json.loads( - self.env.metadata[self.env.docname].setdefault(REPLAY_FILES_KEY, "{}") - ) - counter = len(replay_files) + 1 - script_name = f"{env.docname.replace('/', '-')}-{counter}.py" - replay_files[script_name] = body_text - self.env.metadata[self.env.docname][REPLAY_FILES_KEY] = json.dumps( - replay_files - ) - replay_src = f"_static/pyrepl/{script_name}" + replay_src = register_replay_script(env, env.docname, list(self.content)) attrs.append(f'replay-src="{replay_src}"') self.env.metadata[self.env.docname]["pyrepl"] = True diff --git a/sphinx_pyrepl_web/autodoc.py b/sphinx_pyrepl_web/autodoc.py new file mode 100644 index 0000000..34dd173 --- /dev/null +++ b/sphinx_pyrepl_web/autodoc.py @@ -0,0 +1,214 @@ +"""Autodoc integration: convert docstring Example blocks to py-repl directives.""" + +from __future__ import annotations + +import re +from typing import Any + +from sphinx.application import Sphinx + +from sphinx_pyrepl_web.doctest import _is_doctest_input_line, extract_doctest_inputs + +# Common docstring section headers that end an Example doctest block. +_OTHER_SECTION_HEADERS = frozenset( + { + "Args", + "Arguments", + "Attributes", + "Caution", + "Danger", + "Error", + "Hint", + "Important", + "Keyword Args", + "Keyword Arguments", + "Methods", + "Note", + "Notes", + "Parameters", + "Raises", + "Return", + "Returns", + "See Also", + "Tip", + "Todo", + "Warning", + "Warnings", + "Warns", + "Yield", + "Yields", + } +) + +_DEFAULT_WHAT = frozenset({"class", "function", "method"}) + + +def _section_header_pattern(sections: list[str]) -> re.Pattern[str]: + names = "|".join(re.escape(section) for section in sections) + return re.compile(rf"^\s*({names})\s*:?\s*$") + + +def _matches_configured_section(line: str, pattern: re.Pattern[str]) -> bool: + return bool(pattern.match(line)) + + +def _matches_other_section_header(line: str) -> bool: + stripped = line.lstrip() + if not stripped.endswith(":"): + return False + title = stripped[:-1].strip() + return title in _OTHER_SECTION_HEADERS + + +def _collect_doctest_block( + lines: list[str], start: int, section_pattern: re.Pattern[str] +) -> tuple[list[str], int]: + """Return doctest block lines and index after the block.""" + block: list[str] = [] + block_started = False + i = start + + while i < len(lines): + line = lines[i] + stripped = line.lstrip() + + if stripped == "": + if block_started: + block.append(line) + i += 1 + continue + + if _matches_configured_section(line, section_pattern): + break + + if _matches_other_section_header(stripped): + break + + if _is_doctest_input_line(line): + block_started = True + block.append(line) + i += 1 + continue + + if block_started and (line.startswith(" ") or line.startswith("\t")): + block.append(line) + i += 1 + continue + + break + + return block, i + + +def _has_primary_prompt(lines: list[str]) -> bool: + return any(line.lstrip().startswith(">>>") for line in lines) + + +def format_pyrepl_directive( + input_lines: list[str], options: dict[str, str | bool] +) -> list[str]: + """Format a ``.. py-repl::`` directive block for injection into a docstring.""" + result = [".. py-repl::"] + + for key in sorted(options): + value = options[key] + if value is True: + result.append(f" :{key}:") + elif value: + result.append(f" :{key}: {value}") + + if input_lines: + result.append("") + for line in input_lines: + normalized = line.lstrip() + result.append(f" {normalized}" if normalized else "") + + return result + + +def transform_docstring_lines( + lines: list[str], + sections: list[str], + options: dict[str, str | bool], +) -> None: + """Replace configured Example sections with ``.. py-repl::`` directives in place.""" + section_pattern = _section_header_pattern(sections) + i = 0 + + while i < len(lines): + line = lines[i] + if not _matches_configured_section(line, section_pattern): + i += 1 + continue + + header_line = line + block, next_i = _collect_doctest_block(lines, i + 1, section_pattern) + + if not block or not _has_primary_prompt(block): + i = next_i + continue + + input_lines = extract_doctest_inputs(block) + if not input_lines: + i = next_i + continue + + replacement = [header_line, ""] + format_pyrepl_directive(input_lines, options) + lines[i:next_i] = replacement + i += len(replacement) + + +def derive_packages(qualified_name: str) -> str: + """Derive a PyPI/import package name from a documented object's qualified name.""" + if "." in qualified_name: + return qualified_name.rsplit(".", 1)[0] + return qualified_name + + +def process_autodoc_docstring( + app: Sphinx, + what: str, + name: str, + obj: Any, + options: Any, + lines: list[str], +) -> None: + """Convert matching docstring Example blocks into ``.. py-repl::`` directives.""" + if what not in _DEFAULT_WHAT: + return + + sections: list[str] = app.config.pyrepl_autodoc_sections + if not sections: + return + + directive_options: dict[str, str | bool] = dict(app.config.pyrepl_autodoc_options) + packages = app.config.pyrepl_autodoc_packages + if packages is None: + packages = derive_packages(name) + if packages: + directive_options["packages"] = packages + + transform_docstring_lines(lines, sections, directive_options) + + +def register(app: Sphinx) -> None: + """Register autodoc config values and hook when enabled.""" + app.add_config_value("pyrepl_autodoc", False, "env") + app.add_config_value("pyrepl_autodoc_packages", None, "env") + app.add_config_value("pyrepl_autodoc_sections", ["Example", "Examples"], "env") + app.add_config_value("pyrepl_autodoc_options", {}, "env") + + if not app.config.pyrepl_autodoc: + return + + if "sphinx.ext.autodoc" not in app.config.extensions: + from sphinx.util import logging + + logger = logging.getLogger(__name__) + logger.warning( + "pyrepl_autodoc is enabled but sphinx.ext.autodoc is not loaded; " + "autodoc integration will have no effect." + ) + return + + app.connect("autodoc-process-docstring", process_autodoc_docstring) diff --git a/sphinx_pyrepl_web/doctest.py b/sphinx_pyrepl_web/doctest.py new file mode 100644 index 0000000..8e6becb --- /dev/null +++ b/sphinx_pyrepl_web/doctest.py @@ -0,0 +1,50 @@ +"""Utilities for parsing doctest blocks in docstrings.""" + + +def _is_doctest_input_line(line: str) -> bool: + """Return True if *line* is a doctest input or continuation prompt.""" + stripped = line.lstrip() + if stripped.startswith(">>> "): + return True + if stripped == ">>>": + return True + if stripped.startswith("..."): + return True + return False + + +def extract_doctest_inputs(lines: list[str]) -> list[str]: + """Return input-only doctest lines, dropping expected output and tracebacks. + + Keeps lines starting with ``>>>`` or ``...`` (including bare ``...`` block + terminators). Drops expected output lines and traceback blocks (from + ``Traceback`` through the error message). + """ + result: list[str] = [] + in_traceback = False + in_block = False + + for line in lines: + stripped = line.lstrip() + + if stripped.startswith("Traceback"): + in_traceback = True + continue + + if in_traceback: + # Only resume on a new primary prompt; traceback ellipsis uses ``...``. + if stripped.startswith(">>> "): + in_traceback = False + else: + continue + + if _is_doctest_input_line(line): + in_block = True + result.append(line) + elif in_block and stripped == "": + result.append(line) + else: + # Expected output or trailing non-doctest content. + pass + + return result diff --git a/tests/test_autodoc.py b/tests/test_autodoc.py new file mode 100644 index 0000000..2a9ccd8 --- /dev/null +++ b/tests/test_autodoc.py @@ -0,0 +1,92 @@ +from sphinx_pyrepl_web.autodoc import ( + derive_packages, + format_pyrepl_directive, + transform_docstring_lines, +) + + +def test_format_pyrepl_directive_options_and_body(): + result = format_pyrepl_directive( + [" >>> x = 1", " >>> x + 1"], + {"no-banner": True, "packages": "naming"}, + ) + assert result == [ + ".. py-repl::", + " :no-banner:", + " :packages: naming", + "", + " >>> x = 1", + " >>> x + 1", + ] + + +def test_transform_docstring_injects_pyrepl(): + lines = [ + "Base class for name objects.", + "", + "Example:", + " >>> from naming import Name", + " >>> n = MyName()", + " >>> n.get()", + " '{base}'", + ] + transform_docstring_lines( + lines, + sections=["Example"], + options={"no-banner": True, "packages": "naming"}, + ) + assert lines[2] == "Example:" + assert lines[3] == "" + assert lines[4] == ".. py-repl::" + assert " :packages: naming" in lines + assert " >>> from naming import Name" in lines + assert "'{base}'" not in lines + + +def test_transform_docstring_leaves_non_example_unchanged(): + lines = [ + "A helper function.", + "", + "Returns:", + " A string value.", + ] + original = list(lines) + transform_docstring_lines(lines, sections=["Example"], options={}) + assert lines == original + + +def test_transform_docstring_skips_example_without_doctest(): + lines = [ + "Example:", + "Use this class for naming things.", + ] + original = list(lines) + transform_docstring_lines(lines, sections=["Example"], options={}) + assert lines == original + + +def test_transform_docstring_pipefile_traceback(): + lines = [ + "Example:", + " >>> pf.year = 'nondigits'", + " Traceback (most recent call last):", + " ...", + " ValueError: invalid", + " >>> pf.year = 1907", + ] + transform_docstring_lines( + lines, + sections=["Example"], + options={"packages": "naming", "no-banner": True}, + ) + body = "\n".join(lines) + assert ".. py-repl::" in body + assert "Traceback" not in body + assert "ValueError" not in body + assert " >>> pf.year = 'nondigits'" in lines + assert " >>> pf.year = 1907" in lines + + +def test_derive_packages(): + assert derive_packages("naming.Name") == "naming" + assert derive_packages("naming") == "naming" diff --git a/tests/test_build_autodoc_replay.py b/tests/test_build_autodoc_replay.py new file mode 100644 index 0000000..707721b --- /dev/null +++ b/tests/test_build_autodoc_replay.py @@ -0,0 +1,157 @@ +import json +import py_compile +import sys +from pathlib import Path + +import pytest +from sphinx.application import Sphinx + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + + +@pytest.fixture +def autodoc_project(tmp_path): + srcdir = tmp_path / "docs" + pkgdir = tmp_path / "mockpkg" + srcdir.mkdir() + pkgdir.mkdir() + (pkgdir / "__init__.py").write_text( + r''' +class Name: + r"""Base class for name objects. + + Example: + >>> from naming import Name + >>> class MyName(Name): + ... config = dict(base=r'\w+') + ... + >>> n = MyName() + >>> n.get() + '{base}' + >>> n.name = 'hello_world' + >>> n + Name("hello_world") + """ + +class PipeFile: + """Pipeline file example. + + Example: + >>> from naming import PipeFile + >>> pf = PipeFile('wipfile.7.ext') + >>> pf.values + {'base': 'wipfile', 'version': '7'} + >>> pf.year = 'nondigits' + Traceback (most recent call last): + ... + ValueError: invalid year + >>> pf.year = 1907 + """ +'''.strip() + + "\n", + encoding="utf-8", + ) + + outdir = tmp_path / "_build" + doctreedir = tmp_path / "_doctree" + (srcdir / "conf.py").write_text( + f"import sys\n" + f"sys.path.insert(0, {str(tmp_path)!r})\n" + "extensions = ['sphinx.ext.autodoc', 'sphinx_pyrepl_web']\n" + "pyrepl_js = 'pyrepl.js'\n" + "master_doc = 'index'\n" + "pyrepl_autodoc = True\n" + "pyrepl_autodoc_packages = 'mockpkg'\n" + "pyrepl_autodoc_sections = ['Example']\n" + "pyrepl_autodoc_options = {'no-banner': True}\n", + encoding="utf-8", + ) + (srcdir / "index.rst").write_text( + """ +API +=== + +.. autoclass:: mockpkg.Name + :members: + +.. autoclass:: mockpkg.PipeFile + :members: +""".strip() + + "\n", + encoding="utf-8", + ) + return srcdir, outdir, doctreedir + + +def _build(srcdir, outdir, doctreedir, parallel=0): + outdir.mkdir(parents=True, exist_ok=True) + doctreedir.mkdir(parents=True, exist_ok=True) + with open(outdir / "warnings.txt", "w", encoding="utf-8") as warning_file: + app = Sphinx( + srcdir=str(srcdir), + confdir=str(srcdir), + outdir=str(outdir), + doctreedir=str(doctreedir), + buildername="html", + warning=warning_file, + freshenv=True, + parallel=parallel, + ) + app.build() + return app + + +def test_build_autodoc_injects_pyrepl_and_replay_scripts(autodoc_project): + srcdir, outdir, doctreedir = autodoc_project + app = _build(srcdir, outdir, doctreedir) + + html = (outdir / "index.html").read_text(encoding="utf-8") + assert html.count("= 2 + assert 'packages="mockpkg"' in html + assert "no-banner" in html + + replay_dir = outdir / "_static" / "pyrepl" + scripts = sorted(replay_dir.glob("index-*.py")) + assert len(scripts) >= 2 + + for script in scripts: + content = script.read_text(encoding="utf-8") + assert ">>>" not in content + assert "'{base}'" not in content + assert "Traceback" not in content + py_compile.compile(str(script), doraise=True) + + replay_files = json.loads( + app.env.metadata["index"].get("pyrepl-replay-files", "{}") + ) + assert len(replay_files) >= 2 + + +def test_build_autodoc_disabled_by_default(tmp_path): + srcdir = tmp_path / "docs" + pkgdir = tmp_path / "mockpkg" + srcdir.mkdir() + pkgdir.mkdir() + (pkgdir / "__init__.py").write_text( + 'class Name:\n """Example:\n >>> x = 1\n 1\n """\n', + encoding="utf-8", + ) + outdir = tmp_path / "_build" + doctreedir = tmp_path / "_doctree" + (srcdir / "conf.py").write_text( + f"import sys\nsys.path.insert(0, {str(tmp_path)!r})\n" + "extensions = ['sphinx.ext.autodoc', 'sphinx_pyrepl_web']\n" + "pyrepl_js = 'pyrepl.js'\n" + "master_doc = 'index'\n", + encoding="utf-8", + ) + (srcdir / "index.rst").write_text( + ".. autoclass:: mockpkg.Name\n :members:\n", + encoding="utf-8", + ) + app = _build(srcdir, outdir, doctreedir) + + html = (outdir / "index.html").read_text(encoding="utf-8") + assert ">> from naming import Name", + " >>> class MyName(Name):", + " ... config = dict(base=r'\\w+')", + " ...", + " >>> n = MyName()", + " >>> n.get()", + " '{base}'", + " >>> n.values", + " {}", + " >>> n.name = 'hello_world'", + " >>> n", + ' Name("hello_world")', + ] + assert extract_doctest_inputs(lines) == [ + " >>> from naming import Name", + " >>> class MyName(Name):", + " ... config = dict(base=r'\\w+')", + " ...", + " >>> n = MyName()", + " >>> n.get()", + " >>> n.values", + " >>> n.name = 'hello_world'", + " >>> n", + ] + + +def test_pipefile_list_output(): + lines = [ + " >>> [p.get(index=x, output='render') for x in range(10)]", + " ['wipfile.render.7.0.ext',", + " 'wipfile.render.7.1.ext',", + " 'wipfile.render.7.2.ext',", + " 'wipfile.render.7.3.ext',", + " 'wipfile.render.7.4.ext',", + " 'wipfile.render.7.5.ext',", + " 'wipfile.render.7.6.ext',", + " 'wipfile.render.7.7.ext',", + " 'wipfile.render.7.8.ext',", + " 'wipfile.render.7.9.ext']", + ] + assert extract_doctest_inputs(lines) == [ + " >>> [p.get(index=x, output='render') for x in range(10)]", + ] + + +def test_pipefile_traceback_block(): + lines = [ + " >>> pf.year = 'nondigits'", + " Traceback (most recent call last):", + " ...", + " ValueError: Can't set field 'year' with invalid value 'nondigits'", + " >>> pf.year = 1907", + " >>> pf", + ' ProjectFile("project_data_name_1907_christianl_constant_iamlast.data.17.abc")', + ] + assert extract_doctest_inputs(lines) == [ + " >>> pf.year = 'nondigits'", + " >>> pf.year = 1907", + " >>> pf", + ] + + +def test_bare_terminator_preserved(): + lines = [ + ">>> class Foo:", + "... x = 1", + "...", + ">>> Foo()", + ] + assert extract_doctest_inputs(lines) == lines + + +def test_comment_input_line(): + lines = [ + " >>> n.name = 'hello_world'", + " >>> n", + ' Name("hello_world")', + " >>> # modify name and get values from field names", + " >>> n.base = 'through_field_name'", + ] + assert extract_doctest_inputs(lines) == [ + " >>> n.name = 'hello_world'", + " >>> n", + " >>> # modify name and get values from field names", + " >>> n.base = 'through_field_name'", + ] + + +def test_input_only_passthrough(): + lines = [ + ">>> x = 1", + ">>> x + 1", + ] + assert extract_doctest_inputs(lines) == lines + + +def test_empty_block(): + assert extract_doctest_inputs([]) == []