diff --git a/.gitignore b/.gitignore index e707d28..5d09aa7 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ __pycache__/ .venv-build-test/ .venv-testpypi/ docs/_build/ +docs/_static/wheels/*.whl dist/ build/ *.egg diff --git a/README.md b/README.md index d0c13f4..28c33f2 100644 --- a/README.md +++ b/README.md @@ -76,11 +76,19 @@ Enable [doctest style examples](https://docs.python.org/3/library/doctest.html) | Value | Outcome | |-------------------------|--------------------------------------------------------------------| | `None` (default) | Replay doctest input without preloading packages | +| `:project:` | Build a wheel from the documented project and preload it | | Wheel path / PyPI names | Install the package and import the documented object before replay | +Optional when using `:project:`: + +| Value | Default | Outcome | +|-------|---------|---------| +| `pyrepl_project_root` | auto-detect | Project root containing `pyproject.toml` (relative to `conf.py`) | +| `pyrepl_wheel_dir` | `_static/wheels` | Directory for the built wheel, relative to `conf.py` | + ### Local wheels -Unreleased package wheels can be available in the REPL by building them under Sphinx's `html_static_path`. +Unreleased package wheels can be available in the REPL by building them under Sphinx's `html_static_path`, or by using `:project:` to build automatically at doc-build time. All options combined: @@ -93,5 +101,12 @@ extensions = [ html_static_path = ["_static"] pyrepl_doctest_blocks = "autodoc" -pyrepl_autodoc_packages = "_static/wheels/my_package-1.0.0-py3-none-any.whl" +pyrepl_autodoc_packages = ":project:" +``` + +For a monorepo or non-default layout, point at the package root explicitly: + +```python +pyrepl_autodoc_packages = ":project:" +pyrepl_project_root = ".." ``` diff --git a/docs/conf.py b/docs/conf.py index 9980427..a51e2ca 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,7 +14,8 @@ "sphinx_pyrepl_web", ] pyrepl_doctest_blocks = "autodoc" -pyrepl_autodoc_packages = "_static/wheels/pyrepl_test_pkg-1.0.0-py3-none-any.whl" +pyrepl_autodoc_packages = ":project:" +pyrepl_project_root = "../tests/fixtures/pyrepl_test_pkg" html_static_path = ["_static"] exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "example.rst"] diff --git a/docs/development.rst b/docs/development.rst index 61d8152..c291c86 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -56,6 +56,11 @@ Static wheels Wheel packages must be `Pyodide `_ compatible (pure-python packages work out of the box). For ``CPython`` extensions, visit `pyodide-build `_. +Set ``pyrepl_autodoc_packages = ":project:"`` to build a wheel from the documented +project automatically during the Sphinx HTML build. The wheel is written to +``_static/wheels/`` (configurable via ``pyrepl_wheel_dir``) and reused on +incremental builds until project sources change. + Wheels under ``_static/`` are copied into the HTML output when ``_static`` is listed in ``html_static_path``. At runtime, `pyrepl-web `_ resolves site-relative wheel paths to absolute URLs before calling diff --git a/docs/example.rst b/docs/example.rst index bfc7464..5199ec1 100644 --- a/docs/example.rst +++ b/docs/example.rst @@ -156,7 +156,8 @@ Set ``pyrepl_autodoc_packages`` to install the documented package and automatica # conf.py html_static_path = ["_static"] pyrepl_doctest_blocks = "autodoc" - pyrepl_autodoc_packages = "_static/wheels/pyrepl_test_pkg-1.0.0-py3-none-any.whl" + pyrepl_autodoc_packages = ":project:" + pyrepl_project_root = "../tests/fixtures/pyrepl_test_pkg" Source module: diff --git a/sphinx_pyrepl_web/__init__.py b/sphinx_pyrepl_web/__init__.py index 9997be9..04acb7f 100644 --- a/sphinx_pyrepl_web/__init__.py +++ b/sphinx_pyrepl_web/__init__.py @@ -1,6 +1,6 @@ """A Sphinx extension for embedding pyrepl-web Python REPLs in documentation.""" -__version__ = "0.3.0" +__version__ = "0.4.0" import json from doctest import DocTestParser @@ -16,6 +16,8 @@ from sphinx.util.fileutil import copy_asset_file from sphinx.util.osutil import relative_uri +from sphinx_pyrepl_web.wheel import ensure_project_wheel_on_init + PYREPL_DIR = Path(__file__).parent / "pyrepl" STARTUP_FILES_KEY = "pyrepl-startup-files" REPLAY_FILES_KEY = "pyrepl-replay-files" @@ -55,7 +57,10 @@ def setup(app: Sphinx): app.add_config_value("pyrepl_js", "../pyrepl.js", "env") app.add_config_value("pyrepl_doctest_blocks", False, "env", types=(bool, str)) app.add_config_value("pyrepl_autodoc_packages", None, "env") + app.add_config_value("pyrepl_project_root", None, "env") + app.add_config_value("pyrepl_wheel_dir", "_static/wheels", "env") app.add_directive("py-repl", PyRepl) + app.connect("builder-inited", ensure_project_wheel_on_init) app.connect("doctree-read", doctree_read) app.connect("doctree-read", transform_doctest_blocks) app.connect("html-page-context", add_html_context) @@ -167,7 +172,14 @@ def _find_autodoc_desc(node: nodes.Node) -> addnodes.desc | None: def _autodoc_packages(app: Sphinx) -> str | None: """Return configured package preload for autodoc doctest REPLs.""" - return app.config.pyrepl_autodoc_packages or None + try: + resolved = object.__getattribute__(app, "_pyrepl_resolved_autodoc_packages") + except AttributeError: + resolved = None + if resolved is not None: + return resolved + raw = app.config.pyrepl_autodoc_packages + return raw or None def _inside_autodoc_desc(node: nodes.Node) -> bool: @@ -356,3 +368,14 @@ def copy_asset_files(app, _): dest = outdir / rel dest.parent.mkdir(parents=True, exist_ok=True) copy_asset_file(str(path.resolve()), str(dest.resolve())) + + wheel_dir_setting = app.config.pyrepl_wheel_dir + src_wheel_dir = Path(app.confdir) / wheel_dir_setting + if src_wheel_dir.is_dir(): + dest_wheel_dir = outdir / wheel_dir_setting + dest_wheel_dir.mkdir(parents=True, exist_ok=True) + for wheel in src_wheel_dir.glob("*.whl"): + copy_asset_file( + str(wheel.resolve()), + str((dest_wheel_dir / wheel.name).resolve()), + ) diff --git a/sphinx_pyrepl_web/wheel.py b/sphinx_pyrepl_web/wheel.py new file mode 100644 index 0000000..81b4fde --- /dev/null +++ b/sphinx_pyrepl_web/wheel.py @@ -0,0 +1,247 @@ +"""Build and resolve project wheels for autodoc REPL package preload.""" + +from __future__ import annotations + +import os +import re +import subprocess +import sys +import tomllib +from configparser import ConfigParser +from pathlib import Path + +from sphinx.application import Sphinx +from sphinx.errors import ConfigError +from sphinx.util import logging + +PROJECT_SENTINEL = ":project:" +_PROJECT_MARKERS = ("pyproject.toml", "setup.cfg", "setup.py") +_SKIP_DIRS = frozenset( + { + ".git", + ".hg", + ".svn", + ".tox", + ".venv", + "__pycache__", + "build", + "dist", + "doc", + "docs", + "node_modules", + "venv", + } +) +logger = logging.getLogger(__name__) + + +def is_project_sentinel(value: object) -> bool: + """Return True if *value* requests automatic project wheel resolution.""" + return value == PROJECT_SENTINEL + + +def _is_project_root(path: Path) -> bool: + return any((path / marker).is_file() for marker in _PROJECT_MARKERS) + + +def find_project_root(confdir: Path, override: str | None = None) -> Path: + """Locate the Python project root for wheel builds.""" + if override is not None: + root = (confdir / override).resolve() + if not _is_project_root(root): + raise ConfigError( + f"pyrepl_project_root {override!r} does not contain " + f"pyproject.toml, setup.cfg, or setup.py " + f"(resolved to {root})" + ) + return root + + for candidate in (confdir, *confdir.parents): + if _is_project_root(candidate): + return candidate + + raise ConfigError( + "pyrepl_autodoc_packages is ':project:' but no project root was found " + f"searching upward from {confdir}" + ) + + +def read_distribution_name(project_root: Path) -> str: + """Return the distribution name declared by the project.""" + pyproject = project_root / "pyproject.toml" + if pyproject.is_file(): + data = tomllib.loads(pyproject.read_text(encoding="utf-8")) + name = data.get("project", {}).get("name") + if isinstance(name, str) and name: + return name + + setup_cfg = project_root / "setup.cfg" + if setup_cfg.is_file(): + parser = ConfigParser() + parser.read(setup_cfg, encoding="utf-8") + if parser.has_option("metadata", "name"): + name = parser.get("metadata", "name") + if name: + return name + + raise ConfigError( + f"could not determine distribution name from {project_root}" + ) + + +def normalize_distribution_name(name: str) -> str: + """Normalize a distribution name for wheel filename matching.""" + return re.sub(r"[-_.]+", "_", name).lower() + + +def wheel_dir_path(confdir: Path, wheel_dir: str) -> Path: + """Return the absolute wheel output directory.""" + return (confdir / wheel_dir).resolve() + + +def resolved_wheel_href(wheel_dir: str, wheel_name: str) -> str: + """Return the Sphinx site path for a built wheel.""" + return f"{wheel_dir.rstrip('/')}/{wheel_name}" + + +def find_newest_wheel(wheel_dir: Path, distribution_name: str) -> Path | None: + """Return the newest wheel matching *distribution_name*, if any.""" + normalized = normalize_distribution_name(distribution_name) + matches = [ + wheel + for wheel in wheel_dir.glob("*.whl") + if normalize_distribution_name(wheel.name.split("-", 1)[0]) == normalized + ] + if not matches: + return None + return max(matches, key=lambda path: path.stat().st_mtime) + + +def project_latest_mtime(project_root: Path, *, wheel_dir: Path) -> float: + """Return the latest mtime among project sources that affect wheels.""" + latest = 0.0 + for marker in _PROJECT_MARKERS: + path = project_root / marker + if path.is_file(): + latest = max(latest, path.stat().st_mtime) + + for path in project_root.rglob("*.py"): + if wheel_dir in path.parents or path.parents[0] == wheel_dir: + continue + if _SKIP_DIRS.intersection(path.relative_to(project_root).parts): + continue + latest = max(latest, path.stat().st_mtime) + + return latest + + +def wheel_is_fresh(wheel_path: Path, project_root: Path, *, wheel_dir: Path) -> bool: + """Return True if *wheel_path* is newer than relevant project sources.""" + if not wheel_path.is_file(): + return False + # Compare whole seconds; subsecond utime precision varies across platforms. + wheel_mtime = int(wheel_path.stat().st_mtime) + project_mtime = int(project_latest_mtime(project_root, wheel_dir=wheel_dir)) + return wheel_mtime >= project_mtime + + +def _wheel_build_env() -> dict[str, str]: + """Return a subprocess environment safe for ``pip wheel`` under strict warnings.""" + env = os.environ.copy() + # CI sets PYTHONWARNINGS=error; pip's pkg_resources import emits DeprecationWarning. + env.pop("PYTHONWARNINGS", None) + return env + + +def build_wheel(project_root: Path, wheel_dir: Path) -> None: + """Build a wheel for *project_root* into *wheel_dir*.""" + wheel_dir.mkdir(parents=True, exist_ok=True) + try: + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "wheel", + str(project_root), + "--no-deps", + "-w", + str(wheel_dir), + ], + check=True, + capture_output=True, + text=True, + env=_wheel_build_env(), + ) + except subprocess.CalledProcessError as exc: + detail = (exc.stderr or exc.stdout or "").strip() + raise ConfigError( + f"failed to build project wheel for {project_root}: {detail}" + ) from exc + + +def warn_if_wheel_dir_not_static(app: Sphinx, wheel_dir: str) -> None: + """Warn when wheels will not be copied into HTML output.""" + static_paths = list(getattr(app.config, "html_static_path", None) or []) + covered = any( + wheel_dir == static_path + or wheel_dir.startswith(f"{static_path.rstrip('/')}/") + for static_path in static_paths + ) + if covered: + return + logger.warning( + "pyrepl wheel directory %r is not covered by html_static_path; " + "add its parent directory so wheels are copied into HTML output", + wheel_dir, + type="pyrepl", + subtype="wheel", + ) + + +def ensure_project_wheel(app: Sphinx) -> str: + """Build or reuse a project wheel and return its Sphinx path.""" + confdir = Path(app.confdir) + wheel_dir_setting = app.config.pyrepl_wheel_dir + wheel_dir = wheel_dir_path(confdir, wheel_dir_setting) + project_root = find_project_root(confdir, app.config.pyrepl_project_root) + distribution_name = read_distribution_name(project_root) + + warn_if_wheel_dir_not_static(app, wheel_dir_setting) + + wheel_path = find_newest_wheel(wheel_dir, distribution_name) + if wheel_path is None or not wheel_is_fresh( + wheel_path, project_root, wheel_dir=wheel_dir + ): + logger.info( + "building project wheel for %r into %s", + distribution_name, + wheel_dir, + type="pyrepl", + subtype="wheel", + ) + build_wheel(project_root, wheel_dir) + wheel_path = find_newest_wheel(wheel_dir, distribution_name) + + if wheel_path is None: + raise ConfigError( + f"project wheel for {distribution_name!r} was not found in {wheel_dir} " + "after build" + ) + + logger.info( + "using project wheel %s", + wheel_path.name, + type="pyrepl", + subtype="wheel", + ) + return resolved_wheel_href(wheel_dir_setting, wheel_path.name) + + +def ensure_project_wheel_on_init(app: Sphinx) -> None: + """Resolve ``:project:`` before doctrees are read.""" + if not is_project_sentinel(app.config.pyrepl_autodoc_packages): + return + if app.builder.format != "html": + return + app._pyrepl_resolved_autodoc_packages = ensure_project_wheel(app) diff --git a/tests/integration/test_project_wheel.py b/tests/integration/test_project_wheel.py new file mode 100644 index 0000000..d7e7aff --- /dev/null +++ b/tests/integration/test_project_wheel.py @@ -0,0 +1,42 @@ +from tests.helpers import load_bootstrap_files, pyrepl_tag +from tests.support import ( + FIXTURES, + autodoc_conf_header, + build_sphinx, + project_wheel_conf_extra, +) + + +def test_autodoc_with_project_sentinel_builds_wheel(tmp_path): + srcdir = tmp_path / "docs" + srcdir.mkdir() + outdir = tmp_path / "_build" + doctreedir = tmp_path / "_doctree" + + (srcdir / "conf.py").write_text( + autodoc_conf_header( + sys_path=str(FIXTURES / "pyrepl_test_pkg"), + extra=project_wheel_conf_extra(srcdir=srcdir), + ), + encoding="utf-8", + ) + (srcdir / "index.rst").write_text( + ".. autofunction:: pyrepl_test_pkg.demo.example_generator\n", + encoding="utf-8", + ) + + app = build_sphinx(srcdir, outdir, doctreedir) + + built_wheel = next((srcdir / "_static" / "wheels").glob("pyrepl_test_pkg-*.whl")) + wheel_path = f"_static/wheels/{built_wheel.name}" + html = (outdir / "index.html").read_text(encoding="utf-8") + assert f'packages="{wheel_path}"' in html + assert 'replay-src="_static/pyrepl/index-1.py"' in html + tag = pyrepl_tag(html) + assert 'src="_static/pyrepl/index-1-bootstrap.py"' in tag + assert (outdir / "_static" / "wheels" / built_wheel.name).is_file() + assert built_wheel.is_file() + assert (outdir / "_static" / "pyrepl" / "index-1.py").is_file() + assert (outdir / "_static" / "pyrepl" / "index-1-bootstrap.py").is_file() + assert app.env.get_doctree("index").get("pyrepl") + assert list(load_bootstrap_files(app, "index")) == ["index-1-bootstrap.py"] diff --git a/tests/support.py b/tests/support.py index 7742620..fcccec8 100644 --- a/tests/support.py +++ b/tests/support.py @@ -54,6 +54,26 @@ def wheel_conf_extra() -> str: """ +def project_wheel_conf_extra(*, srcdir: Path | None = None) -> str: + """Return conf.py snippet enabling automatic project wheel builds.""" + fixture = FIXTURES / "pyrepl_test_pkg" + if srcdir is not None: + import os + + try: + project_root = os.path.relpath(fixture.resolve(), srcdir.resolve()) + except ValueError: + # Windows: fixture and srcdir may live on different drives. + project_root = str(fixture.resolve()) + else: + project_root = str(fixture.resolve()) + return f""" +html_static_path = ["_static"] +pyrepl_autodoc_packages = ":project:" +pyrepl_project_root = {project_root!r} +""" + + def autodoc_conf_header(*, sys_path: str, extra: str = "") -> str: """Return a minimal autodoc-enabled conf.py header.""" return f""" diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index de0668f..ba695fd 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -18,3 +18,10 @@ def test_autodoc_packages(configured, expected): app = MagicMock() app.config.pyrepl_autodoc_packages = configured assert _autodoc_packages(app) == expected + + +def test_autodoc_packages_uses_resolved_project_wheel(): + app = MagicMock() + app.config.pyrepl_autodoc_packages = ":project:" + app._pyrepl_resolved_autodoc_packages = WHEEL_PATH + assert _autodoc_packages(app) == WHEEL_PATH diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py new file mode 100644 index 0000000..65a0531 --- /dev/null +++ b/tests/unit/test_wheel.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +import os +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from sphinx.errors import ConfigError + +from sphinx_pyrepl_web.wheel import ( + PROJECT_SENTINEL, + build_wheel, + ensure_project_wheel, + find_newest_wheel, + find_project_root, + is_project_sentinel, + normalize_distribution_name, + project_latest_mtime, + read_distribution_name, + resolved_wheel_href, + wheel_is_fresh, +) +from tests.support import FIXTURES + +PKG_ROOT = FIXTURES / "pyrepl_test_pkg" + + +def test_is_project_sentinel(): + assert is_project_sentinel(PROJECT_SENTINEL) + assert not is_project_sentinel("_static/wheels/foo.whl") + assert not is_project_sentinel(None) + + +def test_find_project_root_from_nested_confdir(tmp_path): + project = tmp_path / "project" + docs = project / "docs" + docs.mkdir(parents=True) + (project / "pyproject.toml").write_text( + '[project]\nname = "demo"\nversion = "0.1.0"\n', + encoding="utf-8", + ) + assert find_project_root(docs) == project + + +def test_find_project_root_honors_override(tmp_path): + pkg = tmp_path / "pkg" + docs = tmp_path / "docs" + docs.mkdir() + pkg.mkdir() + (pkg / "pyproject.toml").write_text( + '[project]\nname = "demo"\nversion = "0.1.0"\n', + encoding="utf-8", + ) + assert find_project_root(docs, "../pkg") == pkg.resolve() + + +def test_find_project_root_invalid_override(tmp_path): + docs = tmp_path / "docs" + docs.mkdir() + with pytest.raises(ConfigError, match="pyrepl_project_root"): + find_project_root(docs, "../missing") + + +def test_read_distribution_name_from_fixture(): + assert read_distribution_name(PKG_ROOT) == "pyrepl_test_pkg" + + +def test_normalize_distribution_name(): + assert normalize_distribution_name("my-package") == "my_package" + assert normalize_distribution_name("PyRepl.Test") == "pyrepl_test" + + +def test_find_newest_wheel_picks_latest(tmp_path): + wheel_dir = tmp_path / "wheels" + wheel_dir.mkdir() + older = wheel_dir / "pyrepl_test_pkg-1.0.0-py3-none-any.whl" + newer = wheel_dir / "pyrepl_test_pkg-2.0.0-py3-none-any.whl" + older.write_bytes(b"old") + newer.write_bytes(b"new") + now = os.path.getmtime(newer) + os.utime(older, (now - 10, now - 10)) + os.utime(newer, (now, now)) + assert find_newest_wheel(wheel_dir, "pyrepl_test_pkg") == newer + + +def test_resolved_wheel_href(): + assert ( + resolved_wheel_href("_static/wheels", "demo-1.0.0-py3-none-any.whl") + == "_static/wheels/demo-1.0.0-py3-none-any.whl" + ) + + +def test_wheel_is_fresh_when_source_unchanged(tmp_path): + project = tmp_path / "project" + wheel_dir = tmp_path / "wheels" + project.mkdir() + wheel_dir.mkdir() + (project / "pyproject.toml").write_text( + '[project]\nname = "demo"\nversion = "1.0.0"\n', + encoding="utf-8", + ) + pkg = project / "demo_pkg" + pkg.mkdir() + source = pkg / "__init__.py" + source.write_text("x = 1\n", encoding="utf-8") + wheel = wheel_dir / "demo-1.0.0-py3-none-any.whl" + wheel.write_bytes(b"wheel") + assert wheel_is_fresh(wheel, project, wheel_dir=wheel_dir) + + +def test_wheel_is_stale_when_source_changes(tmp_path): + project = tmp_path / "project" + wheel_dir = tmp_path / "wheels" + project.mkdir() + wheel_dir.mkdir() + (project / "pyproject.toml").write_text( + '[project]\nname = "demo"\nversion = "1.0.0"\n', + encoding="utf-8", + ) + pkg = project / "demo_pkg" + pkg.mkdir() + source = pkg / "__init__.py" + source.write_text("x = 1\n", encoding="utf-8") + wheel = wheel_dir / "demo-1.0.0-py3-none-any.whl" + wheel.write_bytes(b"wheel") + source.write_text("x = 2\n", encoding="utf-8") + now = os.path.getmtime(source) + os.utime(wheel, (now - 10, now - 10)) + assert not wheel_is_fresh(wheel, project, wheel_dir=wheel_dir) + + +def test_build_wheel_creates_fixture_wheel(tmp_path, monkeypatch): + monkeypatch.setenv("PYTHONWARNINGS", "error") + wheel_dir = tmp_path / "wheels" + build_wheel(PKG_ROOT, wheel_dir) + wheel = find_newest_wheel(wheel_dir, "pyrepl_test_pkg") + assert wheel is not None + assert wheel.name.startswith("pyrepl_test_pkg-1.0.0-") + assert wheel.name.endswith("-none-any.whl") + + +def test_ensure_project_wheel_builds_and_returns_path(tmp_path): + docs = tmp_path / "docs" + docs.mkdir() + app = MagicMock() + app.confdir = str(docs) + app.config.pyrepl_project_root = str(PKG_ROOT) + app.config.pyrepl_wheel_dir = "_static/wheels" + app.config.html_static_path = ["_static"] + + path = ensure_project_wheel(app) + + assert path.startswith("_static/wheels/pyrepl_test_pkg-1.0.0-") + assert path.endswith("-none-any.whl") + assert (docs / "_static" / "wheels" / path.rsplit("/", 1)[-1]).is_file() + + +def test_ensure_project_wheel_reuses_fresh_wheel(tmp_path): + docs = tmp_path / "docs" + wheel_dir = docs / "_static" / "wheels" + wheel_dir.mkdir(parents=True) + wheel = wheel_dir / "pyrepl_test_pkg-1.0.0-py2.py3-none-any.whl" + wheel.write_bytes(b"wheel") + latest = project_latest_mtime(PKG_ROOT, wheel_dir=wheel_dir) + os.utime(wheel, (latest + 10, latest + 10)) + + app = MagicMock() + app.confdir = str(docs) + app.config.pyrepl_project_root = str(PKG_ROOT) + app.config.pyrepl_wheel_dir = "_static/wheels" + app.config.html_static_path = ["_static"] + + with patch("sphinx_pyrepl_web.wheel.build_wheel") as build_mock: + first = ensure_project_wheel(app) + second = ensure_project_wheel(app) + + assert first == second + build_mock.assert_not_called()