diff --git a/.github/workflows/docs-pr-failure-post-comment.yml b/.github/workflows/docs-pr-failure-post-comment.yml new file mode 100644 index 00000000..2a8928b2 --- /dev/null +++ b/.github/workflows/docs-pr-failure-post-comment.yml @@ -0,0 +1,66 @@ +name: Post PR comment with doc-build failure log + +env: + DOCS_FAILURE_ARTIFACT: test-build-docs-container_failed + +on: + workflow_run: + workflows: ["Test building docs when they're updated"] + types: [completed] + +jobs: + comment: + if: >- + github.event.workflow_run.event == 'pull_request' + && github.event.workflow_run.conclusion == 'failure' + runs-on: ubuntu-latest + permissions: + pull-requests: write + actions: read + steps: + - name: Check for failure artifact + id: check + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + run: | + gh api repos/$REPO/actions/runs/${{ github.event.workflow_run.id }}/artifacts \ + --jq '.artifacts[] | select(.name == "${{ env.DOCS_FAILURE_ARTIFACT }}") | .id' > artifact_id.txt + + if [ -s artifact_id.txt ]; then + echo "found=true" >> $GITHUB_OUTPUT + else + echo "found=false" >> $GITHUB_OUTPUT + fi + + - name: Download logs + if: steps.check.outputs.found == 'true' + uses: actions/download-artifact@v4 + with: + name: ${{ env.DOCS_FAILURE_ARTIFACT }} + path: ${{ env.DOCS_FAILURE_ARTIFACT }} + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Post comment + if: steps.check.outputs.found == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + run: | + PR_NUMBER=$(cat "${DOCS_FAILURE_ARTIFACT}/pr_number.txt") + + { + echo "### ❌ Docs build failed" + echo + echo '
' + echo "Build logs" + echo + echo '```' + cat "${DOCS_FAILURE_ARTIFACT}/build.log" + echo '```' + echo + echo "
" + } > comment-body.md + + gh pr comment "$PR_NUMBER" --repo "$REPO" --body-file comment-body.md diff --git a/.github/workflows/docs-to-publish.yml b/.github/workflows/docs-to-publish.yml new file mode 100644 index 00000000..38f333f7 --- /dev/null +++ b/.github/workflows/docs-to-publish.yml @@ -0,0 +1,59 @@ +name: Test building docs "to publish" + +on: + push: + # Run when a change to these files is pushed to any branch. Without the "branches:" line, for some reason this will be run whenever a tag is pushed, even if the listed files aren't changed. + branches: ['*'] + paths: + - 'doc/doc-builder/**' + - 'doc/build_docs_to_publish' + + pull_request: + # Run on pull requests that change the listed files + paths: + - 'doc/doc-builder/**' + - 'doc/build_docs_to_publish' + + workflow_dispatch: + +permissions: + contents: read +jobs: + + test-build-docs-to-publish-container: + if: ${{ always() }} + name: With container from registry + runs-on: ubuntu-latest + steps: + + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: true + + - name: Build docs using Docker (Podman has trouble on GitHub runners) + id: build-docs + run: | + set -o pipefail + mkdir -p build-logs + [[ -e doc ]] && cd doc + git fetch + PYTHONUNBUFFERED=1 ./build_docs_to_publish -r ${PWD}/_build -d --verbose --site-root "$PWD/_publish" 2>&1 | tee >(sed -E $'s/\x1b\\[[0-9;]*[a-zA-Z]//g' > "${GITHUB_WORKSPACE}/build-logs/build.log") + # The tee writes build.log for the PR comment (posted by docs-pr-failure-post-comment.yml); the inner sed strips ANSI color codes that would otherwise render as garbage in the comment. + # PYTHONUNBUFFERED=1 because otherwise the teed log will be out of order + + # The rest of the steps only trigger on failure of above build-docs step. + # They upload logs that will be used by the docs-pr-failure-post-comment.yml workflow. + + - name: Record PR number on failure + if: failure() && steps.build-docs.outcome == 'failure' && github.event_name == 'pull_request' + run: | + mkdir -p build-logs + echo "${{ github.event.pull_request.number }}" > build-logs/pr_number.txt + + - name: Upload logs on failure + if: failure() && steps.build-docs.outcome == 'failure' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-build-docs-to-publish-container_failed + path: build-logs/ diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..43ee1dfa --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,58 @@ +name: Test building docs when they're updated + +on: + push: + # Run when a change to these files is pushed to any branch. Without the "branches:" line, for some reason this will be run whenever a tag is pushed, even if the listed files aren't changed. + branches: ['*'] + paths: + - 'doc/**' + # And also make sure to add any files that you "include" in your docs that live outside doc/ + + pull_request: + # Run on pull requests that change the listed files + paths: + - 'doc/**' + # And also make sure to add any files that you "include" in your docs that live outside doc/ + + workflow_dispatch: + +permissions: + contents: read +jobs: + + test-build-docs-container: + if: ${{ always() }} + name: With container from registry + runs-on: ubuntu-latest + steps: + + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: true + + - name: Build docs using Docker (Podman has trouble on GitHub runners) + id: build-docs + run: | + set -o pipefail + mkdir -p build-logs + [[ -e doc ]] && cd doc + PYTHONUNBUFFERED=1 ./build_docs -b ${PWD}/_build -c -d 2>&1 | tee >(sed -E $'s/\x1b\\[[0-9;]*[a-zA-Z]//g' > "${GITHUB_WORKSPACE}/build-logs/build.log") + # The tee writes build.log for the PR comment (posted by docs-pr-failure-post-comment.yml); the inner sed strips ANSI color codes that would otherwise render as garbage in the comment. + # PYTHONUNBUFFERED=1 because otherwise the teed log will be out of order + + # The rest of the steps only trigger on failure of above build-docs step. + # They upload logs that will be used by the docs-pr-failure-post-comment.yml workflow. + + - name: Record PR number on failure + if: failure() && steps.build-docs.outcome == 'failure' && github.event_name == 'pull_request' + run: | + mkdir -p build-logs + echo "${{ github.event.pull_request.number }}" > build-logs/pr_number.txt + + - name: Upload logs on failure + if: failure() && steps.build-docs.outcome == 'failure' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-build-docs-container_failed + path: build-logs/ diff --git a/.gitignore b/.gitignore index 5361b611..73f751cb 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ CMakeCache.txt CMakeFiles Makefile +!doc/Makefile autocopy.log Makefile.am build @@ -68,3 +69,4 @@ cube_to_target/smooth_topo_cube_sph.mod # Ignore docs files _build/ +_publish/ diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 00000000..2048c35f --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,28 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SOURCEDIR = source +DIRWITHCONFPY = $(if $(wildcard $(dir $(lastword $(MAKEFILE_LIST)))doc-builder),doc-builder,.) +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" -c "$(DIRWITHCONFPY)" $(SPHINXOPTS) $(O) + +# 'make fetch-images' should be run before building the documentation. (If building via +# the build_docs command, this is run automatically for you.) This is needed because we +# have configured this repository (via an .lfsconfig file at the top level) to NOT +# automatically fetch any of the large files when cloning / fetching. +fetch-images: + git lfs install --force + git lfs pull --exclude="" --include="" + +.PHONY: help fetch-images Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" -c "$(DIRWITHCONFPY)" $(SPHINXOPTS) $(O) diff --git a/doc/build_docs b/doc/build_docs index f62db80b..16bdb3f0 100755 --- a/doc/build_docs +++ b/doc/build_docs @@ -1,6 +1,33 @@ #!/usr/bin/env bash set -e +# $scriptname will already be set if build_docs is getting called from build_docs_to_publish. +# Otherwise, we need to set it to the basename of this script. +if [[ -z "${scriptname}" ]]; then + scriptname="${0##*/}" +fi + +if [ ! -e doc-builder ]; then + # We're in the doc-builder directory itself + script="./tools/${scriptname}.py" + files_to_copy_to_source="conf.py substitutions.py _templates _static" + cp -a ${files_to_copy_to_source} source/ +else + # We're in a repo that *includes* doc-builder + script="./doc-builder/tools/${scriptname}.py" + + # Make sure the doc-builder submodule is checked out. If doc-builder isn't a submodule but is + # instead manually included, we will perform the `else`, which is safe because it's a no-op in + # that situation. + # (Assume that if the repo uses git-fleximod, doc-builder is handled with it.) + git_fleximod_path="$(git rev-parse --show-toplevel)/bin/git-fleximod" + if [[ -e "${git_fleximod_path}" ]]; then + "${git_fleximod_path}" update doc-builder + else + git submodule update --init -- doc-builder + fi +fi + # Check if --verbose or -V was passed verbose=false for arg in "$@"; do @@ -17,8 +44,18 @@ else fi if $verbose; then - echo "Running: ./doc-builder/build_docs $@" + echo "Running: ${script} $@" +fi +set +e +${script} "$@" +exit_code=$? +set -e + +if [ ! -e doc-builder ]; then + # Clean up these things we copied + for f in ${files_to_copy_to_source}; do + rm -r "source/$f" + done fi -./doc-builder/build_docs "$@" -exit 0 +exit $exit_code diff --git a/doc/build_docs_to_publish b/doc/build_docs_to_publish index b1d58387..dc176c60 100755 --- a/doc/build_docs_to_publish +++ b/doc/build_docs_to_publish @@ -1,27 +1,6 @@ #!/usr/bin/env bash set -e -# Check if --verbose or -V was passed -verbose=false -for arg in "$@"; do - case "$arg" in - --verbose|-V) verbose=true; break ;; - esac -done +scriptname=build_docs_to_publish -cd "${script_dir}" - -if $verbose; then - echo "Running: make fetch-images" - make fetch-images -else - make fetch-images > /dev/null 2>&1 -fi - -if $verbose; then - echo "Running: ./doc-builder/build_docs_to_publish $@" - pwd -fi -./doc-builder/build_docs_to_publish "$@" - -exit 0 +. ./build_docs diff --git a/doc/doc-builder/README.md b/doc/doc-builder/README.md index 37e1a602..8e459ca6 100644 --- a/doc/doc-builder/README.md +++ b/doc/doc-builder/README.md @@ -1,5 +1,5 @@ $$$$$$$$$$$$$$$$$$$ -This instance of doc-builder was copied from https://github.com/ESMCI/doc-builder/tree/v3.4.1 +This instance of doc-builder was copied from https://github.com/ESMCI/doc-builder/tree/v3.5.9 $$$$$$$$$$$$$$$$$$$ This tool wraps the build command to build sphinx-based documentation. diff --git a/doc/doc-builder/build_docs b/doc/doc-builder/build_docs index f05599ca..16bdb3f0 100755 --- a/doc/doc-builder/build_docs +++ b/doc/doc-builder/build_docs @@ -1,14 +1,61 @@ -#!/usr/bin/env python3 +#!/usr/bin/env bash +set -e -"""Main driver wrapper around the doc_builder/build_docs utility. +# $scriptname will already be set if build_docs is getting called from build_docs_to_publish. +# Otherwise, we need to set it to the basename of this script. +if [[ -z "${scriptname}" ]]; then + scriptname="${0##*/}" +fi -This tool wraps the build command to build sphinx-based documentation. +if [ ! -e doc-builder ]; then + # We're in the doc-builder directory itself + script="./tools/${scriptname}.py" + files_to_copy_to_source="conf.py substitutions.py _templates _static" + cp -a ${files_to_copy_to_source} source/ +else + # We're in a repo that *includes* doc-builder + script="./doc-builder/tools/${scriptname}.py" -This should be run from the directory that contains the Makefile for -building the documentation. -""" + # Make sure the doc-builder submodule is checked out. If doc-builder isn't a submodule but is + # instead manually included, we will perform the `else`, which is safe because it's a no-op in + # that situation. + # (Assume that if the repo uses git-fleximod, doc-builder is handled with it.) + git_fleximod_path="$(git rev-parse --show-toplevel)/bin/git-fleximod" + if [[ -e "${git_fleximod_path}" ]]; then + "${git_fleximod_path}" update doc-builder + else + git submodule update --init -- doc-builder + fi +fi -from doc_builder import build_docs # pylint: disable=import-error +# Check if --verbose or -V was passed +verbose=false +for arg in "$@"; do + case "$arg" in + --verbose|-V) verbose=true; break ;; + esac +done -if __name__ == "__main__": - build_docs.main() +if $verbose; then + echo "Running: make fetch-images" + make fetch-images +else + make fetch-images > /dev/null 2>&1 +fi + +if $verbose; then + echo "Running: ${script} $@" +fi +set +e +${script} "$@" +exit_code=$? +set -e + +if [ ! -e doc-builder ]; then + # Clean up these things we copied + for f in ${files_to_copy_to_source}; do + rm -r "source/$f" + done +fi + +exit $exit_code diff --git a/doc/doc-builder/build_docs_to_publish b/doc/doc-builder/build_docs_to_publish index df663700..dc176c60 100755 --- a/doc/doc-builder/build_docs_to_publish +++ b/doc/doc-builder/build_docs_to_publish @@ -1,214 +1,6 @@ -#!/usr/bin/env python3 +#!/usr/bin/env bash +set -e -""" -Loop through all versions of the documentation, building each and moving it to a directory for -publication. +scriptname=build_docs_to_publish -Adapted from https://www.codingwiththomas.com/blog/my-sphinx-best-practice-for-a-multiversion- -documentation-in-different-languages -(last visited 2025-05-20) -""" - -import sys -import os -import subprocess -import argparse - -# pylint: disable=import-error,no-name-in-module -from doc_builder.build_docs import ( - main as build_docs, -) -from doc_builder.build_docs_shared_args import main as build_docs_shared_args -from doc_builder.sys_utils import ( - get_git_head_or_branch, - check_permanent_file, - get_toplevel_git_dir, -) -from doc_builder.build_commands import get_container_cli_tool - -# Change to the parent director of doc-builder and add to Python path -os.chdir(os.path.join(os.path.dirname(__file__), os.pardir)) -sys.path.insert(0, os.getcwd()) - -# Import our definitions of each documentation version. -# pylint: disable=wrong-import-position -from version_list import ( - LATEST_REF, - VERSION_LIST, -) - - -# Path to certain important files -SOURCE = "source" -VERSIONS_PY = os.path.join("version_list.py") -MAKEFILE = "Makefile" - - -def get_static_templates_path_relative_to_here(args, path): - """ - _static and _templates paths are relative to conf.py, but for operations here we need them - relative to our current directory. - """ - full_path = os.path.join(os.path.dirname(args.conf_py_path), path) - return os.path.relpath(full_path) - - -def setup_this_ref(args): - """ - Check that the repo is ready for the checkouts that are about to happen - """ - # Get the current branch, or SHA if detached HEAD - orig_ref = get_git_head_or_branch() - - # Some files/directories/submodules must stay the same for all builds. We list these in - # the permanent_files list. - permanent_files = [ - VERSIONS_PY, - "doc-builder", - MAKEFILE, - args.conf_py_path, - get_static_templates_path_relative_to_here(args, args.templates_path), - get_static_templates_path_relative_to_here(args, args.static_path), - ] - - # Check some things about "permanent" files before checkout - for filename in permanent_files: - check_permanent_file(filename) - - return orig_ref, permanent_files - - -def checkout_and_build(version, permanent_files, args): - """ - Check out docs for a version and build - """ - - # Check out the git reference of this version (branch name, tag, or commit SHA) - subprocess.check_output("git checkout " + version.ref, shell=True) - - # Check out LATEST_REF version of permanent files - our_toplevel = get_toplevel_git_dir(os.getcwd()) - for filepath in permanent_files: - # Skip if not in our working tree (i.e., in a submodule or a totally different place) - file_toplevel = get_toplevel_git_dir(filepath) - if file_toplevel != our_toplevel: - continue - - # Check out file - filename = os.path.basename(filepath) - subprocess.check_output(f"git checkout {LATEST_REF} -- {filename}", shell=True) - - # Build the docs for this version - build_args = [ - "-r", - args.repo_root, - "-v", - version.short_name, - "--version-display-name", - version.display_name, - "--versions", - "--site-root", - args.site_root, - "--clean", - ] - if args.build_in_container: - build_args += ["-d"] - if args.conf_py_path: - build_args += ["--conf-py-path", args.conf_py_path] - if args.static_path: - build_args += ["--static-path", args.static_path] - if args.templates_path: - build_args += ["--templates-path", args.templates_path] - if args.container_cli_tool: - build_args += ["--container-cli-tool", args.container_cli_tool] - print(" ".join(build_args)) - build_docs(build_args) - - -def reset_git(orig_ref): - """ - Restore the repo to how it was before we did our checkouts - """ - # 1. Get the current ref's version of doc-builder to avoid "would be overwritten by checkout" - # errors. - cmd = "git submodule update --checkout doc-builder".split(" ") - result = subprocess.run(cmd, check=False, capture_output=True, text=True) - # If doc-builder not tracked by git, that's fine. Otherwise error. - if result.returncode and "did not match any file(s) known to git" not in result.stderr: - print("PWD: " + os.getcwd()) - print(result.stdout) - print(result.stderr) - raise subprocess.CalledProcessError(result.returncode, cmd) - - # 2. Check out the original git ref (branch or commit SHA) - subprocess.check_output("git checkout " + orig_ref, shell=True) - - # 3. Restore the current version's doc-builder - subprocess.check_output("git submodule update --checkout doc-builder", shell=True) - - -def check_version_list(): - """ - Check version list for problems - """ - has_default = False - for version in VERSION_LIST: - # Expect at most one version with landing_version True - if version.landing_version: - if has_default: - raise RuntimeError("Expected at most one version with landing_version True") - has_default = True - - -def main(): - """ - Loop through all versions of the documentation, building each and moving it to a directory for - publication. - """ - # Set up parser - parser = argparse.ArgumentParser() - - # Arguments shared with build_docs - parser = build_docs_shared_args(parser) - - # Custom arguments for build_docs_to_publish - parser.add_argument( - "--publish-dir", - default="_publish", - help="Where the docs should be moved after being built", - ) - - # Parse arguments - args = parser.parse_args() - if args.container_cli_tool is None: - args.container_cli_tool = get_container_cli_tool() - - # Check version list for problems - check_version_list() - - # Loop over all documentation versions - for version in VERSION_LIST: - # Check that repo is ready and get our current state - orig_ref, permanent_files = setup_this_ref(args) - - # Build this version - try: - checkout_and_build(version, permanent_files, args) - # Restore the repo to how it was before we did our checkouts, even if checkout_and_build() - # errored - finally: - reset_git(orig_ref) - - # Copy this version to the publication directory - src = os.path.join(args.repo_root, "versions", version.short_name, "html") - if version.landing_version: - dst = args.publish_dir - else: - dst = os.path.join(args.publish_dir, version.short_name) - if not os.path.exists(dst): - os.makedirs(dst) - subprocess.check_output(f"mv '{src}'/* '{dst}'/", shell=True) - - -if __name__ == "__main__": - main() +. ./build_docs diff --git a/doc/doc-builder/doc_builder/build_commands.py b/doc/doc-builder/doc_builder/build_commands.py index 7a34a798..52d7660f 100644 --- a/doc/doc-builder/doc_builder/build_commands.py +++ b/doc/doc-builder/doc_builder/build_commands.py @@ -7,7 +7,7 @@ import shutil from doc_builder import sys_utils # pylint: disable=import-error -DEFAULT_IMAGE = "ghcr.io/esmci/doc-builder/doc-build-container:v2.0.1" +DEFAULT_IMAGE = "ghcr.io/esmci/doc-builder/doc-build-container:v2.0.1a" # The path in the container's filesystem where doc-builder's parent repo checkout is mounted _CONTAINER_HOME = "/home/user/mounted_home" diff --git a/doc/doc-builder/tools/build_docs.py b/doc/doc-builder/tools/build_docs.py new file mode 100755 index 00000000..c77fa880 --- /dev/null +++ b/doc/doc-builder/tools/build_docs.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 + +"""Main driver wrapper around the doc_builder/build_docs utility. + +This tool wraps the build command to build sphinx-based documentation. + +This should be run from the directory that contains the Makefile for +building the documentation. +""" + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir)) + +from doc_builder import build_docs # pylint: disable=import-error,wrong-import-position + +if __name__ == "__main__": + build_docs.main() diff --git a/doc/doc-builder/tools/build_docs_to_publish.py b/doc/doc-builder/tools/build_docs_to_publish.py new file mode 100755 index 00000000..387a32e1 --- /dev/null +++ b/doc/doc-builder/tools/build_docs_to_publish.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 + +""" +Loop through all versions of the documentation, building each and moving it to a directory for +publication. + +Adapted from https://www.codingwiththomas.com/blog/my-sphinx-best-practice-for-a-multiversion- +documentation-in-different-languages +(last visited 2025-05-20) +""" + +import sys +import os +import subprocess +import argparse + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir)) + +# pylint: disable=import-error,no-name-in-module,wrong-import-position +from doc_builder.build_docs import ( + main as build_docs, +) +from doc_builder.build_docs_shared_args import main as build_docs_shared_args +from doc_builder.sys_utils import ( + get_git_head_or_branch, + check_permanent_file, + get_toplevel_git_dir, +) +from doc_builder.build_commands import get_container_cli_tool + +# Change to the parent directory of our version_list.py so we can add it to Python path +DOC_BUILDER_IS_SUBMODULE = os.path.exists("doc-builder") +new_wd = os.path.join(os.path.dirname(__file__), os.pardir) +if DOC_BUILDER_IS_SUBMODULE: + new_wd = os.path.join(new_wd, os.pardir) +os.chdir(new_wd) +sys.path.insert(0, os.getcwd()) + +# Import our definitions of each documentation version. +# pylint: disable=wrong-import-position +from version_list import ( + LATEST_REF, + VERSION_LIST, +) + +# Path to certain important files +SOURCE = "source" +VERSIONS_PY = os.path.join("version_list.py") +MAKEFILE = "Makefile" + + +def get_static_templates_path_relative_to_here(args, path): + """ + _static and _templates paths are relative to conf.py, but for operations here we need them + relative to our current directory. + """ + full_path = os.path.join(os.path.dirname(args.conf_py_path), path) + return os.path.relpath(full_path) + + +def setup_this_ref(args): + """ + Check that the repo is ready for the checkouts that are about to happen + """ + # Get the current branch, or SHA if detached HEAD + orig_ref = get_git_head_or_branch() + + # Some files/directories/submodules must stay the same for all builds. We list these in + # the permanent_files list. + permanent_files = [ + VERSIONS_PY, + MAKEFILE, + args.conf_py_path, + get_static_templates_path_relative_to_here(args, args.templates_path), + get_static_templates_path_relative_to_here(args, args.static_path), + ] + if DOC_BUILDER_IS_SUBMODULE: + permanent_files.append("doc-builder") + + # Check some things about "permanent" files before checkout + for filename in permanent_files: + check_permanent_file(filename) + + return orig_ref, permanent_files + + +def checkout_and_build(version, permanent_files, args): + """ + Check out docs for a version and build + """ + + # Check out the git reference of this version (branch name, tag, or commit SHA) + subprocess.check_output("git checkout " + version.ref, shell=True) + + # Check out LATEST_REF version of permanent files + our_toplevel = get_toplevel_git_dir(os.getcwd()) + for filepath in permanent_files: + # Skip if not in our working tree (i.e., in a submodule or a totally different place) + file_toplevel = get_toplevel_git_dir(filepath) + if file_toplevel != our_toplevel: + continue + + # Check out file + relpath = os.path.relpath(filepath) + subprocess.check_output(["git", "checkout", LATEST_REF, "--", relpath]) + + # Build the docs for this version + build_args = [ + "-r", + args.repo_root, + "-v", + version.short_name, + "--version-display-name", + version.display_name, + "--versions", + "--site-root", + args.site_root, + "--clean", + ] + if args.build_in_container: + build_args += ["-d"] + if args.conf_py_path: + build_args += ["--conf-py-path", args.conf_py_path] + if args.static_path: + build_args += ["--static-path", args.static_path] + if args.templates_path: + build_args += ["--templates-path", args.templates_path] + if args.container_cli_tool: + build_args += ["--container-cli-tool", args.container_cli_tool] + if args.verbose: + build_args.append("--verbose") + print(f"build_logs command-line args: {build_args}") + build_docs(build_args) + + +def reset_git(orig_ref): + """ + Restore the repo to how it was before we did our checkouts + """ + # 1. Get the current ref's version of doc-builder to avoid "would be overwritten by checkout" + # errors. + if DOC_BUILDER_IS_SUBMODULE: + cmd = "git submodule update --checkout doc-builder".split(" ") + result = subprocess.run(cmd, check=False, capture_output=True, text=True) + # If doc-builder not tracked by git, that's fine. Otherwise error. + if result.returncode and "did not match any file(s) known to git" not in result.stderr: + print("PWD: " + os.getcwd()) + print(result.stdout) + print(result.stderr) + raise subprocess.CalledProcessError(result.returncode, cmd) + + # 2. Check out the original git ref (branch or commit SHA) + subprocess.check_output("git checkout " + orig_ref, shell=True) + + # 3. Restore the current version's doc-builder + if DOC_BUILDER_IS_SUBMODULE: + subprocess.check_output("git submodule update --checkout doc-builder", shell=True) + + +def check_version_list(): + """ + Check version list for problems + """ + has_default = False + for version in VERSION_LIST: + # Expect at most one version with landing_version True + if version.landing_version: + if has_default: + raise RuntimeError("Expected at most one version with landing_version True") + has_default = True + + +def main(): + """ + Loop through all versions of the documentation, building each and moving it to a directory for + publication. + """ + # Set up parser + parser = argparse.ArgumentParser() + + # Arguments shared with build_docs + parser = build_docs_shared_args(parser) + + # Custom arguments for build_docs_to_publish + parser.add_argument( + "--publish-dir", + default="_publish", + help="Where the docs should be moved after being built", + ) + + # Parse arguments + args = parser.parse_args() + if args.container_cli_tool is None: + args.container_cli_tool = get_container_cli_tool() + args.conf_py_path = os.path.relpath(os.path.abspath(args.conf_py_path)) + + # Check version list for problems + check_version_list() + + # Loop over all documentation versions + for version in VERSION_LIST: + # Check that repo is ready and get our current state + orig_ref, permanent_files = setup_this_ref(args) + + # Build this version + try: + checkout_and_build(version, permanent_files, args) + # Restore the repo to how it was before we did our checkouts, even if checkout_and_build() + # errored + finally: + reset_git(orig_ref) + + # Copy this version to the publication directory + src = os.path.join(args.repo_root, "versions", version.short_name, "html") + if version.landing_version: + dst = args.publish_dir + else: + dst = os.path.join(args.publish_dir, version.short_name) + if not os.path.exists(dst): + os.makedirs(dst) + subprocess.check_output(f"mv '{src}'/* '{dst}'/", shell=True) + + +if __name__ == "__main__": + main() diff --git a/doc/doc-builder/tools/preview_docs_pr b/doc/doc-builder/tools/preview_docs_pr new file mode 100755 index 00000000..a2f166aa --- /dev/null +++ b/doc/doc-builder/tools/preview_docs_pr @@ -0,0 +1,459 @@ +#!/usr/bin/env python +"""Given a GitHub PR URL, download the code as if it had been merged and then build the docs""" + +import urllib.request +import urllib.error +import json +import re +import os +import sys +import argparse +import subprocess +import time +import zipfile +import stat +import shutil + +# Get default location in which to clone the code +SCRATCH = os.getenv("SCRATCH") +DEFAULT_EXTRACTION_DIR_BASENAME = "preview_docs_pr.{}.{}.pr-{}" # repo owner, repo name, PR number +DEFAULT_EXTRACTION_DIR = os.path.join( + SCRATCH if SCRATCH else "", + DEFAULT_EXTRACTION_DIR_BASENAME, +) + +INDENT = 4 * " " + + +def parse_pr_url(url): + """Extract owner, repo, and PR number from a GitHub PR URL.""" + pattern = r"https://github\.com/([^/]+)/([^/]+)/pull/(\d+)" + match = re.match(pattern, url.strip()) + if not match: + raise ValueError(f"Invalid GitHub PR URL: {url}") + owner, repo, pr_number = match.groups() + return owner, repo, int(pr_number) + + +def make_api_call(url, token): + """Helper function to make a GitHub API call""" + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + if token: + headers["Authorization"] = f"Bearer {token}" + + req = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read().decode()) + + +def fetch_pr_info(owner, repo, pr_number, token=None): + """Fetch PR metadata from the GitHub API.""" + + # Get overall PR info + pr_url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}" + pr_info = make_api_call(pr_url, token) + + # Get files touched by PR + files_url = pr_url + "/files" + pr_files = make_api_call(files_url, token) + + return pr_info, pr_files + + +def fetch_pr_info_with_mergeability(owner, repo, pr_number, token=None, retries=5): + """Fetch the PR info, retrying as needed if mergeability hasn't yet been computed""" + wait_time = 10 # seconds + for attempt in range(retries): + pr, files = fetch_pr_info(owner, repo, pr_number, token) + if pr["state"] != "open" or pr.get("mergeable") is not None: + return pr, files + print( + f" Mergeability not yet computed, waiting {wait_time} seconds and retrying" + f"({attempt + 1}/{retries})..." + ) + time.sleep(wait_time) + raise RuntimeError(f"Mergeability still unknown after {retries} retries.") + + +def pick_ref(pr): + """ + Choose the best ref to download, with explanation. + """ + state = pr["state"] + merge_commit_sha = pr.get("merge_commit_sha") + mergeable = pr.get("mergeable") + + # If the PR can't be merged, we can't preview what it'd look like after merge. + # Note that "mergeable" only refers to whether there are Git conflicts; it doesn't "know" + # anything about PR tests that are passing or failing. + if not mergeable: + print("PR branch has merge conflicts, so merged docs can't be previewed. Exiting.") + sys.exit(1) + + # If PR was closed without merging, GitHub supposedly nulls out merge_commit_sha, so that + # situation would require special handling. + if state == "closed" and not pr.get("merged_at"): + raise NotImplementedError("PR closed without merge") + + if not merge_commit_sha: + raise RuntimeError("How is this possible?") + + return merge_commit_sha + + +def download_zip(extraction_dir, token, pr_info, merge_commit_sha): + """Download the code as of the merge commit from GitHub""" + print("Downloading zip from GitHub...") + owner, repo, pr_number = pr_info + zip_url = f"https://api.github.com/repos/{owner}/{repo}/zipball/{merge_commit_sha}" + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + if token: + headers["Authorization"] = f"Bearer {token}" + req = urllib.request.Request(zip_url, headers=headers) + pr_number = pr_info[2] + zip_path = os.path.join(extraction_dir, f"pr-{pr_number}.zip") + try: + with urllib.request.urlopen(req) as resp, open(zip_path, "wb") as f: + f.write(resp.read()) + except urllib.error.HTTPError as e: + raise RuntimeError(f"Failed to download zip from {zip_url}: {e}") from e + return zip_path + + +def extract_zip(extraction_dir, zip_path): + """Unzip the code""" + try: + # Unzip + extracted_path = get_extracted_path(extraction_dir, zip_path) + with zipfile.ZipFile(zip_path, "r") as zf: + print("Unzipping...") + zf.extractall(extraction_dir) + os.remove(zip_path) + except: + os.remove(zip_path) + raise + + print(f"Code downloaded to: '{extracted_path}'") + return extracted_path + + +def get_extracted_path(extraction_dir, zip_path): + """Get the path where the code will actually be extracted to""" + with zipfile.ZipFile(zip_path, "r") as zf: + # GitHub zips have a single top-level folder like "owner-repo-/" + top_level = zf.namelist()[0].split("/")[0] + if extraction_dir is None: + extracted_path = top_level + else: + extracted_path = os.path.join(extraction_dir, top_level) + return extracted_path + + +def submodule_method_to_use(zip_path: str): + """Get the submodule update method to use, if any""" + submodule_method = None + top_level = get_extracted_path(None, zip_path) + with zipfile.ZipFile(zip_path, "r") as zf: + if os.path.join(top_level, ".gitmodules") in zf.namelist(): + git_fleximod = os.path.join("bin", "git-fleximod") + if os.path.join(top_level, git_fleximod) in zf.namelist(): + submodule_method = git_fleximod + else: + submodule_method = "git submodule" + return submodule_method + + +def download_pr_code(pr_url, extraction_dir, overwrite, stdout, token=None): + """Download code as if the pull request had been merged""" + verbose = stdout is None + + # Fetch PR information via GitHub API + pr_info = parse_pr_url(pr_url) + pr, files = fetch_pr_info_with_mergeability(*pr_info, token) + + # Get the ref to download + merge_commit_sha = pick_ref(pr) + + if not os.path.exists(extraction_dir): + os.makedirs(extraction_dir) + + # Download the zip (follows redirects automatically) + zip_path = download_zip(extraction_dir, token, pr_info, merge_commit_sha) + code_dir = get_extracted_path(extraction_dir, zip_path) + + # For maximum safety, just delete any existing copies + if os.path.exists(code_dir): + if not overwrite: + raise FileExistsError( + f"Clone directory exists; add -o/--overwrite to overwrite: '{code_dir}'" + ) + print(f"Deleting existing clone dir: '{code_dir}'") + shutil.rmtree(code_dir) + + # If the repo uses git-fleximod OR .gitmodules doesn't exist, we can proceed by just extracting + # the zip. Otherwise, the repo uses regular git submodules, but the zip will not contain the + # data necessary for `git submodule update` to work. In that case, we need to actually clone + # the repo. + submodule_method = submodule_method_to_use(zip_path) + if submodule_method == "git submodule": + # We don't need the zip file anymore + print("Using regular git submodules, so deleting zip...") + os.remove(zip_path) + + download_pr_code_clone(stdout, verbose, pr_info, code_dir, submodule_method) + else: + download_pr_code_zip(extraction_dir, stdout, pr_info, zip_path, submodule_method) + + return code_dir, files + + +def download_pr_code_zip(extraction_dir, stdout, pr_info, zip_path, submodule_method): + """Extract downloaded code from ZIP and get submodules if needed""" + + # Unzip + code_dir = extract_zip(extraction_dir, zip_path) + + # Set up so that git lfs works + os.chdir(code_dir) + set_up_git(pr_info, stdout) + + # Get submodules, if needed + if os.path.exists(".gitmodules"): + print("Getting submodules...") + os.chmod(submodule_method, os.stat(submodule_method).st_mode | stat.S_IXUSR) + subprocess.check_call([submodule_method, "update"], stdout=stdout) + + +def download_pr_code_clone(stdout, verbose, pr_info, code_dir, submodule_method): + """Clone the merge commit and get submodules""" + print("Cloning...") + owner, repo, pr_number = pr_info + + # Initialize git repo in code_dir + cmd = ["git", "init", code_dir] + if verbose: + print(cmd) + subprocess.check_call(cmd, stdout=stdout) + + # cd to code_dir + os.chdir(code_dir) + + # Add origin + repo_url = f"https://github.com/{owner}/{repo}.git" + cmd = f"git remote add origin {repo_url}" + if verbose: + print(cmd) + subprocess.check_call(cmd.split(" "), stdout=stdout) + + # For the next two commands, pipe stderr to stdout to suppress it if we're not verbose. + # Unfortunately this is needed because this command puts normal printout in stderr. + + # Fetch origin to depth of 2, so we see a little of the real history if we do `git log` + cmd = f"git fetch --depth 2 origin refs/pull/{pr_number}/merge" + if verbose: + print(cmd) + subprocess.check_call(cmd.split(" "), stdout=stdout, stderr=stdout) + + # Check out the merge commit + cmd = "git checkout FETCH_HEAD" + if verbose: + print(cmd) + subprocess.check_call(cmd.split(" "), stdout=stdout, stderr=stdout) + + # Get submodules + print("Getting submodules...") + subprocess.check_call( + f"{submodule_method} update --init --recursive --depth 1".split(" "), + stdout=stdout, + ) + + +def build_docs(code_dir, files, verbose): + """Build the documentation""" + + print("Building docs...") + os.chdir("doc") + path = "./build_docs" + os.chmod(path, os.stat(path).st_mode | stat.S_IXUSR) + output = [] + cmd = [path, "-b", "_build", "-d", "-c"] + if verbose: + cmd.append("--verbose") + with subprocess.Popen( + cmd, + cwd=os.path.join(code_dir, "doc"), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, # Merge stderr into stdout + text=True, + bufsize=1, # Line buffering + env={**os.environ, "PYTHONUNBUFFERED": "1"}, # Force unbuffered output + ) as process: + for line in process.stdout: + print(line, end="") # Print to screen + output.append(line) # Save for later + output = "".join(output) + if output and os.path.exists(os.path.join("_build", "html", "index.html")): + print_files_msg(code_dir, files) + + +def set_up_git(pr_info, stdout): + """Initialize a git repo in the downloaded code so that git lfs works right""" + + owner, repo, _ = pr_info + remote_url = f"https://github.com/{owner}/{repo}.git" + subprocess.check_call(["git", "init", "-b", "preview-docs"], stdout=stdout) + subprocess.check_call(["git", "remote", "add", "origin", remote_url], stdout=stdout) + subprocess.check_call(["git", "add", "-A"], stdout=stdout) + subprocess.check_call( + [ + "git", + "commit", + "-m", + "PR snapshot for docs preview", + ], + stdout=stdout, + ) + + +def print_files_msg(code_dir, files): + """Print a message about the directly-affected files""" + build_dir = os.path.join(code_dir, "doc", "_build") + html_dir = os.path.join(build_dir, "html") + print(f"\nThe updated files are in {html_dir}") + print("Doc source files directly touched (not deleted) by PR:") + for f_dict in files: + f = f_dict["filename"] + # Slashes here are platform-independent, because f is returned from GitHub API call + # Skip deleted or otherwise nonexistent files + full_path = os.path.join(code_dir, f) + if f_dict["status"] == "removed" or not os.path.exists(full_path): + continue + + # Skip files not in doc source + if not f.startswith("doc/source/"): + continue + + # Get string to print + f_print = "/".join(f.split("/")[2:]) # Remove leading doc/source/ + root, extension = os.path.splitext(f_print) + basename = f.split("/")[-1] # pylint: disable=use-maxsplit-arg + if extension in [".rst", ".md"]: + # These types get converted to HTML + f_print = root + ".html" + assert os.path.exists(os.path.join(html_dir, f_print)) + elif os.path.exists(os.path.join(html_dir, "_images", basename)): + # Image files get put in _build/html/_images/ + f_print = os.path.join("_images", basename) + else: + f_print = "[NOT SURE WHERE THIS IS BUILT TO] " + f_print + print(INDENT + f_print) + print( + "Note that changes to these or other files may indirectly affect other doc files!" + " For example, if one of these files is a new/changed image, or a new/changed text" + " file that's `include`d somewhere, or a text file with an updated label that's cross-" + "referenced elsewhere. Or if a file outside doc/source/ that's `include`d in a doc file" + " got changed." + ) + + +def parse_args(): + """Parse arguments""" + parser = argparse.ArgumentParser( + description="Given a GitHub PR URL, download the merged version and build the docs." + ) + + parser.add_argument( + "pr_url", + help="GitHub pull request URL", + type=str, + ) + + extraction_dir_help_default = DEFAULT_EXTRACTION_DIR.format("REPO_OWNER", "REPO", "PR_NUM") + parser.add_argument( + "--dir", + "--extraction-dir", + dest="extraction_dir", + help=( + "Directory to download and extract code versions to." + f" Default: {extraction_dir_help_default}" + ), + type=str, + default=None, + ) + + parser.add_argument( + "-o", + "--overwrite", + help="Overwrite existing clone dir, if any.", + action="store_true", + ) + + parser.add_argument( + "-v", + "--verbose", + help="Verbose output, including for build_docs command.", + action="store_true", + ) + + args = parser.parse_args(sys.argv[1:]) + + # Get clone dir, if not provided + pr_info = parse_pr_url(args.pr_url) + if not args.extraction_dir: + args.extraction_dir = DEFAULT_EXTRACTION_DIR.format(*pr_info) + elif os.path.abspath(args.extraction_dir) == os.getcwd(): + args.extraction_dir = os.path.join( + os.getcwd(), DEFAULT_EXTRACTION_DIR_BASENAME.format(*pr_info) + ) + print(f"Will clone to '{args.extraction_dir}'") + + # Check that clone dir parent exists + args.extraction_dir = os.path.abspath(args.extraction_dir) + clone_parent = os.path.dirname(args.extraction_dir) + if not os.path.exists(clone_parent): + raise NotImplementedError(f"Clone parent directory does not exist: '{clone_parent}'") + + # Check that clone dir parent is not (in) a git repo + result = subprocess.run( + ["git", "-C", clone_parent, "rev-parse", "--is-inside-work-tree"], + check=False, + capture_output=True, + ) + if not result.returncode: + raise RuntimeError( + "Clone parent directory is (in) a git repo, which would cause problems: " + f"'{clone_parent}'" + ) + + return args + + +def main(): + """Main function""" + args = parse_args() + + # Handle verbosity for subprocess calls + if args.verbose: + stdout = None + else: + stdout = subprocess.DEVNULL + + # GitHub personal access tokens are currently not supported, but the code is all there. Just + # need to think about how to implement them in a way that will prevent people from putting their + # secrets into their shell history. + ( + code_dir, + files, + ) = download_pr_code(args.pr_url, args.extraction_dir, args.overwrite, stdout, token=None) + + build_docs(code_dir, files, args.verbose) + + +if __name__ == "__main__": + main()