diff --git a/README.md b/README.md index cdc83433..89a52b57 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,18 @@ codenib doctor --require core --require wiki codenib index /path/to/repository ``` +Export that indexed commit as a serverless Wiki when a live Ask backend is not +needed: + +```bash +codenib export /path/to/repository --output /tmp/repository-wiki +``` + +The export contains a versioned provenance manifest, precomputed Wiki pages, +source citations, and available page-level dependency data. It contains no +provider credential; interactive Ask and runtime graph exploration remain on +the local or MCP serving path. + See the [Quickstart](https://docs.codenib.ai/quickstart/) for ports, advanced indexing, and troubleshooting. diff --git a/codenib/cli.py b/codenib/cli.py index a8fc3f49..aa480f8b 100644 --- a/codenib/cli.py +++ b/codenib/cli.py @@ -238,6 +238,35 @@ def _run_mcp(args: argparse.Namespace) -> int: return 0 +def _run_export(args: argparse.Namespace) -> int: + repo_path = resolve_repo_path(args.repo) + manifest_path = resolve_manifest_path(str(repo_path)) + output_dir = ( + Path(args.output).expanduser().resolve() + if args.output + else manifest_path.parent / "static-wiki" + ) + + from .web.static_export import export_static_wiki + + try: + result = export_static_wiki( + repo_path, + manifest_path, + output_dir, + frontend_dir=args.frontend_dir, + base_path=args.base_path, + ) + except (OSError, RuntimeError, ValueError) as exc: + raise CLIError(str(exc)) from exc + + print(f"Static Wiki: {result.output_dir}") + print(f"Repository: {result.repo_id}") + print(f"Pages: {result.page_count}") + print(f"Manifest: {result.manifest_path}") + return 0 + + def _model_options_for_args( args: argparse.Namespace, *, @@ -976,6 +1005,27 @@ def build_parser() -> argparse.ArgumentParser: ) wiki_parser.set_defaults(handler=_run_wiki) + export_parser = subparsers.add_parser( + "export", + help="export an indexed repository Wiki for static hosting", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + export_parser.add_argument("repo", nargs="?", default=".") + export_parser.add_argument( + "--output", + help="output directory; defaults beside the repository manifest", + ) + export_parser.add_argument( + "--base-path", + default="/", + help="URL path where the static site will be mounted", + ) + export_parser.add_argument( + "--frontend-dir", + help="path to a prebuilt CodeNib frontend or web source checkout", + ) + export_parser.set_defaults(handler=_run_export) + mcp_parser = subparsers.add_parser( "mcp", help="serve an indexed repository over MCP stdio", diff --git a/codenib/web/static_export.py b/codenib/web/static_export.py new file mode 100644 index 00000000..a220694c --- /dev/null +++ b/codenib/web/static_export.py @@ -0,0 +1,684 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +"""Export a manifest-backed repository Wiki for static hosting.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import subprocess +import tempfile +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Iterable, Mapping +from urllib.parse import quote, unquote, urlsplit + +from .._version import package_version +from ..compiler.manifest import RepoManifest +from .launcher import find_frontend_dir +from .local import prepare_local_wiki + +STATIC_EXPORT_SCHEMA_VERSION = "1.0" +STATIC_EXPORT_MANIFEST = "codenib-static.json" + +_SENSITIVE_ENV_NAMES = { + "ANTHROPIC_API_KEY", + "AZURE_API_KEY", + "CODENIB_DEMO_API_KEY", + "GITHUB_TOKEN", + "GOOGLE_API_KEY", + "OPENAI_API_KEY", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", +} +_SENSITIVE_ENV_SUFFIXES = ("_API_KEY", "_TOKEN", "_SECRET") +_DOCUMENT_BASE_RE = re.compile(r"]*href=(['\"])[^'\"]*\1[^>]*>", re.I) +_DOCUMENT_REFERENCE_RE = re.compile( + r"(?P\b(?:src|href)=)(?P['\"])(?P[^'\"]*)(?P=quote)", + re.I, +) +_SAFE_BASE_PATH_RE = re.compile(r"^/[A-Za-z0-9._~!$&()*+,;=:@%/-]*$") +_SAFE_PAGE_ID_RE = re.compile(r"^[A-Za-z0-9_.-]+$") + + +@dataclass(frozen=True, slots=True) +class StaticExportResult: + """Summary returned after publishing one static Wiki directory.""" + + output_dir: Path + manifest_path: Path + repo_id: str + page_count: int + file_count: int + + +def normalize_base_path(value: str) -> str: + """Return a canonical URL path used as the static site's mount point.""" + + raw = (value or "/").strip() + parsed = urlsplit(raw) + if parsed.scheme or parsed.netloc or parsed.query or parsed.fragment: + raise ValueError("base path must be a URL path without a host or query") + if not raw.startswith("/") or "\\" in raw: + raise ValueError("base path must start with '/' and use forward slashes") + + decoded = unquote(parsed.path) + parts = PurePosixPath(decoded).parts + if any(part in {".", ".."} for part in parts): + raise ValueError("base path must not contain '.' or '..' segments") + if not _SAFE_BASE_PATH_RE.fullmatch( + parsed.path + ) or not _SAFE_BASE_PATH_RE.fullmatch(decoded): + raise ValueError("base path contains unsupported URL characters") + + normalized = "/" + "/".join(part for part in parts if part != "/") + return "/" if normalized == "/" else normalized.rstrip("/") + + +def _json_bytes(value: Any) -> bytes: + return ( + json.dumps( + value, + ensure_ascii=True, + indent=2, + sort_keys=True, + separators=(",", ": "), + ) + + "\n" + ).encode("utf-8") + + +def _write_bytes(root: Path, relative: str, content: bytes) -> Path: + target = root.joinpath(*PurePosixPath(relative).parts).resolve() + if root != target and root not in target.parents: + raise ValueError(f"export path escapes the output directory: {relative}") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + return target + + +def _write_json(root: Path, relative: str, value: Any) -> Path: + return _write_bytes(root, relative, _json_bytes(value)) + + +def _model_dict(value: Any) -> dict[str, Any]: + if hasattr(value, "model_dump"): + return dict(value.model_dump(mode="json")) + if hasattr(value, "dict"): + return dict(value.dict()) + if isinstance(value, Mapping): + return dict(value) + raise TypeError(f"unsupported repository metadata type: {type(value).__name__}") + + +def _page_ids(nodes: Iterable[Mapping[str, Any]]) -> list[str]: + result: list[str] = [] + for node in nodes: + page_id = str(node.get("id") or "").strip() + if not page_id: + raise ValueError("Wiki page tree contains an empty page id") + if not _SAFE_PAGE_ID_RE.fullmatch(page_id): + raise ValueError(f"Wiki page id is not URL-safe: {page_id!r}") + result.append(page_id) + children = node.get("children") or () + if not isinstance(children, (list, tuple)): + raise ValueError(f"Wiki page {page_id!r} has invalid children") + result.extend(_page_ids(children)) + if len(result) != len(set(result)): + raise ValueError("Wiki page tree contains duplicate page ids") + return result + + +def _source_path(value: str) -> str: + path = value.replace("\\", "/") + parts = PurePosixPath(path).parts + if path.startswith("/") or re.match(r"^[A-Za-z]:/", path): + raise ValueError(f"Wiki source path must be repository-relative: {path}") + if any(part == ".." for part in parts): + raise ValueError(f"Wiki source path contains traversal: {path}") + return PurePosixPath(path).as_posix() if path else "" + + +def _normalize_source_fields(value: Any) -> Any: + if isinstance(value, Mapping): + return { + key: ( + _source_path(str(item)) + if key == "file" and item is not None + else _normalize_source_fields(item) + ) + for key, item in value.items() + } + if isinstance(value, list): + return [_normalize_source_fields(item) for item in value] + if isinstance(value, tuple): + return [_normalize_source_fields(item) for item in value] + return value + + +def _normalize_page(builder: Any, page: Mapping[str, Any]) -> dict[str, Any]: + payload = dict(page) + citations = [] + for value in payload.get("citations") or (): + citation = dict(value) + file = _source_path(str(citation.get("file") or "")) + citation["file"] = file or None + if file and not citation.get("content"): + source = builder.source( + file, + citation.get("start_line"), + citation.get("end_line"), + ) + if source is not None: + citation["content"] = source.get("content") + citations.append(citation) + payload["citations"] = citations + + if "generation" not in payload: + payload["generation"] = { + "mode": "offline", + "model": None, + "repaired": False, + } + if "grounding" not in payload: + count = len(citations) + payload["grounding"] = { + "valid": True, + "citation_coverage": 1.0, + "cited_evidence": count, + "evidence_count": count, + "relation_count": 0, + } + return _normalize_source_fields(payload) + + +def _unavailable_page_graph(note: str = "") -> dict[str, Any]: + payload: dict[str, Any] = { + "available": False, + "nodes": [], + "edges": [], + "mermaid": "", + } + if note: + payload["note"] = note + return payload + + +def _page_graph(bundle: Any, page: Mapping[str, Any]) -> dict[str, Any]: + try: + graph = bundle.code_graph() + if graph is None: + return _unavailable_page_graph() + from .codemap import build_page_subgraph + + return _normalize_source_fields( + build_page_subgraph( + graph, + page.get("citations") or (), + repo_dir=bundle.entry.repo_dir, + hierarchy_graph=bundle.hierarchical_graph(), + ) + ) + except Exception: # noqa: BLE001 - an optional graph must not block the Wiki + return _unavailable_page_graph("The indexed dependency view was unavailable.") + + +def _prebuilt_frontend(explicit: str | os.PathLike[str] | None) -> Path: + frontend = find_frontend_dir(explicit) + if frontend is None: + raise ValueError( + "CodeNib Wiki frontend was not found; pass --frontend-dir or install " + "a release wheel containing the prebuilt frontend" + ) + if (frontend / "package.json").is_file(): + frontend = frontend / "dist" + if not (frontend / "index.html").is_file(): + raise ValueError( + f"prebuilt frontend not found under {frontend}; run " + "`cd web && npm ci && npm run build` first" + ) + return frontend + + +def _copy_frontend(source: Path, target: Path, *, base_path: str) -> None: + for candidate in source.rglob("*"): + if candidate.is_symlink(): + raise ValueError(f"frontend contains a symbolic link: {candidate}") + shutil.copytree(source, target, dirs_exist_ok=True) + + index = target / "index.html" + index_text = index.read_text(encoding="utf-8") + index_text = _DOCUMENT_BASE_RE.sub("", index_text) + + def mount_reference(match: re.Match[str]) -> str: + url = match.group("url") + if not url or url.startswith(("#", "?")): + return match.group(0) + + parsed = urlsplit(url) + if parsed.scheme or parsed.netloc: + return match.group(0) + if parsed.path.startswith("/"): + if base_path != "/": + raise ValueError( + "prebuilt frontend contains root-relative assets and cannot be " + "mounted below '/'; rebuild it with the current CodeNib frontend" + ) + return match.group(0) + + decoded = unquote(parsed.path) + if any(part in {".", ".."} for part in PurePosixPath(decoded).parts): + if not decoded.startswith("./") or any( + part == ".." for part in PurePosixPath(decoded).parts + ): + raise ValueError("prebuilt frontend asset path contains traversal") + relative = parsed.path[2:] if parsed.path.startswith("./") else parsed.path + mount = "" if base_path == "/" else base_path + mounted = f"{mount}/{relative}" + if parsed.query: + mounted += f"?{parsed.query}" + if parsed.fragment: + mounted += f"#{parsed.fragment}" + return ( + f'{match.group("prefix")}{match.group("quote")}' + f'{mounted}{match.group("quote")}' + ) + + index_text = _DOCUMENT_REFERENCE_RE.sub(mount_reference, index_text) + index.write_text(index_text, encoding="utf-8") + + +def _runtime_config(base_path: str) -> bytes: + data_base = f"{base_path.rstrip('/')}/data" if base_path != "/" else "/data" + config = { + "mode": "static", + "basePath": base_path, + "dataBase": data_base, + } + encoded = json.dumps( + config, ensure_ascii=True, sort_keys=True, separators=(",", ":") + ) + return ( + f"window.__CODENIB_RUNTIME__ = Object.freeze({encoded});\n" + 'window.__CODENIB_API_BASE__ = "";\n' + ).encode("utf-8") + + +def _not_found_page(base_path: str) -> bytes: + root = f"{base_path.rstrip('/')}/" if base_path != "/" else "/" + encoded_root = json.dumps(root) + return ( + 'CodeNib Wiki' + "" + ).encode("utf-8") + + +def _view_provenance(manifest: RepoManifest) -> dict[str, Any]: + result = {} + for name, entry in sorted(manifest.indexes.items()): + result[name] = { + "status": entry.status, + "current": manifest.index_is_current(name), + "commit": entry.commit, + "source_fingerprint": entry.source_fingerprint, + } + return result + + +def _github_repository_url(repo_path: Path) -> str | None: + result = subprocess.run( + ["git", "-C", str(repo_path), "remote", "get-url", "origin"], + capture_output=True, + check=False, + text=True, + ) + if result.returncode != 0: + return None + origin = result.stdout.strip() + if origin.startswith("git@github.com:"): + path = origin.split(":", 1)[1] + else: + parsed = urlsplit(origin) + if (parsed.hostname or "").lower() != "github.com": + return None + path = parsed.path.lstrip("/") + if path.endswith(".git"): + path = path[:-4] + path = path.strip("/") + if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", path): + return None + return f"https://github.com/{path}" + + +def _load_static_bundle(local: Any, manifest_path: Path) -> Any: + """Load only the persisted views required to render static Wiki pages.""" + + from ..index.sparse_idx.bm25_index import BM25CodeIndexer + from .config import REGISTRY_FILENAME, load_registry + from .repo_registry import RepoBundle + + manifest = RepoManifest.load(manifest_path) + bm25_entry = manifest.indexes.get("bm25") + if bm25_entry is None or not manifest.index_is_current("bm25"): + raise ValueError("static Wiki export requires a current bm25 view") + + entries = load_registry(str(local.data_dir / REGISTRY_FILENAME)) + entry = next((item for item in entries if item.instance_id == local.repo_id), None) + if entry is None: + raise ValueError(f"prepared repository {local.repo_id!r} could not be loaded") + + def load_views(bundle: Any) -> None: + index = BM25CodeIndexer() + index.load_index(bm25_entry.path) + bundle.bm25 = index + bundle.vector_store = None + + return RepoBundle( + entry=entry, + manifest=manifest, + chat_available=False, + view_loader=load_views, + runtime_loader=None, + ) + + +def _generation_summary(pages: Iterable[Mapping[str, Any]]) -> dict[str, Any]: + page_list = list(pages) + modes = sorted( + { + str((page.get("generation") or {}).get("mode") or "offline") + for page in page_list + } + ) + models = sorted( + { + str((page.get("generation") or {}).get("model")) + for page in page_list + if (page.get("generation") or {}).get("model") + } + ) + policies = sorted( + { + str((page.get("generation") or {}).get("renderer") or "index-derived") + for page in page_list + } + ) + prompt_versions = sorted( + { + str((page.get("generation") or {}).get("prompt_version")) + for page in page_list + if (page.get("generation") or {}).get("prompt_version") + } + ) + grounded = sum( + bool((page.get("grounding") or {}).get("valid")) for page in page_list + ) + return { + "modes": modes, + "models": models, + "policies": policies, + "prompt_versions": prompt_versions, + "page_count": len(page_list), + "grounding_valid_pages": grounded, + } + + +def _file_inventory(root: Path) -> list[dict[str, Any]]: + files = [] + for path in sorted(root.rglob("*")): + if not path.is_file() or path.name == STATIC_EXPORT_MANIFEST: + continue + data = path.read_bytes() + files.append( + { + "path": path.relative_to(root).as_posix(), + "bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + } + ) + return files + + +def _secret_values(environ: Mapping[str, str]) -> list[bytes]: + values = [] + for name, value in environ.items(): + upper = name.upper() + sensitive = upper in _SENSITIVE_ENV_NAMES or upper.endswith( + _SENSITIVE_ENV_SUFFIXES + ) + if sensitive and len(value) >= 8: + values.append(value.encode("utf-8")) + return values + + +def _serialized_patterns(values: Iterable[str]) -> set[bytes]: + patterns: set[bytes] = set() + for value in values: + if not value: + continue + patterns.add(value.encode("utf-8")) + escaped = json.dumps(value, ensure_ascii=True)[1:-1] + patterns.add(escaped.encode("utf-8")) + return patterns + + +def _assert_publishable( + root: Path, + *, + forbidden_paths: Iterable[Path], + environ: Mapping[str, str], +) -> None: + forbidden_values: list[str] = [] + for path in forbidden_paths: + resolved = path.resolve() + forbidden_values.extend((str(resolved), resolved.as_posix())) + forbidden = _serialized_patterns(forbidden_values) + secrets = _serialized_patterns( + value.decode("utf-8") for value in _secret_values(environ) + ) + for path in root.rglob("*"): + if path.is_symlink(): + raise ValueError(f"static export contains a symbolic link: {path}") + if not path.is_file(): + continue + data = path.read_bytes() + if any(value and value in data for value in forbidden): + raise ValueError( + "static export contains an absolute build-machine path in " + f"{path.relative_to(root)}" + ) + if any(secret in data for secret in secrets): + raise ValueError( + "static export contains a configured credential in " + f"{path.relative_to(root)}" + ) + + +def _validated_output(repo_path: Path, output_dir: Path) -> Path: + repo_path = repo_path.resolve() + output_dir = output_dir.expanduser().resolve() + if output_dir == repo_path or repo_path in output_dir.parents: + raise ValueError("static export output must be outside the target repository") + if output_dir in repo_path.parents: + raise ValueError("static export output must not contain the target repository") + if output_dir.exists() and not output_dir.is_dir(): + raise ValueError(f"static export output is not a directory: {output_dir}") + if output_dir.exists() and any(output_dir.iterdir()): + if not (output_dir / STATIC_EXPORT_MANIFEST).is_file(): + raise ValueError( + "refusing to replace a non-empty directory that is not a CodeNib " + f"static export: {output_dir}" + ) + return output_dir + + +def export_static_wiki( + repo_path: Path, + manifest_path: Path, + output_dir: Path, + *, + frontend_dir: str | os.PathLike[str] | None = None, + base_path: str = "/", + environ: Mapping[str, str] | None = None, +) -> StaticExportResult: + """Build a deterministic static Wiki from an existing repository manifest.""" + + repo_path = repo_path.expanduser().resolve() + manifest_path = manifest_path.expanduser().resolve() + output_dir = _validated_output(repo_path, output_dir) + base_path = normalize_base_path(base_path) + frontend = _prebuilt_frontend(frontend_dir) + environment = os.environ if environ is None else environ + + local = prepare_local_wiki( + repo_path, + manifest_path, + frontend_port=0, + agent_wiki=False, + ) + + from ..wiki import WikiBuilder + + bundle = _load_static_bundle(local, manifest_path) + builder = WikiBuilder(bundle) + + tree = builder.page_tree() + page_ids = _page_ids(tree) + pages = [] + graphs: dict[str, dict[str, Any]] = {} + for page_id in page_ids: + page = builder.page(page_id) + if page is None: + raise ValueError(f"Wiki page tree references a missing page: {page_id}") + normalized = _normalize_page(builder, page) + pages.append(normalized) + graphs[page_id] = _page_graph(bundle, normalized) + + repo_info = _model_dict(bundle.info()) + capabilities = {name: False for name in dict(repo_info.get("capabilities") or {})} + capabilities.update( + { + "chat": False, + "codemap": False, + "edge_labels": False, + "source_citations": True, + "static_wiki": True, + "wiki_graph": any(graph.get("available") for graph in graphs.values()), + } + ) + repo_info["capabilities"] = capabilities + repo_info["problem_statement"] = "" + repo_info["incremental"] = None + repo_info["source_url"] = _github_repository_url(repo_path) + + output_dir.parent.mkdir(parents=True, exist_ok=True) + stage = Path( + tempfile.mkdtemp( + prefix=f".{output_dir.name}.tmp-", + dir=str(output_dir.parent), + ) + ).resolve() + try: + _copy_frontend(frontend, stage, base_path=base_path) + _write_bytes(stage, "runtime-config.js", _runtime_config(base_path)) + _write_bytes(stage, "404.html", _not_found_page(base_path)) + _write_json(stage, "data/repos.json", [repo_info]) + + repo_component = quote(local.repo_id, safe="") + repo_root = f"data/repos/{repo_component}" + _write_json( + stage, + f"{repo_root}/wiki.json", + {"repo": bundle.entry.repo, "pages": tree}, + ) + _write_json( + stage, + f"{repo_root}/commits.json", + {"available": False, "commits": [], "selected": None}, + ) + for page, page_id in zip(pages, page_ids, strict=True): + component = quote(page_id, safe="") + _write_json(stage, f"{repo_root}/pages/{component}.json", page) + _write_json( + stage, + f"{repo_root}/page-graphs/{component}.json", + graphs[page_id], + ) + + _assert_publishable( + stage, + forbidden_paths=(repo_path, manifest_path.parent), + environ=environment, + ) + source_manifest = bundle.manifest + export_manifest = { + "schema_version": STATIC_EXPORT_SCHEMA_VERSION, + "repository": { + "id": local.repo_id, + "slug": bundle.entry.repo, + "url": repo_info["source_url"], + "commit": source_manifest.commit, + "source_fingerprint": source_manifest.source_fingerprint, + "languages": list(source_manifest.languages), + }, + "builder": { + "codenib_version": package_version(), + "manifest_version": source_manifest.version, + "profile": sorted( + name + for name in source_manifest.indexes + if source_manifest.index_is_current(name) + ), + }, + "source_locations": { + "path": "repository-relative-posix", + "line_base": 1, + "end_line": "inclusive", + "commit": source_manifest.commit, + }, + "views": _view_provenance(source_manifest), + "capabilities": capabilities, + "generation": _generation_summary(pages), + "base_path": base_path, + "files": _file_inventory(stage), + } + manifest_file = _write_json(stage, STATIC_EXPORT_MANIFEST, export_manifest) + _assert_publishable( + stage, + forbidden_paths=(repo_path, manifest_path.parent), + environ=environment, + ) + + if output_dir.exists(): + shutil.rmtree(output_dir) + stage.rename(output_dir) + manifest_file = output_dir / manifest_file.relative_to(stage) + except Exception: + shutil.rmtree(stage, ignore_errors=True) + raise + + return StaticExportResult( + output_dir=output_dir, + manifest_path=manifest_file, + repo_id=local.repo_id, + page_count=len(pages), + file_count=len(export_manifest["files"]), + ) + + +__all__ = [ + "STATIC_EXPORT_MANIFEST", + "STATIC_EXPORT_SCHEMA_VERSION", + "StaticExportResult", + "export_static_wiki", + "normalize_base_path", +] diff --git a/docs/quickstart.md b/docs/quickstart.md index 090bb027..63e0b3f5 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -73,6 +73,39 @@ Force a clean rebuild with: codenib wiki /path/to/repository --rebuild ``` +## Export A Static Wiki + +An existing manifest can be frozen into a serverless directory for GitHub +Pages or another static host: + +```bash +codenib index /path/to/repository +codenib export /path/to/repository --output /tmp/repository-wiki +``` + +The export records the repository commit, source fingerprint, view +capabilities, source-location convention, generation mode, and content hashes +in `codenib-static.json`. It also embeds source slices used by page citations, +so reading a page does not require the local FastAPI process. CodeNib rejects +an export when the checkout no longer matches the manifest. + +For a GitHub project Pages site, set its mount path to the repository name: + +```bash +codenib export . \ + --output /tmp/my-project \ + --base-path /my-project +``` + +The output must be outside the target checkout. CodeNib can replace a prior +CodeNib export, but it refuses to delete an unrelated non-empty directory. + +Static mode publishes only capabilities with a serverless implementation: +Wiki navigation, embedded citations, and precomputed page dependency views. +Interactive Ask, on-demand edge labels, and arbitrary dependency queries need +an authenticated CodeNib runtime and are not exposed by the export. No API key, +GitHub token, endpoint credential, or build-machine absolute path is serialized. + ## Select Repository Views | Preset | Required package | Views | diff --git a/test/test_cli.py b/test/test_cli.py index b57a2851..2ad07fc8 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -20,11 +20,30 @@ def test_parser_exposes_release_commands() -> None: parser = cli.build_parser() - for command in ("index", "wiki", "mcp", "doctor"): + for command in ("index", "wiki", "export", "mcp", "doctor"): args = parser.parse_args([command]) assert args.command == command +def test_export_parser_accepts_pages_mount_options() -> None: + args = cli.build_parser().parse_args( + [ + "export", + ".", + "--output", + "/tmp/wiki", + "--base-path", + "/project", + "--frontend-dir", + "/tmp/frontend", + ] + ) + + assert args.output == "/tmp/wiki" + assert args.base_path == "/project" + assert args.frontend_dir == "/tmp/frontend" + + def test_wiki_parser_accepts_headless_quality_audit() -> None: args = cli.build_parser().parse_args(["wiki", ".", "--audit", "--audit-json"]) diff --git a/test/web/test_static_export.py b/test/web/test_static_export.py new file mode 100644 index 00000000..94cbb183 --- /dev/null +++ b/test/web/test_static_export.py @@ -0,0 +1,380 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +from pathlib import Path, PureWindowsPath +from types import SimpleNamespace + +import pytest + +from codenib.compiler.manifest import IndexEntry, RepoManifest +from codenib.web.static_export import ( + STATIC_EXPORT_MANIFEST, + _assert_publishable, + export_static_wiki, + normalize_base_path, +) + + +class _Builder: + def __init__(self, _bundle) -> None: + self.pages = { + "overview": { + "id": "overview", + "title": "Overview", + "markdown": "# Overview\n\nThe runtime is source linked.", + "citations": [ + { + "file": "src/runtime.py", + "start_line": 1, + "end_line": 2, + "node_name": "run", + "type": "function", + "score": None, + "content": None, + } + ], + "diagram": "", + }, + "architecture": { + "id": "architecture", + "title": "Architecture", + "markdown": "# Architecture", + "citations": [], + "diagram": "", + }, + } + + def page_tree(self): + return [ + {"id": "overview", "title": "Overview", "children": []}, + {"id": "architecture", "title": "Architecture", "children": []}, + ] + + def page(self, page_id): + return self.pages.get(page_id) + + def source(self, file, start, end): + assert (file, start, end) == ("src/runtime.py", 1, 2) + return { + "file": file, + "start_line": start, + "end_line": end, + "content": "def run():\n return 'ready'\n", + } + + +def _frontend(root: Path) -> Path: + frontend = root / "frontend" + (frontend / "assets").mkdir(parents=True) + (frontend / "index.html").write_text( + "" + "" + "" + "ModuleHeading" + "", + encoding="utf-8", + ) + (frontend / "runtime-config.js").write_text( + 'window.__CODENIB_API_BASE__ = "";\n', encoding="utf-8" + ) + (frontend / "assets" / "app.js").write_text( + "console.log('wiki');\n", encoding="utf-8" + ) + return frontend + + +def _manifest(repo: Path, artifact: Path) -> tuple[RepoManifest, Path]: + entry = IndexEntry( + index_type="bm25", + path=str(artifact / "bm25"), + built_at="2026-08-04T00:00:00Z", + built_at_epoch=0.0, + status="fresh", + commit="abc123", + source_fingerprint="source-1", + ) + manifest = RepoManifest( + repo_path=str(repo), + commit="abc123", + source_fingerprint="source-1", + languages=["python"], + file_count=1, + indexes={"bm25": entry}, + capabilities={"sparse_search": True}, + ) + path = artifact / "repo_manifest.json" + manifest.save(path) + return manifest, path + + +@pytest.fixture +def export_setup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + repo = tmp_path / "repo" + repo.mkdir() + (repo / "src").mkdir() + (repo / "src" / "runtime.py").write_text( + "def run():\n return 'ready'\n", encoding="utf-8" + ) + artifact = tmp_path / "artifact" + manifest, manifest_path = _manifest(repo, artifact) + bundle = SimpleNamespace( + entry=SimpleNamespace( + instance_id="demo", + repo="owner/demo", + repo_dir=str(repo), + ), + manifest=manifest, + info=lambda: { + "id": "demo", + "name": "owner/demo @ abc123", + "repo": "owner/demo", + "base_commit": "abc123", + "commit_short": "abc123", + "language": "python", + "description": "A demo repository.", + "problem_statement": "private benchmark text", + "languages": ["python"], + "file_count": 1, + "capabilities": {"sparse_search": True, "chat": True}, + "graph_coverage": None, + }, + code_graph=lambda: None, + hierarchical_graph=lambda: None, + ) + local = SimpleNamespace( + repo_id="demo", + data_dir=artifact / "wiki", + config_path=artifact / "wiki" / "config.yaml", + ) + monkeypatch.setattr( + "codenib.web.static_export.prepare_local_wiki", + lambda *_args, **_kwargs: local, + ) + monkeypatch.setattr( + "codenib.web.static_export._load_static_bundle", + lambda _local, _manifest_path: bundle, + ) + monkeypatch.setattr("codenib.wiki.WikiBuilder", _Builder) + + return SimpleNamespace( + repo=repo, + artifact=artifact, + manifest_path=manifest_path, + frontend=_frontend(tmp_path), + output=tmp_path / "site", + ) + + +def _tree_bytes(root: Path) -> dict[str, bytes]: + return { + path.relative_to(root).as_posix(): path.read_bytes() + for path in sorted(root.rglob("*")) + if path.is_file() + } + + +def test_static_export_is_deterministic_and_publishable( + export_setup, monkeypatch: pytest.MonkeyPatch +) -> None: + setup = export_setup + monkeypatch.setattr( + "codenib.web.static_export._github_repository_url", + lambda _repo: "https://github.com/owner/demo", + ) + first = export_static_wiki( + setup.repo, + setup.manifest_path, + setup.output, + frontend_dir=setup.frontend, + base_path="/demo", + environ={"OPENAI_API_KEY": "not-present-in-output"}, + ) + first_bytes = _tree_bytes(setup.output) + + second = export_static_wiki( + setup.repo, + setup.manifest_path, + setup.output, + frontend_dir=setup.frontend, + base_path="/demo", + environ={"OPENAI_API_KEY": "not-present-in-output"}, + ) + + assert first.page_count == second.page_count == 2 + assert _tree_bytes(setup.output) == first_bytes + manifest = json.loads((setup.output / STATIC_EXPORT_MANIFEST).read_text()) + assert manifest["schema_version"] == "1.0" + assert manifest["repository"]["commit"] == "abc123" + assert manifest["repository"]["url"] == "https://github.com/owner/demo" + assert manifest["source_locations"]["line_base"] == 1 + assert manifest["capabilities"]["static_wiki"] is True + assert manifest["capabilities"]["chat"] is False + assert manifest["capabilities"]["codemap"] is False + assert manifest["capabilities"]["sparse_search"] is False + assert manifest["views"]["bm25"]["current"] is True + + repos = json.loads((setup.output / "data" / "repos.json").read_text()) + assert repos[0]["problem_statement"] == "" + assert repos[0]["source_url"] == "https://github.com/owner/demo" + page = json.loads( + ( + setup.output / "data" / "repos" / "demo" / "pages" / "overview.json" + ).read_text() + ) + assert page["citations"][0]["content"].startswith("def run") + assert page["generation"]["mode"] == "offline" + assert b"not-present-in-output" not in b"".join(first_bytes.values()) + assert str(setup.repo).encode() not in b"".join(first_bytes.values()) + runtime = (setup.output / "runtime-config.js").read_text() + assert '"basePath":"/demo"' in runtime + assert '"mode":"static"' in runtime + index = (setup.output / "index.html").read_text() + assert " None: + root = tmp_path / "site" + root.mkdir() + windows_path = PureWindowsPath(r"C:\build\repository") + (root / "page.json").write_text( + json.dumps({"source": str(windows_path)}), + encoding="utf-8", + ) + + class ResolvedWindowsPath: + def resolve(self): + return windows_path + + with pytest.raises(ValueError, match="absolute build-machine path"): + _assert_publishable( + root, + forbidden_paths=(ResolvedWindowsPath(),), + environ={}, + ) + + +def test_static_export_rejects_a_configured_secret( + export_setup, monkeypatch: pytest.MonkeyPatch +) -> None: + secret = "provider-secret-value" + + class SecretBuilder(_Builder): + def __init__(self, bundle) -> None: + super().__init__(bundle) + self.pages["overview"]["markdown"] = f"# Overview\n\n{secret}" + + monkeypatch.setattr("codenib.wiki.WikiBuilder", SecretBuilder) + + with pytest.raises(ValueError, match="configured credential"): + export_static_wiki( + export_setup.repo, + export_setup.manifest_path, + export_setup.output, + frontend_dir=export_setup.frontend, + environ={"OPENAI_API_KEY": secret}, + ) + assert not export_setup.output.exists() + + +def test_static_export_advertises_only_precomputed_page_graphs( + export_setup, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + "codenib.web.static_export._page_graph", + lambda _bundle, _page: { + "available": True, + "nodes": [{"id": "run"}], + "edges": [], + "mermaid": "", + }, + ) + + export_static_wiki( + export_setup.repo, + export_setup.manifest_path, + export_setup.output, + frontend_dir=export_setup.frontend, + ) + + repos = json.loads((export_setup.output / "data" / "repos.json").read_text()) + assert repos[0]["capabilities"]["wiki_graph"] is True + assert repos[0]["capabilities"]["codemap"] is False + graph = json.loads( + ( + export_setup.output + / "data" + / "repos" + / "demo" + / "page-graphs" + / "overview.json" + ).read_text() + ) + assert graph["available"] is True + + +def test_static_export_does_not_replace_an_unrelated_directory(export_setup) -> None: + export_setup.output.mkdir() + (export_setup.output / "keep.txt").write_text("keep", encoding="utf-8") + + with pytest.raises(ValueError, match="refusing to replace"): + export_static_wiki( + export_setup.repo, + export_setup.manifest_path, + export_setup.output, + frontend_dir=export_setup.frontend, + ) + + assert (export_setup.output / "keep.txt").read_text() == "keep" + + +def test_static_export_rejects_absolute_citation_paths( + export_setup, monkeypatch: pytest.MonkeyPatch +) -> None: + class AbsolutePathBuilder(_Builder): + def __init__(self, bundle) -> None: + super().__init__(bundle) + self.pages["overview"]["citations"][0]["file"] = "/tmp/source.py" + + monkeypatch.setattr("codenib.wiki.WikiBuilder", AbsolutePathBuilder) + + with pytest.raises(ValueError, match="repository-relative"): + export_static_wiki( + export_setup.repo, + export_setup.manifest_path, + export_setup.output, + frontend_dir=export_setup.frontend, + ) + + +@pytest.mark.parametrize( + ("value", "expected"), + [("/", "/"), ("/demo/", "/demo"), ("/org/demo", "/org/demo")], +) +def test_normalize_base_path(value: str, expected: str) -> None: + assert normalize_base_path(value) == expected + + +@pytest.mark.parametrize( + "value", + [ + "demo", + "https://example.com/demo", + "/demo?token=x", + "/demo/../other", + "/demo%22onload", + "/demo%3Fquery", + "/demo%23fragment", + '/demo" onload="alert(1)', + ], +) +def test_normalize_base_path_rejects_unsafe_values(value: str) -> None: + with pytest.raises(ValueError): + normalize_base_path(value) diff --git a/web/app/[repoId]/page.tsx b/web/app/[repoId]/page.tsx index 905e985f..96b3270c 100644 --- a/web/app/[repoId]/page.tsx +++ b/web/app/[repoId]/page.tsx @@ -5,6 +5,7 @@ import Header from "@/components/Header"; import Markdown from "@/components/Markdown"; import AskBar from "@/components/AskBar"; import { AppLink } from "@/lib/router"; +import { isStaticRuntime } from "@/lib/runtime"; import { fetchCommits, fetchRepos, @@ -54,6 +55,7 @@ function stripGeneratedDiagrams( // Link a repo-relative source path to the exact blob on GitHub at the indexed commit. function ghFileUrl( repo: string | undefined, + sourceUrl: string | null | undefined, commit: string | undefined, file: string, start?: number | null, @@ -61,7 +63,8 @@ function ghFileUrl( ): string | null { if (!repo) return null; const lines = start ? `#L${start}${end && end !== start ? `-L${end}` : ""}` : ""; - return `https://github.com/${repo}/blob/${commit || "HEAD"}/${file}${lines}`; + const root = (sourceUrl || `https://github.com/${repo}`).replace(/\/+$/, ""); + return `${root}/blob/${commit || "HEAD"}/${file}${lines}`; } function TocTree({ @@ -123,6 +126,8 @@ function commitEvidence(commits: CommitRef[], selected?: string): string | null export default function WikiPageView({ repoId }: { repoId: string }) { + const staticRuntime = isStaticRuntime(); + const [repo, setRepo] = useState(null); const [pages, setPages] = useState([]); const [activeId, setActiveId] = useState("overview"); @@ -289,6 +294,7 @@ export default function WikiPageView({ repoId }: { repoId: string }) { }, [sourceCitation]); const hasGraph = !!repo?.capabilities?.codemap; + const hasPageGraph = hasGraph || !!repo?.capabilities?.wiki_graph; const generationMode = page?.generation?.mode ?? "offline"; // "Source checked" is the strict claim: every substantial block is cited // and every referenced source identifier resolves. Generated pages that only @@ -329,36 +335,38 @@ export default function WikiPageView({ repoId }: { repoId: string }) { / {repo ? repo.repo : repoId} - + + + + + + + Dependency Map + + )} {page && /} {page && {page.title}} @@ -445,7 +453,7 @@ export default function WikiPageView({ repoId }: { repoId: string }) { )} )} - {hasGraph && ( + {hasPageGraph && (
openGraph(label)} + onFocus={hasGraph ? (label) => openGraph(label) : undefined} repoFullName={repo?.repo} commit={repo?.base_commit} /> @@ -500,6 +508,7 @@ export default function WikiPageView({ repoId }: { repoId: string }) { {page.evidence.items.map((item) => { const url = ghFileUrl( repo?.repo, + repo?.source_url, repo?.base_commit, item.file, item.start_line, @@ -556,7 +565,12 @@ export default function WikiPageView({ repoId }: { repoId: string }) { Relevant source files ({wikiFiles.length})
{wikiFiles.map((f) => { - const url = ghFileUrl(repo?.repo, repo?.base_commit, f); + const url = ghFileUrl( + repo?.repo, + repo?.source_url, + repo?.base_commit, + f, + ); return url ? ( new Promise((resolve) => window.setTimeout(resolve, ms)); export default function Landing() { + const staticRuntime = isStaticRuntime(); const [repos, setRepos] = useState([]); const [error, setError] = useState(null); const [loading, setLoading] = useState(true); @@ -78,8 +80,9 @@ export default function Landing() { const run = async () => { let lastError: unknown = null; - for (let attempt = 0; attempt < repoRetryDelays.length; attempt += 1) { - const delay = repoRetryDelays[attempt]; + const delays = staticRuntime ? [0] : repoRetryDelays; + for (let attempt = 0; attempt < delays.length; attempt += 1) { + const delay = delays[attempt]; if (delay > 0) { if (active) setError("Connecting to backend; retrying repository list..."); await sleep(delay); @@ -152,24 +155,30 @@ export default function Landing() {
- - + - Add repo - - - - - - + {!staticRuntime && ( + + + + Add repo + + + + + + + )} {error && (

- Backend unavailable — start it with codenib-web after building an index. + {staticRuntime ? ( + "Static Wiki data is unavailable." + ) : ( + <>Backend unavailable — start it with codenib-web after building an index. + )}

Request failed: {error}