From 3c29ddd7561a98ad1bf1ea6709b479130bba3020 Mon Sep 17 00:00:00 2001 From: fishmingyu <1661342068@qq.com> Date: Tue, 4 Aug 2026 23:06:42 -0700 Subject: [PATCH 1/8] feat(artifacts): package portable repository context Add commit-validated BM25/vector serving artifacts and a publish CLI that pairs them with static Wiki exports. Keep credentials, machine paths, mutable vector state, and overlapping output roots out of distributable data.\n\nVerified with focused artifact/CLI/static-export tests and the full unit tier. --- codenib/artifacts/__init__.py | 21 ++ codenib/artifacts/context.py | 432 ++++++++++++++++++++++++ codenib/artifacts/security.py | 173 ++++++++++ codenib/cli.py | 175 ++++++++++ codenib/compiler/checkout_identity.py | 63 ++++ codenib/web/local.py | 47 +-- codenib/web/static_export.py | 101 ++---- test/artifacts/test_context_artifact.py | 364 ++++++++++++++++++++ test/artifacts/test_publish.py | 179 ++++++++++ test/test_cli.py | 44 ++- test/test_cli_remote_embeddings.py | 19 ++ test/web/test_static_export.py | 15 +- 12 files changed, 1506 insertions(+), 127 deletions(-) create mode 100644 codenib/artifacts/__init__.py create mode 100644 codenib/artifacts/context.py create mode 100644 codenib/artifacts/security.py create mode 100644 codenib/compiler/checkout_identity.py create mode 100644 test/artifacts/test_context_artifact.py create mode 100644 test/artifacts/test_publish.py diff --git a/codenib/artifacts/__init__.py b/codenib/artifacts/__init__.py new file mode 100644 index 00000000..b06e05d8 --- /dev/null +++ b/codenib/artifacts/__init__.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +"""Portable repository-context artifacts.""" + +from .context import ( + CONTEXT_ARTIFACT_MANIFEST, + CONTEXT_ARTIFACT_SCHEMA, + PORTABLE_CONTEXT_VIEWS, + ContextArtifactResult, + stage_context_artifact, +) + +__all__ = [ + "CONTEXT_ARTIFACT_MANIFEST", + "CONTEXT_ARTIFACT_SCHEMA", + "PORTABLE_CONTEXT_VIEWS", + "ContextArtifactResult", + "stage_context_artifact", +] diff --git a/codenib/artifacts/context.py b/codenib/artifacts/context.py new file mode 100644 index 00000000..8b7f000f --- /dev/null +++ b/codenib/artifacts/context.py @@ -0,0 +1,432 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +"""Stage a portable, commit-addressed repository-context artifact.""" + +from __future__ import annotations + +import json +import os +import pickle +import re +import shutil +import subprocess +import tempfile +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Iterable, Mapping, Sequence + +from .. import compat_pickle +from .._version import package_version +from ..compiler.artifact_fingerprints import bm25_artifact_file_fingerprints +from ..compiler.checkout_identity import validate_checkout_identity +from ..compiler.manifest import MANIFEST_FILENAME, RepoManifest +from ..compiler.snapshot_store import normalize_repo +from ..provider_routes import resolve_embedding_artifact_route +from .security import assert_no_credential_fields, assert_publishable_tree, file_sha256 + +CONTEXT_ARTIFACT_SCHEMA = "codenib.context-artifact.v1" +CONTEXT_ARTIFACT_MANIFEST = "codenib-context.json" +PORTABLE_CONTEXT_VIEWS = frozenset({"bm25", "vector"}) +_VIEW_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+$") + + +@dataclass(frozen=True, slots=True) +class ContextArtifactResult: + """Paths and identity of one staged context artifact.""" + + output_dir: Path + metadata_path: Path + manifest_path: Path + repository: str + commit: str + views: tuple[str, ...] + file_count: int + byte_count: int + + +def _json_bytes(value: Any) -> bytes: + return ( + json.dumps( + value, + ensure_ascii=True, + indent=2, + sort_keys=True, + separators=(",", ": "), + ) + + "\n" + ).encode("utf-8") + + +def _write_json(root: Path, relative: str, value: Any) -> Path: + target = root.joinpath(*PurePosixPath(relative).parts).resolve() + if root != target and root not in target.parents: + raise ValueError(f"artifact path escapes the output directory: {relative}") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(_json_bytes(value)) + return target + + +def _repository_slug(repo_path: Path, explicit: str | None) -> str: + if explicit: + return normalize_repo(explicit) + try: + result = subprocess.run( + ["git", "-C", str(repo_path), "config", "--get", "remote.origin.url"], + check=True, + capture_output=True, + text=True, + timeout=10, + ) + origin = result.stdout.strip() + except (OSError, subprocess.SubprocessError): + origin = "" + return normalize_repo(origin or repo_path.name) + + +def _validated_output(repo_path: Path, manifest_root: Path, output_dir: Path) -> Path: + output = output_dir.expanduser().resolve() + for source, label in ( + (repo_path.resolve(), "repository"), + (manifest_root.resolve(), "index root"), + ): + if output == source or source in output.parents or output in source.parents: + raise ValueError(f"context artifact output overlaps the {label}: {output}") + if output.exists() and not output.is_dir(): + raise ValueError(f"context artifact output is not a directory: {output}") + if output.exists() and any(output.iterdir()): + if not (output / CONTEXT_ARTIFACT_MANIFEST).is_file(): + raise ValueError( + "refusing to replace a non-empty directory that is not a CodeNib " + f"context artifact: {output}" + ) + return output + + +def _view_source(entry_path: str, manifest_root: Path, *, view: str) -> Path: + source = Path(entry_path).expanduser() + if not source.is_absolute(): + source = manifest_root / source + source = source.resolve() + if source != manifest_root and manifest_root not in source.parents: + raise ValueError(f"view {view!r} is outside the manifest index root: {source}") + if not source.exists(): + raise ValueError(f"view {view!r} is missing: {source}") + return source + + +def _copy_view(source: Path, stage: Path, view: str) -> str: + if not _VIEW_NAME_RE.fullmatch(view): + raise ValueError(f"invalid context artifact view name: {view!r}") + for candidate in (source, *source.rglob("*")) if source.is_dir() else (source,): + if candidate.is_symlink(): + raise ValueError(f"view {view!r} contains a symbolic link: {candidate}") + + destination = stage / "views" / view + if source.is_dir(): + shutil.copytree(source, destination) + return destination.relative_to(stage).as_posix() + destination.mkdir(parents=True) + target = destination / source.name + shutil.copy2(source, target) + return target.relative_to(stage).as_posix() + + +def _normalize_copied_view( + stage: Path, + *, + repo_path: Path, + view: str, + relative: str, +) -> dict[str, Any]: + """Rewrite view-local machine paths and return identity adjustments.""" + + target = stage.joinpath(*PurePosixPath(relative).parts) + if view == "vector": + return _normalize_vector_view(target, repo_path) + if view != "bm25": + raise ValueError( + f"view {view!r} is not yet supported by portable context artifacts; " + "select bm25 and/or vector" + ) + root = target if target.is_dir() else target.parent + metadata_path = root / "bm25_metadata.json" + if not metadata_path.is_file(): + raise ValueError("portable BM25 view is missing bm25_metadata.json") + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + if not isinstance(metadata, dict): + raise ValueError("portable BM25 metadata must be a JSON object") + metadata["project_root"] = "source" + metadata_path.write_bytes(_json_bytes(metadata)) + return {"artifact_file_fingerprints": bm25_artifact_file_fingerprints(root)} + + +def _portable_source_path(value: object, repo_path: Path, *, source: str) -> str: + raw = str(value or "") + if not raw: + return "" + path = Path(raw).expanduser() + if path.is_absolute(): + try: + path = path.resolve().relative_to(repo_path) + except ValueError as exc: + raise ValueError(f"{source} points outside the repository: {raw}") from exc + normalized = PurePosixPath(path.as_posix()) + if normalized.is_absolute() or ".." in normalized.parts: + raise ValueError(f"{source} is not repository-relative: {raw}") + return normalized.as_posix() + + +def _normalize_vector_documents(path: Path, repo_path: Path) -> None: + with path.open("rb") as handle: + documents = compat_pickle.load(handle) + if not isinstance(documents, list): + raise ValueError(f"portable vector documents must be a list: {path.name}") + for index, document in enumerate(documents): + metadata = getattr(document, "metadata", None) + if not isinstance(metadata, dict): + raise ValueError( + f"portable vector document {index} has invalid metadata: {path.name}" + ) + metadata["file"] = _portable_source_path( + metadata.get("file"), + repo_path, + source=f"vector document {index} file", + ) + with path.open("wb") as handle: + pickle.dump(documents, handle, protocol=pickle.HIGHEST_PROTOCOL) + + +def _normalize_vector_view(target: Path, repo_path: Path) -> dict[str, Any]: + if not target.is_dir(): + raise ValueError("portable vector view must be a directory") + + # Query serving does not need the mutable state used to build the next + # commit. Excluding it keeps the downloadable artifact smaller and avoids + # publishing build-machine paths from incremental caches. + for name in ( + "chunk_store.json", + "chunk_store.pkl", + "embeddings_cache.json", + "embeddings_cache.npz", + "embeddings_cache.pkl", + "incremental_state.json", + ): + (target / name).unlink(missing_ok=True) + + document_files = sorted(target.glob("l[02]/documents_*.pkl")) + if not document_files: + raise ValueError( + "portable vector view requires the current documents_*.pkl format; " + "rebuild the vector view" + ) + for path in document_files: + _normalize_vector_documents(path, repo_path) + + # The current document files supersede legacy LangChain docstore pickles. + # Leaving both formats would retain duplicate absolute source paths. + for legacy in target.glob("l[02]/index_*.pkl"): + legacy.unlink() + return {"artifact_scope": "query-serving"} + + +def _inventory(root: Path) -> list[dict[str, Any]]: + files = [] + for path in sorted( + candidate for candidate in root.rglob("*") if candidate.is_file() + ): + size, digest = file_sha256(path) + files.append( + { + "path": path.relative_to(root).as_posix(), + "bytes": size, + "sha256": digest, + } + ) + return files + + +def _selected_views( + manifest: RepoManifest, + requested: Sequence[str] | None, +) -> tuple[str, ...]: + available = {name for name in manifest.indexes if manifest.index_is_current(name)} + if requested is None: + selected = sorted(available) + else: + selected = list(dict.fromkeys(str(name).strip() for name in requested)) + missing = sorted(name for name in selected if name not in available) + if missing: + raise ValueError( + "context artifact requires current views: " + ", ".join(missing) + ) + if not selected: + raise ValueError("context artifact requires at least one current view") + return tuple(selected) + + +def _portable_capabilities( + capabilities: Mapping[str, bool], + views: Iterable[str], +) -> dict[str, bool]: + selected = set(views) + result = dict(capabilities) + result.update( + { + "sparse_search": "bm25" in selected, + "dense_search": "vector" in selected, + "hybrid_search": {"bm25", "vector"} <= selected, + "symbol_navigation": "symbol_graph" in selected, + } + ) + return result + + +def stage_context_artifact( + repo_path: Path, + manifest_path: Path, + output_dir: Path, + *, + repository: str | None = None, + views: Sequence[str] | None = None, + environ: Mapping[str, str] | None = None, + validate_checkout: bool = True, +) -> ContextArtifactResult: + """Copy current views into an atomic, path-independent artifact directory.""" + + repo_path = repo_path.expanduser().resolve() + manifest_path = manifest_path.expanduser().resolve() + if not repo_path.is_dir(): + raise ValueError(f"repository directory does not exist: {repo_path}") + if not manifest_path.is_file(): + raise ValueError(f"repository manifest does not exist: {manifest_path}") + manifest_root = manifest_path.parent + output_dir = _validated_output(repo_path, manifest_root, output_dir) + environment = os.environ if environ is None else environ + manifest = RepoManifest.load(manifest_path) + if validate_checkout: + validate_checkout_identity( + repo_path, + manifest, + artifact_root=manifest_root, + ) + selected = _selected_views(manifest, views) + unsupported = sorted(set(selected) - PORTABLE_CONTEXT_VIEWS) + if unsupported: + raise ValueError( + "portable context artifacts do not yet support views: " + + ", ".join(unsupported) + ) + slug = _repository_slug(repo_path, repository) + + output_dir.parent.mkdir(parents=True, exist_ok=True) + stage = Path( + tempfile.mkdtemp( + prefix=f".{output_dir.name}.tmp-", + dir=str(output_dir.parent), + ) + ).resolve() + try: + portable = manifest.to_dict() + portable["repo"]["path"] = "source" + portable_indexes: dict[str, Any] = {} + for view in selected: + entry = manifest.indexes[view] + assert_no_credential_fields(entry.config, source=f"view {view!r} config") + assert_no_credential_fields( + entry.metadata, + source=f"view {view!r} metadata", + ) + if view == "vector": + resolve_embedding_artifact_route(entry.config) + source = _view_source(entry.path, manifest_root, view=view) + relative = _copy_view(source, stage, view) + adjustments = _normalize_copied_view( + stage, + repo_path=repo_path, + view=view, + relative=relative, + ) + entry_data = entry.to_dict() + entry_data["path"] = relative + for section in ("config", "metadata"): + entry_data[section].update(adjustments) + portable_indexes[view] = entry_data + portable["indexes"] = portable_indexes + portable["capabilities"] = _portable_capabilities( + manifest.capabilities, + selected, + ) + portable_manifest = _write_json(stage, MANIFEST_FILENAME, portable) + + assert_publishable_tree( + stage, + forbidden_paths=(repo_path, manifest_root), + environ=environment, + label="context artifact", + ) + files = _inventory(stage) + metadata = { + "schema": CONTEXT_ARTIFACT_SCHEMA, + "repository": { + "slug": slug, + "commit": manifest.commit, + "source_fingerprint": manifest.source_fingerprint, + "languages": list(manifest.languages), + }, + "builder": { + "codenib_version": package_version(), + "manifest_version": manifest.version, + "compiled_at": manifest.compiled_at, + }, + "manifest": { + "path": MANIFEST_FILENAME, + "repository_path": "source", + "paths": "artifact-relative-posix", + }, + "source_locations": { + "path": "repository-relative-posix", + "line_base": 1, + "end_line": "inclusive", + "commit": manifest.commit, + }, + "views": list(selected), + "capabilities": portable["capabilities"], + "files": files, + } + metadata_path = _write_json(stage, CONTEXT_ARTIFACT_MANIFEST, metadata) + assert_publishable_tree( + stage, + forbidden_paths=(repo_path, manifest_root), + environ=environment, + label="context artifact", + ) + + if output_dir.exists(): + shutil.rmtree(output_dir) + os.replace(stage, output_dir) + except BaseException: + shutil.rmtree(stage, ignore_errors=True) + raise + + byte_count = sum(int(item["bytes"]) for item in files) + return ContextArtifactResult( + output_dir=output_dir, + metadata_path=output_dir / metadata_path.relative_to(stage), + manifest_path=output_dir / portable_manifest.relative_to(stage), + repository=slug, + commit=manifest.commit, + views=selected, + file_count=len(files), + byte_count=byte_count, + ) + + +__all__ = [ + "CONTEXT_ARTIFACT_MANIFEST", + "CONTEXT_ARTIFACT_SCHEMA", + "PORTABLE_CONTEXT_VIEWS", + "ContextArtifactResult", + "stage_context_artifact", +] diff --git a/codenib/artifacts/security.py b/codenib/artifacts/security.py new file mode 100644 index 00000000..cd1dfb9f --- /dev/null +++ b/codenib/artifacts/security.py @@ -0,0 +1,173 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +"""Publication guards shared by static sites and context artifacts.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any, Iterable, Mapping + +_SENSITIVE_ENV_NAMES = { + "ANTHROPIC_API_KEY", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AZURE_API_KEY", + "CODENIB_DEMO_API_KEY", + "CODENIB_ACTION_EMBEDDING_KEY", + "CODENIB_EMBEDDING_API_KEY", + "GH_TOKEN", + "GITHUB_TOKEN", + "GOOGLE_API_KEY", + "OPENAI_API_KEY", +} +_SENSITIVE_ENV_SUFFIXES = ("_API_KEY", "_TOKEN", "_SECRET") +_SCAN_CHUNK_BYTES = 1024 * 1024 +_CREDENTIAL_FIELDS = { + "access_token", + "api_key", + "apikey", + "authorization", + "bearer_token", + "client_secret", + "headers", + "password", + "private_key", + "secret_key", + "token", +} + + +def assert_no_credential_fields(value: Any, *, source: str) -> None: + """Reject credential-shaped keys in metadata intended for publication.""" + + if isinstance(value, Mapping): + for key, item in value.items(): + name = str(key).strip().lower().replace("-", "_") + if name in _CREDENTIAL_FIELDS: + raise ValueError(f"{source} contains a credential field: {key}") + assert_no_credential_fields(item, source=source) + elif isinstance(value, (list, tuple)): + for item in value: + assert_no_credential_fields(item, source=source) + + +def _serialized_patterns(values: Iterable[str]) -> tuple[bytes, ...]: + patterns: set[bytes] = set() + for value in values: + if not value: + continue + patterns.add(value.encode("utf-8")) + escaped = json.dumps(value, ensure_ascii=True)[1:-1] + patterns.add(escaped.encode("utf-8")) + return tuple(sorted(patterns, key=lambda item: (-len(item), item))) + + +def _secret_values(environ: Mapping[str, str]) -> tuple[bytes, ...]: + values: set[str] = set() + for name, value in environ.items(): + upper = name.upper() + sensitive = upper in _SENSITIVE_ENV_NAMES or upper.endswith( + _SENSITIVE_ENV_SUFFIXES + ) + if sensitive and len(value) >= 8: + values.add(value) + return _serialized_patterns(values) + + +def file_sha256(path: Path) -> tuple[int, str]: + """Return byte length and SHA-256 without loading a full artifact file.""" + + digest = hashlib.sha256() + size = 0 + with path.open("rb") as handle: + while chunk := handle.read(_SCAN_CHUNK_BYTES): + size += len(chunk) + digest.update(chunk) + return size, digest.hexdigest() + + +def _matching_kind( + path: Path, + *, + forbidden: tuple[bytes, ...], + secrets: tuple[bytes, ...], +) -> str | None: + needles = tuple( + (kind, needle) + for kind, values in (("path", forbidden), ("secret", secrets)) + for needle in values + if needle + ) + if not needles: + return None + overlap = max(len(needle) for _kind, needle in needles) - 1 + tail = b"" + with path.open("rb") as handle: + while chunk := handle.read(_SCAN_CHUNK_BYTES): + window = tail + chunk + for kind, needle in needles: + if needle in window: + return kind + tail = window[-overlap:] if overlap else b"" + return None + + +def _assert_publishable_file( + path: Path, + *, + root: Path, + forbidden: tuple[bytes, ...], + secrets: tuple[bytes, ...], + label: str, +) -> None: + relative = path.relative_to(root) + if path.is_symlink(): + raise ValueError(f"{label} contains a symbolic link: {relative}") + if any(secret in relative.as_posix().encode("utf-8") for secret in secrets): + raise ValueError(f"{label} contains a configured credential in {relative}") + if not path.is_file(): + return + match = _matching_kind(path, forbidden=forbidden, secrets=secrets) + if match == "path": + raise ValueError( + f"{label} contains an absolute build-machine path in {relative}" + ) + if match == "secret": + raise ValueError(f"{label} contains a configured credential in {relative}") + + +def assert_publishable_tree( + root: Path, + *, + forbidden_paths: Iterable[Path], + environ: Mapping[str, str], + label: str, +) -> None: + """Reject links, build-machine paths, and configured secrets in a tree.""" + + resolved_root = root.expanduser().resolve() + forbidden_values: list[str] = [] + for path in forbidden_paths: + resolved = path.expanduser().resolve() + forbidden_values.extend((str(resolved), resolved.as_posix())) + forbidden = _serialized_patterns(forbidden_values) + secrets = _secret_values(environ) + for path in sorted(resolved_root.rglob("*")): + _assert_publishable_file( + path, + root=resolved_root, + forbidden=forbidden, + secrets=secrets, + label=label, + ) + + +__all__ = [ + "assert_no_credential_fields", + "assert_publishable_tree", + "file_sha256", +] diff --git a/codenib/cli.py b/codenib/cli.py index 90443515..2cb5c4b2 100644 --- a/codenib/cli.py +++ b/codenib/cli.py @@ -475,6 +475,121 @@ def _run_export(args: argparse.Namespace) -> int: return 0 +def _default_distribution_dir(manifest_path: Path, name: str, commit: str) -> Path: + identity = (commit or "working-tree")[:12] + return manifest_path.parent.parent / "exports" / f"{name}-{identity}" + + +def _publication_environment(credential_env: str | None = None) -> dict[str, str]: + environment = dict(os.environ) + selected = credential_env or environment.get("CODENIB_EMBEDDING_API_KEY_ENV") + if selected and environment.get(selected): + environment["CODENIB_PUBLICATION_CREDENTIAL_SECRET"] = environment[selected] + return environment + + +def _run_artifact_pack(args: argparse.Namespace) -> int: + repo_path = resolve_repo_path(args.repo) + manifest_path = resolve_manifest_path(str(repo_path)) + from .artifacts import stage_context_artifact + from .compiler.manifest import RepoManifest + + manifest = RepoManifest.load(manifest_path) + output_dir = ( + Path(args.output).expanduser().resolve() + if args.output + else _default_distribution_dir( + manifest_path, + "context", + manifest.commit, + ) + ) + selected_views = _split_values(args.view) or None + try: + result = stage_context_artifact( + repo_path, + manifest_path, + output_dir, + repository=args.repository or os.environ.get("GITHUB_REPOSITORY"), + views=selected_views, + environ=_publication_environment(), + ) + except (OSError, RuntimeError, ValueError) as exc: + raise CLIError(str(exc)) from exc + + print(f"Context artifact: {result.output_dir}") + print(f"Repository: {result.repository}") + print(f"Commit: {result.commit or 'working tree'}") + print(f"Views: {', '.join(result.views)}") + print(f"Manifest: {result.metadata_path}") + return 0 + + +def _run_publish(args: argparse.Namespace) -> int: + selected_views = _selected_views(args.preset, args.view) + unsupported = sorted(set(selected_views) - {"bm25", "vector"}) + if unsupported: + raise CLIError( + "portable publication currently supports bm25 and vector views; " + f"unsupported: {', '.join(unsupported)}" + ) + index_result = _run_index(args) + if index_result: + return index_result + + repo_path = resolve_repo_path(args.repo) + manifest_path = resolve_manifest_path(str(repo_path)) + from .artifacts import stage_context_artifact + from .compiler.manifest import RepoManifest + from .web.static_export import export_static_wiki + + manifest = RepoManifest.load(manifest_path) + site_output = ( + Path(args.site_output).expanduser().resolve() + if args.site_output + else _default_distribution_dir(manifest_path, "wiki", manifest.commit) + ) + context_output = ( + Path(args.context_output).expanduser().resolve() + if args.context_output + else _default_distribution_dir(manifest_path, "context", manifest.commit) + ) + if ( + site_output == context_output + or site_output in context_output.parents + or context_output in site_output.parents + ): + raise CLIError("Wiki and context artifact outputs must not overlap") + publication_environment = _publication_environment(args.embedding_api_key_env) + try: + site = export_static_wiki( + repo_path, + manifest_path, + site_output, + frontend_dir=args.frontend_dir, + base_path=args.base_path, + environ=publication_environment, + ) + context = stage_context_artifact( + repo_path, + manifest_path, + context_output, + repository=args.repository or os.environ.get("GITHUB_REPOSITORY"), + views=selected_views, + environ=publication_environment, + validate_checkout=False, + ) + except (OSError, RuntimeError, ValueError) as exc: + raise CLIError(str(exc)) from exc + + print(f"Published Wiki: {site.output_dir}") + print(f"Context artifact: {context.output_dir}") + print(f"Repository: {context.repository}") + print(f"Commit: {context.commit or 'working tree'}") + print(f"Views: {', '.join(context.views)}") + return 0 + + def _model_options_for_args( args: argparse.Namespace, *, @@ -1415,6 +1530,66 @@ def build_parser() -> argparse.ArgumentParser: ) export_parser.set_defaults(handler=_run_export) + artifact_parser = subparsers.add_parser( + "artifact", + help="package portable repository-context artifacts", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + artifact_subparsers = artifact_parser.add_subparsers( + dest="artifact_command", + required=True, + ) + artifact_pack_parser = artifact_subparsers.add_parser( + "pack", + help="stage current manifest views as a portable context artifact", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + artifact_pack_parser.add_argument("repo", nargs="?", default=".") + artifact_pack_parser.add_argument("--output") + artifact_pack_parser.add_argument( + "--repository", + help="stable owner/repository identity; defaults to origin or directory name", + ) + artifact_pack_parser.add_argument( + "--view", + action="append", + default=[], + help="current view to include; repeat or use a comma-separated list", + ) + artifact_pack_parser.set_defaults(handler=_run_artifact_pack) + + publish_parser = subparsers.add_parser( + "publish", + help="incrementally build and publish a static Wiki plus context artifact", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + publish_parser.add_argument("repo", nargs="?", default=".") + publish_parser.add_argument( + "--preset", + choices=("fast", "semantic"), + default="fast", + ) + publish_parser.add_argument("--language", action="append", default=[]) + publish_parser.add_argument("--view", action="append", default=[]) + publish_parser.add_argument("--rebuild", action="store_true") + _add_embedding_route_arguments(publish_parser) + publish_parser.add_argument("--site-output") + publish_parser.add_argument("--context-output") + publish_parser.add_argument( + "--repository", + help="stable owner/repository identity; defaults to origin or directory name", + ) + publish_parser.add_argument( + "--base-path", + default="/", + help="URL path where the static site will be mounted", + ) + publish_parser.add_argument( + "--frontend-dir", + help="path to a prebuilt CodeNib frontend or web source checkout", + ) + publish_parser.set_defaults(handler=_run_publish) + mcp_parser = subparsers.add_parser( "mcp", help="serve an indexed repository over MCP stdio", diff --git a/codenib/compiler/checkout_identity.py b/codenib/compiler/checkout_identity.py new file mode 100644 index 00000000..399f76f0 --- /dev/null +++ b/codenib/compiler/checkout_identity.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +"""Validate that a repository checkout matches a compiled manifest.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +from ..source_fingerprint import fingerprint_repository +from .manifest import RepoManifest + + +def checkout_commit(repo_path: Path) -> str | None: + """Return the checkout's current commit when the directory is in Git.""" + + result = subprocess.run( + ["git", "-C", str(repo_path), "rev-parse", "HEAD"], + capture_output=True, + check=False, + text=True, + ) + return result.stdout.strip() if result.returncode == 0 else None + + +def validate_checkout_identity( + repo_path: Path, + manifest: RepoManifest, + *, + artifact_root: Path, +) -> None: + """Reject source or commit drift between a checkout and its manifest.""" + + expected_source = (manifest.source_fingerprint or "").strip() + if expected_source: + actual_source = fingerprint_repository( + repo_path, + exclude_roots=(artifact_root,), + ).value + if actual_source != expected_source: + raise ValueError( + "repository source files do not match the indexed content. " + "Rebuild the index for the current working tree before serving " + "or publishing context." + ) + + expected = (manifest.commit or "").strip() + actual = checkout_commit(repo_path) + if not expected or not actual: + return + if actual.startswith(expected) or expected.startswith(actual): + return + raise ValueError( + "repository checkout does not match the indexed snapshot: " + f"HEAD is {actual[:12]}, manifest is {expected[:12]}. " + "Rebuild the index or check out the manifest commit before serving " + "or publishing context." + ) + + +__all__ = ["checkout_commit", "validate_checkout_identity"] diff --git a/codenib/web/local.py b/codenib/web/local.py index 3b05e457..b78efd02 100644 --- a/codenib/web/local.py +++ b/codenib/web/local.py @@ -16,10 +16,10 @@ import yaml +from ..compiler.checkout_identity import validate_checkout_identity from ..compiler.manifest import RepoManifest from ..compiler.snapshot_store import normalize_repo from ..llm.options import validate_model_options -from ..source_fingerprint import fingerprint_repository from .config import RepoEntry, save_registry @@ -48,49 +48,6 @@ def _origin_url(repo_path: Path) -> str | None: return result.stdout.strip() if result.returncode == 0 else None -def _checkout_commit(repo_path: Path) -> str | None: - result = subprocess.run( - ["git", "-C", str(repo_path), "rev-parse", "HEAD"], - capture_output=True, - check=False, - text=True, - ) - return result.stdout.strip() if result.returncode == 0 else None - - -def _validate_checkout_identity( - repo_path: Path, - manifest: RepoManifest, - *, - artifact_root: Path, -) -> None: - expected_source = (manifest.source_fingerprint or "").strip() - if expected_source: - actual_source = fingerprint_repository( - repo_path, - exclude_roots=(artifact_root,), - ).value - if actual_source != expected_source: - raise ValueError( - "repository source files do not match the indexed content. " - "Rebuild the index for the current working tree before starting " - "the Wiki." - ) - - expected = (manifest.commit or "").strip() - actual = _checkout_commit(repo_path) - if not expected or not actual: - return - if actual.startswith(expected) or expected.startswith(actual): - return - raise ValueError( - "repository checkout does not match the indexed snapshot: " - f"HEAD is {actual[:12]}, manifest is {expected[:12]}. " - "Rebuild the index or check out the manifest commit before starting " - "the Wiki." - ) - - def _repository_slug(repo_path: Path) -> str: origin = _origin_url(repo_path) if origin: @@ -119,7 +76,7 @@ def prepare_local_wiki( repo_path = repo_path.expanduser().resolve() manifest_path = manifest_path.expanduser().resolve() manifest = RepoManifest.load(str(manifest_path)) - _validate_checkout_identity( + validate_checkout_identity( repo_path, manifest, artifact_root=manifest_path.parent, diff --git a/codenib/web/static_export.py b/codenib/web/static_export.py index a220694c..ffeabdc6 100644 --- a/codenib/web/static_export.py +++ b/codenib/web/static_export.py @@ -6,7 +6,6 @@ from __future__ import annotations -import hashlib import json import os import re @@ -19,6 +18,7 @@ from urllib.parse import quote, unquote, urlsplit from .._version import package_version +from ..artifacts.security import assert_publishable_tree, file_sha256 from ..compiler.manifest import RepoManifest from .launcher import find_frontend_dir from .local import prepare_local_wiki @@ -26,17 +26,6 @@ STATIC_EXPORT_SCHEMA_VERSION = "1.0" STATIC_EXPORT_MANIFEST = "codenib-static.json" -_SENSITIVE_ENV_NAMES = { - "ANTHROPIC_API_KEY", - "AZURE_API_KEY", - "CODENIB_DEMO_API_KEY", - "GITHUB_TOKEN", - "GOOGLE_API_KEY", - "OPENAI_API_KEY", - "AWS_SECRET_ACCESS_KEY", - "AWS_SESSION_TOKEN", -} -_SENSITIVE_ENV_SUFFIXES = ("_API_KEY", "_TOKEN", "_SECRET") _DOCUMENT_BASE_RE = re.compile(r"]*href=(['\"])[^'\"]*\1[^>]*>", re.I) _DOCUMENT_REFERENCE_RE = re.compile( r"(?P\b(?:src|href)=)(?P['\"])(?P[^'\"]*)(?P=quote)", @@ -437,79 +426,33 @@ def _file_inventory(root: Path) -> list[dict[str, Any]]: for path in sorted(root.rglob("*")): if not path.is_file() or path.name == STATIC_EXPORT_MANIFEST: continue - data = path.read_bytes() + size, digest = file_sha256(path) files.append( { "path": path.relative_to(root).as_posix(), - "bytes": len(data), - "sha256": hashlib.sha256(data).hexdigest(), + "bytes": size, + "sha256": digest, } ) return files -def _secret_values(environ: Mapping[str, str]) -> list[bytes]: - values = [] - for name, value in environ.items(): - upper = name.upper() - sensitive = upper in _SENSITIVE_ENV_NAMES or upper.endswith( - _SENSITIVE_ENV_SUFFIXES - ) - if sensitive and len(value) >= 8: - values.append(value.encode("utf-8")) - return values - - -def _serialized_patterns(values: Iterable[str]) -> set[bytes]: - patterns: set[bytes] = set() - for value in values: - if not value: - continue - patterns.add(value.encode("utf-8")) - escaped = json.dumps(value, ensure_ascii=True)[1:-1] - patterns.add(escaped.encode("utf-8")) - return patterns - - -def _assert_publishable( - root: Path, - *, - forbidden_paths: Iterable[Path], - environ: Mapping[str, str], -) -> None: - forbidden_values: list[str] = [] - for path in forbidden_paths: - resolved = path.resolve() - forbidden_values.extend((str(resolved), resolved.as_posix())) - forbidden = _serialized_patterns(forbidden_values) - secrets = _serialized_patterns( - value.decode("utf-8") for value in _secret_values(environ) - ) - for path in root.rglob("*"): - if path.is_symlink(): - raise ValueError(f"static export contains a symbolic link: {path}") - if not path.is_file(): - continue - data = path.read_bytes() - if any(value and value in data for value in forbidden): - raise ValueError( - "static export contains an absolute build-machine path in " - f"{path.relative_to(root)}" - ) - if any(secret in data for secret in secrets): - raise ValueError( - "static export contains a configured credential in " - f"{path.relative_to(root)}" - ) - - -def _validated_output(repo_path: Path, output_dir: Path) -> Path: +def _validated_output( + repo_path: Path, + manifest_root: Path, + output_dir: Path, +) -> Path: repo_path = repo_path.resolve() + manifest_root = manifest_root.resolve() output_dir = output_dir.expanduser().resolve() - if output_dir == repo_path or repo_path in output_dir.parents: - raise ValueError("static export output must be outside the target repository") - if output_dir in repo_path.parents: - raise ValueError("static export output must not contain the target repository") + for source, label in ( + (repo_path, "target repository"), + (manifest_root, "index root"), + ): + if output_dir == source or source in output_dir.parents: + raise ValueError(f"static export output must be outside the {label}") + if output_dir in source.parents: + raise ValueError(f"static export output must not contain the {label}") if output_dir.exists() and not output_dir.is_dir(): raise ValueError(f"static export output is not a directory: {output_dir}") if output_dir.exists() and any(output_dir.iterdir()): @@ -534,7 +477,7 @@ def export_static_wiki( repo_path = repo_path.expanduser().resolve() manifest_path = manifest_path.expanduser().resolve() - output_dir = _validated_output(repo_path, output_dir) + output_dir = _validated_output(repo_path, manifest_path.parent, output_dir) base_path = normalize_base_path(base_path) frontend = _prebuilt_frontend(frontend_dir) environment = os.environ if environ is None else environ @@ -614,10 +557,11 @@ def export_static_wiki( graphs[page_id], ) - _assert_publishable( + assert_publishable_tree( stage, forbidden_paths=(repo_path, manifest_path.parent), environ=environment, + label="static export", ) source_manifest = bundle.manifest export_manifest = { @@ -652,10 +596,11 @@ def export_static_wiki( "files": _file_inventory(stage), } manifest_file = _write_json(stage, STATIC_EXPORT_MANIFEST, export_manifest) - _assert_publishable( + assert_publishable_tree( stage, forbidden_paths=(repo_path, manifest_path.parent), environ=environment, + label="static export", ) if output_dir.exists(): diff --git a/test/artifacts/test_context_artifact.py b/test/artifacts/test_context_artifact.py new file mode 100644 index 00000000..d7ed8bfc --- /dev/null +++ b/test/artifacts/test_context_artifact.py @@ -0,0 +1,364 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import hashlib +import json +import pickle +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from codenib.artifacts import ( + CONTEXT_ARTIFACT_MANIFEST, + CONTEXT_ARTIFACT_SCHEMA, + stage_context_artifact, +) +from codenib.compiler.manifest import IndexEntry, RepoManifest +from codenib.source_fingerprint import fingerprint_repository + + +def _fixture_manifest( + root: Path, + *, + view_path: Path | None = None, + config: dict | None = None, + status: str = "fresh", +) -> tuple[Path, Path, Path]: + repo = root / "repo" + repo.mkdir(parents=True) + (repo / "sample.py").write_text("VALUE = 1\n") + index_root = root / "state" / "indexes" + index_root.mkdir(parents=True) + if view_path is None: + view_path = index_root / "bm25" + view_path.mkdir() + (view_path / "documents.json").write_text( + '[{"page_content": "value", "metadata": {"file": "sample.py"}}]\n' + ) + (view_path / "bm25_metadata.json").write_text( + json.dumps( + { + "project_root": str(repo), + "max_k": 128, + "language": "english", + } + ) + + "\n" + ) + manifest_path = index_root / "repo_manifest.json" + source_fingerprint = fingerprint_repository(repo).value + RepoManifest( + repo_path=str(repo), + commit="a" * 40, + last_indexed_commit="a" * 40, + source_fingerprint=source_fingerprint, + last_indexed_source_fingerprint=source_fingerprint, + languages=["python"], + file_count=1, + indexes={ + "bm25": IndexEntry( + index_type="bm25", + path=str(view_path), + built_at="2026-08-04T00:00:00+00:00", + built_at_epoch=1.0, + status=status, + config=dict(config or {}), + commit="a" * 40, + source_fingerprint=source_fingerprint, + ) + }, + capabilities={"sparse_search": status == "fresh"}, + compiled_at="2026-08-04T00:00:00+00:00", + compiled_at_epoch=1.0, + ).save(manifest_path) + return repo, manifest_path, view_path + + +def _tree(root: Path) -> dict[str, bytes]: + return { + path.relative_to(root).as_posix(): path.read_bytes() + for path in sorted(root.rglob("*")) + if path.is_file() + } + + +def _fixture_vector_manifest(root: Path) -> tuple[Path, Path, Path]: + repo = root / "repo" + repo.mkdir(parents=True) + source = repo / "sample.py" + source.write_text("VALUE = 1\n") + index_root = root / "state" / "indexes" + vector = index_root / "vector" + level = vector / "l2" + level.mkdir(parents=True) + with (level / "documents_test__model.pkl").open("wb") as handle: + pickle.dump( + [ + SimpleNamespace( + page_content="VALUE = 1", + metadata={"file": str(source), "node_id": "sample.py"}, + ) + ], + handle, + ) + (level / "index_test__model.faiss").write_bytes(b"serving-index") + (level / "index_test__model.pkl").write_bytes( + pickle.dumps({"legacy_path": str(source)}) + ) + (vector / "chunk_store.json").write_text( + json.dumps({str(source): [{"file": str(source)}]}) + ) + (vector / "embeddings_cache.npz").write_bytes(b"mutable-cache") + (vector / "incremental_state.json").write_text( + json.dumps({"index_path": str(vector)}) + ) + config = { + "builder_schema": 2, + "embedding_model": "test/model", + "embedding_provider": "huggingface", + "embedding_dimension": 4, + "dimension": 4, + "embedding_kwargs": {}, + "index_metric": "ip", + } + manifest_path = index_root / "repo_manifest.json" + source_fingerprint = fingerprint_repository(repo).value + RepoManifest( + repo_path=str(repo), + commit="b" * 40, + last_indexed_commit="b" * 40, + source_fingerprint=source_fingerprint, + last_indexed_source_fingerprint=source_fingerprint, + languages=["python"], + file_count=1, + indexes={ + "vector": IndexEntry( + index_type="vector", + path=str(vector), + built_at="2026-08-04T00:00:00+00:00", + built_at_epoch=1.0, + status="fresh", + config=config, + metadata=dict(config), + commit="b" * 40, + source_fingerprint=source_fingerprint, + ) + }, + capabilities={"dense_search": True}, + compiled_at="2026-08-04T00:00:00+00:00", + compiled_at_epoch=1.0, + ).save(manifest_path) + return repo, manifest_path, vector + + +def test_context_artifact_rewrites_paths_and_hashes_current_views( + tmp_path: Path, +) -> None: + repo, manifest_path, _view_path = _fixture_manifest(tmp_path) + output = tmp_path / "publish" / "context" + + result = stage_context_artifact( + repo, + manifest_path, + output, + repository="Example/Project", + views=["bm25"], + environ={"GITHUB_TOKEN": "configured-secret-value"}, + ) + + assert result.repository == "example/project" + assert result.commit == "a" * 40 + assert result.views == ("bm25",) + metadata = json.loads((output / CONTEXT_ARTIFACT_MANIFEST).read_text()) + assert metadata["schema"] == CONTEXT_ARTIFACT_SCHEMA + assert metadata["repository"]["slug"] == "example/project" + portable = json.loads((output / "repo_manifest.json").read_text()) + assert portable["repo"]["path"] == "source" + assert portable["indexes"]["bm25"]["path"] == "views/bm25" + assert portable["capabilities"]["sparse_search"] is True + bm25_metadata = json.loads( + (output / "views" / "bm25" / "bm25_metadata.json").read_text() + ) + assert bm25_metadata["project_root"] == "source" + + files = {item["path"]: item for item in metadata["files"]} + assert set(files) == { + "repo_manifest.json", + "views/bm25/bm25_metadata.json", + "views/bm25/documents.json", + } + for relative, record in files.items(): + payload = (output / relative).read_bytes() + assert record["bytes"] == len(payload) + assert record["sha256"] == hashlib.sha256(payload).hexdigest() + serialized = b"".join(_tree(output).values()) + assert str(repo).encode() not in serialized + assert str(manifest_path.parent).encode() not in serialized + assert b"configured-secret-value" not in serialized + fingerprints = portable["indexes"]["bm25"]["config"]["artifact_file_fingerprints"] + for relative, record in fingerprints.items(): + payload = (output / "views" / "bm25" / relative).read_bytes() + assert record["size"] == len(payload) + assert record["sha256"] == hashlib.sha256(payload).hexdigest() + + +def test_context_artifact_is_deterministic_for_one_manifest(tmp_path: Path) -> None: + repo, manifest_path, _view_path = _fixture_manifest(tmp_path) + + stage_context_artifact( + repo, + manifest_path, + tmp_path / "first" / "context", + repository="example/project", + ) + stage_context_artifact( + repo, + manifest_path, + tmp_path / "second" / "context", + repository="example/project", + ) + + assert _tree(tmp_path / "first" / "context") == _tree( + tmp_path / "second" / "context" + ) + + +def test_context_artifact_keeps_only_portable_vector_serving_state( + tmp_path: Path, +) -> None: + repo, manifest_path, _vector = _fixture_vector_manifest(tmp_path) + output = tmp_path / "publish" / "context" + + stage_context_artifact( + repo, + manifest_path, + output, + repository="example/vector-project", + ) + + vector = output / "views" / "vector" + assert not (vector / "chunk_store.json").exists() + assert not (vector / "embeddings_cache.npz").exists() + assert not (vector / "incremental_state.json").exists() + assert not (vector / "l2" / "index_test__model.pkl").exists() + assert (vector / "l2" / "index_test__model.faiss").is_file() + with (vector / "l2" / "documents_test__model.pkl").open("rb") as handle: + documents = pickle.load(handle) + assert documents[0].metadata["file"] == "sample.py" + portable = json.loads((output / "repo_manifest.json").read_text()) + assert portable["indexes"]["vector"]["config"]["artifact_scope"] == ( + "query-serving" + ) + assert str(repo).encode() not in b"".join(_tree(output).values()) + + +def test_context_artifact_rejects_credential_shaped_config(tmp_path: Path) -> None: + repo, manifest_path, _view_path = _fixture_manifest( + tmp_path, + config={"api_key": "must-not-persist"}, + ) + + with pytest.raises(ValueError, match="credential field"): + stage_context_artifact( + repo, + manifest_path, + tmp_path / "publish" / "context", + ) + + +def test_context_artifact_rejects_credential_shaped_metadata(tmp_path: Path) -> None: + repo, manifest_path, _view_path = _fixture_manifest(tmp_path) + manifest = RepoManifest.load(manifest_path) + manifest.indexes["bm25"].metadata["authorization"] = "must-not-persist" + manifest.save(manifest_path) + + with pytest.raises(ValueError, match="credential field"): + stage_context_artifact( + repo, + manifest_path, + tmp_path / "publish" / "context", + ) + + +def test_context_artifact_rejects_source_drift(tmp_path: Path) -> None: + repo, manifest_path, _view_path = _fixture_manifest(tmp_path) + (repo / "sample.py").write_text("VALUE = 2\n") + + with pytest.raises(ValueError, match="source files do not match"): + stage_context_artifact( + repo, + manifest_path, + tmp_path / "publish" / "context", + ) + + +def test_context_artifact_rejects_configured_secret_in_view(tmp_path: Path) -> None: + repo, manifest_path, view_path = _fixture_manifest(tmp_path) + (view_path / "leak.bin").write_bytes(b"runtime-secret-value") + + with pytest.raises(ValueError, match="configured credential"): + stage_context_artifact( + repo, + manifest_path, + tmp_path / "publish" / "context", + environ={"MODELS_TOKEN": "runtime-secret-value"}, + ) + + +def test_context_artifact_stream_scan_finds_boundary_spanning_secret( + tmp_path: Path, +) -> None: + repo, manifest_path, view_path = _fixture_manifest(tmp_path) + secret = b"boundary-spanning-secret" + (view_path / "large.bin").write_bytes(b"x" * (1024 * 1024 - 5) + secret) + + with pytest.raises(ValueError, match="configured credential"): + stage_context_artifact( + repo, + manifest_path, + tmp_path / "publish" / "context", + environ={"CODENIB_ACTION_EMBEDDING_KEY": secret.decode()}, + ) + + +def test_context_artifact_rejects_view_outside_manifest_root(tmp_path: Path) -> None: + outside = tmp_path / "outside" + outside.mkdir() + (outside / "documents.json").write_text("[]\n") + (outside / "bm25_metadata.json").write_text("{}\n") + repo, manifest_path, _view_path = _fixture_manifest( + tmp_path, + view_path=outside, + ) + + with pytest.raises(ValueError, match="outside the manifest index root"): + stage_context_artifact( + repo, + manifest_path, + tmp_path / "publish" / "context", + ) + + +def test_context_artifact_rejects_stale_or_linked_views(tmp_path: Path) -> None: + stale_root = tmp_path / "stale" + repo, manifest_path, _view_path = _fixture_manifest(stale_root, status="stale") + with pytest.raises(ValueError, match="requires at least one current view"): + stage_context_artifact( + repo, + manifest_path, + stale_root / "publish" / "context", + ) + + linked_root = tmp_path / "linked" + repo, manifest_path, view_path = _fixture_manifest(linked_root) + (view_path / "alias").symlink_to(view_path / "index.json") + with pytest.raises(ValueError, match="symbolic link"): + stage_context_artifact( + repo, + manifest_path, + linked_root / "publish" / "context", + ) diff --git a/test/artifacts/test_publish.py b/test/artifacts/test_publish.py new file mode 100644 index 00000000..77749bf2 --- /dev/null +++ b/test/artifacts/test_publish.py @@ -0,0 +1,179 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + +from codenib import cli +from codenib.artifacts import CONTEXT_ARTIFACT_MANIFEST +from codenib.compiler.index_compiler import IndexCompiler +from codenib.web.static_export import STATIC_EXPORT_MANIFEST + + +def _frontend(root: Path) -> Path: + frontend = root / "frontend" + (frontend / "assets").mkdir(parents=True) + (frontend / "index.html").write_text( + "" + "" + "" + ) + (frontend / "runtime-config.js").write_text('window.__CODENIB_API_BASE__ = "";\n') + (frontend / "assets" / "app.js").write_text("console.log('wiki');\n") + return frontend + + +def test_publish_builds_static_site_and_portable_context_without_model( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + (repo / "runtime.py").write_text( + "def run(value: int) -> int:\n return value + 1\n" + ) + site = tmp_path / "published" / "site" + context = tmp_path / "published" / "context" + monkeypatch.setenv("CODENIB_HOME", str(tmp_path / "home")) + monkeypatch.setenv("GITHUB_REPOSITORY", "Example/Project") + + result = cli.run( + [ + "publish", + str(repo), + "--preset", + "fast", + "--site-output", + str(site), + "--context-output", + str(context), + "--base-path", + "/project", + "--frontend-dir", + str(_frontend(tmp_path)), + ] + ) + + assert result == 0 + assert (site / "index.html").is_file() + static_metadata = json.loads((site / STATIC_EXPORT_MANIFEST).read_text()) + context_metadata = json.loads((context / CONTEXT_ARTIFACT_MANIFEST).read_text()) + assert static_metadata["base_path"] == "/project" + assert context_metadata["repository"]["slug"] == "example/project" + assert context_metadata["views"] == ["bm25"] + assert (context / "views" / "bm25").is_dir() + output = capsys.readouterr().out + assert "Published Wiki:" in output + assert "Context artifact:" in output + + +def _git(repo: Path, *args: str) -> str: + result = subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def test_publish_second_commit_uses_incremental_compiler_state( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _git(repo, "init", "--quiet") + _git(repo, "config", "user.name", "CodeNib Test") + _git(repo, "config", "user.email", "codenib@example.invalid") + source = repo / "runtime.py" + source.write_text("def run() -> int:\n return 1\n") + _git(repo, "add", "runtime.py") + _git(repo, "commit", "--quiet", "-m", "initial") + monkeypatch.setenv("CODENIB_HOME", str(tmp_path / "home")) + frontend = _frontend(tmp_path) + site = tmp_path / "published" / "site" + context = tmp_path / "published" / "context" + command = [ + "publish", + str(repo), + "--preset", + "fast", + "--site-output", + str(site), + "--context-output", + str(context), + "--frontend-dir", + str(frontend), + ] + + assert cli.run(command) == 0 + first_commit = json.loads((context / CONTEXT_ARTIFACT_MANIFEST).read_text())[ + "repository" + ]["commit"] + + source.write_text("def run() -> int:\n return 2\n") + _git(repo, "add", "runtime.py") + _git(repo, "commit", "--quiet", "-m", "update") + calls: list[tuple[tuple, dict]] = [] + original = IndexCompiler.update_repo + + def recording_update(self, *args, **kwargs): + calls.append((args, kwargs)) + return original(self, *args, **kwargs) + + monkeypatch.setattr(IndexCompiler, "update_repo", recording_update) + + assert cli.run(command) == 0 + second_commit = _git(repo, "rev-parse", "HEAD") + metadata = json.loads((context / CONTEXT_ARTIFACT_MANIFEST).read_text()) + assert calls + assert first_commit != second_commit + assert metadata["repository"]["commit"] == second_commit + assert metadata["source_locations"]["commit"] == second_commit + + +def test_publish_rejects_nested_site_and_context_outputs( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + (repo / "runtime.py").write_text("VALUE = 1\n") + monkeypatch.setenv("CODENIB_HOME", str(tmp_path / "home")) + output = tmp_path / "published" + + result = cli.run( + [ + "publish", + str(repo), + "--site-output", + str(output), + "--context-output", + str(output / "context"), + "--frontend-dir", + str(_frontend(tmp_path)), + ] + ) + + assert result == 2 + assert not output.exists() + + +def test_publication_environment_marks_custom_embedding_key_as_secret( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("CUSTOM_EMBEDDING_CREDENTIAL", "runtime-secret-value") + + environment = cli._publication_environment("CUSTOM_EMBEDDING_CREDENTIAL") + + assert environment["CODENIB_PUBLICATION_CREDENTIAL_SECRET"] == ( + "runtime-secret-value" + ) diff --git a/test/test_cli.py b/test/test_cli.py index 7467b754..1b67e503 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -20,7 +20,7 @@ def test_parser_exposes_release_commands() -> None: parser = cli.build_parser() - for command in ("index", "wiki", "export", "mcp", "doctor"): + for command in ("index", "wiki", "export", "publish", "mcp", "doctor"): args = parser.parse_args([command]) assert args.command == command @@ -44,6 +44,46 @@ def test_export_parser_accepts_pages_mount_options() -> None: assert args.frontend_dir == "/tmp/frontend" +def test_publish_and_artifact_parsers_expose_distribution_options() -> None: + publish = cli.build_parser().parse_args( + [ + "publish", + ".", + "--preset", + "semantic", + "--site-output", + "/tmp/site", + "--context-output", + "/tmp/context", + "--repository", + "example/project", + "--base-path", + "/project", + "--embedding-provider", + "github_models", + ] + ) + artifact = cli.build_parser().parse_args( + [ + "artifact", + "pack", + ".", + "--output", + "/tmp/context", + "--view", + "bm25,vector", + ] + ) + + assert publish.preset == "semantic" + assert publish.site_output == "/tmp/site" + assert publish.context_output == "/tmp/context" + assert publish.repository == "example/project" + assert publish.embedding_provider == "github_models" + assert artifact.artifact_command == "pack" + assert artifact.view == ["bm25,vector"] + + def test_wiki_parser_accepts_headless_quality_audit() -> None: args = cli.build_parser().parse_args(["wiki", ".", "--audit", "--audit-json"]) @@ -792,7 +832,7 @@ def test_prepare_local_wiki_rejects_mismatched_checkout( languages=["python"], ).save(str(manifest_path)) monkeypatch.setattr( - "codenib.web.local._checkout_commit", + "codenib.compiler.checkout_identity.checkout_commit", lambda _repo_path: "b" * 40, ) diff --git a/test/test_cli_remote_embeddings.py b/test/test_cli_remote_embeddings.py index 1e639fa9..fba57182 100644 --- a/test/test_cli_remote_embeddings.py +++ b/test/test_cli_remote_embeddings.py @@ -12,6 +12,9 @@ import pytest from codenib import cli +from codenib.artifacts import stage_context_artifact +from codenib.compiler.manifest import MANIFEST_FILENAME +from codenib.paths import repo_index_dir class _EmbeddingHandler(BaseHTTPRequestHandler): @@ -104,3 +107,19 @@ def test_openai_semantic_build_uses_remote_sdk_without_sentence_transformers( request["authorization"] == "Bearer runtime-token" for request in _EmbeddingHandler.requests ) + + artifact = tmp_path / "portable-context" + stage_context_artifact( + repo, + repo_index_dir(repo) / MANIFEST_FILENAME, + artifact, + repository="example/semantic-project", + views=["vector"], + environ={"GITHUB_TOKEN": "runtime-token"}, + ) + serialized = b"".join( + path.read_bytes() for path in artifact.rglob("*") if path.is_file() + ) + assert str(repo).encode() not in serialized + assert b"runtime-token" not in serialized + assert not (artifact / "views" / "vector" / "incremental_state.json").exists() diff --git a/test/web/test_static_export.py b/test/web/test_static_export.py index 94cbb183..cc147bb4 100644 --- a/test/web/test_static_export.py +++ b/test/web/test_static_export.py @@ -11,9 +11,9 @@ import pytest from codenib.compiler.manifest import IndexEntry, RepoManifest +from codenib.artifacts.security import assert_publishable_tree from codenib.web.static_export import ( STATIC_EXPORT_MANIFEST, - _assert_publishable, export_static_wiki, normalize_base_path, ) @@ -254,10 +254,11 @@ def resolve(self): return windows_path with pytest.raises(ValueError, match="absolute build-machine path"): - _assert_publishable( + assert_publishable_tree( root, forbidden_paths=(ResolvedWindowsPath(),), environ={}, + label="static export", ) @@ -335,6 +336,16 @@ def test_static_export_does_not_replace_an_unrelated_directory(export_setup) -> assert (export_setup.output / "keep.txt").read_text() == "keep" +def test_static_export_rejects_index_root_overlap(export_setup) -> None: + with pytest.raises(ValueError, match="outside the index root"): + export_static_wiki( + export_setup.repo, + export_setup.manifest_path, + export_setup.manifest_path.parent, + frontend_dir=export_setup.frontend, + ) + + def test_static_export_rejects_absolute_citation_paths( export_setup, monkeypatch: pytest.MonkeyPatch ) -> None: From 2d904aadf989d1c9edf382c05a3f325c96a3e401 Mon Sep 17 00:00:00 2001 From: fishmingyu <1661342068@qq.com> Date: Tue, 4 Aug 2026 23:15:32 -0700 Subject: [PATCH 2/8] feat(actions): publish repository context to Pages Add a reusable no-model or semantic publishing workflow that builds the static Wiki and matching commit-addressed context artifact. Bind cache and artifact identity to the indexed checkout, constrain credential exposure, reject unsafe inputs, and document GitHub Models and BYO routes.\n\nVerified with the full unit tier, pre-commit, action-validator, and strict MkDocs build. --- .github/actions/publish/action.yml | 330 ++++++++++++++++++++ .github/workflows/codenib-pages.yml | 134 ++++++++ .github/workflows/codenib-publish-smoke.yml | 93 ++++++ README.md | 14 + docs/github_pages.md | 153 +++++++++ docs/quickstart.md | 17 + mkdocs.yml | 1 + test/actions/test_publish_action.py | 304 ++++++++++++++++++ test/fixtures/publish_repo/sample.py | 7 + 9 files changed, 1053 insertions(+) create mode 100644 .github/actions/publish/action.yml create mode 100644 .github/workflows/codenib-pages.yml create mode 100644 .github/workflows/codenib-publish-smoke.yml create mode 100644 docs/github_pages.md create mode 100644 test/actions/test_publish_action.py create mode 100644 test/fixtures/publish_repo/sample.py diff --git a/.github/actions/publish/action.yml b/.github/actions/publish/action.yml new file mode 100644 index 00000000..96c18841 --- /dev/null +++ b/.github/actions/publish/action.yml @@ -0,0 +1,330 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +name: Publish CodeNib repository context +description: Build a static Wiki and a portable, commit-addressed context artifact. + +inputs: + repository-path: + description: Repository checkout to index. + default: "." + repository: + description: Stable owner/repository identity. + default: "" + preset: + description: Portable CodeNib view preset (fast or semantic). + default: fast + python-version: + description: Python runtime used to build the artifact. + default: "3.12" + embedding-provider: + description: Embedding provider; semantic defaults to github_models. + default: "" + embedding-model: + description: Embedding model id. + default: "" + embedding-dimension: + description: Embedding vector width for a non-default model. + default: "" + embedding-endpoint: + description: BYO OpenAI-compatible embedding API base. + default: "" + embedding-api-key-env: + description: Name of an environment variable containing the embedding key. + default: "" + base-path: + description: URL path where the static Wiki is mounted. + default: "/" + site-output: + description: Static site output directory; defaults under RUNNER_TEMP. + default: "" + context-output: + description: Context artifact output directory; defaults under RUNNER_TEMP. + default: "" + artifact-name: + description: Uploaded context artifact name. + default: "" + upload-context: + description: Upload the context artifact with actions/upload-artifact. + default: "true" + retention-days: + description: Context artifact retention in days. + default: "14" + cache: + description: Restore and save incremental CodeNib repository state. + default: "true" + revision: + description: Optional CodeNib revision included in cache compatibility. + default: "" + +outputs: + site-path: + description: Absolute static site path. + value: ${{ steps.inputs.outputs.site_path }} + context-path: + description: Absolute portable context artifact path. + value: ${{ steps.inputs.outputs.context_path }} + context-manifest: + description: Portable context metadata path. + value: ${{ steps.inputs.outputs.context_manifest }} + artifact-name: + description: Uploaded context artifact name. + value: ${{ steps.inputs.outputs.artifact_name }} + cache-hit: + description: Whether the exact commit cache key was restored. + value: ${{ steps.restore.outputs.cache-hit }} + cache-key: + description: Immutable cache key for this build. + value: ${{ steps.inputs.outputs.cache_key }} + source-commit: + description: Exact commit indexed from repository-path. + value: ${{ steps.inputs.outputs.source_commit }} + +runs: + using: composite + steps: + - name: Resolve public build identity + id: inputs + shell: bash + env: + ACTION_REF: ${{ github.action_ref }} + INPUT_ARTIFACT_NAME: ${{ inputs.artifact-name }} + INPUT_BASE_PATH: ${{ inputs.base-path }} + INPUT_CACHE: ${{ inputs.cache }} + INPUT_CONTEXT_OUTPUT: ${{ inputs.context-output }} + INPUT_EMBEDDING_DIMENSION: ${{ inputs.embedding-dimension }} + INPUT_EMBEDDING_ENDPOINT: ${{ inputs.embedding-endpoint }} + INPUT_EMBEDDING_MODEL: ${{ inputs.embedding-model }} + INPUT_EMBEDDING_PROVIDER: ${{ inputs.embedding-provider }} + INPUT_PRESET: ${{ inputs.preset }} + INPUT_PYTHON_VERSION: ${{ inputs.python-version }} + INPUT_REPOSITORY: ${{ inputs.repository }} + INPUT_REPOSITORY_PATH: ${{ inputs.repository-path }} + INPUT_REVISION: ${{ inputs.revision }} + INPUT_SITE_OUTPUT: ${{ inputs.site-output }} + INPUT_RETENTION_DAYS: ${{ inputs.retention-days }} + INPUT_UPLOAD_CONTEXT: ${{ inputs.upload-context }} + run: | + set -euo pipefail + + require_single_line() { + local name="$1" + local value="${!name}" + if [[ "$value" == *$'\n'* || "$value" == *$'\r'* ]]; then + echo "$name must not contain a line break" >&2 + exit 2 + fi + } + for name in \ + INPUT_ARTIFACT_NAME INPUT_BASE_PATH INPUT_CACHE INPUT_CONTEXT_OUTPUT \ + INPUT_EMBEDDING_DIMENSION INPUT_EMBEDDING_ENDPOINT \ + INPUT_EMBEDDING_MODEL INPUT_EMBEDDING_PROVIDER INPUT_PRESET \ + INPUT_PYTHON_VERSION INPUT_REPOSITORY INPUT_REPOSITORY_PATH \ + INPUT_RETENTION_DAYS INPUT_REVISION INPUT_SITE_OUTPUT \ + INPUT_UPLOAD_CONTEXT; do + require_single_line "$name" + done + + case "$INPUT_PRESET" in + fast|semantic) ;; + *) echo "unsupported CodeNib preset: $INPUT_PRESET" >&2; exit 2 ;; + esac + case "$INPUT_CACHE:$INPUT_UPLOAD_CONTEXT" in + true:true|true:false|false:true|false:false) ;; + *) echo "cache and upload-context must be true or false" >&2; exit 2 ;; + esac + if [[ ! "$INPUT_RETENTION_DAYS" =~ ^[0-9]+$ ]] || + (( INPUT_RETENTION_DAYS < 1 || INPUT_RETENTION_DAYS > 90 )); then + echo "retention-days must be an integer from 1 through 90" >&2 + exit 2 + fi + + embedding_provider="$INPUT_EMBEDDING_PROVIDER" + if [[ "$INPUT_PRESET" == "semantic" && -z "$embedding_provider" ]]; then + embedding_provider="github_models" + fi + case "$embedding_provider" in + ""|github_models|huggingface|openai) ;; + *) echo "unsupported embedding provider: $embedding_provider" >&2; exit 2 ;; + esac + + repo_path="$(realpath -e "$INPUT_REPOSITORY_PATH" 2>/dev/null || true)" + if [[ -z "$repo_path" || ! -d "$repo_path" ]]; then + echo "repository-path must resolve to a directory" >&2 + exit 2 + fi + source_commit="$(git -C "$repo_path" rev-parse --verify HEAD 2>/dev/null || true)" + if [[ ! "$source_commit" =~ ^[0-9a-f]{40}$ ]]; then + echo "repository-path must be a Git checkout with a resolved HEAD" >&2 + exit 2 + fi + + case "$INPUT_PRESET" in + fast) extras="" ;; + semantic) + if [[ "$embedding_provider" == "huggingface" ]]; then + extras="semantic" + else + extras="semantic-remote" + fi + ;; + esac + + site_path="${INPUT_SITE_OUTPUT:-$RUNNER_TEMP/codenib-site}" + context_path="${INPUT_CONTEXT_OUTPUT:-$RUNNER_TEMP/codenib-context}" + site_path="$(realpath -m "$site_path")" + context_path="$(realpath -m "$context_path")" + if [[ "$site_path" == "$context_path" || + "$site_path" == "$context_path"/* || + "$context_path" == "$site_path"/* ]]; then + echo "site-output and context-output must not overlap" >&2 + exit 2 + fi + frontend_path="$(realpath -m "$GITHUB_ACTION_PATH/../../../web/dist")" + repository="${INPUT_REPOSITORY:-$GITHUB_REPOSITORY}" + if [[ ! "$repository" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then + echo "repository must use owner/name form" >&2 + exit 2 + fi + repository_key="${GITHUB_REPOSITORY_ID:-$repository}" + revision="${INPUT_REVISION:-${ACTION_REF:-source}}" + source_hash="$({ + sha256sum "$GITHUB_ACTION_PATH/action.yml" + sha256sum "$GITHUB_ACTION_PATH/../../../pyproject.toml" + } | sha256sum | cut -d' ' -f1)" + identity="$({ + printf '%s\0' "$revision" "$source_hash" "$INPUT_PYTHON_VERSION" + printf '%s\0' "$INPUT_PRESET" "$embedding_provider" + printf '%s\0' "$INPUT_EMBEDDING_MODEL" "$INPUT_EMBEDDING_DIMENSION" + printf '%s\0' "$INPUT_EMBEDDING_ENDPOINT" + } | sha256sum | cut -d' ' -f1)" + cache_prefix="codenib-${RUNNER_OS}-${repository_key}-${identity}" + cache_key="${cache_prefix}-${source_commit}" + safe_repository="$(printf '%s' "$repository" | tr '/:@ ' '----' | tr -cd 'A-Za-z0-9_.-')" + safe_repository="${safe_repository:-repository}" + artifact_name="${INPUT_ARTIFACT_NAME:-codenib-context-${safe_repository}-${source_commit:0:12}}" + if [[ ! "$artifact_name" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$ ]]; then + echo "artifact-name must be 1-128 portable filename characters" >&2 + exit 2 + fi + + { + echo "artifact_name=$artifact_name" + echo "cache_key=$cache_key" + echo "cache_prefix=$cache_prefix-" + echo "context_manifest=$context_path/codenib-context.json" + echo "context_path=$context_path" + echo "embedding_provider=$embedding_provider" + echo "extras=$extras" + echo "frontend_path=$frontend_path" + echo "repository=$repository" + echo "site_path=$site_path" + echo "source_commit=$source_commit" + } >> "$GITHUB_OUTPUT" + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: ${{ inputs.python-version }} + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: "22" + cache: npm + cache-dependency-path: ${{ github.action_path }}/../../../web/package-lock.json + + - name: Restore incremental repository state + id: restore + if: ${{ inputs.cache == 'true' }} + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ~/.codenib/repositories + key: ${{ steps.inputs.outputs.cache_key }} + restore-keys: ${{ steps.inputs.outputs.cache_prefix }} + + - name: Build static frontend + shell: bash + env: + FRONTEND_OUTPUT: ${{ steps.inputs.outputs.frontend_path }} + run: | + set -euo pipefail + npm ci --prefix "$GITHUB_ACTION_PATH/../../../web" + npm run build --prefix "$GITHUB_ACTION_PATH/../../../web" -- \ + --outDir "$FRONTEND_OUTPUT" + + - name: Install CodeNib + shell: bash + env: + EXTRAS: ${{ steps.inputs.outputs.extras }} + run: | + set -euo pipefail + if [[ -n "$EXTRAS" ]]; then + python -m pip install "$GITHUB_ACTION_PATH/../../..[$EXTRAS]" + else + python -m pip install "$GITHUB_ACTION_PATH/../../.." + fi + + - name: Publish repository context + shell: bash + env: + CODENIB_ACTION_EMBEDDING_KEY_ENV: ${{ inputs.embedding-api-key-env }} + CONTEXT_OUTPUT: ${{ steps.inputs.outputs.context_path }} + EMBEDDING_DIMENSION: ${{ inputs.embedding-dimension }} + EMBEDDING_ENDPOINT: ${{ inputs.embedding-endpoint }} + EMBEDDING_MODEL: ${{ inputs.embedding-model }} + EMBEDDING_PROVIDER: ${{ steps.inputs.outputs.embedding_provider }} + FRONTEND_OUTPUT: ${{ steps.inputs.outputs.frontend_path }} + GITHUB_TOKEN: >- + ${{ inputs.preset == 'semantic' && + steps.inputs.outputs.embedding_provider == 'github_models' && + github.token || '' }} + INPUT_BASE_PATH: ${{ inputs.base-path }} + INPUT_REPOSITORY_PATH: ${{ inputs.repository-path }} + PRESET: ${{ inputs.preset }} + REPOSITORY: ${{ steps.inputs.outputs.repository }} + SITE_OUTPUT: ${{ steps.inputs.outputs.site_path }} + run: | + set -euo pipefail + command=( + codenib publish "$INPUT_REPOSITORY_PATH" + --preset "$PRESET" + --site-output "$SITE_OUTPUT" + --context-output "$CONTEXT_OUTPUT" + --repository "$REPOSITORY" + --base-path "$INPUT_BASE_PATH" + --frontend-dir "$FRONTEND_OUTPUT" + ) + if [[ -n "$EMBEDDING_PROVIDER" ]]; then + command+=(--embedding-provider "$EMBEDDING_PROVIDER") + fi + if [[ -n "$EMBEDDING_MODEL" ]]; then + command+=(--embedding-model "$EMBEDDING_MODEL") + fi + if [[ -n "$EMBEDDING_DIMENSION" ]]; then + command+=(--embedding-dimension "$EMBEDDING_DIMENSION") + fi + if [[ -n "$EMBEDDING_ENDPOINT" ]]; then + command+=(--embedding-endpoint "$EMBEDDING_ENDPOINT") + fi + if [[ -n "$CODENIB_ACTION_EMBEDDING_KEY_ENV" ]]; then + command+=(--embedding-api-key-env "$CODENIB_ACTION_EMBEDDING_KEY_ENV") + fi + "${command[@]}" + + - name: Save incremental repository state + if: ${{ inputs.cache == 'true' && steps.restore.outputs.cache-hit != 'true' }} + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ~/.codenib/repositories + key: ${{ steps.inputs.outputs.cache_key }} + + - name: Upload portable context artifact + if: ${{ inputs.upload-context == 'true' }} + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5 + with: + name: ${{ steps.inputs.outputs.artifact_name }} + path: ${{ steps.inputs.outputs.context_path }} + if-no-files-found: error + retention-days: ${{ inputs.retention-days }} diff --git a/.github/workflows/codenib-pages.yml b/.github/workflows/codenib-pages.yml new file mode 100644 index 00000000..977e3aaa --- /dev/null +++ b/.github/workflows/codenib-pages.yml @@ -0,0 +1,134 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +name: CodeNib Pages + +on: + workflow_call: + inputs: + preset: + description: CodeNib view preset. + type: string + default: fast + python-version: + description: Python runtime used for indexing. + type: string + default: "3.12" + embedding-provider: + description: Embedding provider; semantic defaults to github_models. + type: string + default: "" + embedding-model: + description: Embedding model id. + type: string + default: "" + embedding-dimension: + description: Embedding vector width. + type: string + default: "" + embedding-endpoint: + description: BYO OpenAI-compatible embedding API base. + type: string + default: "" + retention-days: + description: Portable context artifact retention. + type: number + default: 14 + secrets: + embedding_api_key: + description: Optional BYO embedding credential. + required: false + outputs: + page-url: + description: Deployed Pages URL. + value: ${{ jobs.deploy.outputs.page-url }} + context-artifact: + description: Downloadable context artifact name. + value: ${{ jobs.build.outputs.context-artifact }} + +concurrency: + group: codenib-pages-${{ github.repository_id }} + cancel-in-progress: false + +jobs: + build: + if: >- + github.event_name != 'pull_request_target' && + (github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository) + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + models: read + pages: write + outputs: + context-artifact: ${{ steps.publish.outputs.artifact-name }} + site-path: ${{ steps.publish.outputs.site-path }} + steps: + - name: Checkout caller repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Checkout the exact CodeNib workflow revision + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} + path: .codenib-action + persist-credentials: false + + - name: Configure GitHub Pages + id: pages + uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6 + + - name: Build Wiki and context artifact + id: publish + uses: ./.codenib-action/.github/actions/publish + env: + CODENIB_ACTION_EMBEDDING_KEY: >- + ${{ inputs.preset == 'semantic' && + inputs.embedding-provider == 'openai' && + secrets.embedding_api_key || '' }} + with: + base-path: ${{ steps.pages.outputs.base_path }} + embedding-api-key-env: >- + ${{ inputs.preset == 'semantic' && + inputs.embedding-provider == 'openai' && + secrets.embedding_api_key && + 'CODENIB_ACTION_EMBEDDING_KEY' || '' }} + embedding-dimension: ${{ inputs.embedding-dimension }} + embedding-endpoint: ${{ inputs.embedding-endpoint }} + embedding-model: ${{ inputs.embedding-model }} + embedding-provider: ${{ inputs.embedding-provider }} + preset: ${{ inputs.preset }} + python-version: ${{ inputs.python-version }} + repository: ${{ github.repository }} + retention-days: ${{ inputs.retention-days }} + revision: ${{ job.workflow_sha }} + + - name: Upload GitHub Pages artifact + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5 + with: + path: ${{ steps.publish.outputs.site-path }} + + deploy: + if: ${{ needs.build.result == 'success' }} + needs: build + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + outputs: + page-url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy GitHub Pages + id: deployment + uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5 diff --git a/.github/workflows/codenib-publish-smoke.yml b/.github/workflows/codenib-publish-smoke.yml new file mode 100644 index 00000000..844804ba --- /dev/null +++ b/.github/workflows/codenib-publish-smoke.yml @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +name: CodeNib Publish Smoke + +on: + pull_request: + paths: + - ".github/actions/publish/**" + - ".github/workflows/codenib-pages.yml" + - ".github/workflows/codenib-publish-smoke.yml" + - "codenib/artifacts/**" + - "codenib/cli.py" + - "codenib/web/**" + - "pyproject.toml" + - "setup.py" + - "web/**" + push: + branches: [main] + paths: + - ".github/actions/publish/**" + - ".github/workflows/codenib-pages.yml" + - ".github/workflows/codenib-publish-smoke.yml" + - "codenib/artifacts/**" + - "codenib/cli.py" + - "codenib/web/**" + - "pyproject.toml" + - "setup.py" + - "web/**" + workflow_dispatch: + +permissions: + contents: read + +jobs: + no-model: + if: >- + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Prepare fixture repository + id: fixture + shell: bash + env: + FIXTURE_SOURCE: ${{ github.workspace }}/test/fixtures/publish_repo/sample.py + REPOSITORY_PATH: ${{ runner.temp }}/codenib-publish-repo + run: | + set -euo pipefail + mkdir -p "$REPOSITORY_PATH" + cp "$FIXTURE_SOURCE" "$REPOSITORY_PATH/sample.py" + git -C "$REPOSITORY_PATH" init --quiet + git -C "$REPOSITORY_PATH" config user.name "CodeNib Smoke" + git -C "$REPOSITORY_PATH" config user.email "codenib@example.invalid" + git -C "$REPOSITORY_PATH" add sample.py + git -C "$REPOSITORY_PATH" commit --quiet -m "fixture" + echo "path=$REPOSITORY_PATH" >> "$GITHUB_OUTPUT" + + - name: Exercise local publish Action + id: publish + uses: ./.github/actions/publish + env: + CODENIB_ACTION_TEST_TOKEN: never-publish-this-value + with: + base-path: /publish-smoke + cache: "false" + repository-path: ${{ steps.fixture.outputs.path }} + upload-context: "false" + + - name: Validate published identities + shell: bash + env: + CONTEXT_MANIFEST: ${{ steps.publish.outputs.context-manifest }} + SITE_PATH: ${{ steps.publish.outputs.site-path }} + run: | + set -euo pipefail + test -f "$SITE_PATH/index.html" + test -f "$SITE_PATH/codenib-static.json" + test -f "$CONTEXT_MANIFEST" + jq -e '.schema == "codenib.context-artifact.v1"' "$CONTEXT_MANIFEST" + jq -e '.views == ["bm25"]' "$CONTEXT_MANIFEST" + if grep -R -F "never-publish-this-value" "$SITE_PATH" "$(dirname "$CONTEXT_MANIFEST")"; then + echo "configured test token leaked into publication output" >&2 + exit 1 + fi diff --git a/README.md b/README.md index 89a52b57..3c6657a1 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,12 @@ agent or code-Wiki framework. ## News +- **2026-08-04 — Commit-addressed Pages publishing.** + [`codenib publish`](docs/github_pages.md) and the reusable GitHub workflow + build a no-model BM25 or opt-in semantic Wiki, deploy the static inspection + surface, and retain the matching portable context views for the indexed + commit. Incremental caches remain private build state rather than part of the + downloadable serving artifact. - **2026-08-04 — Native repository explorer.** [`RepositoryContextExplorer`](codenib/agent/runtime/explorer.py) plans BM25, dense, hybrid, reranked, and graph routes over manifest-backed views and @@ -129,6 +135,13 @@ source citations, and available page-level dependency data. It contains no provider credential; interactive Ask and runtime graph exploration remain on the local or MCP serving path. +For a repository-hosted Wiki, CodeNib also ships a reusable GitHub workflow +that incrementally builds the same manifest, deploys the static site to Pages, +and uploads the matching commit-addressed context artifact. Its default `fast` +route needs no model credential; semantic search can opt into GitHub Models or +a BYO OpenAI-compatible endpoint. See +[GitHub Pages](https://docs.codenib.ai/github_pages/). + See the [Quickstart](https://docs.codenib.ai/quickstart/) for ports, advanced indexing, and troubleshooting. @@ -192,6 +205,7 @@ records chunking, graph, incremental, and C++ decoder support. ## Documentation - [Quickstart](https://docs.codenib.ai/quickstart/) +- [GitHub Pages](https://docs.codenib.ai/github_pages/) - [MCP Server](https://docs.codenib.ai/mcp/) - [Agent Integrations](https://docs.codenib.ai/agent_integrations/) - [Web UI](https://docs.codenib.ai/web_demo/) diff --git a/docs/github_pages.md b/docs/github_pages.md new file mode 100644 index 00000000..b55fc55b --- /dev/null +++ b/docs/github_pages.md @@ -0,0 +1,153 @@ + + +# Publish With GitHub Pages + +CodeNib can build repository context in GitHub Actions, deploy a source-linked +static Wiki to GitHub Pages, and retain the matching context views as one +downloadable artifact. The default path uses BM25 and needs no model, API key, +or model download. + +## No-Model Starter + +Create a caller workflow in the repository that should receive a Wiki: + +```yaml +name: CodeNib Pages + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +jobs: + publish: + uses: sysevol-ai/CodeNib/.github/workflows/codenib-pages.yml@ +``` + +Replace `` with a published CodeNib release commit. A commit SHA +keeps the compiler, frontend, action, and artifact schema on one reviewed +revision. In the repository's **Settings > Pages**, select **GitHub Actions** as +the source. + +The workflow checks out the caller's exact commit, incrementally builds the +`fast` preset, exports the Wiki at the Pages-provided mount path, and deploys it +through the `github-pages` environment. It also uploads an artifact named from +the repository and commit. + +## Semantic Search With GitHub Models + +GitHub Models is an opt-in embedding route. Add `models: read` and select the +semantic preset: + +```yaml +permissions: + contents: read + models: read + pages: write + id-token: write + +jobs: + publish: + uses: sysevol-ai/CodeNib/.github/workflows/codenib-pages.yml@ + with: + preset: semantic +``` + +CodeNib uses the workflow's process-local `GITHUB_TOKEN`; it does not write the +token or its environment-variable name into the vector manifest. The default +route is `openai/text-embedding-3-small`. GitHub Models usage and billing are +separate from GitHub Copilot; review GitHub's +[Models billing documentation](https://docs.github.com/en/billing/concepts/product-billing/github-models) +before enabling it broadly. + +## Bring Your Own Embedding Endpoint + +An OpenAI-compatible endpoint can replace GitHub Models without changing the +artifact or Pages workflow: + +```yaml +jobs: + publish: + uses: sysevol-ai/CodeNib/.github/workflows/codenib-pages.yml@ + with: + preset: semantic + embedding-provider: openai + embedding-model: text-embedding-3-small + embedding-dimension: "1536" + embedding-endpoint: https://embeddings.example.com/v1 + secrets: + embedding_api_key: ${{ secrets.CODENIB_EMBEDDING_API_KEY }} +``` + +Provider, model, vector dimension, endpoint, Python version, and CodeNib source +revision participate in cache compatibility. The credential value does not. +Endpoints containing user information, a query, or a fragment are rejected. + +## What Gets Published + +The Pages artifact is a serverless inspection surface. It contains generated +pages, source slices used by citations, page-level dependency data when +available, and `codenib-static.json`. It does not contain an API endpoint, +credential, interactive Ask backend, or unrestricted source-reading service. + +The separate context artifact contains: + +- `codenib-context.json`, with repository, commit, schema, capabilities, and + file hashes; +- an artifact-relative `repo_manifest.json`; +- the BM25 view and, for `semantic`, FAISS indexes plus repository-relative + document locations. + +Mutable vector maintenance caches are deliberately excluded. The downloadable +artifact represents query-serving state for one commit; it is not a substitute +for the Action cache used to update a later commit. Portable publication +currently supports the `fast` and `semantic` presets. Graph and Zoekt indexes +remain available in the local/MCP runtime but are not yet promised as portable +Pages artifacts. + +## Incremental Builds + +The Action caches `~/.codenib/repositories` under a key that includes the +repository, platform, Python version, profile, provider identity, and CodeNib +revision. A prefix restore may supply the previous commit's state, but it never +declares that state current. The compiler compares the checkout and manifest, +updates supported views, and rebuilds when reuse is not valid. The newly +uploaded context artifact always records the indexed checkout's resolved Git +commit rather than assuming that it matches the surrounding event SHA. + +## Security Boundary + +The reusable workflow rejects `pull_request_target` and skips pull requests +whose head repository differs from the base repository. It therefore does not +pass GitHub Models or BYO credentials to untrusted fork code. All shipped +third-party Actions are pinned to immutable commits, checkout credentials are +not persisted, and publication fails if an output contains a configured secret, +a symbolic link, or a build-machine source/index path. + +Use `push` or `workflow_dispatch` for normal publication. Do not wrap the +reusable workflow in `pull_request_target`. + +## Build Without Deployment + +The composite Action can be used directly when another static host or artifact +store owns deployment: + +```yaml +- uses: sysevol-ai/CodeNib/.github/actions/publish@ + id: codenib + with: + preset: fast + base-path: /repository +``` + +Its outputs include `site-path`, `context-path`, `context-manifest`, +`artifact-name`, `cache-hit`, `cache-key`, and `source-commit`. diff --git a/docs/quickstart.md b/docs/quickstart.md index d749f059..c3fa7d58 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -106,6 +106,23 @@ Interactive Ask, on-demand edge labels, and arbitrary dependency queries need an authenticated CodeNib runtime and are not exposed by the export. No API key, GitHub token, endpoint credential, or build-machine absolute path is serialized. +To build the index and both distribution surfaces in one command: + +```bash +codenib publish . \ + --preset fast \ + --site-output /tmp/repository-wiki \ + --context-output /tmp/repository-context \ + --base-path /repository +``` + +The context directory is commit-addressed query-serving state with an +artifact-relative manifest and file hashes. It excludes mutable maintenance +caches. Portable publication currently supports `fast` and `semantic`; use the +local or MCP runtime for graph and Zoekt views. See +[Publish With GitHub Pages](github_pages.md) for the no-model Action, GitHub +Models, and BYO endpoint configurations. + ## Select Repository Views | Preset | Required package | Views | diff --git a/mkdocs.yml b/mkdocs.yml index a37b04d1..87da3ee3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -165,6 +165,7 @@ nav: - Get Started: - get-started/index.md - Quickstart: quickstart.md + - GitHub Pages: github_pages.md - Web UI: web_demo.md - Running Locally: running-locally.md - Language Capabilities: language_capabilities.md diff --git a/test/actions/test_publish_action.py b/test/actions/test_publish_action.py new file mode 100644 index 00000000..2a2a7f17 --- /dev/null +++ b/test/actions/test_publish_action.py @@ -0,0 +1,304 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +import re +import subprocess +from pathlib import Path +from typing import Any + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +ACTION_PATH = ROOT / ".github" / "actions" / "publish" / "action.yml" +WORKFLOW_PATH = ROOT / ".github" / "workflows" / "codenib-pages.yml" +SMOKE_WORKFLOW_PATH = ROOT / ".github" / "workflows" / "codenib-publish-smoke.yml" +PINNED_ACTION_RE = re.compile(r"^[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_.-]+)+@[0-9a-f]{40}$") + + +def _load(path: Path) -> dict[str, Any]: + value = yaml.load(path.read_text(encoding="utf-8"), Loader=yaml.BaseLoader) + assert isinstance(value, dict) + return value + + +def _steps(value: dict[str, Any]) -> list[dict[str, Any]]: + if "runs" in value: + return list(value["runs"]["steps"]) + result = [] + for job in value["jobs"].values(): + result.extend(job.get("steps") or []) + return result + + +def _assert_external_actions_are_sha_pinned(value: dict[str, Any]) -> None: + for step in _steps(value): + uses = step.get("uses") + if not uses or str(uses).startswith("./"): + continue + assert PINNED_ACTION_RE.fullmatch(str(uses)), uses + + +def _run_resolve_step( + tmp_path: Path, **env_overrides: str +) -> tuple[subprocess.CompletedProcess[str], str]: + action = _load(ACTION_PATH) + resolve = next(step for step in _steps(action) if step.get("id") == "inputs") + output_path = tmp_path / "github-output" + env = os.environ.copy() + env.update( + { + "ACTION_REF": "0123456789abcdef", + "GITHUB_ACTION_PATH": str(ACTION_PATH.parent), + "GITHUB_OUTPUT": str(output_path), + "GITHUB_REPOSITORY": "example/project", + "GITHUB_REPOSITORY_ID": "123456", + "GITHUB_SHA": "a" * 40, + "INPUT_ARTIFACT_NAME": "", + "INPUT_BASE_PATH": "/", + "INPUT_CACHE": "true", + "INPUT_CONTEXT_OUTPUT": "", + "INPUT_EMBEDDING_DIMENSION": "", + "INPUT_EMBEDDING_ENDPOINT": "", + "INPUT_EMBEDDING_MODEL": "", + "INPUT_EMBEDDING_PROVIDER": "", + "INPUT_PRESET": "fast", + "INPUT_PYTHON_VERSION": "3.12", + "INPUT_REPOSITORY": "", + "INPUT_REPOSITORY_PATH": str(ROOT), + "INPUT_RETENTION_DAYS": "14", + "INPUT_REVISION": "", + "INPUT_SITE_OUTPUT": "", + "INPUT_UPLOAD_CONTEXT": "true", + "RUNNER_OS": "Linux", + "RUNNER_TEMP": str(tmp_path), + } + ) + env.update(env_overrides) + result = subprocess.run( + ["bash", "-c", str(resolve["run"])], + text=True, + capture_output=True, + check=False, + env=env, + ) + output = output_path.read_text(encoding="utf-8") if output_path.exists() else "" + return result, output + + +def test_publish_action_has_secret_free_inputs_and_stable_outputs() -> None: + action = _load(ACTION_PATH) + + assert action["runs"]["using"] == "composite" + assert "api-key" not in action["inputs"] + assert "embedding-api-key-env" in action["inputs"] + assert { + "site-path", + "context-path", + "context-manifest", + "artifact-name", + "cache-hit", + "cache-key", + "source-commit", + } <= set(action["outputs"]) + assert action["inputs"]["preset"]["default"] == "fast" + resolve = next(step for step in _steps(action) if step.get("id") == "inputs") + assert "fast|semantic)" in resolve["run"] + assert "graph|full" not in resolve["run"] + _assert_external_actions_are_sha_pinned(action) + + +def test_publish_action_shell_blocks_parse_with_bash() -> None: + action = _load(ACTION_PATH) + + for index, step in enumerate(_steps(action)): + script = str(step.get("run") or "") + if not script: + continue + result = subprocess.run( + ["bash", "-n"], + input=script, + text=True, + capture_output=True, + check=False, + ) + assert result.returncode == 0, f"step {index}: {result.stderr}" + + +def test_publish_action_builds_frontend_before_source_install() -> None: + action = _load(ACTION_PATH) + names = [step.get("name") for step in _steps(action)] + + assert names.index("Build static frontend") < names.index("Install CodeNib") + resolve = next(step for step in _steps(action) if step.get("id") == "inputs") + assert "web/dist" in resolve["run"] + + +def test_publish_action_keeps_untrusted_inputs_out_of_shell_source() -> None: + action = _load(ACTION_PATH) + + for step in _steps(action): + script = str(step.get("run") or "") + assert "${{ inputs." not in script + publish = next( + step + for step in _steps(action) + if step.get("name") == "Publish repository context" + ) + token_expression = publish["env"]["GITHUB_TOKEN"] + assert "inputs.preset == 'semantic'" in token_expression + assert "embedding_provider == 'github_models'" in token_expression + assert "github.token" in token_expression + assert "command=(" in publish["run"] + assert '"${command[@]}"' in publish["run"] + + +def test_publish_action_resolves_valid_inputs(tmp_path: Path) -> None: + result, output = _run_resolve_step(tmp_path) + source_commit = subprocess.run( + ["git", "-C", str(ROOT), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + assert result.returncode == 0, result.stderr + assert ( + f"artifact_name=codenib-context-example-project-{source_commit[:12]}" in output + ) + assert f"site_path={tmp_path / 'codenib-site'}" in output + assert f"context_path={tmp_path / 'codenib-context'}" in output + assert "embedding_provider=" in output + assert "extras=" in output + assert f"source_commit={source_commit}" in output + + +def test_publish_action_rejects_output_command_injection(tmp_path: Path) -> None: + result, output = _run_resolve_step( + tmp_path, + INPUT_EMBEDDING_PROVIDER="openai\nartifact_name=forged", + ) + + assert result.returncode == 2 + assert "must not contain a line break" in result.stderr + assert "forged" not in output + + +def test_publish_action_rejects_unsupported_provider(tmp_path: Path) -> None: + result, output = _run_resolve_step( + tmp_path, + INPUT_PRESET="semantic", + INPUT_EMBEDDING_PROVIDER="unknown", + ) + + assert result.returncode == 2 + assert "unsupported embedding provider" in result.stderr + assert output == "" + + +def test_publish_action_rejects_overlapping_outputs(tmp_path: Path) -> None: + site_path = tmp_path / "publish" + result, output = _run_resolve_step( + tmp_path, + INPUT_SITE_OUTPUT=str(site_path), + INPUT_CONTEXT_OUTPUT=str(site_path / "context"), + ) + + assert result.returncode == 2 + assert "site-output and context-output must not overlap" in result.stderr + assert output == "" + + +def test_publish_action_cache_identity_is_public_and_commit_addressed() -> None: + action = _load(ACTION_PATH) + resolve = next(step for step in _steps(action) if step.get("id") == "inputs") + script = resolve["run"] + + assert "git -C" in script + assert "source_commit" in script + assert "GITHUB_SHA" not in script + assert "INPUT_PRESET" in script + assert "INPUT_EMBEDDING_PROVIDER" in script + assert "INPUT_EMBEDDING_ENDPOINT" in script + assert "API_KEY" not in script + restore = next( + step + for step in _steps(action) + if step.get("name") == "Restore incremental repository state" + ) + save = next( + step + for step in _steps(action) + if step.get("name") == "Save incremental repository state" + ) + assert restore["with"]["key"] == "${{ steps.inputs.outputs.cache_key }}" + assert save["with"]["key"] == "${{ steps.inputs.outputs.cache_key }}" + + +def test_reusable_workflow_is_fork_safe_and_binds_its_exact_revision() -> None: + workflow = _load(WORKFLOW_PATH) + build = workflow["jobs"]["build"] + condition = build["if"] + + assert "pull_request_target" in condition + assert "head.repo.full_name == github.repository" in condition + assert build["permissions"] == { + "contents": "read", + "models": "read", + "pages": "write", + } + exact_checkout = next( + step + for step in build["steps"] + if step.get("name") == "Checkout the exact CodeNib workflow revision" + ) + assert exact_checkout["with"]["repository"] == "${{ job.workflow_repository }}" + assert exact_checkout["with"]["ref"] == "${{ job.workflow_sha }}" + assert exact_checkout["with"]["persist-credentials"] == "false" + secret_steps = [ + step for step in build["steps"] if "secrets.embedding_api_key" in str(step) + ] + assert [step["name"] for step in secret_steps] == [ + "Build Wiki and context artifact" + ] + publish = secret_steps[0] + secret_expression = publish["env"]["CODENIB_ACTION_EMBEDDING_KEY"] + assert "inputs.preset == 'semantic'" in secret_expression + assert "inputs.embedding-provider == 'openai'" in secret_expression + _assert_external_actions_are_sha_pinned(workflow) + + +def test_reusable_workflow_deploys_only_the_published_site() -> None: + workflow = _load(WORKFLOW_PATH) + build = workflow["jobs"]["build"] + upload = next( + step + for step in build["steps"] + if step.get("name") == "Upload GitHub Pages artifact" + ) + deploy = workflow["jobs"]["deploy"] + + assert upload["with"]["path"] == "${{ steps.publish.outputs.site-path }}" + assert deploy["needs"] == "build" + assert deploy["permissions"] == {"pages": "write", "id-token": "write"} + assert deploy["environment"]["name"] == "github-pages" + assert workflow["concurrency"]["cancel-in-progress"] == "false" + + +def test_hosted_publish_smoke_is_narrow_and_sha_pinned() -> None: + workflow = _load(SMOKE_WORKFLOW_PATH) + job = workflow["jobs"]["no-model"] + + assert "head.repo.full_name == github.repository" in job["if"] + publish = next( + step + for step in job["steps"] + if step.get("name") == "Exercise local publish Action" + ) + assert publish["with"]["cache"] == "false" + assert publish["with"]["upload-context"] == "false" + assert publish["with"]["repository-path"] == "${{ steps.fixture.outputs.path }}" + _assert_external_actions_are_sha_pinned(workflow) diff --git a/test/fixtures/publish_repo/sample.py b/test/fixtures/publish_repo/sample.py new file mode 100644 index 00000000..8bdee228 --- /dev/null +++ b/test/fixtures/publish_repo/sample.py @@ -0,0 +1,7 @@ +"""Small source fixture used by the no-model publish Action smoke.""" + + +def repository_name() -> str: + """Return a deterministic value that BM25 can index.""" + + return "CodeNib" From 950d1c827090fae387c832c4c6a5ac887c8022d3 Mon Sep 17 00:00:00 2001 From: fishmingyu <1661342068@qq.com> Date: Tue, 4 Aug 2026 23:41:22 -0700 Subject: [PATCH 3/8] fix(actions): canonicalize context artifact names Lowercase repository identity before deriving the default artifact name so GitHub lookup and cache consumers use one stable owner/repository spelling.\n\nVerified with the Action tests, action-validator, and pre-commit. --- .github/actions/publish/action.yml | 5 ++++- test/actions/test_publish_action.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/actions/publish/action.yml b/.github/actions/publish/action.yml index 96c18841..bca33f20 100644 --- a/.github/actions/publish/action.yml +++ b/.github/actions/publish/action.yml @@ -201,7 +201,10 @@ runs: } | sha256sum | cut -d' ' -f1)" cache_prefix="codenib-${RUNNER_OS}-${repository_key}-${identity}" cache_key="${cache_prefix}-${source_commit}" - safe_repository="$(printf '%s' "$repository" | tr '/:@ ' '----' | tr -cd 'A-Za-z0-9_.-')" + safe_repository="$({ + printf '%s' "$repository" | tr '[:upper:]' '[:lower:]' | \ + tr '/:@ ' '----' | tr -cd 'a-z0-9_.-' + })" safe_repository="${safe_repository:-repository}" artifact_name="${INPUT_ARTIFACT_NAME:-codenib-context-${safe_repository}-${source_commit:0:12}}" if [[ ! "$artifact_name" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$ ]]; then diff --git a/test/actions/test_publish_action.py b/test/actions/test_publish_action.py index 2a2a7f17..b82023f8 100644 --- a/test/actions/test_publish_action.py +++ b/test/actions/test_publish_action.py @@ -220,6 +220,7 @@ def test_publish_action_cache_identity_is_public_and_commit_addressed() -> None: assert "git -C" in script assert "source_commit" in script assert "GITHUB_SHA" not in script + assert "tr '[:upper:]' '[:lower:]'" in script assert "INPUT_PRESET" in script assert "INPUT_EMBEDDING_PROVIDER" in script assert "INPUT_EMBEDDING_ENDPOINT" in script From dc02f3b6ac8b360891444c79154c10eacb5111a6 Mon Sep 17 00:00:00 2001 From: fishmingyu <1661342068@qq.com> Date: Wed, 5 Aug 2026 00:21:43 -0700 Subject: [PATCH 4/8] fix(actions): replace retired model route --- .github/actions/publish/action.yml | 10 +++---- .github/workflows/codenib-pages.yml | 3 +-- README.md | 4 +-- docs/github_pages.md | 26 +++++++----------- test/actions/test_publish_action.py | 41 +++++++++++++++++++++++++---- test/test_cli.py | 4 +-- 6 files changed, 53 insertions(+), 35 deletions(-) diff --git a/.github/actions/publish/action.yml b/.github/actions/publish/action.yml index bca33f20..38c0559b 100644 --- a/.github/actions/publish/action.yml +++ b/.github/actions/publish/action.yml @@ -19,7 +19,7 @@ inputs: description: Python runtime used to build the artifact. default: "3.12" embedding-provider: - description: Embedding provider; semantic defaults to github_models. + description: Embedding provider; semantic defaults to local Hugging Face. default: "" embedding-model: description: Embedding model id. @@ -142,10 +142,10 @@ runs: embedding_provider="$INPUT_EMBEDDING_PROVIDER" if [[ "$INPUT_PRESET" == "semantic" && -z "$embedding_provider" ]]; then - embedding_provider="github_models" + embedding_provider="huggingface" fi case "$embedding_provider" in - ""|github_models|huggingface|openai) ;; + ""|huggingface|openai) ;; *) echo "unsupported embedding provider: $embedding_provider" >&2; exit 2 ;; esac @@ -279,10 +279,6 @@ runs: EMBEDDING_MODEL: ${{ inputs.embedding-model }} EMBEDDING_PROVIDER: ${{ steps.inputs.outputs.embedding_provider }} FRONTEND_OUTPUT: ${{ steps.inputs.outputs.frontend_path }} - GITHUB_TOKEN: >- - ${{ inputs.preset == 'semantic' && - steps.inputs.outputs.embedding_provider == 'github_models' && - github.token || '' }} INPUT_BASE_PATH: ${{ inputs.base-path }} INPUT_REPOSITORY_PATH: ${{ inputs.repository-path }} PRESET: ${{ inputs.preset }} diff --git a/.github/workflows/codenib-pages.yml b/.github/workflows/codenib-pages.yml index 977e3aaa..682774c2 100644 --- a/.github/workflows/codenib-pages.yml +++ b/.github/workflows/codenib-pages.yml @@ -16,7 +16,7 @@ on: type: string default: "3.12" embedding-provider: - description: Embedding provider; semantic defaults to github_models. + description: Embedding provider; semantic defaults to local Hugging Face. type: string default: "" embedding-model: @@ -61,7 +61,6 @@ jobs: timeout-minutes: 30 permissions: contents: read - models: read pages: write outputs: context-artifact: ${{ steps.publish.outputs.artifact-name }} diff --git a/README.md b/README.md index 3c6657a1..82812135 100644 --- a/README.md +++ b/README.md @@ -138,8 +138,8 @@ the local or MCP serving path. For a repository-hosted Wiki, CodeNib also ships a reusable GitHub workflow that incrementally builds the same manifest, deploys the static site to Pages, and uploads the matching commit-addressed context artifact. Its default `fast` -route needs no model credential; semantic search can opt into GitHub Models or -a BYO OpenAI-compatible endpoint. See +route needs no model credential or model download; semantic search can use a +local Hugging Face model or an explicit BYO OpenAI-compatible endpoint. See [GitHub Pages](https://docs.codenib.ai/github_pages/). See the diff --git a/docs/github_pages.md b/docs/github_pages.md index b55fc55b..403a0e3a 100644 --- a/docs/github_pages.md +++ b/docs/github_pages.md @@ -43,18 +43,12 @@ The workflow checks out the caller's exact commit, incrementally builds the through the `github-pages` environment. It also uploads an artifact named from the repository and commit. -## Semantic Search With GitHub Models +## Semantic Search With A Local Model -GitHub Models is an opt-in embedding route. Add `models: read` and select the -semantic preset: +Select the semantic preset to build BM25 and dense-vector views. It defaults to +a local Hugging Face embedding model and needs no API credential: ```yaml -permissions: - contents: read - models: read - pages: write - id-token: write - jobs: publish: uses: sysevol-ai/CodeNib/.github/workflows/codenib-pages.yml@ @@ -62,16 +56,14 @@ jobs: preset: semantic ``` -CodeNib uses the workflow's process-local `GITHUB_TOKEN`; it does not write the -token or its environment-variable name into the vector manifest. The default -route is `openai/text-embedding-3-small`. GitHub Models usage and billing are -separate from GitHub Copilot; review GitHub's -[Models billing documentation](https://docs.github.com/en/billing/concepts/product-billing/github-models) -before enabling it broadly. +The first build downloads the default embedding model into the ephemeral Action +runner. The resulting vector view is stored in the commit-addressed context +artifact and reused through CodeNib's repository cache on later builds. Keep the +default `fast` preset when a model download is undesirable. ## Bring Your Own Embedding Endpoint -An OpenAI-compatible endpoint can replace GitHub Models without changing the +An OpenAI-compatible endpoint can replace the local model without changing the artifact or Pages workflow: ```yaml @@ -128,7 +120,7 @@ commit rather than assuming that it matches the surrounding event SHA. The reusable workflow rejects `pull_request_target` and skips pull requests whose head repository differs from the base repository. It therefore does not -pass GitHub Models or BYO credentials to untrusted fork code. All shipped +pass BYO credentials to untrusted fork code. All shipped third-party Actions are pinned to immutable commits, checkout credentials are not persisted, and publication fails if an output contains a configured secret, a symbolic link, or a build-machine source/index path. diff --git a/test/actions/test_publish_action.py b/test/actions/test_publish_action.py index b82023f8..be8128da 100644 --- a/test/actions/test_publish_action.py +++ b/test/actions/test_publish_action.py @@ -148,10 +148,8 @@ def test_publish_action_keeps_untrusted_inputs_out_of_shell_source() -> None: for step in _steps(action) if step.get("name") == "Publish repository context" ) - token_expression = publish["env"]["GITHUB_TOKEN"] - assert "inputs.preset == 'semantic'" in token_expression - assert "embedding_provider == 'github_models'" in token_expression - assert "github.token" in token_expression + assert "GITHUB_TOKEN" not in publish["env"] + assert "github.token" not in str(publish) assert "command=(" in publish["run"] assert '"${command[@]}"' in publish["run"] @@ -176,6 +174,40 @@ def test_publish_action_resolves_valid_inputs(tmp_path: Path) -> None: assert f"source_commit={source_commit}" in output +def test_publish_action_defaults_semantic_to_local_huggingface( + tmp_path: Path, +) -> None: + result, output = _run_resolve_step(tmp_path, INPUT_PRESET="semantic") + + assert result.returncode == 0, result.stderr + assert "embedding_provider=huggingface\n" in output + assert "extras=semantic\n" in output + + +def test_publish_action_selects_remote_extra_for_openai(tmp_path: Path) -> None: + result, output = _run_resolve_step( + tmp_path, + INPUT_PRESET="semantic", + INPUT_EMBEDDING_PROVIDER="openai", + ) + + assert result.returncode == 0, result.stderr + assert "embedding_provider=openai\n" in output + assert "extras=semantic-remote\n" in output + + +def test_publish_action_rejects_retired_github_models(tmp_path: Path) -> None: + result, output = _run_resolve_step( + tmp_path, + INPUT_PRESET="semantic", + INPUT_EMBEDDING_PROVIDER="github_models", + ) + + assert result.returncode == 2 + assert "unsupported embedding provider" in result.stderr + assert output == "" + + def test_publish_action_rejects_output_command_injection(tmp_path: Path) -> None: result, output = _run_resolve_step( tmp_path, @@ -248,7 +280,6 @@ def test_reusable_workflow_is_fork_safe_and_binds_its_exact_revision() -> None: assert "head.repo.full_name == github.repository" in condition assert build["permissions"] == { "contents": "read", - "models": "read", "pages": "write", } exact_checkout = next( diff --git a/test/test_cli.py b/test/test_cli.py index 1b67e503..1d476d83 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -60,7 +60,7 @@ def test_publish_and_artifact_parsers_expose_distribution_options() -> None: "--base-path", "/project", "--embedding-provider", - "github_models", + "openai", ] ) artifact = cli.build_parser().parse_args( @@ -79,7 +79,7 @@ def test_publish_and_artifact_parsers_expose_distribution_options() -> None: assert publish.site_output == "/tmp/site" assert publish.context_output == "/tmp/context" assert publish.repository == "example/project" - assert publish.embedding_provider == "github_models" + assert publish.embedding_provider == "openai" assert artifact.artifact_command == "pack" assert artifact.view == ["bm25,vector"] From a54fab3af05ec717341a2acdfe1b4d735f2cf16f Mon Sep 17 00:00:00 2001 From: fishmingyu <1661342068@qq.com> Date: Wed, 5 Aug 2026 00:25:59 -0700 Subject: [PATCH 5/8] fix(actions): canonicalize frontend cache path --- .github/actions/publish/action.yml | 16 ++++++++++------ test/actions/test_publish_action.py | 9 +++++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/actions/publish/action.yml b/.github/actions/publish/action.yml index 38c0559b..4fb9d101 100644 --- a/.github/actions/publish/action.yml +++ b/.github/actions/publish/action.yml @@ -181,7 +181,8 @@ runs: echo "site-output and context-output must not overlap" >&2 exit 2 fi - frontend_path="$(realpath -m "$GITHUB_ACTION_PATH/../../../web/dist")" + source_path="$(realpath -e "$GITHUB_ACTION_PATH/../../..")" + frontend_path="$source_path/web/dist" repository="${INPUT_REPOSITORY:-$GITHUB_REPOSITORY}" if [[ ! "$repository" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then echo "repository must use owner/name form" >&2 @@ -223,6 +224,7 @@ runs: echo "frontend_path=$frontend_path" echo "repository=$repository" echo "site_path=$site_path" + echo "source_path=$source_path" echo "source_commit=$source_commit" } >> "$GITHUB_OUTPUT" @@ -236,7 +238,7 @@ runs: with: node-version: "22" cache: npm - cache-dependency-path: ${{ github.action_path }}/../../../web/package-lock.json + cache-dependency-path: ${{ steps.inputs.outputs.source_path }}/web/package-lock.json - name: Restore incremental repository state id: restore @@ -251,22 +253,24 @@ runs: shell: bash env: FRONTEND_OUTPUT: ${{ steps.inputs.outputs.frontend_path }} + SOURCE_PATH: ${{ steps.inputs.outputs.source_path }} run: | set -euo pipefail - npm ci --prefix "$GITHUB_ACTION_PATH/../../../web" - npm run build --prefix "$GITHUB_ACTION_PATH/../../../web" -- \ + npm ci --prefix "$SOURCE_PATH/web" + npm run build --prefix "$SOURCE_PATH/web" -- \ --outDir "$FRONTEND_OUTPUT" - name: Install CodeNib shell: bash env: EXTRAS: ${{ steps.inputs.outputs.extras }} + SOURCE_PATH: ${{ steps.inputs.outputs.source_path }} run: | set -euo pipefail if [[ -n "$EXTRAS" ]]; then - python -m pip install "$GITHUB_ACTION_PATH/../../..[$EXTRAS]" + python -m pip install "$SOURCE_PATH[$EXTRAS]" else - python -m pip install "$GITHUB_ACTION_PATH/../../.." + python -m pip install "$SOURCE_PATH" fi - name: Publish repository context diff --git a/test/actions/test_publish_action.py b/test/actions/test_publish_action.py index be8128da..4af34c65 100644 --- a/test/actions/test_publish_action.py +++ b/test/actions/test_publish_action.py @@ -135,6 +135,14 @@ def test_publish_action_builds_frontend_before_source_install() -> None: assert names.index("Build static frontend") < names.index("Install CodeNib") resolve = next(step for step in _steps(action) if step.get("id") == "inputs") assert "web/dist" in resolve["run"] + setup_node = next( + step for step in _steps(action) if step.get("name") == "Set up Node.js" + ) + dependency_path = setup_node["with"]["cache-dependency-path"] + assert dependency_path == ( + "${{ steps.inputs.outputs.source_path }}/web/package-lock.json" + ) + assert ".." not in dependency_path def test_publish_action_keeps_untrusted_inputs_out_of_shell_source() -> None: @@ -171,6 +179,7 @@ def test_publish_action_resolves_valid_inputs(tmp_path: Path) -> None: assert f"context_path={tmp_path / 'codenib-context'}" in output assert "embedding_provider=" in output assert "extras=" in output + assert f"source_path={ROOT}" in output assert f"source_commit={source_commit}" in output From 18b191fe1c65eb4e89d079cb6f47c94b888bfb16 Mon Sep 17 00:00:00 2001 From: fishmingyu <1661342068@qq.com> Date: Wed, 5 Aug 2026 01:20:19 -0700 Subject: [PATCH 6/8] test(artifacts): cover escaped Windows paths --- test/web/test_static_export.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/web/test_static_export.py b/test/web/test_static_export.py index cc147bb4..5f59dce6 100644 --- a/test/web/test_static_export.py +++ b/test/web/test_static_export.py @@ -10,8 +10,8 @@ import pytest -from codenib.compiler.manifest import IndexEntry, RepoManifest from codenib.artifacts.security import assert_publishable_tree +from codenib.compiler.manifest import IndexEntry, RepoManifest from codenib.web.static_export import ( STATIC_EXPORT_MANIFEST, export_static_wiki, @@ -250,6 +250,9 @@ def test_publishability_rejects_json_escaped_windows_path(tmp_path: Path) -> Non ) class ResolvedWindowsPath: + def expanduser(self): + return self + def resolve(self): return windows_path From ee0eb66c1409b7c0f61df32421b4dd5911e7271d Mon Sep 17 00:00:00 2001 From: fishmingyu <1661342068@qq.com> Date: Wed, 5 Aug 2026 01:44:16 -0700 Subject: [PATCH 7/8] docs(pages): separate static and query capabilities --- README.md | 15 ++++++++------- docs/github_pages.md | 12 ++++++++---- docs/product_roadmap.md | 5 +++-- docs/quickstart.md | 4 ++-- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 82812135..407a86f0 100644 --- a/README.md +++ b/README.md @@ -52,10 +52,10 @@ agent or code-Wiki framework. - **2026-08-04 — Commit-addressed Pages publishing.** [`codenib publish`](docs/github_pages.md) and the reusable GitHub workflow - build a no-model BM25 or opt-in semantic Wiki, deploy the static inspection - surface, and retain the matching portable context views for the indexed - commit. Incremental caches remain private build state rather than part of the - downloadable serving artifact. + deploy a no-model static Wiki and retain a matching BM25 or opt-in + vector-enhanced context artifact for the indexed commit. Incremental caches + remain private build state rather than part of the downloadable serving + artifact. - **2026-08-04 — Native repository explorer.** [`RepositoryContextExplorer`](codenib/agent/runtime/explorer.py) plans BM25, dense, hybrid, reranked, and graph routes over manifest-backed views and @@ -138,9 +138,10 @@ the local or MCP serving path. For a repository-hosted Wiki, CodeNib also ships a reusable GitHub workflow that incrementally builds the same manifest, deploys the static site to Pages, and uploads the matching commit-addressed context artifact. Its default `fast` -route needs no model credential or model download; semantic search can use a -local Hugging Face model or an explicit BYO OpenAI-compatible endpoint. See -[GitHub Pages](https://docs.codenib.ai/github_pages/). +route needs no model credential or model download. The optional `semantic` +route builds the matching vector artifact with a local Hugging Face model or an +explicit BYO OpenAI-compatible endpoint; query-time search remains in the local +or MCP runtime. See [GitHub Pages](https://docs.codenib.ai/github_pages/). See the [Quickstart](https://docs.codenib.ai/quickstart/) diff --git a/docs/github_pages.md b/docs/github_pages.md index 403a0e3a..5585d24d 100644 --- a/docs/github_pages.md +++ b/docs/github_pages.md @@ -41,9 +41,11 @@ the source. The workflow checks out the caller's exact commit, incrementally builds the `fast` preset, exports the Wiki at the Pages-provided mount path, and deploys it through the `github-pages` environment. It also uploads an artifact named from -the repository and commit. +the repository and commit. BM25 belongs to that context artifact; the static +Wiki serves precomputed pages, citations, and navigation without executing a +query engine in the browser. -## Semantic Search With A Local Model +## Build Semantic Context With A Local Model Select the semantic preset to build BM25 and dense-vector views. It defaults to a local Hugging Face embedding model and needs no API credential: @@ -58,8 +60,10 @@ jobs: The first build downloads the default embedding model into the ephemeral Action runner. The resulting vector view is stored in the commit-addressed context -artifact and reused through CodeNib's repository cache on later builds. Keep the -default `fast` preset when a model download is undesirable. +artifact and reused through CodeNib's repository cache on later builds. Serve +that artifact through the local or MCP runtime for semantic queries; the Pages +site remains a precomputed inspection surface. Keep the default `fast` preset +when a model download is undesirable. ## Bring Your Own Embedding Endpoint diff --git a/docs/product_roadmap.md b/docs/product_roadmap.md index 32e27e1e..5851c026 100644 --- a/docs/product_roadmap.md +++ b/docs/product_roadmap.md @@ -252,8 +252,9 @@ The program has two user-facing surfaces that share one artifact contract: Generation and embeddings are build-time provider capabilities in the static surface. A Pages export never contains a provider credential, and it does not claim query-time semantic search when no authenticated runtime exists. The -offline fallback remains useful through lexical search, source navigation, -pre-generated pages, and dependency data when those views are available. +Pages surface remains useful through source navigation, pre-generated pages, +and dependency data when those views are available. The matching artifact adds +lexical and optional semantic search when loaded by the local or MCP runtime. ### H1: Static artifact contract ([#416](https://github.com/sysevol-ai/CodeNib/issues/416)) diff --git a/docs/quickstart.md b/docs/quickstart.md index c3fa7d58..ea56defc 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -120,8 +120,8 @@ The context directory is commit-addressed query-serving state with an artifact-relative manifest and file hashes. It excludes mutable maintenance caches. Portable publication currently supports `fast` and `semantic`; use the local or MCP runtime for graph and Zoekt views. See -[Publish With GitHub Pages](github_pages.md) for the no-model Action, GitHub -Models, and BYO endpoint configurations. +[Publish With GitHub Pages](github_pages.md) for the no-model Action, local +embedding model, and BYO endpoint configurations. ## Select Repository Views From 5c5576def6899611324ad99879afb8075dd7190d Mon Sep 17 00:00:00 2001 From: fishmingyu <1661342068@qq.com> Date: Tue, 4 Aug 2026 23:59:07 -0700 Subject: [PATCH 8/8] refactor(artifacts): make vector payloads portable Convert trusted local vector document pickles into inert JSON when packaging query-serving artifacts. Prefer that portable format at load time while retaining the legacy pickle fallback for local indexes. Verified with focused artifact and vector-store tests. --- codenib/artifacts/context.py | 34 +++++++++++---- codenib/index/embedding/vector_store.py | 38 ++++++++++++++++- test/artifacts/test_context_artifact.py | 11 +++-- test/index/test_vector_store_ivf.py | 55 +++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 12 deletions(-) diff --git a/codenib/artifacts/context.py b/codenib/artifacts/context.py index 8b7f000f..fbced665 100644 --- a/codenib/artifacts/context.py +++ b/codenib/artifacts/context.py @@ -8,7 +8,6 @@ import json import os -import pickle import re import shutil import subprocess @@ -178,24 +177,42 @@ def _portable_source_path(value: object, repo_path: Path, *, source: str) -> str return normalized.as_posix() -def _normalize_vector_documents(path: Path, repo_path: Path) -> None: +def _convert_vector_documents(path: Path, repo_path: Path) -> Path: + """Convert a trusted local document pickle to portable, inert JSON.""" + with path.open("rb") as handle: documents = compat_pickle.load(handle) if not isinstance(documents, list): raise ValueError(f"portable vector documents must be a list: {path.name}") + payload: list[dict[str, Any]] = [] for index, document in enumerate(documents): + page_content = getattr(document, "page_content", None) + if not isinstance(page_content, str): + raise ValueError( + f"portable vector document {index} has invalid content: {path.name}" + ) metadata = getattr(document, "metadata", None) if not isinstance(metadata, dict): raise ValueError( f"portable vector document {index} has invalid metadata: {path.name}" ) - metadata["file"] = _portable_source_path( + normalized_metadata = dict(metadata) + normalized_metadata["file"] = _portable_source_path( metadata.get("file"), repo_path, source=f"vector document {index} file", ) - with path.open("wb") as handle: - pickle.dump(documents, handle, protocol=pickle.HIGHEST_PROTOCOL) + payload.append( + { + "page_content": page_content, + "metadata": normalized_metadata, + } + ) + + output = path.with_suffix(".json") + output.write_bytes(_json_bytes(payload)) + path.unlink() + return output def _normalize_vector_view(target: Path, repo_path: Path) -> dict[str, Any]: @@ -222,13 +239,16 @@ def _normalize_vector_view(target: Path, repo_path: Path) -> dict[str, Any]: "rebuild the vector view" ) for path in document_files: - _normalize_vector_documents(path, repo_path) + _convert_vector_documents(path, repo_path) # The current document files supersede legacy LangChain docstore pickles. # Leaving both formats would retain duplicate absolute source paths. for legacy in target.glob("l[02]/index_*.pkl"): legacy.unlink() - return {"artifact_scope": "query-serving"} + return { + "artifact_scope": "query-serving", + "portable_document_format": "codenib.vector-documents.v1", + } def _inventory(root: Path) -> list[dict[str, Any]]: diff --git a/codenib/index/embedding/vector_store.py b/codenib/index/embedding/vector_store.py index 3218760b..445e5781 100644 --- a/codenib/index/embedding/vector_store.py +++ b/codenib/index/embedding/vector_store.py @@ -1138,8 +1138,16 @@ def _load_level( f"expected {self.dimension}, found {int(index.d)}" ) - # Try loading documents pickle (works for both new _Document and - # legacy LangChain Document objects via duck-typing conversion). + # Portable artifacts use inert JSON so a downloaded document store is + # never unpickled. Local indexes retain the pickle fallback for + # compatibility with previously built artifacts. + json_path = level_path / f"documents_{model_suffix}.json" + if json_path.exists(): + documents = self._load_documents_json(json_path) + return index, documents + + # Try loading the local documents pickle (works for both new _Document + # and legacy LangChain Document objects via duck-typing conversion). docs_path = level_path / f"documents_{model_suffix}.pkl" if docs_path.exists(): try: @@ -1169,6 +1177,32 @@ def _load_level( logger.warning(f"No document store found for {level_path}") return index, [] + @staticmethod + def _load_documents_json(path: Path) -> List[_Document]: + """Load the non-executable portable vector document format.""" + + with path.open(encoding="utf-8") as handle: + payload = json.load(handle) + if not isinstance(payload, list): + raise ValueError(f"vector documents must be a JSON list: {path}") + + documents: List[_Document] = [] + for index, item in enumerate(payload): + if not isinstance(item, dict): + raise ValueError( + f"vector document {index} must be a JSON object: {path}" + ) + page_content = item.get("page_content") + metadata = item.get("metadata") + if not isinstance(page_content, str) or not isinstance(metadata, dict): + raise ValueError( + f"vector document {index} has invalid content or metadata: {path}" + ) + documents.append( + _Document(page_content=page_content, metadata=dict(metadata)) + ) + return documents + @staticmethod def _load_langchain_pkl(pkl_path: Path) -> List[_Document]: """Extract documents from a LangChain FAISS pkl file. diff --git a/test/artifacts/test_context_artifact.py b/test/artifacts/test_context_artifact.py index d7ed8bfc..05a3cada 100644 --- a/test/artifacts/test_context_artifact.py +++ b/test/artifacts/test_context_artifact.py @@ -246,13 +246,18 @@ def test_context_artifact_keeps_only_portable_vector_serving_state( assert not (vector / "incremental_state.json").exists() assert not (vector / "l2" / "index_test__model.pkl").exists() assert (vector / "l2" / "index_test__model.faiss").is_file() - with (vector / "l2" / "documents_test__model.pkl").open("rb") as handle: - documents = pickle.load(handle) - assert documents[0].metadata["file"] == "sample.py" + assert not (vector / "l2" / "documents_test__model.pkl").exists() + documents = json.loads((vector / "l2" / "documents_test__model.json").read_text()) + assert documents[0]["metadata"]["file"] == "sample.py" portable = json.loads((output / "repo_manifest.json").read_text()) assert portable["indexes"]["vector"]["config"]["artifact_scope"] == ( "query-serving" ) + assert ( + portable["indexes"]["vector"]["config"]["portable_document_format"] + == "codenib.vector-documents.v1" + ) + assert not list(output.rglob("*.pkl")) assert str(repo).encode() not in b"".join(_tree(output).values()) diff --git a/test/index/test_vector_store_ivf.py b/test/index/test_vector_store_ivf.py index a04ef30d..1b5521a8 100644 --- a/test/index/test_vector_store_ivf.py +++ b/test/index/test_vector_store_ivf.py @@ -169,6 +169,61 @@ def test_ivf_save_load_roundtrip(tmp_path): assert res and res[0].node_name == chunks[1]["name"] +def test_load_prefers_portable_json_documents(tmp_path): + path = tmp_path / "vs" + store = _make_store(embedding_model="test/model") + chunks = _chunks(2) + store.add_code_chunks(chunks) + store.save(str(path)) + + documents_path = path / "l2" / "documents_test__model.pkl" + documents_path.unlink() + portable_path = documents_path.with_suffix(".json") + portable_path.write_text( + json.dumps( + [ + { + "page_content": chunk["content"], + "metadata": { + "name": chunk["name"], + "file": chunk["file"], + "start_line": chunk["start_line"], + "end_line": chunk["end_line"], + }, + } + for chunk in chunks + ] + ), + encoding="utf-8", + ) + + loaded = _make_store(embedding_model="test/model", store_path=str(path)) + loaded.load() + + assert [document.metadata["file"] for document in loaded.l2_documents] == [ + "m0.py", + "m1.py", + ] + + +def test_load_rejects_invalid_portable_json_documents(tmp_path): + path = tmp_path / "vs" + store = _make_store(embedding_model="test/model") + store.add_code_chunks(_chunks(1)) + store.save(str(path)) + + documents_path = path / "l2" / "documents_test__model.pkl" + documents_path.unlink() + documents_path.with_suffix(".json").write_text( + '[{"page_content": 7, "metadata": {}}]', + encoding="utf-8", + ) + + loaded = _make_store(embedding_model="test/model", store_path=str(path)) + with pytest.raises(ValueError, match="invalid content or metadata"): + loaded.load() + + def test_load_rejects_faiss_dimension_mismatch(tmp_path): path = tmp_path / "vs" model = "test/model"