diff --git a/README.md b/README.md index 407a86f0..ceb539da 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,8 @@ route needs no model credential or model download. The optional `semantic` route builds the matching vector artifact with a local Hugging Face model or an explicit BYO OpenAI-compatible endpoint; query-time search remains in the local or MCP runtime. See [GitHub Pages](https://docs.codenib.ai/github_pages/). +The published BM25/vector artifact can then be verified against an exact local +checkout and served through MCP without rebuilding the repository views. See the [Quickstart](https://docs.codenib.ai/quickstart/) diff --git a/codenib/artifacts/__init__.py b/codenib/artifacts/__init__.py index b06e05d8..0a37e75b 100644 --- a/codenib/artifacts/__init__.py +++ b/codenib/artifacts/__init__.py @@ -4,6 +4,7 @@ """Portable repository-context artifacts.""" +from .archive import extract_context_artifact_archive from .context import ( CONTEXT_ARTIFACT_MANIFEST, CONTEXT_ARTIFACT_SCHEMA, @@ -11,11 +12,35 @@ ContextArtifactResult, stage_context_artifact, ) +from .github import ( + GitHubArtifactFetchResult, + GitHubArtifactRecord, + fetch_github_context_artifact, + resolve_github_context_artifact, +) +from .mcp_config import MCP_CONFIG_HOSTS, render_artifact_mcp_config +from .runtime import ( + ContextArtifactBinding, + VerifiedContextArtifact, + bind_context_artifact, + verify_context_artifact, +) __all__ = [ "CONTEXT_ARTIFACT_MANIFEST", "CONTEXT_ARTIFACT_SCHEMA", "PORTABLE_CONTEXT_VIEWS", "ContextArtifactResult", + "ContextArtifactBinding", + "GitHubArtifactFetchResult", + "GitHubArtifactRecord", + "MCP_CONFIG_HOSTS", + "VerifiedContextArtifact", + "bind_context_artifact", + "extract_context_artifact_archive", + "fetch_github_context_artifact", + "resolve_github_context_artifact", + "render_artifact_mcp_config", "stage_context_artifact", + "verify_context_artifact", ] diff --git a/codenib/artifacts/archive.py b/codenib/artifacts/archive.py new file mode 100644 index 00000000..8425f0d0 --- /dev/null +++ b/codenib/artifacts/archive.py @@ -0,0 +1,216 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +"""Bounded, traversal-safe extraction for context artifact archives.""" + +from __future__ import annotations + +import os +import shutil +import stat +import tempfile +import zipfile +from pathlib import Path, PurePosixPath + +from .context import CONTEXT_ARTIFACT_MANIFEST +from .runtime import VerifiedContextArtifact, verify_context_artifact + +DEFAULT_MAX_ARCHIVE_FILES = 100_000 +DEFAULT_MAX_EXPANDED_BYTES = 64 * 1024 * 1024 * 1024 +_COPY_CHUNK_BYTES = 1024 * 1024 + + +def _member_path(value: str) -> PurePosixPath: + if not value or "\\" in value or "\x00" in value: + raise ValueError("context artifact archive contains an invalid path") + path = PurePosixPath(value) + normalized = path.as_posix().rstrip("/") + if ( + path.is_absolute() + or not normalized + or value.rstrip("/") != normalized + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise ValueError(f"context artifact archive path is unsafe: {value!r}") + return PurePosixPath(normalized) + + +def _member_kind(info: zipfile.ZipInfo) -> int: + return stat.S_IFMT(info.external_attr >> 16) + + +def _validated_members( + archive: zipfile.ZipFile, + *, + max_files: int, + max_bytes: int, +) -> list[tuple[zipfile.ZipInfo, PurePosixPath]]: + entries = archive.infolist() + max_entries = max_files * 2 + 64 + if len(entries) > max_entries: + raise ValueError(f"context artifact archive exceeds {max_entries} entries") + raw: list[tuple[zipfile.ZipInfo, PurePosixPath]] = [] + metadata_paths: list[PurePosixPath] = [] + for info in entries: + path = _member_path(info.filename) + kind = _member_kind(info) + if kind == stat.S_IFLNK: + raise ValueError( + f"context artifact archive contains a symbolic link: {path}" + ) + if kind not in {0, stat.S_IFREG, stat.S_IFDIR}: + raise ValueError( + f"context artifact archive contains a special file: {path}" + ) + if info.flag_bits & 0x1: + raise ValueError(f"context artifact archive member is encrypted: {path}") + raw.append((info, path)) + if not info.is_dir() and path.name == CONTEXT_ARTIFACT_MANIFEST: + metadata_paths.append(path) + + if len(metadata_paths) != 1: + raise ValueError( + "context artifact archive must contain exactly one metadata file" + ) + prefix = metadata_paths[0].parent + prefix_parts = () if str(prefix) == "." else prefix.parts + + result: list[tuple[zipfile.ZipInfo, PurePosixPath]] = [] + seen: set[str] = set() + file_count = 0 + total_bytes = 0 + for info, path in raw: + if prefix_parts: + if path.parts[: len(prefix_parts)] != prefix_parts: + raise ValueError( + "context artifact archive contains files outside its root" + ) + stripped_parts = path.parts[len(prefix_parts) :] + if not stripped_parts: + continue + path = PurePosixPath(*stripped_parts) + relative = path.as_posix() + if relative in seen: + raise ValueError(f"duplicate context artifact archive path: {relative}") + seen.add(relative) + if not info.is_dir(): + file_count += 1 + total_bytes += info.file_size + if file_count > max_files: + raise ValueError(f"context artifact archive exceeds {max_files} files") + if total_bytes > max_bytes: + raise ValueError( + f"context artifact archive exceeds {max_bytes} expanded bytes" + ) + result.append((info, path)) + return result + + +def _extract_member( + archive: zipfile.ZipFile, + info: zipfile.ZipInfo, + relative: PurePosixPath, + root: Path, +) -> None: + output = root.joinpath(*relative.parts) + if info.is_dir(): + output.mkdir(parents=True, exist_ok=True) + return + output.parent.mkdir(parents=True, exist_ok=True) + written = 0 + with archive.open(info, "r") as source, output.open("xb") as destination: + while chunk := source.read(_COPY_CHUNK_BYTES): + written += len(chunk) + if written > info.file_size: + raise ValueError( + f"context artifact archive member exceeded declared size: {relative}" + ) + destination.write(chunk) + if written != info.file_size: + raise ValueError(f"context artifact archive member size mismatch: {relative}") + + +def extract_context_artifact_archive( + archive_path: str | Path, + output_dir: str | Path, + *, + expected_repository: str | None = None, + expected_commit: str | None = None, + max_files: int = DEFAULT_MAX_ARCHIVE_FILES, + max_bytes: int = DEFAULT_MAX_EXPANDED_BYTES, +) -> VerifiedContextArtifact: + """Extract and verify an artifact ZIP before publishing it.""" + + archive_candidate = Path(archive_path).expanduser() + if archive_candidate.is_symlink(): + raise ValueError( + f"context artifact archive must not be a symbolic link: {archive_candidate}" + ) + archive_path = archive_candidate.resolve() + if not archive_path.is_file(): + raise ValueError(f"context artifact archive does not exist: {archive_path}") + output_candidate = Path(output_dir).expanduser() + if output_candidate.is_symlink(): + raise ValueError( + f"context artifact output must not be a symbolic link: {output_candidate}" + ) + output = output_candidate.resolve() + if output.exists() and not output.is_dir(): + raise ValueError(f"context artifact output is not a directory: {output}") + if output.exists() and any(output.iterdir()): + if not (output / CONTEXT_ARTIFACT_MANIFEST).is_file(): + raise ValueError( + "refusing to replace a non-empty directory that is not a " + f"CodeNib context artifact: {output}" + ) + + output.parent.mkdir(parents=True, exist_ok=True) + stage = Path( + tempfile.mkdtemp( + prefix=f".{output.name}.extract-", + dir=str(output.parent), + ) + ).resolve() + try: + with zipfile.ZipFile(archive_path) as archive: + members = _validated_members( + archive, + max_files=max_files, + max_bytes=max_bytes, + ) + for info, relative in members: + _extract_member(archive, info, relative, stage) + verify_context_artifact( + stage, + expected_repository=expected_repository, + expected_commit=expected_commit, + max_files=max_files, + max_bytes=max_bytes, + ) + if output.exists(): + shutil.rmtree(output) + os.replace(stage, output) + except zipfile.BadZipFile as exc: + shutil.rmtree(stage, ignore_errors=True) + raise ValueError( + f"context artifact archive is not a valid ZIP: {archive_path}" + ) from exc + except BaseException: + shutil.rmtree(stage, ignore_errors=True) + raise + + return verify_context_artifact( + output, + expected_repository=expected_repository, + expected_commit=expected_commit, + max_files=max_files, + max_bytes=max_bytes, + ) + + +__all__ = [ + "DEFAULT_MAX_ARCHIVE_FILES", + "DEFAULT_MAX_EXPANDED_BYTES", + "extract_context_artifact_archive", +] diff --git a/codenib/artifacts/github.py b/codenib/artifacts/github.py new file mode 100644 index 00000000..69a8c32c --- /dev/null +++ b/codenib/artifacts/github.py @@ -0,0 +1,424 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +"""Resolve commit-addressed CodeNib artifacts through the GitHub REST API.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import tempfile +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +from ..compiler.snapshot_store import normalize_repo +from ..paths import user_state_dir +from .archive import ( + DEFAULT_MAX_ARCHIVE_FILES, + DEFAULT_MAX_EXPANDED_BYTES, + extract_context_artifact_archive, +) +from .runtime import VerifiedContextArtifact, verify_context_artifact + +GITHUB_API_VERSION = "2026-03-10" +DEFAULT_GITHUB_TOKEN_ENV = "GH_TOKEN" +DEFAULT_MAX_DOWNLOAD_BYTES = 16 * 1024 * 1024 * 1024 +_MAX_API_RESPONSE_BYTES = 16 * 1024 * 1024 +_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") +_DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") +_GITHUB_REPOSITORY_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") +_ENV_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_USER_AGENT = "CodeNib-context-artifact" + + +@dataclass(frozen=True, slots=True) +class GitHubArtifactRecord: + """GitHub provenance for one resolved workflow artifact.""" + + artifact_id: int + name: str + repository: str + head_sha: str + archive_download_url: str + archive_digest: str + size_in_bytes: int + created_at: str + + +@dataclass(frozen=True, slots=True) +class GitHubArtifactFetchResult: + """Verified cache result from a GitHub artifact lookup.""" + + artifact: VerifiedContextArtifact + record: GitHubArtifactRecord | None + downloaded: bool + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +def _github_repository(value: str) -> tuple[str, str, str]: + raw = value.strip().removesuffix(".git") + if not _GITHUB_REPOSITORY_RE.fullmatch(raw): + raise ValueError("GitHub repository must use owner/name form") + normalized = normalize_repo(raw) + owner, name = normalized.split("/", 1) + return normalized, owner, name + + +def _full_commit(value: str) -> str: + commit = value.strip().lower() + if not _COMMIT_RE.fullmatch(commit): + raise ValueError("GitHub artifact resolution requires a full Git SHA") + return commit + + +def default_github_artifact_name(repository: str, commit: str) -> str: + """Return the name emitted by the CodeNib publishing Action.""" + + normalized, _owner, _name = _github_repository(repository) + resolved_commit = _full_commit(commit) + return f"codenib-context-{normalized.replace('/', '-')}-{resolved_commit[:12]}" + + +def default_context_artifact_dir(repository: str, commit: str) -> Path: + """Return the user-owned cache path for one repository commit.""" + + _normalized, owner, name = _github_repository(repository) + return user_state_dir() / "artifacts" / owner / name / _full_commit(commit) + + +def _validated_api_url(value: str) -> str: + parsed = urllib.parse.urlsplit(value.rstrip("/")) + if ( + parsed.scheme != "https" + or not parsed.netloc + or parsed.username + or parsed.password + or parsed.query + or parsed.fragment + ): + raise ValueError("GitHub API URL must be an HTTPS origin or path") + return urllib.parse.urlunsplit(parsed) + + +def _request_headers(token: str | None = None) -> dict[str, str]: + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": _USER_AGENT, + "X-GitHub-Api-Version": GITHUB_API_VERSION, + } + if token: + headers["Authorization"] = f"Bearer {token}" + return headers + + +def _github_json(url: str, *, token: str | None) -> dict[str, Any]: + request = urllib.request.Request(url, headers=_request_headers(token)) + # The API response never needs a redirect. Refusing one prevents a custom + # or compromised API origin from forwarding the bearer token elsewhere. + opener = urllib.request.build_opener(_NoRedirect()) + try: + with opener.open(request, timeout=30) as response: + payload = response.read(_MAX_API_RESPONSE_BYTES + 1) + except urllib.error.HTTPError as exc: + try: + code = exc.code + finally: + exc.close() + raise RuntimeError(f"GitHub API request failed with HTTP {code}") from exc + except urllib.error.URLError as exc: + raise RuntimeError("GitHub API request failed") from exc + if len(payload) > _MAX_API_RESPONSE_BYTES: + raise RuntimeError("GitHub API response exceeded the safety limit") + try: + value = json.loads(payload) + except (UnicodeError, json.JSONDecodeError) as exc: + raise RuntimeError("GitHub API returned invalid JSON") from exc + if not isinstance(value, dict): + raise RuntimeError("GitHub API returned an unexpected response") + return value + + +def _record( + value: object, + *, + repository: str, + expected_name: str, + expected_commit: str, + api_origin: tuple[str, str], +) -> GitHubArtifactRecord | None: + if not isinstance(value, Mapping): + return None + if value.get("name") != expected_name or value.get("expired") is not False: + return None + workflow_run = value.get("workflow_run") + if not isinstance(workflow_run, Mapping): + return None + head_sha = workflow_run.get("head_sha") + if head_sha != expected_commit: + return None + artifact_id = value.get("id") + size = value.get("size_in_bytes") + digest = value.get("digest") + download_url = value.get("archive_download_url") + created_at = value.get("created_at") + if ( + not isinstance(artifact_id, int) + or isinstance(artifact_id, bool) + or artifact_id <= 0 + or not isinstance(size, int) + or isinstance(size, bool) + or size < 0 + or not isinstance(digest, str) + or not _DIGEST_RE.fullmatch(digest) + or not isinstance(download_url, str) + or not isinstance(created_at, str) + ): + return None + parsed_download = urllib.parse.urlsplit(download_url) + if (parsed_download.scheme, parsed_download.netloc) != api_origin: + return None + return GitHubArtifactRecord( + artifact_id=artifact_id, + name=expected_name, + repository=repository, + head_sha=head_sha, + archive_download_url=download_url, + archive_digest=digest, + size_in_bytes=size, + created_at=created_at, + ) + + +def resolve_github_context_artifact( + repository: str, + commit: str, + *, + artifact_name: str | None = None, + token: str | None = None, + api_url: str = "https://api.github.com", +) -> GitHubArtifactRecord: + """Resolve the newest non-expired artifact for one exact workflow SHA.""" + + normalized, owner, name = _github_repository(repository) + commit = _full_commit(commit) + artifact_name = artifact_name or default_github_artifact_name( + normalized, + commit, + ) + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", artifact_name): + raise ValueError("GitHub artifact name is invalid") + api_url = _validated_api_url(api_url) + parsed_api = urllib.parse.urlsplit(api_url) + api_origin = (parsed_api.scheme, parsed_api.netloc) + encoded_name = urllib.parse.urlencode( + {"name": artifact_name, "per_page": 100}, + ) + endpoint = ( + f"{api_url}/repos/{urllib.parse.quote(owner, safe='')}/" + f"{urllib.parse.quote(name, safe='')}/actions/artifacts?{encoded_name}" + ) + payload = _github_json(endpoint, token=token) + artifacts = payload.get("artifacts") + if not isinstance(artifacts, list): + raise RuntimeError("GitHub artifact listing has no artifact array") + records = [ + record + for item in artifacts + if ( + record := _record( + item, + repository=normalized, + expected_name=artifact_name, + expected_commit=commit, + api_origin=api_origin, + ) + ) + ] + if not records: + raise FileNotFoundError( + "no non-expired CodeNib artifact matches " + f"{normalized}@{commit[:12]} with name {artifact_name!r}" + ) + return max(records, key=lambda item: item.artifact_id) + + +def _download_archive( + url: str, + output: Path, + *, + token: str, + max_bytes: int, +) -> tuple[int, str]: + """Download through GitHub's redirect without forwarding its token.""" + + request = urllib.request.Request(url, headers=_request_headers(token)) + opener = urllib.request.build_opener(_NoRedirect()) + try: + response = opener.open(request, timeout=30) + except urllib.error.HTTPError as exc: + try: + if exc.code != 302: + raise RuntimeError( + f"GitHub artifact download failed with HTTP {exc.code}" + ) from exc + location = exc.headers.get("Location") + finally: + exc.close() + except urllib.error.URLError as exc: + raise RuntimeError("GitHub artifact download request failed") from exc + else: + response.close() + raise RuntimeError("GitHub artifact download did not return a redirect") + if not location: + raise RuntimeError("GitHub artifact download redirect is missing") + parsed = urllib.parse.urlsplit(location) + if ( + parsed.scheme != "https" + or not parsed.netloc + or parsed.username + or parsed.password + ): + raise RuntimeError("GitHub artifact download redirect is unsafe") + + redirected = urllib.request.Request( + location, + headers={"User-Agent": _USER_AGENT}, + ) + digest = hashlib.sha256() + size = 0 + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + with urllib.request.urlopen(redirected, timeout=60) as response: + with os.fdopen(os.open(output, flags, 0o600), "wb") as handle: + while chunk := response.read(1024 * 1024): + size += len(chunk) + if size > max_bytes: + raise RuntimeError( + f"GitHub artifact download exceeds {max_bytes} bytes" + ) + digest.update(chunk) + handle.write(chunk) + except urllib.error.HTTPError as exc: + raise RuntimeError( + f"GitHub artifact object download failed with HTTP {exc.code}" + ) from exc + except urllib.error.URLError as exc: + raise RuntimeError("GitHub artifact object download failed") from exc + return size, digest.hexdigest() + + +def fetch_github_context_artifact( + repository: str, + commit: str, + *, + output_dir: str | Path | None = None, + artifact_name: str | None = None, + token: str | None = None, + token_env: str = DEFAULT_GITHUB_TOKEN_ENV, + api_url: str = "https://api.github.com", + force: bool = False, + max_download_bytes: int = DEFAULT_MAX_DOWNLOAD_BYTES, + max_files: int = DEFAULT_MAX_ARCHIVE_FILES, + max_expanded_bytes: int = DEFAULT_MAX_EXPANDED_BYTES, +) -> GitHubArtifactFetchResult: + """Download, archive-verify, extract, and cache one GitHub artifact.""" + + repository, _owner, _name = _github_repository(repository) + commit = _full_commit(commit) + if not _ENV_NAME_RE.fullmatch(token_env): + raise ValueError("GitHub token environment variable name is invalid") + output_candidate = ( + Path(output_dir).expanduser() + if output_dir is not None + else default_context_artifact_dir(repository, commit) + ) + if output_candidate.is_symlink(): + raise ValueError( + f"context artifact output must not be a symbolic link: {output_candidate}" + ) + output = output_candidate.resolve() + if output.is_dir() and (output / "codenib-context.json").is_file() and not force: + return GitHubArtifactFetchResult( + artifact=verify_context_artifact( + output, + expected_repository=repository, + expected_commit=commit, + max_files=max_files, + max_bytes=max_expanded_bytes, + ), + record=None, + downloaded=False, + ) + token = token or os.environ.get(token_env) + if not token: + raise ValueError(f"GitHub artifact download requires a token in {token_env}") + + record = resolve_github_context_artifact( + repository, + commit, + artifact_name=artifact_name, + token=token, + api_url=api_url, + ) + if record.size_in_bytes > max_download_bytes: + raise ValueError( + "GitHub artifact exceeds the configured download limit: " + f"{record.size_in_bytes} > {max_download_bytes}" + ) + output.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{output.name}.", + suffix=".zip", + dir=str(output.parent), + ) + os.close(descriptor) + archive = Path(temporary_name) + try: + _size, digest = _download_archive( + record.archive_download_url, + archive, + token=token, + max_bytes=max_download_bytes, + ) + if f"sha256:{digest}" != record.archive_digest: + raise ValueError("GitHub artifact archive digest does not match the API") + artifact = extract_context_artifact_archive( + archive, + output, + expected_repository=repository, + expected_commit=commit, + max_files=max_files, + max_bytes=max_expanded_bytes, + ) + finally: + archive.unlink(missing_ok=True) + return GitHubArtifactFetchResult( + artifact=artifact, + record=record, + downloaded=True, + ) + + +__all__ = [ + "DEFAULT_GITHUB_TOKEN_ENV", + "DEFAULT_MAX_DOWNLOAD_BYTES", + "GITHUB_API_VERSION", + "GitHubArtifactFetchResult", + "GitHubArtifactRecord", + "default_context_artifact_dir", + "default_github_artifact_name", + "fetch_github_context_artifact", + "resolve_github_context_artifact", +] diff --git a/codenib/artifacts/mcp_config.py b/codenib/artifacts/mcp_config.py new file mode 100644 index 00000000..c5b9367c --- /dev/null +++ b/codenib/artifacts/mcp_config.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +"""Generate reviewable MCP client configuration for a verified artifact.""" + +from __future__ import annotations + +import json +import re +import shlex +from pathlib import Path + +from .runtime import bind_context_artifact + +MCP_CONFIG_HOSTS = ("claude", "codex", "json") +_SERVER_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+$") + + +def render_artifact_mcp_config( + artifact_path: str | Path, + repo_path: str | Path, + *, + host: str, + server_name: str = "codenib", + command: str = "codenib", + expected_repository: str | None = None, + claude_scope: str = "project", +) -> str: + """Verify a binding and render a host command or generic JSON config.""" + + if host not in MCP_CONFIG_HOSTS: + raise ValueError(f"unsupported MCP config host: {host}") + if not _SERVER_NAME_RE.fullmatch(server_name): + raise ValueError( + "MCP server name must use letters, digits, dot, dash, or underscore" + ) + if not command or "\x00" in command or "\n" in command or "\r" in command: + raise ValueError("MCP command must be a non-empty single-line value") + if claude_scope not in {"local", "project", "user"}: + raise ValueError("Claude MCP scope must be local, project, or user") + + binding = bind_context_artifact( + artifact_path, + repo_path, + expected_repository=expected_repository, + ) + args = [ + "mcp", + "--artifact", + str(binding.artifact.root), + "--repo", + str(binding.repo_path), + "--repository", + binding.artifact.repository, + ] + server = { + "type": "stdio", + "command": command, + "args": args, + } + if host == "json": + return ( + json.dumps( + {"mcpServers": {server_name: server}}, + ensure_ascii=True, + indent=2, + sort_keys=True, + ) + + "\n" + ) + if host == "claude": + payload = json.dumps(server, ensure_ascii=True, separators=(",", ":")) + return ( + shlex.join( + [ + "claude", + "mcp", + "add-json", + "--scope", + claude_scope, + server_name, + payload, + ] + ) + + "\n" + ) + return shlex.join(["codex", "mcp", "add", server_name, "--", command, *args]) + "\n" + + +__all__ = ["MCP_CONFIG_HOSTS", "render_artifact_mcp_config"] diff --git a/codenib/artifacts/runtime.py b/codenib/artifacts/runtime.py new file mode 100644 index 00000000..ed406bd9 --- /dev/null +++ b/codenib/artifacts/runtime.py @@ -0,0 +1,608 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +"""Verify and bind portable context artifacts for query-only runtimes.""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Mapping + +from ..compiler.checkout_identity import checkout_commit +from ..compiler.manifest import MANIFEST_FILENAME, MANIFEST_VERSION, RepoManifest +from ..compiler.snapshot_store import normalize_repo +from ..source_fingerprint import fingerprint_repository +from .context import ( + CONTEXT_ARTIFACT_MANIFEST, + CONTEXT_ARTIFACT_SCHEMA, + PORTABLE_CONTEXT_VIEWS, +) +from .security import file_sha256 + +_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") +_DIGEST_RE = re.compile(r"^[0-9a-f]{64}$") +_SOURCE_FINGERPRINT_RE = re.compile(r"^sha256:[0-9a-f]{64}$") +_REPOSITORY_RE = re.compile(r"^[a-z0-9_.-]+(?:/[a-z0-9_.-]+)*$") +_DEFAULT_MAX_FILES = 100_000 +_DEFAULT_MAX_BYTES = 64 * 1024 * 1024 * 1024 +_MAX_METADATA_BYTES = 16 * 1024 * 1024 + + +@dataclass(frozen=True, slots=True) +class VerifiedContextArtifact: + """Integrity-checked, still-unbound portable context artifact.""" + + root: Path + metadata_path: Path + manifest_path: Path + metadata: Mapping[str, Any] + manifest: RepoManifest + repository: str + commit: str + source_fingerprint: str + views: tuple[str, ...] + source_paths: tuple[str, ...] + file_count: int + byte_count: int + + +@dataclass(frozen=True, slots=True) +class ContextArtifactBinding: + """Verified artifact rebound in memory to one exact source checkout.""" + + artifact: VerifiedContextArtifact + repo_path: Path + manifest: RepoManifest + + +def _load_json_object(path: Path, *, label: str, max_bytes: int) -> dict[str, Any]: + try: + size = path.stat().st_size + except OSError as exc: + raise ValueError(f"{label} is not readable: {path}") from exc + if size > max_bytes: + raise ValueError(f"{label} exceeds {max_bytes} bytes: {path}") + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ValueError(f"{label} is not valid UTF-8 JSON: {path}") from exc + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object: {path}") + return value + + +def _relative_path(value: object, *, label: str) -> PurePosixPath: + if not isinstance(value, str) or not value or "\\" in value or "\x00" in value: + raise ValueError(f"{label} must be a normalized relative POSIX path") + path = PurePosixPath(value) + if ( + path.is_absolute() + or value != path.as_posix() + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise ValueError(f"{label} must be a normalized relative POSIX path") + return path + + +def _artifact_path(root: Path, value: object, *, label: str) -> Path: + relative = _relative_path(value, label=label) + path = root.joinpath(*relative.parts) + resolved = path.resolve() + if root != resolved and root not in resolved.parents: + raise ValueError(f"{label} escapes the context artifact") + return resolved + + +def _actual_files(root: Path, *, max_files: int) -> set[str]: + files: set[str] = set() + for path in sorted(root.rglob("*")): + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + raise ValueError(f"context artifact contains a symbolic link: {relative}") + if path.is_file(): + files.add(relative) + # The metadata file is intentionally outside its own inventory. + if len(files) > max_files + 1: + raise ValueError( + f"context artifact contains more than {max_files} inventoried files" + ) + elif not path.is_dir(): + raise ValueError(f"context artifact contains a special file: {relative}") + return files + + +def _inventory( + metadata: Mapping[str, Any], + *, + max_files: int, + max_bytes: int, +) -> tuple[dict[str, tuple[int, str]], int]: + raw_files = metadata.get("files") + if not isinstance(raw_files, list) or not raw_files: + raise ValueError("context artifact inventory must be a non-empty list") + if len(raw_files) > max_files: + raise ValueError(f"context artifact inventory exceeds {max_files} files") + + result: dict[str, tuple[int, str]] = {} + total_bytes = 0 + for index, record in enumerate(raw_files): + if not isinstance(record, dict): + raise ValueError(f"context artifact file record {index} must be an object") + relative = _relative_path( + record.get("path"), + label=f"context artifact file record {index} path", + ).as_posix() + if relative == CONTEXT_ARTIFACT_MANIFEST: + raise ValueError("context artifact metadata cannot inventory itself") + if relative in result: + raise ValueError(f"duplicate context artifact inventory path: {relative}") + size = record.get("bytes") + digest = record.get("sha256") + if not isinstance(size, int) or isinstance(size, bool) or size < 0: + raise ValueError(f"invalid byte size for context artifact file: {relative}") + if not isinstance(digest, str) or not _DIGEST_RE.fullmatch(digest): + raise ValueError(f"invalid SHA-256 for context artifact file: {relative}") + if PurePosixPath(relative).suffix.lower() in {".pickle", ".pkl"}: + raise ValueError( + f"portable context artifacts must not contain pickle: {relative}" + ) + total_bytes += size + if total_bytes > max_bytes: + raise ValueError(f"context artifact exceeds {max_bytes} inventoried bytes") + result[relative] = (size, digest) + return result, total_bytes + + +def _repository_identity(metadata: Mapping[str, Any]) -> tuple[str, str, str]: + repository = metadata.get("repository") + if not isinstance(repository, dict): + raise ValueError("context artifact repository identity must be an object") + slug = repository.get("slug") + commit = repository.get("commit") + source_fingerprint = repository.get("source_fingerprint") + if not isinstance(slug, str): + raise ValueError("context artifact repository slug must be a string") + try: + normalized_slug = normalize_repo(slug) + except ValueError as exc: + raise ValueError("context artifact repository slug is invalid") from exc + if slug != normalized_slug or not _REPOSITORY_RE.fullmatch(slug): + raise ValueError("context artifact repository slug is not canonical") + if not isinstance(commit, str) or not _COMMIT_RE.fullmatch(commit): + raise ValueError("context artifact commit must be a full lowercase Git SHA") + if not isinstance(source_fingerprint, str) or not _SOURCE_FINGERPRINT_RE.fullmatch( + source_fingerprint + ): + raise ValueError("context artifact source fingerprint is invalid") + return slug, commit, source_fingerprint + + +def _artifact_views(metadata: Mapping[str, Any]) -> tuple[str, ...]: + raw_views = metadata.get("views") + if not isinstance(raw_views, list) or not raw_views: + raise ValueError("context artifact views must be a non-empty list") + if not all(isinstance(view, str) for view in raw_views): + raise ValueError("context artifact view names must be strings") + views = tuple(raw_views) + if len(set(views)) != len(views): + raise ValueError("context artifact view names must be unique") + unsupported = sorted(set(views) - PORTABLE_CONTEXT_VIEWS) + if unsupported: + raise ValueError( + "context artifact contains unsupported portable views: " + + ", ".join(unsupported) + ) + return views + + +def _source_path(value: object, *, label: str) -> str: + if not isinstance(value, str) or not value or "\\" in value or "\x00" in value: + raise ValueError(f"{label} must be a repository-relative POSIX path") + path = PurePosixPath(value) + if ( + path.is_absolute() + or value != path.as_posix() + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise ValueError(f"{label} must be a repository-relative POSIX path") + return path.as_posix() + + +def _document_source_paths( + path: Path, + *, + label: str, + max_bytes: int, +) -> set[str]: + try: + size = path.stat().st_size + except OSError as exc: + raise ValueError(f"{label} is not readable: {path}") from exc + if size > max_bytes: + raise ValueError(f"{label} exceeds {max_bytes} bytes: {path}") + try: + documents = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ValueError(f"{label} is not valid UTF-8 JSON: {path}") from exc + if not isinstance(documents, list): + raise ValueError(f"{label} must be a JSON list: {path}") + + result: set[str] = set() + for index, document in enumerate(documents): + if not isinstance(document, dict): + raise ValueError(f"{label} document {index} must be an object") + page_content = document.get("page_content") + metadata = document.get("metadata") + if not isinstance(page_content, str) or not isinstance(metadata, dict): + raise ValueError( + f"{label} document {index} has invalid content or metadata" + ) + raw_file = metadata.get("file") + if raw_file is not None and raw_file != "": + result.add( + _source_path( + raw_file, + label=f"{label} document {index} file", + ) + ) + for field in ("start_line", "end_line"): + value = metadata.get(field) + if value is not None and ( + not isinstance(value, int) or isinstance(value, bool) or value < 0 + ): + raise ValueError(f"{label} document {index} has invalid {field}") + return result + + +def _validate_view_payloads( + root: Path, + inventory: Mapping[str, tuple[int, str]], + *, + views: tuple[str, ...], +) -> tuple[str, ...]: + source_paths: set[str] = set() + inventory_paths = set(inventory) + + if "bm25" in views: + metadata_relative = "views/bm25/bm25_metadata.json" + documents_relative = "views/bm25/documents.json" + required = {metadata_relative, documents_relative} + if not required <= inventory_paths: + raise ValueError("portable BM25 view is missing its serving files") + metadata = _load_json_object( + root / metadata_relative, + label="portable BM25 metadata", + max_bytes=inventory[metadata_relative][0], + ) + if metadata.get("project_root") != "source": + raise ValueError("portable BM25 project root must be 'source'") + source_paths.update( + _document_source_paths( + root / documents_relative, + label="portable BM25 documents", + max_bytes=inventory[documents_relative][0], + ) + ) + + if "vector" in views: + document_paths = sorted( + relative + for relative in inventory_paths + if PurePosixPath(relative).parent.name in {"l0", "l2"} + and PurePosixPath(relative).parent.parent.as_posix() == "views/vector" + and PurePosixPath(relative).name.startswith("documents_") + and PurePosixPath(relative).suffix == ".json" + ) + if not document_paths: + raise ValueError("portable vector view has no JSON document store") + for relative in document_paths: + path = PurePosixPath(relative) + suffix = path.name.removeprefix("documents_").removesuffix(".json") + index_relative = (path.parent / f"index_{suffix}.faiss").as_posix() + if not suffix or index_relative not in inventory_paths: + raise ValueError( + f"portable vector documents have no matching FAISS index: {relative}" + ) + source_paths.update( + _document_source_paths( + root.joinpath(*path.parts), + label=f"portable vector documents {relative}", + max_bytes=inventory[relative][0], + ) + ) + return tuple(sorted(source_paths)) + + +def _validate_manifest( + root: Path, + metadata: Mapping[str, Any], + *, + commit: str, + source_fingerprint: str, + views: tuple[str, ...], + inventoried_paths: set[str], +) -> tuple[Path, RepoManifest]: + manifest_record = metadata.get("manifest") + if not isinstance(manifest_record, dict): + raise ValueError("context artifact manifest descriptor must be an object") + if manifest_record.get("repository_path") != "source": + raise ValueError("context artifact manifest repository path must be 'source'") + if manifest_record.get("paths") != "artifact-relative-posix": + raise ValueError("context artifact manifest path contract is unsupported") + if manifest_record.get("path") != MANIFEST_FILENAME: + raise ValueError("context artifact manifest filename is unsupported") + manifest_path = _artifact_path( + root, + manifest_record.get("path"), + label="context artifact manifest path", + ) + manifest_relative = manifest_path.relative_to(root).as_posix() + if manifest_relative not in inventoried_paths: + raise ValueError("context artifact manifest is absent from its inventory") + try: + manifest = RepoManifest.load(manifest_path) + except (KeyError, OSError, TypeError, ValueError) as exc: + raise ValueError("context artifact repository manifest is invalid") from exc + if manifest.version != MANIFEST_VERSION: + raise ValueError( + "context artifact repository manifest version is incompatible: " + f"expected {MANIFEST_VERSION}, found {manifest.version}" + ) + if manifest.repo_path != "source": + raise ValueError("portable repository manifest path must be 'source'") + if ( + not isinstance(manifest.file_count, int) + or isinstance(manifest.file_count, bool) + or manifest.file_count < 0 + ): + raise ValueError("context artifact repository file count is invalid") + if manifest.commit != commit or manifest.source_fingerprint != source_fingerprint: + raise ValueError("context artifact and repository manifest identities differ") + if set(manifest.indexes) != set(views): + raise ValueError( + "context artifact view list differs from its repository manifest" + ) + + metadata_capabilities = metadata.get("capabilities") + if ( + not isinstance(metadata_capabilities, dict) + or not all(isinstance(value, bool) for value in metadata_capabilities.values()) + or metadata_capabilities != manifest.capabilities + ): + raise ValueError("context artifact capability records differ") + metadata_languages = metadata.get("repository", {}).get("languages") + if ( + not isinstance(metadata_languages, list) + or not all(isinstance(language, str) for language in metadata_languages) + or metadata_languages != manifest.languages + ): + raise ValueError("context artifact language records differ") + source_locations = metadata.get("source_locations") + if not isinstance(source_locations, dict) or source_locations != { + "path": "repository-relative-posix", + "line_base": 1, + "end_line": "inclusive", + "commit": commit, + }: + raise ValueError("context artifact source-location contract is unsupported") + builder = metadata.get("builder") + if ( + not isinstance(builder, dict) + or builder.get("manifest_version") != MANIFEST_VERSION + or not isinstance(builder.get("codenib_version"), str) + or not isinstance(builder.get("compiled_at"), str) + ): + raise ValueError("context artifact builder identity is invalid") + + for view in views: + entry = manifest.indexes[view] + if not isinstance(entry.config, dict) or not isinstance(entry.metadata, dict): + raise ValueError(f"context artifact view metadata is invalid: {view}") + if not manifest.index_is_current(view): + raise ValueError(f"context artifact view is not current: {view}") + if entry.index_type != view: + raise ValueError(f"context artifact view type differs from its key: {view}") + expected_view_path = f"views/{view}" + if entry.path != expected_view_path: + raise ValueError( + f"context artifact {view} view path must be {expected_view_path!r}" + ) + view_path = _artifact_path( + root, + entry.path, + label=f"context artifact {view} view path", + ) + if not view_path.is_dir(): + raise ValueError(f"context artifact view is missing: {view}") + view_relative = view_path.relative_to(root).as_posix() + if not any( + relative == view_relative or relative.startswith(f"{view_relative}/") + for relative in inventoried_paths + ): + raise ValueError(f"context artifact view has no inventoried files: {view}") + if view == "vector" and entry.config.get("portable_document_format") != ( + "codenib.vector-documents.v1" + ): + raise ValueError("portable vector document format is unsupported") + return manifest_path, manifest + + +def verify_context_artifact( + root: str | Path, + *, + expected_repository: str | None = None, + expected_commit: str | None = None, + max_files: int = _DEFAULT_MAX_FILES, + max_bytes: int = _DEFAULT_MAX_BYTES, +) -> VerifiedContextArtifact: + """Verify an artifact completely without opening any persisted index.""" + + candidate = Path(root).expanduser() + if candidate.is_symlink(): + raise ValueError( + f"context artifact root must not be a symbolic link: {candidate}" + ) + resolved = candidate.resolve() + if not resolved.is_dir(): + raise ValueError(f"context artifact directory does not exist: {resolved}") + metadata_path = resolved / CONTEXT_ARTIFACT_MANIFEST + if metadata_path.is_symlink() or not metadata_path.is_file(): + raise ValueError(f"context artifact metadata is missing: {metadata_path}") + metadata = _load_json_object( + metadata_path, + label="context artifact metadata", + max_bytes=_MAX_METADATA_BYTES, + ) + if metadata.get("schema") != CONTEXT_ARTIFACT_SCHEMA: + raise ValueError( + "context artifact schema is incompatible: " + f"expected {CONTEXT_ARTIFACT_SCHEMA}, found {metadata.get('schema')!r}" + ) + repository, commit, source_fingerprint = _repository_identity(metadata) + views = _artifact_views(metadata) + inventory, byte_count = _inventory( + metadata, + max_files=max_files, + max_bytes=max_bytes, + ) + + expected_files = set(inventory) | {CONTEXT_ARTIFACT_MANIFEST} + actual_files = _actual_files(resolved, max_files=max_files) + if actual_files != expected_files: + missing = sorted(expected_files - actual_files) + extra = sorted(actual_files - expected_files) + raise ValueError( + "context artifact file set differs from its inventory: " + f"missing={missing}, extra={extra}" + ) + for relative, (expected_size, expected_digest) in inventory.items(): + path = _artifact_path( + resolved, + relative, + label=f"context artifact inventory path {relative!r}", + ) + size, digest = file_sha256(path) + if size != expected_size or digest != expected_digest: + raise ValueError(f"context artifact file digest mismatch: {relative}") + + manifest_path, manifest = _validate_manifest( + resolved, + metadata, + commit=commit, + source_fingerprint=source_fingerprint, + views=views, + inventoried_paths=set(inventory), + ) + source_paths = _validate_view_payloads( + resolved, + inventory, + views=views, + ) + if expected_repository is not None: + expected = normalize_repo(expected_repository) + if repository != expected: + raise ValueError( + "context artifact repository mismatch: " + f"expected {expected}, found {repository}" + ) + if expected_commit is not None: + expected = expected_commit.strip().lower() + if not _COMMIT_RE.fullmatch(expected): + raise ValueError("expected context artifact commit must be a full Git SHA") + if commit != expected: + raise ValueError( + "context artifact commit mismatch: " + f"expected {expected[:12]}, found {commit[:12]}" + ) + + return VerifiedContextArtifact( + root=resolved, + metadata_path=metadata_path, + manifest_path=manifest_path, + metadata=metadata, + manifest=manifest, + repository=repository, + commit=commit, + source_fingerprint=source_fingerprint, + views=views, + source_paths=source_paths, + file_count=len(inventory), + byte_count=byte_count, + ) + + +def bind_context_artifact( + root: str | Path, + repo_path: str | Path, + *, + expected_repository: str | None = None, + expected_commit: str | None = None, +) -> ContextArtifactBinding: + """Verify an artifact and bind it to an exact, unchanged checkout.""" + + artifact = verify_context_artifact( + root, + expected_repository=expected_repository, + expected_commit=expected_commit, + ) + repo = Path(repo_path).expanduser().resolve() + if not repo.is_dir(): + raise ValueError(f"repository checkout does not exist: {repo}") + actual_commit = checkout_commit(repo) + if actual_commit != artifact.commit: + actual_label = actual_commit[:12] if actual_commit else "not-a-git-checkout" + raise ValueError( + "repository checkout commit does not match the context artifact: " + f"checkout={actual_label}, artifact={artifact.commit[:12]}" + ) + source = fingerprint_repository( + repo, + exclude_roots=(artifact.root,), + ) + if source.value != artifact.source_fingerprint: + raise ValueError( + "repository source files do not match the context artifact fingerprint" + ) + if source.file_count != artifact.manifest.file_count: + raise ValueError( + "repository file count does not match the context artifact manifest" + ) + for relative in artifact.source_paths: + candidate = repo.joinpath(*PurePosixPath(relative).parts) + resolved_source = candidate.resolve() + if not resolved_source.is_file() or ( + resolved_source != repo and repo not in resolved_source.parents + ): + raise ValueError( + "context artifact source path does not resolve to a file inside " + f"the repository checkout: {relative}" + ) + + manifest_data = artifact.manifest.to_dict() + manifest_data["repo"]["path"] = str(repo) + for view, entry in manifest_data["indexes"].items(): + entry["path"] = str( + _artifact_path( + artifact.root, + entry["path"], + label=f"context artifact {view} view path", + ) + ) + manifest = RepoManifest.from_dict(manifest_data) + return ContextArtifactBinding( + artifact=artifact, + repo_path=repo, + manifest=manifest, + ) + + +__all__ = [ + "ContextArtifactBinding", + "VerifiedContextArtifact", + "bind_context_artifact", + "verify_context_artifact", +] diff --git a/codenib/cli.py b/codenib/cli.py index 2cb5c4b2..82c1ce20 100644 --- a/codenib/cli.py +++ b/codenib/cli.py @@ -439,10 +439,24 @@ def _run_index(args: argparse.Namespace) -> int: def _run_mcp(args: argparse.Namespace) -> int: _require_modules(("mcp",), extra="mcp", feature="the MCP server") - manifest_path = resolve_manifest_path(args.path) from .mcp.server import main as mcp_main - mcp_main([str(manifest_path), "--log-level", args.log_level]) + if args.artifact: + repo_path = resolve_repo_path(args.repo) + command = [ + "--artifact", + str(Path(args.artifact).expanduser().resolve()), + "--repo", + str(repo_path), + "--log-level", + args.log_level, + ] + if args.repository: + command.extend(("--repository", args.repository)) + mcp_main(command) + else: + manifest_path = resolve_manifest_path(args.path) + mcp_main([str(manifest_path), "--log-level", args.log_level]) return 0 @@ -525,6 +539,104 @@ def _run_artifact_pack(args: argparse.Namespace) -> int: return 0 +def _artifact_checkout_commit(repo_path: Path, explicit: str | None) -> str: + from .compiler.checkout_identity import checkout_commit + + commit = (explicit or checkout_commit(repo_path) or "").strip().lower() + if not re.fullmatch(r"[0-9a-f]{40}", commit): + raise CLIError( + "artifact operations require a Git checkout with a full resolved HEAD" + ) + return commit + + +def _run_artifact_verify(args: argparse.Namespace) -> int: + from .artifacts import bind_context_artifact, verify_context_artifact + + try: + if args.repo: + binding = bind_context_artifact( + args.path, + resolve_repo_path(args.repo), + expected_repository=args.repository, + expected_commit=args.commit, + ) + artifact = binding.artifact + checkout = binding.repo_path + else: + artifact = verify_context_artifact( + args.path, + expected_repository=args.repository, + expected_commit=args.commit, + ) + checkout = None + except (OSError, RuntimeError, ValueError) as exc: + raise CLIError(str(exc)) from exc + + print(f"Context artifact: {artifact.root}") + print(f"Repository: {artifact.repository}") + print(f"Commit: {artifact.commit}") + print(f"Views: {', '.join(artifact.views)}") + print(f"Files: {artifact.file_count}") + print(f"Bytes: {artifact.byte_count}") + print(f"Checkout: {checkout or 'not checked'}") + return 0 + + +def _run_artifact_fetch(args: argparse.Namespace) -> int: + from .artifacts import bind_context_artifact, fetch_github_context_artifact + + repo_path = resolve_repo_path(args.repo) + commit = _artifact_checkout_commit(repo_path, args.commit) + try: + result = fetch_github_context_artifact( + args.repository, + commit, + output_dir=args.output, + artifact_name=args.artifact_name, + token_env=args.token_env, + api_url=args.github_api_url, + force=args.force, + ) + binding = bind_context_artifact( + result.artifact.root, + repo_path, + expected_repository=args.repository, + expected_commit=commit, + ) + except (OSError, RuntimeError, ValueError) as exc: + raise CLIError(str(exc)) from exc + + state = "downloaded" if result.downloaded else "cached" + print(f"Context artifact: {binding.artifact.root}") + print(f"Repository: {binding.artifact.repository}") + print(f"Commit: {binding.artifact.commit}") + print(f"Views: {', '.join(binding.artifact.views)}") + print(f"State: {state}") + if result.record is not None: + print(f"GitHub artifact: {result.record.artifact_id}") + return 0 + + +def _run_artifact_mcp_config(args: argparse.Namespace) -> int: + from .artifacts import render_artifact_mcp_config + + try: + config = render_artifact_mcp_config( + args.path, + resolve_repo_path(args.repo), + host=args.host, + server_name=args.name, + command=args.command, + expected_repository=args.repository, + claude_scope=args.claude_scope, + ) + except (OSError, RuntimeError, ValueError) as exc: + raise CLIError(str(exc)) from exc + print(config, end="") + return 0 + + def _run_publish(args: argparse.Namespace) -> int: selected_views = _selected_views(args.preset, args.view) unsupported = sorted(set(selected_views) - {"bm25", "vector"}) @@ -1558,6 +1670,74 @@ def build_parser() -> argparse.ArgumentParser: ) artifact_pack_parser.set_defaults(handler=_run_artifact_pack) + artifact_verify_parser = artifact_subparsers.add_parser( + "verify", + help="verify artifact integrity and optionally bind an exact checkout", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + artifact_verify_parser.add_argument("path") + artifact_verify_parser.add_argument( + "--repo", + help="exact checkout used to verify commit and source identity", + ) + artifact_verify_parser.add_argument( + "--repository", + help="expected owner/repository identity", + ) + artifact_verify_parser.add_argument( + "--commit", + help="expected full Git commit", + ) + artifact_verify_parser.set_defaults(handler=_run_artifact_verify) + + artifact_fetch_parser = artifact_subparsers.add_parser( + "fetch", + help="download and verify a commit-matched GitHub Actions artifact", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + artifact_fetch_parser.add_argument("repository", help="GitHub owner/repository") + artifact_fetch_parser.add_argument( + "--repo", + default=".", + help="exact local checkout to bind", + ) + artifact_fetch_parser.add_argument("--commit", help="full commit; defaults to HEAD") + artifact_fetch_parser.add_argument("--output") + artifact_fetch_parser.add_argument("--artifact-name") + artifact_fetch_parser.add_argument( + "--token-env", + default="GH_TOKEN", + help="environment variable containing a GitHub token with Actions read", + ) + artifact_fetch_parser.add_argument( + "--github-api-url", + default=os.environ.get("GITHUB_API_URL", "https://api.github.com"), + ) + artifact_fetch_parser.add_argument("--force", action="store_true") + artifact_fetch_parser.set_defaults(handler=_run_artifact_fetch) + + artifact_config_parser = artifact_subparsers.add_parser( + "mcp-config", + help="render a Codex, Claude, or generic MCP configuration", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + artifact_config_parser.add_argument("path") + artifact_config_parser.add_argument("--repo", default=".") + artifact_config_parser.add_argument( + "--host", + choices=("claude", "codex", "json"), + required=True, + ) + artifact_config_parser.add_argument("--name", default="codenib") + artifact_config_parser.add_argument("--command", default="codenib") + artifact_config_parser.add_argument("--repository") + artifact_config_parser.add_argument( + "--claude-scope", + choices=("local", "project", "user"), + default="project", + ) + artifact_config_parser.set_defaults(handler=_run_artifact_mcp_config) + publish_parser = subparsers.add_parser( "publish", help="incrementally build and publish a static Wiki plus context artifact", @@ -1601,6 +1781,19 @@ def build_parser() -> argparse.ArgumentParser: default=".", help="repository directory or repo_manifest.json", ) + mcp_parser.add_argument( + "--artifact", + help="verified portable context artifact directory", + ) + mcp_parser.add_argument( + "--repo", + default=".", + help="exact checkout bound to --artifact", + ) + mcp_parser.add_argument( + "--repository", + help="expected owner/repository identity for --artifact", + ) mcp_parser.add_argument( "--log-level", choices=("DEBUG", "INFO", "WARNING", "ERROR"), diff --git a/codenib/mcp/context.py b/codenib/mcp/context.py index 9626ecdd..8a2a39b1 100644 --- a/codenib/mcp/context.py +++ b/codenib/mcp/context.py @@ -17,7 +17,7 @@ from dataclasses import dataclass, field from pathlib import Path from threading import RLock -from typing import TYPE_CHECKING, Dict, Optional +from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional from ..compiler.manifest import RepoManifest from ..provider_routes import resolve_embedding_artifact_route @@ -122,6 +122,7 @@ class ServerContext: zoekt: Optional[ZoektSearcher] = None vector: Optional[CodeVectorStore] = None errors: Dict[str, str] = field(default_factory=dict) + artifact: Optional[Mapping[str, Any]] = None _view_lock: RLock = field(default_factory=RLock, init=False, repr=False) @classmethod @@ -130,6 +131,7 @@ def load( manifest_path: RepoManifest | str | Path, *, views: Iterable[str] | None = None, + artifact: Mapping[str, Any] | None = None, ) -> ServerContext: """Load a manifest and the selected runtime views. @@ -144,7 +146,7 @@ def load( if isinstance(manifest_path, RepoManifest) else RepoManifest.load(manifest_path) ) - ctx = cls(manifest=manifest) + ctx = cls(manifest=manifest, artifact=dict(artifact) if artifact else None) ctx.load_views(selected) @@ -283,6 +285,8 @@ def _load_bm25(self) -> None: indexer = BM25CodeIndexer() indexer.load_index(entry.path) + if indexer.project_root == "source": + indexer.project_root = self.manifest.repo_path self.bm25 = indexer logger.info("Loaded BM25 index from %s", entry.path) except Exception as exc: diff --git a/codenib/mcp/server.py b/codenib/mcp/server.py index c7706407..6566327a 100644 --- a/codenib/mcp/server.py +++ b/codenib/mcp/server.py @@ -27,6 +27,7 @@ from mcp.server import MCPServer +from ..compiler.manifest import RepoManifest from .context import ServerContext from .prompts import CODENIB_GUIDE from .tools.dependency import dependency_subgraph_impl @@ -303,7 +304,14 @@ async def get_manifest() -> dict[str, Any]: """Return the repo manifest as a dict.""" if _ctx is None: raise RuntimeError("Server not initialized") - return _ctx.manifest.to_dict() + result = _ctx.manifest.to_dict() + result["runtime"] = { + "loaded_views": sorted(_ctx.loaded_views), + "view_errors": dict(sorted(_ctx.errors.items())), + } + if _ctx.artifact is not None: + result["artifact"] = dict(_ctx.artifact) + return result # ------------------------------------------------------------------ @@ -401,6 +409,21 @@ def _parse_args( type=str, help="Path to repo_manifest.json produced by IndexCompiler.", ) + parser.add_argument( + "--artifact", + type=str, + help="Verified portable context artifact directory.", + ) + parser.add_argument( + "--repo", + type=str, + help="Exact repository checkout bound to --artifact.", + ) + parser.add_argument( + "--repository", + type=str, + help="Expected owner/repository identity for --artifact.", + ) parser.add_argument( "--log-level", type=str, @@ -411,10 +434,14 @@ def _parse_args( return parser.parse_args(argv) -def init_server(manifest_path: str | Path) -> None: +def init_server( + manifest_path: RepoManifest | str | Path, + *, + artifact: dict[str, Any] | None = None, +) -> None: """Initialize the global ServerContext from a manifest file. - Loads the manifest and hydrates all available indexes into the + Loads the manifest and opens all available indexes in the module-level ``_ctx``. Safe to call from tests with a temporary manifest path. @@ -422,11 +449,22 @@ def init_server(manifest_path: str | Path) -> None: FileNotFoundError: if ``manifest_path`` does not exist. """ global _ctx - resolved = Path(manifest_path).resolve() - if not resolved.exists(): - raise FileNotFoundError(f"Manifest not found: {resolved}") - logger.info("Loading manifest from %s", resolved) - _ctx = ServerContext.load(resolved) + if isinstance(manifest_path, RepoManifest): + manifest = manifest_path + logger.info( + "Loading in-memory manifest for %s@%s", + manifest.repo_path, + (manifest.commit or "")[:12], + ) + else: + resolved = Path(manifest_path).resolve() + if not resolved.exists(): + raise FileNotFoundError(f"Manifest not found: {resolved}") + logger.info("Loading manifest from %s", resolved) + manifest = RepoManifest.load(resolved) + if _ctx is not None: + _ctx.close() + _ctx = ServerContext.load(manifest, artifact=artifact) def main(argv: list[str] | None = None) -> None: @@ -434,9 +472,15 @@ def main(argv: list[str] | None = None) -> None: program_name = _cli_program_name() args = _parse_args(argv) manifest_path = args.manifest_flag or args.manifest - if not manifest_path: + if args.artifact and manifest_path: + logger.error("Choose either a manifest or --artifact, not both") + sys.exit(1) + if args.artifact and not args.repo: + logger.error("--artifact requires --repo with the exact checkout") + sys.exit(1) + if not args.artifact and not manifest_path: logger.error( - "No manifest provided. Use: %s or --manifest ", + "No context provided. Use: %s or --artifact --repo ", program_name, ) sys.exit(1) @@ -448,7 +492,27 @@ def main(argv: list[str] | None = None) -> None: ) try: - init_server(manifest_path) + if args.artifact: + from ..artifacts import bind_context_artifact + + binding = bind_context_artifact( + args.artifact, + args.repo, + expected_repository=args.repository, + ) + artifact = binding.artifact + init_server( + binding.manifest, + artifact={ + "verified": True, + "schema": artifact.metadata["schema"], + "repository": artifact.repository, + "commit": artifact.commit, + "views": list(artifact.views), + }, + ) + else: + init_server(manifest_path) logger.info("Starting MCP server on stdio...") mcp.run(transport="stdio") except FileNotFoundError as exc: diff --git a/docs/github_pages.md b/docs/github_pages.md index 5585d24d..9bf3db0e 100644 --- a/docs/github_pages.md +++ b/docs/github_pages.md @@ -147,3 +147,60 @@ store owns deployment: Its outputs include `site-path`, `context-path`, `context-manifest`, `artifact-name`, `cache-hit`, `cache-key`, and `source-commit`. + +## Reuse the Artifact Through MCP + +The uploaded context artifact can serve an exact local checkout without +rebuilding its BM25 or vector views. Check out the commit first, then fetch the +artifact with a token that has **Actions: read** permission: + +```bash +git -C /path/to/repository checkout +export GH_TOKEN=github_pat_... + +codenib artifact fetch owner/repository \ + --repo /path/to/repository \ + --commit +``` + +CodeNib resolves the newest non-expired artifact whose workflow +`head_sha` exactly matches the commit. It checks GitHub's archive digest, +extracts with file-count, expanded-size, symlink, and traversal limits, verifies +every inventoried file, and compares the local checkout's commit and source +fingerprint before an index loader runs. The downloaded artifact uses JSON for +portable vector documents; CodeNib never loads a pickle from this path. + +These checks establish artifact integrity and source compatibility, not trust in +an arbitrary workflow publisher. Fetch only artifacts produced by a workflow +and pinned CodeNib revision that you trust. Semantic artifacts also contain a +provider and endpoint identity; review that identity before exposing model +credentials to the MCP process. + +The command prints the verified cache directory. Start MCP directly: + +```bash +codenib mcp \ + --artifact ~/.codenib/artifacts/owner/repository/ \ + --repo /path/to/repository \ + --repository owner/repository +``` + +Or generate a reviewable client configuration command: + +```bash +codenib artifact mcp-config \ + ~/.codenib/artifacts/owner/repository/ \ + --repo /path/to/repository \ + --repository owner/repository \ + --host codex +``` + +`--host claude` emits the corresponding `claude mcp add-json` command; +`--host json` emits a project-scoped `.mcp.json` document. Review the output +before running or placing it. CodeNib does not edit a client configuration +automatically. + +BM25 serving requires no model credential. A semantic artifact reuses its +stored vectors but still needs the manifest-selected embedding provider for +each query embedding. Provider credentials stay in the MCP process environment +and are never copied into client configuration or the context artifact. diff --git a/docs/mcp.md b/docs/mcp.md index bbbaf07b..48eb5376 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -51,6 +51,26 @@ The command also accepts the manifest path directly: codenib mcp ~/.codenib/repositories/-/indexes/repo_manifest.json ``` +### Load a Published Artifact + +A Pages publishing run can produce the same query-serving views once and reuse +them in MCP at the indexed commit: + +```bash +codenib artifact fetch owner/repository --repo /path/to/repository +codenib artifact mcp-config \ + ~/.codenib/artifacts/owner/repository/ \ + --repo /path/to/repository \ + --host codex +``` + +`artifact fetch` derives the full commit from the checkout unless `--commit` is +provided. It requires `GH_TOKEN` with Actions read access. The MCP process +rechecks artifact hashes, repository identity, commit, and the filtered source +fingerprint on every start; it does not rebuild or silently substitute a stale +view. See [Publish With GitHub Pages](github_pages.md#reuse-the-artifact-through-mcp) +for the trust boundary and Claude/generic configuration options. + Transport is stdio and logs go to stderr. A typical client configuration is: ```json diff --git a/test/artifacts/test_runtime.py b/test/artifacts/test_runtime.py new file mode 100644 index 00000000..dc605345 --- /dev/null +++ b/test/artifacts/test_runtime.py @@ -0,0 +1,891 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import asyncio +import hashlib +import io +import json +import math +import shlex +import shutil +import stat +import subprocess +import urllib.error +import urllib.request +import zipfile +from pathlib import Path +from unittest.mock import patch + +import pytest + +import codenib.artifacts.github as github_artifacts +import codenib.mcp.server as server_module +from codenib.artifacts import ( + CONTEXT_ARTIFACT_MANIFEST, + bind_context_artifact, + extract_context_artifact_archive, + fetch_github_context_artifact, + render_artifact_mcp_config, + resolve_github_context_artifact, + stage_context_artifact, + verify_context_artifact, +) +from codenib.cli import run +from codenib.compiler.index_builders import VectorIndexBuilder +from codenib.compiler.manifest import IndexEntry, RepoManifest +from codenib.index.embedding.vector_store import CodeVectorStore +from codenib.mcp.context import ServerContext +from codenib.source_fingerprint import fingerprint_repository + + +def _git(repo: Path, *args: str) -> str: + return subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def _bm25_artifact( + tmp_path: Path, + *, + source_symlink: bool = False, +) -> tuple[Path, Path, str]: + repo = tmp_path / "repo" + repo.mkdir() + source = repo / "sample.py" + if source_symlink: + outside = tmp_path / "outside.py" + outside.write_text("SECRET = 'outside checkout'\n", encoding="utf-8") + source.symlink_to(outside) + else: + source.write_text("VALUE = 1\n", encoding="utf-8") + _git(repo, "init", "--quiet") + _git(repo, "config", "user.name", "CodeNib Test") + _git(repo, "config", "user.email", "codenib@example.invalid") + _git(repo, "add", "sample.py") + _git(repo, "commit", "--quiet", "-m", "fixture") + commit = _git(repo, "rev-parse", "HEAD") + + index_root = tmp_path / "indexes" + view = index_root / "bm25" + view.mkdir(parents=True) + (view / "documents.json").write_text( + json.dumps( + [ + { + "page_content": "sample py value 1", + "metadata": { + "file": "sample.py", + "name": "VALUE", + "node_id": "sample.py:VALUE", + "type": "variable", + "start_line": 0, + "end_line": 0, + }, + } + ] + ), + encoding="utf-8", + ) + (view / "bm25_metadata.json").write_text( + json.dumps( + { + "project_root": str(repo), + "max_k": 15, + "language": "english", + } + ), + encoding="utf-8", + ) + source_identity = fingerprint_repository(repo) + manifest_path = index_root / "repo_manifest.json" + RepoManifest( + repo_path=str(repo), + commit=commit, + last_indexed_commit=commit, + source_fingerprint=source_identity.value, + last_indexed_source_fingerprint=source_identity.value, + languages=["python"], + file_count=source_identity.file_count, + indexes={ + "bm25": IndexEntry( + index_type="bm25", + path=str(view), + built_at="2026-08-04T00:00:00+00:00", + built_at_epoch=1.0, + status="fresh", + commit=commit, + source_fingerprint=source_identity.value, + ) + }, + capabilities={ + "sparse_search": True, + "dense_search": False, + "hybrid_search": False, + "symbol_navigation": False, + }, + compiled_at="2026-08-04T00:00:00+00:00", + compiled_at_epoch=1.0, + ).save(manifest_path) + artifact = tmp_path / "artifact" + stage_context_artifact( + repo, + manifest_path, + artifact, + repository="example/project", + ) + return repo, artifact, commit + + +class _PortableEmbedding: + def embed_query(self, text: str) -> list[float]: + values = [ + float(value + 1) for value in hashlib.sha256(text.encode()).digest()[:4] + ] + norm = math.sqrt(sum(value * value for value in values)) + return [value / norm for value in values] + + def embed_documents(self, texts: list[str]) -> list[list[float]]: + return [self.embed_query(text) for text in texts] + + +def _vector_artifact(tmp_path: Path) -> tuple[Path, Path, str]: + repo = tmp_path / "semantic-repo" + repo.mkdir() + (repo / "search.py").write_text( + "def locate_symbol(query):\n return query.casefold()\n", + encoding="utf-8", + ) + _git(repo, "init", "--quiet") + _git(repo, "config", "user.name", "CodeNib Test") + _git(repo, "config", "user.email", "codenib@example.invalid") + _git(repo, "add", "search.py") + _git(repo, "commit", "--quiet", "-m", "fixture") + commit = _git(repo, "rev-parse", "HEAD") + config = VectorIndexBuilder( + languages=["python"], + embedding_model="test/model", + embedding_provider="huggingface", + embedding_dimension=4, + build_levels=["l2"], + ).artifact_identity() + index_root = tmp_path / "semantic-indexes" + vector = index_root / "vector" + with patch.object( + CodeVectorStore, + "_initialize_embedding_model", + return_value=_PortableEmbedding(), + ): + store = CodeVectorStore( + embedding_model="test/model", + embedding_provider="huggingface", + dimension=4, + index_metric="ip", + store_path=str(vector), + artifact_metadata=config, + ) + store.add_code_chunks( + [ + { + "content": "def locate_symbol(query): return query.casefold()", + "chunk_type": "function", + "name": "locate_symbol", + "file": str(repo / "search.py"), + "start_line": 0, + "end_line": 1, + } + ], + level="l2", + ) + store.save() + + source_identity = fingerprint_repository(repo) + manifest_path = index_root / "repo_manifest.json" + RepoManifest( + repo_path=str(repo), + commit=commit, + last_indexed_commit=commit, + source_fingerprint=source_identity.value, + last_indexed_source_fingerprint=source_identity.value, + languages=["python"], + file_count=source_identity.file_count, + indexes={ + "vector": IndexEntry( + index_type="vector", + path=str(vector), + built_at="2026-08-04T00:00:00+00:00", + built_at_epoch=1.0, + status="fresh", + config=dict(config), + metadata=dict(config), + commit=commit, + source_fingerprint=source_identity.value, + ) + }, + capabilities={ + "sparse_search": False, + "dense_search": True, + "hybrid_search": False, + "symbol_navigation": False, + }, + compiled_at="2026-08-04T00:00:00+00:00", + compiled_at_epoch=1.0, + ).save(manifest_path) + artifact = tmp_path / "semantic-artifact" + stage_context_artifact( + repo, + manifest_path, + artifact, + repository="example/semantic-project", + ) + return repo, artifact, commit + + +def _metadata(artifact: Path) -> dict: + return json.loads((artifact / CONTEXT_ARTIFACT_MANIFEST).read_text()) + + +def _write_metadata(artifact: Path, metadata: dict) -> None: + (artifact / CONTEXT_ARTIFACT_MANIFEST).write_text( + json.dumps(metadata, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def _refresh_inventory_file(artifact: Path, metadata: dict, relative: str) -> None: + path = artifact / relative + record = next(item for item in metadata["files"] if item["path"] == relative) + record["bytes"] = path.stat().st_size + record["sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + + +def _zip_tree(source: Path, output: Path, *, prefix: str = "") -> None: + with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for path in sorted(source.rglob("*")): + if path.is_file(): + relative = path.relative_to(source).as_posix() + archive.write(path, f"{prefix}{relative}") + + +def test_verify_and_bind_artifact_before_loading_bm25(tmp_path: Path) -> None: + repo, artifact, commit = _bm25_artifact(tmp_path) + + verified = verify_context_artifact( + artifact, + expected_repository="example/project", + expected_commit=commit, + ) + binding = bind_context_artifact( + artifact, + repo, + expected_repository="example/project", + expected_commit=commit, + ) + + assert verified.views == ("bm25",) + assert binding.manifest.repo_path == str(repo) + assert binding.manifest.indexes["bm25"].path == str(artifact / "views" / "bm25") + context = ServerContext.load(binding.manifest, views=["bm25"]) + assert context.bm25 is not None + assert context.bm25.project_root == str(repo) + results = context.bm25.search("value", return_code_content=True) + assert results[0].file == "sample.py" + assert "VALUE = 1" in (results[0].content or "") + + +def test_bind_and_query_portable_semantic_artifact_without_pickle( + tmp_path: Path, +) -> None: + repo, artifact, commit = _vector_artifact(tmp_path) + assert not list(artifact.rglob("*.pkl")) + binding = bind_context_artifact( + artifact, + repo, + expected_repository="example/semantic-project", + expected_commit=commit, + ) + + with patch.object( + CodeVectorStore, + "_initialize_embedding_model", + return_value=_PortableEmbedding(), + ): + context = ServerContext.load(binding.manifest, views=["vector"]) + assert context.vector is not None + results = context.vector.search("locate a symbol", top_k=1) + + assert results[0].node_name == "locate_symbol" + assert results[0].file == "search.py" + + +def test_verify_rejects_digest_mismatch(tmp_path: Path) -> None: + _repo, artifact, _commit = _bm25_artifact(tmp_path) + (artifact / "views" / "bm25" / "documents.json").write_text("[]\n") + + with pytest.raises(ValueError, match="digest mismatch"): + verify_context_artifact(artifact) + + +def test_verify_rejects_extra_file_and_symlink(tmp_path: Path) -> None: + _repo, artifact, _commit = _bm25_artifact(tmp_path) + (artifact / "extra.txt").write_text("not inventoried") + with pytest.raises(ValueError, match="file set differs"): + verify_context_artifact(artifact) + + (artifact / "extra.txt").unlink() + (artifact / "link").symlink_to(artifact / "repo_manifest.json") + with pytest.raises(ValueError, match="symbolic link"): + verify_context_artifact(artifact) + + +def test_verify_rejects_pickle_even_when_inventoried(tmp_path: Path) -> None: + _repo, artifact, _commit = _bm25_artifact(tmp_path) + payload = b"not executable, but the format is forbidden" + (artifact / "views" / "bm25" / "documents.pkl").write_bytes(payload) + metadata = _metadata(artifact) + metadata["files"].append( + { + "path": "views/bm25/documents.pkl", + "bytes": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + ) + _write_metadata(artifact, metadata) + + with pytest.raises(ValueError, match="must not contain pickle"): + verify_context_artifact(artifact) + + +def test_verify_rejects_inventory_path_escape(tmp_path: Path) -> None: + _repo, artifact, _commit = _bm25_artifact(tmp_path) + metadata = _metadata(artifact) + metadata["files"][0]["path"] = "../outside" + _write_metadata(artifact, metadata) + + with pytest.raises(ValueError, match="normalized relative POSIX path"): + verify_context_artifact(artifact) + + +def test_verify_rejects_repository_and_commit_mismatch(tmp_path: Path) -> None: + _repo, artifact, commit = _bm25_artifact(tmp_path) + with pytest.raises(ValueError, match="repository mismatch"): + verify_context_artifact( + artifact, + expected_repository="different/project", + ) + with pytest.raises(ValueError, match="commit mismatch"): + verify_context_artifact( + artifact, + expected_commit="f" * 40 if commit != "f" * 40 else "e" * 40, + ) + + +def test_verify_rejects_unsafe_bm25_source_contract(tmp_path: Path) -> None: + _repo, artifact, _commit = _bm25_artifact(tmp_path) + metadata_path = artifact / "views" / "bm25" / "bm25_metadata.json" + metadata_path.write_text('{"project_root": "/tmp"}\n', encoding="utf-8") + metadata = _metadata(artifact) + _refresh_inventory_file( + artifact, + metadata, + "views/bm25/bm25_metadata.json", + ) + _write_metadata(artifact, metadata) + with pytest.raises(ValueError, match="project root"): + verify_context_artifact(artifact) + + metadata_path.write_text('{"project_root": "source"}\n', encoding="utf-8") + documents_path = artifact / "views" / "bm25" / "documents.json" + documents = json.loads(documents_path.read_text()) + documents[0]["metadata"]["file"] = "../../outside.py" + documents_path.write_text(json.dumps(documents), encoding="utf-8") + metadata = _metadata(artifact) + _refresh_inventory_file( + artifact, + metadata, + "views/bm25/bm25_metadata.json", + ) + _refresh_inventory_file( + artifact, + metadata, + "views/bm25/documents.json", + ) + _write_metadata(artifact, metadata) + with pytest.raises(ValueError, match="repository-relative POSIX path"): + verify_context_artifact(artifact) + + +def test_bind_rejects_source_symlink_outside_checkout(tmp_path: Path) -> None: + repo, artifact, _commit = _bm25_artifact(tmp_path, source_symlink=True) + + with pytest.raises(ValueError, match="inside the repository checkout"): + bind_context_artifact(artifact, repo) + + +def test_bind_rejects_checkout_commit_drift(tmp_path: Path) -> None: + repo, artifact, _commit = _bm25_artifact(tmp_path) + (repo / "second.py").write_text("SECOND = 2\n", encoding="utf-8") + _git(repo, "add", "second.py") + _git(repo, "commit", "--quiet", "-m", "second") + + with pytest.raises(ValueError, match="checkout commit does not match"): + bind_context_artifact(artifact, repo) + + +def test_bind_rejects_checkout_source_drift(tmp_path: Path) -> None: + repo, artifact, _commit = _bm25_artifact(tmp_path) + (repo / "sample.py").write_text("VALUE = 2\n", encoding="utf-8") + + with pytest.raises(ValueError, match="source files do not match"): + bind_context_artifact(artifact, repo) + + +def test_extract_archive_verifies_and_removes_single_root_prefix( + tmp_path: Path, +) -> None: + _repo, artifact, commit = _bm25_artifact(tmp_path) + archive = tmp_path / "artifact.zip" + _zip_tree(artifact, archive, prefix="download/") + + verified = extract_context_artifact_archive( + archive, + tmp_path / "extracted", + expected_repository="example/project", + expected_commit=commit, + ) + + assert verified.commit == commit + assert verified.metadata_path == tmp_path / "extracted" / ( + CONTEXT_ARTIFACT_MANIFEST + ) + + +def test_extract_archive_rejects_traversal_and_symlink(tmp_path: Path) -> None: + traversal = tmp_path / "traversal.zip" + with zipfile.ZipFile(traversal, "w") as archive: + archive.writestr(CONTEXT_ARTIFACT_MANIFEST, "{}") + archive.writestr("../outside", "bad") + with pytest.raises(ValueError, match="path is unsafe"): + extract_context_artifact_archive(traversal, tmp_path / "traversal-output") + + linked = tmp_path / "linked.zip" + with zipfile.ZipFile(linked, "w") as archive: + archive.writestr(CONTEXT_ARTIFACT_MANIFEST, "{}") + info = zipfile.ZipInfo("link") + info.create_system = 3 + info.external_attr = (stat.S_IFLNK | 0o777) << 16 + archive.writestr(info, "repo_manifest.json") + with pytest.raises(ValueError, match="symbolic link"): + extract_context_artifact_archive(linked, tmp_path / "linked-output") + + +def test_extract_archive_enforces_expanded_size_before_writing(tmp_path: Path) -> None: + archive = tmp_path / "large.zip" + with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as bundle: + bundle.writestr(CONTEXT_ARTIFACT_MANIFEST, "{}") + bundle.writestr("large.bin", "x" * 1024) + + with pytest.raises(ValueError, match="expanded bytes"): + extract_context_artifact_archive( + archive, + tmp_path / "large-output", + max_bytes=128, + ) + assert not (tmp_path / "large-output").exists() + + +def test_extract_archive_limits_directory_entries(tmp_path: Path) -> None: + archive = tmp_path / "directories.zip" + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr(CONTEXT_ARTIFACT_MANIFEST, "{}") + for index in range(67): + bundle.writestr(f"directory-{index}/", "") + + with pytest.raises(ValueError, match="archive exceeds .* entries"): + extract_context_artifact_archive( + archive, + tmp_path / "directory-output", + max_files=1, + ) + + +def _github_record( + *, + artifact_id: int, + commit: str, + digest: str, + expired: bool = False, +) -> dict: + return { + "id": artifact_id, + "name": f"codenib-context-example-project-{commit[:12]}", + "size_in_bytes": 1024, + "archive_download_url": ( + "https://api.github.com/repos/example/project/actions/artifacts/" + f"{artifact_id}/zip" + ), + "digest": digest, + "expired": expired, + "created_at": f"2026-08-0{artifact_id}T00:00:00Z", + "workflow_run": {"head_sha": commit}, + } + + +def test_resolve_github_artifact_uses_exact_head_sha_and_newest_id( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _repo, _artifact, commit = _bm25_artifact(tmp_path) + digest = f"sha256:{'a' * 64}" + observed: dict = {} + + def fake_json(url: str, *, token: str | None) -> dict: + observed.update(url=url, token=token) + return { + "artifacts": [ + _github_record(artifact_id=1, commit="f" * 40, digest=digest), + _github_record(artifact_id=2, commit=commit, digest=digest), + _github_record(artifact_id=3, commit=commit, digest=digest), + _github_record( + artifact_id=4, + commit=commit, + digest=digest, + expired=True, + ), + ] + } + + monkeypatch.setattr(github_artifacts, "_github_json", fake_json) + record = resolve_github_context_artifact( + "Example/Project", + commit, + token="runtime-token", + ) + + assert record.artifact_id == 3 + assert record.head_sha == commit + assert observed["token"] == "runtime-token" + assert "name=codenib-context-example-project-" in observed["url"] + + +def test_fetch_github_artifact_verifies_archive_digest_and_reuses_cache( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _repo, artifact, commit = _bm25_artifact(tmp_path) + archive = tmp_path / "source.zip" + _zip_tree(artifact, archive) + archive_bytes = archive.read_bytes() + archive_digest = hashlib.sha256(archive_bytes).hexdigest() + record = _github_record( + artifact_id=7, + commit=commit, + digest=f"sha256:{archive_digest}", + ) + record["size_in_bytes"] = len(archive_bytes) + calls = {"list": 0, "download": 0} + + def fake_json(_url: str, *, token: str | None) -> dict: + assert token == "runtime-token" + calls["list"] += 1 + return {"artifacts": [record]} + + def fake_download( + _url: str, + output: Path, + *, + token: str, + max_bytes: int, + ) -> tuple[int, str]: + assert token == "runtime-token" + assert len(archive_bytes) < max_bytes + calls["download"] += 1 + shutil.copyfile(archive, output) + return len(archive_bytes), archive_digest + + monkeypatch.setattr(github_artifacts, "_github_json", fake_json) + monkeypatch.setattr(github_artifacts, "_download_archive", fake_download) + output = tmp_path / "cache" / commit + first = fetch_github_context_artifact( + "example/project", + commit, + output_dir=output, + token="runtime-token", + ) + second = fetch_github_context_artifact( + "example/project", + commit, + output_dir=output, + ) + + assert first.downloaded is True + assert first.record is not None and first.record.artifact_id == 7 + assert second.downloaded is False + assert second.record is None + assert calls == {"list": 1, "download": 1} + with pytest.raises(ValueError, match="inventory exceeds 2 files"): + fetch_github_context_artifact( + "example/project", + commit, + output_dir=output, + max_files=2, + ) + + +def test_fetch_github_artifact_rejects_archive_digest_mismatch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _repo, artifact, commit = _bm25_artifact(tmp_path) + archive = tmp_path / "source.zip" + _zip_tree(artifact, archive) + record = _github_record( + artifact_id=8, + commit=commit, + digest=f"sha256:{'0' * 64}", + ) + + monkeypatch.setattr( + github_artifacts, + "_github_json", + lambda _url, *, token: {"artifacts": [record]}, + ) + + def fake_download( + _url: str, + output: Path, + *, + token: str, + max_bytes: int, + ) -> tuple[int, str]: + shutil.copyfile(archive, output) + return output.stat().st_size, "f" * 64 + + monkeypatch.setattr(github_artifacts, "_download_archive", fake_download) + output = tmp_path / "cache" / commit + with pytest.raises(ValueError, match="archive digest"): + fetch_github_context_artifact( + "example/project", + commit, + output_dir=output, + token="runtime-token", + ) + assert not output.exists() + + +def test_github_download_does_not_forward_token_to_redirect( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + payload = b"zip-payload" + requests: list[urllib.request.Request] = [] + + class RedirectingOpener: + def open(self, request: urllib.request.Request, timeout: int): + requests.append(request) + headers = {"Location": "https://objects.example/artifact.zip"} + raise urllib.error.HTTPError( + request.full_url, + 302, + "Found", + headers, + None, + ) + + class ObjectResponse(io.BytesIO): + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + def fake_urlopen(request: urllib.request.Request, timeout: int): + requests.append(request) + return ObjectResponse(payload) + + monkeypatch.setattr( + urllib.request, + "build_opener", + lambda *_handlers: RedirectingOpener(), + ) + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + output = tmp_path / "artifact.zip" + size, digest = github_artifacts._download_archive( + "https://api.github.com/repos/example/project/actions/artifacts/1/zip", + output, + token="never-forward-this-token", + max_bytes=1024, + ) + + assert size == len(payload) + assert digest == hashlib.sha256(payload).hexdigest() + assert requests[0].get_header("Authorization") == ( + "Bearer never-forward-this-token" + ) + assert requests[1].get_header("Authorization") is None + + +def test_github_api_refuses_redirect_with_token( + monkeypatch: pytest.MonkeyPatch, +) -> None: + requests: list[urllib.request.Request] = [] + + class RedirectingOpener: + def open(self, request: urllib.request.Request, timeout: int): + requests.append(request) + raise urllib.error.HTTPError( + request.full_url, + 302, + "Found", + {"Location": "https://attacker.example/artifacts"}, + None, + ) + + monkeypatch.setattr( + urllib.request, + "build_opener", + lambda *_handlers: RedirectingOpener(), + ) + + with pytest.raises(RuntimeError, match="HTTP 302"): + github_artifacts._github_json( + "https://api.github.com/repos/example/project/actions/artifacts", + token="never-forward-this-token", + ) + + assert len(requests) == 1 + assert requests[0].get_header("Authorization") == ( + "Bearer never-forward-this-token" + ) + + +def test_artifact_cli_verifies_binding_and_renders_configs( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + repo, artifact, commit = _bm25_artifact(tmp_path) + + assert ( + run( + [ + "artifact", + "verify", + str(artifact), + "--repo", + str(repo), + "--repository", + "example/project", + "--commit", + commit, + ] + ) + == 0 + ) + output = capsys.readouterr().out + assert "Repository: example/project" in output + assert f"Commit: {commit}" in output + assert f"Checkout: {repo}" in output + + assert ( + run( + [ + "artifact", + "mcp-config", + str(artifact), + "--repo", + str(repo), + "--repository", + "example/project", + "--host", + "json", + ] + ) + == 0 + ) + config = json.loads(capsys.readouterr().out) + args = config["mcpServers"]["codenib"]["args"] + assert args[:2] == ["mcp", "--artifact"] + assert args[-2:] == ["--repository", "example/project"] + + +def test_render_artifact_mcp_host_commands_are_shell_safe(tmp_path: Path) -> None: + repo, artifact, _commit = _bm25_artifact(tmp_path) + codex = shlex.split( + render_artifact_mcp_config( + artifact, + repo, + host="codex", + server_name="repository-context", + ) + ) + claude = shlex.split( + render_artifact_mcp_config( + artifact, + repo, + host="claude", + server_name="repository-context", + ) + ) + + assert codex[:6] == [ + "codex", + "mcp", + "add", + "repository-context", + "--", + "codenib", + ] + assert claude[:7] == [ + "claude", + "mcp", + "add-json", + "--scope", + "project", + "repository-context", + json.dumps( + { + "type": "stdio", + "command": "codenib", + "args": codex[6:], + }, + ensure_ascii=True, + separators=(",", ":"), + ), + ] + + +def test_mcp_server_starts_from_verified_artifact(tmp_path: Path) -> None: + repo, artifact, commit = _bm25_artifact(tmp_path) + + with patch.object(server_module.mcp, "run") as run_server: + server_module.main( + [ + "--artifact", + str(artifact), + "--repo", + str(repo), + "--repository", + "example/project", + "--log-level", + "ERROR", + ] + ) + + run_server.assert_called_once_with(transport="stdio") + context = server_module.get_context() + assert context.bm25 is not None + manifest = asyncio.run(server_module.get_manifest()) + assert manifest["repo"]["commit"] == commit + assert manifest["artifact"] == { + "verified": True, + "schema": "codenib.context-artifact.v1", + "repository": "example/project", + "commit": commit, + "views": ["bm25"], + } + assert manifest["runtime"]["loaded_views"] == ["bm25"]