diff --git a/.github/actions/publish/action.yml b/.github/actions/publish/action.yml new file mode 100644 index 00000000..4fb9d101 --- /dev/null +++ b/.github/actions/publish/action.yml @@ -0,0 +1,333 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +name: Publish CodeNib repository context +description: Build a static Wiki and a portable, commit-addressed context artifact. + +inputs: + repository-path: + description: Repository checkout to index. + default: "." + repository: + description: Stable owner/repository identity. + default: "" + preset: + description: Portable CodeNib view preset (fast or semantic). + default: fast + python-version: + description: Python runtime used to build the artifact. + default: "3.12" + embedding-provider: + description: Embedding provider; semantic defaults to local Hugging Face. + default: "" + embedding-model: + description: Embedding model id. + default: "" + embedding-dimension: + description: Embedding vector width for a non-default model. + default: "" + embedding-endpoint: + description: BYO OpenAI-compatible embedding API base. + default: "" + embedding-api-key-env: + description: Name of an environment variable containing the embedding key. + default: "" + base-path: + description: URL path where the static Wiki is mounted. + default: "/" + site-output: + description: Static site output directory; defaults under RUNNER_TEMP. + default: "" + context-output: + description: Context artifact output directory; defaults under RUNNER_TEMP. + default: "" + artifact-name: + description: Uploaded context artifact name. + default: "" + upload-context: + description: Upload the context artifact with actions/upload-artifact. + default: "true" + retention-days: + description: Context artifact retention in days. + default: "14" + cache: + description: Restore and save incremental CodeNib repository state. + default: "true" + revision: + description: Optional CodeNib revision included in cache compatibility. + default: "" + +outputs: + site-path: + description: Absolute static site path. + value: ${{ steps.inputs.outputs.site_path }} + context-path: + description: Absolute portable context artifact path. + value: ${{ steps.inputs.outputs.context_path }} + context-manifest: + description: Portable context metadata path. + value: ${{ steps.inputs.outputs.context_manifest }} + artifact-name: + description: Uploaded context artifact name. + value: ${{ steps.inputs.outputs.artifact_name }} + cache-hit: + description: Whether the exact commit cache key was restored. + value: ${{ steps.restore.outputs.cache-hit }} + cache-key: + description: Immutable cache key for this build. + value: ${{ steps.inputs.outputs.cache_key }} + source-commit: + description: Exact commit indexed from repository-path. + value: ${{ steps.inputs.outputs.source_commit }} + +runs: + using: composite + steps: + - name: Resolve public build identity + id: inputs + shell: bash + env: + ACTION_REF: ${{ github.action_ref }} + INPUT_ARTIFACT_NAME: ${{ inputs.artifact-name }} + INPUT_BASE_PATH: ${{ inputs.base-path }} + INPUT_CACHE: ${{ inputs.cache }} + INPUT_CONTEXT_OUTPUT: ${{ inputs.context-output }} + INPUT_EMBEDDING_DIMENSION: ${{ inputs.embedding-dimension }} + INPUT_EMBEDDING_ENDPOINT: ${{ inputs.embedding-endpoint }} + INPUT_EMBEDDING_MODEL: ${{ inputs.embedding-model }} + INPUT_EMBEDDING_PROVIDER: ${{ inputs.embedding-provider }} + INPUT_PRESET: ${{ inputs.preset }} + INPUT_PYTHON_VERSION: ${{ inputs.python-version }} + INPUT_REPOSITORY: ${{ inputs.repository }} + INPUT_REPOSITORY_PATH: ${{ inputs.repository-path }} + INPUT_REVISION: ${{ inputs.revision }} + INPUT_SITE_OUTPUT: ${{ inputs.site-output }} + INPUT_RETENTION_DAYS: ${{ inputs.retention-days }} + INPUT_UPLOAD_CONTEXT: ${{ inputs.upload-context }} + run: | + set -euo pipefail + + require_single_line() { + local name="$1" + local value="${!name}" + if [[ "$value" == *$'\n'* || "$value" == *$'\r'* ]]; then + echo "$name must not contain a line break" >&2 + exit 2 + fi + } + for name in \ + INPUT_ARTIFACT_NAME INPUT_BASE_PATH INPUT_CACHE INPUT_CONTEXT_OUTPUT \ + INPUT_EMBEDDING_DIMENSION INPUT_EMBEDDING_ENDPOINT \ + INPUT_EMBEDDING_MODEL INPUT_EMBEDDING_PROVIDER INPUT_PRESET \ + INPUT_PYTHON_VERSION INPUT_REPOSITORY INPUT_REPOSITORY_PATH \ + INPUT_RETENTION_DAYS INPUT_REVISION INPUT_SITE_OUTPUT \ + INPUT_UPLOAD_CONTEXT; do + require_single_line "$name" + done + + case "$INPUT_PRESET" in + fast|semantic) ;; + *) echo "unsupported CodeNib preset: $INPUT_PRESET" >&2; exit 2 ;; + esac + case "$INPUT_CACHE:$INPUT_UPLOAD_CONTEXT" in + true:true|true:false|false:true|false:false) ;; + *) echo "cache and upload-context must be true or false" >&2; exit 2 ;; + esac + if [[ ! "$INPUT_RETENTION_DAYS" =~ ^[0-9]+$ ]] || + (( INPUT_RETENTION_DAYS < 1 || INPUT_RETENTION_DAYS > 90 )); then + echo "retention-days must be an integer from 1 through 90" >&2 + exit 2 + fi + + embedding_provider="$INPUT_EMBEDDING_PROVIDER" + if [[ "$INPUT_PRESET" == "semantic" && -z "$embedding_provider" ]]; then + embedding_provider="huggingface" + fi + case "$embedding_provider" in + ""|huggingface|openai) ;; + *) echo "unsupported embedding provider: $embedding_provider" >&2; exit 2 ;; + esac + + repo_path="$(realpath -e "$INPUT_REPOSITORY_PATH" 2>/dev/null || true)" + if [[ -z "$repo_path" || ! -d "$repo_path" ]]; then + echo "repository-path must resolve to a directory" >&2 + exit 2 + fi + source_commit="$(git -C "$repo_path" rev-parse --verify HEAD 2>/dev/null || true)" + if [[ ! "$source_commit" =~ ^[0-9a-f]{40}$ ]]; then + echo "repository-path must be a Git checkout with a resolved HEAD" >&2 + exit 2 + fi + + case "$INPUT_PRESET" in + fast) extras="" ;; + semantic) + if [[ "$embedding_provider" == "huggingface" ]]; then + extras="semantic" + else + extras="semantic-remote" + fi + ;; + esac + + site_path="${INPUT_SITE_OUTPUT:-$RUNNER_TEMP/codenib-site}" + context_path="${INPUT_CONTEXT_OUTPUT:-$RUNNER_TEMP/codenib-context}" + site_path="$(realpath -m "$site_path")" + context_path="$(realpath -m "$context_path")" + if [[ "$site_path" == "$context_path" || + "$site_path" == "$context_path"/* || + "$context_path" == "$site_path"/* ]]; then + echo "site-output and context-output must not overlap" >&2 + exit 2 + fi + source_path="$(realpath -e "$GITHUB_ACTION_PATH/../../..")" + frontend_path="$source_path/web/dist" + repository="${INPUT_REPOSITORY:-$GITHUB_REPOSITORY}" + if [[ ! "$repository" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then + echo "repository must use owner/name form" >&2 + exit 2 + fi + repository_key="${GITHUB_REPOSITORY_ID:-$repository}" + revision="${INPUT_REVISION:-${ACTION_REF:-source}}" + source_hash="$({ + sha256sum "$GITHUB_ACTION_PATH/action.yml" + sha256sum "$GITHUB_ACTION_PATH/../../../pyproject.toml" + } | sha256sum | cut -d' ' -f1)" + identity="$({ + printf '%s\0' "$revision" "$source_hash" "$INPUT_PYTHON_VERSION" + printf '%s\0' "$INPUT_PRESET" "$embedding_provider" + printf '%s\0' "$INPUT_EMBEDDING_MODEL" "$INPUT_EMBEDDING_DIMENSION" + printf '%s\0' "$INPUT_EMBEDDING_ENDPOINT" + } | sha256sum | cut -d' ' -f1)" + cache_prefix="codenib-${RUNNER_OS}-${repository_key}-${identity}" + cache_key="${cache_prefix}-${source_commit}" + safe_repository="$({ + printf '%s' "$repository" | tr '[:upper:]' '[:lower:]' | \ + tr '/:@ ' '----' | tr -cd 'a-z0-9_.-' + })" + safe_repository="${safe_repository:-repository}" + artifact_name="${INPUT_ARTIFACT_NAME:-codenib-context-${safe_repository}-${source_commit:0:12}}" + if [[ ! "$artifact_name" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$ ]]; then + echo "artifact-name must be 1-128 portable filename characters" >&2 + exit 2 + fi + + { + echo "artifact_name=$artifact_name" + echo "cache_key=$cache_key" + echo "cache_prefix=$cache_prefix-" + echo "context_manifest=$context_path/codenib-context.json" + echo "context_path=$context_path" + echo "embedding_provider=$embedding_provider" + echo "extras=$extras" + echo "frontend_path=$frontend_path" + echo "repository=$repository" + echo "site_path=$site_path" + echo "source_path=$source_path" + echo "source_commit=$source_commit" + } >> "$GITHUB_OUTPUT" + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: ${{ inputs.python-version }} + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: "22" + cache: npm + cache-dependency-path: ${{ steps.inputs.outputs.source_path }}/web/package-lock.json + + - name: Restore incremental repository state + id: restore + if: ${{ inputs.cache == 'true' }} + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ~/.codenib/repositories + key: ${{ steps.inputs.outputs.cache_key }} + restore-keys: ${{ steps.inputs.outputs.cache_prefix }} + + - name: Build static frontend + shell: bash + env: + FRONTEND_OUTPUT: ${{ steps.inputs.outputs.frontend_path }} + SOURCE_PATH: ${{ steps.inputs.outputs.source_path }} + run: | + set -euo pipefail + npm ci --prefix "$SOURCE_PATH/web" + npm run build --prefix "$SOURCE_PATH/web" -- \ + --outDir "$FRONTEND_OUTPUT" + + - name: Install CodeNib + shell: bash + env: + EXTRAS: ${{ steps.inputs.outputs.extras }} + SOURCE_PATH: ${{ steps.inputs.outputs.source_path }} + run: | + set -euo pipefail + if [[ -n "$EXTRAS" ]]; then + python -m pip install "$SOURCE_PATH[$EXTRAS]" + else + python -m pip install "$SOURCE_PATH" + fi + + - name: Publish repository context + shell: bash + env: + CODENIB_ACTION_EMBEDDING_KEY_ENV: ${{ inputs.embedding-api-key-env }} + CONTEXT_OUTPUT: ${{ steps.inputs.outputs.context_path }} + EMBEDDING_DIMENSION: ${{ inputs.embedding-dimension }} + EMBEDDING_ENDPOINT: ${{ inputs.embedding-endpoint }} + EMBEDDING_MODEL: ${{ inputs.embedding-model }} + EMBEDDING_PROVIDER: ${{ steps.inputs.outputs.embedding_provider }} + FRONTEND_OUTPUT: ${{ steps.inputs.outputs.frontend_path }} + INPUT_BASE_PATH: ${{ inputs.base-path }} + INPUT_REPOSITORY_PATH: ${{ inputs.repository-path }} + PRESET: ${{ inputs.preset }} + REPOSITORY: ${{ steps.inputs.outputs.repository }} + SITE_OUTPUT: ${{ steps.inputs.outputs.site_path }} + run: | + set -euo pipefail + command=( + codenib publish "$INPUT_REPOSITORY_PATH" + --preset "$PRESET" + --site-output "$SITE_OUTPUT" + --context-output "$CONTEXT_OUTPUT" + --repository "$REPOSITORY" + --base-path "$INPUT_BASE_PATH" + --frontend-dir "$FRONTEND_OUTPUT" + ) + if [[ -n "$EMBEDDING_PROVIDER" ]]; then + command+=(--embedding-provider "$EMBEDDING_PROVIDER") + fi + if [[ -n "$EMBEDDING_MODEL" ]]; then + command+=(--embedding-model "$EMBEDDING_MODEL") + fi + if [[ -n "$EMBEDDING_DIMENSION" ]]; then + command+=(--embedding-dimension "$EMBEDDING_DIMENSION") + fi + if [[ -n "$EMBEDDING_ENDPOINT" ]]; then + command+=(--embedding-endpoint "$EMBEDDING_ENDPOINT") + fi + if [[ -n "$CODENIB_ACTION_EMBEDDING_KEY_ENV" ]]; then + command+=(--embedding-api-key-env "$CODENIB_ACTION_EMBEDDING_KEY_ENV") + fi + "${command[@]}" + + - name: Save incremental repository state + if: ${{ inputs.cache == 'true' && steps.restore.outputs.cache-hit != 'true' }} + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ~/.codenib/repositories + key: ${{ steps.inputs.outputs.cache_key }} + + - name: Upload portable context artifact + if: ${{ inputs.upload-context == 'true' }} + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5 + with: + name: ${{ steps.inputs.outputs.artifact_name }} + path: ${{ steps.inputs.outputs.context_path }} + if-no-files-found: error + retention-days: ${{ inputs.retention-days }} diff --git a/.github/workflows/codenib-pages.yml b/.github/workflows/codenib-pages.yml new file mode 100644 index 00000000..682774c2 --- /dev/null +++ b/.github/workflows/codenib-pages.yml @@ -0,0 +1,133 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +name: CodeNib Pages + +on: + workflow_call: + inputs: + preset: + description: CodeNib view preset. + type: string + default: fast + python-version: + description: Python runtime used for indexing. + type: string + default: "3.12" + embedding-provider: + description: Embedding provider; semantic defaults to local Hugging Face. + type: string + default: "" + embedding-model: + description: Embedding model id. + type: string + default: "" + embedding-dimension: + description: Embedding vector width. + type: string + default: "" + embedding-endpoint: + description: BYO OpenAI-compatible embedding API base. + type: string + default: "" + retention-days: + description: Portable context artifact retention. + type: number + default: 14 + secrets: + embedding_api_key: + description: Optional BYO embedding credential. + required: false + outputs: + page-url: + description: Deployed Pages URL. + value: ${{ jobs.deploy.outputs.page-url }} + context-artifact: + description: Downloadable context artifact name. + value: ${{ jobs.build.outputs.context-artifact }} + +concurrency: + group: codenib-pages-${{ github.repository_id }} + cancel-in-progress: false + +jobs: + build: + if: >- + github.event_name != 'pull_request_target' && + (github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository) + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + pages: write + outputs: + context-artifact: ${{ steps.publish.outputs.artifact-name }} + site-path: ${{ steps.publish.outputs.site-path }} + steps: + - name: Checkout caller repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Checkout the exact CodeNib workflow revision + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} + path: .codenib-action + persist-credentials: false + + - name: Configure GitHub Pages + id: pages + uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6 + + - name: Build Wiki and context artifact + id: publish + uses: ./.codenib-action/.github/actions/publish + env: + CODENIB_ACTION_EMBEDDING_KEY: >- + ${{ inputs.preset == 'semantic' && + inputs.embedding-provider == 'openai' && + secrets.embedding_api_key || '' }} + with: + base-path: ${{ steps.pages.outputs.base_path }} + embedding-api-key-env: >- + ${{ inputs.preset == 'semantic' && + inputs.embedding-provider == 'openai' && + secrets.embedding_api_key && + 'CODENIB_ACTION_EMBEDDING_KEY' || '' }} + embedding-dimension: ${{ inputs.embedding-dimension }} + embedding-endpoint: ${{ inputs.embedding-endpoint }} + embedding-model: ${{ inputs.embedding-model }} + embedding-provider: ${{ inputs.embedding-provider }} + preset: ${{ inputs.preset }} + python-version: ${{ inputs.python-version }} + repository: ${{ github.repository }} + retention-days: ${{ inputs.retention-days }} + revision: ${{ job.workflow_sha }} + + - name: Upload GitHub Pages artifact + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5 + with: + path: ${{ steps.publish.outputs.site-path }} + + deploy: + if: ${{ needs.build.result == 'success' }} + needs: build + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + outputs: + page-url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy GitHub Pages + id: deployment + uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5 diff --git a/.github/workflows/codenib-publish-smoke.yml b/.github/workflows/codenib-publish-smoke.yml new file mode 100644 index 00000000..844804ba --- /dev/null +++ b/.github/workflows/codenib-publish-smoke.yml @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +name: CodeNib Publish Smoke + +on: + pull_request: + paths: + - ".github/actions/publish/**" + - ".github/workflows/codenib-pages.yml" + - ".github/workflows/codenib-publish-smoke.yml" + - "codenib/artifacts/**" + - "codenib/cli.py" + - "codenib/web/**" + - "pyproject.toml" + - "setup.py" + - "web/**" + push: + branches: [main] + paths: + - ".github/actions/publish/**" + - ".github/workflows/codenib-pages.yml" + - ".github/workflows/codenib-publish-smoke.yml" + - "codenib/artifacts/**" + - "codenib/cli.py" + - "codenib/web/**" + - "pyproject.toml" + - "setup.py" + - "web/**" + workflow_dispatch: + +permissions: + contents: read + +jobs: + no-model: + if: >- + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Prepare fixture repository + id: fixture + shell: bash + env: + FIXTURE_SOURCE: ${{ github.workspace }}/test/fixtures/publish_repo/sample.py + REPOSITORY_PATH: ${{ runner.temp }}/codenib-publish-repo + run: | + set -euo pipefail + mkdir -p "$REPOSITORY_PATH" + cp "$FIXTURE_SOURCE" "$REPOSITORY_PATH/sample.py" + git -C "$REPOSITORY_PATH" init --quiet + git -C "$REPOSITORY_PATH" config user.name "CodeNib Smoke" + git -C "$REPOSITORY_PATH" config user.email "codenib@example.invalid" + git -C "$REPOSITORY_PATH" add sample.py + git -C "$REPOSITORY_PATH" commit --quiet -m "fixture" + echo "path=$REPOSITORY_PATH" >> "$GITHUB_OUTPUT" + + - name: Exercise local publish Action + id: publish + uses: ./.github/actions/publish + env: + CODENIB_ACTION_TEST_TOKEN: never-publish-this-value + with: + base-path: /publish-smoke + cache: "false" + repository-path: ${{ steps.fixture.outputs.path }} + upload-context: "false" + + - name: Validate published identities + shell: bash + env: + CONTEXT_MANIFEST: ${{ steps.publish.outputs.context-manifest }} + SITE_PATH: ${{ steps.publish.outputs.site-path }} + run: | + set -euo pipefail + test -f "$SITE_PATH/index.html" + test -f "$SITE_PATH/codenib-static.json" + test -f "$CONTEXT_MANIFEST" + jq -e '.schema == "codenib.context-artifact.v1"' "$CONTEXT_MANIFEST" + jq -e '.views == ["bm25"]' "$CONTEXT_MANIFEST" + if grep -R -F "never-publish-this-value" "$SITE_PATH" "$(dirname "$CONTEXT_MANIFEST")"; then + echo "configured test token leaked into publication output" >&2 + exit 1 + fi diff --git a/README.md b/README.md index 89a52b57..407a86f0 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,12 @@ agent or code-Wiki framework. ## News +- **2026-08-04 — Commit-addressed Pages publishing.** + [`codenib publish`](docs/github_pages.md) and the reusable GitHub workflow + deploy a no-model static Wiki and retain a matching BM25 or opt-in + vector-enhanced context artifact for the indexed commit. Incremental caches + remain private build state rather than part of the downloadable serving + artifact. - **2026-08-04 — Native repository explorer.** [`RepositoryContextExplorer`](codenib/agent/runtime/explorer.py) plans BM25, dense, hybrid, reranked, and graph routes over manifest-backed views and @@ -129,6 +135,14 @@ source citations, and available page-level dependency data. It contains no provider credential; interactive Ask and runtime graph exploration remain on the local or MCP serving path. +For a repository-hosted Wiki, CodeNib also ships a reusable GitHub workflow +that incrementally builds the same manifest, deploys the static site to Pages, +and uploads the matching commit-addressed context artifact. Its default `fast` +route needs no model credential or model download. The optional `semantic` +route builds the matching vector artifact with a local Hugging Face model or an +explicit BYO OpenAI-compatible endpoint; query-time search remains in the local +or MCP runtime. See [GitHub Pages](https://docs.codenib.ai/github_pages/). + See the [Quickstart](https://docs.codenib.ai/quickstart/) for ports, advanced indexing, and troubleshooting. @@ -192,6 +206,7 @@ records chunking, graph, incremental, and C++ decoder support. ## Documentation - [Quickstart](https://docs.codenib.ai/quickstart/) +- [GitHub Pages](https://docs.codenib.ai/github_pages/) - [MCP Server](https://docs.codenib.ai/mcp/) - [Agent Integrations](https://docs.codenib.ai/agent_integrations/) - [Web UI](https://docs.codenib.ai/web_demo/) diff --git a/codenib/artifacts/__init__.py b/codenib/artifacts/__init__.py new file mode 100644 index 00000000..b06e05d8 --- /dev/null +++ b/codenib/artifacts/__init__.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +"""Portable repository-context artifacts.""" + +from .context import ( + CONTEXT_ARTIFACT_MANIFEST, + CONTEXT_ARTIFACT_SCHEMA, + PORTABLE_CONTEXT_VIEWS, + ContextArtifactResult, + stage_context_artifact, +) + +__all__ = [ + "CONTEXT_ARTIFACT_MANIFEST", + "CONTEXT_ARTIFACT_SCHEMA", + "PORTABLE_CONTEXT_VIEWS", + "ContextArtifactResult", + "stage_context_artifact", +] diff --git a/codenib/artifacts/context.py b/codenib/artifacts/context.py new file mode 100644 index 00000000..fbced665 --- /dev/null +++ b/codenib/artifacts/context.py @@ -0,0 +1,452 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +"""Stage a portable, commit-addressed repository-context artifact.""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import tempfile +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Iterable, Mapping, Sequence + +from .. import compat_pickle +from .._version import package_version +from ..compiler.artifact_fingerprints import bm25_artifact_file_fingerprints +from ..compiler.checkout_identity import validate_checkout_identity +from ..compiler.manifest import MANIFEST_FILENAME, RepoManifest +from ..compiler.snapshot_store import normalize_repo +from ..provider_routes import resolve_embedding_artifact_route +from .security import assert_no_credential_fields, assert_publishable_tree, file_sha256 + +CONTEXT_ARTIFACT_SCHEMA = "codenib.context-artifact.v1" +CONTEXT_ARTIFACT_MANIFEST = "codenib-context.json" +PORTABLE_CONTEXT_VIEWS = frozenset({"bm25", "vector"}) +_VIEW_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+$") + + +@dataclass(frozen=True, slots=True) +class ContextArtifactResult: + """Paths and identity of one staged context artifact.""" + + output_dir: Path + metadata_path: Path + manifest_path: Path + repository: str + commit: str + views: tuple[str, ...] + file_count: int + byte_count: int + + +def _json_bytes(value: Any) -> bytes: + return ( + json.dumps( + value, + ensure_ascii=True, + indent=2, + sort_keys=True, + separators=(",", ": "), + ) + + "\n" + ).encode("utf-8") + + +def _write_json(root: Path, relative: str, value: Any) -> Path: + target = root.joinpath(*PurePosixPath(relative).parts).resolve() + if root != target and root not in target.parents: + raise ValueError(f"artifact path escapes the output directory: {relative}") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(_json_bytes(value)) + return target + + +def _repository_slug(repo_path: Path, explicit: str | None) -> str: + if explicit: + return normalize_repo(explicit) + try: + result = subprocess.run( + ["git", "-C", str(repo_path), "config", "--get", "remote.origin.url"], + check=True, + capture_output=True, + text=True, + timeout=10, + ) + origin = result.stdout.strip() + except (OSError, subprocess.SubprocessError): + origin = "" + return normalize_repo(origin or repo_path.name) + + +def _validated_output(repo_path: Path, manifest_root: Path, output_dir: Path) -> Path: + output = output_dir.expanduser().resolve() + for source, label in ( + (repo_path.resolve(), "repository"), + (manifest_root.resolve(), "index root"), + ): + if output == source or source in output.parents or output in source.parents: + raise ValueError(f"context artifact output overlaps the {label}: {output}") + if output.exists() and not output.is_dir(): + raise ValueError(f"context artifact output is not a directory: {output}") + if output.exists() and any(output.iterdir()): + if not (output / CONTEXT_ARTIFACT_MANIFEST).is_file(): + raise ValueError( + "refusing to replace a non-empty directory that is not a CodeNib " + f"context artifact: {output}" + ) + return output + + +def _view_source(entry_path: str, manifest_root: Path, *, view: str) -> Path: + source = Path(entry_path).expanduser() + if not source.is_absolute(): + source = manifest_root / source + source = source.resolve() + if source != manifest_root and manifest_root not in source.parents: + raise ValueError(f"view {view!r} is outside the manifest index root: {source}") + if not source.exists(): + raise ValueError(f"view {view!r} is missing: {source}") + return source + + +def _copy_view(source: Path, stage: Path, view: str) -> str: + if not _VIEW_NAME_RE.fullmatch(view): + raise ValueError(f"invalid context artifact view name: {view!r}") + for candidate in (source, *source.rglob("*")) if source.is_dir() else (source,): + if candidate.is_symlink(): + raise ValueError(f"view {view!r} contains a symbolic link: {candidate}") + + destination = stage / "views" / view + if source.is_dir(): + shutil.copytree(source, destination) + return destination.relative_to(stage).as_posix() + destination.mkdir(parents=True) + target = destination / source.name + shutil.copy2(source, target) + return target.relative_to(stage).as_posix() + + +def _normalize_copied_view( + stage: Path, + *, + repo_path: Path, + view: str, + relative: str, +) -> dict[str, Any]: + """Rewrite view-local machine paths and return identity adjustments.""" + + target = stage.joinpath(*PurePosixPath(relative).parts) + if view == "vector": + return _normalize_vector_view(target, repo_path) + if view != "bm25": + raise ValueError( + f"view {view!r} is not yet supported by portable context artifacts; " + "select bm25 and/or vector" + ) + root = target if target.is_dir() else target.parent + metadata_path = root / "bm25_metadata.json" + if not metadata_path.is_file(): + raise ValueError("portable BM25 view is missing bm25_metadata.json") + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + if not isinstance(metadata, dict): + raise ValueError("portable BM25 metadata must be a JSON object") + metadata["project_root"] = "source" + metadata_path.write_bytes(_json_bytes(metadata)) + return {"artifact_file_fingerprints": bm25_artifact_file_fingerprints(root)} + + +def _portable_source_path(value: object, repo_path: Path, *, source: str) -> str: + raw = str(value or "") + if not raw: + return "" + path = Path(raw).expanduser() + if path.is_absolute(): + try: + path = path.resolve().relative_to(repo_path) + except ValueError as exc: + raise ValueError(f"{source} points outside the repository: {raw}") from exc + normalized = PurePosixPath(path.as_posix()) + if normalized.is_absolute() or ".." in normalized.parts: + raise ValueError(f"{source} is not repository-relative: {raw}") + return normalized.as_posix() + + +def _convert_vector_documents(path: Path, repo_path: Path) -> Path: + """Convert a trusted local document pickle to portable, inert JSON.""" + + with path.open("rb") as handle: + documents = compat_pickle.load(handle) + if not isinstance(documents, list): + raise ValueError(f"portable vector documents must be a list: {path.name}") + payload: list[dict[str, Any]] = [] + for index, document in enumerate(documents): + page_content = getattr(document, "page_content", None) + if not isinstance(page_content, str): + raise ValueError( + f"portable vector document {index} has invalid content: {path.name}" + ) + metadata = getattr(document, "metadata", None) + if not isinstance(metadata, dict): + raise ValueError( + f"portable vector document {index} has invalid metadata: {path.name}" + ) + normalized_metadata = dict(metadata) + normalized_metadata["file"] = _portable_source_path( + metadata.get("file"), + repo_path, + source=f"vector document {index} file", + ) + payload.append( + { + "page_content": page_content, + "metadata": normalized_metadata, + } + ) + + output = path.with_suffix(".json") + output.write_bytes(_json_bytes(payload)) + path.unlink() + return output + + +def _normalize_vector_view(target: Path, repo_path: Path) -> dict[str, Any]: + if not target.is_dir(): + raise ValueError("portable vector view must be a directory") + + # Query serving does not need the mutable state used to build the next + # commit. Excluding it keeps the downloadable artifact smaller and avoids + # publishing build-machine paths from incremental caches. + for name in ( + "chunk_store.json", + "chunk_store.pkl", + "embeddings_cache.json", + "embeddings_cache.npz", + "embeddings_cache.pkl", + "incremental_state.json", + ): + (target / name).unlink(missing_ok=True) + + document_files = sorted(target.glob("l[02]/documents_*.pkl")) + if not document_files: + raise ValueError( + "portable vector view requires the current documents_*.pkl format; " + "rebuild the vector view" + ) + for path in document_files: + _convert_vector_documents(path, repo_path) + + # The current document files supersede legacy LangChain docstore pickles. + # Leaving both formats would retain duplicate absolute source paths. + for legacy in target.glob("l[02]/index_*.pkl"): + legacy.unlink() + return { + "artifact_scope": "query-serving", + "portable_document_format": "codenib.vector-documents.v1", + } + + +def _inventory(root: Path) -> list[dict[str, Any]]: + files = [] + for path in sorted( + candidate for candidate in root.rglob("*") if candidate.is_file() + ): + size, digest = file_sha256(path) + files.append( + { + "path": path.relative_to(root).as_posix(), + "bytes": size, + "sha256": digest, + } + ) + return files + + +def _selected_views( + manifest: RepoManifest, + requested: Sequence[str] | None, +) -> tuple[str, ...]: + available = {name for name in manifest.indexes if manifest.index_is_current(name)} + if requested is None: + selected = sorted(available) + else: + selected = list(dict.fromkeys(str(name).strip() for name in requested)) + missing = sorted(name for name in selected if name not in available) + if missing: + raise ValueError( + "context artifact requires current views: " + ", ".join(missing) + ) + if not selected: + raise ValueError("context artifact requires at least one current view") + return tuple(selected) + + +def _portable_capabilities( + capabilities: Mapping[str, bool], + views: Iterable[str], +) -> dict[str, bool]: + selected = set(views) + result = dict(capabilities) + result.update( + { + "sparse_search": "bm25" in selected, + "dense_search": "vector" in selected, + "hybrid_search": {"bm25", "vector"} <= selected, + "symbol_navigation": "symbol_graph" in selected, + } + ) + return result + + +def stage_context_artifact( + repo_path: Path, + manifest_path: Path, + output_dir: Path, + *, + repository: str | None = None, + views: Sequence[str] | None = None, + environ: Mapping[str, str] | None = None, + validate_checkout: bool = True, +) -> ContextArtifactResult: + """Copy current views into an atomic, path-independent artifact directory.""" + + repo_path = repo_path.expanduser().resolve() + manifest_path = manifest_path.expanduser().resolve() + if not repo_path.is_dir(): + raise ValueError(f"repository directory does not exist: {repo_path}") + if not manifest_path.is_file(): + raise ValueError(f"repository manifest does not exist: {manifest_path}") + manifest_root = manifest_path.parent + output_dir = _validated_output(repo_path, manifest_root, output_dir) + environment = os.environ if environ is None else environ + manifest = RepoManifest.load(manifest_path) + if validate_checkout: + validate_checkout_identity( + repo_path, + manifest, + artifact_root=manifest_root, + ) + selected = _selected_views(manifest, views) + unsupported = sorted(set(selected) - PORTABLE_CONTEXT_VIEWS) + if unsupported: + raise ValueError( + "portable context artifacts do not yet support views: " + + ", ".join(unsupported) + ) + slug = _repository_slug(repo_path, repository) + + output_dir.parent.mkdir(parents=True, exist_ok=True) + stage = Path( + tempfile.mkdtemp( + prefix=f".{output_dir.name}.tmp-", + dir=str(output_dir.parent), + ) + ).resolve() + try: + portable = manifest.to_dict() + portable["repo"]["path"] = "source" + portable_indexes: dict[str, Any] = {} + for view in selected: + entry = manifest.indexes[view] + assert_no_credential_fields(entry.config, source=f"view {view!r} config") + assert_no_credential_fields( + entry.metadata, + source=f"view {view!r} metadata", + ) + if view == "vector": + resolve_embedding_artifact_route(entry.config) + source = _view_source(entry.path, manifest_root, view=view) + relative = _copy_view(source, stage, view) + adjustments = _normalize_copied_view( + stage, + repo_path=repo_path, + view=view, + relative=relative, + ) + entry_data = entry.to_dict() + entry_data["path"] = relative + for section in ("config", "metadata"): + entry_data[section].update(adjustments) + portable_indexes[view] = entry_data + portable["indexes"] = portable_indexes + portable["capabilities"] = _portable_capabilities( + manifest.capabilities, + selected, + ) + portable_manifest = _write_json(stage, MANIFEST_FILENAME, portable) + + assert_publishable_tree( + stage, + forbidden_paths=(repo_path, manifest_root), + environ=environment, + label="context artifact", + ) + files = _inventory(stage) + metadata = { + "schema": CONTEXT_ARTIFACT_SCHEMA, + "repository": { + "slug": slug, + "commit": manifest.commit, + "source_fingerprint": manifest.source_fingerprint, + "languages": list(manifest.languages), + }, + "builder": { + "codenib_version": package_version(), + "manifest_version": manifest.version, + "compiled_at": manifest.compiled_at, + }, + "manifest": { + "path": MANIFEST_FILENAME, + "repository_path": "source", + "paths": "artifact-relative-posix", + }, + "source_locations": { + "path": "repository-relative-posix", + "line_base": 1, + "end_line": "inclusive", + "commit": manifest.commit, + }, + "views": list(selected), + "capabilities": portable["capabilities"], + "files": files, + } + metadata_path = _write_json(stage, CONTEXT_ARTIFACT_MANIFEST, metadata) + assert_publishable_tree( + stage, + forbidden_paths=(repo_path, manifest_root), + environ=environment, + label="context artifact", + ) + + if output_dir.exists(): + shutil.rmtree(output_dir) + os.replace(stage, output_dir) + except BaseException: + shutil.rmtree(stage, ignore_errors=True) + raise + + byte_count = sum(int(item["bytes"]) for item in files) + return ContextArtifactResult( + output_dir=output_dir, + metadata_path=output_dir / metadata_path.relative_to(stage), + manifest_path=output_dir / portable_manifest.relative_to(stage), + repository=slug, + commit=manifest.commit, + views=selected, + file_count=len(files), + byte_count=byte_count, + ) + + +__all__ = [ + "CONTEXT_ARTIFACT_MANIFEST", + "CONTEXT_ARTIFACT_SCHEMA", + "PORTABLE_CONTEXT_VIEWS", + "ContextArtifactResult", + "stage_context_artifact", +] diff --git a/codenib/artifacts/security.py b/codenib/artifacts/security.py new file mode 100644 index 00000000..cd1dfb9f --- /dev/null +++ b/codenib/artifacts/security.py @@ -0,0 +1,173 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +"""Publication guards shared by static sites and context artifacts.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any, Iterable, Mapping + +_SENSITIVE_ENV_NAMES = { + "ANTHROPIC_API_KEY", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AZURE_API_KEY", + "CODENIB_DEMO_API_KEY", + "CODENIB_ACTION_EMBEDDING_KEY", + "CODENIB_EMBEDDING_API_KEY", + "GH_TOKEN", + "GITHUB_TOKEN", + "GOOGLE_API_KEY", + "OPENAI_API_KEY", +} +_SENSITIVE_ENV_SUFFIXES = ("_API_KEY", "_TOKEN", "_SECRET") +_SCAN_CHUNK_BYTES = 1024 * 1024 +_CREDENTIAL_FIELDS = { + "access_token", + "api_key", + "apikey", + "authorization", + "bearer_token", + "client_secret", + "headers", + "password", + "private_key", + "secret_key", + "token", +} + + +def assert_no_credential_fields(value: Any, *, source: str) -> None: + """Reject credential-shaped keys in metadata intended for publication.""" + + if isinstance(value, Mapping): + for key, item in value.items(): + name = str(key).strip().lower().replace("-", "_") + if name in _CREDENTIAL_FIELDS: + raise ValueError(f"{source} contains a credential field: {key}") + assert_no_credential_fields(item, source=source) + elif isinstance(value, (list, tuple)): + for item in value: + assert_no_credential_fields(item, source=source) + + +def _serialized_patterns(values: Iterable[str]) -> tuple[bytes, ...]: + patterns: set[bytes] = set() + for value in values: + if not value: + continue + patterns.add(value.encode("utf-8")) + escaped = json.dumps(value, ensure_ascii=True)[1:-1] + patterns.add(escaped.encode("utf-8")) + return tuple(sorted(patterns, key=lambda item: (-len(item), item))) + + +def _secret_values(environ: Mapping[str, str]) -> tuple[bytes, ...]: + values: set[str] = set() + for name, value in environ.items(): + upper = name.upper() + sensitive = upper in _SENSITIVE_ENV_NAMES or upper.endswith( + _SENSITIVE_ENV_SUFFIXES + ) + if sensitive and len(value) >= 8: + values.add(value) + return _serialized_patterns(values) + + +def file_sha256(path: Path) -> tuple[int, str]: + """Return byte length and SHA-256 without loading a full artifact file.""" + + digest = hashlib.sha256() + size = 0 + with path.open("rb") as handle: + while chunk := handle.read(_SCAN_CHUNK_BYTES): + size += len(chunk) + digest.update(chunk) + return size, digest.hexdigest() + + +def _matching_kind( + path: Path, + *, + forbidden: tuple[bytes, ...], + secrets: tuple[bytes, ...], +) -> str | None: + needles = tuple( + (kind, needle) + for kind, values in (("path", forbidden), ("secret", secrets)) + for needle in values + if needle + ) + if not needles: + return None + overlap = max(len(needle) for _kind, needle in needles) - 1 + tail = b"" + with path.open("rb") as handle: + while chunk := handle.read(_SCAN_CHUNK_BYTES): + window = tail + chunk + for kind, needle in needles: + if needle in window: + return kind + tail = window[-overlap:] if overlap else b"" + return None + + +def _assert_publishable_file( + path: Path, + *, + root: Path, + forbidden: tuple[bytes, ...], + secrets: tuple[bytes, ...], + label: str, +) -> None: + relative = path.relative_to(root) + if path.is_symlink(): + raise ValueError(f"{label} contains a symbolic link: {relative}") + if any(secret in relative.as_posix().encode("utf-8") for secret in secrets): + raise ValueError(f"{label} contains a configured credential in {relative}") + if not path.is_file(): + return + match = _matching_kind(path, forbidden=forbidden, secrets=secrets) + if match == "path": + raise ValueError( + f"{label} contains an absolute build-machine path in {relative}" + ) + if match == "secret": + raise ValueError(f"{label} contains a configured credential in {relative}") + + +def assert_publishable_tree( + root: Path, + *, + forbidden_paths: Iterable[Path], + environ: Mapping[str, str], + label: str, +) -> None: + """Reject links, build-machine paths, and configured secrets in a tree.""" + + resolved_root = root.expanduser().resolve() + forbidden_values: list[str] = [] + for path in forbidden_paths: + resolved = path.expanduser().resolve() + forbidden_values.extend((str(resolved), resolved.as_posix())) + forbidden = _serialized_patterns(forbidden_values) + secrets = _secret_values(environ) + for path in sorted(resolved_root.rglob("*")): + _assert_publishable_file( + path, + root=resolved_root, + forbidden=forbidden, + secrets=secrets, + label=label, + ) + + +__all__ = [ + "assert_no_credential_fields", + "assert_publishable_tree", + "file_sha256", +] diff --git a/codenib/cli.py b/codenib/cli.py index 90443515..2cb5c4b2 100644 --- a/codenib/cli.py +++ b/codenib/cli.py @@ -475,6 +475,121 @@ def _run_export(args: argparse.Namespace) -> int: return 0 +def _default_distribution_dir(manifest_path: Path, name: str, commit: str) -> Path: + identity = (commit or "working-tree")[:12] + return manifest_path.parent.parent / "exports" / f"{name}-{identity}" + + +def _publication_environment(credential_env: str | None = None) -> dict[str, str]: + environment = dict(os.environ) + selected = credential_env or environment.get("CODENIB_EMBEDDING_API_KEY_ENV") + if selected and environment.get(selected): + environment["CODENIB_PUBLICATION_CREDENTIAL_SECRET"] = environment[selected] + return environment + + +def _run_artifact_pack(args: argparse.Namespace) -> int: + repo_path = resolve_repo_path(args.repo) + manifest_path = resolve_manifest_path(str(repo_path)) + from .artifacts import stage_context_artifact + from .compiler.manifest import RepoManifest + + manifest = RepoManifest.load(manifest_path) + output_dir = ( + Path(args.output).expanduser().resolve() + if args.output + else _default_distribution_dir( + manifest_path, + "context", + manifest.commit, + ) + ) + selected_views = _split_values(args.view) or None + try: + result = stage_context_artifact( + repo_path, + manifest_path, + output_dir, + repository=args.repository or os.environ.get("GITHUB_REPOSITORY"), + views=selected_views, + environ=_publication_environment(), + ) + except (OSError, RuntimeError, ValueError) as exc: + raise CLIError(str(exc)) from exc + + print(f"Context artifact: {result.output_dir}") + print(f"Repository: {result.repository}") + print(f"Commit: {result.commit or 'working tree'}") + print(f"Views: {', '.join(result.views)}") + print(f"Manifest: {result.metadata_path}") + return 0 + + +def _run_publish(args: argparse.Namespace) -> int: + selected_views = _selected_views(args.preset, args.view) + unsupported = sorted(set(selected_views) - {"bm25", "vector"}) + if unsupported: + raise CLIError( + "portable publication currently supports bm25 and vector views; " + f"unsupported: {', '.join(unsupported)}" + ) + index_result = _run_index(args) + if index_result: + return index_result + + repo_path = resolve_repo_path(args.repo) + manifest_path = resolve_manifest_path(str(repo_path)) + from .artifacts import stage_context_artifact + from .compiler.manifest import RepoManifest + from .web.static_export import export_static_wiki + + manifest = RepoManifest.load(manifest_path) + site_output = ( + Path(args.site_output).expanduser().resolve() + if args.site_output + else _default_distribution_dir(manifest_path, "wiki", manifest.commit) + ) + context_output = ( + Path(args.context_output).expanduser().resolve() + if args.context_output + else _default_distribution_dir(manifest_path, "context", manifest.commit) + ) + if ( + site_output == context_output + or site_output in context_output.parents + or context_output in site_output.parents + ): + raise CLIError("Wiki and context artifact outputs must not overlap") + publication_environment = _publication_environment(args.embedding_api_key_env) + try: + site = export_static_wiki( + repo_path, + manifest_path, + site_output, + frontend_dir=args.frontend_dir, + base_path=args.base_path, + environ=publication_environment, + ) + context = stage_context_artifact( + repo_path, + manifest_path, + context_output, + repository=args.repository or os.environ.get("GITHUB_REPOSITORY"), + views=selected_views, + environ=publication_environment, + validate_checkout=False, + ) + except (OSError, RuntimeError, ValueError) as exc: + raise CLIError(str(exc)) from exc + + print(f"Published Wiki: {site.output_dir}") + print(f"Context artifact: {context.output_dir}") + print(f"Repository: {context.repository}") + print(f"Commit: {context.commit or 'working tree'}") + print(f"Views: {', '.join(context.views)}") + return 0 + + def _model_options_for_args( args: argparse.Namespace, *, @@ -1415,6 +1530,66 @@ def build_parser() -> argparse.ArgumentParser: ) export_parser.set_defaults(handler=_run_export) + artifact_parser = subparsers.add_parser( + "artifact", + help="package portable repository-context artifacts", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + artifact_subparsers = artifact_parser.add_subparsers( + dest="artifact_command", + required=True, + ) + artifact_pack_parser = artifact_subparsers.add_parser( + "pack", + help="stage current manifest views as a portable context artifact", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + artifact_pack_parser.add_argument("repo", nargs="?", default=".") + artifact_pack_parser.add_argument("--output") + artifact_pack_parser.add_argument( + "--repository", + help="stable owner/repository identity; defaults to origin or directory name", + ) + artifact_pack_parser.add_argument( + "--view", + action="append", + default=[], + help="current view to include; repeat or use a comma-separated list", + ) + artifact_pack_parser.set_defaults(handler=_run_artifact_pack) + + publish_parser = subparsers.add_parser( + "publish", + help="incrementally build and publish a static Wiki plus context artifact", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + publish_parser.add_argument("repo", nargs="?", default=".") + publish_parser.add_argument( + "--preset", + choices=("fast", "semantic"), + default="fast", + ) + publish_parser.add_argument("--language", action="append", default=[]) + publish_parser.add_argument("--view", action="append", default=[]) + publish_parser.add_argument("--rebuild", action="store_true") + _add_embedding_route_arguments(publish_parser) + publish_parser.add_argument("--site-output") + publish_parser.add_argument("--context-output") + publish_parser.add_argument( + "--repository", + help="stable owner/repository identity; defaults to origin or directory name", + ) + publish_parser.add_argument( + "--base-path", + default="/", + help="URL path where the static site will be mounted", + ) + publish_parser.add_argument( + "--frontend-dir", + help="path to a prebuilt CodeNib frontend or web source checkout", + ) + publish_parser.set_defaults(handler=_run_publish) + mcp_parser = subparsers.add_parser( "mcp", help="serve an indexed repository over MCP stdio", diff --git a/codenib/compiler/checkout_identity.py b/codenib/compiler/checkout_identity.py new file mode 100644 index 00000000..399f76f0 --- /dev/null +++ b/codenib/compiler/checkout_identity.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +"""Validate that a repository checkout matches a compiled manifest.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +from ..source_fingerprint import fingerprint_repository +from .manifest import RepoManifest + + +def checkout_commit(repo_path: Path) -> str | None: + """Return the checkout's current commit when the directory is in Git.""" + + result = subprocess.run( + ["git", "-C", str(repo_path), "rev-parse", "HEAD"], + capture_output=True, + check=False, + text=True, + ) + return result.stdout.strip() if result.returncode == 0 else None + + +def validate_checkout_identity( + repo_path: Path, + manifest: RepoManifest, + *, + artifact_root: Path, +) -> None: + """Reject source or commit drift between a checkout and its manifest.""" + + expected_source = (manifest.source_fingerprint or "").strip() + if expected_source: + actual_source = fingerprint_repository( + repo_path, + exclude_roots=(artifact_root,), + ).value + if actual_source != expected_source: + raise ValueError( + "repository source files do not match the indexed content. " + "Rebuild the index for the current working tree before serving " + "or publishing context." + ) + + expected = (manifest.commit or "").strip() + actual = checkout_commit(repo_path) + if not expected or not actual: + return + if actual.startswith(expected) or expected.startswith(actual): + return + raise ValueError( + "repository checkout does not match the indexed snapshot: " + f"HEAD is {actual[:12]}, manifest is {expected[:12]}. " + "Rebuild the index or check out the manifest commit before serving " + "or publishing context." + ) + + +__all__ = ["checkout_commit", "validate_checkout_identity"] diff --git a/codenib/index/embedding/vector_store.py b/codenib/index/embedding/vector_store.py index 3218760b..445e5781 100644 --- a/codenib/index/embedding/vector_store.py +++ b/codenib/index/embedding/vector_store.py @@ -1138,8 +1138,16 @@ def _load_level( f"expected {self.dimension}, found {int(index.d)}" ) - # Try loading documents pickle (works for both new _Document and - # legacy LangChain Document objects via duck-typing conversion). + # Portable artifacts use inert JSON so a downloaded document store is + # never unpickled. Local indexes retain the pickle fallback for + # compatibility with previously built artifacts. + json_path = level_path / f"documents_{model_suffix}.json" + if json_path.exists(): + documents = self._load_documents_json(json_path) + return index, documents + + # Try loading the local documents pickle (works for both new _Document + # and legacy LangChain Document objects via duck-typing conversion). docs_path = level_path / f"documents_{model_suffix}.pkl" if docs_path.exists(): try: @@ -1169,6 +1177,32 @@ def _load_level( logger.warning(f"No document store found for {level_path}") return index, [] + @staticmethod + def _load_documents_json(path: Path) -> List[_Document]: + """Load the non-executable portable vector document format.""" + + with path.open(encoding="utf-8") as handle: + payload = json.load(handle) + if not isinstance(payload, list): + raise ValueError(f"vector documents must be a JSON list: {path}") + + documents: List[_Document] = [] + for index, item in enumerate(payload): + if not isinstance(item, dict): + raise ValueError( + f"vector document {index} must be a JSON object: {path}" + ) + page_content = item.get("page_content") + metadata = item.get("metadata") + if not isinstance(page_content, str) or not isinstance(metadata, dict): + raise ValueError( + f"vector document {index} has invalid content or metadata: {path}" + ) + documents.append( + _Document(page_content=page_content, metadata=dict(metadata)) + ) + return documents + @staticmethod def _load_langchain_pkl(pkl_path: Path) -> List[_Document]: """Extract documents from a LangChain FAISS pkl file. diff --git a/codenib/web/local.py b/codenib/web/local.py index 3b05e457..b78efd02 100644 --- a/codenib/web/local.py +++ b/codenib/web/local.py @@ -16,10 +16,10 @@ import yaml +from ..compiler.checkout_identity import validate_checkout_identity from ..compiler.manifest import RepoManifest from ..compiler.snapshot_store import normalize_repo from ..llm.options import validate_model_options -from ..source_fingerprint import fingerprint_repository from .config import RepoEntry, save_registry @@ -48,49 +48,6 @@ def _origin_url(repo_path: Path) -> str | None: return result.stdout.strip() if result.returncode == 0 else None -def _checkout_commit(repo_path: Path) -> str | None: - result = subprocess.run( - ["git", "-C", str(repo_path), "rev-parse", "HEAD"], - capture_output=True, - check=False, - text=True, - ) - return result.stdout.strip() if result.returncode == 0 else None - - -def _validate_checkout_identity( - repo_path: Path, - manifest: RepoManifest, - *, - artifact_root: Path, -) -> None: - expected_source = (manifest.source_fingerprint or "").strip() - if expected_source: - actual_source = fingerprint_repository( - repo_path, - exclude_roots=(artifact_root,), - ).value - if actual_source != expected_source: - raise ValueError( - "repository source files do not match the indexed content. " - "Rebuild the index for the current working tree before starting " - "the Wiki." - ) - - expected = (manifest.commit or "").strip() - actual = _checkout_commit(repo_path) - if not expected or not actual: - return - if actual.startswith(expected) or expected.startswith(actual): - return - raise ValueError( - "repository checkout does not match the indexed snapshot: " - f"HEAD is {actual[:12]}, manifest is {expected[:12]}. " - "Rebuild the index or check out the manifest commit before starting " - "the Wiki." - ) - - def _repository_slug(repo_path: Path) -> str: origin = _origin_url(repo_path) if origin: @@ -119,7 +76,7 @@ def prepare_local_wiki( repo_path = repo_path.expanduser().resolve() manifest_path = manifest_path.expanduser().resolve() manifest = RepoManifest.load(str(manifest_path)) - _validate_checkout_identity( + validate_checkout_identity( repo_path, manifest, artifact_root=manifest_path.parent, diff --git a/codenib/web/static_export.py b/codenib/web/static_export.py index a220694c..ffeabdc6 100644 --- a/codenib/web/static_export.py +++ b/codenib/web/static_export.py @@ -6,7 +6,6 @@ from __future__ import annotations -import hashlib import json import os import re @@ -19,6 +18,7 @@ from urllib.parse import quote, unquote, urlsplit from .._version import package_version +from ..artifacts.security import assert_publishable_tree, file_sha256 from ..compiler.manifest import RepoManifest from .launcher import find_frontend_dir from .local import prepare_local_wiki @@ -26,17 +26,6 @@ STATIC_EXPORT_SCHEMA_VERSION = "1.0" STATIC_EXPORT_MANIFEST = "codenib-static.json" -_SENSITIVE_ENV_NAMES = { - "ANTHROPIC_API_KEY", - "AZURE_API_KEY", - "CODENIB_DEMO_API_KEY", - "GITHUB_TOKEN", - "GOOGLE_API_KEY", - "OPENAI_API_KEY", - "AWS_SECRET_ACCESS_KEY", - "AWS_SESSION_TOKEN", -} -_SENSITIVE_ENV_SUFFIXES = ("_API_KEY", "_TOKEN", "_SECRET") _DOCUMENT_BASE_RE = re.compile(r"]*href=(['\"])[^'\"]*\1[^>]*>", re.I) _DOCUMENT_REFERENCE_RE = re.compile( r"(?P\b(?:src|href)=)(?P['\"])(?P[^'\"]*)(?P=quote)", @@ -437,79 +426,33 @@ def _file_inventory(root: Path) -> list[dict[str, Any]]: for path in sorted(root.rglob("*")): if not path.is_file() or path.name == STATIC_EXPORT_MANIFEST: continue - data = path.read_bytes() + size, digest = file_sha256(path) files.append( { "path": path.relative_to(root).as_posix(), - "bytes": len(data), - "sha256": hashlib.sha256(data).hexdigest(), + "bytes": size, + "sha256": digest, } ) return files -def _secret_values(environ: Mapping[str, str]) -> list[bytes]: - values = [] - for name, value in environ.items(): - upper = name.upper() - sensitive = upper in _SENSITIVE_ENV_NAMES or upper.endswith( - _SENSITIVE_ENV_SUFFIXES - ) - if sensitive and len(value) >= 8: - values.append(value.encode("utf-8")) - return values - - -def _serialized_patterns(values: Iterable[str]) -> set[bytes]: - patterns: set[bytes] = set() - for value in values: - if not value: - continue - patterns.add(value.encode("utf-8")) - escaped = json.dumps(value, ensure_ascii=True)[1:-1] - patterns.add(escaped.encode("utf-8")) - return patterns - - -def _assert_publishable( - root: Path, - *, - forbidden_paths: Iterable[Path], - environ: Mapping[str, str], -) -> None: - forbidden_values: list[str] = [] - for path in forbidden_paths: - resolved = path.resolve() - forbidden_values.extend((str(resolved), resolved.as_posix())) - forbidden = _serialized_patterns(forbidden_values) - secrets = _serialized_patterns( - value.decode("utf-8") for value in _secret_values(environ) - ) - for path in root.rglob("*"): - if path.is_symlink(): - raise ValueError(f"static export contains a symbolic link: {path}") - if not path.is_file(): - continue - data = path.read_bytes() - if any(value and value in data for value in forbidden): - raise ValueError( - "static export contains an absolute build-machine path in " - f"{path.relative_to(root)}" - ) - if any(secret in data for secret in secrets): - raise ValueError( - "static export contains a configured credential in " - f"{path.relative_to(root)}" - ) - - -def _validated_output(repo_path: Path, output_dir: Path) -> Path: +def _validated_output( + repo_path: Path, + manifest_root: Path, + output_dir: Path, +) -> Path: repo_path = repo_path.resolve() + manifest_root = manifest_root.resolve() output_dir = output_dir.expanduser().resolve() - if output_dir == repo_path or repo_path in output_dir.parents: - raise ValueError("static export output must be outside the target repository") - if output_dir in repo_path.parents: - raise ValueError("static export output must not contain the target repository") + for source, label in ( + (repo_path, "target repository"), + (manifest_root, "index root"), + ): + if output_dir == source or source in output_dir.parents: + raise ValueError(f"static export output must be outside the {label}") + if output_dir in source.parents: + raise ValueError(f"static export output must not contain the {label}") if output_dir.exists() and not output_dir.is_dir(): raise ValueError(f"static export output is not a directory: {output_dir}") if output_dir.exists() and any(output_dir.iterdir()): @@ -534,7 +477,7 @@ def export_static_wiki( repo_path = repo_path.expanduser().resolve() manifest_path = manifest_path.expanduser().resolve() - output_dir = _validated_output(repo_path, output_dir) + output_dir = _validated_output(repo_path, manifest_path.parent, output_dir) base_path = normalize_base_path(base_path) frontend = _prebuilt_frontend(frontend_dir) environment = os.environ if environ is None else environ @@ -614,10 +557,11 @@ def export_static_wiki( graphs[page_id], ) - _assert_publishable( + assert_publishable_tree( stage, forbidden_paths=(repo_path, manifest_path.parent), environ=environment, + label="static export", ) source_manifest = bundle.manifest export_manifest = { @@ -652,10 +596,11 @@ def export_static_wiki( "files": _file_inventory(stage), } manifest_file = _write_json(stage, STATIC_EXPORT_MANIFEST, export_manifest) - _assert_publishable( + assert_publishable_tree( stage, forbidden_paths=(repo_path, manifest_path.parent), environ=environment, + label="static export", ) if output_dir.exists(): diff --git a/docs/github_pages.md b/docs/github_pages.md new file mode 100644 index 00000000..5585d24d --- /dev/null +++ b/docs/github_pages.md @@ -0,0 +1,149 @@ + + +# Publish With GitHub Pages + +CodeNib can build repository context in GitHub Actions, deploy a source-linked +static Wiki to GitHub Pages, and retain the matching context views as one +downloadable artifact. The default path uses BM25 and needs no model, API key, +or model download. + +## No-Model Starter + +Create a caller workflow in the repository that should receive a Wiki: + +```yaml +name: CodeNib Pages + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +jobs: + publish: + uses: sysevol-ai/CodeNib/.github/workflows/codenib-pages.yml@ +``` + +Replace `` with a published CodeNib release commit. A commit SHA +keeps the compiler, frontend, action, and artifact schema on one reviewed +revision. In the repository's **Settings > Pages**, select **GitHub Actions** as +the source. + +The workflow checks out the caller's exact commit, incrementally builds the +`fast` preset, exports the Wiki at the Pages-provided mount path, and deploys it +through the `github-pages` environment. It also uploads an artifact named from +the repository and commit. BM25 belongs to that context artifact; the static +Wiki serves precomputed pages, citations, and navigation without executing a +query engine in the browser. + +## Build Semantic Context With A Local Model + +Select the semantic preset to build BM25 and dense-vector views. It defaults to +a local Hugging Face embedding model and needs no API credential: + +```yaml +jobs: + publish: + uses: sysevol-ai/CodeNib/.github/workflows/codenib-pages.yml@ + with: + preset: semantic +``` + +The first build downloads the default embedding model into the ephemeral Action +runner. The resulting vector view is stored in the commit-addressed context +artifact and reused through CodeNib's repository cache on later builds. Serve +that artifact through the local or MCP runtime for semantic queries; the Pages +site remains a precomputed inspection surface. Keep the default `fast` preset +when a model download is undesirable. + +## Bring Your Own Embedding Endpoint + +An OpenAI-compatible endpoint can replace the local model without changing the +artifact or Pages workflow: + +```yaml +jobs: + publish: + uses: sysevol-ai/CodeNib/.github/workflows/codenib-pages.yml@ + with: + preset: semantic + embedding-provider: openai + embedding-model: text-embedding-3-small + embedding-dimension: "1536" + embedding-endpoint: https://embeddings.example.com/v1 + secrets: + embedding_api_key: ${{ secrets.CODENIB_EMBEDDING_API_KEY }} +``` + +Provider, model, vector dimension, endpoint, Python version, and CodeNib source +revision participate in cache compatibility. The credential value does not. +Endpoints containing user information, a query, or a fragment are rejected. + +## What Gets Published + +The Pages artifact is a serverless inspection surface. It contains generated +pages, source slices used by citations, page-level dependency data when +available, and `codenib-static.json`. It does not contain an API endpoint, +credential, interactive Ask backend, or unrestricted source-reading service. + +The separate context artifact contains: + +- `codenib-context.json`, with repository, commit, schema, capabilities, and + file hashes; +- an artifact-relative `repo_manifest.json`; +- the BM25 view and, for `semantic`, FAISS indexes plus repository-relative + document locations. + +Mutable vector maintenance caches are deliberately excluded. The downloadable +artifact represents query-serving state for one commit; it is not a substitute +for the Action cache used to update a later commit. Portable publication +currently supports the `fast` and `semantic` presets. Graph and Zoekt indexes +remain available in the local/MCP runtime but are not yet promised as portable +Pages artifacts. + +## Incremental Builds + +The Action caches `~/.codenib/repositories` under a key that includes the +repository, platform, Python version, profile, provider identity, and CodeNib +revision. A prefix restore may supply the previous commit's state, but it never +declares that state current. The compiler compares the checkout and manifest, +updates supported views, and rebuilds when reuse is not valid. The newly +uploaded context artifact always records the indexed checkout's resolved Git +commit rather than assuming that it matches the surrounding event SHA. + +## Security Boundary + +The reusable workflow rejects `pull_request_target` and skips pull requests +whose head repository differs from the base repository. It therefore does not +pass BYO credentials to untrusted fork code. All shipped +third-party Actions are pinned to immutable commits, checkout credentials are +not persisted, and publication fails if an output contains a configured secret, +a symbolic link, or a build-machine source/index path. + +Use `push` or `workflow_dispatch` for normal publication. Do not wrap the +reusable workflow in `pull_request_target`. + +## Build Without Deployment + +The composite Action can be used directly when another static host or artifact +store owns deployment: + +```yaml +- uses: sysevol-ai/CodeNib/.github/actions/publish@ + id: codenib + with: + preset: fast + base-path: /repository +``` + +Its outputs include `site-path`, `context-path`, `context-manifest`, +`artifact-name`, `cache-hit`, `cache-key`, and `source-commit`. diff --git a/docs/product_roadmap.md b/docs/product_roadmap.md index 32e27e1e..5851c026 100644 --- a/docs/product_roadmap.md +++ b/docs/product_roadmap.md @@ -252,8 +252,9 @@ The program has two user-facing surfaces that share one artifact contract: Generation and embeddings are build-time provider capabilities in the static surface. A Pages export never contains a provider credential, and it does not claim query-time semantic search when no authenticated runtime exists. The -offline fallback remains useful through lexical search, source navigation, -pre-generated pages, and dependency data when those views are available. +Pages surface remains useful through source navigation, pre-generated pages, +and dependency data when those views are available. The matching artifact adds +lexical and optional semantic search when loaded by the local or MCP runtime. ### H1: Static artifact contract ([#416](https://github.com/sysevol-ai/CodeNib/issues/416)) diff --git a/docs/quickstart.md b/docs/quickstart.md index d749f059..ea56defc 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -106,6 +106,23 @@ Interactive Ask, on-demand edge labels, and arbitrary dependency queries need an authenticated CodeNib runtime and are not exposed by the export. No API key, GitHub token, endpoint credential, or build-machine absolute path is serialized. +To build the index and both distribution surfaces in one command: + +```bash +codenib publish . \ + --preset fast \ + --site-output /tmp/repository-wiki \ + --context-output /tmp/repository-context \ + --base-path /repository +``` + +The context directory is commit-addressed query-serving state with an +artifact-relative manifest and file hashes. It excludes mutable maintenance +caches. Portable publication currently supports `fast` and `semantic`; use the +local or MCP runtime for graph and Zoekt views. See +[Publish With GitHub Pages](github_pages.md) for the no-model Action, local +embedding model, and BYO endpoint configurations. + ## Select Repository Views | Preset | Required package | Views | diff --git a/mkdocs.yml b/mkdocs.yml index a37b04d1..87da3ee3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -165,6 +165,7 @@ nav: - Get Started: - get-started/index.md - Quickstart: quickstart.md + - GitHub Pages: github_pages.md - Web UI: web_demo.md - Running Locally: running-locally.md - Language Capabilities: language_capabilities.md diff --git a/test/actions/test_publish_action.py b/test/actions/test_publish_action.py new file mode 100644 index 00000000..4af34c65 --- /dev/null +++ b/test/actions/test_publish_action.py @@ -0,0 +1,345 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +import re +import subprocess +from pathlib import Path +from typing import Any + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +ACTION_PATH = ROOT / ".github" / "actions" / "publish" / "action.yml" +WORKFLOW_PATH = ROOT / ".github" / "workflows" / "codenib-pages.yml" +SMOKE_WORKFLOW_PATH = ROOT / ".github" / "workflows" / "codenib-publish-smoke.yml" +PINNED_ACTION_RE = re.compile(r"^[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_.-]+)+@[0-9a-f]{40}$") + + +def _load(path: Path) -> dict[str, Any]: + value = yaml.load(path.read_text(encoding="utf-8"), Loader=yaml.BaseLoader) + assert isinstance(value, dict) + return value + + +def _steps(value: dict[str, Any]) -> list[dict[str, Any]]: + if "runs" in value: + return list(value["runs"]["steps"]) + result = [] + for job in value["jobs"].values(): + result.extend(job.get("steps") or []) + return result + + +def _assert_external_actions_are_sha_pinned(value: dict[str, Any]) -> None: + for step in _steps(value): + uses = step.get("uses") + if not uses or str(uses).startswith("./"): + continue + assert PINNED_ACTION_RE.fullmatch(str(uses)), uses + + +def _run_resolve_step( + tmp_path: Path, **env_overrides: str +) -> tuple[subprocess.CompletedProcess[str], str]: + action = _load(ACTION_PATH) + resolve = next(step for step in _steps(action) if step.get("id") == "inputs") + output_path = tmp_path / "github-output" + env = os.environ.copy() + env.update( + { + "ACTION_REF": "0123456789abcdef", + "GITHUB_ACTION_PATH": str(ACTION_PATH.parent), + "GITHUB_OUTPUT": str(output_path), + "GITHUB_REPOSITORY": "example/project", + "GITHUB_REPOSITORY_ID": "123456", + "GITHUB_SHA": "a" * 40, + "INPUT_ARTIFACT_NAME": "", + "INPUT_BASE_PATH": "/", + "INPUT_CACHE": "true", + "INPUT_CONTEXT_OUTPUT": "", + "INPUT_EMBEDDING_DIMENSION": "", + "INPUT_EMBEDDING_ENDPOINT": "", + "INPUT_EMBEDDING_MODEL": "", + "INPUT_EMBEDDING_PROVIDER": "", + "INPUT_PRESET": "fast", + "INPUT_PYTHON_VERSION": "3.12", + "INPUT_REPOSITORY": "", + "INPUT_REPOSITORY_PATH": str(ROOT), + "INPUT_RETENTION_DAYS": "14", + "INPUT_REVISION": "", + "INPUT_SITE_OUTPUT": "", + "INPUT_UPLOAD_CONTEXT": "true", + "RUNNER_OS": "Linux", + "RUNNER_TEMP": str(tmp_path), + } + ) + env.update(env_overrides) + result = subprocess.run( + ["bash", "-c", str(resolve["run"])], + text=True, + capture_output=True, + check=False, + env=env, + ) + output = output_path.read_text(encoding="utf-8") if output_path.exists() else "" + return result, output + + +def test_publish_action_has_secret_free_inputs_and_stable_outputs() -> None: + action = _load(ACTION_PATH) + + assert action["runs"]["using"] == "composite" + assert "api-key" not in action["inputs"] + assert "embedding-api-key-env" in action["inputs"] + assert { + "site-path", + "context-path", + "context-manifest", + "artifact-name", + "cache-hit", + "cache-key", + "source-commit", + } <= set(action["outputs"]) + assert action["inputs"]["preset"]["default"] == "fast" + resolve = next(step for step in _steps(action) if step.get("id") == "inputs") + assert "fast|semantic)" in resolve["run"] + assert "graph|full" not in resolve["run"] + _assert_external_actions_are_sha_pinned(action) + + +def test_publish_action_shell_blocks_parse_with_bash() -> None: + action = _load(ACTION_PATH) + + for index, step in enumerate(_steps(action)): + script = str(step.get("run") or "") + if not script: + continue + result = subprocess.run( + ["bash", "-n"], + input=script, + text=True, + capture_output=True, + check=False, + ) + assert result.returncode == 0, f"step {index}: {result.stderr}" + + +def test_publish_action_builds_frontend_before_source_install() -> None: + action = _load(ACTION_PATH) + names = [step.get("name") for step in _steps(action)] + + assert names.index("Build static frontend") < names.index("Install CodeNib") + resolve = next(step for step in _steps(action) if step.get("id") == "inputs") + assert "web/dist" in resolve["run"] + setup_node = next( + step for step in _steps(action) if step.get("name") == "Set up Node.js" + ) + dependency_path = setup_node["with"]["cache-dependency-path"] + assert dependency_path == ( + "${{ steps.inputs.outputs.source_path }}/web/package-lock.json" + ) + assert ".." not in dependency_path + + +def test_publish_action_keeps_untrusted_inputs_out_of_shell_source() -> None: + action = _load(ACTION_PATH) + + for step in _steps(action): + script = str(step.get("run") or "") + assert "${{ inputs." not in script + publish = next( + step + for step in _steps(action) + if step.get("name") == "Publish repository context" + ) + assert "GITHUB_TOKEN" not in publish["env"] + assert "github.token" not in str(publish) + assert "command=(" in publish["run"] + assert '"${command[@]}"' in publish["run"] + + +def test_publish_action_resolves_valid_inputs(tmp_path: Path) -> None: + result, output = _run_resolve_step(tmp_path) + source_commit = subprocess.run( + ["git", "-C", str(ROOT), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + assert result.returncode == 0, result.stderr + assert ( + f"artifact_name=codenib-context-example-project-{source_commit[:12]}" in output + ) + assert f"site_path={tmp_path / 'codenib-site'}" in output + assert f"context_path={tmp_path / 'codenib-context'}" in output + assert "embedding_provider=" in output + assert "extras=" in output + assert f"source_path={ROOT}" in output + assert f"source_commit={source_commit}" in output + + +def test_publish_action_defaults_semantic_to_local_huggingface( + tmp_path: Path, +) -> None: + result, output = _run_resolve_step(tmp_path, INPUT_PRESET="semantic") + + assert result.returncode == 0, result.stderr + assert "embedding_provider=huggingface\n" in output + assert "extras=semantic\n" in output + + +def test_publish_action_selects_remote_extra_for_openai(tmp_path: Path) -> None: + result, output = _run_resolve_step( + tmp_path, + INPUT_PRESET="semantic", + INPUT_EMBEDDING_PROVIDER="openai", + ) + + assert result.returncode == 0, result.stderr + assert "embedding_provider=openai\n" in output + assert "extras=semantic-remote\n" in output + + +def test_publish_action_rejects_retired_github_models(tmp_path: Path) -> None: + result, output = _run_resolve_step( + tmp_path, + INPUT_PRESET="semantic", + INPUT_EMBEDDING_PROVIDER="github_models", + ) + + assert result.returncode == 2 + assert "unsupported embedding provider" in result.stderr + assert output == "" + + +def test_publish_action_rejects_output_command_injection(tmp_path: Path) -> None: + result, output = _run_resolve_step( + tmp_path, + INPUT_EMBEDDING_PROVIDER="openai\nartifact_name=forged", + ) + + assert result.returncode == 2 + assert "must not contain a line break" in result.stderr + assert "forged" not in output + + +def test_publish_action_rejects_unsupported_provider(tmp_path: Path) -> None: + result, output = _run_resolve_step( + tmp_path, + INPUT_PRESET="semantic", + INPUT_EMBEDDING_PROVIDER="unknown", + ) + + assert result.returncode == 2 + assert "unsupported embedding provider" in result.stderr + assert output == "" + + +def test_publish_action_rejects_overlapping_outputs(tmp_path: Path) -> None: + site_path = tmp_path / "publish" + result, output = _run_resolve_step( + tmp_path, + INPUT_SITE_OUTPUT=str(site_path), + INPUT_CONTEXT_OUTPUT=str(site_path / "context"), + ) + + assert result.returncode == 2 + assert "site-output and context-output must not overlap" in result.stderr + assert output == "" + + +def test_publish_action_cache_identity_is_public_and_commit_addressed() -> None: + action = _load(ACTION_PATH) + resolve = next(step for step in _steps(action) if step.get("id") == "inputs") + script = resolve["run"] + + assert "git -C" in script + assert "source_commit" in script + assert "GITHUB_SHA" not in script + assert "tr '[:upper:]' '[:lower:]'" in script + assert "INPUT_PRESET" in script + assert "INPUT_EMBEDDING_PROVIDER" in script + assert "INPUT_EMBEDDING_ENDPOINT" in script + assert "API_KEY" not in script + restore = next( + step + for step in _steps(action) + if step.get("name") == "Restore incremental repository state" + ) + save = next( + step + for step in _steps(action) + if step.get("name") == "Save incremental repository state" + ) + assert restore["with"]["key"] == "${{ steps.inputs.outputs.cache_key }}" + assert save["with"]["key"] == "${{ steps.inputs.outputs.cache_key }}" + + +def test_reusable_workflow_is_fork_safe_and_binds_its_exact_revision() -> None: + workflow = _load(WORKFLOW_PATH) + build = workflow["jobs"]["build"] + condition = build["if"] + + assert "pull_request_target" in condition + assert "head.repo.full_name == github.repository" in condition + assert build["permissions"] == { + "contents": "read", + "pages": "write", + } + exact_checkout = next( + step + for step in build["steps"] + if step.get("name") == "Checkout the exact CodeNib workflow revision" + ) + assert exact_checkout["with"]["repository"] == "${{ job.workflow_repository }}" + assert exact_checkout["with"]["ref"] == "${{ job.workflow_sha }}" + assert exact_checkout["with"]["persist-credentials"] == "false" + secret_steps = [ + step for step in build["steps"] if "secrets.embedding_api_key" in str(step) + ] + assert [step["name"] for step in secret_steps] == [ + "Build Wiki and context artifact" + ] + publish = secret_steps[0] + secret_expression = publish["env"]["CODENIB_ACTION_EMBEDDING_KEY"] + assert "inputs.preset == 'semantic'" in secret_expression + assert "inputs.embedding-provider == 'openai'" in secret_expression + _assert_external_actions_are_sha_pinned(workflow) + + +def test_reusable_workflow_deploys_only_the_published_site() -> None: + workflow = _load(WORKFLOW_PATH) + build = workflow["jobs"]["build"] + upload = next( + step + for step in build["steps"] + if step.get("name") == "Upload GitHub Pages artifact" + ) + deploy = workflow["jobs"]["deploy"] + + assert upload["with"]["path"] == "${{ steps.publish.outputs.site-path }}" + assert deploy["needs"] == "build" + assert deploy["permissions"] == {"pages": "write", "id-token": "write"} + assert deploy["environment"]["name"] == "github-pages" + assert workflow["concurrency"]["cancel-in-progress"] == "false" + + +def test_hosted_publish_smoke_is_narrow_and_sha_pinned() -> None: + workflow = _load(SMOKE_WORKFLOW_PATH) + job = workflow["jobs"]["no-model"] + + assert "head.repo.full_name == github.repository" in job["if"] + publish = next( + step + for step in job["steps"] + if step.get("name") == "Exercise local publish Action" + ) + assert publish["with"]["cache"] == "false" + assert publish["with"]["upload-context"] == "false" + assert publish["with"]["repository-path"] == "${{ steps.fixture.outputs.path }}" + _assert_external_actions_are_sha_pinned(workflow) diff --git a/test/artifacts/test_context_artifact.py b/test/artifacts/test_context_artifact.py new file mode 100644 index 00000000..05a3cada --- /dev/null +++ b/test/artifacts/test_context_artifact.py @@ -0,0 +1,369 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import hashlib +import json +import pickle +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from codenib.artifacts import ( + CONTEXT_ARTIFACT_MANIFEST, + CONTEXT_ARTIFACT_SCHEMA, + stage_context_artifact, +) +from codenib.compiler.manifest import IndexEntry, RepoManifest +from codenib.source_fingerprint import fingerprint_repository + + +def _fixture_manifest( + root: Path, + *, + view_path: Path | None = None, + config: dict | None = None, + status: str = "fresh", +) -> tuple[Path, Path, Path]: + repo = root / "repo" + repo.mkdir(parents=True) + (repo / "sample.py").write_text("VALUE = 1\n") + index_root = root / "state" / "indexes" + index_root.mkdir(parents=True) + if view_path is None: + view_path = index_root / "bm25" + view_path.mkdir() + (view_path / "documents.json").write_text( + '[{"page_content": "value", "metadata": {"file": "sample.py"}}]\n' + ) + (view_path / "bm25_metadata.json").write_text( + json.dumps( + { + "project_root": str(repo), + "max_k": 128, + "language": "english", + } + ) + + "\n" + ) + manifest_path = index_root / "repo_manifest.json" + source_fingerprint = fingerprint_repository(repo).value + RepoManifest( + repo_path=str(repo), + commit="a" * 40, + last_indexed_commit="a" * 40, + source_fingerprint=source_fingerprint, + last_indexed_source_fingerprint=source_fingerprint, + languages=["python"], + file_count=1, + indexes={ + "bm25": IndexEntry( + index_type="bm25", + path=str(view_path), + built_at="2026-08-04T00:00:00+00:00", + built_at_epoch=1.0, + status=status, + config=dict(config or {}), + commit="a" * 40, + source_fingerprint=source_fingerprint, + ) + }, + capabilities={"sparse_search": status == "fresh"}, + compiled_at="2026-08-04T00:00:00+00:00", + compiled_at_epoch=1.0, + ).save(manifest_path) + return repo, manifest_path, view_path + + +def _tree(root: Path) -> dict[str, bytes]: + return { + path.relative_to(root).as_posix(): path.read_bytes() + for path in sorted(root.rglob("*")) + if path.is_file() + } + + +def _fixture_vector_manifest(root: Path) -> tuple[Path, Path, Path]: + repo = root / "repo" + repo.mkdir(parents=True) + source = repo / "sample.py" + source.write_text("VALUE = 1\n") + index_root = root / "state" / "indexes" + vector = index_root / "vector" + level = vector / "l2" + level.mkdir(parents=True) + with (level / "documents_test__model.pkl").open("wb") as handle: + pickle.dump( + [ + SimpleNamespace( + page_content="VALUE = 1", + metadata={"file": str(source), "node_id": "sample.py"}, + ) + ], + handle, + ) + (level / "index_test__model.faiss").write_bytes(b"serving-index") + (level / "index_test__model.pkl").write_bytes( + pickle.dumps({"legacy_path": str(source)}) + ) + (vector / "chunk_store.json").write_text( + json.dumps({str(source): [{"file": str(source)}]}) + ) + (vector / "embeddings_cache.npz").write_bytes(b"mutable-cache") + (vector / "incremental_state.json").write_text( + json.dumps({"index_path": str(vector)}) + ) + config = { + "builder_schema": 2, + "embedding_model": "test/model", + "embedding_provider": "huggingface", + "embedding_dimension": 4, + "dimension": 4, + "embedding_kwargs": {}, + "index_metric": "ip", + } + manifest_path = index_root / "repo_manifest.json" + source_fingerprint = fingerprint_repository(repo).value + RepoManifest( + repo_path=str(repo), + commit="b" * 40, + last_indexed_commit="b" * 40, + source_fingerprint=source_fingerprint, + last_indexed_source_fingerprint=source_fingerprint, + languages=["python"], + file_count=1, + indexes={ + "vector": IndexEntry( + index_type="vector", + path=str(vector), + built_at="2026-08-04T00:00:00+00:00", + built_at_epoch=1.0, + status="fresh", + config=config, + metadata=dict(config), + commit="b" * 40, + source_fingerprint=source_fingerprint, + ) + }, + capabilities={"dense_search": True}, + compiled_at="2026-08-04T00:00:00+00:00", + compiled_at_epoch=1.0, + ).save(manifest_path) + return repo, manifest_path, vector + + +def test_context_artifact_rewrites_paths_and_hashes_current_views( + tmp_path: Path, +) -> None: + repo, manifest_path, _view_path = _fixture_manifest(tmp_path) + output = tmp_path / "publish" / "context" + + result = stage_context_artifact( + repo, + manifest_path, + output, + repository="Example/Project", + views=["bm25"], + environ={"GITHUB_TOKEN": "configured-secret-value"}, + ) + + assert result.repository == "example/project" + assert result.commit == "a" * 40 + assert result.views == ("bm25",) + metadata = json.loads((output / CONTEXT_ARTIFACT_MANIFEST).read_text()) + assert metadata["schema"] == CONTEXT_ARTIFACT_SCHEMA + assert metadata["repository"]["slug"] == "example/project" + portable = json.loads((output / "repo_manifest.json").read_text()) + assert portable["repo"]["path"] == "source" + assert portable["indexes"]["bm25"]["path"] == "views/bm25" + assert portable["capabilities"]["sparse_search"] is True + bm25_metadata = json.loads( + (output / "views" / "bm25" / "bm25_metadata.json").read_text() + ) + assert bm25_metadata["project_root"] == "source" + + files = {item["path"]: item for item in metadata["files"]} + assert set(files) == { + "repo_manifest.json", + "views/bm25/bm25_metadata.json", + "views/bm25/documents.json", + } + for relative, record in files.items(): + payload = (output / relative).read_bytes() + assert record["bytes"] == len(payload) + assert record["sha256"] == hashlib.sha256(payload).hexdigest() + serialized = b"".join(_tree(output).values()) + assert str(repo).encode() not in serialized + assert str(manifest_path.parent).encode() not in serialized + assert b"configured-secret-value" not in serialized + fingerprints = portable["indexes"]["bm25"]["config"]["artifact_file_fingerprints"] + for relative, record in fingerprints.items(): + payload = (output / "views" / "bm25" / relative).read_bytes() + assert record["size"] == len(payload) + assert record["sha256"] == hashlib.sha256(payload).hexdigest() + + +def test_context_artifact_is_deterministic_for_one_manifest(tmp_path: Path) -> None: + repo, manifest_path, _view_path = _fixture_manifest(tmp_path) + + stage_context_artifact( + repo, + manifest_path, + tmp_path / "first" / "context", + repository="example/project", + ) + stage_context_artifact( + repo, + manifest_path, + tmp_path / "second" / "context", + repository="example/project", + ) + + assert _tree(tmp_path / "first" / "context") == _tree( + tmp_path / "second" / "context" + ) + + +def test_context_artifact_keeps_only_portable_vector_serving_state( + tmp_path: Path, +) -> None: + repo, manifest_path, _vector = _fixture_vector_manifest(tmp_path) + output = tmp_path / "publish" / "context" + + stage_context_artifact( + repo, + manifest_path, + output, + repository="example/vector-project", + ) + + vector = output / "views" / "vector" + assert not (vector / "chunk_store.json").exists() + assert not (vector / "embeddings_cache.npz").exists() + assert not (vector / "incremental_state.json").exists() + assert not (vector / "l2" / "index_test__model.pkl").exists() + assert (vector / "l2" / "index_test__model.faiss").is_file() + assert not (vector / "l2" / "documents_test__model.pkl").exists() + documents = json.loads((vector / "l2" / "documents_test__model.json").read_text()) + assert documents[0]["metadata"]["file"] == "sample.py" + portable = json.loads((output / "repo_manifest.json").read_text()) + assert portable["indexes"]["vector"]["config"]["artifact_scope"] == ( + "query-serving" + ) + assert ( + portable["indexes"]["vector"]["config"]["portable_document_format"] + == "codenib.vector-documents.v1" + ) + assert not list(output.rglob("*.pkl")) + assert str(repo).encode() not in b"".join(_tree(output).values()) + + +def test_context_artifact_rejects_credential_shaped_config(tmp_path: Path) -> None: + repo, manifest_path, _view_path = _fixture_manifest( + tmp_path, + config={"api_key": "must-not-persist"}, + ) + + with pytest.raises(ValueError, match="credential field"): + stage_context_artifact( + repo, + manifest_path, + tmp_path / "publish" / "context", + ) + + +def test_context_artifact_rejects_credential_shaped_metadata(tmp_path: Path) -> None: + repo, manifest_path, _view_path = _fixture_manifest(tmp_path) + manifest = RepoManifest.load(manifest_path) + manifest.indexes["bm25"].metadata["authorization"] = "must-not-persist" + manifest.save(manifest_path) + + with pytest.raises(ValueError, match="credential field"): + stage_context_artifact( + repo, + manifest_path, + tmp_path / "publish" / "context", + ) + + +def test_context_artifact_rejects_source_drift(tmp_path: Path) -> None: + repo, manifest_path, _view_path = _fixture_manifest(tmp_path) + (repo / "sample.py").write_text("VALUE = 2\n") + + with pytest.raises(ValueError, match="source files do not match"): + stage_context_artifact( + repo, + manifest_path, + tmp_path / "publish" / "context", + ) + + +def test_context_artifact_rejects_configured_secret_in_view(tmp_path: Path) -> None: + repo, manifest_path, view_path = _fixture_manifest(tmp_path) + (view_path / "leak.bin").write_bytes(b"runtime-secret-value") + + with pytest.raises(ValueError, match="configured credential"): + stage_context_artifact( + repo, + manifest_path, + tmp_path / "publish" / "context", + environ={"MODELS_TOKEN": "runtime-secret-value"}, + ) + + +def test_context_artifact_stream_scan_finds_boundary_spanning_secret( + tmp_path: Path, +) -> None: + repo, manifest_path, view_path = _fixture_manifest(tmp_path) + secret = b"boundary-spanning-secret" + (view_path / "large.bin").write_bytes(b"x" * (1024 * 1024 - 5) + secret) + + with pytest.raises(ValueError, match="configured credential"): + stage_context_artifact( + repo, + manifest_path, + tmp_path / "publish" / "context", + environ={"CODENIB_ACTION_EMBEDDING_KEY": secret.decode()}, + ) + + +def test_context_artifact_rejects_view_outside_manifest_root(tmp_path: Path) -> None: + outside = tmp_path / "outside" + outside.mkdir() + (outside / "documents.json").write_text("[]\n") + (outside / "bm25_metadata.json").write_text("{}\n") + repo, manifest_path, _view_path = _fixture_manifest( + tmp_path, + view_path=outside, + ) + + with pytest.raises(ValueError, match="outside the manifest index root"): + stage_context_artifact( + repo, + manifest_path, + tmp_path / "publish" / "context", + ) + + +def test_context_artifact_rejects_stale_or_linked_views(tmp_path: Path) -> None: + stale_root = tmp_path / "stale" + repo, manifest_path, _view_path = _fixture_manifest(stale_root, status="stale") + with pytest.raises(ValueError, match="requires at least one current view"): + stage_context_artifact( + repo, + manifest_path, + stale_root / "publish" / "context", + ) + + linked_root = tmp_path / "linked" + repo, manifest_path, view_path = _fixture_manifest(linked_root) + (view_path / "alias").symlink_to(view_path / "index.json") + with pytest.raises(ValueError, match="symbolic link"): + stage_context_artifact( + repo, + manifest_path, + linked_root / "publish" / "context", + ) diff --git a/test/artifacts/test_publish.py b/test/artifacts/test_publish.py new file mode 100644 index 00000000..77749bf2 --- /dev/null +++ b/test/artifacts/test_publish.py @@ -0,0 +1,179 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + +from codenib import cli +from codenib.artifacts import CONTEXT_ARTIFACT_MANIFEST +from codenib.compiler.index_compiler import IndexCompiler +from codenib.web.static_export import STATIC_EXPORT_MANIFEST + + +def _frontend(root: Path) -> Path: + frontend = root / "frontend" + (frontend / "assets").mkdir(parents=True) + (frontend / "index.html").write_text( + "" + "" + "" + ) + (frontend / "runtime-config.js").write_text('window.__CODENIB_API_BASE__ = "";\n') + (frontend / "assets" / "app.js").write_text("console.log('wiki');\n") + return frontend + + +def test_publish_builds_static_site_and_portable_context_without_model( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + (repo / "runtime.py").write_text( + "def run(value: int) -> int:\n return value + 1\n" + ) + site = tmp_path / "published" / "site" + context = tmp_path / "published" / "context" + monkeypatch.setenv("CODENIB_HOME", str(tmp_path / "home")) + monkeypatch.setenv("GITHUB_REPOSITORY", "Example/Project") + + result = cli.run( + [ + "publish", + str(repo), + "--preset", + "fast", + "--site-output", + str(site), + "--context-output", + str(context), + "--base-path", + "/project", + "--frontend-dir", + str(_frontend(tmp_path)), + ] + ) + + assert result == 0 + assert (site / "index.html").is_file() + static_metadata = json.loads((site / STATIC_EXPORT_MANIFEST).read_text()) + context_metadata = json.loads((context / CONTEXT_ARTIFACT_MANIFEST).read_text()) + assert static_metadata["base_path"] == "/project" + assert context_metadata["repository"]["slug"] == "example/project" + assert context_metadata["views"] == ["bm25"] + assert (context / "views" / "bm25").is_dir() + output = capsys.readouterr().out + assert "Published Wiki:" in output + assert "Context artifact:" in output + + +def _git(repo: Path, *args: str) -> str: + result = subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def test_publish_second_commit_uses_incremental_compiler_state( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + _git(repo, "init", "--quiet") + _git(repo, "config", "user.name", "CodeNib Test") + _git(repo, "config", "user.email", "codenib@example.invalid") + source = repo / "runtime.py" + source.write_text("def run() -> int:\n return 1\n") + _git(repo, "add", "runtime.py") + _git(repo, "commit", "--quiet", "-m", "initial") + monkeypatch.setenv("CODENIB_HOME", str(tmp_path / "home")) + frontend = _frontend(tmp_path) + site = tmp_path / "published" / "site" + context = tmp_path / "published" / "context" + command = [ + "publish", + str(repo), + "--preset", + "fast", + "--site-output", + str(site), + "--context-output", + str(context), + "--frontend-dir", + str(frontend), + ] + + assert cli.run(command) == 0 + first_commit = json.loads((context / CONTEXT_ARTIFACT_MANIFEST).read_text())[ + "repository" + ]["commit"] + + source.write_text("def run() -> int:\n return 2\n") + _git(repo, "add", "runtime.py") + _git(repo, "commit", "--quiet", "-m", "update") + calls: list[tuple[tuple, dict]] = [] + original = IndexCompiler.update_repo + + def recording_update(self, *args, **kwargs): + calls.append((args, kwargs)) + return original(self, *args, **kwargs) + + monkeypatch.setattr(IndexCompiler, "update_repo", recording_update) + + assert cli.run(command) == 0 + second_commit = _git(repo, "rev-parse", "HEAD") + metadata = json.loads((context / CONTEXT_ARTIFACT_MANIFEST).read_text()) + assert calls + assert first_commit != second_commit + assert metadata["repository"]["commit"] == second_commit + assert metadata["source_locations"]["commit"] == second_commit + + +def test_publish_rejects_nested_site_and_context_outputs( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + (repo / "runtime.py").write_text("VALUE = 1\n") + monkeypatch.setenv("CODENIB_HOME", str(tmp_path / "home")) + output = tmp_path / "published" + + result = cli.run( + [ + "publish", + str(repo), + "--site-output", + str(output), + "--context-output", + str(output / "context"), + "--frontend-dir", + str(_frontend(tmp_path)), + ] + ) + + assert result == 2 + assert not output.exists() + + +def test_publication_environment_marks_custom_embedding_key_as_secret( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("CUSTOM_EMBEDDING_CREDENTIAL", "runtime-secret-value") + + environment = cli._publication_environment("CUSTOM_EMBEDDING_CREDENTIAL") + + assert environment["CODENIB_PUBLICATION_CREDENTIAL_SECRET"] == ( + "runtime-secret-value" + ) diff --git a/test/fixtures/publish_repo/sample.py b/test/fixtures/publish_repo/sample.py new file mode 100644 index 00000000..8bdee228 --- /dev/null +++ b/test/fixtures/publish_repo/sample.py @@ -0,0 +1,7 @@ +"""Small source fixture used by the no-model publish Action smoke.""" + + +def repository_name() -> str: + """Return a deterministic value that BM25 can index.""" + + return "CodeNib" diff --git a/test/index/test_vector_store_ivf.py b/test/index/test_vector_store_ivf.py index a04ef30d..1b5521a8 100644 --- a/test/index/test_vector_store_ivf.py +++ b/test/index/test_vector_store_ivf.py @@ -169,6 +169,61 @@ def test_ivf_save_load_roundtrip(tmp_path): assert res and res[0].node_name == chunks[1]["name"] +def test_load_prefers_portable_json_documents(tmp_path): + path = tmp_path / "vs" + store = _make_store(embedding_model="test/model") + chunks = _chunks(2) + store.add_code_chunks(chunks) + store.save(str(path)) + + documents_path = path / "l2" / "documents_test__model.pkl" + documents_path.unlink() + portable_path = documents_path.with_suffix(".json") + portable_path.write_text( + json.dumps( + [ + { + "page_content": chunk["content"], + "metadata": { + "name": chunk["name"], + "file": chunk["file"], + "start_line": chunk["start_line"], + "end_line": chunk["end_line"], + }, + } + for chunk in chunks + ] + ), + encoding="utf-8", + ) + + loaded = _make_store(embedding_model="test/model", store_path=str(path)) + loaded.load() + + assert [document.metadata["file"] for document in loaded.l2_documents] == [ + "m0.py", + "m1.py", + ] + + +def test_load_rejects_invalid_portable_json_documents(tmp_path): + path = tmp_path / "vs" + store = _make_store(embedding_model="test/model") + store.add_code_chunks(_chunks(1)) + store.save(str(path)) + + documents_path = path / "l2" / "documents_test__model.pkl" + documents_path.unlink() + documents_path.with_suffix(".json").write_text( + '[{"page_content": 7, "metadata": {}}]', + encoding="utf-8", + ) + + loaded = _make_store(embedding_model="test/model", store_path=str(path)) + with pytest.raises(ValueError, match="invalid content or metadata"): + loaded.load() + + def test_load_rejects_faiss_dimension_mismatch(tmp_path): path = tmp_path / "vs" model = "test/model" diff --git a/test/test_cli.py b/test/test_cli.py index 7467b754..1d476d83 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -20,7 +20,7 @@ def test_parser_exposes_release_commands() -> None: parser = cli.build_parser() - for command in ("index", "wiki", "export", "mcp", "doctor"): + for command in ("index", "wiki", "export", "publish", "mcp", "doctor"): args = parser.parse_args([command]) assert args.command == command @@ -44,6 +44,46 @@ def test_export_parser_accepts_pages_mount_options() -> None: assert args.frontend_dir == "/tmp/frontend" +def test_publish_and_artifact_parsers_expose_distribution_options() -> None: + publish = cli.build_parser().parse_args( + [ + "publish", + ".", + "--preset", + "semantic", + "--site-output", + "/tmp/site", + "--context-output", + "/tmp/context", + "--repository", + "example/project", + "--base-path", + "/project", + "--embedding-provider", + "openai", + ] + ) + artifact = cli.build_parser().parse_args( + [ + "artifact", + "pack", + ".", + "--output", + "/tmp/context", + "--view", + "bm25,vector", + ] + ) + + assert publish.preset == "semantic" + assert publish.site_output == "/tmp/site" + assert publish.context_output == "/tmp/context" + assert publish.repository == "example/project" + assert publish.embedding_provider == "openai" + assert artifact.artifact_command == "pack" + assert artifact.view == ["bm25,vector"] + + def test_wiki_parser_accepts_headless_quality_audit() -> None: args = cli.build_parser().parse_args(["wiki", ".", "--audit", "--audit-json"]) @@ -792,7 +832,7 @@ def test_prepare_local_wiki_rejects_mismatched_checkout( languages=["python"], ).save(str(manifest_path)) monkeypatch.setattr( - "codenib.web.local._checkout_commit", + "codenib.compiler.checkout_identity.checkout_commit", lambda _repo_path: "b" * 40, ) diff --git a/test/test_cli_remote_embeddings.py b/test/test_cli_remote_embeddings.py index 1e639fa9..fba57182 100644 --- a/test/test_cli_remote_embeddings.py +++ b/test/test_cli_remote_embeddings.py @@ -12,6 +12,9 @@ import pytest from codenib import cli +from codenib.artifacts import stage_context_artifact +from codenib.compiler.manifest import MANIFEST_FILENAME +from codenib.paths import repo_index_dir class _EmbeddingHandler(BaseHTTPRequestHandler): @@ -104,3 +107,19 @@ def test_openai_semantic_build_uses_remote_sdk_without_sentence_transformers( request["authorization"] == "Bearer runtime-token" for request in _EmbeddingHandler.requests ) + + artifact = tmp_path / "portable-context" + stage_context_artifact( + repo, + repo_index_dir(repo) / MANIFEST_FILENAME, + artifact, + repository="example/semantic-project", + views=["vector"], + environ={"GITHUB_TOKEN": "runtime-token"}, + ) + serialized = b"".join( + path.read_bytes() for path in artifact.rglob("*") if path.is_file() + ) + assert str(repo).encode() not in serialized + assert b"runtime-token" not in serialized + assert not (artifact / "views" / "vector" / "incremental_state.json").exists() diff --git a/test/web/test_static_export.py b/test/web/test_static_export.py index 94cbb183..5f59dce6 100644 --- a/test/web/test_static_export.py +++ b/test/web/test_static_export.py @@ -10,10 +10,10 @@ import pytest +from codenib.artifacts.security import assert_publishable_tree from codenib.compiler.manifest import IndexEntry, RepoManifest from codenib.web.static_export import ( STATIC_EXPORT_MANIFEST, - _assert_publishable, export_static_wiki, normalize_base_path, ) @@ -250,14 +250,18 @@ def test_publishability_rejects_json_escaped_windows_path(tmp_path: Path) -> Non ) class ResolvedWindowsPath: + def expanduser(self): + return self + def resolve(self): return windows_path with pytest.raises(ValueError, match="absolute build-machine path"): - _assert_publishable( + assert_publishable_tree( root, forbidden_paths=(ResolvedWindowsPath(),), environ={}, + label="static export", ) @@ -335,6 +339,16 @@ def test_static_export_does_not_replace_an_unrelated_directory(export_setup) -> assert (export_setup.output / "keep.txt").read_text() == "keep" +def test_static_export_rejects_index_root_overlap(export_setup) -> None: + with pytest.raises(ValueError, match="outside the index root"): + export_static_wiki( + export_setup.repo, + export_setup.manifest_path, + export_setup.manifest_path.parent, + frontend_dir=export_setup.frontend, + ) + + def test_static_export_rejects_absolute_citation_paths( export_setup, monkeypatch: pytest.MonkeyPatch ) -> None: