Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
228 changes: 228 additions & 0 deletions .github/workflows/publish-eval-image.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,228 @@
# Publish the measured eval image:
# ghcr.io/baseintelligence/agent-challenge-canonical
#
# Consumes golden assets via ORAS (never from this git tree):
# ghcr.io/baseintelligence/agent-challenge-golden-assets@sha256:…
#
# Co-pin AGENT_CHALLENGE_REF (git SHA or tag of BaseIntelligence/agent-challenge)
# with GOLDEN_ASSETS_REF (digest-pinned OCI artefact). Update both together.
#
# Package visibility: after the first successful push, set the GHCR package
# "agent-challenge-canonical" to Public (GitHub → Packages → package settings)
# so miners can `docker pull` by digest without credentials. DSTACK_DOCKER_*
# remains available for private pulls but is not required for this image.
#
# Bootstrap: until the first golden-assets digest exists, set GOLDEN_ASSETS_REF
# via workflow_dispatch input after running agent-challenge publish-golden-assets.

name: Publish eval image

on:
workflow_dispatch:
inputs:
agent_challenge_ref:
description: "agent-challenge git ref (SHA or tag), co-pinned with golden assets"
required: true
type: string
golden_assets_ref:
description: "Digest-pinned golden assets (ghcr.io/.../agent-challenge-golden-assets@sha256:...)"
required: true
type: string
image_tag:
description: "Optional extra tag (e.g. v1.2.3); sha-<challenge-sha> always applied"
required: false
default: ""
type: string
push:
description: "Push to GHCR (false = build-only smoke)"
required: true
default: true
type: boolean
push:
tags:
- "eval-image-v*.*.*"

permissions:
contents: read

env:
IMAGE_NAME: ghcr.io/baseintelligence/agent-challenge-canonical
# Default co-pins — replace GOLDEN_ASSETS_REF after first ORAS publish (T1 bootstrap).
# AGENT_CHALLENGE_REF must match the freeze that produced the golden-assets digest.
AGENT_CHALLENGE_REF: ${{ inputs.agent_challenge_ref || 'main' }}
GOLDEN_ASSETS_REF: ${{ inputs.golden_assets_ref || 'ghcr.io/baseintelligence/agent-challenge-golden-assets@sha256:REPLACE_AFTER_FIRST_PUBLISH' }}
SOURCE_DATE_EPOCH: "1700000000"

jobs:
publish-eval-image:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout agent-recipe
uses: actions/checkout@v4

- name: Require digest-pinned GOLDEN_ASSETS_REF
env:
REF: ${{ env.GOLDEN_ASSETS_REF }}
run: |
set -euo pipefail
case "$REF" in
*@sha256:[0-9a-f][0-9a-f]*)
echo "GOLDEN_ASSETS_REF ok: $REF"
;;
*)
echo "::error::GOLDEN_ASSETS_REF must be digest-pinned (repo@sha256:64hex); got: $REF"
echo "Publish golden assets first (agent-challenge publish-golden-assets), then re-run with the digest."
exit 1
;;
esac

- name: Install ORAS
uses: oras-project/setup-oras@v1
with:
version: "1.2.2"

- name: Log in to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Pull golden assets (digest-pinned ORAS)
env:
GOLDEN_ASSETS_REF: ${{ env.GOLDEN_ASSETS_REF }}
run: |
set -euo pipefail
DEST="${RUNNER_TEMP}/golden-assets"
mkdir -p "$DEST"
oras pull "$GOLDEN_ASSETS_REF" -o "$DEST"
test -f "$DEST/MANIFEST.json"
test -f "$DEST/golden/dataset-digest.json"
test -f "$DEST/golden/live-registry-refs.json"
test -f "$DEST/golden/tbench-2.1-oracle.json.enc"
test -d "$DEST/docker/canonical/live-task-cache"
python3 - <<'PY'
import hashlib, json, os, sys
from pathlib import Path
dest = Path(os.environ["RUNNER_TEMP"]) / "golden-assets"
manifest = json.loads((dest / "MANIFEST.json").read_text())
assert manifest.get("schema") == "agent-challenge/golden-assets@1", manifest.get("schema")
errors = []
for rel, expected in manifest.get("files", {}).items():
path = dest / rel
if not path.is_file():
errors.append(f"missing {rel}")
continue
got = "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest()
if got != expected:
errors.append(f"digest mismatch {rel}: {got} != {expected}")
if errors:
print("\n".join(errors), file=sys.stderr)
sys.exit(1)
print("MANIFEST.json file digests ok; version=", manifest.get("version"))
PY
echo "GOLDEN_ASSETS_DIR=$DEST" >> "$GITHUB_ENV"

- name: Checkout agent-challenge at co-pinned ref
uses: actions/checkout@v4
with:
repository: BaseIntelligence/agent-challenge
ref: ${{ env.AGENT_CHALLENGE_REF }}
path: ${{ runner.temp }}/agent-challenge
# GITHUB_TOKEN can read public repos; for private, use a PAT with contents:read.
token: ${{ secrets.GITHUB_TOKEN }}

- name: Record co-pin metadata
id: pins
run: |
set -euo pipefail
AC_SHA=$(git -C "${RUNNER_TEMP}/agent-challenge" rev-parse HEAD)
echo "agent_challenge_sha=$AC_SHA" >> "$GITHUB_OUTPUT"
echo "AGENT_CHALLENGE_SHA=$AC_SHA" >> "$GITHUB_ENV"
echo "co-pin AGENT_CHALLENGE_REF=${AGENT_CHALLENGE_REF} -> $AC_SHA"
echo "co-pin GOLDEN_ASSETS_REF=${GOLDEN_ASSETS_REF}"

- name: Set up Docker Buildx (docker-container driver)
uses: docker/setup-buildx-action@v3
with:
driver: docker-container

- name: Build (and optionally push) eval image
id: build
env:
PUSH: ${{ inputs.push == true || inputs.push == 'true' || github.event_name == 'push' }}
EXTRA_TAG: ${{ inputs.image_tag }}
run: |
set -euo pipefail
META="${RUNNER_TEMP}/build-metadata.json"
TAGS=(-t "${IMAGE_NAME}:sha-${AGENT_CHALLENGE_SHA}")
if [[ -n "${EXTRA_TAG}" ]]; then
TAGS+=(-t "${IMAGE_NAME}:${EXTRA_TAG}")
fi
if [[ "${GITHUB_REF_TYPE:-}" == "tag" ]]; then
TAGS+=(-t "${IMAGE_NAME}:${GITHUB_REF_NAME}")
fi

OUTPUT_ARGS=()
if [[ "${PUSH}" == "true" ]]; then
# Single primary tag for rewrite-timestamp push; retag after if needed.
PRIMARY="${IMAGE_NAME}:sha-${AGENT_CHALLENGE_SHA}"
OUTPUT_ARGS=(--output "type=image,name=${PRIMARY},push=true,rewrite-timestamp=true")
if [[ -n "${EXTRA_TAG}" ]]; then
OUTPUT_ARGS+=(--output "type=image,name=${IMAGE_NAME}:${EXTRA_TAG},push=true,rewrite-timestamp=true")
fi
else
OUTPUT_ARGS=(--output "type=oci,dest=${RUNNER_TEMP}/eval-image.oci.tar,rewrite-timestamp=true")
fi

docker buildx build \
--progress=plain \
-f Dockerfile \
--provenance=false \
--sbom=false \
--build-arg "SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}" \
--build-context "golden_assets=${GOLDEN_ASSETS_DIR}" \
--build-context "agent_challenge=${RUNNER_TEMP}/agent-challenge" \
--metadata-file "${META}" \
"${OUTPUT_ARGS[@]}" \
.

DIGEST=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("containerimage.digest",""))' "${META}")
if [[ ! "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then
echo "::error::missing containerimage.digest in build metadata: ${DIGEST}"
cat "${META}" || true
exit 1
fi
echo "digest=${DIGEST}" >> "$GITHUB_OUTPUT"
echo "image_ref=${IMAGE_NAME}@${DIGEST}" >> "$GITHUB_OUTPUT"
echo "Built ${IMAGE_NAME}@${DIGEST}"
echo "### Eval image" >> "$GITHUB_STEP_SUMMARY"
echo "" >> "$GITHUB_STEP_SUMMARY"
echo "- Image: \`${IMAGE_NAME}@${DIGEST}\`" >> "$GITHUB_STEP_SUMMARY"
echo "- AGENT_CHALLENGE_SHA: \`${AGENT_CHALLENGE_SHA}\`" >> "$GITHUB_STEP_SUMMARY"
echo "- GOLDEN_ASSETS_REF: \`${GOLDEN_ASSETS_REF}\`" >> "$GITHUB_STEP_SUMMARY"
echo "- SOURCE_DATE_EPOCH: \`${SOURCE_DATE_EPOCH}\`" >> "$GITHUB_STEP_SUMMARY"
echo "" >> "$GITHUB_STEP_SUMMARY"
echo "Set GHCR package **agent-challenge-canonical** visibility to **Public** after first push." >> "$GITHUB_STEP_SUMMARY"

- name: Emit digest artefact
if: always() && steps.build.outputs.digest != ''
run: |
mkdir -p "${RUNNER_TEMP}/out"
{
echo "image_ref=${{ steps.build.outputs.image_ref }}"
echo "digest=${{ steps.build.outputs.digest }}"
echo "agent_challenge_sha=${{ steps.pins.outputs.agent_challenge_sha }}"
echo "golden_assets_ref=${GOLDEN_ASSETS_REF}"
} > "${RUNNER_TEMP}/out/eval-image-pins.txt"
cat "${RUNNER_TEMP}/out/eval-image-pins.txt"

- name: Upload pin receipt
if: always() && steps.build.outputs.digest != ''
uses: actions/upload-artifact@v4
with:
name: eval-image-pins
path: ${{ runner.temp }}/out/eval-image-pins.txt
9 changes: 9 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -44,3 +44,12 @@ secrets/
*.zip
submissions/
.local/

# Eval-image local build inputs (ORAS pull / multi-checkout — never commit)
golden/
live-task-cache/
**/live-task-cache/
golden-assets/
/tmp-golden-assets/
*.oci.tar
.eval-image-pins.txt
55 changes: 55 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# syntax=docker/dockerfile:1
#
# Measured agent-challenge eval / orchestrator image.
# Owned by agent-recipe (D7). Built reproducibly; published as
# ghcr.io/baseintelligence/agent-challenge-canonical
#
# Golden + live-task-cache are NEVER taken from this git tree. They arrive only
# via BuildKit named context `golden_assets` (ORAS pull of
# ghcr.io/baseintelligence/agent-challenge-golden-assets@sha256:… per T0 §5).
#
# agent_challenge package source arrives via named context `agent_challenge`
# (git SHA checkout co-pinned with GOLDEN_ASSETS_REF).
#
# Reproducibility (match agent_challenge.canonical.build):
# SOURCE_DATE_EPOCH=1700000000 docker buildx build \
# --build-arg SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH \
# --provenance=false --sbom=false \
# --build-context golden_assets=/path/to/pulled-assets \
# --build-context agent_challenge=/path/to/agent-challenge@SHA \
# --output type=image,name=ghcr.io/baseintelligence/agent-challenge-canonical:TAG,push=true,rewrite-timestamp=true \
# -f Dockerfile .
#
# No secrets: ciphertext golden only; CHALLENGE_GOLDEN_KEY_FILE never baked.

ARG BASE_IMAGE=python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a1bfc852199fbf
FROM ${BASE_IMAGE}

# Declared so BuildKit normalises layer timestamps (rewrite-timestamp).
ARG SOURCE_DATE_EPOCH

ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PYTHONPATH=/app/src

WORKDIR /app

# Locked + hashed runtime deps from the co-pinned agent-challenge tree.
COPY --from=agent_challenge docker/canonical/requirements.txt /app/canonical-requirements.txt
RUN --mount=type=cache,target=/root/.cache/pip \
pip install --no-compile --require-hashes --requirement /app/canonical-requirements.txt

COPY --from=agent_challenge pyproject.toml uv.lock README.md /app/
COPY --from=agent_challenge src /app/src

# Golden assets from ORAS-pulled build-context only (T0 §5.4).
COPY --from=golden_assets golden/dataset-digest.json \
golden/live-registry-refs.json \
golden/tbench-2.1-oracle.json.enc \
/opt/agent-challenge/golden/
RUN ln -sfn /opt/agent-challenge/golden /app/golden
COPY --from=golden_assets docker/canonical/live-task-cache/ \
/opt/agent-challenge/task-cache/

ENTRYPOINT ["python", "-m", "agent_challenge.canonical.entrypoint"]
CMD ["--help"]
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,10 @@ dry-run paths only. Never print secrets.
| Self-deploy module | `python -m agent_challenge.selfdeploy` |
| Challenge repo | [BaseIntelligence/agent-challenge](https://github.com/BaseIntelligence/agent-challenge) |
| Measured OpenRouter model (docs pin) | `x-ai/grok-4.5` |
| Measured eval image | `ghcr.io/baseintelligence/agent-challenge-canonical` (digest-only pins) |
| Golden assets (build input) | `ghcr.io/baseintelligence/agent-challenge-golden-assets@sha256:…` via `GOLDEN_ASSETS_REF` |

See **[docs/eval-image.md](docs/eval-image.md)** for Dockerfile / ORAS co-pin (`AGENT_CHALLENGE_REF` + `GOLDEN_ASSETS_REF`) and reproducible build flags.
| Default Phala money cap | `$20` (CPU TDX; prefer `tdx.small`) |
| Public API | `https://chain.joinbase.ai` |
| Twin recipe (PRISM) | [BaseIntelligence/prism-recipe](https://github.com/BaseIntelligence/prism-recipe) |
Expand Down
59 changes: 59 additions & 0 deletions docs/eval-image.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
# Measured eval image (agent-recipe)

This repository owns the **Dockerfile + GHCR publish workflow** for the measured
agent-challenge eval / orchestrator image (plan D7 / T0–T1).

| Image | Purpose |
| --- | --- |
| `ghcr.io/baseintelligence/agent-challenge-canonical` | Runtime eval image (digest-only pins in compose / validator) |
| `ghcr.io/baseintelligence/agent-challenge-golden-assets` | **Build input only** (ORAS artefact). Owned and published by [agent-challenge](https://github.com/BaseIntelligence/agent-challenge). |

## Co-pins (required)

Builds must set **both** of these together (same freeze):

| Env | Meaning |
| --- | --- |
| `AGENT_CHALLENGE_REF` | Git SHA or tag of `BaseIntelligence/agent-challenge` (source + `docker/canonical/requirements.txt`) |
| `GOLDEN_ASSETS_REF` | Digest-pinned OCI ref, e.g. `ghcr.io/baseintelligence/agent-challenge-golden-assets@sha256:…` |

Never pin golden assets by floating tag alone. Never commit `golden/` or
`live-task-cache/` into this git tree.

## Local / CI build shape

```bash
# 1) Pull golden assets (after agent-challenge has published them)
oras pull "$GOLDEN_ASSETS_REF" -o /tmp/golden-assets

# 2) Checkout agent-challenge at the co-pinned SHA
git clone https://github.com/BaseIntelligence/agent-challenge.git /tmp/agent-challenge
git -C /tmp/agent-challenge checkout "$AGENT_CHALLENGE_REF"

# 3) Reproducible buildx (matches agent_challenge.canonical.build)
export SOURCE_DATE_EPOCH=1700000000
docker buildx build \
--provenance=false --sbom=false \
--build-arg SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH \
--build-context golden_assets=/tmp/golden-assets \
--build-context agent_challenge=/tmp/agent-challenge \
--output type=image,name=ghcr.io/baseintelligence/agent-challenge-canonical:local,push=false,rewrite-timestamp=true \
-f Dockerfile .
```

Workflow: [`.github/workflows/publish-eval-image.yml`](../.github/workflows/publish-eval-image.yml).

## Package visibility

After the first successful GHCR push, set package
**agent-challenge-canonical** to **Public** so miners can pull by digest without
credentials. Ciphertext golden may appear in image layers (miner-visible by
design); the decryption key never ships in git or the image.

## ENTRYPOINT

Unchanged from agent-challenge canonical:

```text
["python", "-m", "agent_challenge.canonical.entrypoint"]
```
4 changes: 2 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name = "agent-recipe"
version = "0.1.0"
description = "Miner-facing agent-challenge recipe: baseagent template + Phala self-deploy attestation path"
readme = "README.md"
requires-python = ">=3.11"
requires-python = ">=3.12"
license = { text = "Apache-2.0" }
authors = [
{ name = "BaseIntelligence" },
Expand Down Expand Up @@ -37,7 +37,7 @@ pythonpath = ["src"]

[tool.ruff]
line-length = 100
target-version = "py311"
target-version = "py312"

[tool.ruff.lint]
select = ["E", "F", "I", "UP", "B"]
Expand Down
Loading