From 10d7fbede54e3ee0e0ecd06687831f756a0408c4 Mon Sep 17 00:00:00 2001 From: Rashid Kaleem <230885705+arekay-nv@users.noreply.github.com> Date: Thu, 9 Jul 2026 11:17:42 -0500 Subject: [PATCH 1/3] Testing workflow to push images Signed-off-by: Rashid Kaleem <230885705+arekay-nv@users.noreply.github.com> --- .github/workflows/publish-image.yml | 82 +++++++++++ scripts/push_docker_image.sh | 208 ++++++++++++++++++++++++++++ 2 files changed, 290 insertions(+) create mode 100644 .github/workflows/publish-image.yml create mode 100755 scripts/push_docker_image.sh diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml new file mode 100644 index 00000000..1474eb29 --- /dev/null +++ b/.github/workflows/publish-image.yml @@ -0,0 +1,82 @@ +# Manually-triggered build+push of the endpoints client image to GHCR. +# +# Run it from the Actions tab ("Run workflow"): pick the branch/tag in the UI (that +# selection is github.sha), or type an explicit SHA/branch/tag in the `ref` input. +# The image is pushed to ghcr.io/mlcommons/endpoints tagged with the short commit +# SHA and the ref name, using the automatic GITHUB_TOKEN (no stored secret needed). +name: Publish client image + +on: + workflow_dispatch: + inputs: + ref: + description: "Git ref to build (branch/tag/SHA). Blank = the ref selected above." + required: false + type: string + platforms: + description: "Target platform(s)" + required: true + default: linux/amd64 + type: choice + options: + - linux/amd64 + - linux/amd64,linux/arm64 + provision_dsr1: + description: "Bake in the DeepSeek-R1 accuracy evaluator (heavier; needs build-time network)" + required: true + default: "1" + type: choice + options: + - "1" + - "0" + no_cache: + description: "Build with --no-cache (clean/reproducible, slower)" + required: true + default: "true" + type: choice + options: + - "true" + - "false" + +# GITHUB_TOKEN needs packages:write to push to the org's GHCR package. +permissions: + contents: read + packages: write + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - name: Checkout selected ref + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # Check out the ref *name* (not github.sha) so a local branch head exists + # and the script can resolve/tag it; blank input falls back to the + # dispatched branch/tag. Full history for `git rev-parse --short`. + ref: ${{ inputs.ref || github.ref_name }} + fetch-depth: 0 + + # Registers QEMU binfmt handlers so the docker-container builder can build + # the non-native arch when platforms includes linux/arm64. + - name: Set up QEMU + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + # Logs in as the user who triggered the run; the password is this run's + # automatic GITHUB_TOKEN (scoped by the permissions block above). + - name: Log in to GitHub Container Registry + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + env: + ENDPOINTS_REF: ${{ inputs.ref || github.ref_name }} + PLATFORM: ${{ inputs.platforms }} + PROVISION_DSR1: ${{ inputs.provision_dsr1 }} + NO_CACHE: ${{ inputs.no_cache == 'true' && '1' || '0' }} + run: ./scripts/push_docker_image.sh diff --git a/scripts/push_docker_image.sh b/scripts/push_docker_image.sh new file mode 100755 index 00000000..dde371e1 --- /dev/null +++ b/scripts/push_docker_image.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash +# push_docker_image.sh — build and push the endpoints client image to a registry. +# +# Builds scripts/Dockerfile.dev at a given git ref and pushes it to GitHub +# Container Registry (default: ghcr.io/mlcommons/endpoints), tagged with both the +# short commit SHA and the ref name so consumers can pin either. +# +# Prerequisites: +# - docker with buildx (the script provisions a docker-container builder; the +# default 'docker' driver can neither --push nor build multi-platform). +# - Authenticated to the target registry. Either run `docker login ghcr.io` +# yourself, or export GHCR_USER + GHCR_TOKEN (a GitHub PAT / GITHUB_TOKEN with +# write:packages) and this script logs in for you. +# +# Usage: +# ./scripts/push_docker_image.sh # build+push current HEAD (linux/amd64) +# ENDPOINTS_REF=v1.2.0 ./scripts/push_docker_image.sh # build+push a specific tag/branch/sha +# ./scripts/push_docker_image.sh --ref main # same, via flag +# ./scripts/push_docker_image.sh --multi-arch # linux/amd64,linux/arm64 manifest list +# ./scripts/push_docker_image.sh --platform linux/arm64 # single non-native arch +# ./scripts/push_docker_image.sh --cache # allow layer cache (default: --no-cache) +# ./scripts/push_docker_image.sh --multi-arch --no-push # dry build (validate both arches, publish nothing) +# ./scripts/push_docker_image.sh --allow-dirty # build HEAD despite uncommitted tracked changes +# +# Multi-arch note: --multi-arch (or --platform with a comma) pushes a manifest +# list usable on BOTH x86 and arm hosts; Docker pulls the matching arch. The +# non-native leg needs QEMU registered on the host once: +# docker run --privileged --rm tonistiigi/binfmt --install all +# Dockerfile.dev's DSR1 provisioning runs under emulation and is MUCH slower. +# +# Environment variables (flags take precedence): +# IMAGE full image ref w/o tag (default: ghcr.io/mlcommons/endpoints) +# ENDPOINTS_REF git ref to build (default: current HEAD; checked out if it differs) +# PLATFORM target platform(s) (default: linux/amd64) +# DOCKERFILE Dockerfile path (default: scripts/Dockerfile.dev) +# NO_CACHE 1=--no-cache, 0=cache (default: 1) +# ALLOW_DIRTY 1=build HEAD with uncommitted tracked changes (default: 0) +# PROVISION_DSR1 passthrough build-arg (default: Dockerfile default = 1) +# GHCR_USER/GHCR_TOKEN optional registry login (fallback: GITHUB_ACTOR/GITHUB_TOKEN) +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BUILDX_BUILDER="endpoints-buildx" + +usage() { + sed -n '2,/^set -euo/p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//; /^set -euo/d' +} + +IMAGE="${IMAGE:-ghcr.io/mlcommons/endpoints}" +ENDPOINTS_REF="${ENDPOINTS_REF:-}" +PLATFORM="${PLATFORM:-linux/amd64}" +DOCKERFILE="${DOCKERFILE:-scripts/Dockerfile.dev}" +NO_CACHE="${NO_CACHE:-1}" +ALLOW_DIRTY="${ALLOW_DIRTY:-0}" +PUSH=1 + +while [[ $# -gt 0 ]]; do + case "$1" in + --ref) + [[ -n "${2:-}" && "$2" != --* ]] || { echo "error: --ref requires a value" >&2; exit 1; } + ENDPOINTS_REF="$2"; shift ;; + --ref=*) ENDPOINTS_REF="${1#*=}" ;; + --image) + [[ -n "${2:-}" && "$2" != --* ]] || { echo "error: --image requires a value" >&2; exit 1; } + IMAGE="$2"; shift ;; + --image=*) IMAGE="${1#*=}" ;; + --platform) + [[ -n "${2:-}" && "$2" != --* ]] || { echo "error: --platform requires a value, e.g. linux/arm64" >&2; exit 1; } + PLATFORM="$2"; shift ;; + --platform=*) PLATFORM="${1#*=}" ;; + --multi-arch) PLATFORM="linux/amd64,linux/arm64" ;; + --no-cache) NO_CACHE=1 ;; + --cache) NO_CACHE=0 ;; + --no-push) PUSH=0 ;; + --allow-dirty) ALLOW_DIRTY=1 ;; + --dockerfile) + [[ -n "${2:-}" && "$2" != --* ]] || { echo "error: --dockerfile requires a value" >&2; exit 1; } + DOCKERFILE="$2"; shift ;; + --dockerfile=*) DOCKERFILE="${1#*=}" ;; + -h | --help) usage; exit 0 ;; + *) echo "error: unknown argument '$1'" >&2; usage; exit 1 ;; + esac + shift +done + +cd "$REPO_ROOT" + +if ! docker buildx version >/dev/null 2>&1; then + echo "error: 'docker buildx' is required but not available." >&2 + exit 1 +fi + +# --------------------------------------------------------------------------- +# Optional registry login. Skipped silently if no token is provided (assumes the +# caller already ran `docker login`). Registry host is the first path segment of +# IMAGE (e.g. ghcr.io from ghcr.io/mlcommons/endpoints). +# --------------------------------------------------------------------------- +REGISTRY_HOST="${IMAGE%%/*}" +LOGIN_USER="${GHCR_USER:-${GITHUB_ACTOR:-}}" +LOGIN_TOKEN="${GHCR_TOKEN:-${GITHUB_TOKEN:-}}" +if [[ "$PUSH" == "1" && -n "$LOGIN_TOKEN" ]]; then + echo ">> Logging in to ${REGISTRY_HOST} as ${LOGIN_USER:-}" + echo "$LOGIN_TOKEN" | docker login "$REGISTRY_HOST" -u "${LOGIN_USER:-x-access-token}" --password-stdin +fi + +# --------------------------------------------------------------------------- +# Resolve the ref to build. If ENDPOINTS_REF is set and points at a different +# commit than the current HEAD, check it out (requiring a clean tree) and restore +# the original ref on exit so the working copy is left as we found it. +# --------------------------------------------------------------------------- +ORIGINAL_REF="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || true)" +[[ "$ORIGINAL_REF" == "HEAD" ]] && ORIGINAL_REF="$(git rev-parse HEAD)" + +restore_ref() { + if [[ -n "${DID_CHECKOUT:-}" && -n "$ORIGINAL_REF" ]]; then + echo ">> Restoring original ref ${ORIGINAL_REF}" + git checkout --quiet "$ORIGINAL_REF" + fi +} +trap restore_ref EXIT + +# Uncommitted changes to TRACKED files (staged or unstaged) make the working tree +# differ from HEAD. Untracked files are ignored here: they are not part of any commit +# and .dockerignore governs the build context. +tree_has_tracked_changes() { ! git diff --quiet HEAD 2>/dev/null; } + +if [[ -n "$ENDPOINTS_REF" ]]; then + # Fall back to the remote-tracking ref: after actions/checkout lands a detached + # SHA, a bare branch name like 'main' exists only as origin/main, so the plain + # resolve would fail and abort the CI default-dispatch path. + target_sha="$(git rev-parse --verify "${ENDPOINTS_REF}^{commit}" 2>/dev/null \ + || git rev-parse --verify "origin/${ENDPOINTS_REF}^{commit}" 2>/dev/null || true)" + [[ -n "$target_sha" ]] || { echo "error: '$ENDPOINTS_REF' is not a valid git ref." >&2; exit 1; } + if [[ "$target_sha" != "$(git rev-parse HEAD)" ]]; then + if tree_has_tracked_changes; then + echo "error: working tree has uncommitted changes to tracked files; commit/stash before building a different ref ($ENDPOINTS_REF)." >&2 + exit 1 + fi + echo ">> Checking out ${ENDPOINTS_REF}" + git checkout --quiet "$ENDPOINTS_REF" + DID_CHECKOUT=1 + fi +else + # Tag the built image with the current branch/ref name for readability. + ENDPOINTS_REF="$ORIGINAL_REF" +fi + +SHORT_SHA="$(git rev-parse --short HEAD)" +# Docker tags allow only [A-Za-z0-9_.-], must not start with '.' or '-', and cap at +# 128 chars. Map every other char (e.g. '/', '+', '~', '#') to '-' so a legitimate +# git ref name can't yield an invalid tag that fails `-t` only after the whole build. +REF_TAG="$(printf '%s' "$ENDPOINTS_REF" | tr -c 'A-Za-z0-9_.-' '-')" +[[ "$REF_TAG" == [A-Za-z0-9_]* ]] || REF_TAG="_${REF_TAG}" +REF_TAG="${REF_TAG:0:128}" + +# Building the working tree as-is (no distinct ref checked out): don't publish a +# :$SHORT_SHA tag whose '.' build context doesn't match commit $SHORT_SHA. +if [[ -z "${DID_CHECKOUT:-}" && "$ALLOW_DIRTY" != "1" ]] && tree_has_tracked_changes; then + dirty_msg="working tree has uncommitted changes to tracked files; :${SHORT_SHA} would not match commit ${SHORT_SHA}" + if [[ "$PUSH" == "1" ]]; then + echo "error: ${dirty_msg}." >&2 + echo " Commit/stash the changes, or pass --allow-dirty to publish anyway." >&2 + exit 1 + fi + echo ">> warning: ${dirty_msg} (dry run; nothing pushed)." >&2 +fi + +# --------------------------------------------------------------------------- +# Ensure a docker-container builder (supports --push and multi-platform). +# --------------------------------------------------------------------------- +if ! docker buildx inspect "$BUILDX_BUILDER" >/dev/null 2>&1; then + echo ">> Creating buildx builder '${BUILDX_BUILDER}' (docker-container driver)" + docker buildx create --name "$BUILDX_BUILDER" --driver docker-container >/dev/null +fi + +if [[ "$PLATFORM" == *,* ]]; then + echo ">> Multi-arch build for ${PLATFORM} — requires QEMU for non-native archs" + echo " (register once with: docker run --privileged --rm tonistiigi/binfmt --install all)" +fi + +BUILD_ARGS=() +[[ "$NO_CACHE" == "1" ]] && BUILD_ARGS+=(--no-cache) +[[ -n "${PROVISION_DSR1:-}" ]] && BUILD_ARGS+=(--build-arg "PROVISION_DSR1=${PROVISION_DSR1}") +# With --no-push we build to cache and discard (validation only). A multi-platform +# result cannot be --load-ed into the local store, so no output flag is added. +[[ "$PUSH" == "1" ]] && BUILD_ARGS+=(--push) + +BUILD_VERB=$([[ "$PUSH" == "1" ]] && echo "and pushing" || echo "(dry run, --no-push)") +echo ">> Building ${IMAGE}:${SHORT_SHA} (ref: ${ENDPOINTS_REF}) for ${PLATFORM} ${BUILD_VERB}" +docker buildx build \ + --builder "$BUILDX_BUILDER" \ + --platform "$PLATFORM" \ + -f "$DOCKERFILE" \ + -t "${IMAGE}:${SHORT_SHA}" \ + -t "${IMAGE}:${REF_TAG}" \ + ${BUILD_ARGS[@]+"${BUILD_ARGS[@]}"} \ + . + +echo +if [[ "$PUSH" == "1" ]]; then + echo "Done. Pushed:" + echo " ${IMAGE}:${SHORT_SHA}" + echo " ${IMAGE}:${REF_TAG}" + echo "Pull with:" + echo " docker pull ${IMAGE}:${SHORT_SHA}" +else + echo "Done. Dry build succeeded for ${PLATFORM} (nothing pushed)." +fi From 097ee8735b3734c53e114a15037ac355d2c9f843 Mon Sep 17 00:00:00 2001 From: Rashid Kaleem <230885705+arekay-nv@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:14:47 -0500 Subject: [PATCH 2/3] Add metadata Signed-off-by: Rashid Kaleem <230885705+arekay-nv@users.noreply.github.com> --- scripts/push_docker_image.sh | 30 +++++++++++++++++++ .../livecodebench/lcb_serve.dockerfile | 9 ++++++ .../evaluation/livecodebench/push_image.sh | 20 +++++++++++-- 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/scripts/push_docker_image.sh b/scripts/push_docker_image.sh index dde371e1..465d00ee 100755 --- a/scripts/push_docker_image.sh +++ b/scripts/push_docker_image.sh @@ -36,6 +36,7 @@ # NO_CACHE 1=--no-cache, 0=cache (default: 1) # ALLOW_DIRTY 1=build HEAD with uncommitted tracked changes (default: 0) # PROVISION_DSR1 passthrough build-arg (default: Dockerfile default = 1) +# IMAGE_DESCRIPTION GHCR package-page description (default: "endpoints client image (ref , commit )") # GHCR_USER/GHCR_TOKEN optional registry login (fallback: GITHUB_ACTOR/GITHUB_TOKEN) set -euo pipefail @@ -181,10 +182,39 @@ fi BUILD_ARGS=() [[ "$NO_CACHE" == "1" ]] && BUILD_ARGS+=(--no-cache) [[ -n "${PROVISION_DSR1:-}" ]] && BUILD_ARGS+=(--build-arg "PROVISION_DSR1=${PROVISION_DSR1}") + +# Don't attach SLSA provenance attestations. buildx adds them by default on --push, +# which surfaces a spurious `unknown/unknown` entry on the GHCR package page and +# wraps even a single-arch build in an image index. Opting out yields a clean, +# single-manifest artifact; the source/revision/version annotations below preserve +# the "which commit built this" traceability the attestation would have carried. +BUILD_ARGS+=(--provenance=false) + # With --no-push we build to cache and discard (validation only). A multi-platform # result cannot be --load-ed into the local store, so no output flag is added. [[ "$PUSH" == "1" ]] && BUILD_ARGS+=(--push) +# OCI annotations for the GHCR package page (description + repo/commit provenance). +# --annotation requires the build to actually produce the component named by the +# level prefix, so gate on --push and pick the level by artifact shape: a multi-arch +# build pushes an image index (GHCR reads the description from the index); a +# single-arch build (provenance off, above) pushes a lone manifest, for which +# `index:` errors with "index annotations not supported for single platform export". +if [[ "$PUSH" == "1" ]]; then + if [[ "$PLATFORM" == *,* ]]; then + ANNOTATION_LEVEL="index" + else + ANNOTATION_LEVEL="manifest" + fi + IMAGE_DESCRIPTION="${IMAGE_DESCRIPTION:-endpoints client image (ref ${ENDPOINTS_REF}, commit ${SHORT_SHA})}" + BUILD_ARGS+=( + --annotation "${ANNOTATION_LEVEL}:org.opencontainers.image.source=https://github.com/mlcommons/endpoints" + --annotation "${ANNOTATION_LEVEL}:org.opencontainers.image.revision=${SHORT_SHA}" + --annotation "${ANNOTATION_LEVEL}:org.opencontainers.image.version=${ENDPOINTS_REF}" + --annotation "${ANNOTATION_LEVEL}:org.opencontainers.image.description=${IMAGE_DESCRIPTION}" + ) +fi + BUILD_VERB=$([[ "$PUSH" == "1" ]] && echo "and pushing" || echo "(dry run, --no-push)") echo ">> Building ${IMAGE}:${SHORT_SHA} (ref: ${ENDPOINTS_REF}) for ${PLATFORM} ${BUILD_VERB}" docker buildx build \ diff --git a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.dockerfile b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.dockerfile index 83b1b79d..a8f80c95 100644 --- a/src/inference_endpoint/evaluation/livecodebench/lcb_serve.dockerfile +++ b/src/inference_endpoint/evaluation/livecodebench/lcb_serve.dockerfile @@ -63,6 +63,15 @@ COPY _server.py /app/server.py # Make lcb_serve.py available as a module ENV PYTHONPATH="/app" +# Provenance: the endpoints repo commit this image was built from, supplied by +# push_image.sh via --build-arg. Recorded as a config LABEL (not a manifest +# annotation) so it survives the oci-mediatypes=false push and is visible in +# `docker inspect`. Declared after the COPYs so a new SHA only rebuilds this +# metadata layer, never the expensive dataset-generation stage above. +ARG ENDPOINTS_SHA=unknown +LABEL org.opencontainers.image.source="https://github.com/mlcommons/endpoints" \ + org.opencontainers.image.revision="${ENDPOINTS_SHA}" + # Launch the WebSocket server with long-running connection support # Default port 13835 # - timeout-keep-alive: Allow connections to stay open for hours diff --git a/src/inference_endpoint/evaluation/livecodebench/push_image.sh b/src/inference_endpoint/evaluation/livecodebench/push_image.sh index 3e92af49..f9738d4f 100755 --- a/src/inference_endpoint/evaluation/livecodebench/push_image.sh +++ b/src/inference_endpoint/evaluation/livecodebench/push_image.sh @@ -83,6 +83,20 @@ done # shellcheck source=_image_env.sh source "${SCRIPT_DIR}/_image_env.sh" +# ---------------------------------------------------------------------------- +# Provenance: the endpoints repo commit this image is built from, baked into the +# image config as an OCI label by both build paths below. A config LABEL (not a +# manifest --annotation) is deliberate: the buildx push sets oci-mediatypes=false, +# and Docker-media-type manifests have no annotations field, whereas config labels +# are representable in both formats and survive `docker inspect`. Scope the dirty +# check to this build context so unrelated edits elsewhere in the repo don't mark +# the image -dirty. +# ---------------------------------------------------------------------------- +ENDPOINTS_SHA="$(git -C "$SCRIPT_DIR" rev-parse --short HEAD 2>/dev/null || echo unknown)" +if [[ "$ENDPOINTS_SHA" != "unknown" ]] && ! git -C "$SCRIPT_DIR" diff --quiet HEAD -- "$SCRIPT_DIR" 2>/dev/null; then + ENDPOINTS_SHA="${ENDPOINTS_SHA}-dirty" +fi + # ---------------------------------------------------------------------------- # Cross-architecture path: build with buildx and push directly to the registry. # A non-native image cannot be `docker load`-ed into the local store, so buildx @@ -127,12 +141,13 @@ if [[ -n "$PLATFORM" ]]; then exit 1 fi - echo ">> Building ${LCB_IMAGE_REF} for ${PLATFORM} and pushing (buildx) ..." + echo ">> Building ${LCB_IMAGE_REF} for ${PLATFORM} (endpoints ${ENDPOINTS_SHA}) and pushing (buildx) ..." docker buildx build \ --builder "$BUILDX_BUILDER" \ --platform "$PLATFORM" \ -f "${SCRIPT_DIR}/lcb_serve.dockerfile" \ --secret id=HF_TOKEN,env=HF_TOKEN \ + --build-arg "ENDPOINTS_SHA=${ENDPOINTS_SHA}" \ -t "$LCB_IMAGE_REF" \ --provenance=false \ --output "type=image,push=true,compression=gzip,force-compression=true,oci-mediatypes=false" \ @@ -147,10 +162,11 @@ else exit 1 fi - echo ">> Building ${LCB_LOCAL_TAG} ..." + echo ">> Building ${LCB_LOCAL_TAG} (endpoints ${ENDPOINTS_SHA}) ..." docker build \ -f "${SCRIPT_DIR}/lcb_serve.dockerfile" \ --secret id=HF_TOKEN,env=HF_TOKEN \ + --build-arg "ENDPOINTS_SHA=${ENDPOINTS_SHA}" \ -t "$LCB_LOCAL_TAG" \ "$SCRIPT_DIR" fi From da7d3a9119ef9d33c7550ed2a62555bae7637a57 Mon Sep 17 00:00:00 2001 From: Rashid Kaleem <230885705+arekay-nv@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:31:35 -0500 Subject: [PATCH 3/3] More fixups. Signed-off-by: Rashid Kaleem <230885705+arekay-nv@users.noreply.github.com> --- .../evaluation/livecodebench/README.md | 35 ++++---- .../evaluation/livecodebench/_image_env.sh | 13 ++- .../evaluation/livecodebench/pull_image.sh | 12 +-- .../evaluation/livecodebench/push_image.sh | 80 ++++++++++++++++--- 4 files changed, 105 insertions(+), 35 deletions(-) diff --git a/src/inference_endpoint/evaluation/livecodebench/README.md b/src/inference_endpoint/evaluation/livecodebench/README.md index a5b7fb5d..07e6cf42 100644 --- a/src/inference_endpoint/evaluation/livecodebench/README.md +++ b/src/inference_endpoint/evaluation/livecodebench/README.md @@ -182,14 +182,14 @@ consumer can **pull and run it with no `HF_TOKEN` and no rebuild**. Two scripts in this directory wrap the docker build/tag/push/pull steps. Both resolve the image reference from environment variables (shared via `_image_env.sh`): -| Variable | Required | Default | Meaning | -| -------------------- | -------- | -------------------- | --------------------------------------------------------------------------- | -| `LCB_IMAGE_REGISTRY` | yes | — | registry + namespace, e.g. `myregistry.com/team` | -| `LCB_IMAGE_NAME` | no | `lcb-service` | image repo name | -| `LCB_IMAGE_TAG` | no | `release_v6` | tag; defaults to the baked-in dataset version so the tag is self-describing | -| `LCB_LOCAL_TAG` | no | `lcb-service:latest` | local tag the run command / scorer expect | +| Variable | Required | Default | Meaning | +| -------------------- | -------------------- | -------------------- | --------------------------------------------------------------------------------------------------------------------------- | +| `LCB_IMAGE_REGISTRY` | yes | — | registry + namespace, e.g. `myregistry.com/team` | +| `LCB_IMAGE_NAME` | no | `lcb-service` | image repo name | +| `LCB_IMAGE_TAG` | push: no · pull: yes | endpoints short SHA | image tag; `push_image.sh` defaults it to the endpoints commit SHA (one immutable tag per build). Pull must name the build. | +| `LCB_LOCAL_TAG` | no | `lcb-service:latest` | local tag the run command / scorer expect | -The resolved remote reference is `${LCB_IMAGE_REGISTRY}/${LCB_IMAGE_NAME}:${LCB_IMAGE_TAG}`. +The resolved remote reference is `${LCB_IMAGE_REGISTRY}/${LCB_IMAGE_NAME}:${LCB_IMAGE_TAG}`. `push_image.sh` sets `LCB_IMAGE_TAG` to the endpoints commit short SHA by default, so each build publishes an immutable `…/lcb-service:` — there is no moving `latest`/`release_v6` tag. The same SHA is also baked into the image as the `org.opencontainers.image.revision` label. #### Push (maintainer) @@ -200,8 +200,14 @@ Requires `docker login dhi.io` (base images) and `docker login` to your target r LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN= \ ./push_image.sh -# Or push an already-built local image without rebuilding: -LCB_IMAGE_REGISTRY=myregistry.com/team ./push_image.sh --no-build +# Or push an already-built local image without rebuilding (name the build explicitly): +LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./push_image.sh --no-build +``` + +Each build publishes an immutable `:` tag, so **re-pushing an existing `:` is refused** to protect it. Pass `--force` to overwrite deliberately (e.g. re-running a partially failed push): + +```bash +LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN= ./push_image.sh --force ``` **Cross-architecture builds.** To build for an architecture other than the host's (e.g. build `arm64` on an @@ -227,11 +233,12 @@ docker run --privileged --rm tonistiigi/binfmt --install all #### Pull (consumer / eval side) -No `HF_TOKEN` needed. By default the image is re-tagged locally as `lcb-service:latest`, so the -[hardened run command](#hardened-run-command) and the scorer's `ws://localhost:13835/evaluate` expectation work unchanged. +No `HF_TOKEN` needed. Set `LCB_IMAGE_TAG` to the build (short SHA) you want to pull. By default the image is re-tagged +locally as `lcb-service:latest`, so the [hardened run command](#hardened-run-command) and the scorer's +`ws://localhost:13835/evaluate` expectation work unchanged. ```bash -LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh +LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh ``` ### (Only if using enroot) Generating a .sqsh file for enroot @@ -240,8 +247,8 @@ The pull script can produce an enroot `.sqsh` from the pulled image with the `-- [enroot](https://github.com/NVIDIA/enroot/tree/main) (e.g. SLURM clusters): ```bash -LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh --sqsh # writes lcb_service.sqsh -LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh --sqsh out.sqsh # custom output path +LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh --sqsh # writes lcb_service.sqsh +LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh --sqsh out.sqsh # custom output path ``` This runs `enroot import --output dockerd://${LCB_IMAGE_REF}` on the just-pulled image. Running the service via enroot is diff --git a/src/inference_endpoint/evaluation/livecodebench/_image_env.sh b/src/inference_endpoint/evaluation/livecodebench/_image_env.sh index ba615cb6..32464b83 100644 --- a/src/inference_endpoint/evaluation/livecodebench/_image_env.sh +++ b/src/inference_endpoint/evaluation/livecodebench/_image_env.sh @@ -8,7 +8,7 @@ # Inputs (environment variables): # LCB_IMAGE_REGISTRY (required) registry + namespace, e.g. myregistry.com/team # LCB_IMAGE_NAME (optional) image repo name (default: lcb-service) -# LCB_IMAGE_TAG (optional) tag (default: release_v6) +# LCB_IMAGE_TAG (required) tag — push_image.sh defaults it to the endpoints short SHA # LCB_LOCAL_TAG (optional) local tag used by run/scorer (default: lcb-service:latest) # # Exports: @@ -23,8 +23,15 @@ if [[ -z "${LCB_IMAGE_REGISTRY:-}" ]]; then fi LCB_IMAGE_NAME="${LCB_IMAGE_NAME:-lcb-service}" -# Defaults to the baked-in dataset version so the artifact is self-describing. -LCB_IMAGE_TAG="${LCB_IMAGE_TAG:-release_v6}" +# Images are tagged by the endpoints commit SHA (one immutable tag per build), so +# there is no channel default. push_image.sh sets this to the SHA automatically; +# a consumer pulling must name the specific build. +if [[ -z "${LCB_IMAGE_TAG:-}" ]]; then + echo "error: LCB_IMAGE_TAG is not set." >&2 + echo " push_image.sh defaults it to the endpoints commit SHA; for pull, set it to" >&2 + echo " the build you want, e.g. LCB_IMAGE_TAG=\$(git rev-parse --short HEAD)" >&2 + return 1 2>/dev/null || exit 1 +fi LCB_LOCAL_TAG="${LCB_LOCAL_TAG:-lcb-service:latest}" # Strip any trailing slash on the registry to avoid a double slash in the ref. diff --git a/src/inference_endpoint/evaluation/livecodebench/pull_image.sh b/src/inference_endpoint/evaluation/livecodebench/pull_image.sh index dc679d78..3b3127d2 100755 --- a/src/inference_endpoint/evaluation/livecodebench/pull_image.sh +++ b/src/inference_endpoint/evaluation/livecodebench/pull_image.sh @@ -6,13 +6,13 @@ # lcb-service:latest so the hardened run command in README.md and the scorer's # ws://localhost:13835/evaluate expectation work unchanged. # -# Usage: -# LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh -# LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh --sqsh # also create lcb_service.sqsh -# LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh --sqsh out.sqsh # enroot import to out.sqsh -# LCB_IMAGE_REGISTRY=myregistry.com/team ./pull_image.sh --no-local-tag # skip the lcb-service:latest tag +# Usage (images are tagged by short SHA, so LCB_IMAGE_TAG selects the build): +# LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh +# LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh --sqsh # also create lcb_service.sqsh +# LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh --sqsh out.sqsh # enroot import to out.sqsh +# LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./pull_image.sh --no-local-tag # skip the lcb-service:latest tag # -# Environment variables: see _image_env.sh (LCB_IMAGE_REGISTRY required). +# Environment variables: see _image_env.sh (LCB_IMAGE_REGISTRY + LCB_IMAGE_TAG required). set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/src/inference_endpoint/evaluation/livecodebench/push_image.sh b/src/inference_endpoint/evaluation/livecodebench/push_image.sh index f9738d4f..c83ba77c 100755 --- a/src/inference_endpoint/evaluation/livecodebench/push_image.sh +++ b/src/inference_endpoint/evaluation/livecodebench/push_image.sh @@ -5,13 +5,17 @@ # build time. Pushing the built image lets consumers pull-and-run with no HF_TOKEN # and no rebuild (see pull_image.sh). # +# The image is tagged by the endpoints commit short SHA — one immutable tag per +# build. LCB_IMAGE_TAG overrides it. There is no moving channel tag. +# # Prerequisites: # - docker login dhi.io (base images are Docker Hardened Images) # - docker login (target for the push) # # Usage: # LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN=hf_xxx ./push_image.sh -# LCB_IMAGE_REGISTRY=myregistry.com/team ./push_image.sh --no-build # push existing local image +# LCB_IMAGE_REGISTRY=myregistry.com/team LCB_IMAGE_TAG= ./push_image.sh --no-build # push existing local image +# ... --force # overwrite an existing : tag (default: refuse) # # Cross-architecture build (e.g. build arm64 on an x86 node, or a multi-arch manifest): # LCB_IMAGE_REGISTRY=myregistry.com/team HF_TOKEN=hf_xxx ./push_image.sh --platform linux/arm64 @@ -54,10 +58,12 @@ unsupported_platforms() { platform_supported() { [[ -z "$(unsupported_platforms "$1" "$2")" ]]; } NO_BUILD=0 +FORCE=0 PLATFORM="${LCB_IMAGE_PLATFORM:-}" while [[ $# -gt 0 ]]; do case "$1" in --no-build) NO_BUILD=1 ;; + --force) FORCE=1 ;; --platform) if [[ -z "${2:-}" || "$2" == --* ]]; then echo "error: --platform requires a value, e.g. linux/arm64" >&2 @@ -80,23 +86,71 @@ while [[ $# -gt 0 ]]; do shift done -# shellcheck source=_image_env.sh -source "${SCRIPT_DIR}/_image_env.sh" - # ---------------------------------------------------------------------------- -# Provenance: the endpoints repo commit this image is built from, baked into the -# image config as an OCI label by both build paths below. A config LABEL (not a -# manifest --annotation) is deliberate: the buildx push sets oci-mediatypes=false, -# and Docker-media-type manifests have no annotations field, whereas config labels -# are representable in both formats and survive `docker inspect`. Scope the dirty -# check to this build context so unrelated edits elsewhere in the repo don't mark -# the image -dirty. +# Provenance: the endpoints repo commit this image is built from. It is both baked +# into the image config as an OCI label (see --build-arg below) AND used as the +# image tag — one immutable tag per build. A config LABEL (not a manifest +# --annotation) is deliberate: the buildx push sets oci-mediatypes=false, and +# Docker-media-type manifests have no annotations field, whereas config labels are +# representable in both formats and survive `docker inspect`. Scope the dirty check +# to this build context so unrelated edits elsewhere in the repo don't mark it -dirty. # ---------------------------------------------------------------------------- ENDPOINTS_SHA="$(git -C "$SCRIPT_DIR" rev-parse --short HEAD 2>/dev/null || echo unknown)" if [[ "$ENDPOINTS_SHA" != "unknown" ]] && ! git -C "$SCRIPT_DIR" diff --quiet HEAD -- "$SCRIPT_DIR" 2>/dev/null; then ENDPOINTS_SHA="${ENDPOINTS_SHA}-dirty" fi +# Tag the image by the endpoints commit SHA (LCB_IMAGE_TAG overrides). The auto SHA +# tag is only trustworthy when the tree is clean AND we build now, so when the tag is +# not given explicitly, refuse the cases where : would misrepresent the image: +# - no git → nothing to name the image by +# - --no-build → pushes a pre-built local image whose baked revision label may be +# a different commit than HEAD; : would lie about provenance +# - dirty context → :-dirty is a moving, non-reproducible tag +# An explicit LCB_IMAGE_TAG is the escape hatch for all three. Set before sourcing so +# _image_env.sh builds LCB_IMAGE_REF from it. +if [[ -z "${LCB_IMAGE_TAG:-}" ]]; then + if [[ "$ENDPOINTS_SHA" == "unknown" ]]; then + echo "error: cannot determine a short SHA to tag the image (no git); set LCB_IMAGE_TAG." >&2 + exit 1 + fi + if [[ "$NO_BUILD" -eq 1 ]]; then + echo "error: --no-build pushes a pre-built local image whose baked revision may not match" >&2 + echo " HEAD (${ENDPOINTS_SHA}); set LCB_IMAGE_TAG explicitly to name it." >&2 + exit 1 + fi + if [[ "$ENDPOINTS_SHA" == *-dirty ]]; then + echo "error: build context has uncommitted changes; :${ENDPOINTS_SHA} is a moving tag." >&2 + echo " Commit the changes, or set LCB_IMAGE_TAG explicitly to push anyway." >&2 + exit 1 + fi +fi +export LCB_IMAGE_TAG="${LCB_IMAGE_TAG:-$ENDPOINTS_SHA}" + +# shellcheck source=_image_env.sh +source "${SCRIPT_DIR}/_image_env.sh" + +# Refuse to overwrite an existing remote tag unless --force: the SHA tag is meant to +# be immutable, so a second push to the same ref would silently replace a published +# image. `imagetools inspect` queries the registry (no layer pull). Fail CLOSED: a +# clean exit means the tag exists (block); a non-zero exit is trusted as "absent" ONLY +# when the output confirms not-found. Any other failure (buildx plugin missing, auth +# denied, registry/network error) blocks too — otherwise the guard silently no-ops on +# exactly the hosts/creds where it can't verify, and immutability goes unenforced. +# --force skips the check entirely. +if [[ "$FORCE" -eq 0 ]]; then + if inspect_out="$(docker buildx imagetools inspect "$LCB_IMAGE_REF" 2>&1)"; then + echo "error: ${LCB_IMAGE_REF} already exists in the registry." >&2 + echo " The SHA tag is meant to be immutable; re-run with --force to overwrite it." >&2 + exit 1 + elif ! grep -qiE 'not found|manifest unknown|manifest_unknown|name_unknown|no such manifest' <<<"$inspect_out"; then + echo "error: could not verify whether ${LCB_IMAGE_REF} already exists:" >&2 + printf ' %s\n' "$inspect_out" >&2 + echo " Fix registry access (or install docker buildx), or re-run with --force to skip this check." >&2 + exit 1 + fi +fi + # ---------------------------------------------------------------------------- # Cross-architecture path: build with buildx and push directly to the registry. # A non-native image cannot be `docker load`-ed into the local store, so buildx @@ -178,9 +232,11 @@ else docker push "$LCB_IMAGE_REF" fi +echo +echo "Pushed: ${LCB_IMAGE_REF}" echo echo "Done. Consumers can now pull with:" echo " LCB_IMAGE_REGISTRY=${LCB_IMAGE_REGISTRY%/} \\" [[ "$LCB_IMAGE_NAME" != "lcb-service" ]] && echo " LCB_IMAGE_NAME=${LCB_IMAGE_NAME} \\" -[[ "$LCB_IMAGE_TAG" != "release_v6" ]] && echo " LCB_IMAGE_TAG=${LCB_IMAGE_TAG} \\" +echo " LCB_IMAGE_TAG=${LCB_IMAGE_TAG} \\" echo " ./pull_image.sh"