diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d91e5c8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.git +.github +.codex +.agents +build_*.log +README.md diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a3849bb..c42b484 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -1,4 +1,5 @@ name: CI + on: push: branches: @@ -6,52 +7,363 @@ on: pull_request: branches: - main - schedule: - - cron: '0 7 * * WED' concurrency: - group: ${ {github.event_name }}-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{github.event_name == 'pull_request'}} + group: ${{ github.event_name }}-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: - contents: read + contents: write packages: write jobs: - CI: - continue-on-error: ${{ matrix.config.continue-on-error == 'true' }} - strategy: - matrix: - config: - - {dockerfile: 'ubuntu', tag: 'dev', build_args: 'TAG=dev'} + validate: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Run fast checks + env: + BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }} + shell: bash + run: | + set -euo pipefail + if [[ "$BASE_SHA" =~ ^0+$ ]]; then + BASE_SHA="$(git rev-list --max-parents=0 "$GITHUB_SHA")" + fi + PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -v + docker buildx bake --print + git diff --check "$BASE_SHA" "$GITHUB_SHA" + + detect: + runs-on: ubuntu-latest + outputs: + catalog_targets: ${{ steps.changes.outputs.catalog_targets }} + refresh_locks: ${{ steps.changes.outputs.refresh_locks }} + should_build: ${{ steps.changes.outputs.should_build }} + targets: ${{ steps.changes.outputs.targets }} + steps: + - name: Check out code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Detect affected image chain + id: changes + shell: bash + run: | + set -euo pipefail + + head_sha="$GITHUB_SHA" + if [ "${{ github.event_name }}" = "pull_request" ]; then + base_sha="${{ github.event.pull_request.base.sha }}" + else + base_sha="${{ github.event.before }}" + if [[ "$base_sha" =~ ^0+$ ]]; then + base_sha="$(git rev-list --max-parents=0 "$head_sha")" + fi + fi + + changed_files="$(mktemp)" + bake_json="$(mktemp)" + git diff --name-only "$base_sha" "$head_sha" > "$changed_files" + docker buildx bake --print > "$bake_json" + selection="$(python3 scripts/select_build_targets.py \ + --bake-json "$bake_json" \ + --changed-files "$changed_files")" + echo "$selection" + + catalog_targets="$(jq -r '.catalog_targets | join(" ")' <<< "$selection")" + targets="$(jq -r '.targets | join(" ")' <<< "$selection")" + refresh_locks="$(jq -r '.refresh_locks | join(" ")' <<< "$selection")" + + echo "catalog_targets=${catalog_targets}" >> "$GITHUB_OUTPUT" + echo "refresh_locks=${refresh_locks}" >> "$GITHUB_OUTPUT" + echo "targets=${targets}" >> "$GITHUB_OUTPUT" + if [ -n "$targets" ]; then + echo "should_build=true" >> "$GITHUB_OUTPUT" + else + echo "should_build=false" >> "$GITHUB_OUTPUT" + fi + + build: runs-on: ubuntu-latest - env: - docker-tag: ghcr.io/ornl-mdf/containers/${{ matrix.config.dockerfile }}:${{ matrix.config.tag }} + needs: + - validate + - detect + if: needs.detect.outputs.should_build == 'true' steps: - - name: Checkout out code - uses: actions/checkout@v3 + - name: Check out code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Github Container Registry - uses: docker/login-action@v2 + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + if: github.event_name == 'push' + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build new image and load it into Docker to test - uses: docker/build-push-action@v4 - with: - tags: ${{ env.docker-tag }} - file: docker/${{ matrix.config.dockerfile }} - build-args: ${{ matrix.config.build_args }} - pull: true - push: false - load: true - - name: Push Docker images for Github Container Registry - uses: docker/build-push-action@v3 - if: ${{ github.event_name == 'push' || github.event_name == 'schedule' }} - with: - tags: ${{ env.docker-tag }} - file: docker/${{ matrix.config.dockerfile }} - build-args: ${{ matrix.config.build_args }} - push: true + + - name: Allocate immutable release tag + if: github.event_name == 'push' + id: release + env: + CATALOG_TARGETS: ${{ needs.detect.outputs.catalog_targets }} + TARGETS: ${{ needs.detect.outputs.targets }} + shell: bash + run: | + set -euo pipefail + command=( + python3 scripts/select_release_tag.py + --registry ghcr.io/ornl-mdf/containers + --date "$(date -u +%F)" + --revision "$GITHUB_SHA" + ) + for target in $CATALOG_TARGETS; do + command+=(--catalog-target "$target") + done + for target in $TARGETS; do + command+=(--selected-target "$target") + done + result="$("${command[@]}")" + echo "$result" + echo "release_tag=$(jq -r '.tag' <<< "$result")" >> "$GITHUB_OUTPUT" + status="$(jq -r '.status' <<< "$result")" + candidate="$(jq -r '.candidate // empty' <<< "$result")" + if [ "$status" = new ]; then + candidate="candidate-$(jq -r '.tag' <<< "$result")-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + fi + echo "status=${status}" >> "$GITHUB_OUTPUT" + echo "candidate_tag=${candidate}" >> "$GITHUB_OUTPUT" + echo "release_created=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT" + + - name: Refresh changed Spack locks + if: needs.detect.outputs.refresh_locks != '' && (github.event_name == 'pull_request' || steps.release.outputs.status == 'new') + env: + LOCKS: ${{ needs.detect.outputs.refresh_locks }} + shell: bash + run: | + set -euo pipefail + for image in $LOCKS; do + rm -f "config/spack/${image}.lock" + done + + - name: Build affected targets + if: github.event_name == 'pull_request' || steps.release.outputs.status == 'new' + env: + GIT_REVISION: ${{ github.sha }} + OUTPUT_TYPE: docker + PUSH: "false" + RELEASE_CREATED: ${{ steps.release.outputs.release_created || '' }} + RELEASE_TAG: ${{ steps.release.outputs.release_tag || 'unreleased' }} + RELEASE_CANDIDATE: ${{ steps.release.outputs.candidate_tag || '' }} + run: docker buildx bake ${{ needs.detect.outputs.targets }} + + - name: Smoke test built containers + if: github.event_name == 'pull_request' || steps.release.outputs.status == 'new' + env: + RELEASE_TAG: ${{ steps.release.outputs.release_tag || 'unreleased' }} + TARGETS: ${{ needs.detect.outputs.targets }} + shell: bash + run: scripts/container-smoke-tests.sh $TARGETS + + - name: Preflight release inventory artifacts + if: github.event_name == 'push' && steps.release.outputs.status == 'new' + env: + RELEASE_TAG: ${{ steps.release.outputs.release_tag }} + TARGETS: ${{ needs.detect.outputs.targets }} + shell: bash + run: | + set -euo pipefail + inventory_root="$RUNNER_TEMP/release-inventories" + for image in $TARGETS; do + reference="ghcr.io/ornl-mdf/containers/${image}:${RELEASE_TAG}" + image_root="${inventory_root}/${image}" + mkdir -p "${image_root}/${image}" + container="$(docker create "$reference")" + docker cp "${container}:/usr/share/ornl-mdf/inventory/${image}/." "${image_root}/${image}" + docker rm -f "$container" >/dev/null + done + for image in $TARGETS; do + image_root="${inventory_root}/${image}" + base_args=() + if [ "$image" = exaca ] || [ "$image" = thesis ]; then + base_args=(--base-inventory "${inventory_root}/ubuntu") + fi + python3 scripts/generate-container-docs.py \ + --image "$image" \ + --tag "$RELEASE_TAG" \ + --digest sha256:preflight \ + --inventory "$image_root" \ + --output-root "$RUNNER_TEMP/preflight-container-docs/${image}" \ + "${base_args[@]}" + done + + - name: Stage verified images in the private registry + if: github.event_name == 'push' && steps.release.outputs.status == 'new' + env: + CANDIDATE_TAG: ${{ steps.release.outputs.candidate_tag }} + RELEASE_TAG: ${{ steps.release.outputs.release_tag }} + TARGETS: ${{ needs.detect.outputs.targets }} + shell: bash + run: | + set -euo pipefail + for image in $TARGETS; do + public_reference="ghcr.io/ornl-mdf/containers/${image}:${RELEASE_TAG}" + candidate_reference="ghcr.io/ornl-mdf/containers-staging/${image}:${CANDIDATE_TAG}" + docker tag "$public_reference" "$candidate_reference" + docker image push "$candidate_reference" + labels="$(docker buildx imagetools inspect "$candidate_reference" --format '{{json .Config.Labels}}')" + test "$(jq -r '."org.opencontainers.image.revision"' <<< "$labels")" = "$GITHUB_SHA" + test "$(jq -r '."org.opencontainers.image.version"' <<< "$labels")" = "$RELEASE_TAG" + test "$(jq -r '."org.ornl-mdf.containers.candidate"' <<< "$labels")" = "$CANDIDATE_TAG" + done + + - name: Promote complete private candidate batch + if: github.event_name == 'push' && (steps.release.outputs.status == 'new' || steps.release.outputs.status == 'resume') + id: promote + env: + CANDIDATE_TAG: ${{ steps.release.outputs.candidate_tag }} + RELEASE_TAG: ${{ steps.release.outputs.release_tag }} + TARGETS: ${{ needs.detect.outputs.targets }} + shell: bash + run: | + set -euo pipefail + echo "started=true" >> "$GITHUB_OUTPUT" + for image in $TARGETS; do + candidate_repository="ghcr.io/ornl-mdf/containers-staging/${image}" + candidate_reference="${candidate_repository}:${CANDIDATE_TAG}" + public_reference="ghcr.io/ornl-mdf/containers/${image}:${RELEASE_TAG}" + labels="$(docker buildx imagetools inspect "$candidate_reference" --format '{{json .Config.Labels}}')" + test "$(jq -r '."org.opencontainers.image.revision"' <<< "$labels")" = "$GITHUB_SHA" + test "$(jq -r '."org.opencontainers.image.version"' <<< "$labels")" = "$RELEASE_TAG" + test "$(jq -r '."org.ornl-mdf.containers.candidate"' <<< "$labels")" = "$CANDIDATE_TAG" + candidate_digest="sha256:$(docker buildx imagetools inspect "$candidate_reference" --raw | sha256sum | cut -d' ' -f1)" + if docker buildx imagetools inspect "$public_reference" --raw >/dev/null 2>&1; then + public_digest="sha256:$(docker buildx imagetools inspect "$public_reference" --raw | sha256sum | cut -d' ' -f1)" + test "$public_digest" = "$candidate_digest" + else + docker buildx imagetools create --tag "$public_reference" "${candidate_repository}@${candidate_digest}" + public_digest="sha256:$(docker buildx imagetools inspect "$public_reference" --raw | sha256sum | cut -d' ' -f1)" + test "$public_digest" = "$candidate_digest" + fi + done + + - name: Stage published release inventories + if: github.event_name == 'push' + env: + RELEASE_TAG: ${{ steps.release.outputs.release_tag }} + TARGETS: ${{ needs.detect.outputs.targets }} + shell: bash + run: | + set -euo pipefail + inventory_root="$RUNNER_TEMP/release-inventories" + for image in $TARGETS; do + reference="ghcr.io/ornl-mdf/containers/${image}:${RELEASE_TAG}" + image_root="${inventory_root}/${image}" + docker pull "$reference" + mkdir -p "${image_root}/${image}" + container="$(docker create "$reference")" + docker cp "${container}:/usr/share/ornl-mdf/inventory/${image}/." "${image_root}/${image}" + docker rm -f "$container" >/dev/null + done + + - name: Generate release inventories and locks + if: github.event_name == 'push' + env: + RELEASE_TAG: ${{ steps.release.outputs.release_tag }} + TARGETS: ${{ needs.detect.outputs.targets }} + shell: bash + run: | + set -euo pipefail + inventory_root="$RUNNER_TEMP/release-inventories" + for image in $TARGETS; do + reference="ghcr.io/ornl-mdf/containers/${image}:${RELEASE_TAG}" + image_root="${inventory_root}/${image}" + digest="sha256:$(docker buildx imagetools inspect "$reference" --raw | sha256sum | cut -d' ' -f1)" + base_args=() + if [ "$image" = exaca ] || [ "$image" = thesis ]; then + base_args=(--base-inventory "${inventory_root}/ubuntu") + fi + python3 scripts/generate-container-docs.py \ + --image "$image" \ + --tag "$RELEASE_TAG" \ + --digest "$digest" \ + --inventory "$image_root" \ + "${base_args[@]}" + if [ -f "${image_root}/${image}/spack.lock" ]; then + cp "${image_root}/${image}/spack.lock" "config/spack/${image}.lock" + echo "Captured locked Spack environment for ${image}" + fi + done + + - name: Commit generated release records + if: github.event_name == 'push' + env: + RELEASE_TAG: ${{ steps.release.outputs.release_tag }} + shell: bash + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add docs/containers + git add config/spack/*.lock 2>/dev/null || true + if git diff --cached --quiet; then + exit 0 + fi + git commit -m "docs: record container inventories for ${RELEASE_TAG}" + git fetch origin main + git rebase origin/main + git push origin HEAD:main + + - name: Remove private candidate images after release completion + if: github.event_name == 'push' && (steps.release.outputs.status == 'new' || steps.release.outputs.status == 'resume' || steps.release.outputs.status == 'complete') + env: + CANDIDATE_TAG: ${{ steps.release.outputs.candidate_tag }} + GH_TOKEN: ${{ github.token }} + TARGETS: ${{ needs.detect.outputs.targets }} + shell: bash + run: | + set -euo pipefail + for image in $TARGETS; do + package="containers-staging/${image}" + encoded_package="$(jq -rn --arg value "$package" '$value | @uri')" + version_ids="$(gh api --paginate "/orgs/ORNL-MDF/packages/container/${encoded_package}/versions" \ + --jq '.[] | select(.metadata.container.tags | index(env.CANDIDATE_TAG)) | .id')" + for version_id in $version_ids; do + gh api --method DELETE "/orgs/ORNL-MDF/packages/container/${encoded_package}/versions/${version_id}" + done + done + + - name: Remove incomplete private candidates + if: failure() && github.event_name == 'push' && steps.release.outputs.status == 'new' && steps.promote.outputs.started != 'true' + env: + CANDIDATE_TAG: ${{ steps.release.outputs.candidate_tag }} + GH_TOKEN: ${{ github.token }} + TARGETS: ${{ needs.detect.outputs.targets }} + shell: bash + run: | + set -euo pipefail + for image in $TARGETS; do + package="containers-staging/${image}" + encoded_package="$(jq -rn --arg value "$package" '$value | @uri')" + version_ids="$(gh api --paginate "/orgs/ORNL-MDF/packages/container/${encoded_package}/versions" \ + --jq '.[] | select(.metadata.container.tags | index(env.CANDIDATE_TAG)) | .id')" + for version_id in $version_ids; do + gh api --method DELETE "/orgs/ORNL-MDF/packages/container/${encoded_package}/versions/${version_id}" + done + done diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..333c1e9 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +logs/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..2a83c4f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,36 @@ +# Agent Guide + +## Repository Model + +- `docker-bake.hcl` is the build and tag source of truth. +- `images/ubuntu` is the shared base for `exaca` and `thesis`. +- Spack manifests live in `config/spack`; solver lockfiles are generated by CI. +- Published tags are immutable UTC release identifiers. The first daily batch uses + `YYYY-MM-DD`; later batches append `-`, progressing from `a` through `z`, + then `aa`. Local builds use `unreleased`. +- A package is a public Bake target with `images//Dockerfile`. Solver Spack + manifests use the matching `config/spack/.yaml` name; `target:` contexts + define image dependencies. + +## Release Rules + +- Do not introduce moving release tags such as `main`, `master`, `develop`, or `latest`. +- Keep direct external build inputs immutable in CI: image digests and Git commits. +- Preserve the embedded `/usr/share/ornl-mdf/inventory` artifacts. CI uses them to + generate `docs/containers/` without starting published containers. +- Changing `config/spack/exaca.yaml` or `config/spack/thesis.yaml` must allow CI to + refresh the matching lockfile. Do not hand-edit generated `*.lock` files. +- Do not overwrite a published image release tag or alter historical inventory pages. + +## Change Discipline + +- Inspect the worktree before editing; it may contain unrelated user changes. +- Use `apply_patch` for manual edits. Do not reset, checkout, or otherwise discard + existing work. +- Keep Dockerfiles compatible with Buildx Bake and the non-root `mdf` runtime user. +- Update `README.md` when public build, tag, or inventory behavior changes. + +## Required Checks + +Run the focused checks in `TESTING.md` for every change. A full image build needs +registry/network access and is normally validated by GitHub Actions. diff --git a/README.md b/README.md index 4c742e4..54ec9e1 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,184 @@ # Containers for ORNL-MDF Myna workflows -## Usage +This repository builds Docker images primarily intended for use with Myna workflows and +other workflows that require containerized ORNL-MDF simulation tools. -See available containers at https://github.com/orgs/ORNL-MDF/packages +> [!NOTE] +> Myna itself is expected to be installed in CI or user environments with `uv` or +> other Python package managers, so it does not have a dedicated container here. -Options to install are listed for each package. A local Docker installation is -necessary. Once installed, the container can be started with, e.g.: +## Repository Layout + +```text +images/ + / + Dockerfile + ... +config/ + spack/ + base.yaml + ubuntu.yaml + exaca.yaml + ... +docker-bake.hcl +``` + +Each image has its own directory, while shared Spack environment definitions +live under `config/spack`. `docker-bake.hcl` is the single source of truth for +local builds and CI. + +## Image Layout + +```text +spack/ubuntu-noble@sha256: -> ghcr.io/ornl-mdf/containers/ubuntu: +openfoam/openfoam10-paraview510@sha256: + AdditiveFOAM commit -> ghcr.io/ornl-mdf/containers/additivefoam: +ubuntu target/image -> exaca target +ubuntu target/image -> thesis target +``` + +`ubuntu:` is the repo-owned Spack base image. It contains the shared +GCC, MPICH, and Kokkos toolchain used by the solver images, together with their +Spack activation. ExaCA and Thesis reuse this installed store and add only their +solver-specific packages. This is intended to be the base image for other +Spack-derived containers; use a different base only when a software environment +has a specific compatibility constraint. + +`exaca:` and `thesis:` reuse the `ubuntu` container as their build base. + +`additivefoam:` repackages the OpenFOAM Foundation v10 image and layers +AdditiveFOAM 1.2.0 on top of it. + +All repo-owned images default to the non-root `mdf` runtime user. Build steps +that need elevated privileges still run as `root` inside the Dockerfile, but +interactive shells and normal container commands start as `mdf`. +They start in `/workspace`, which is owned by `mdf`. + +## Building Locally + +Build all images and load them into the local Docker daemon: + +```sh +mkdir -p logs +for target in ubuntu additivefoam exaca thesis; do + docker buildx bake "$target" 2>&1 | tee "logs/${target}.log" +done +``` + +When buildign a single target buildx resolves additional context needed to a build a target from +definitions in `docker-bake.hcl`. Run one target and write its log: + +```sh +mkdir -p logs +docker buildx bake additivefoam 2>&1 | tee logs/additivefoam.log +``` + +Override the published namespace or tag: + +```sh +mkdir -p logs +for target in ubuntu additivefoam exaca thesis; do + REGISTRY=ghcr.io/ornl-mdf/containers \ + docker buildx bake "$target" 2>&1 | tee "logs/${target}.log" +done +``` + +`docker-bake.hcl` defaults to `type=docker`, so successful local builds are +loaded directly into the local Docker image store. + +Published tags are immutable UTC release identifiers. The first daily batch uses +`YYYY-MM-DD`; CI assigns later same-day batches `YYYY-MM-DD-`, progressing +from `a` through `z`, then `aa`. The exact installed package versions, base-image +digest, and source revisions for each tag are listed in +[`docs/containers/`](docs/containers/README.md), so no container needs to be started +to inspect its software. + +A date-based tag is a completed public release only after it appears in that catalog. +CI first stages verified images in a private GHCR package, then promotes their exact +digests to public tags and commits the catalog and generated Spack locks. A failed +release is resumed only from that same private candidate; CI never rebuilds a +different image under an allocated public tag. +The `ghcr.io/ornl-mdf/containers-staging/` packages must remain private and +grant the repository workflow package write/delete access. + +Local builds default to `unreleased`: + +```text +ubuntu:unreleased +additivefoam:unreleased +exaca:unreleased +thesis:unreleased +``` + +After a successful build, smoke test the locally loaded images: + +```sh +scripts/container-smoke-tests.sh +``` + +To smoke test only the solver images, pass their target names: + +```sh +scripts/container-smoke-tests.sh exaca thesis +``` + +The same tests run in CI for every affected image before it can be published. + +Override them individually when needed: + +```sh +REGISTRY=ghcr.io/ornl-mdf/containers \ +RELEASE_TAG="$(date -u +%F)" \ +docker buildx bake ``` -docker run -it ghcr.io/ornl-mdf/containers/[os]:[version] /bin/bash + +The deafult user for the containers is `mdf`. If a debugging session requires +root inside a container, override the runtime user explicitly: + +```sh +docker run --user root -it ghcr.io/ornl-mdf/containers/ubuntu: /bin/bash ``` +To generate a local-only copy of the container documentation after building +local unreleased images with the default Bake settings, run: + +```sh +mkdir -p logs/container-docs + +for image in ubuntu additivefoam exaca thesis; do + ref="ghcr.io/ornl-mdf/containers/${image}:unreleased" + digest="$(docker image inspect --format '{{.Id}}' "$ref")" + + base_args=() + if [ "$image" = exaca ] || [ "$image" = thesis ]; then + base_args=(--base-image "ghcr.io/ornl-mdf/containers/ubuntu:unreleased") + fi + python3 scripts/generate-container-docs.py \ + --image "$image" \ + --tag unreleased \ + --digest "$digest" \ + --output-root logs/container-docs \ + "${base_args[@]}" +done +``` + +ExaCA and Thesis pages link to the matching Ubuntu page for shared package names +and versions; their Installed Software tables list only software added or changed +by the solver image. + +## CI Rebuild Policy + +CI discovers packages from public Bake targets whose Dockerfile is +`images//Dockerfile`. It rebuilds the changed target and every target that +depends on it through a Bake `target:` context. `config/spack/.yaml` changes +rebuild that target and refresh its generated lockfile; `config/spack/base.yaml` +rebuilds `ubuntu` and its dependents. Changes to `docker-bake.hcl` or `.dockerignore` +rebuild every discovered package. -## Updating +The Ubuntu Spack manifest is the shared toolchain contract. Changing +`config/spack/ubuntu.yaml` rebuilds `ubuntu`, refreshes its lockfile, and rebuilds +all of its dependent solver images and lockfiles. Changes to the shared +`config/spack/base.yaml` receive the same treatment. -If a new package is needed, add it to the `spack.yaml` and run `./build.sh` +`config/spack/ubuntu.lock` is a build input when present, just like the solver +lockfiles. CI removes it only when the shared environment manifests require a new +concrete solution. diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 0000000..f8089d9 --- /dev/null +++ b/TESTING.md @@ -0,0 +1,83 @@ +# Testing + +## Fast Local Checks + +Run these after any change: + +```sh +PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -v +docker buildx bake --print +git diff --check +``` + +The unit test validates Markdown catalog generation from a fixture inventory. The +Bake command validates target inheritance, build arguments, labels, and tags without +pulling or building images. `git diff --check` catches whitespace errors. + +## Container Smoke Tests + +After a successful local build, run the runtime smoke tests against all locally +loaded `unreleased` images: + +```sh +scripts/container-smoke-tests.sh +``` + +Pass one or more target names to limit the test run, for example: + +```sh +scripts/container-smoke-tests.sh exaca thesis +``` + +The script checks the non-root Ubuntu runtime environment, the AdditiveFOAM +tutorial completion marker, and the serial and MPI error paths of ExaCA and +3DThesis. It expects the solver programs to reject a missing input file; ExaCA +must also print its version banner. CI runs the same script against every +affected image after it builds and before a push can publish it. + +## Validate Release Inputs + +Use `--print` with representative immutable inputs when modifying Bake variables, +Docker build arguments, or OCI labels: + +```sh +RELEASE_TAG=2026-07-10 \ +RELEASE_CREATED=2026-07-10T12:00:00Z \ +RELEASE_CANDIDATE=candidate-2026-07-10-123-1 \ +GIT_REVISION=abc123 \ +SPACK_UBUNTU_NOBLE_IMAGE='spack/ubuntu-noble@sha256:c5286e543f226f2c36a6a5ae4c845bc1cd78fad9ece2704dd16256ae774a5d4f' \ +OPENFOAM_IMAGE='openfoam/openfoam10-paraview510@sha256:d6ff1f9a2e7bc3c9177f373bebbdeb542fd8b49144afc24d5e3a3cd9bfae253d' \ +ADDITIVEFOAM_REF=b8f6d48c53555c303fa8186c895aee5712b6ea02 \ +docker buildx bake --print ubuntu additivefoam +``` + +Confirm the rendered output has a date tag, optionally followed by an alphabetic suffix, +and OCI `source`, `version`, `revision`, `created`, and candidate labels. Do not +substitute a real published date and push from a local machine. + +## Change-Specific Coverage + +| Change | Minimum validation | +| --- | --- | +| `scripts/` or `docs/` generation | Unit test and inspect generated Markdown fixture output. | +| `docker-bake.hcl` | Bake print with defaults and injected immutable inputs. | +| Dockerfile | Bake print; after a successful local build, run the relevant container smoke tests. CI runs them for every affected image. | +| `config/spack/.yaml` | Verify its Bake target is selected; CI refreshes its lockfile and any dependent lockfiles. | +| `config/spack/base.yaml` or `images/ubuntu/` | Verify `ubuntu` and its Bake dependents are selected; shared-manifest changes also refresh their locks. | +| Workflow | Run the fast checks and review shell quoting, permissions, and push-only steps. | + +## CI-Only Validation + +GitHub Actions is the release test for registry-dependent behavior. On a `main` push +it uses the pinned external inputs, assigns the next immutable UTC release tag, +preflights embedded inventory before publication, stages candidates in the private +`ghcr.io/ornl-mdf/containers-staging` package, promotes verified digests, and commits +catalog pages and new Spack locks. The catalog commit marks release completion. +Validate the workflow log and resulting `docs/containers/` pages after such a release. +Before the first run, provision `ghcr.io/ornl-mdf/containers-staging/` as a +private package namespace and grant this repository's workflow token package +write/delete access. + +If DNS, registry authentication, or external network access is unavailable locally, +do not treat a failed full Docker build as a Dockerfile failure; report the limitation +and run the fast local checks instead. diff --git a/build.sh b/build.sh deleted file mode 100644 index 37e189b..0000000 --- a/build.sh +++ /dev/null @@ -1,2 +0,0 @@ -# TODO: build dev and release containers -spack containerize > docker/ubuntu diff --git a/config/spack/base.yaml b/config/spack/base.yaml new file mode 100644 index 0000000..2a6a03e --- /dev/null +++ b/config/spack/base.yaml @@ -0,0 +1,6 @@ +concretizer: + unify: true +config: + install_tree: + root: /opt/software +view: /opt/views/view diff --git a/config/spack/exaca.yaml b/config/spack/exaca.yaml new file mode 100644 index 0000000..645b87e --- /dev/null +++ b/config/spack/exaca.yaml @@ -0,0 +1,5 @@ +spack: + include: + - base.yaml + specs: + - exaca@2.0.1 ^mpich@4.3.0 ^kokkos@4.7.04 diff --git a/config/spack/thesis.yaml b/config/spack/thesis.yaml new file mode 100644 index 0000000..996b3ce --- /dev/null +++ b/config/spack/thesis.yaml @@ -0,0 +1,5 @@ +spack: + include: + - base.yaml + specs: + - thesis@4.0.0+mpi ^mpich@4.3.0 diff --git a/config/spack/ubuntu.yaml b/config/spack/ubuntu.yaml new file mode 100644 index 0000000..1e0a521 --- /dev/null +++ b/config/spack/ubuntu.yaml @@ -0,0 +1,6 @@ +spack: + include: + - base.yaml + specs: + - mpich@4.3.0 + - kokkos@4.7.04 diff --git a/container_specific_tests.md b/container_specific_tests.md new file mode 100644 index 0000000..1bd11d1 --- /dev/null +++ b/container_specific_tests.md @@ -0,0 +1,34 @@ +additivefoam +``` +# Launch container +docker run -it --rm ghcr.io/ornl-mdf/containers/additivefoam:unreleased + +# Check OpenFOAM is on path +which checkMesh # should not be blank + +# Check AdditiveFOAM tutorial works +cd /opt/AdditiveFOAM/tutorials/AMB2018-02-B +tail -n 1 log.additiveFoam # should say "Finalising parallel run" +``` + +exaca +``` +# Launch container +docker run -it --rm ghcr.io/ornl-mdf/containers/exaca:unreleased + +# Ensure executable is found and launches +which ExaCA +ExaCA # should throw an error saying "must provide path to input file" but should also contain "ExaCA version:" line +mpirun -n 2 ExaCA # should throw input file error, but should also contain "ExaCA version:" line +``` + +thesis +``` +# Launch container +docker run -it --rm ghcr.io/ornl-mdf/containers/thesis:unreleased + +# Ensure executable is found and launches +which 3DThesis +3DThesis # should throw an error saying "must provide path to input file" +mpirun -n 2 3DThesis # should throw input file error +``` \ No newline at end of file diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 0000000..1e1f3ad --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,119 @@ +variable "REGISTRY" { + default = "ghcr.io/ornl-mdf/containers" +} + +variable "RELEASE_TAG" { + # CI supplies an immutable UTC date, with a suffix for later daily releases. + default = "unreleased" +} + +variable "SPACK_UBUNTU_NOBLE_IMAGE" { + default = "spack/ubuntu-noble@sha256:c5286e543f226f2c36a6a5ae4c845bc1cd78fad9ece2704dd16256ae774a5d4f" +} + +variable "OPENFOAM_IMAGE" { + default = "openfoam/openfoam10-paraview510@sha256:d6ff1f9a2e7bc3c9177f373bebbdeb542fd8b49144afc24d5e3a3cd9bfae253d" +} + +variable "ADDITIVEFOAM_REF" { + default = "b8f6d48c53555c303fa8186c895aee5712b6ea02" +} + +variable "GIT_REVISION" { + default = "local" +} + +variable "RELEASE_CREATED" { + default = "" +} + +variable "RELEASE_CANDIDATE" { + # CI records the private staging tag used to promote an immutable release. + default = "" +} + +variable "OUTPUT_TYPE" { + default = "docker" +} + +variable "PUSH" { + default = "false" +} + +group "default" { + targets = ["ubuntu", "additivefoam", "exaca", "thesis"] +} + +group "base" { + targets = ["ubuntu", "additivefoam"] +} + +group "solvers" { + targets = ["exaca", "thesis"] +} + +target "_common" { + context = "." + output = ["type=${OUTPUT_TYPE},push=${PUSH}"] + args = { + GIT_REVISION = "${GIT_REVISION}" + } + labels = { + "org.opencontainers.image.source" = "https://github.com/ORNL-MDF/containers" + "org.opencontainers.image.version" = "${RELEASE_TAG}" + "org.opencontainers.image.revision" = "${GIT_REVISION}" + "org.opencontainers.image.created" = "${RELEASE_CREATED}" + "org.ornl-mdf.containers.candidate" = "${RELEASE_CANDIDATE}" + } +} + +target "ubuntu" { + inherits = ["_common"] + dockerfile = "images/ubuntu/Dockerfile" + tags = ["${REGISTRY}/ubuntu:${RELEASE_TAG}"] + args = { + SPACK_UBUNTU_NOBLE_IMAGE = "${SPACK_UBUNTU_NOBLE_IMAGE}" + } + labels = { + "org.opencontainers.image.description" = "ORNL-MDF Ubuntu and Spack base environment. Full inventory: repository docs/containers." + } + pull = true +} + +target "additivefoam" { + inherits = ["_common"] + dockerfile = "images/additivefoam/Dockerfile" + tags = ["${REGISTRY}/additivefoam:${RELEASE_TAG}"] + args = { + OPENFOAM_IMAGE = "${OPENFOAM_IMAGE}" + ADDITIVEFOAM_REF = "${ADDITIVEFOAM_REF}" + } + labels = { + "org.opencontainers.image.description" = "ORNL-MDF AdditiveFOAM environment. Full inventory: repository docs/containers." + } + pull = true +} + +target "exaca" { + inherits = ["_common"] + dockerfile = "images/exaca/Dockerfile" + tags = ["${REGISTRY}/exaca:${RELEASE_TAG}"] + contexts = { + ubuntu-base = "target:ubuntu" + } + labels = { + "org.opencontainers.image.description" = "ORNL-MDF ExaCA simulation environment. Full inventory: repository docs/containers." + } +} + +target "thesis" { + inherits = ["_common"] + dockerfile = "images/thesis/Dockerfile" + tags = ["${REGISTRY}/thesis:${RELEASE_TAG}"] + contexts = { + ubuntu-base = "target:ubuntu" + } + labels = { + "org.opencontainers.image.description" = "ORNL-MDF 3DThesis simulation environment. Full inventory: repository docs/containers." + } +} diff --git a/docker/ubuntu b/docker/ubuntu deleted file mode 100644 index 79aa26e..0000000 --- a/docker/ubuntu +++ /dev/null @@ -1,90 +0,0 @@ -# Build stage with Spack pre-installed and ready to be used -FROM spack/ubuntu-jammy:develop - -# Get Ubuntu packages -# 1. apt-get install necessary build tools -# 2. apt-get install dependencies for spack to build correctly -RUN apt-get update \ -&& apt-get install -y \ - python3-dev \ - python-is-python3 \ - build-essential \ - cmake \ - vim \ - emacs \ - wget \ - libssl-dev \ - libffi-dev \ - libxrender1 \ -&& apt-get clean \ -&& rm -rf /var/lib/apt/lists/* - -# Get Myna -ENV MYNA_DIR=/opt/myna -RUN MYNA_VERSION=main.tar.gz && \ - MYNA_ARCHIVE=myna-${MYNA_VERSION} && \ - MYNA_URL=https://github.com/ORNL-MDF/Myna/archive/refs/heads/${MYNA_VERSION} && \ - wget --quiet ${MYNA_URL} --output-document=${MYNA_ARCHIVE} && \ - mkdir -p ${MYNA_DIR} && \ - tar -xf ${MYNA_ARCHIVE} -C ${MYNA_DIR} --strip-components=1 && \ - rm -f ${MYNA_ARCHIVE} && \ - python3 -m pip install --upgrade pip && \ - python3 -m pip install ${MYNA_DIR} --root-user-action=ignore && \ - python3 -m pip cache purge -ENV PYTHONPATH=${PYTHONPATH}:${MYNA_DIR} - -# What we want to install and how we want to install it -# is specified in a manifest file (spack.yaml) -# mpich added because of issues with openmpi -RUN mkdir -p /opt/spack-environment && \ -set -o noclobber \ -&& (echo spack: \ -&& echo ' specs:' \ -&& echo ' - mpich@4.3.0' \ -&& echo ' - exaca@master' \ -&& echo ' - additivefoam@main' \ -&& echo ' - thesis@master+mpi' \ -&& echo ' concretizer:' \ -&& echo ' unify: true' \ -&& echo ' config:' \ -&& echo ' install_tree:' \ -&& echo ' root: /opt/software' \ -&& echo ' view: /opt/views/view') > /opt/spack-environment/spack.yaml - -# Find apt-get packages, then build the spack environment -# NOTE: could use "spack --backtrace external find openmpi" etc. -RUN cd /opt/spack-environment \ -&& spack env activate . \ -&& spack external find \ -&& spack compiler find \ -&& spack install --fail-fast \ -&& find -L /opt/views/view/* -type f -exec readlink -f '{}' \; | \ - xargs file -i | \ - grep 'charset=binary' | \ - grep 'x-executable\|x-archive\|x-sharedlib' | \ - awk -F: '{print $1}' | xargs strip \ -&& spack gc -y - -# Modifications to the environment that are necessary to run -RUN cd /opt/spack-environment && \ - spack env activate --sh -d . > activate.sh - -# Make .bashrc root profile (used on container restarts) -# and add container entry point -RUN mkdir -p /root \ -&& { \ - echo "export OMPI_ALLOW_RUN_AS_ROOT=1"; \ - echo "export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1"; \ - echo ". /opt/spack/share/spack/setup-env.sh"; \ - echo ". /opt/spack-environment/activate.sh"; \ - } > /root/.bashrc \ -&& { \ - echo '#!/bin/sh'; \ - echo ". /root/.bashrc"; \ - echo 'exec "$@"'; \ - } > /entrypoint.sh \ -&& chmod a+x /entrypoint.sh \ -&& ln -s /opt/views/view /opt/view - -ENTRYPOINT [ "/entrypoint.sh" ] -CMD [ "/bin/bash" ] diff --git a/docs/containers/README.md b/docs/containers/README.md new file mode 100644 index 0000000..62972e7 --- /dev/null +++ b/docs/containers/README.md @@ -0,0 +1,3 @@ +# Container Software Inventory + +Published image inventories are generated by CI after each release. diff --git a/images/additivefoam/Dockerfile b/images/additivefoam/Dockerfile new file mode 100644 index 0000000..3abe074 --- /dev/null +++ b/images/additivefoam/Dockerfile @@ -0,0 +1,72 @@ +ARG OPENFOAM_IMAGE=openfoam/openfoam10-paraview510@sha256:d6ff1f9a2e7bc3c9177f373bebbdeb542fd8b49144afc24d5e3a3cd9bfae253d +FROM ${OPENFOAM_IMAGE} +ARG OPENFOAM_IMAGE +ARG ADDITIVEFOAM_REF=b8f6d48c53555c303fa8186c895aee5712b6ea02 +ARG GIT_REVISION=local + +ENV MDF_USER=mdf +ENV MDF_UID=1000 +ENV MDF_GID=1000 +ENV MDF_WORKDIR=/workspace + +USER root + +SHELL ["/bin/bash", "-lc"] + +ENV ADDITIVEFOAM_DIR=/opt/AdditiveFOAM +ENV ADDITIVEFOAM_VERSION=1.2.0 + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends git; \ + apt-get clean; \ + rm -rf /var/lib/apt/lists/*; \ + git init "${ADDITIVEFOAM_DIR}"; \ + git -C "${ADDITIVEFOAM_DIR}" remote add origin https://github.com/ORNL/AdditiveFOAM.git; \ + git -C "${ADDITIVEFOAM_DIR}" fetch --depth 1 origin "${ADDITIVEFOAM_REF}"; \ + git -C "${ADDITIVEFOAM_DIR}" checkout --detach FETCH_HEAD; \ + additivefoam_revision="$(git -C "${ADDITIVEFOAM_DIR}" rev-parse HEAD)"; \ + for candidate in /opt/openfoam10/etc/bashrc /usr/lib/openfoam/openfoam10/etc/bashrc /usr/lib/openfoam/openfoam*/etc/bashrc; do \ + if [ -f "$candidate" ]; then \ + set +u; \ + . "$candidate" || true; \ + set -u; \ + break; \ + fi; \ + done; \ + cd "${ADDITIVEFOAM_DIR}"; \ + ./Allwmake; \ + rm -rf "${ADDITIVEFOAM_DIR}/.git"; \ + if ! getent group "${MDF_USER}" >/dev/null; then \ + if getent group "${MDF_GID}" >/dev/null; then \ + groupadd "${MDF_USER}"; \ + else \ + groupadd --gid "${MDF_GID}" "${MDF_USER}"; \ + fi; \ + fi; \ + if ! id -u "${MDF_USER}" >/dev/null 2>&1; then \ + if getent passwd "${MDF_UID}" >/dev/null; then \ + useradd --create-home --gid "${MDF_USER}" --shell /bin/bash "${MDF_USER}"; \ + else \ + useradd --create-home --uid "${MDF_UID}" --gid "${MDF_USER}" --shell /bin/bash "${MDF_USER}"; \ + fi; \ + fi; \ + home_dir="$(getent passwd "${MDF_USER}" | cut -d: -f6)"; \ + if [ -z "${home_dir}" ]; then \ + echo "Unable to resolve home directory for ${MDF_USER}" >&2; \ + exit 1; \ + fi; \ + mkdir -p "${home_dir}" "${MDF_WORKDIR}"; \ + chown -R "${MDF_USER}:${MDF_USER}" "${ADDITIVEFOAM_DIR}" "${home_dir}" "${MDF_WORKDIR}"; \ + mkdir -p /usr/share/ornl-mdf/inventory/additivefoam; \ + { \ + echo "image=additivefoam"; \ + echo "base_image=${OPENFOAM_IMAGE}"; \ + echo "additivefoam_version=${ADDITIVEFOAM_VERSION}"; \ + echo "additivefoam_revision=${additivefoam_revision}"; \ + echo "repository_revision=${GIT_REVISION}"; \ + } > /usr/share/ornl-mdf/inventory/additivefoam/metadata.env; \ + dpkg-query -W -f='${binary:Package}\t${Version}\n' | sort > /usr/share/ornl-mdf/inventory/additivefoam/apt.tsv + +USER mdf +WORKDIR /workspace diff --git a/images/exaca/Dockerfile b/images/exaca/Dockerfile new file mode 100644 index 0000000..db1102f --- /dev/null +++ b/images/exaca/Dockerfile @@ -0,0 +1,33 @@ +FROM ubuntu-base + +USER root +ARG GIT_REVISION=local + +COPY config/spack/ /tmp/spack-config/ + +RUN cp /tmp/spack-config/base.yaml /opt/spack-environment/base.yaml \ +&& cp /tmp/spack-config/exaca.yaml /opt/spack-environment/spack.yaml \ +&& rm -f /opt/spack-environment/spack.lock \ +&& if [ -f /tmp/spack-config/exaca.lock ]; then cp /tmp/spack-config/exaca.lock /opt/spack-environment/spack.lock; fi \ +&& cd /opt/spack-environment \ +&& spack env activate . \ +&& spack external find \ +&& spack install --fail-fast \ +&& find -L /opt/views/view/* -type f -exec readlink -f '{}' \; | \ + xargs -r file -i | \ + grep 'charset=binary' | \ + grep 'x-executable\|x-archive\|x-sharedlib' | \ + awk -F: '{print $1}' | xargs -r strip \ +&& spack clean --stage \ +&& spack env activate --sh -d . > activate.sh \ +&& mkdir -p /usr/share/ornl-mdf/inventory/exaca \ +&& { \ + echo "image=exaca"; \ + echo "repository_revision=${GIT_REVISION}"; \ + } > /usr/share/ornl-mdf/inventory/exaca/metadata.env \ +&& spack find --json > /usr/share/ornl-mdf/inventory/exaca/spack.json \ +&& cp spack.lock /usr/share/ornl-mdf/inventory/exaca/spack.lock + +ENTRYPOINT ["/entrypoint.sh"] +USER mdf +CMD ["/bin/bash"] diff --git a/images/thesis/Dockerfile b/images/thesis/Dockerfile new file mode 100644 index 0000000..7102281 --- /dev/null +++ b/images/thesis/Dockerfile @@ -0,0 +1,33 @@ +FROM ubuntu-base + +USER root +ARG GIT_REVISION=local + +COPY config/spack/ /tmp/spack-config/ + +RUN cp /tmp/spack-config/base.yaml /opt/spack-environment/base.yaml \ +&& cp /tmp/spack-config/thesis.yaml /opt/spack-environment/spack.yaml \ +&& rm -f /opt/spack-environment/spack.lock \ +&& if [ -f /tmp/spack-config/thesis.lock ]; then cp /tmp/spack-config/thesis.lock /opt/spack-environment/spack.lock; fi \ +&& cd /opt/spack-environment \ +&& spack env activate . \ +&& spack external find \ +&& spack install --fail-fast \ +&& find -L /opt/views/view/* -type f -exec readlink -f '{}' \; | \ + xargs -r file -i | \ + grep 'charset=binary' | \ + grep 'x-executable\|x-archive\|x-sharedlib' | \ + awk -F: '{print $1}' | xargs -r strip \ +&& spack clean --stage \ +&& spack env activate --sh -d . > activate.sh \ +&& mkdir -p /usr/share/ornl-mdf/inventory/thesis \ +&& { \ + echo "image=thesis"; \ + echo "repository_revision=${GIT_REVISION}"; \ + } > /usr/share/ornl-mdf/inventory/thesis/metadata.env \ +&& spack find --json > /usr/share/ornl-mdf/inventory/thesis/spack.json \ +&& cp spack.lock /usr/share/ornl-mdf/inventory/thesis/spack.lock + +ENTRYPOINT ["/entrypoint.sh"] +USER mdf +CMD ["/bin/bash"] diff --git a/images/ubuntu/Dockerfile b/images/ubuntu/Dockerfile new file mode 100644 index 0000000..9da2db2 --- /dev/null +++ b/images/ubuntu/Dockerfile @@ -0,0 +1,97 @@ +# Repository-owned Ubuntu Noble base for ORNL-MDF workflow images. +ARG SPACK_UBUNTU_NOBLE_IMAGE=spack/ubuntu-noble@sha256:c5286e543f226f2c36a6a5ae4c845bc1cd78fad9ece2704dd16256ae774a5d4f +FROM ${SPACK_UBUNTU_NOBLE_IMAGE} +ARG SPACK_UBUNTU_NOBLE_IMAGE +ARG GIT_REVISION=local + +ENV MDF_USER=mdf +ENV MDF_UID=1000 +ENV MDF_GID=1000 +ENV MDF_WORKDIR=/workspace +ENV MAKEFLAGS= + +COPY config/spack/base.yaml /opt/spack-environment/base.yaml +COPY config/spack/ubuntu.yaml /opt/spack-environment/spack.yaml +COPY config/spack/ /tmp/spack-config/ + +RUN apt-get update \ +&& apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + file \ +&& apt-get clean \ +&& rm -rf /var/lib/apt/lists/* + +RUN set -eux; \ + if ! getent group "${MDF_USER}" >/dev/null; then \ + if getent group "${MDF_GID}" >/dev/null; then \ + groupadd "${MDF_USER}"; \ + else \ + groupadd --gid "${MDF_GID}" "${MDF_USER}"; \ + fi; \ + fi; \ + if ! id -u "${MDF_USER}" >/dev/null 2>&1; then \ + if getent passwd "${MDF_UID}" >/dev/null; then \ + useradd --create-home --gid "${MDF_USER}" --shell /bin/bash "${MDF_USER}"; \ + else \ + useradd --create-home --uid "${MDF_UID}" --gid "${MDF_USER}" --shell /bin/bash "${MDF_USER}"; \ + fi; \ + fi; \ + mkdir -p /opt/software /opt/views "${MDF_WORKDIR}"; \ + home_dir="$(getent passwd "${MDF_USER}" | cut -d: -f6)"; \ + if [ -z "${home_dir}" ]; then \ + echo "Unable to resolve home directory for ${MDF_USER}" >&2; \ + exit 1; \ + fi; \ + mkdir -p "${home_dir}"; \ + if [ -f /tmp/spack-config/ubuntu.lock ]; then cp /tmp/spack-config/ubuntu.lock /opt/spack-environment/spack.lock; fi; \ + cd /opt/spack-environment; \ + spack env activate .; \ + spack external find; \ + spack compiler find; \ + spack install --fail-fast; \ + find -L /opt/views/view/* -type f -exec readlink -f '{}' \; | \ + xargs -r file -i | \ + grep 'charset=binary' | \ + grep 'x-executable\|x-archive\|x-sharedlib' | \ + awk -F: '{print $1}' | xargs -r strip; \ + spack clean --stage; \ + spack env activate --sh -d . > activate.sh; \ + { \ + echo ". /opt/spack/share/spack/setup-env.sh"; \ + echo ". /opt/spack-environment/activate.sh"; \ + } > /etc/profile.d/spack-env.sh; \ + { \ + echo "#!/bin/sh"; \ + echo 'set -e'; \ + echo ". /opt/spack/share/spack/setup-env.sh"; \ + echo ". /opt/spack-environment/activate.sh"; \ + echo 'export PATH="/opt/views/view/bin:${PATH}"'; \ + echo 'export HOME="$(getent passwd "${MDF_USER}" | cut -d: -f6)"'; \ + echo 'cd "${MDF_WORKDIR}"'; \ + echo 'exec "$@"'; \ + } > /entrypoint.sh; \ + chmod a+x /entrypoint.sh; \ + { \ + echo ". /opt/spack/share/spack/setup-env.sh"; \ + echo ". /opt/spack-environment/activate.sh"; \ + echo 'export PATH="/opt/views/view/bin:${PATH}"'; \ + echo 'cd "${MDF_WORKDIR}"'; \ + } > "${home_dir}/.bashrc"; \ + chown "${MDF_USER}:${MDF_USER}" "${home_dir}/.bashrc"; \ + chown -R "${MDF_USER}:${MDF_USER}" /opt/spack-environment /opt/software /opt/views "${MDF_WORKDIR}" "${home_dir}"; \ + ln -s /opt/views/view /opt/view; \ + mkdir -p /usr/share/ornl-mdf/inventory/ubuntu; \ + { \ + echo "image=ubuntu"; \ + echo "base_image=${SPACK_UBUNTU_NOBLE_IMAGE}"; \ + echo "repository_revision=${GIT_REVISION}"; \ + } > /usr/share/ornl-mdf/inventory/ubuntu/metadata.env; \ + spack find --json > /usr/share/ornl-mdf/inventory/ubuntu/spack.json; \ + cp spack.lock /usr/share/ornl-mdf/inventory/ubuntu/spack.lock; \ + dpkg-query -W -f='${binary:Package}\t${Version}\n' | sort > /usr/share/ornl-mdf/inventory/ubuntu/apt.tsv + +ENTRYPOINT ["/entrypoint.sh"] +USER mdf +WORKDIR /workspace +CMD ["/bin/bash"] diff --git a/scripts/container-smoke-tests.sh b/scripts/container-smoke-tests.sh new file mode 100755 index 0000000..752af32 --- /dev/null +++ b/scripts/container-smoke-tests.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Exercise the essential runtime interfaces of images built by docker-bake.hcl. +set -euo pipefail + +registry="${REGISTRY:-ghcr.io/ornl-mdf/containers}" +release_tag="${RELEASE_TAG:-unreleased}" + +if [ "$#" -eq 0 ]; then + set -- ubuntu additivefoam exaca thesis +fi + +image_ref() { + printf '%s/%s:%s' "$registry" "$1" "$release_tag" +} + +expect_failure() { + local description="$1" + shift + local output status + + set +e + output="$("$@" 2>&1)" + status=$? + set -e + printf '%s\n' "$output" + if [ "$status" -eq 0 ]; then + echo "Expected ${description} to fail without an input file" >&2 + return 1 + fi + printf '%s' "$output" +} + +for target in "$@"; do + image="$(image_ref "$target")" + echo "Smoke testing ${image}" + + case "$target" in + ubuntu) + docker run --rm "$image" /bin/bash -lc \ + 'test "$(id -un)" = mdf && test "$PWD" = /workspace && test -f /usr/share/ornl-mdf/inventory/ubuntu/metadata.env' + ;; + additivefoam) + docker run --rm "$image" /bin/bash -lc \ + 'command -v checkMesh >/dev/null && cd /opt/AdditiveFOAM/tutorials/AMB2018-02-B && test "$(tail -n 1 log.additiveFoam)" = "Finalising parallel run"' + ;; + exaca) + output="$(expect_failure 'ExaCA serial invocation' docker run --rm "$image" ExaCA)" + case "$output" in + *"ExaCA version:"*) ;; + *) echo 'ExaCA did not print its version banner' >&2; exit 1 ;; + esac + output="$(expect_failure 'ExaCA MPI invocation' docker run --rm "$image" mpirun -n 2 ExaCA)" + case "$output" in + *"ExaCA version:"*) ;; + *) echo 'ExaCA MPI invocation did not print its version banner' >&2; exit 1 ;; + esac + ;; + thesis) + expect_failure '3DThesis serial invocation' docker run --rm "$image" 3DThesis >/dev/null + expect_failure '3DThesis MPI invocation' docker run --rm "$image" mpirun -n 2 3DThesis >/dev/null + ;; + *) + echo "Unknown smoke-test target: ${target}" >&2 + exit 2 + ;; + esac +done diff --git a/scripts/generate-container-docs.py b/scripts/generate-container-docs.py new file mode 100644 index 0000000..1833f81 --- /dev/null +++ b/scripts/generate-container-docs.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Render a repository catalog from inventory files embedded in a container.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import tempfile + + +def read_metadata(path: Path) -> dict[str, str]: + values: dict[str, str] = {} + for line in path.read_text().splitlines(): + if "=" in line: + key, value = line.split("=", 1) + values[key] = value + return values + + +def extract_inventory(image: str) -> Path: + tempdir = Path(tempfile.mkdtemp(prefix="container-inventory-")) + container = subprocess.check_output(["docker", "create", image], text=True).strip() + try: + subprocess.run( + ["docker", "cp", f"{container}:/usr/share/ornl-mdf/inventory", str(tempdir)], + check=True, + ) + finally: + subprocess.run(["docker", "rm", "-f", container], check=True, stdout=subprocess.DEVNULL) + return tempdir / "inventory" + + +def software_rows(inventory: Path) -> list[tuple[str, str]]: + rows: list[tuple[str, str]] = [] + for apt_file in sorted(inventory.glob("*/apt.tsv")): + rows.extend(tuple(line.split("\t", 1)) for line in apt_file.read_text().splitlines() if "\t" in line) + for spack_file in sorted(inventory.glob("*/spack.json")): + try: + packages = json.loads(spack_file.read_text()) + except json.JSONDecodeError: + continue + if isinstance(packages, dict): + packages = packages.get("specs", []) + if not isinstance(packages, list): + continue + for package in packages: + name = package.get("name", "unknown") + version = package.get("version", "unknown") + rows.append((f"spack:{name}", str(version))) + return sorted(set(rows)) + + +def render( + image: str, + tag: str, + digest: str, + inventory: Path, + base_inventory: Path | None = None, +) -> str: + metadata = {} + for metadata_file in sorted(inventory.glob("*/metadata.env")): + metadata.update(read_metadata(metadata_file)) + lines = [ + f"# {image}:{tag}", + "", + f"- Image: `ghcr.io/ornl-mdf/containers/{image}:{tag}`", + f"- Digest: `{digest}`", + f"- Repository revision: `{metadata.get('repository_revision', 'not recorded')}`", + "", + "## Build Inputs", + "", + ] + for key in sorted(key for key in metadata if key not in {"image", "repository_revision"}): + lines.append(f"- {key.replace('_', ' ')}: `{metadata[key]}`") + for lock_file in sorted(inventory.glob("*/spack.lock")): + lock_digest = hashlib.sha256(lock_file.read_bytes()).hexdigest() + lines.append(f"- {lock_file.parent.name} Spack lock SHA-256: `{lock_digest}`") + rows = software_rows(inventory) + if base_inventory is not None: + base_rows = set(software_rows(base_inventory)) + rows = [row for row in rows if row not in base_rows] + lines.extend( + [ + "", + f"- Shared packages and versions: [ubuntu:{tag}](../ubuntu/{tag}.md)", + ] + ) + lines.extend(["", "## Installed Software", "", "| Package | Version |", "| --- | --- |"]) + lines.extend(f"| `{name}` | `{version}` |" for name, version in rows) + lines.append("") + return "\n".join(lines) + + +def update_index(root: Path) -> None: + entries = sorted(root.glob("*/*.md")) + lines = ["# Container Software Inventory", "", "Generated from artifacts embedded in published images.", ""] + for entry in entries: + if entry.name == "README.md": + continue + lines.append(f"- [{entry.parent.name}:{entry.stem}]({entry.relative_to(root).as_posix()})") + lines.append("") + (root / "README.md").write_text("\n".join(lines)) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--image", required=True) + parser.add_argument("--tag", required=True) + parser.add_argument("--digest", required=True) + parser.add_argument("--output-root", type=Path, default=Path("docs/containers")) + parser.add_argument("--inventory", type=Path) + base_group = parser.add_mutually_exclusive_group() + base_group.add_argument("--base-inventory", type=Path) + base_group.add_argument("--base-image") + args = parser.parse_args() + + inventory = args.inventory or extract_inventory(f"ghcr.io/ornl-mdf/containers/{args.image}:{args.tag}") + base_inventory = args.base_inventory + if args.base_image: + base_inventory = extract_inventory(args.base_image) + output_dir = args.output_root / args.image + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / f"{args.tag}.md").write_text( + render(args.image, args.tag, args.digest, inventory, base_inventory) + ) + update_index(args.output_root) + if args.inventory is None: + shutil.rmtree(inventory.parent) + if args.base_image: + shutil.rmtree(base_inventory.parent) + + +if __name__ == "__main__": + main() diff --git a/scripts/select_build_targets.py b/scripts/select_build_targets.py new file mode 100644 index 0000000..795191f --- /dev/null +++ b/scripts/select_build_targets.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Select affected container packages from a rendered Bake definition.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + + +def package_targets(bake: dict[str, Any]) -> dict[str, dict[str, Any]]: + """Return public Bake targets that follow the repository package convention.""" + targets = bake.get("target", {}) + if not isinstance(targets, dict): + raise ValueError("Bake output does not contain a target object") + + packages: dict[str, dict[str, Any]] = {} + for name, target in targets.items(): + if not isinstance(name, str) or not isinstance(target, dict) or name.startswith("_"): + continue + if target.get("dockerfile") == f"images/{name}/Dockerfile": + packages[name] = target + return packages + + +def reverse_dependencies(packages: dict[str, dict[str, Any]]) -> dict[str, set[str]]: + """Map a target to packages that consume it through target: contexts.""" + dependents = {name: set() for name in packages} + for name, target in packages.items(): + contexts = target.get("contexts", {}) + if not isinstance(contexts, dict): + continue + for context in contexts.values(): + if isinstance(context, str) and context.startswith("target:"): + dependency = context.removeprefix("target:") + if dependency in dependents: + dependents[dependency].add(name) + return dependents + + +def dependencies(packages: dict[str, dict[str, Any]]) -> dict[str, set[str]]: + """Map a package to the repository targets it consumes through target: contexts.""" + result = {name: set() for name in packages} + for name, target in packages.items(): + contexts = target.get("contexts", {}) + if not isinstance(contexts, dict): + continue + for context in contexts.values(): + if isinstance(context, str) and context.startswith("target:"): + dependency = context.removeprefix("target:") + if dependency in result: + result[name].add(dependency) + return result + + +def select_targets(bake: dict[str, Any], changed_files: list[str]) -> tuple[list[str], list[str]]: + """Return selected packages and Spack lockfiles to regenerate.""" + packages = package_targets(bake) + package_names = set(packages) + changed = set(changed_files) + selected: set[str] = set() + refresh_locks: set[str] = set() + + if {"docker-bake.hcl", ".dockerignore"} & changed: + selected.update(package_names) + + for path in changed: + parts = Path(path).parts + if len(parts) >= 3 and parts[0] == "images" and parts[1] in package_names: + selected.add(parts[1]) + + if path == "config/spack/base.yaml" and "ubuntu" in package_names: + selected.add("ubuntu") + refresh_locks.add("ubuntu") + + if len(parts) == 3 and parts[:2] == ("config", "spack") and path.endswith(".yaml"): + package = Path(parts[2]).stem + if package in package_names and package != "base": + selected.add(package) + refresh_locks.add(package) + + dependents = reverse_dependencies(packages) + pending = list(selected) + while pending: + dependency = pending.pop() + for dependent in dependents[dependency]: + if dependent not in selected: + selected.add(dependent) + pending.append(dependent) + if dependency in refresh_locks: + refresh_locks.add(dependent) + + required = list(selected) + package_dependencies = dependencies(packages) + while required: + package = required.pop() + for dependency in package_dependencies[package]: + if dependency not in selected: + selected.add(dependency) + required.append(dependency) + + return sorted(selected), sorted(refresh_locks) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--bake-json", type=Path, required=True) + parser.add_argument("--changed-files", type=Path, required=True) + args = parser.parse_args() + + bake = json.loads(args.bake_json.read_text()) + changed_files = [line for line in args.changed_files.read_text().splitlines() if line] + targets, refresh_locks = select_targets(bake, changed_files) + print( + json.dumps( + { + "catalog_targets": sorted(package_targets(bake)), + "refresh_locks": refresh_locks, + "targets": targets, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/select_release_tag.py b/scripts/select_release_tag.py new file mode 100644 index 0000000..f16889c --- /dev/null +++ b/scripts/select_release_tag.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Allocate immutable daily release tags from the registry state.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import json +import subprocess +from typing import Callable, Iterable + + +@dataclass(frozen=True) +class RegistryEntry: + revision: str | None + candidate: str | None = None + + +@dataclass(frozen=True) +class ReleaseSelection: + """A safe action for a release tag discovered in the public registry.""" + + tag: str + status: str + candidate: str | None = None + + +Probe = Callable[[str, str], RegistryEntry | None] + + +def suffix_for_index(index: int) -> str: + """Return a, b, ..., z, aa, ab, ... for a zero-based index.""" + if index < 0: + raise ValueError("suffix index must be non-negative") + + characters: list[str] = [] + while True: + index, remainder = divmod(index, 26) + characters.append(chr(ord("a") + remainder)) + if index == 0: + return "".join(reversed(characters)) + index -= 1 + + +def tag_for_index(date: str, index: int) -> str: + """Return the bare date first, followed by alphabetically suffixed tags.""" + if index == 0: + return date + return f"{date}-{suffix_for_index(index - 1)}" + + +def select_release_tag( + date: str, + catalog_targets: Iterable[str], + selected_targets: Iterable[str], + revision: str, + probe: Probe, +) -> ReleaseSelection: + """Select a new, resumable, or complete immutable release tag. + + A partial public batch can only be resumed when every existing selected image + identifies the same private candidate. This prevents a retry from mixing + independently built images under one release tag. + """ + catalog = sorted(set(catalog_targets)) + selected = sorted(set(selected_targets)) + if not catalog or not selected: + raise ValueError("catalog and selected targets must both be non-empty") + if not set(selected).issubset(catalog): + raise ValueError("selected targets must be in the package catalog") + + for index in range(26**3): + tag = tag_for_index(date, index) + entries = {target: probe(target, tag) for target in catalog} + + selected_entries = [entries[target] for target in selected] + selected_present = [entry for entry in selected_entries if entry is not None] + batch_consistent = all(entry is None or entry.revision == revision for entry in entries.values()) + if selected_present and batch_consistent: + candidates = {entry.candidate for entry in selected_present} + if None in candidates or len(candidates) != 1: + raise RuntimeError( + f"release tag {tag} has inconsistent or missing candidate metadata" + ) + candidate = candidates.pop() + if len(selected_present) == len(selected) and all( + entry is not None and entry.revision == revision for entry in selected_entries + ): + return ReleaseSelection(tag, "complete", candidate) + return ReleaseSelection(tag, "resume", candidate) + + if all(entry is None for entry in entries.values()): + return ReleaseSelection(tag, "new") + + raise RuntimeError("could not find an available release suffix") + + +def registry_probe(registry: str) -> Probe: + def probe(target: str, tag: str) -> RegistryEntry | None: + reference = f"{registry}/{target}:{tag}" + result = subprocess.run( + ["docker", "buildx", "imagetools", "inspect", reference, "--format", "{{json .Config.Labels}}"], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + return None + try: + labels = json.loads(result.stdout) + except json.JSONDecodeError: + labels = {} + if not isinstance(labels, dict): + labels = {} + return RegistryEntry( + labels.get("org.opencontainers.image.revision"), + labels.get("org.ornl-mdf.containers.candidate"), + ) + + return probe + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--registry", required=True) + parser.add_argument("--date", required=True) + parser.add_argument("--revision", required=True) + parser.add_argument("--catalog-target", action="append", required=True) + parser.add_argument("--selected-target", action="append", required=True) + args = parser.parse_args() + + selection = select_release_tag( + args.date, + args.catalog_target, + args.selected_target, + args.revision, + registry_probe(args.registry), + ) + print( + json.dumps( + { + "candidate": selection.candidate, + "status": selection.status, + "tag": selection.tag, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/spack.yaml b/spack.yaml deleted file mode 100644 index 3627a46..0000000 --- a/spack.yaml +++ /dev/null @@ -1,5 +0,0 @@ -spack: - specs: - # Myna apps (can eventually be replaced with "myna +exaca +additivefoam") - - exaca@master - - additivefoam@main diff --git a/tests/test_dockerfiles.py b/tests/test_dockerfiles.py new file mode 100644 index 0000000..050cd20 --- /dev/null +++ b/tests/test_dockerfiles.py @@ -0,0 +1,19 @@ +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + + +class DockerfileTests(unittest.TestCase): + def test_ubuntu_uses_a_committed_lock_when_available(self): + dockerfile = (ROOT / "images" / "ubuntu" / "Dockerfile").read_text() + self.assertIn("COPY config/spack/ /tmp/spack-config/", dockerfile) + self.assertIn( + "if [ -f /tmp/spack-config/ubuntu.lock ]; then cp /tmp/spack-config/ubuntu.lock /opt/spack-environment/spack.lock; fi;", + dockerfile, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_generate_container_docs.py b/tests/test_generate_container_docs.py new file mode 100644 index 0000000..dd56725 --- /dev/null +++ b/tests/test_generate_container_docs.py @@ -0,0 +1,95 @@ +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "generate-container-docs.py" + + +class GenerateContainerDocsTests(unittest.TestCase): + def test_renders_embedded_inventory_and_updates_index(self): + with tempfile.TemporaryDirectory() as temp: + temp_path = Path(temp) + inventory = temp_path / "inventory" / "exaca" + inventory.mkdir(parents=True) + (inventory / "metadata.env").write_text( + "image=exaca\nbase_image=spack/ubuntu-noble@sha256:base\nrepository_revision=abc123\n" + ) + (inventory / "apt.tsv").write_text("cmake\t3.28.3\n") + (inventory / "spack.json").write_text(json.dumps([{"name": "exaca", "version": "1.2.3"}])) + + output = temp_path / "docs" + subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--image", + "exaca", + "--tag", + "2026-07-10", + "--digest", + "sha256:published", + "--inventory", + str(temp_path / "inventory"), + "--output-root", + str(output), + ], + check=True, + ) + + page = (output / "exaca" / "2026-07-10.md").read_text() + self.assertIn("`sha256:published`", page) + self.assertIn("`spack:exaca`", page) + self.assertIn("`1.2.3`", page) + self.assertIn("[exaca:2026-07-10](exaca/2026-07-10.md)", (output / "README.md").read_text()) + + def test_omits_software_shared_with_ubuntu(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + ubuntu = root / "ubuntu" / "ubuntu" + exaca = root / "exaca" / "exaca" + ubuntu.mkdir(parents=True) + exaca.mkdir(parents=True) + (ubuntu / "apt.tsv").write_text("cmake\t3.28.3\npython\t3.12.0\n") + (exaca / "apt.tsv").write_text( + "cmake\t3.28.3\nninja-build\t1.11.1\npython\t3.13.0\n" + ) + (exaca / "spack.json").write_text( + json.dumps([{"name": "exaca", "version": "1.2.3"}]) + ) + + output = root / "docs" + subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--image", + "exaca", + "--tag", + "test", + "--digest", + "sha256:test", + "--inventory", + str(root / "exaca"), + "--base-inventory", + str(root / "ubuntu"), + "--output-root", + str(output), + ], + check=True, + ) + + page = (output / "exaca" / "test.md").read_text() + self.assertIn("[ubuntu:test](../ubuntu/test.md)", page) + self.assertNotIn("`cmake`", page) + self.assertIn("`ninja-build`", page) + self.assertIn("`python` | `3.13.0`", page) + self.assertIn("`spack:exaca`", page) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_select_build_targets.py b/tests/test_select_build_targets.py new file mode 100644 index 0000000..7622641 --- /dev/null +++ b/tests/test_select_build_targets.py @@ -0,0 +1,72 @@ +from pathlib import Path +import sys +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +from select_build_targets import select_targets + + +BAKE = { + "target": { + "_common": {"dockerfile": "ignored"}, + "ubuntu": {"dockerfile": "images/ubuntu/Dockerfile"}, + "exaca": { + "dockerfile": "images/exaca/Dockerfile", + "contexts": {"ubuntu-base": "target:ubuntu"}, + }, + "thesis": { + "dockerfile": "images/thesis/Dockerfile", + "contexts": {"ubuntu-base": "target:ubuntu"}, + }, + "tooling": {"dockerfile": "images/not-tooling/Dockerfile"}, + } +} + + +class SelectBuildTargetsTests(unittest.TestCase): + def test_base_change_includes_reverse_dependencies(self): + self.assertEqual( + select_targets(BAKE, ["config/spack/base.yaml"]), + (["exaca", "thesis", "ubuntu"], ["exaca", "thesis", "ubuntu"]), + ) + + def test_manifest_change_selects_package_and_lock(self): + self.assertEqual( + select_targets(BAKE, ["config/spack/exaca.yaml"]), + (["exaca", "ubuntu"], ["exaca"]), + ) + + def test_shared_toolchain_manifest_rebuilds_dependents_and_lock(self): + self.assertEqual( + select_targets(BAKE, ["config/spack/ubuntu.yaml"]), + (["exaca", "thesis", "ubuntu"], ["exaca", "thesis", "ubuntu"]), + ) + + def test_image_change_selects_package(self): + self.assertEqual( + select_targets(BAKE, ["images/thesis/Dockerfile"]), + (["thesis", "ubuntu"], []), + ) + + def test_bake_change_includes_all_public_packages(self): + self.assertEqual( + select_targets(BAKE, ["docker-bake.hcl"]), + (["exaca", "thesis", "ubuntu"], []), + ) + + def test_new_bake_package_is_discovered(self): + bake = {"target": {**BAKE["target"], "newsolver": {"dockerfile": "images/newsolver/Dockerfile"}}} + self.assertEqual( + select_targets(bake, ["docker-bake.hcl"]), + (["exaca", "newsolver", "thesis", "ubuntu"], []), + ) + + def test_unrelated_change_selects_nothing(self): + self.assertEqual(select_targets(BAKE, ["README.md"]), ([], [])) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_select_release_tag.py b/tests/test_select_release_tag.py new file mode 100644 index 0000000..3e0bfd3 --- /dev/null +++ b/tests/test_select_release_tag.py @@ -0,0 +1,83 @@ +from pathlib import Path +import sys +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +from select_release_tag import RegistryEntry, ReleaseSelection, select_release_tag, suffix_for_index, tag_for_index + + +class SelectReleaseTagTests(unittest.TestCase): + def test_suffix_progression(self): + self.assertEqual([suffix_for_index(index) for index in (0, 25, 26, 27, 701)], ["a", "z", "aa", "ab", "zz"]) + + def test_release_tag_starts_with_bare_date(self): + self.assertEqual( + [tag_for_index("2026-07-10", index) for index in (0, 1, 2, 26, 27)], + ["2026-07-10", "2026-07-10-a", "2026-07-10-b", "2026-07-10-z", "2026-07-10-aa"], + ) + + def test_selects_first_globally_available_tag(self): + existing = {("ubuntu", "2026-07-10"): RegistryEntry("old-revision")} + + def probe(target, tag): + return existing.get((target, tag)) + + self.assertEqual( + select_release_tag("2026-07-10", ["ubuntu", "exaca"], ["exaca"], "new-revision", probe), + ReleaseSelection("2026-07-10-a", "new"), + ) + + def test_reuses_complete_matching_release(self): + existing = { + ("ubuntu", "2026-07-10"): RegistryEntry("revision", "candidate-1"), + ("exaca", "2026-07-10"): RegistryEntry("revision", "candidate-1"), + } + + def probe(target, tag): + return existing.get((target, tag)) + + self.assertEqual( + select_release_tag("2026-07-10", ["ubuntu", "exaca"], ["ubuntu", "exaca"], "revision", probe), + ReleaseSelection("2026-07-10", "complete", "candidate-1"), + ) + + def test_mismatched_batch_does_not_reuse_tag(self): + existing = { + ("ubuntu", "2026-07-10"): RegistryEntry("revision", "candidate-1"), + ("exaca", "2026-07-10"): RegistryEntry("other-revision"), + } + + def probe(target, tag): + return existing.get((target, tag)) + + self.assertEqual( + select_release_tag("2026-07-10", ["ubuntu", "exaca"], ["ubuntu"], "revision", probe), + ReleaseSelection("2026-07-10-a", "new"), + ) + + def test_resumes_a_consistent_partial_release(self): + existing = {("ubuntu", "2026-07-10"): RegistryEntry("revision", "candidate-1")} + + def probe(target, tag): + return existing.get((target, tag)) + + self.assertEqual( + select_release_tag("2026-07-10", ["ubuntu", "exaca"], ["ubuntu", "exaca"], "revision", probe), + ReleaseSelection("2026-07-10", "resume", "candidate-1"), + ) + + def test_rejects_partial_release_without_candidate_metadata(self): + existing = {("ubuntu", "2026-07-10"): RegistryEntry("revision")} + + def probe(target, tag): + return existing.get((target, tag)) + + with self.assertRaisesRegex(RuntimeError, "candidate metadata"): + select_release_tag("2026-07-10", ["ubuntu", "exaca"], ["ubuntu", "exaca"], "revision", probe) + + +if __name__ == "__main__": + unittest.main()