From 9b9f2a174c4800c62cfc7fe5fa0cdfc5409df1a3 Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Mon, 22 Jun 2026 13:11:59 -0500 Subject: [PATCH 01/25] pin action versions to sha supply chain best practice --- .github/workflows/release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f122d5d..02678e1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,7 +30,7 @@ jobs: name: build-${{ matrix.arch }}-${{ matrix.runtime }} timeout-minutes: 180 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 1 persist-credentials: false @@ -41,7 +41,7 @@ jobs: ls -lah dist/img - name: Upload artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: img-${{ matrix.arch }}-${{ matrix.runtime }} path: dist/img/*-${{ matrix.runtime }}.* @@ -59,12 +59,12 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 1 persist-credentials: false - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: dist/img From e81f0b82d94b278abaa315761c9ad61145aa5971 Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Mon, 22 Jun 2026 13:28:54 -0500 Subject: [PATCH 02/25] add make DIST variable --- Makefile | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index c38eba1..41e31e1 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ # image and tool versions include dependencies.env +# which dist image to build (debian or ubuntu) +DIST ?= ubuntu + # runtime RUNTIME ?= docker @@ -22,24 +25,29 @@ ifeq ($(strip $(OS_ARCH)),aarch64) BINFMT_ARCH = x86_64 endif +export ARCH +export BINFMT_ARCH + # # targets # all: image -.PHONY: clean +.PHONY: clean cloud-image clean: rm -rf dist -cloud-image: - ARCH=$(ARCH) UBUNTU_VERSION=$(UBUNTU_VERSION) UBUNTU_CODENAME=$(UBUNTU_CODENAME) scripts/cloud-image.sh +cloud-image: $(DIST)-cloud-image + +ubuntu-cloud-image: + UBUNTU_VERSION=$(UBUNTU_VERSION) UBUNTU_CODENAME=$(UBUNTU_CODENAME) scripts/cloud-image.sh binfmt: - ARCH=$(ARCH) BINFMT_ARCH=$(BINFMT_ARCH) BINFMT_VERSION=$(BINFMT_VERSION) BINFMT_QEMU_VERSION=$(BINFMT_QEMU_VERSION) scripts/binfmt.sh + BINFMT_VERSION=$(BINFMT_VERSION) BINFMT_QEMU_VERSION=$(BINFMT_QEMU_VERSION) scripts/binfmt.sh containerd: - ARCH=$(ARCH) NERDCTL_VERSION=$(NERDCTL_VERSION) FLANNEL_VERSION=$(FLANNEL_VERSION) FLANNEL_MINI_VERSION=$(FLANNEL_MINI_VERSION) RUNTIME=$(RUNTIME) scripts/containerd.sh + NERDCTL_VERSION=$(NERDCTL_VERSION) FLANNEL_VERSION=$(FLANNEL_VERSION) FLANNEL_MINI_VERSION=$(FLANNEL_MINI_VERSION) RUNTIME=$(RUNTIME) scripts/containerd.sh -image: cloud-image binfmt containerd - ARCH=$(ARCH) BINFMT_ARCH=$(BINFMT_ARCH) UBUNTU_VERSION=$(UBUNTU_VERSION) DOCKER_VERSION=$(DOCKER_VERSION) RUNTIME=$(RUNTIME) scripts/image.docker.sh +image: $(DIST)-cloud-image binfmt containerd + UBUNTU_VERSION=$(UBUNTU_VERSION) DOCKER_VERSION=$(DOCKER_VERSION) RUNTIME=$(RUNTIME) scripts/image.docker.sh From d16fc700e0dd7673bb73cb6479bbcd5e3b5bfc56 Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Wed, 24 Jun 2026 20:56:26 -0500 Subject: [PATCH 03/25] rewrite downloads scripts into makefile --- Makefile | 122 +++++++++++++++++++++++++++++++++++++---- dependencies.env | 17 +++--- scripts/binfmt.sh | 67 ---------------------- scripts/cloud-image.sh | 32 ----------- scripts/containerd.sh | 76 ------------------------- 5 files changed, 121 insertions(+), 193 deletions(-) delete mode 100755 scripts/binfmt.sh delete mode 100755 scripts/cloud-image.sh delete mode 100755 scripts/containerd.sh diff --git a/Makefile b/Makefile index 41e31e1..6b09b45 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,15 @@ # image and tool versions include dependencies.env +to_upper = $(shell echo '$1' | tr '[:lower:]' '[:upper:]') + +# quiet target commands by default +ifeq ($(filter $(DEBUG),1 true),) +MAKEFLAGS += -s +else +TARV = v +endif + # which dist image to build (debian or ubuntu) DIST ?= ubuntu @@ -17,7 +26,7 @@ endif # is required for Docker and asset downloads. ARCH_x86_64 = amd64 ARCH_aarch64 = arm64 -ARCH = $(shell echo "$(ARCH_$(OS_ARCH))") +ARCH = $(ARCH_$(OS_ARCH)) # binfmt needs the opposite of OS_ARCH BINFMT_ARCH = aarch64 @@ -28,26 +37,119 @@ endif export ARCH export BINFMT_ARCH +# containerd +CONTAINERD_ARCHIVE ?= dist/containerd/containerd-utils-$(ARCH).tar.gz +NERDCTL_FILE ?= dist/containerd/nerdctl-full-$(NERDCTL_VERSION)-linux-$(ARCH).tar.gz +NERDCTL_URL ?= https://github.com/containerd/nerdctl/releases/download/v$(NERDCTL_VERSION)/$(notdir $(NERDCTL_FILE)) +FLANNEL_FILE ?= dist/containerd/cni-plugin-flannel-linux-$(ARCH)-v$(FLANNEL_MINI_VERSION).tgz +FLANNEL_URL ?= https://github.com/flannel-io/cni-plugin/releases/download/v$(FLANNEL_VERSION)/$(notdir $(FLANNEL_FILE)) + +# binfmt +BINFMT_ARCHIVE = dist/binfmt/binfmt-$(ARCH).tar.gz +BINFMT_DOWNLOAD_URL ?= https://github.com/tonistiigi/binfmt/releases/download/$(BINFMT_VERSION) +BINFMT_FILE ?= dist/binfmt/$(BINFMT_VERSION)/binfmt_linux-$(ARCH).tar.gz +BINFMT_URL ?= $(BINFMT_DOWNLOAD_URL)/$(notdir $(BINFMT_FILE)) +BINFMT_QEMU_FILE ?= dist/binfmt/$(BINFMT_QEMU_VERSION)/qemu_v$(BINFMT_QEMU_VERSION)_linux-$(ARCH).tar.gz +BINFMT_QEMU_URL ?= $(BINFMT_DOWNLOAD_URL)/$(notdir $(BINFMT_QEMU_FILE)) + +# ubuntu +UBUNTU_IMAGE_BASE_URL ?= https://cloud-images.ubuntu.com/minimal/releases/$(UBUNTU_CODENAME)/release-$(UBUNTU_BUILD) +UBUNTU_IMAGE_FILE ?= dist/img/ubuntu-$(UBUNTU_VERSION)-minimal-cloudimg-$(ARCH).img +UBUNTU_IMAGE_SHA_FILE ?= $(UBUNTU_IMAGE_FILE).sha256sum + +# DIST resolved variables +IMAGE_BASE_URL ?= $($(call to_upper,$(DIST))_IMAGE_BASE_URL) +IMAGE_FILE ?= $($(call to_upper,$(DIST))_IMAGE_FILE) +IMAGE_SHA_FILE ?= $($(call to_upper,$(DIST))_IMAGE_SHA_FILE) +IMAGE_SHA_SIZE ?= $(patsubst .sha%sum,%,$(suffix $(IMAGE_SHA_FILE))) + +# +# defines +# + +define download_and_verify +$(1): + @echo "target: $$@" + mkdir -p $$(@D) && \ + curl -o$$@.download -L $(2) && \ + tar $$(TARV)xzOf $$@.download &> /dev/null || { \ + echo >&2 "error downloading"; \ + exit 1; \ + } + mv -f $$@.download $$@ +endef + # # targets # all: image -.PHONY: clean cloud-image +.PHONY: clean cloud-image binfmt containerd image clean: rm -rf dist -cloud-image: $(DIST)-cloud-image +# base image +cloud-image: $(IMAGE_SHA_FILE) + +$(IMAGE_FILE): + @echo "target: $@" + mkdir -p dist/img && cd dist/img && curl -o"$@" -L $(IMAGE_BASE_URL)/$(notdir $@) + +$(IMAGE_SHA_FILE): $(IMAGE_FILE) + @echo "target: $@" + shasum -a $(IMAGE_SHA_SIZE) $< > $@.tmp + cd dist/img && ( \ + curl -sL $(IMAGE_BASE_URL)/SHA$(IMAGE_SHA_SIZE)SUMS | \ + grep $(notdir $<) | \ + shasum -a $(IMAGE_SHA_SIZE) --check --status \ + ) || { \ + echo >&2 "checksum did not match!"; \ + rm -f $@.tmp; \ + mv -f $< $<.invalid; \ + exit 1; \ + } + mv $@.tmp $@ + +# checksum +%.sha512sum: % + @echo "target: $@" + shasum -a 512 $< > $@ + +# binfmt +binfmt: $(BINFMT_ARCHIVE).sha512sum + +$(BINFMT_ARCHIVE): $(BINFMT_FILE) $(BINFMT_QEMU_FILE) + @echo "target: $@" + rm -f '$@' + TMP_DIR=$$(mktemp -d); \ + trap 'rm -rf "$$TMP_DIR"' EXIT; \ + for f in $^; do tar $(TARX)zxf "$$f" -C "$$TMP_DIR"; done; \ + cd "$$TMP_DIR" && tar $(TARV)czf '$(CURDIR)/$@' binfmt qemu-i386 qemu-$(BINFMT_ARCH) || { \ + echo >&2 "failed to create $@" ; \ + rm -f '$(CURDIR)/$@'; exit 1; \ + } + +$(eval $(call download_and_verify,$(BINFMT_FILE),$(BINFMT_URL))) +$(eval $(call download_and_verify,$(BINFMT_QEMU_FILE),$(BINFMT_QEMU_URL))) -ubuntu-cloud-image: - UBUNTU_VERSION=$(UBUNTU_VERSION) UBUNTU_CODENAME=$(UBUNTU_CODENAME) scripts/cloud-image.sh +# containerd +containerd: $(CONTAINERD_ARCHIVE).sha512sum -binfmt: - BINFMT_VERSION=$(BINFMT_VERSION) BINFMT_QEMU_VERSION=$(BINFMT_QEMU_VERSION) scripts/binfmt.sh +$(CONTAINERD_ARCHIVE): $(NERDCTL_FILE) $(FLANNEL_FILE) + @echo "target: $@" + rm -f '$@' + TMP_DIR=$$(mktemp -d); \ + trap 'rm -rf "$$TMP_DIR"' EXIT; \ + for f in $^; do tar $(TARV)xzf "$$f" -C "$$TMP_DIR"; done; \ + cd "$$TMP_DIR" && tar $(TARV)czf '$(CURDIR)/$@' bin lib libexec share || { \ + echo >&2 "failed to create $@" ; \ + rm -f '$(CURDIR)/$@'; exit 1; \ + } -containerd: - NERDCTL_VERSION=$(NERDCTL_VERSION) FLANNEL_VERSION=$(FLANNEL_VERSION) FLANNEL_MINI_VERSION=$(FLANNEL_MINI_VERSION) RUNTIME=$(RUNTIME) scripts/containerd.sh +$(eval $(call download_and_verify,$(NERDCTL_FILE),$(NERDCTL_URL))) +$(eval $(call download_and_verify,$(FLANNEL_FILE),$(FLANNEL_URL))) -image: $(DIST)-cloud-image binfmt containerd +# primary target +image: cloud-image binfmt containerd UBUNTU_VERSION=$(UBUNTU_VERSION) DOCKER_VERSION=$(DOCKER_VERSION) RUNTIME=$(RUNTIME) scripts/image.docker.sh diff --git a/dependencies.env b/dependencies.env index 9572740..dd69de8 100644 --- a/dependencies.env +++ b/dependencies.env @@ -1,8 +1,9 @@ -UBUNTU_VERSION="24.04" -UBUNTU_CODENAME="noble" -BINFMT_VERSION="deploy/v7.0.0-28" -BINFMT_QEMU_VERSION="7.0.0" -DOCKER_VERSION="29.5.2" -NERDCTL_VERSION="2.3.1" -FLANNEL_VERSION="1.9.1-flannel1" -FLANNEL_MINI_VERSION="1.9.1" +UBUNTU_VERSION=24.04 +UBUNTU_CODENAME=noble +UBUNTU_BUILD=20260617 +BINFMT_VERSION=deploy/v7.0.0-28 +BINFMT_QEMU_VERSION=7.0.0 +DOCKER_VERSION=29.5.2 +NERDCTL_VERSION=2.3.1 +FLANNEL_VERSION=1.9.1-flannel1 +FLANNEL_MINI_VERSION=1.9.1 diff --git a/scripts/binfmt.sh b/scripts/binfmt.sh deleted file mode 100755 index 2c492d0..0000000 --- a/scripts/binfmt.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash - -set -eux - -# external variables that must be set -echo vars: $ARCH $BINFMT_ARCH - -SCRIPT_DIR=$(realpath "$(dirname "$(dirname $0)")") -DIST_DIR="${SCRIPT_DIR}/dist/binfmt" -mkdir -p $DIST_DIR - -TEMP_DIR=/tmp/binfmt -mkdir -p $TEMP_DIR -cd $TEMP_DIR - -download_binfmt() ( - # download archive - FILE="binfmt_linux-${1}.tar.gz" - URL="https://github.com/tonistiigi/binfmt/releases/download/${BINFMT_VERSION}/${FILE}" - curl -LO $URL - - # extract - tar xvfz $FILE -) - -download_qemu() ( - # download archive - FILE="qemu_v${BINFMT_QEMU_VERSION}_linux-${1}.tar.gz" - URL="https://github.com/tonistiigi/binfmt/releases/download/${BINFMT_VERSION}/${FILE}" - curl -LO $URL - - # extract - tar xvfz $FILE -) - -create_archive() ( - # move required components - FILE="binfmt-${1}.tar.gz" - tar cfz $FILE \ - binfmt \ - qemu-i386 \ - qemu-${2} - - shasum -a 512 "${FILE}" >"${FILE}.sha512sum" -) - -copy_to_dist() ( - # copy to dist dir - cp "binfmt-${1}.tar.gz" $DIST_DIR - cp "binfmt-${1}.tar.gz.sha512sum" $DIST_DIR - - # cleanup - rm -rf "$TEMP_DIR/*" -) - -download() ( - download_binfmt "${1}" - download_qemu "${1}" - create_archive "${1}" "${2}" - copy_to_dist "${1}" -) - -# download -download $ARCH $BINFMT_ARCH - -echo download successful -ls -lh $DIST_DIR diff --git a/scripts/cloud-image.sh b/scripts/cloud-image.sh deleted file mode 100755 index fc674a8..0000000 --- a/scripts/cloud-image.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -set -eux - -# external variables that must be set -echo vars: $ARCH - -# switch to dist dir -SCRIPT_DIR=$(realpath "$(dirname "$(dirname $0)")") -DIST_DIR="${SCRIPT_DIR}/dist/img" -mkdir -p $DIST_DIR - -cd $DIST_DIR - -download() ( - FILE="ubuntu-${UBUNTU_VERSION}-minimal-cloudimg-${1}.img" - URL="https://cloud-images.ubuntu.com/minimal/releases/${UBUNTU_CODENAME}/release/${FILE}" - curl -LO $URL - - shasum -a 512 "${FILE}" >"${FILE}.sha512sum" -) - -# download -download $ARCH - -# validate -( - curl -sL https://cloud-images.ubuntu.com/minimal/releases/${UBUNTU_CODENAME}/release/SHA256SUMS | grep "${ARCH}\.img$" | shasum -a 256 --check --status -) - -echo download successful -ls -lh . diff --git a/scripts/containerd.sh b/scripts/containerd.sh deleted file mode 100755 index 5e467bc..0000000 --- a/scripts/containerd.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash - -if [ "$RUNTIME" != "containerd" ]; then - exit 0 -fi - -set -ex - -SCRIPT_DIR=$(realpath "$(dirname "$(dirname $0)")") -DIST_DIR="${SCRIPT_DIR}/dist/containerd" -mkdir -p $DIST_DIR - -TEMP_DIR=/tmp/containerd -mkdir -p $TEMP_DIR -cd $TEMP_DIR - -download_containerd() ( - # download archive - FILE="nerdctl-full-${NERDCTL_VERSION}-linux-${1}.tar.gz" - URL="https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VERSION}/${FILE}" - curl -LO $URL - - # validate - curl -sL "https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VERSION}/SHA256SUMS" | grep "${FILE}" | shasum -a 256 --check --status - - # extract - tar xvfz $FILE -) - -download_flannel() ( - # download archive - FILE="cni-plugin-flannel-linux-${1}-v${FLANNEL_MINI_VERSION}.tgz" - URL="https://github.com/flannel-io/cni-plugin/releases/download/v${FLANNEL_VERSION}/${FILE}" - curl -LO $URL - - # validate - curl -sL "${URL}.sha512" | shasum -a 512 --check --status - - # extract - tar xvfz "$FILE" - mv "flannel-${1}" libexec/cni/flannel -) - -create_archive() ( - # move required components - FILE="containerd-utils-${1}.tar.gz" - tar cfz $FILE \ - bin \ - lib \ - libexec \ - share - - shasum -a 512 "${FILE}" >"${FILE}.sha512sum" -) - -copy_to_dist() ( - # copy to dist dir - cp "containerd-utils-${1}.tar.gz" $DIST_DIR - cp "containerd-utils-${1}.tar.gz.sha512sum" $DIST_DIR - - # cleanup - rm -rf "$TEMP_DIR/*" -) - -download() ( - download_containerd "${1}" - download_flannel "${1}" - create_archive "${1}" - copy_to_dist "${1}" -) - -# download -download $ARCH - -echo download successful -ls -lh $DIST_DIR From e3b8d842079cbbc3c5c0801511f0c3d3ee7db68a Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Wed, 24 Jun 2026 21:14:24 -0500 Subject: [PATCH 04/25] quiet apt for log output --- scripts/image.sh | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/image.sh b/scripts/image.sh index 48a0c25..093c43c 100755 --- a/scripts/image.sh +++ b/scripts/image.sh @@ -17,8 +17,8 @@ CHROOT_DIR=/mnt/colima-img FILE="$IMG_DIR/$FILENAME" install_dependencies() ( - apt-get update - apt-get install -y file fdisk libdigest-sha-perl qemu-utils + apt-get -qq update + apt-get -qq install -y file fdisk libdigest-sha-perl qemu-utils ) convert_file() ( @@ -52,16 +52,16 @@ install_packages() ( echo 'nameserver 1.1.1.1' >$CHROOT_DIR/etc/resolv.conf # prepare packages - chroot_exec apt-get update + chroot_exec apt-get -qq update # packages common to all runtimes, to prevent from final purging - chroot_exec apt-get install -y iptables socat sshfs cloud-init lsb-release python3-apt gnupg curl wget dnsmasq + chroot_exec apt-get -qq install -y iptables socat sshfs cloud-init lsb-release python3-apt gnupg curl wget dnsmasq # none if [ "$RUNTIME" == "none" ]; then ( - chroot_exec apt-get install -y htop inetutils-ping dnsutils net-tools netcat-openbsd telnet vim-tiny nano - chroot_exec apt-get purge -y dmsetup xz-utils + chroot_exec apt-get -qq install -y htop inetutils-ping dnsutils net-tools netcat-openbsd telnet vim-tiny nano + chroot_exec apt-get -qq purge -y dmsetup xz-utils ) fi @@ -72,7 +72,7 @@ install_packages() ( chroot_exec sh /tmp/get-docker.sh --version $DOCKER_VERSION chroot_exec rm /tmp/get-docker.sh chroot_exec apt-mark hold docker-ce docker-ce-cli containerd.io - chroot_exec apt-get purge -y dmsetup xz-utils + chroot_exec apt-get -qq purge -y dmsetup xz-utils ) fi @@ -83,7 +83,7 @@ install_packages() ( tar Cxfz ${CHROOT_DIR}/usr/local /build/dist/containerd/containerd-utils-${ARCH}.tar.gz chroot_exec mkdir -p /opt/cni chroot_exec mv /usr/local/libexec/cni /opt/cni/bin - chroot_exec apt-get purge -y dmsetup xz-utils + chroot_exec apt-get -qq purge -y dmsetup xz-utils ) fi @@ -102,15 +102,15 @@ Architectures: $(dpkg --print-architecture) Signed-By: /etc/apt/keyrings/zabbly.asc EOF' - chroot_exec apt-get update - chroot_exec apt-get install -y htop inetutils-ping dnsutils net-tools netcat-openbsd telnet vim-tiny nano - chroot_exec apt-get install -y incus incus-base incus-client incus-extra incus-ui-canonical zfsutils-linux btrfs-progs lvm2 thin-provisioning-tools + chroot_exec apt-get -qq update + chroot_exec apt-get -qq install -y htop inetutils-ping dnsutils net-tools netcat-openbsd telnet vim-tiny nano + chroot_exec apt-get -qq install -y incus incus-base incus-client incus-extra incus-ui-canonical zfsutils-linux btrfs-progs lvm2 thin-provisioning-tools chroot_exec apt-mark hold incus incus-base incus-client incus-extra incus-ui-canonical zfsutils-linux btrfs-progs lvm2 thin-provisioning-tools ) fi - chroot_exec apt-get purge -y apport console-setup-linux dbus-user-session liblocale-gettext-perl lxd-agent-loader lxd-installer parted pciutils pollinate python3-gi snapd ssh-import-id - chroot_exec apt-get purge -y ubuntu-advantage-tools ubuntu-cloud-minimal ubuntu-drivers-common ubuntu-release-upgrader-core unattended-upgrades systemd-resolved + chroot_exec apt-get -qq purge -y apport console-setup-linux dbus-user-session liblocale-gettext-perl lxd-agent-loader lxd-installer parted pciutils pollinate python3-gi snapd ssh-import-id + chroot_exec apt-get -qq purge -y ubuntu-advantage-tools ubuntu-cloud-minimal ubuntu-drivers-common ubuntu-release-upgrader-core unattended-upgrades systemd-resolved chroot_exec apt-get autoremove -y chroot_exec apt-get clean -y From 757a7b4724b800c838067da9b82c7429f5e841d8 Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Wed, 24 Jun 2026 21:39:22 -0500 Subject: [PATCH 05/25] fix building images on fips enabled hosts https://bugs.launchpad.net/ubuntu/+source/openssl/+bug/2141933 --- scripts/image.docker.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/image.docker.sh b/scripts/image.docker.sh index c5bf403..dc69977 100755 --- a/scripts/image.docker.sh +++ b/scripts/image.docker.sh @@ -11,6 +11,9 @@ echo vars: $ARCH $BINFMT_ARCH $UBUNTU_VERSION $DOCKER_VERSION $RUNTIME # computed variables SCRIPT_DIR=$(realpath "$(dirname "$(dirname $0)")") +# https://bugs.launchpad.net/ubuntu/+source/openssl/+bug/2141933 +export OPENSSL_FORCE_FIPS_MODE=0 + # dependencies in case of cross-arch docker run --privileged --rm tonistiigi/binfmt --install $BINFMT_ARCH @@ -23,4 +26,5 @@ docker run --rm --privileged \ --env UBUNTU_VERSION \ --env DOCKER_VERSION \ --env RUNTIME \ + --env OPENSSL_FORCE_FIPS_MODE \ ubuntu:${UBUNTU_VERSION} /build/scripts/image.sh From 3a6a7c48fea7a621f61160b5916c2bf062d7b9ec Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Wed, 24 Jun 2026 22:07:04 -0500 Subject: [PATCH 06/25] use parallel gzip --- scripts/image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/image.sh b/scripts/image.sh index 093c43c..1a62d57 100755 --- a/scripts/image.sh +++ b/scripts/image.sh @@ -18,7 +18,7 @@ FILE="$IMG_DIR/$FILENAME" install_dependencies() ( apt-get -qq update - apt-get -qq install -y file fdisk libdigest-sha-perl qemu-utils + apt-get -qq install -y file fdisk libdigest-sha-perl qemu-utils pigz ) convert_file() ( @@ -144,7 +144,7 @@ EOF compress_file() ( raw_file="${FILE}-${RUNTIME}.raw" mv $FILE.raw $raw_file - gzip -9 -n -f $raw_file + pigz -9 -n -f $raw_file dir="$(dirname $raw_file)" filename="$(basename $raw_file)" (cd $dir && shasum -a 512 "${filename}.gz" >"${filename}.gz.sha512sum") From dd035afc952bf35a891ddd1d23419e01508dc89f Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Wed, 24 Jun 2026 23:56:01 -0500 Subject: [PATCH 07/25] convert RUNTIMEs into targets --- .github/workflows/release.yml | 2 +- Makefile | 39 +++++++++++++++++++++++++---------- README.md | 17 +++++++++++---- 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 02678e1..5cb7221 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,7 +37,7 @@ jobs: - name: Build run: | - make image OS_ARCH=${{ matrix.arch }} RUNTIME=${{ matrix.runtime }} + make OS_ARCH=${{ matrix.arch }} ${{ matrix.runtime }} ls -lah dist/img - name: Upload artifact diff --git a/Makefile b/Makefile index 6b09b45..1270971 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,18 @@ # image and tool versions include dependencies.env +UNAME_S := $(shell uname -s) + to_upper = $(shell echo '$1' | tr '[:lower:]' '[:upper:]') +ifeq ($(UNAME_S),Darwin) + # macOS uses BSD stat + PRINT_STATS_CMD = stat -f "%N%n |- %z bytes%n '- %Sm" +else + # Linux (and most others) use GNU stat + PRINT_STATS_CMD = stat -c "%N\n |- %s bytes\n '- %y" +endif + # quiet target commands by default ifeq ($(filter $(DEBUG),1 true),) MAKEFLAGS += -s @@ -14,6 +24,7 @@ endif DIST ?= ubuntu # runtime +RUNTIMES = docker containerd incus none RUNTIME ?= docker # architecture defaults to the current system's. @@ -83,15 +94,21 @@ endef # targets # -all: image +.PHONY: clean distclean image $(RUNTIMES) + +# deprecated (default) target +image: $(RUNTIME) + +all: $(RUNTIMES) -.PHONY: clean cloud-image binfmt containerd image clean: + @echo "target: $@" + rm -rf dist/img/*.raw.gz* + +distclean: rm -rf dist # base image -cloud-image: $(IMAGE_SHA_FILE) - $(IMAGE_FILE): @echo "target: $@" mkdir -p dist/img && cd dist/img && curl -o"$@" -L $(IMAGE_BASE_URL)/$(notdir $@) @@ -117,8 +134,6 @@ $(IMAGE_SHA_FILE): $(IMAGE_FILE) shasum -a 512 $< > $@ # binfmt -binfmt: $(BINFMT_ARCHIVE).sha512sum - $(BINFMT_ARCHIVE): $(BINFMT_FILE) $(BINFMT_QEMU_FILE) @echo "target: $@" rm -f '$@' @@ -134,8 +149,6 @@ $(eval $(call download_and_verify,$(BINFMT_FILE),$(BINFMT_URL))) $(eval $(call download_and_verify,$(BINFMT_QEMU_FILE),$(BINFMT_QEMU_URL))) # containerd -containerd: $(CONTAINERD_ARCHIVE).sha512sum - $(CONTAINERD_ARCHIVE): $(NERDCTL_FILE) $(FLANNEL_FILE) @echo "target: $@" rm -f '$@' @@ -150,6 +163,10 @@ $(CONTAINERD_ARCHIVE): $(NERDCTL_FILE) $(FLANNEL_FILE) $(eval $(call download_and_verify,$(NERDCTL_FILE),$(NERDCTL_URL))) $(eval $(call download_and_verify,$(FLANNEL_FILE),$(FLANNEL_URL))) -# primary target -image: cloud-image binfmt containerd - UBUNTU_VERSION=$(UBUNTU_VERSION) DOCKER_VERSION=$(DOCKER_VERSION) RUNTIME=$(RUNTIME) scripts/image.docker.sh +# images +$(basename $(UBUNTU_IMAGE_FILE))-%.raw.gz: $(UBUNTU_IMAGE_SHA_FILE) $(CONTAINERD_ARCHIVE).sha512sum $(BINFMT_ARCHIVE).sha512sum + UBUNTU_VERSION=$(UBUNTU_VERSION) DOCKER_VERSION=$(DOCKER_VERSION) RUNTIME=$* scripts/image.docker.sh + touch "$@" + +$(RUNTIMES): %: $(basename $(IMAGE_FILE))-%.raw.gz + $(PRINT_STATS_CMD) $< diff --git a/README.md b/README.md index e61a59e..1e01308 100644 --- a/README.md +++ b/README.md @@ -7,17 +7,26 @@ Dependencies for Colima Generate a raw disk image compressed with gzip (`.raw.gz`) for the OS architecture and default runtime (docker). ```sh -make image +make ``` Generate a `.raw.gz` image for another architecture. `OS_ARCH` must be one of `aarch64`, `x86_64` ```sh -OS_ARCH=x86_64 make image +OS_ARCH=x86_64 make ``` -Generate a `.raw.gz` image for another runtime. `RUNTIME` must be one of `docker`, `containerd`, `incus`, `none` +Generate a `.raw.gz` image for another runtime. ```sh -RUNTIME=containerd make image +make docker # default make target +make containerd +make incus +make none +``` + +Generate `.raw.gz` images for all runtimes. + +```sh +make all ``` From 8913f0f98d98ffb29d41fc124a7344ded01284d5 Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Thu, 25 Jun 2026 10:30:24 -0500 Subject: [PATCH 08/25] use kpartx instead of offset hunting with fdisk --- scripts/image.sh | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/scripts/image.sh b/scripts/image.sh index 1a62d57..0bd5109 100755 --- a/scripts/image.sh +++ b/scripts/image.sh @@ -18,21 +18,42 @@ FILE="$IMG_DIR/$FILENAME" install_dependencies() ( apt-get -qq update - apt-get -qq install -y file fdisk libdigest-sha-perl qemu-utils pigz + apt-get -qq install -y file kpartx libdigest-sha-perl qemu-utils pigz ) convert_file() ( qemu-img convert -p -f qcow2 -O raw $FILE.img $FILE.raw ) -extract_partition_offset() ( - fdisk -l $FILE.raw | grep "$FILE.raw1 " | awk -F' ' '{print $2}' -) - -mount_partition() ( +mount_partition() { + LOOP_DEV=$(losetup -Pf --show "$FILE.raw") + kpartx -avs "$LOOP_DEV" + LOOP_NAME=$(basename "$LOOP_DEV") + ROOT_PART="/dev/mapper/${LOOP_NAME}p1" + ROOT_UUID=$(blkid -s UUID -o value "$ROOT_PART") + mkdir -p "/dev/disk/by-uuid" + ln -s "/dev/mapper/${LOOP_NAME}p1" "/dev/disk/by-uuid/$ROOT_UUID" mkdir -p $CHROOT_DIR - mount -o loop,offset=$(($1 * 512)) $FILE.raw $CHROOT_DIR -) + mount "$ROOT_PART" "$CHROOT_DIR" +} + +cleanup() { + if [ -n "$CHROOT_DIR" ] && mountpoint -q "$CHROOT_DIR"; then + umount "$CHROOT_DIR/dev/pts" ||: + umount "$CHROOT_DIR/proc" ||: + umount "$CHROOT_DIR" + fi + if [ -n "$ROOT_UUID" ] && [ -h "/dev/disk/by-uuid/$ROOT_UUID" ]; then + rm -f "/dev/disk/by-uuid/$ROOT_UUID" + fi + if [ -n "$LOOP_DEV" ]; then + kpartx -dvs "$LOOP_DEV" 2>/dev/null || true + fi + if [ -n "$LOOP_DEV" ] && losetup "$LOOP_DEV" >/dev/null 2>&1; then + losetup -d "$LOOP_DEV" + fi +} +trap 'cleanup' EXIT unmount_partition() ( umount $CHROOT_DIR @@ -153,7 +174,7 @@ compress_file() ( # perform all actions install_dependencies convert_file -mount_partition "$(extract_partition_offset)" +mount_partition install_packages unmount_partition compress_file From 6bf842e72e1a25f39e3df311a87611d0d077b777 Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Thu, 25 Jun 2026 10:51:12 -0500 Subject: [PATCH 09/25] image target rebuild on script changes --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1270971..af5db92 100644 --- a/Makefile +++ b/Makefile @@ -164,7 +164,7 @@ $(eval $(call download_and_verify,$(NERDCTL_FILE),$(NERDCTL_URL))) $(eval $(call download_and_verify,$(FLANNEL_FILE),$(FLANNEL_URL))) # images -$(basename $(UBUNTU_IMAGE_FILE))-%.raw.gz: $(UBUNTU_IMAGE_SHA_FILE) $(CONTAINERD_ARCHIVE).sha512sum $(BINFMT_ARCHIVE).sha512sum +$(basename $(UBUNTU_IMAGE_FILE))-%.raw.gz: $(UBUNTU_IMAGE_SHA_FILE) $(CONTAINERD_ARCHIVE).sha512sum $(BINFMT_ARCHIVE).sha512sum scripts/image.docker.sh scripts/image.sh UBUNTU_VERSION=$(UBUNTU_VERSION) DOCKER_VERSION=$(DOCKER_VERSION) RUNTIME=$* scripts/image.docker.sh touch "$@" From 6de4409becc3821c5e767b6b9d9bf516459bab7e Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Thu, 25 Jun 2026 14:06:30 -0500 Subject: [PATCH 10/25] implement builder image --- .gitignore | 1 + Makefile | 30 +++++++++++++++++++++++++----- scripts/Dockerfile | 21 +++++++++++++++++++++ scripts/image.docker.sh | 30 ------------------------------ scripts/image.sh | 37 +++++++++++++------------------------ 5 files changed, 60 insertions(+), 59 deletions(-) create mode 100644 scripts/Dockerfile delete mode 100755 scripts/image.docker.sh diff --git a/.gitignore b/.gitignore index 849ddff..5401e88 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ dist/ +scripts/.build-image-stamp diff --git a/Makefile b/Makefile index af5db92..bcbd824 100644 --- a/Makefile +++ b/Makefile @@ -90,6 +90,12 @@ $(1): mv -f $$@.download $$@ endef +# image builder container image +DOCKER_BUILD_IMAGE = scripts/.build-image-stamp +DOCKER_BUILD_IMAGE_SOURCES = scripts/Dockerfile scripts/image.sh +DOCKER_BUILD_IMAGE_TAG = colima-core-builder:latest + +IMAGE_DEPENDENCIES = $(DOCKER_BUILD_IMAGE) $(CONTAINERD_ARCHIVE).sha512sum $(BINFMT_ARCHIVE).sha512sum # # targets # @@ -101,11 +107,12 @@ image: $(RUNTIME) all: $(RUNTIMES) +# rm build targets clean: - @echo "target: $@" - rm -rf dist/img/*.raw.gz* + rm -rf $(IMAGE_DEPENDENCIES) dist/img/*.raw.gz* -distclean: +# rm + cache +distclean: clean rm -rf dist # base image @@ -163,9 +170,22 @@ $(CONTAINERD_ARCHIVE): $(NERDCTL_FILE) $(FLANNEL_FILE) $(eval $(call download_and_verify,$(NERDCTL_FILE),$(NERDCTL_URL))) $(eval $(call download_and_verify,$(FLANNEL_FILE),$(FLANNEL_URL))) +# builder +$(DOCKER_BUILD_IMAGE): $(DOCKER_BUILD_IMAGE_SOURCES) Makefile + docker build --build-arg UBUNTU_VERSION=$(UBUNTU_VERSION) -t $(DOCKER_BUILD_IMAGE_TAG) --iidfile $@ $(dir $@) + # images -$(basename $(UBUNTU_IMAGE_FILE))-%.raw.gz: $(UBUNTU_IMAGE_SHA_FILE) $(CONTAINERD_ARCHIVE).sha512sum $(BINFMT_ARCHIVE).sha512sum scripts/image.docker.sh scripts/image.sh - UBUNTU_VERSION=$(UBUNTU_VERSION) DOCKER_VERSION=$(DOCKER_VERSION) RUNTIME=$* scripts/image.docker.sh +$(basename $(UBUNTU_IMAGE_FILE))-%.raw.gz: $(UBUNTU_IMAGE_SHA_FILE) $(IMAGE_DEPENDENCIES) $(DOCKER_BUILD_IMAGE) Makefile + if [ $(OS_ARCH) != $(ARCH) ] ; then docker run --privileged --rm tonistiigi/binfmt --install $(BINFMT_ARCH); fi + docker run --rm --privileged \ + --platform linux/$(ARCH) \ + --volume $(CURDIR):/build \ + --env BINFMT_ARCHIVE=$(BINFMT_ARCHIVE) \ + --env CONTAINERD_ARCHIVE=$(CONTAINERD_ARCHIVE) \ + --env IMAGE_FILE=$(IMAGE_FILE) \ + --env DOCKER_VERSION=$(DOCKER_VERSION) \ + --env RUNTIME=$* \ + $(DOCKER_BUILD_IMAGE_TAG) touch "$@" $(RUNTIMES): %: $(basename $(IMAGE_FILE))-%.raw.gz diff --git a/scripts/Dockerfile b/scripts/Dockerfile new file mode 100644 index 0000000..2754248 --- /dev/null +++ b/scripts/Dockerfile @@ -0,0 +1,21 @@ +ARG UBUNTU_VERSION + +FROM ubuntu:${UBUNTU_VERSION?} AS os + +ENV DEBIAN_FRONTEND=noninteractive +# https://bugs.launchpad.net/ubuntu/+source/openssl/+bug/2141933 +ENV OPENSSL_FORCE_FIPS_MODE=0 +SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] + +RUN --mount=type=cache,target=/var/cache/apt,rw --mount=type=cache,target=/var/lib/apt,rw < /etc/apt/apt.conf.d/colima +apt-get -qq update +apt-get -qq upgrade +apt-get -qq install -y file kpartx libdigest-sha-perl qemu-utils pigz +EOD + +COPY image.sh /image.sh + +VOLUME /build + +CMD ["/image.sh"] diff --git a/scripts/image.docker.sh b/scripts/image.docker.sh deleted file mode 100755 index dc69977..0000000 --- a/scripts/image.docker.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -set -eux - -# disable apt prompts -export DEBIAN_FRONTEND=noninteractive - -# external variables that must be set -echo vars: $ARCH $BINFMT_ARCH $UBUNTU_VERSION $DOCKER_VERSION $RUNTIME - -# computed variables -SCRIPT_DIR=$(realpath "$(dirname "$(dirname $0)")") - -# https://bugs.launchpad.net/ubuntu/+source/openssl/+bug/2141933 -export OPENSSL_FORCE_FIPS_MODE=0 - -# dependencies in case of cross-arch -docker run --privileged --rm tonistiigi/binfmt --install $BINFMT_ARCH - -# build disk image -docker run --rm --privileged \ - --platform linux/$ARCH \ - --volume $SCRIPT_DIR:/build \ - --env ARCH \ - --env BINFMT_ARCH \ - --env UBUNTU_VERSION \ - --env DOCKER_VERSION \ - --env RUNTIME \ - --env OPENSSL_FORCE_FIPS_MODE \ - ubuntu:${UBUNTU_VERSION} /build/scripts/image.sh diff --git a/scripts/image.sh b/scripts/image.sh index 0bd5109..8153a2d 100755 --- a/scripts/image.sh +++ b/scripts/image.sh @@ -6,27 +6,20 @@ set -eux export DEBIAN_FRONTEND=noninteractive # external variables that must be set -echo vars: $ARCH $BINFMT_ARCH $UBUNTU_VERSION $DOCKER_VERSION $RUNTIME +echo vars: $DOCKER_VERSION $RUNTIME -FILENAME="ubuntu-${UBUNTU_VERSION}-minimal-cloudimg-${ARCH}" - -SCRIPT_DIR=$(realpath "$(dirname "$(dirname $0)")") -IMG_DIR="$SCRIPT_DIR/dist/img" +BUILD_DIR="/build" CHROOT_DIR=/mnt/colima-img -FILE="$IMG_DIR/$FILENAME" - -install_dependencies() ( - apt-get -qq update - apt-get -qq install -y file kpartx libdigest-sha-perl qemu-utils pigz -) +IMAGE_FILE="${BUILD_DIR}/${IMAGE_FILE}" +RAW_FILE="${IMAGE_FILE%.*}-${RUNTIME}.raw" convert_file() ( - qemu-img convert -p -f qcow2 -O raw $FILE.img $FILE.raw + qemu-img convert -p -f qcow2 -O raw "${IMAGE_FILE}" "${RAW_FILE}" ) mount_partition() { - LOOP_DEV=$(losetup -Pf --show "$FILE.raw") + LOOP_DEV=$(losetup -Pf --show "${RAW_FILE}") kpartx -avs "$LOOP_DEV" LOOP_NAME=$(basename "$LOOP_DEV") ROOT_PART="/dev/mapper/${LOOP_NAME}p1" @@ -101,7 +94,7 @@ install_packages() ( if [ "$RUNTIME" == "containerd" ]; then ( cd /tmp - tar Cxfz ${CHROOT_DIR}/usr/local /build/dist/containerd/containerd-utils-${ARCH}.tar.gz + tar Cxfz ${CHROOT_DIR}/usr/local "${BUILD_DIR}/${CONTAINERD_ARCHIVE}" chroot_exec mkdir -p /opt/cni chroot_exec mv /usr/local/libexec/cni /opt/cni/bin chroot_exec apt-get -qq purge -y dmsetup xz-utils @@ -140,9 +133,9 @@ EOF' # binfmt ( cd /tmp - tar xfz /build/dist/binfmt/binfmt-${ARCH}.tar.gz - chown root:root binfmt qemu-i386 qemu-${BINFMT_ARCH} - mv binfmt qemu-i386 qemu-${BINFMT_ARCH} ${CHROOT_DIR}/usr/bin + tar xfz "${BUILD_DIR}/${BINFMT_ARCHIVE}" + chown root:root binfmt qemu-* + mv binfmt qemu-* ${CHROOT_DIR}/usr/bin ) # enable vsock modules at boot @@ -152,6 +145,7 @@ virtio_vsock EOF # clean traces + chroot_exec find /tmp -mindepth 1 -delete chroot_exec rm /etc/resolv.conf chroot_exec mv /etc/resolv.conf.bak /etc/resolv.conf chroot_exec umount /dev/pts @@ -163,16 +157,11 @@ EOF ) compress_file() ( - raw_file="${FILE}-${RUNTIME}.raw" - mv $FILE.raw $raw_file - pigz -9 -n -f $raw_file - dir="$(dirname $raw_file)" - filename="$(basename $raw_file)" - (cd $dir && shasum -a 512 "${filename}.gz" >"${filename}.gz.sha512sum") + pigz -9 -n -f "${RAW_FILE}" + shasum -a 512 "${RAW_FILE}.gz" >"${RAW_FILE}.gz.sha512sum" ) # perform all actions -install_dependencies convert_file mount_partition install_packages From f279605336ad8bc1639b39011ec256cb44571284 Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Thu, 25 Jun 2026 14:30:28 -0500 Subject: [PATCH 11/25] fix cleanup on error or abort --- Makefile | 7 +++++-- scripts/image.sh | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index bcbd824..07dc549 100644 --- a/Makefile +++ b/Makefile @@ -177,7 +177,7 @@ $(DOCKER_BUILD_IMAGE): $(DOCKER_BUILD_IMAGE_SOURCES) Makefile # images $(basename $(UBUNTU_IMAGE_FILE))-%.raw.gz: $(UBUNTU_IMAGE_SHA_FILE) $(IMAGE_DEPENDENCIES) $(DOCKER_BUILD_IMAGE) Makefile if [ $(OS_ARCH) != $(ARCH) ] ; then docker run --privileged --rm tonistiigi/binfmt --install $(BINFMT_ARCH); fi - docker run --rm --privileged \ + docker run --rm -i --tty --privileged \ --platform linux/$(ARCH) \ --volume $(CURDIR):/build \ --env BINFMT_ARCHIVE=$(BINFMT_ARCHIVE) \ @@ -185,7 +185,10 @@ $(basename $(UBUNTU_IMAGE_FILE))-%.raw.gz: $(UBUNTU_IMAGE_SHA_FILE) $(IMAGE_DEPE --env IMAGE_FILE=$(IMAGE_FILE) \ --env DOCKER_VERSION=$(DOCKER_VERSION) \ --env RUNTIME=$* \ - $(DOCKER_BUILD_IMAGE_TAG) + $(DOCKER_BUILD_IMAGE_TAG) || { \ + echo >&2 "failed to create $@"; \ + rm -f '$@'* ; exit 1; \ + } touch "$@" $(RUNTIMES): %: $(basename $(IMAGE_FILE))-%.raw.gz diff --git a/scripts/image.sh b/scripts/image.sh index 8153a2d..6332f84 100755 --- a/scripts/image.sh +++ b/scripts/image.sh @@ -31,6 +31,7 @@ mount_partition() { } cleanup() { + rm -f "${RAW_FILE}" if [ -n "$CHROOT_DIR" ] && mountpoint -q "$CHROOT_DIR"; then umount "$CHROOT_DIR/dev/pts" ||: umount "$CHROOT_DIR/proc" ||: From 42fb94ecac1454c8567d309495a5f864736f36ad Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Thu, 25 Jun 2026 15:12:22 -0500 Subject: [PATCH 12/25] Debian OS image support --- .github/workflows/release.yml | 11 ++++--- Makefile | 8 ++++- README.md | 8 ++++- dependencies.env | 3 ++ scripts/image.sh | 59 +++++++++++++++++++++++++---------- 5 files changed, 65 insertions(+), 24 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5cb7221..52af820 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,18 +16,19 @@ concurrency: jobs: build: strategy: - max-parallel: 2 + max-parallel: 4 fail-fast: true matrix: arch: [x86_64, aarch64] runtime: [none, docker, containerd, incus] + dist: [ubuntu, debian] include: - arch: x86_64 os: ubuntu-latest - arch: aarch64 os: ubuntu-24.04-arm runs-on: ${{ matrix.os }} - name: build-${{ matrix.arch }}-${{ matrix.runtime }} + name: build-${{ matrix.arch }}-${{ matrix.dist }}-${{ matrix.runtime }} timeout-minutes: 180 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -37,14 +38,14 @@ jobs: - name: Build run: | - make OS_ARCH=${{ matrix.arch }} ${{ matrix.runtime }} + make DIST=${{ matrix.dist }} OS_ARCH=${{ matrix.arch }} ${{ matrix.runtime }} ls -lah dist/img - name: Upload artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: img-${{ matrix.arch }}-${{ matrix.runtime }} - path: dist/img/*-${{ matrix.runtime }}.* + name: img-${{ matrix.arch }}-${{ matrix.dist}}-${{ matrix.runtime }} + path: dist/img/${{ matrix.dist}}-*-${{ matrix.runtime }}.* overwrite: true retention-days: 1 compression-level: 0 diff --git a/Makefile b/Makefile index 07dc549..9ad8be7 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,11 @@ UBUNTU_IMAGE_BASE_URL ?= https://cloud-images.ubuntu.com/minimal/releases/$(UBUN UBUNTU_IMAGE_FILE ?= dist/img/ubuntu-$(UBUNTU_VERSION)-minimal-cloudimg-$(ARCH).img UBUNTU_IMAGE_SHA_FILE ?= $(UBUNTU_IMAGE_FILE).sha256sum +# debian +DEBIAN_IMAGE_BASE_URL ?= https://cloud.debian.org/images/cloud/$(DEBIAN_CODENAME)/$(DEBIAN_BUILD) +DEBIAN_IMAGE_FILE ?= dist/img/debian-$(DEBIAN_VERSION)-genericcloud-$(ARCH)-$(DEBIAN_BUILD).qcow2 +DEBIAN_IMAGE_SHA_FILE ?= $(DEBIAN_IMAGE_FILE).sha512sum + # DIST resolved variables IMAGE_BASE_URL ?= $($(call to_upper,$(DIST))_IMAGE_BASE_URL) IMAGE_FILE ?= $($(call to_upper,$(DIST))_IMAGE_FILE) @@ -175,11 +180,12 @@ $(DOCKER_BUILD_IMAGE): $(DOCKER_BUILD_IMAGE_SOURCES) Makefile docker build --build-arg UBUNTU_VERSION=$(UBUNTU_VERSION) -t $(DOCKER_BUILD_IMAGE_TAG) --iidfile $@ $(dir $@) # images -$(basename $(UBUNTU_IMAGE_FILE))-%.raw.gz: $(UBUNTU_IMAGE_SHA_FILE) $(IMAGE_DEPENDENCIES) $(DOCKER_BUILD_IMAGE) Makefile +$(basename $(IMAGE_FILE))-%.raw.gz: $(IMAGE_SHA_FILE) $(IMAGE_DEPENDENCIES) $(DOCKER_BUILD_IMAGE) Makefile if [ $(OS_ARCH) != $(ARCH) ] ; then docker run --privileged --rm tonistiigi/binfmt --install $(BINFMT_ARCH); fi docker run --rm -i --tty --privileged \ --platform linux/$(ARCH) \ --volume $(CURDIR):/build \ + --env DIST=$(DIST) \ --env BINFMT_ARCHIVE=$(BINFMT_ARCHIVE) \ --env CONTAINERD_ARCHIVE=$(CONTAINERD_ARCHIVE) \ --env IMAGE_FILE=$(IMAGE_FILE) \ diff --git a/README.md b/README.md index 1e01308..5d77b3b 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Dependencies for Colima ## Generating image -Generate a raw disk image compressed with gzip (`.raw.gz`) for the OS architecture and default runtime (docker). +Generate a raw disk image compressed with gzip (`.raw.gz`) for the OS architecture, default runtime (docker), and default OS (Ubuntu). ```sh make @@ -30,3 +30,9 @@ Generate `.raw.gz` images for all runtimes. ```sh make all ``` + +Generate images based on Debian. + +```sh +DIST=debian make +``` diff --git a/dependencies.env b/dependencies.env index dd69de8..3f762a1 100644 --- a/dependencies.env +++ b/dependencies.env @@ -1,6 +1,9 @@ UBUNTU_VERSION=24.04 UBUNTU_CODENAME=noble UBUNTU_BUILD=20260617 +DEBIAN_VERSION=13 +DEBIAN_CODENAME=trixie +DEBIAN_BUILD=20260525-2489 BINFMT_VERSION=deploy/v7.0.0-28 BINFMT_QEMU_VERSION=7.0.0 DOCKER_VERSION=29.5.2 diff --git a/scripts/image.sh b/scripts/image.sh index 6332f84..5d3ca3b 100755 --- a/scripts/image.sh +++ b/scripts/image.sh @@ -27,15 +27,18 @@ mount_partition() { mkdir -p "/dev/disk/by-uuid" ln -s "/dev/mapper/${LOOP_NAME}p1" "/dev/disk/by-uuid/$ROOT_UUID" mkdir -p $CHROOT_DIR + mount "$ROOT_PART" "$CHROOT_DIR" + mount --bind /proc "$CHROOT_DIR/proc" + mount --bind /dev "$CHROOT_DIR/dev" + mount --bind /sys "$CHROOT_DIR/sys" + chroot_exec mount -t devpts devpts /dev/pts } cleanup() { rm -f "${RAW_FILE}" if [ -n "$CHROOT_DIR" ] && mountpoint -q "$CHROOT_DIR"; then - umount "$CHROOT_DIR/dev/pts" ||: - umount "$CHROOT_DIR/proc" ||: - umount "$CHROOT_DIR" + unmount_partition ||: fi if [ -n "$ROOT_UUID" ] && [ -h "/dev/disk/by-uuid/$ROOT_UUID" ]; then rm -f "/dev/disk/by-uuid/$ROOT_UUID" @@ -50,6 +53,10 @@ cleanup() { trap 'cleanup' EXIT unmount_partition() ( + umount $CHROOT_DIR/sys + umount $CHROOT_DIR/dev/pts + umount $CHROOT_DIR/dev + umount $CHROOT_DIR/proc umount $CHROOT_DIR ) @@ -58,25 +65,39 @@ chroot_exec() ( ) install_packages() ( - # necessary - chroot_exec mount -t proc proc /proc - chroot_exec mount -t devpts devpts /dev/pts + echo 'Dpkg::Use-Pty "0"; Dpkg::Progress-Fancy="0";' > $CHROOT_DIR/etc/apt/apt.conf.d/colima # internet chroot_exec mv /etc/resolv.conf /etc/resolv.conf.bak echo 'nameserver 1.1.1.1' >$CHROOT_DIR/etc/resolv.conf + # minimal + echo 'Binary::apt::APT::Keep-Downloaded-Packages "0";' > $CHROOT_DIR/etc/apt/apt.conf.d/01_nocache + echo 'APT::Install-Recommends "0"; APT::Install-Suggests "0"; Acquire::Retries "5";' >> $CHROOT_DIR/etc/apt/apt.conf.d/colima + if [ "${DIST}" == "debian" ] ; then + chroot_exec apt-get -qq purge -y groff-base man-db manpages + fi + cat >$CHROOT_DIR/etc/dpkg/dpkg.cfg.d/01_nodoc <<"EOF" +path-exclude=/usr/share/locale/*; +path-exclude=/usr/share/man/*; +path-exclude=/usr/share/doc/*; +path-include=/usr/share/doc/*/copyright; +EOF + pushd $CHROOT_DIR + find usr/share/doc/ usr/share/man usr/share/locale/ -mindepth 1 -delete + popd + # prepare packages chroot_exec apt-get -qq update # packages common to all runtimes, to prevent from final purging chroot_exec apt-get -qq install -y iptables socat sshfs cloud-init lsb-release python3-apt gnupg curl wget dnsmasq + chroot_exec apt-get -qq install -y htop inetutils-ping dnsutils net-tools netcat-openbsd telnet vim-tiny nano # none if [ "$RUNTIME" == "none" ]; then ( - chroot_exec apt-get -qq install -y htop inetutils-ping dnsutils net-tools netcat-openbsd telnet vim-tiny nano - chroot_exec apt-get -qq purge -y dmsetup xz-utils + chroot_exec apt-get -qq purge -y xz-utils ) fi @@ -87,7 +108,7 @@ install_packages() ( chroot_exec sh /tmp/get-docker.sh --version $DOCKER_VERSION chroot_exec rm /tmp/get-docker.sh chroot_exec apt-mark hold docker-ce docker-ce-cli containerd.io - chroot_exec apt-get -qq purge -y dmsetup xz-utils + chroot_exec apt-get -qq purge -y xz-utils ) fi @@ -98,7 +119,7 @@ install_packages() ( tar Cxfz ${CHROOT_DIR}/usr/local "${BUILD_DIR}/${CONTAINERD_ARCHIVE}" chroot_exec mkdir -p /opt/cni chroot_exec mv /usr/local/libexec/cni /opt/cni/bin - chroot_exec apt-get -qq purge -y dmsetup xz-utils + chroot_exec apt-get -qq purge -y xz-utils ) fi @@ -118,14 +139,19 @@ Signed-By: /etc/apt/keyrings/zabbly.asc EOF' chroot_exec apt-get -qq update - chroot_exec apt-get -qq install -y htop inetutils-ping dnsutils net-tools netcat-openbsd telnet vim-tiny nano - chroot_exec apt-get -qq install -y incus incus-base incus-client incus-extra incus-ui-canonical zfsutils-linux btrfs-progs lvm2 thin-provisioning-tools - chroot_exec apt-mark hold incus incus-base incus-client incus-extra incus-ui-canonical zfsutils-linux btrfs-progs lvm2 thin-provisioning-tools + chroot_exec apt-get -qq install -y incus incus-base incus-client incus-extra incus-ui-canonical zfsutils-linux btrfs-progs lvm2 thin-provisioning-tools nftables + chroot_exec apt-mark hold incus incus-base incus-client incus-extra incus-ui-canonical zfsutils-linux btrfs-progs lvm2 thin-provisioning-tools nftables ) fi - chroot_exec apt-get -qq purge -y apport console-setup-linux dbus-user-session liblocale-gettext-perl lxd-agent-loader lxd-installer parted pciutils pollinate python3-gi snapd ssh-import-id - chroot_exec apt-get -qq purge -y ubuntu-advantage-tools ubuntu-cloud-minimal ubuntu-drivers-common ubuntu-release-upgrader-core unattended-upgrades systemd-resolved + chroot_exec apt-get -qq purge -y apport console-setup-linux dbus-user-session liblocale-gettext-perl parted pciutils pollinate python3-gi snapd ssh-import-id + chroot_exec apt-get -qq purge -y unattended-upgrades systemd-resolved + chroot_exec apt-get -qq purge -y apt-listchanges apt-utils reportbug screen whiptail xml-core + chroot_exec apt-get -qq purge -y bash-completion + + if [ "$DIST" == "ubuntu" ] ; then + chroot_exec apt-get -qq purge -y lxd-agent-loader lxd-installer ubuntu-advantage-tools ubuntu-cloud-minimal ubuntu-drivers-common ubuntu-release-upgrader-core + fi chroot_exec apt-get autoremove -y chroot_exec apt-get clean -y @@ -149,12 +175,11 @@ EOF chroot_exec find /tmp -mindepth 1 -delete chroot_exec rm /etc/resolv.conf chroot_exec mv /etc/resolv.conf.bak /etc/resolv.conf - chroot_exec umount /dev/pts - chroot_exec umount /proc # fill partition with zeros, to recover space during compression chroot_exec dd if=/dev/zero of=/root/zero || echo done chroot_exec rm -f /root/zero + chroot_exec fstrim -av ) compress_file() ( From 0cb922b90237b62fd2106a75bd047a887c482cd0 Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Thu, 25 Jun 2026 16:37:14 -0500 Subject: [PATCH 13/25] clear machine id --- scripts/image.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/image.sh b/scripts/image.sh index 5d3ca3b..b8a62b7 100755 --- a/scripts/image.sh +++ b/scripts/image.sh @@ -173,11 +173,11 @@ EOF # clean traces chroot_exec find /tmp -mindepth 1 -delete - chroot_exec rm /etc/resolv.conf - chroot_exec mv /etc/resolv.conf.bak /etc/resolv.conf + chroot_exec mv -f /etc/resolv.conf.bak /etc/resolv.conf + truncate -s 0 $CHROOT_DIR/etc/machine-id # fill partition with zeros, to recover space during compression - chroot_exec dd if=/dev/zero of=/root/zero || echo done + chroot_exec dd if=/dev/zero of=/root/zero ||: chroot_exec rm -f /root/zero chroot_exec fstrim -av ) From 1bf1459021de71cefa35d704fe17cf4509e01569 Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Thu, 25 Jun 2026 16:57:07 -0500 Subject: [PATCH 14/25] fix image download --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9ad8be7..e1f0f98 100644 --- a/Makefile +++ b/Makefile @@ -123,7 +123,7 @@ distclean: clean # base image $(IMAGE_FILE): @echo "target: $@" - mkdir -p dist/img && cd dist/img && curl -o"$@" -L $(IMAGE_BASE_URL)/$(notdir $@) + mkdir -p $(@D) && curl -o"$@" -L $(IMAGE_BASE_URL)/$(notdir $@) $(IMAGE_SHA_FILE): $(IMAGE_FILE) @echo "target: $@" From 67358a359af18ce162948767c931b72ebf7530f0 Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Thu, 25 Jun 2026 17:10:16 -0500 Subject: [PATCH 15/25] fixes testing in github actions --- .github/workflows/release.yml | 4 ++-- Makefile | 23 +++++++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 52af820..87fe929 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,7 +16,7 @@ concurrency: jobs: build: strategy: - max-parallel: 4 + max-parallel: 1 fail-fast: true matrix: arch: [x86_64, aarch64] @@ -38,7 +38,7 @@ jobs: - name: Build run: | - make DIST=${{ matrix.dist }} OS_ARCH=${{ matrix.arch }} ${{ matrix.runtime }} + make DEBUG=1 DIST=${{ matrix.dist }} ${{ matrix.runtime }} ls -lah dist/img - name: Upload artifact diff --git a/Makefile b/Makefile index e1f0f98..f3c6003 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,17 @@ UNAME_S := $(shell uname -s) to_upper = $(shell echo '$1' | tr '[:lower:]' '[:upper:]') +ifeq ($(MAKE_TERMOUT),) + # Make < 4 doesn't set MAKE_TERMOUT + INTERACTIVE ?= $(shell [ -t 0 ] && echo 1) +else + INTERACTIVE ?= 1 +endif + +ifeq ($(INTERACTIVE),1) +DOCKER_RUN_ARGS := -i --tty +endif + ifeq ($(UNAME_S),Darwin) # macOS uses BSD stat PRINT_STATS_CMD = stat -f "%N%n |- %z bytes%n '- %Sm" @@ -85,10 +96,11 @@ IMAGE_SHA_SIZE ?= $(patsubst .sha%sum,%,$(suffix $(IMAGE_SHA_FILE))) define download_and_verify $(1): - @echo "target: $$@" + @echo "downloading $(2)" mkdir -p $$(@D) && \ curl -o$$@.download -L $(2) && \ - tar $$(TARV)xzOf $$@.download &> /dev/null || { \ + $$(PRINT_STATS_CMD) $$@.download && \ + tar -xzOf $$@.download >/dev/null || { \ echo >&2 "error downloading"; \ exit 1; \ } @@ -151,7 +163,7 @@ $(BINFMT_ARCHIVE): $(BINFMT_FILE) $(BINFMT_QEMU_FILE) rm -f '$@' TMP_DIR=$$(mktemp -d); \ trap 'rm -rf "$$TMP_DIR"' EXIT; \ - for f in $^; do tar $(TARX)zxf "$$f" -C "$$TMP_DIR"; done; \ + for f in $^; do tar $(TARV)zxf "$$f" -C "$$TMP_DIR"; done; \ cd "$$TMP_DIR" && tar $(TARV)czf '$(CURDIR)/$@' binfmt qemu-i386 qemu-$(BINFMT_ARCH) || { \ echo >&2 "failed to create $@" ; \ rm -f '$(CURDIR)/$@'; exit 1; \ @@ -181,8 +193,8 @@ $(DOCKER_BUILD_IMAGE): $(DOCKER_BUILD_IMAGE_SOURCES) Makefile # images $(basename $(IMAGE_FILE))-%.raw.gz: $(IMAGE_SHA_FILE) $(IMAGE_DEPENDENCIES) $(DOCKER_BUILD_IMAGE) Makefile - if [ $(OS_ARCH) != $(ARCH) ] ; then docker run --privileged --rm tonistiigi/binfmt --install $(BINFMT_ARCH); fi - docker run --rm -i --tty --privileged \ + if [ $(OS_ARCH) != $(ARCH) ] ; then docker run $(DOCKER_RUN_ARGS) --privileged --rm tonistiigi/binfmt --install $(BINFMT_ARCH); fi + docker run $(DOCKER_RUN_ARGS) --rm --privileged \ --platform linux/$(ARCH) \ --volume $(CURDIR):/build \ --env DIST=$(DIST) \ @@ -195,7 +207,6 @@ $(basename $(IMAGE_FILE))-%.raw.gz: $(IMAGE_SHA_FILE) $(IMAGE_DEPENDENCIES) $(DO echo >&2 "failed to create $@"; \ rm -f '$@'* ; exit 1; \ } - touch "$@" $(RUNTIMES): %: $(basename $(IMAGE_FILE))-%.raw.gz $(PRINT_STATS_CMD) $< From a2101d2ad9f039c182c2f7482d5eb826193e2c38 Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Thu, 25 Jun 2026 17:57:40 -0500 Subject: [PATCH 16/25] builder image multi-arch --- .github/workflows/release.yml | 44 +++++++++++++++++++++++++++++++---- .gitignore | 2 +- Makefile | 11 +++++---- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 87fe929..10b082a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,37 @@ concurrency: cancel-in-progress: true jobs: - build: + builder: + strategy: + max-parallel: 2 + fail-fast: true + matrix: + arch: [x86_64, aarch64] + include: + - arch: x86_64 + os: ubuntu-latest + - arch: aarch64 + os: ubuntu-24.04-arm + runs-on: ${{ matrix.os }} + name: builder-${{ matrix.arch }} + timeout-minutes: 180 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Build + run: make save-builder + + - name: Upload artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: builder-${{ matrix.arch }} + path: builder-${{ matrix.arch }}.tar + overwrite: true + + image: strategy: max-parallel: 1 fail-fast: true @@ -28,7 +58,8 @@ jobs: - arch: aarch64 os: ubuntu-24.04-arm runs-on: ${{ matrix.os }} - name: build-${{ matrix.arch }}-${{ matrix.dist }}-${{ matrix.runtime }} + needs: [builder] + name: img-${{ matrix.arch }}-${{ matrix.dist }}-${{ matrix.runtime }} timeout-minutes: 180 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -36,9 +67,14 @@ jobs: fetch-depth: 1 persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: builder-${{ matrix.arch }} + - name: Build run: | - make DEBUG=1 DIST=${{ matrix.dist }} ${{ matrix.runtime }} + docker load -i builder-${{ matrix.arch }}.tar + make DIST=${{ matrix.dist }} ${{ matrix.runtime }} ls -lah dist/img - name: Upload artifact @@ -52,7 +88,7 @@ jobs: release: runs-on: ubuntu-latest - needs: [build] + needs: [image] if: github.event_name != 'pull_request' permissions: contents: write diff --git a/.gitignore b/.gitignore index 5401e88..4fc8f1f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ dist/ -scripts/.build-image-stamp +scripts/.build-image-stamp* diff --git a/Makefile b/Makefile index f3c6003..25e93ab 100644 --- a/Makefile +++ b/Makefile @@ -108,16 +108,16 @@ $(1): endef # image builder container image -DOCKER_BUILD_IMAGE = scripts/.build-image-stamp +DOCKER_BUILD_IMAGE = scripts/.build-image-stamp-$(ARCH) DOCKER_BUILD_IMAGE_SOURCES = scripts/Dockerfile scripts/image.sh -DOCKER_BUILD_IMAGE_TAG = colima-core-builder:latest +DOCKER_BUILD_IMAGE_TAG = colima-core-builder:$(ARCH) IMAGE_DEPENDENCIES = $(DOCKER_BUILD_IMAGE) $(CONTAINERD_ARCHIVE).sha512sum $(BINFMT_ARCHIVE).sha512sum # # targets # -.PHONY: clean distclean image $(RUNTIMES) +.PHONY: clean distclean save-builder image $(RUNTIMES) # deprecated (default) target image: $(RUNTIME) @@ -188,8 +188,11 @@ $(eval $(call download_and_verify,$(NERDCTL_FILE),$(NERDCTL_URL))) $(eval $(call download_and_verify,$(FLANNEL_FILE),$(FLANNEL_URL))) # builder +save-builder: $(DOCKER_BUILD_IMAGE) + docker save $(DOCKER_BUILD_IMAGE_TAG) -o builder-$(OS_ARCH).tar + $(DOCKER_BUILD_IMAGE): $(DOCKER_BUILD_IMAGE_SOURCES) Makefile - docker build --build-arg UBUNTU_VERSION=$(UBUNTU_VERSION) -t $(DOCKER_BUILD_IMAGE_TAG) --iidfile $@ $(dir $@) + docker build --platform linux/$(ARCH) --build-arg UBUNTU_VERSION=$(UBUNTU_VERSION) -t $(DOCKER_BUILD_IMAGE_TAG) --iidfile $@ $(dir $@) # images $(basename $(IMAGE_FILE))-%.raw.gz: $(IMAGE_SHA_FILE) $(IMAGE_DEPENDENCIES) $(DOCKER_BUILD_IMAGE) Makefile From d7d5c7e043acc93ac84c1c662df1f9ea0ea9aec2 Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Thu, 25 Jun 2026 18:10:40 -0500 Subject: [PATCH 17/25] optimize parallel actions by arch and dist --- .github/workflows/release.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 10b082a..6e46659 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,11 +46,10 @@ jobs: image: strategy: - max-parallel: 1 + max-parallel: 4 fail-fast: true matrix: arch: [x86_64, aarch64] - runtime: [none, docker, containerd, incus] dist: [ubuntu, debian] include: - arch: x86_64 @@ -59,7 +58,7 @@ jobs: os: ubuntu-24.04-arm runs-on: ${{ matrix.os }} needs: [builder] - name: img-${{ matrix.arch }}-${{ matrix.dist }}-${{ matrix.runtime }} + name: img-${{ matrix.arch }}-${{ matrix.dist }} timeout-minutes: 180 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -74,14 +73,14 @@ jobs: - name: Build run: | docker load -i builder-${{ matrix.arch }}.tar - make DIST=${{ matrix.dist }} ${{ matrix.runtime }} + make DIST=${{ matrix.dist }} none docker containerd incus ls -lah dist/img - name: Upload artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: img-${{ matrix.arch }}-${{ matrix.dist}}-${{ matrix.runtime }} - path: dist/img/${{ matrix.dist}}-*-${{ matrix.runtime }}.* + name: img-${{ matrix.arch }}-${{ matrix.dist}} + path: dist/img/${{ matrix.dist}}-*-*.raw.* overwrite: true retention-days: 1 compression-level: 0 From 7d1dbee77155324cfd2fb728c46aca9e096f9ad2 Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Thu, 25 Jun 2026 18:23:14 -0500 Subject: [PATCH 18/25] fix debian zfs support --- scripts/image.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/image.sh b/scripts/image.sh index b8a62b7..1213b0b 100755 --- a/scripts/image.sh +++ b/scripts/image.sh @@ -126,6 +126,10 @@ EOF # incus if [ "$RUNTIME" == "incus" ]; then ( + # Debian ZFS support is in contrib + if [ "${DIST}" == "debian" ] ; then + chroot_exec sed -i 's/Components: main$/Components: main contrib/' /etc/apt/sources.list.d/debian.sources + fi chroot_exec mkdir -p /etc/apt/keyrings/ chroot_exec curl -fsSL https://pkgs.zabbly.com/key.asc -o /etc/apt/keyrings/zabbly.asc chroot_exec sh -c 'cat < /etc/apt/sources.list.d/zabbly-incus-stable.sources From 2074c65248a57e561dd352f7a1ae136ed8cef246 Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Thu, 25 Jun 2026 18:24:15 -0500 Subject: [PATCH 19/25] nit --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6e46659..8765b36 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -58,7 +58,7 @@ jobs: os: ubuntu-24.04-arm runs-on: ${{ matrix.os }} needs: [builder] - name: img-${{ matrix.arch }}-${{ matrix.dist }} + name: images-${{ matrix.arch }}-${{ matrix.dist }} timeout-minutes: 180 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 From 16c1e368555fe419dad63cd85cccaf622e7a82d2 Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Fri, 26 Jun 2026 09:14:41 -0500 Subject: [PATCH 20/25] curl retry timeout options --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 25e93ab..79b1639 100644 --- a/Makefile +++ b/Makefile @@ -98,7 +98,7 @@ define download_and_verify $(1): @echo "downloading $(2)" mkdir -p $$(@D) && \ - curl -o$$@.download -L $(2) && \ + curl --retry 3 --connect-timeout 10 -o$$@.download -L $(2) && \ $$(PRINT_STATS_CMD) $$@.download && \ tar -xzOf $$@.download >/dev/null || { \ echo >&2 "error downloading"; \ @@ -135,13 +135,13 @@ distclean: clean # base image $(IMAGE_FILE): @echo "target: $@" - mkdir -p $(@D) && curl -o"$@" -L $(IMAGE_BASE_URL)/$(notdir $@) + mkdir -p $(@D) && curl --retry 5 --connect-timeout 10 -o"$@" -L $(IMAGE_BASE_URL)/$(notdir $@) $(IMAGE_SHA_FILE): $(IMAGE_FILE) @echo "target: $@" shasum -a $(IMAGE_SHA_SIZE) $< > $@.tmp cd dist/img && ( \ - curl -sL $(IMAGE_BASE_URL)/SHA$(IMAGE_SHA_SIZE)SUMS | \ + curl --retry 5 --connect-timeout 10 -L $(IMAGE_BASE_URL)/SHA$(IMAGE_SHA_SIZE)SUMS | \ grep $(notdir $<) | \ shasum -a $(IMAGE_SHA_SIZE) --check --status \ ) || { \ From 991e6f128c3a8fdda198c2300419357b91cf814b Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Fri, 26 Jun 2026 12:05:19 -0500 Subject: [PATCH 21/25] implement debian incus support --- scripts/image.sh | 73 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 59 insertions(+), 14 deletions(-) diff --git a/scripts/image.sh b/scripts/image.sh index 1213b0b..8b4c395 100755 --- a/scripts/image.sh +++ b/scripts/image.sh @@ -64,8 +64,50 @@ chroot_exec() ( chroot $CHROOT_DIR "$@" ) +apt_update_source() { + chroot_exec apt-get -qq update -y -o Dir::Etc::sourcelist="sources.list.d/${1}" -o Dir::Etc::sourceparts="-" -o APT::Get::List-Cleanup="0" +} + +apt_keyring_zabbly() { + test -f "$CHROOT_DIR/etc/apt/keyrings/zabbly.asc" || { + chroot_exec install -m 0755 -d /etc/apt/keyrings + chroot_exec curl -fsSLo zabbly.asc https://pkgs.zabbly.com/key.asc + chroot_exec install -m 0644 zabbly.asc /etc/apt/keyrings/zabbly.asc + chroot_exec rm -f zabbly.asc + } +} + +apt_source_zabbly_incus() { + apt_keyring_zabbly + chroot_exec sh -c 'cat < /etc/apt/sources.list.d/zabbly-incus-stable.sources +Enabled: yes +Types: deb +URIs: https://pkgs.zabbly.com/incus/stable +Suites: $(. /etc/os-release && echo ${VERSION_CODENAME}) +Components: main +Architectures: $(dpkg --print-architecture) +Signed-By: /etc/apt/keyrings/zabbly.asc +EOF' + apt_update_source zabbly-incus-stable.sources +} + +apt_source_zabbly_kernel() { + apt_keyring_zabbly + chroot_exec sh -c 'cat < /etc/apt/sources.list.d/zabbly-kernel-stable.sources +Enabled: yes +Types: deb +URIs: https://pkgs.zabbly.com/kernel/stable +Suites: $(. /etc/os-release && echo ${VERSION_CODENAME}) +Components: main +Architectures: $(dpkg --print-architecture) +Signed-By: /etc/apt/keyrings/zabbly.asc +EOF' + apt_update_source zabbly-kernel-stable.sources +} + install_packages() ( echo 'Dpkg::Use-Pty "0"; Dpkg::Progress-Fancy="0";' > $CHROOT_DIR/etc/apt/apt.conf.d/colima + DPKG_ARCH=$(chroot_exec dpkg --print-architecture) # internet chroot_exec mv /etc/resolv.conf /etc/resolv.conf.bak @@ -75,6 +117,7 @@ install_packages() ( echo 'Binary::apt::APT::Keep-Downloaded-Packages "0";' > $CHROOT_DIR/etc/apt/apt.conf.d/01_nocache echo 'APT::Install-Recommends "0"; APT::Install-Suggests "0"; Acquire::Retries "5";' >> $CHROOT_DIR/etc/apt/apt.conf.d/colima if [ "${DIST}" == "debian" ] ; then + chroot_exec sed -i 's/Components: main$/Components: main contrib non-free-firmware/' /etc/apt/sources.list.d/debian.sources chroot_exec apt-get -qq purge -y groff-base man-db manpages fi cat >$CHROOT_DIR/etc/dpkg/dpkg.cfg.d/01_nodoc <<"EOF" @@ -126,23 +169,13 @@ EOF # incus if [ "$RUNTIME" == "incus" ]; then ( - # Debian ZFS support is in contrib + apt_source_zabbly_incus + if [ "${DIST}" == "debian" ] ; then - chroot_exec sed -i 's/Components: main$/Components: main contrib/' /etc/apt/sources.list.d/debian.sources + chroot_exec apt-get -qq install -y dpkg-dev "linux-headers-cloud-${DPKG_ARCH}" + chroot_exec apt-get -qq install -y zfs-dkms fi - chroot_exec mkdir -p /etc/apt/keyrings/ - chroot_exec curl -fsSL https://pkgs.zabbly.com/key.asc -o /etc/apt/keyrings/zabbly.asc - chroot_exec sh -c 'cat < /etc/apt/sources.list.d/zabbly-incus-stable.sources -Enabled: yes -Types: deb -URIs: https://pkgs.zabbly.com/incus/stable -Suites: $(. /etc/os-release && echo ${VERSION_CODENAME}) -Components: main -Architectures: $(dpkg --print-architecture) -Signed-By: /etc/apt/keyrings/zabbly.asc -EOF' - chroot_exec apt-get -qq update chroot_exec apt-get -qq install -y incus incus-base incus-client incus-extra incus-ui-canonical zfsutils-linux btrfs-progs lvm2 thin-provisioning-tools nftables chroot_exec apt-mark hold incus incus-base incus-client incus-extra incus-ui-canonical zfsutils-linux btrfs-progs lvm2 thin-provisioning-tools nftables ) @@ -157,6 +190,18 @@ EOF' chroot_exec apt-get -qq purge -y lxd-agent-loader lxd-installer ubuntu-advantage-tools ubuntu-cloud-minimal ubuntu-drivers-common ubuntu-release-upgrader-core fi + # cleanup any old kernel versions that may be been upgraded + KERNEL_VERSIONS=$(chroot_exec dpkg-query -W -f='${Package}\n' | grep -E '^linux-image-[0-9]' | sed 's/^linux-image-//' | sort -V) + LATEST_VERSION=$(echo "$KERNEL_VERSIONS" | tail -n 1) + for V in $KERNEL_VERSIONS; do + if [ "$V" != "$LATEST_VERSION" ]; then + chroot_exec apt-get -qq purge -y "linux-image-$V" + chroot_exec apt-get -qq purge -y "linux-headers-$V" ||: + chroot_exec apt-get -qq purge -y "linux-modules-$V" ||: + chroot_exec apt-get -qq purge -y "linux-modules-extra-$V" ||: + fi + done + chroot_exec apt-get autoremove -y chroot_exec apt-get clean -y chroot_exec sh -c "rm -rf /var/lib/apt/lists/* /var/cache/apt/*" From f15acca94263837f28b2ccb2c4718fa0cdc8696f Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Fri, 26 Jun 2026 13:30:42 -0500 Subject: [PATCH 22/25] configure kernel output to lima serial.log --- scripts/image.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/scripts/image.sh b/scripts/image.sh index 8b4c395..57ce630 100755 --- a/scripts/image.sh +++ b/scripts/image.sh @@ -214,12 +214,29 @@ EOF mv binfmt qemu-* ${CHROOT_DIR}/usr/bin ) + # console messages to lima serial log + echo "virtio_console" >> $CHROOT_DIR/etc/initramfs-tools/modules + chroot_exec update-initramfs -k all -u + mkdir -p $CHROOT_DIR/etc/default/grub.d + cat >"$CHROOT_DIR"/etc/default/grub.d/99-lima-console.cfg <<"EOF" +GRUB_TERMINAL="serial console" +GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1" +GRUB_CMDLINE_LINUX="$GRUB_CMDLINE_LINUX console=ttyS0 console=hvc0" +EOF + chroot_exec update-grub + # enable vsock modules at boot cat >${CHROOT_DIR}/etc/modules-load.d/vsock.conf < Date: Fri, 26 Jun 2026 14:34:12 -0500 Subject: [PATCH 23/25] basic test script to validate images --- scripts/test.sh | 53 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100755 scripts/test.sh diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 0000000..84731df --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -eu + +IMAGE_FILE=$1 +BN_IMAGE=$(basename "$IMAGE_FILE") +DIST=${BN_IMAGE%%-*} +COMPRESSION=${BN_IMAGE##*.} + +BN_IMAGE_T=$(basename -s ".$COMPRESSION" "$BN_IMAGE") +IMAGE_TYPE=${BN_IMAGE_T##*.} + +BN_IMAGE_T=$(basename -s ".$IMAGE_TYPE" "$BN_IMAGE_T") +RUNTIME=${BN_IMAGE_T##*-} + +echo "DIST=$DIST RUNTIME=$RUNTIME IMAGE=$BN_IMAGE TYPE=$IMAGE_TYPE " + +export COLIMA_PROFILE="$RUNTIME.$$" + +cleanup() { + echo "==== Delete runtime" + colima delete -d -v -f ||: +} +trap 'cleanup' EXIT + +echo "==== Starting runtime" +# gnu timeout (coreutils) is not available everywhere, this perl does the same thing +perl -e 'alarm 60; exec @ARGV' colima start \ + -r "$RUNTIME" \ + --disk-image "$IMAGE_FILE" \ + --force-disk-image + +# check for failed services +echo "==== Test: systemd" +colima exec -- systemctl --failed + +echo "==== Test: $RUNTIME" + +if [ "$RUNTIME" == "incus" ] ; then + colima exec -- incus info + + echo "==== Test: launch, exec, stop, delete" + colima exec -- incus launch images:debian/13 debian13 + colima exec -- incus list + colima exec -- incus exec debian13 cat /etc/os-release + colima exec -- incus stop debian13 + colima exec -- incus delete debian13 + colima exec -- incus list +fi + +if [ "$RUNTIME" == "docker" ] ; then + colima exec -- docker system info +fi From 74147a6679c5055dae1ce2345699ecc508417365 Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Fri, 26 Jun 2026 14:44:11 -0500 Subject: [PATCH 24/25] make apt not so quiet, mostly for debian 13 --- scripts/image.sh | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/scripts/image.sh b/scripts/image.sh index 57ce630..db00339 100755 --- a/scripts/image.sh +++ b/scripts/image.sh @@ -65,7 +65,7 @@ chroot_exec() ( ) apt_update_source() { - chroot_exec apt-get -qq update -y -o Dir::Etc::sourcelist="sources.list.d/${1}" -o Dir::Etc::sourceparts="-" -o APT::Get::List-Cleanup="0" + chroot_exec apt-get -q update -y -o Dir::Etc::sourcelist="sources.list.d/${1}" -o Dir::Etc::sourceparts="-" -o APT::Get::List-Cleanup="0" } apt_keyring_zabbly() { @@ -118,7 +118,7 @@ install_packages() ( echo 'APT::Install-Recommends "0"; APT::Install-Suggests "0"; Acquire::Retries "5";' >> $CHROOT_DIR/etc/apt/apt.conf.d/colima if [ "${DIST}" == "debian" ] ; then chroot_exec sed -i 's/Components: main$/Components: main contrib non-free-firmware/' /etc/apt/sources.list.d/debian.sources - chroot_exec apt-get -qq purge -y groff-base man-db manpages + chroot_exec apt-get -q purge -y groff-base man-db manpages fi cat >$CHROOT_DIR/etc/dpkg/dpkg.cfg.d/01_nodoc <<"EOF" path-exclude=/usr/share/locale/*; @@ -131,16 +131,16 @@ EOF popd # prepare packages - chroot_exec apt-get -qq update + chroot_exec apt-get -q update # packages common to all runtimes, to prevent from final purging - chroot_exec apt-get -qq install -y iptables socat sshfs cloud-init lsb-release python3-apt gnupg curl wget dnsmasq - chroot_exec apt-get -qq install -y htop inetutils-ping dnsutils net-tools netcat-openbsd telnet vim-tiny nano + chroot_exec apt-get -q install -y iptables socat sshfs cloud-init lsb-release python3-apt gnupg curl wget dnsmasq + chroot_exec apt-get -q install -y htop inetutils-ping dnsutils net-tools netcat-openbsd telnet vim-tiny nano # none if [ "$RUNTIME" == "none" ]; then ( - chroot_exec apt-get -qq purge -y xz-utils + chroot_exec apt-get -q purge -y xz-utils ) fi @@ -151,7 +151,7 @@ EOF chroot_exec sh /tmp/get-docker.sh --version $DOCKER_VERSION chroot_exec rm /tmp/get-docker.sh chroot_exec apt-mark hold docker-ce docker-ce-cli containerd.io - chroot_exec apt-get -qq purge -y xz-utils + chroot_exec apt-get -q purge -y xz-utils ) fi @@ -162,7 +162,7 @@ EOF tar Cxfz ${CHROOT_DIR}/usr/local "${BUILD_DIR}/${CONTAINERD_ARCHIVE}" chroot_exec mkdir -p /opt/cni chroot_exec mv /usr/local/libexec/cni /opt/cni/bin - chroot_exec apt-get -qq purge -y xz-utils + chroot_exec apt-get -q purge -y xz-utils ) fi @@ -172,22 +172,22 @@ EOF apt_source_zabbly_incus if [ "${DIST}" == "debian" ] ; then - chroot_exec apt-get -qq install -y dpkg-dev "linux-headers-cloud-${DPKG_ARCH}" - chroot_exec apt-get -qq install -y zfs-dkms + chroot_exec apt-get -q install -y dpkg-dev "linux-headers-cloud-${DPKG_ARCH}" + chroot_exec apt-get -q install -y zfs-dkms fi - chroot_exec apt-get -qq install -y incus incus-base incus-client incus-extra incus-ui-canonical zfsutils-linux btrfs-progs lvm2 thin-provisioning-tools nftables + chroot_exec apt-get -q install -y incus incus-base incus-client incus-extra incus-ui-canonical zfsutils-linux btrfs-progs lvm2 thin-provisioning-tools nftables chroot_exec apt-mark hold incus incus-base incus-client incus-extra incus-ui-canonical zfsutils-linux btrfs-progs lvm2 thin-provisioning-tools nftables ) fi - chroot_exec apt-get -qq purge -y apport console-setup-linux dbus-user-session liblocale-gettext-perl parted pciutils pollinate python3-gi snapd ssh-import-id - chroot_exec apt-get -qq purge -y unattended-upgrades systemd-resolved - chroot_exec apt-get -qq purge -y apt-listchanges apt-utils reportbug screen whiptail xml-core - chroot_exec apt-get -qq purge -y bash-completion + chroot_exec apt-get -q purge -y apport console-setup-linux dbus-user-session liblocale-gettext-perl parted pciutils pollinate python3-gi snapd ssh-import-id + chroot_exec apt-get -q purge -y unattended-upgrades systemd-resolved + chroot_exec apt-get -q purge -y apt-listchanges apt-utils reportbug screen whiptail xml-core + chroot_exec apt-get -q purge -y bash-completion if [ "$DIST" == "ubuntu" ] ; then - chroot_exec apt-get -qq purge -y lxd-agent-loader lxd-installer ubuntu-advantage-tools ubuntu-cloud-minimal ubuntu-drivers-common ubuntu-release-upgrader-core + chroot_exec apt-get -q purge -y lxd-agent-loader lxd-installer ubuntu-advantage-tools ubuntu-cloud-minimal ubuntu-drivers-common ubuntu-release-upgrader-core fi # cleanup any old kernel versions that may be been upgraded @@ -195,10 +195,10 @@ EOF LATEST_VERSION=$(echo "$KERNEL_VERSIONS" | tail -n 1) for V in $KERNEL_VERSIONS; do if [ "$V" != "$LATEST_VERSION" ]; then - chroot_exec apt-get -qq purge -y "linux-image-$V" - chroot_exec apt-get -qq purge -y "linux-headers-$V" ||: - chroot_exec apt-get -qq purge -y "linux-modules-$V" ||: - chroot_exec apt-get -qq purge -y "linux-modules-extra-$V" ||: + chroot_exec apt-get -q purge -y "linux-image-$V" + chroot_exec apt-get -q purge -y "linux-headers-$V" ||: + chroot_exec apt-get -q purge -y "linux-modules-$V" ||: + chroot_exec apt-get -q purge -y "linux-modules-extra-$V" ||: fi done From 31afb5d42cee618b80daa5a0624a6aa955fb8069 Mon Sep 17 00:00:00 2001 From: Jeffrey Clark Date: Thu, 16 Jul 2026 21:03:42 -0400 Subject: [PATCH 25/25] mount all partitions and cleanup images from ubuntu and debian have different partition tables, new method doesn't guess, but looks at each partition and mounts to the correct location. this fixes updating the kernel, rebuilding the initrd, basically anything in /boot or EFI. --- scripts/Dockerfile | 8 ++-- scripts/image.sh | 100 +++++++++++++++++++++++++++++---------------- scripts/test.sh | 3 +- 3 files changed, 72 insertions(+), 39 deletions(-) diff --git a/scripts/Dockerfile b/scripts/Dockerfile index 2754248..a1e46ae 100644 --- a/scripts/Dockerfile +++ b/scripts/Dockerfile @@ -9,9 +9,11 @@ SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] RUN --mount=type=cache,target=/var/cache/apt,rw --mount=type=cache,target=/var/lib/apt,rw < /etc/apt/apt.conf.d/colima -apt-get -qq update -apt-get -qq upgrade -apt-get -qq install -y file kpartx libdigest-sha-perl qemu-utils pigz +apt-get -q update +apt-get -q install -y file libdigest-sha-perl qemu-utils pigz +apt-get -q autoremove -y +apt-get -q clean -y +rm -rf /var/lib/apt/lists/* /var/cache/apt/* EOD COPY image.sh /image.sh diff --git a/scripts/image.sh b/scripts/image.sh index db00339..2f7518c 100755 --- a/scripts/image.sh +++ b/scripts/image.sh @@ -2,9 +2,6 @@ set -eux -# disable apt prompts -export DEBIAN_FRONTEND=noninteractive - # external variables that must be set echo vars: $DOCKER_VERSION $RUNTIME @@ -18,52 +15,80 @@ convert_file() ( qemu-img convert -p -f qcow2 -O raw "${IMAGE_FILE}" "${RAW_FILE}" ) -mount_partition() { +mount_partitions() { + mkdir -p "$CHROOT_DIR" + LOOP_DEV=$(losetup -Pf --show "${RAW_FILE}") - kpartx -avs "$LOOP_DEV" - LOOP_NAME=$(basename "$LOOP_DEV") - ROOT_PART="/dev/mapper/${LOOP_NAME}p1" - ROOT_UUID=$(blkid -s UUID -o value "$ROOT_PART") - mkdir -p "/dev/disk/by-uuid" - ln -s "/dev/mapper/${LOOP_NAME}p1" "/dev/disk/by-uuid/$ROOT_UUID" - mkdir -p $CHROOT_DIR - - mount "$ROOT_PART" "$CHROOT_DIR" + LOOP_NAME="${LOOP_DEV##*/}" + FOUND_ROOT="0:0" + + # generate partition device nodes + for part_sys_path in "/sys/class/block/${LOOP_NAME}/${LOOP_NAME}p"*; do + PART_PART_ENTRY_UUID="" + if [ -e "$part_sys_path" ]; then + PART_NAME="${part_sys_path##*/}" + read -r PART_MAJ_MIN < "$part_sys_path/dev" + if [ ! -e "/dev/$PART_NAME" ]; then + mknod "/dev/$PART_NAME" b "${PART_MAJ_MIN%:*}" "${PART_MAJ_MIN#*:}" + fi + # create part uuid dev entries so mounting is easy later + eval "$(blkid -p -o export /dev/"${PART_NAME}" | sed 's/^/PART_/')" + # Ubuntu uses labels, make fstab consistent with Debian, which uses partuuid + if [[ -n "${PART_PART_ENTRY_UUID:-}" ]] ; then + mkdir -p /dev/disk/by-partuuid + ln -s ../../"${PART_NAME}" /dev/disk/by-partuuid/"${PART_PART_ENTRY_UUID}" + # stage fstab updates so Ubuntu fstab (labels) is like Debian (partuuids) + if [[ -n "${PART_LABEL:-}" ]] ; then + echo "sed -i 's/LABEL=${PART_LABEL}/PARTUUID=${PART_PART_ENTRY_UUID}/' ${CHROOT_DIR}/etc/fstab" >> /tmp/fstab_update + fi + fi + # guess which is the root dev (usually it's 1), but the largest ext4 partition makes more sense + if [[ "${PART_TYPE}" == "ext4" ]] && [[ "${FOUND_ROOT##*:}" -lt "${PART_FSSIZE}" ]] ; then + FOUND_ROOT="/dev/${PART_NAME}:${PART_FSSIZE}" + fi + fi + done + + # mount root so we can mount everything else + mount "${FOUND_ROOT%%:*}" "${CHROOT_DIR}" + + # update fstab with partuuids + if [ -f /tmp/fstab_update ]; then + source /tmp/fstab_update + fi + mount --bind /proc "$CHROOT_DIR/proc" mount --bind /dev "$CHROOT_DIR/dev" mount --bind /sys "$CHROOT_DIR/sys" + chroot_exec mount -t devpts devpts /dev/pts + chroot_exec mount -a } cleanup() { rm -f "${RAW_FILE}" if [ -n "$CHROOT_DIR" ] && mountpoint -q "$CHROOT_DIR"; then - unmount_partition ||: - fi - if [ -n "$ROOT_UUID" ] && [ -h "/dev/disk/by-uuid/$ROOT_UUID" ]; then - rm -f "/dev/disk/by-uuid/$ROOT_UUID" - fi - if [ -n "$LOOP_DEV" ]; then - kpartx -dvs "$LOOP_DEV" 2>/dev/null || true + unmount_partitions ||: fi + rm -rf /dev/disk if [ -n "$LOOP_DEV" ] && losetup "$LOOP_DEV" >/dev/null 2>&1; then losetup -d "$LOOP_DEV" fi } trap 'cleanup' EXIT -unmount_partition() ( +unmount_partitions() ( + chroot_exec umount -a -t nocgroup,nodevtmpfs,noproc,nosysfs umount $CHROOT_DIR/sys - umount $CHROOT_DIR/dev/pts - umount $CHROOT_DIR/dev umount $CHROOT_DIR/proc umount $CHROOT_DIR ) chroot_exec() ( - chroot $CHROOT_DIR "$@" + chroot $CHROOT_DIR env DEBIAN_FRONTEND=noninteractive "$@" ) + apt_update_source() { chroot_exec apt-get -q update -y -o Dir::Etc::sourcelist="sources.list.d/${1}" -o Dir::Etc::sourceparts="-" -o APT::Get::List-Cleanup="0" } @@ -115,7 +140,7 @@ install_packages() ( # minimal echo 'Binary::apt::APT::Keep-Downloaded-Packages "0";' > $CHROOT_DIR/etc/apt/apt.conf.d/01_nocache - echo 'APT::Install-Recommends "0"; APT::Install-Suggests "0"; Acquire::Retries "5";' >> $CHROOT_DIR/etc/apt/apt.conf.d/colima + echo 'APT::Install-Recommends "0"; APT::Install-Suggests "0"; Acquire::Retries "5"; Acquire::Retries::Delay::Maximum "30";' >> $CHROOT_DIR/etc/apt/apt.conf.d/colima if [ "${DIST}" == "debian" ] ; then chroot_exec sed -i 's/Components: main$/Components: main contrib non-free-firmware/' /etc/apt/sources.list.d/debian.sources chroot_exec apt-get -q purge -y groff-base man-db manpages @@ -134,8 +159,9 @@ EOF chroot_exec apt-get -q update # packages common to all runtimes, to prevent from final purging - chroot_exec apt-get -q install -y iptables socat sshfs cloud-init lsb-release python3-apt gnupg curl wget dnsmasq - chroot_exec apt-get -q install -y htop inetutils-ping dnsutils net-tools netcat-openbsd telnet vim-tiny nano + chroot_exec apt-get -q install -y \ + iptables socat sshfs cloud-init lsb-release python3-apt gnupg curl wget dnsmasq rsync \ + htop inetutils-ping dnsutils net-tools netcat-openbsd telnet vim-tiny nano # none if [ "$RUNTIME" == "none" ]; then @@ -215,14 +241,18 @@ EOF ) # console messages to lima serial log - echo "virtio_console" >> $CHROOT_DIR/etc/initramfs-tools/modules - chroot_exec update-initramfs -k all -u + # https://github.com/lima-vm/lima/issues/1659 mkdir -p $CHROOT_DIR/etc/default/grub.d + if [[ "$DIST" == "ubuntu" ]]; then cat >"$CHROOT_DIR"/etc/default/grub.d/99-lima-console.cfg <<"EOF" -GRUB_TERMINAL="serial console" -GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1" -GRUB_CMDLINE_LINUX="$GRUB_CMDLINE_LINUX console=ttyS0 console=hvc0" +GRUB_CMDLINE_LINUX_DEFAULT="console=hvc0 earlycon" EOF + elif [[ "$DIST" == "debian" ]]; then + cat >"$CHROOT_DIR"/etc/default/grub.d/99-lima-console.cfg <<"EOF" +GRUB_CMDLINE_LINUX="$GRUB_CMDLINE_LINUX console=hvc0 earlycon" +EOF + fi + chroot_exec update-grub # enable vsock modules at boot @@ -234,7 +264,7 @@ EOF # logging to console chroot_exec passwd -d root sed -i -e 's/#\(Storage\)=.*/\1=volatile/' \ - -e 's/#\(RuntimeMaxUse\)=.*/100M/' \ + -e 's/#\(RuntimeMaxUse\)=.*/\1=100M/' \ -e 's/#\(ForwardToConsole\)=.*/\1=yes/' $CHROOT_DIR/etc/systemd/journald.conf # clean traces @@ -255,7 +285,7 @@ compress_file() ( # perform all actions convert_file -mount_partition +mount_partitions install_packages -unmount_partition +unmount_partitions compress_file diff --git a/scripts/test.sh b/scripts/test.sh index 84731df..c692921 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -15,9 +15,10 @@ RUNTIME=${BN_IMAGE_T##*-} echo "DIST=$DIST RUNTIME=$RUNTIME IMAGE=$BN_IMAGE TYPE=$IMAGE_TYPE " -export COLIMA_PROFILE="$RUNTIME.$$" +export COLIMA_PROFILE="test" cleanup() { + read -r -t 10 -n 1 -p "Press any key to keep test instance running (deleting in 10s)..." && exit echo "==== Delete runtime" colima delete -d -v -f ||: }