feat(update): 0.3.5.1 → 0.4.0 in-place update flow

.github/workflows/startos-iso.yaml

@@ -349,6 +349,86 @@ jobs:
           retention-days: 14
         if: ${{ matrix.platform == 'raspberrypi' }}
 
+  # The OTA payload legacy (0.3.5.1) boxes pull via their frozen rsync updater:
+  # the 0.4.0 base squashfs (built just above) repackaged onto the published
+  # 0.3.5.1 rootfs. Runs automatically after every image build, in the same run,
+  # consuming that run's <platform>.squashfs artifact. Pure repackaging (no OS
+  # compile), so it runs on x86_64 for every platform. Only platforms that
+  # shipped in 0.3.5.1 (no nvidia/riscv); raspberrypi requires a reflash (#3443).
+  # The <platform>.migration.squashfs artifact is what ops publishes to the
+  # frozen registry as the version's OTA image. See
+  # projects/start-os/build/assemble-migration-payload.sh and PR #3399.
+  migration-image:
+    name: Migration Image
+    needs: [image]
+    if: ${{ !cancelled() && needs.image.result == 'success' }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [x86_64, x86_64-nonfree, aarch64, aarch64-nonfree]
+    env:
+      GH_TOKEN: ${{ github.token }}
+      MIGRATION_FROM_TAG: v0.3.5.1
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Fetch the 0.4.0 base squashfs (this run's image build)
+        id: base
+        continue-on-error: true
+        uses: actions/download-artifact@v8
+        with:
+          name: ${{ matrix.platform }}.squashfs
+          path: base-new
+
+      - name: Skip platforms not built this run
+        id: check
+        run: |
+          if ls base-new/*.squashfs >/dev/null 2>&1; then
+            echo "found=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "::warning::No ${{ matrix.platform }}.squashfs built this run — skipping."
+            echo "found=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Fetch the legacy (0.3.5.1) base image
+        if: steps.check.outputs.found == 'true'
+        run: |
+          mkdir -p base-old
+          gh release download "$MIGRATION_FROM_TAG" --repo "${{ github.repository }}" \
+            --pattern "startos-*_${{ matrix.platform }}.iso" --dir base-old
+
+      # Runs in start9/build-env (the canonical Start9 build image — already has
+      # the squashfs toolchain + b3sum) so unsquashfs/mksquashfs can restore the
+      # rootfs's ownership + device nodes. Only xorriso (to crack the squashfs out
+      # of the 0.3.5.1 ISO) is missing, so we add just that. Not --privileged:
+      # creating device nodes needs only CAP_MKNOD, in docker's default caps.
+      - name: Assemble migration payload
+        if: steps.check.outputs.found == 'true'
+        run: |
+          ARCH=$(printf '%s' "${{ matrix.platform }}" | sed 's/-nonfree$//')
+          NEW=$(ls base-new/*.squashfs | head -n1)
+          OLD=$(ls base-old/startos-*_${{ matrix.platform }}.iso | head -n1)
+          mkdir -p results
+          OUT="results/$(basename "${NEW%.squashfs}").migration.squashfs"
+          docker run --rm -v "$PWD":/w -w /w \
+            -e ARCH="$ARCH" -e NEW="$NEW" -e OLD="$OLD" -e OUT="$OUT" \
+            -e OWNER_UID="$(id -u)" -e OWNER_GID="$(id -g)" \
+            start9/build-env bash -euc '
+              apt-get update -qq
+              apt-get install -yq --no-install-recommends xorriso
+              projects/start-os/build/assemble-migration-payload.sh \
+                --arch "$ARCH" --new-squashfs "$NEW" --old-image "$OLD" --out "$OUT"
+            '
+          echo "Built $OUT"
+
+      - uses: actions/upload-artifact@v7
+        if: steps.check.outputs.found == 'true'
+        with:
+          name: ${{ matrix.platform }}.migration.squashfs
+          path: results/*.migration.squashfs
+          retention-days: 14
+
   deploy:
     name: Deploy
     needs: [image]

projects/start-os/CHANGELOG.md

@@ -8,6 +8,18 @@ Full per-release notes are published on the
 [GitHub releases page](https://github.com/Start9Labs/start-technologies/releases). This
 file tracks notable changes since the move to the monorepo.
 
+## [Unreleased]
+
+### Added
+
+- **In-place 0.3.5.1 → 0.4.0 update path.** Updating the OS from 0.3.5.1 no longer
+  requires syncing a whole root filesystem file-by-file — the step that grew
+  flakier the more files a device had. StartOS 0.3.5.1's existing over-the-air
+  updater instead receives a compact migration payload (the 0.4.0 base image plus
+  a boot-time rewire) and the 0.4.0 initramfs converts the on-disk layout to the
+  0.4.0 format on first boot. The data partition is preserved and the existing
+  package/database migration runs afterward as before.
+
 ## [0.4.0-beta.10]
 
 ### Added

projects/start-os/build.mk

@@ -41,6 +41,20 @@ start-os-squashfs: results/$(BASENAME).squashfs
 results/$(BASENAME).$(IMAGE_TYPE) results/$(BASENAME).squashfs: $(IMAGE_RECIPE_SRC) results/$(BASENAME).deb
 	ARCH=$(ARCH) ./projects/start-os/build/image-recipe/run-local-build.sh "results/$(BASENAME).deb"
 
+# The OTA payload legacy (0.3.5.1) boxes pull via their frozen rsync updater: the
+# 0.4.0 base squashfs repackaged onto the published 0.3.5.1 rootfs. See
+# projects/start-os/build/assemble-migration-payload.sh.
+MIGRATION_FROM_TAG := v0.3.5.1
+
+start-os-migration-squashfs: results/$(BASENAME).migration.squashfs
+
+results/$(BASENAME).migration.squashfs: results/$(BASENAME).squashfs projects/start-os/build/assemble-migration-payload.sh projects/start-os/build/lib/scripts/migration-update-grub
+	@if [ "$(PLATFORM)" = raspberrypi ]; then >&2 echo "migration payload: raspberrypi has no in-place migration — reflash required (#3443)"; exit 1; fi
+	mkdir -p results/migration-base
+	gh release download $(MIGRATION_FROM_TAG) --repo Start9Labs/start-technologies \
+		--pattern 'startos-*_$(PLATFORM).iso' --dir results/migration-base --clobber
+	docker run --rm -v "$(CURDIR)":/w -w /w -e OWNER_UID="$$(id -u)" -e OWNER_GID="$$(id -g)" start9/build-env bash -euc 'apt-get update -qq && apt-get install -yq --no-install-recommends xorriso && projects/start-os/build/assemble-migration-payload.sh --arch $(ARCH) --new-squashfs results/$(BASENAME).squashfs --old-image "$$(ls -1 results/migration-base/startos-*_$(PLATFORM).iso | head -n1)" --out $@'
+
 # For creating os images. DO NOT USE
 start-os-install: $(STARTOS_TARGETS)
 	$(call mkdir,$(DESTDIR)/usr/bin)

projects/start-os/build/assemble-migration-payload.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+# Assemble the 0.3.5.1 -> 0.4.0 migration OTA payload.
+#
+# StartOS 0.3.5.1's (unchangeable) updater rsyncs the *contents* of the published
+# OTA squashfs into the box's `next` subvolume, then reboots. We base the payload
+# on the real 0.3.5.1 rootfs (extracted from its published release image) so the
+# box's existing `next`/`current` make the rsync a small delta, and the apply-time
+# `chroot next update-grub2` runs in a complete userland. On top we layer the
+# 0.4.0 upgrade: the 0.4.0 base image as a nested squashfs (what the 0.4.0
+# initramfs boots), the 0.4.0 kernel/initramfs, a migration sentinel, and our
+# deterministic bootloader updater. The old Haskell registry loop-mounts the
+# result and rsync-serves its tree unchanged.
+set -eo pipefail
+
+SOURCE_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
+
+ARCH=
+NEW_SQUASHFS=
+OLD_IMAGE=
+OUT=
+
+usage() {
+    >&2 echo "usage: $0 --arch ARCH --new-squashfs 0.4.0.squashfs --old-image 0.3.5.1.iso --out payload.squashfs"
+    exit 1
+}
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --arch)         ARCH="$2"; shift 2 ;;
+        --new-squashfs) NEW_SQUASHFS="$2"; shift 2 ;;
+        --old-image)    OLD_IMAGE="$2"; shift 2 ;;
+        --out)          OUT="$2"; shift 2 ;;
+        *) usage ;;
+    esac
+done
+[ -n "$ARCH" ] && [ -f "$NEW_SQUASHFS" ] && [ -f "$OLD_IMAGE" ] && [ -n "$OUT" ] || usage
+
+# Must run as root so unsquashfs/mksquashfs can restore the rootfs's ownership and
+# device nodes (as non-root, unsquashfs can't and exits non-zero). Callers wrap
+# this in a container rather than using host sudo (see build.mk / CI).
+if [ "$(id -u)" -ne 0 ]; then
+    >&2 echo "assemble-migration-payload: must run as root — wrap it in a container (the make target and CI do)"
+    exit 1
+fi
+
+WORK="$(mktemp -d)"
+trap 'rm -rf "$WORK"' EXIT
+
+# 1. Extract the 0.3.5.1 base rootfs (the payload's base) from its release image.
+case "$OLD_IMAGE" in
+    *.iso)
+        xorriso -osirrox on -indev "$OLD_IMAGE" -extract /live/filesystem.squashfs "$WORK/old.squashfs"
+        ;;
+    *)
+        >&2 echo "assemble-migration-payload: unsupported base image $OLD_IMAGE (only .iso wired so far)"
+        exit 1
+        ;;
+esac
+unsquashfs -d "$WORK/payload" "$WORK/old.squashfs"
+rm -f "$WORK/old.squashfs"
+
+# 2. Nest the 0.4.0 base image as images/<b3sum>.rootfs — the rootfs the 0.4.0
+#    initramfs installs and boots (naming matches the fresh-install layout).
+B3SUM="$(b3sum "$NEW_SQUASHFS" | head -c 16)"
+mkdir -p "$WORK/payload/images"
+cp "$NEW_SQUASHFS" "$WORK/payload/images/$B3SUM.rootfs"
+
+# 3. Stage the 0.4.0 kernel/initramfs; 0.3.5.1's sync_boot rsyncs boot/ -> /boot.
+#    Record the exact names so update-grub2 boots the 0.4.0 kernel, not whichever
+#    version happens to sort highest once /boot holds both.
+rm -rf "$WORK/payload/boot"
+unsquashfs -n -f -d "$WORK/payload" "$NEW_SQUASHFS" boot
+mkdir -p "$WORK/payload/usr/lib/startos"
+printf '%s\n%s\n' \
+    "$(cd "$WORK/payload/boot" && ls -1 vmlinuz-* | head -n1)" \
+    "$(cd "$WORK/payload/boot" && ls -1 initrd.img-* | head -n1)" \
+    > "$WORK/payload/usr/lib/startos/migration-boot"
+
+# 4. Sentinel the 0.4.0 initramfs keys on, plus our bootloader updater (0.3.5.1
+#    execs /usr/sbin/update-grub2 in the payload chroot at apply time).
+touch "$WORK/payload/.startos-migration"
+install -m0755 "$SOURCE_DIR/lib/scripts/migration-update-grub" "$WORK/payload/usr/sbin/update-grub2"
+
+# 5. Re-squash into the OTA payload the registry loop-mounts and serves.
+rm -f "$OUT"
+mksquashfs "$WORK/payload" "$OUT" -noappend -comp gzip -b 4096
+# hand the container-created output back to the invoking user (OWNER_* passed in)
+if [ -n "${OWNER_UID:-}" ]; then chown "$OWNER_UID:${OWNER_GID:-$OWNER_UID}" "$OUT"; fi
+
+echo "migration payload for $ARCH -> $OUT (base image $B3SUM.rootfs)"

projects/start-os/build/lib/scripts/migration-update-grub

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Shipped in the 0.3.5.1 -> 0.4.0 migration payload as /usr/sbin/update-grub2.
+#
+# StartOS 0.3.5.1's updater runs `chroot next update-grub2` at apply time, with
+# /boot and /proc bind-mounted into the payload. Rather than regenerate grub from
+# the payload rootfs (grub-mkconfig mis-probes root= outside a native boot), we
+# deterministically point the existing 0.3.5.1 bootloader at the 0.4.0
+# kernel/initramfs that sync_boot just copied into /boot, reusing the exact root=
+# this box already booted with. The 0.4.0 initramfs takes over from there.
+set -e
+
+BOOT=/boot
+MIGRATION_BOOT=/usr/lib/startos/migration-boot
+
+if [ -r "$MIGRATION_BOOT" ]; then
+    kernel="$(sed -n 1p "$MIGRATION_BOOT")"
+    initrd="$(sed -n 2p "$MIGRATION_BOOT")"
+else
+    kernel="$(cd "$BOOT" && ls -1 vmlinuz-* 2>/dev/null | sort -V | tail -n1)"
+    initrd="$(cd "$BOOT" && ls -1 initrd.img-* 2>/dev/null | sort -V | tail -n1)"
+fi
+if [ -z "$kernel" ] || [ -z "$initrd" ]; then
+    >&2 echo "migration-update-grub: 0.4.0 kernel/initramfs not found in $BOOT"
+    exit 1
+fi
+
+root="$(grep -oE 'root=[^ ]+' /proc/cmdline | head -n1)"
+if [ -z "$root" ]; then
+    >&2 echo "migration-update-grub: could not read root= from /proc/cmdline"
+    exit 1
+fi
+
+# boot=startos is what selects the StartOS initramfs module
+# (/etc/initramfs-tools/scripts/startos); without it the 0.4.0 layout rewire
+# never runs. The rest matches 0.4.0's GRUB_CMDLINE_LINUX (debian/postinst).
+CMDLINE="$root boot=startos console=ttyS0,115200n8 console=tty0"
+
+mkdir -p "$BOOT/grub"
+cat > "$BOOT/grub/grub.cfg" <<EOF
+set timeout=1
+set default=0
+insmod all_video
+menuentry "StartOS" {
+    insmod part_gpt
+    insmod part_msdos
+    insmod btrfs
+    insmod ext2
+    insmod fat
+    search --no-floppy --set=root --file "/$kernel"
+    linux "/$kernel" $CMDLINE
+    initrd "/$initrd"
+}
+EOF

projects/start-os/build/lib/scripts/startos-initramfs-module

@@ -53,29 +53,39 @@ local_mount_root()
 			exit 1
 		fi
 	else
-		if [ -f /startos/config/upgrade ] && [ -d /startos/next ]; then
-			oldroot=/startos/next
-		elif [ -d /startos/current ]; then
-			oldroot=/startos/current
-		elif [ -d /startos/prev ]; then
-			oldroot=/startos/prev
-		else
+		if ! [ -f /startos/next/.startos-migration ]; then
 			>&2 echo no StartOS filesystem found
 			exit 1
 		fi
 
-		mkdir -p /startos/config/overlay/etc
-		mv $oldroot/etc/fstab /startos/config/overlay/etc/fstab
-		mv $oldroot/etc/machine-id /startos/config/overlay/etc/machine-id
-		mv $oldroot/etc/ssh /startos/config/overlay/etc/ssh
-
-		mkdir -p /startos/images
-		mv $oldroot /startos/images/legacy.rootfs
+		# First boot after a 0.3.5.1 -> 0.4.0 update. The 0.3.5.1 updater rsynced a
+		# migration payload into `next` (the 0.4.0 base image under next/images plus
+		# this box's real machine-id/ssh/fstab, which it copied out of the running
+		# system) and pointed the bootloader at this initramfs. The old 0.3.x roots
+		# (current/next/prev) are disposable — only the data partition is preserved.
+		echo 'migrating 0.3.x layout to 0.4.0'
 
-		rm -rf /startos/next /startos/current /startos/prev
+		mkdir -p /startos/config/overlay/etc
+		cp -a /startos/next/etc/fstab /startos/config/overlay/etc/fstab
+		cp -a /startos/next/etc/machine-id /startos/config/overlay/etc/machine-id
+		rm -rf /startos/config/overlay/etc/ssh
+		cp -a /startos/next/etc/ssh /startos/config/overlay/etc/ssh
+
+		# Publish images/ by an atomic rename so a crash can never leave an empty
+		# images dir (which the branch above would treat as "no bootable image").
+		rm -rf /startos/images.new
+		mkdir -p /startos/images.new
+		mv /startos/next/images/*.rootfs /startos/images.new/
+		mv /startos/images.new /startos/images
+
+		image="$(ls -t1 /startos/images/*.rootfs | head -n1)"
+		if ! [ -f "$image" ]; then
+			>&2 echo 'migration payload missing 0.4.0 base image'
+			exit 1
+		fi
+		ln -rsf "$image" /startos/config/current.rootfs
 
-		ln -rsf /startos/images/old.squashfs /startos/config/current.rootfs
-		image=$(readlink -f /startos/config/current.rootfs)
+		rm -rf /startos/next /startos/current /startos/prev /startos/config/upgrade
 	fi
 
 	mkdir /lower /upper