Pin container images to digest and automate updates

DEVELOP.md

@@ -37,6 +37,25 @@ $ go get -d k8s.io/client-go@${VERSION} k8s.io/api@${VERSION} k8s.io/apimachiner
             k8s.io/kube-proxy@${VERSION}
 ```
 
+### Update container image digests
+
+`images_gen.go` contains the container image references (tag and digest) used by CKE.
+It is generated automatically by fetching the latest version of each image from the GitHub Packages API.
+
+Prerequisites: the `gh` CLI must be installed and authenticated with the `read:packages` scope.
+
+```console
+$ gh auth login -s "read:packages"
+```
+
+Then run:
+
+```console
+$ make images
+```
+
+This fetches the latest tagged version of each image and rewrites `images_gen.go`.
+
 ### Update the Kubernetes resource definitions embedded in CKE
 
 The Kubernetes resource definitions embedded in CKE is defined in `./static/resource.go`.

Makefile

@@ -32,6 +32,11 @@ test: test-tools
 install:
 	go install ./pkg/...
 
+.PHONY: images
+images:
+	go generate ./
+	$(MAKE) static
+
 .PHONY: static
 static: goimports
 	go generate ./static

container.go

@@ -63,20 +63,26 @@ type docker struct {
 }
 
 func (c docker) PullImage(img Image) error {
-	stdout, stderr, err := c.agent.Run("docker image list --format '{{.Repository}}:{{.Tag}}'")
+	stdout, stderr, err := c.agent.Run("docker image list --digests --format '{{.Repository}}:{{.Tag}}@{{.Digest}}'")
 	if err != nil {
 		return fmt.Errorf("%w, stdout: %s, stderr: %s", err, stdout, stderr)
 	}
 
-	for _, i := range strings.Split(string(stdout), "\n") {
-		if img.Name() == i {
+	noDigest := img.TagRef() + "@<none>"
+	for _, line := range strings.Split(strings.TrimSpace(string(stdout)), "\n") {
+		// Accept if FullRef matches (registry pull) or image has no digest (docker load).
+		if line == img.FullRef() || line == noDigest {
 			return nil
 		}
 	}
 
-	stdout, stderr, err = c.agent.Run("docker image pull " + img.Name())
+	stdout, stderr, err = c.agent.Run("docker image pull " + img.DigestRef())
 	if err != nil {
-		return fmt.Errorf("%w, stdout: %s, stderr: %s", err, stdout, stderr)
+		return fmt.Errorf("docker image pull %s: %w, stdout: %s, stderr: %s", img.DigestRef(), err, stdout, stderr)
+	}
+	stdout, stderr, err = c.agent.Run("docker image tag " + img.DigestRef() + " " + img.TagRef())
+	if err != nil {
+		return fmt.Errorf("docker image tag %s %s: %w, stdout: %s, stderr: %s", img.DigestRef(), img.TagRef(), err, stdout, stderr)
 	}
 	return nil
 }
@@ -86,6 +92,7 @@ func (c docker) Run(img Image, binds []Mount, command string, args ...string) er
 		"docker",
 		"run",
 		"--log-driver=journald",
+		"--pull=never",
 		"--rm",
 		"--network=host",
 		"--uts=host",
@@ -98,7 +105,7 @@ func (c docker) Run(img Image, binds []Mount, command string, args ...string) er
 		}
 		runArgs = append(runArgs, fmt.Sprintf("--volume=%s:%s:%s", m.Source, m.Destination, o))
 	}
-	runArgs = append(runArgs, img.Name(), command)
+	runArgs = append(runArgs, img.TagRef(), command)
 	runArgs = append(runArgs, args...)
 
 	_, _, err := c.agent.Run(strings.Join(runArgs, " "))
@@ -110,6 +117,7 @@ func (c docker) RunWithInput(img Image, binds []Mount, command, input string, ar
 		"docker",
 		"run",
 		"--log-driver=journald",
+		"--pull=never",
 		"--rm",
 		"-i",
 		"--network=host",
@@ -123,7 +131,7 @@ func (c docker) RunWithInput(img Image, binds []Mount, command, input string, ar
 		}
 		runArgs = append(runArgs, fmt.Sprintf("--volume=%s:%s:%s", m.Source, m.Destination, o))
 	}
-	runArgs = append(runArgs, img.Name(), command)
+	runArgs = append(runArgs, img.TagRef(), command)
 	runArgs = append(runArgs, args...)
 
 	return c.agent.RunWithInput(strings.Join(runArgs, " "), input)
@@ -134,6 +142,7 @@ func (c docker) RunWithOutput(img Image, binds []Mount, command string, args ...
 		"docker",
 		"run",
 		"--log-driver=journald",
+		"--pull=never",
 		"--rm",
 		"--network=host",
 		"--uts=host",
@@ -146,7 +155,7 @@ func (c docker) RunWithOutput(img Image, binds []Mount, command string, args ...
 		}
 		runArgs = append(runArgs, fmt.Sprintf("--volume=%s:%s:%s", m.Source, m.Destination, o))
 	}
-	runArgs = append(runArgs, img.Name(), command)
+	runArgs = append(runArgs, img.TagRef(), command)
 	runArgs = append(runArgs, args...)
 
 	stdout, stderr, err := c.agent.Run(strings.Join(runArgs, " "))
@@ -171,6 +180,7 @@ func (c docker) RunSystem(name string, img Image, opts []string, params, extra S
 		"docker",
 		"run",
 		"--log-driver=journald",
+		"--pull=never",
 		"-d",
 		"--name=" + name,
 		"--read-only",
@@ -218,7 +228,7 @@ func (c docker) RunSystem(name string, img Image, opts []string, params, extra S
 	}
 	args = append(args, "--label-file="+labelFile)
 
-	args = append(args, img.Name())
+	args = append(args, img.TagRef())
 
 	args = append(args, params.ExtraArguments...)
 	args = append(args, extra.ExtraArguments...)

docs/design.md

@@ -85,6 +85,7 @@ Implementation policies
 * CKE does not install any tools onto node OS other than containers.
 
     * `kubelet` or other system services run by `docker run`.
+    * Images are identified by digest-pinned references (`repository:tag@sha256:...`). Before running a container, CKE pulls the image if not already present. See [image pull specification](image-pull.md) for details.
 
 * CKE employs CNI network plugins.
 

docs/image-pull.md

@@ -0,0 +1,54 @@
+Image Pull Specification
+========================
+
+Image reference format
+----------------------
+
+CKE manages container images using digest-pinned references in the form:
+
+```
+repository:tag@sha256:<digest>
+```
+
+All image constants (e.g. `EtcdImage`, `KubernetesImage`) are defined in this format and provide three accessors:
+
+| Method | Returns |
+|--------|---------|
+| `FullRef()` | `repository:tag@sha256:<digest>` |
+| `TagRef()` | `repository:tag` |
+
+PullImage behaviour
+-------------------
+
+Before pulling an image, CKE checks whether a suitable image is already present on the node using `docker image list --format '{{.Repository}}:{{.Tag}}@{{.Digest}}'`.
+
+Each line of the output is compared against two conditions:
+
+1. **FullRef match** — the line equals `img.FullRef()` (e.g. `ghcr.io/cybozu/etcd:3.6.11.1@sha256:...`).  
+   This is the normal case after an image has been pulled from a registry.
+
+2. **No-digest match** — the line equals `img.TagRef()+"@<none>"` (e.g. `ghcr.io/cybozu/etcd:3.6.11.1@<none>`).  
+   This covers images loaded via `docker load` from a tar archive, which have a tag but no RepoDigest.
+
+If neither condition is met (including when the tag matches but the digest differs), the image is considered absent and the following steps are executed:
+
+1. `docker image pull <FullRef>` — pulls the image by digest. Docker stores it with `<none>` as the tag.
+2. `docker image tag <FullRef> <TagRef>` — assigns the tag so the image can be addressed by `TagRef` in subsequent `docker run` calls.
+
+Running containers
+------------------
+
+All `docker run` invocations use:
+
+- `--pull=never` — prevents Docker from attempting a pull at run time; the image must already be present from `PullImage`.
+- `TagRef` as the image argument — works for both registry-pulled images (which have the tag) and `docker load` images (which lack a RepoDigest and cannot be addressed by digest).
+
+Air-gap environments
+--------------------
+
+In air-gapped environments, images are pre-loaded onto nodes via `docker load` from a tar archive. These images have a tag but no RepoDigest.
+
+CKE handles this as follows:
+
+1. `PullImage` detects the no-digest match and skips the pull.
+2. `docker run` addresses the image by `TagRef`, which succeeds because the tag is present.

images.go

@@ -1,33 +1,34 @@
 package cke
 
-// Image is the type of container images.
-type Image string
+//go:generate go run ./pkg/update-images/
 
-// Name returns docker image name.
-func (i Image) Name() string {
-	return string(i)
+// Image represents a container image reference.
+type Image struct {
+	fullRef   string
+	tagRef    string
+	digestRef string
 }
 
-// Container image definitions
-const (
-	EtcdImage            = Image("ghcr.io/cybozu/etcd:3.6.11.1")
-	KubernetesImage      = Image("ghcr.io/cybozu/kubernetes:1.35.5.1")
-	ToolsImage           = Image("ghcr.io/cybozu-go/cke-tools:1.35.0")
-	PauseImage           = Image("ghcr.io/cybozu/pause:3.10.1.5")
-	CoreDNSImage         = Image("ghcr.io/cybozu/coredns:1.14.2.1")
-	UnboundImage         = Image("ghcr.io/cybozu/unbound:1.25.1.1")
-	UnboundExporterImage = Image("ghcr.io/cybozu/unbound_exporter:0.5.0.4")
-)
-
-// AllImages return container images list used by CKE
-func AllImages() []string {
-	return []string{
-		EtcdImage.Name(),
-		ToolsImage.Name(),
-		KubernetesImage.Name(),
-		PauseImage.Name(),
-		CoreDNSImage.Name(),
-		UnboundImage.Name(),
-		UnboundExporterImage.Name(),
+func newImage(repository, tag, digest string) Image {
+	return Image{
+		fullRef:   repository + ":" + tag + "@" + digest,
+		tagRef:    repository + ":" + tag,
+		digestRef: repository + "@" + digest,
 	}
 }
+
+// FullRef returns the full image reference (repository:tag@digest).
+func (i Image) FullRef() string {
+	return i.fullRef
+}
+
+// TagRef returns the repository:tag reference without the digest.
+func (i Image) TagRef() string {
+	return i.tagRef
+}
+
+// DigestRef returns the repository@digest reference without the tag.
+func (i Image) DigestRef() string {
+	return i.digestRef
+}
+