From d51e875ee8857b8573cd32a29067ff54e6d5d293 Mon Sep 17 00:00:00 2001 From: Masayuki Ishii Date: Thu, 28 May 2026 17:26:02 +0900 Subject: [PATCH 01/13] Pin container images to digest and automate updates Change container image references from repo:tag to repo:tag@sha256:... to prevent supply chain attacks via tag mutation. Also automate digest fetching to reduce maintenance overhead. Image definitions are split out of images.go into a new images_gen.go. images.go retains the Image type definition and AllImages() function, while individual image constants are managed in images_gen.go, which is written by the generator. The generator (pkg/update-images/main.go) calls the GitHub Packages API via the gh CLI to fetch the latest tag and digest for each image and overwrites images_gen.go. Running it requires gh auth login with the read:packages scope. The PullImage implementation is updated. Previously it listed images via docker image list and matched by tag, which does not work with digest-pinned references. It now checks existence using docker image inspect with the full reference (repo:tag@sha256:...). Error output from inspect and pull failures is also included in returned errors. --- DEVELOP.md | 19 +++++ Makefile | 5 ++ container.go | 5 +- images.go | 36 ++++++--- images_gen.go | 13 +++ localproxy/infrastructure.go | 5 +- localproxy/status.go | 20 ++--- pkg/update-images/main.go | 152 +++++++++++++++++++++++++++++++++++ static/resources.go | 8 +- 9 files changed, 231 insertions(+), 32 deletions(-) create mode 100644 images_gen.go create mode 100644 pkg/update-images/main.go diff --git a/DEVELOP.md b/DEVELOP.md index c8b761dd4..7ad79e8bf 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -37,6 +37,25 @@ $ go get -d k8s.io/client-go@${VERSION} k8s.io/api@${VERSION} k8s.io/apimachiner k8s.io/kube-proxy@${VERSION} ``` +### Update container image digests + +`images_gen.go` contains the container image references (tag and digest) used by CKE. +It is generated automatically by fetching the latest version of each image from the GitHub Packages API. + +Prerequisites: the `gh` CLI must be installed and authenticated with the `read:packages` scope. + +```console +$ gh auth login -s "read:packages" +``` + +Then run: + +```console +$ make images +``` + +This fetches the latest tagged version of each image and rewrites `images_gen.go`. + ### Update the Kubernetes resource definitions embedded in CKE The Kubernetes resource definitions embedded in CKE is defined in `./static/resource.go`. diff --git a/Makefile b/Makefile index b7427a981..5e1c61abc 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,11 @@ test: test-tools install: go install ./pkg/... +.PHONY: images +images: + go generate ./ + $(MAKE) static + .PHONY: static static: goimports go generate ./static diff --git a/container.go b/container.go index d02928734..c44bdd074 100644 --- a/container.go +++ b/container.go @@ -63,13 +63,14 @@ type docker struct { } func (c docker) PullImage(img Image) error { - stdout, stderr, err := c.agent.Run("docker image list --format '{{.Repository}}:{{.Tag}}'") + stdout, stderr, err := c.agent.Run("docker image list --digests --format '{{.Repository}}@{{.Digest}}'") if err != nil { return fmt.Errorf("%w, stdout: %s, stderr: %s", err, stdout, stderr) } + ref := img.Repository() + "@" + img.Digest() for _, i := range strings.Split(string(stdout), "\n") { - if img.Name() == i { + if ref == i { return nil } } diff --git a/images.go b/images.go index facdb7507..958b8dc0a 100644 --- a/images.go +++ b/images.go @@ -1,23 +1,37 @@ package cke +import "strings" + +//go:generate go run ./pkg/update-images/ + // Image is the type of container images. type Image string -// Name returns docker image name. +// Name returns the full image reference. func (i Image) Name() string { return string(i) } -// Container image definitions -const ( - EtcdImage = Image("ghcr.io/cybozu/etcd:3.6.11.1") - KubernetesImage = Image("ghcr.io/cybozu/kubernetes:1.35.5.1") - ToolsImage = Image("ghcr.io/cybozu-go/cke-tools:1.35.0") - PauseImage = Image("ghcr.io/cybozu/pause:3.10.1.5") - CoreDNSImage = Image("ghcr.io/cybozu/coredns:1.14.2.1") - UnboundImage = Image("ghcr.io/cybozu/unbound:1.25.1.1") - UnboundExporterImage = Image("ghcr.io/cybozu/unbound_exporter:0.5.0.4") -) +// Repository returns the repository part of the image reference (without tag and digest). +func (i Image) Repository() string { + name := string(i) + if idx := strings.Index(name, "@"); idx >= 0 { + name = name[:idx] + } + if idx := strings.LastIndex(name, ":"); idx >= 0 && !strings.Contains(name[idx:], "/") { + name = name[:idx] + } + return name +} + +// Digest returns the digest part of the image reference (e.g. "sha256:..."). +func (i Image) Digest() string { + name := string(i) + if idx := strings.Index(name, "@"); idx >= 0 { + return name[idx+1:] + } + return "" +} // AllImages return container images list used by CKE func AllImages() []string { diff --git a/images_gen.go b/images_gen.go new file mode 100644 index 000000000..653d7861a --- /dev/null +++ b/images_gen.go @@ -0,0 +1,13 @@ +// Code generated by pkg/update-images. DO NOT EDIT. + +package cke + +const ( + EtcdImage = Image("ghcr.io/cybozu/etcd:3.6.11.1@sha256:c2e3075893ca9264773d022c4789e401c6ff7346ffd4101a4d79f3bdf37f5637") + KubernetesImage = Image("ghcr.io/cybozu/kubernetes:1.35.5.1@sha256:1051a9fba32c5a095c3895cb1b215d55454ee662d53147dcba56fb1cbda431ea") + ToolsImage = Image("ghcr.io/cybozu-go/cke-tools:1.35.0@sha256:3f0365cf68834dc9e2e082ec901495cae41c8cfc5dcaa9c3a2062d4b50a66f82") + PauseImage = Image("ghcr.io/cybozu/pause:3.10.1.5@sha256:3565a5a085d941c10ef86aaf2bfdaabc8c0f8677d55b2371dce472eb79a79d08") + CoreDNSImage = Image("ghcr.io/cybozu/coredns:1.14.2.1@sha256:e29066d55b9455fdba973ceb6f77232677de15263cc1c78f63707935adc01a14") + UnboundImage = Image("ghcr.io/cybozu/unbound:1.25.1.1@sha256:b84ce87568ed11859891747650a46d8431bac6d1ddda38b542e4d16cd36e0b86") + UnboundExporterImage = Image("ghcr.io/cybozu/unbound_exporter:0.5.0.4@sha256:a7d46220f43bf2ae0c81bbcb0b68801de2d60e87df298db4691ea0dd39b25593") +) diff --git a/localproxy/infrastructure.go b/localproxy/infrastructure.go index 23f583686..ff05a2e28 100644 --- a/localproxy/infrastructure.go +++ b/localproxy/infrastructure.go @@ -113,14 +113,15 @@ var _ cke.ContainerEngine = localDocker{} // PullImage pulls an image. func (l localDocker) PullImage(img cke.Image) error { - cmd := exec.Command("docker", "image", "list", "--format={{.Repository}}:{{.Tag}}") + cmd := exec.Command("docker", "image", "list", "--digests", "--format={{.Repository}}@{{.Digest}}") stdout, err := cmd.Output() if err != nil { return fmt.Errorf("failed to execute docker image list: %w", err) } + ref := img.Repository() + "@" + img.Digest() for _, i := range strings.Fields(string(stdout)) { - if img.Name() == i { + if ref == i { return nil } } diff --git a/localproxy/status.go b/localproxy/status.go index 96918a745..ba7ebee1f 100644 --- a/localproxy/status.go +++ b/localproxy/status.go @@ -33,22 +33,16 @@ var dialer = &net.Dialer{ } func isRunning(name string) (bool, string, error) { - stdout, err := exec.Command("docker", "ps", "--format={{.Names}} {{.Image}}").Output() + out, err := exec.Command("docker", "container", "inspect", "--format={{.State.Running}} {{.Config.Image}}", name).Output() if err != nil { - return false, "", fmt.Errorf("failed to run docker ps: %w", err) + // Container does not exist + return false, "", nil } - - for _, line := range strings.Split(string(stdout), "\n") { - fields := strings.Fields(line) - if len(fields) != 2 { - continue - } - if fields[0] != name { - continue - } - return true, fields[1], nil + fields := strings.Fields(string(out)) + if len(fields) != 2 { + return false, "", fmt.Errorf("unexpected docker inspect output: %s", out) } - return false, "", nil + return fields[0] == "true", fields[1], nil } func getStatus(ctx context.Context, inf cke.Infrastructure) (*status, error) { diff --git a/pkg/update-images/main.go b/pkg/update-images/main.go new file mode 100644 index 000000000..993fcf038 --- /dev/null +++ b/pkg/update-images/main.go @@ -0,0 +1,152 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "go/format" + "net/url" + "os" + "os/exec" + "strings" + "text/template" +) + +type imageEntry struct { + VarName string + Repository string +} + +// Update Repository here when adding or removing images. +var images = []imageEntry{ + {"EtcdImage", "ghcr.io/cybozu/etcd"}, + {"KubernetesImage", "ghcr.io/cybozu/kubernetes"}, + {"ToolsImage", "ghcr.io/cybozu-go/cke-tools"}, + {"PauseImage", "ghcr.io/cybozu/pause"}, + {"CoreDNSImage", "ghcr.io/cybozu/coredns"}, + {"UnboundImage", "ghcr.io/cybozu/unbound"}, + {"UnboundExporterImage", "ghcr.io/cybozu/unbound_exporter"}, +} + +const outputFile = "images_gen.go" + +const tmpl = `// Code generated by pkg/update-images. DO NOT EDIT. + +package cke + +const ( +{{- range .}} + {{.VarName}} = Image("{{.Repository}}:{{.Tag}}@{{.Digest}}") +{{- end}} +) +` + +type packageVersion struct { + Name string `json:"name"` // digest e.g. "sha256:abc..." + CreatedAt string `json:"created_at"` + Metadata struct { + Container struct { + Tags []string `json:"tags"` + } `json:"container"` + } `json:"metadata"` +} + +type imageResult struct { + VarName string + Repository string + Tag string + Digest string +} + +// fetchLatest queries the GitHub Packages API (via gh CLI) for the latest +// version of repository, returning its tag and digest. +func fetchLatest(repository string) (tag, digest string, err error) { + withoutRegistry, _ := strings.CutPrefix(repository, "ghcr.io/") + org, pkg, ok := strings.Cut(withoutRegistry, "/") + if !ok { + return "", "", fmt.Errorf("invalid repository %q: expected ghcr.io/org/package", repository) + } + + apiPath := fmt.Sprintf("/orgs/%s/packages/container/%s/versions?per_page=100", + org, url.PathEscape(pkg)) + + out, err := exec.Command("gh", "api", apiPath, "--paginate").Output() + if err != nil { + return "", "", fmt.Errorf("gh api failed for %s: %w", repository, err) + } + + var versions []packageVersion + dec := json.NewDecoder(bytes.NewReader(out)) + for dec.More() { + var page []packageVersion + if err := dec.Decode(&page); err != nil { + return "", "", fmt.Errorf("failed to decode response for %s: %w", repository, err) + } + versions = append(versions, page...) + } + + // Pick the latest version that has at least one tag. + var latest *packageVersion + for i := range versions { + v := &versions[i] + if len(v.Metadata.Container.Tags) == 0 { + continue + } + if latest == nil || v.CreatedAt > latest.CreatedAt { + latest = v + } + } + if latest == nil { + return "", "", fmt.Errorf("no tagged version found for %s", repository) + } + + // Pick the longest tag excluding "latest" + for _, t := range latest.Metadata.Container.Tags { + if t != "latest" && len(t) > len(tag) { + tag = t + } + } + if tag == "" { + return "", "", fmt.Errorf("no non-latest tag found for %s", repository) + } + + return tag, latest.Name, nil +} + +func main() { + results := make([]imageResult, len(images)) + for i, img := range images { + tag, digest, err := fetchLatest(img.Repository) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to fetch latest for %s: %v\n", img.Repository, err) + os.Exit(1) + } + fmt.Printf("%s %s:%s -> %s\n", img.VarName, img.Repository, tag, digest) + results[i] = imageResult{ + VarName: img.VarName, + Repository: img.Repository, + Tag: tag, + Digest: digest, + } + } + + t := template.Must(template.New("").Parse(tmpl)) + var buf bytes.Buffer + if err := t.Execute(&buf, results); err != nil { + fmt.Fprintf(os.Stderr, "failed to render template: %v\n", err) + os.Exit(1) + } + + formatted, err := format.Source(buf.Bytes()) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to format source: %v\n", err) + os.Exit(1) + } + + if err := os.WriteFile(outputFile, formatted, 0644); err != nil { + fmt.Fprintf(os.Stderr, "failed to write %s: %v\n", outputFile, err) + os.Exit(1) + } + + fmt.Printf("Updated %s\n", outputFile) +} diff --git a/static/resources.go b/static/resources.go index 2f8f6c466..dd1869ad8 100644 --- a/static/resources.go +++ b/static/resources.go @@ -60,8 +60,8 @@ var Resources = []cke.ResourceDefinition{ Namespace: "kube-system", Name: "node-dns", Revision: 4, - Image: "ghcr.io/cybozu/unbound:1.25.1.1,ghcr.io/cybozu/unbound_exporter:0.5.0.4", - Definition: []byte("kind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: node-dns\n namespace: kube-system\n annotations:\n cke.cybozu.com/image: \"ghcr.io/cybozu/unbound:1.25.1.1,ghcr.io/cybozu/unbound_exporter:0.5.0.4\"\n cke.cybozu.com/revision: \"4\"\nspec:\n selector:\n matchLabels:\n cke.cybozu.com/appname: node-dns\n updateStrategy:\n type: RollingUpdate\n rollingUpdate:\n maxSurge: 35%\n maxUnavailable: 0\n template:\n metadata:\n labels:\n cke.cybozu.com/appname: node-dns\n spec:\n priorityClassName: system-node-critical\n nodeSelector:\n kubernetes.io/os: linux\n hostNetwork: true\n tolerations:\n - operator: Exists\n terminationGracePeriodSeconds: 1\n containers:\n - name: unbound\n image: ghcr.io/cybozu/unbound:1.25.1.1\n args:\n - -c\n - /etc/unbound/unbound.conf\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n add:\n - NET_BIND_SERVICE\n drop:\n - all\n readOnlyRootFilesystem: true\n readinessProbe:\n tcpSocket:\n port: 53\n host: localhost\n periodSeconds: 1\n livenessProbe:\n tcpSocket:\n port: 53\n host: localhost\n periodSeconds: 1\n initialDelaySeconds: 1\n failureThreshold: 6\n volumeMounts:\n - name: config-volume\n mountPath: /etc/unbound\n - name: var-run-unbound\n mountPath: /var/run/unbound\n resources:\n requests:\n cpu: 50m\n memory: 250Mi\n - name: reload\n image: ghcr.io/cybozu/unbound:1.25.1.1\n command:\n - /usr/local/bin/reload-unbound\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - all\n readOnlyRootFilesystem: true\n volumeMounts:\n - name: config-volume\n mountPath: /etc/unbound\n - name: var-run-unbound\n mountPath: /var/run/unbound\n - name: exporter\n image: ghcr.io/cybozu/unbound_exporter:0.5.0.4\n args:\n # must be same with the path written in /op/nodedns/nodedns.go\n - --unbound.host=unix:///var/run/unbound/unbound.sock\n - --web.reuse-port=true\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - all\n readOnlyRootFilesystem: true\n volumeMounts:\n - name: var-run-unbound\n mountPath: /var/run/unbound\n volumes:\n - name: config-volume\n configMap:\n name: node-dns\n items:\n - key: unbound.conf\n path: unbound.conf\n - name: var-run-unbound\n emptyDir: {}\n"), + Image: "ghcr.io/cybozu/unbound:1.25.1.1@sha256:b84ce87568ed11859891747650a46d8431bac6d1ddda38b542e4d16cd36e0b86,ghcr.io/cybozu/unbound_exporter:0.5.0.4@sha256:a7d46220f43bf2ae0c81bbcb0b68801de2d60e87df298db4691ea0dd39b25593", + Definition: []byte("kind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: node-dns\n namespace: kube-system\n annotations:\n cke.cybozu.com/image: \"ghcr.io/cybozu/unbound:1.25.1.1@sha256:b84ce87568ed11859891747650a46d8431bac6d1ddda38b542e4d16cd36e0b86,ghcr.io/cybozu/unbound_exporter:0.5.0.4@sha256:a7d46220f43bf2ae0c81bbcb0b68801de2d60e87df298db4691ea0dd39b25593\"\n cke.cybozu.com/revision: \"4\"\nspec:\n selector:\n matchLabels:\n cke.cybozu.com/appname: node-dns\n updateStrategy:\n type: RollingUpdate\n rollingUpdate:\n maxSurge: 35%\n maxUnavailable: 0\n template:\n metadata:\n labels:\n cke.cybozu.com/appname: node-dns\n spec:\n priorityClassName: system-node-critical\n nodeSelector:\n kubernetes.io/os: linux\n hostNetwork: true\n tolerations:\n - operator: Exists\n terminationGracePeriodSeconds: 1\n containers:\n - name: unbound\n image: ghcr.io/cybozu/unbound:1.25.1.1@sha256:b84ce87568ed11859891747650a46d8431bac6d1ddda38b542e4d16cd36e0b86\n args:\n - -c\n - /etc/unbound/unbound.conf\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n add:\n - NET_BIND_SERVICE\n drop:\n - all\n readOnlyRootFilesystem: true\n readinessProbe:\n tcpSocket:\n port: 53\n host: localhost\n periodSeconds: 1\n livenessProbe:\n tcpSocket:\n port: 53\n host: localhost\n periodSeconds: 1\n initialDelaySeconds: 1\n failureThreshold: 6\n volumeMounts:\n - name: config-volume\n mountPath: /etc/unbound\n - name: var-run-unbound\n mountPath: /var/run/unbound\n resources:\n requests:\n cpu: 50m\n memory: 250Mi\n - name: reload\n image: ghcr.io/cybozu/unbound:1.25.1.1@sha256:b84ce87568ed11859891747650a46d8431bac6d1ddda38b542e4d16cd36e0b86\n command:\n - /usr/local/bin/reload-unbound\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - all\n readOnlyRootFilesystem: true\n volumeMounts:\n - name: config-volume\n mountPath: /etc/unbound\n - name: var-run-unbound\n mountPath: /var/run/unbound\n - name: exporter\n image: ghcr.io/cybozu/unbound_exporter:0.5.0.4@sha256:a7d46220f43bf2ae0c81bbcb0b68801de2d60e87df298db4691ea0dd39b25593\n args:\n # must be same with the path written in /op/nodedns/nodedns.go\n - --unbound.host=unix:///var/run/unbound/unbound.sock\n - --web.reuse-port=true\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - all\n readOnlyRootFilesystem: true\n volumeMounts:\n - name: var-run-unbound\n mountPath: /var/run/unbound\n volumes:\n - name: config-volume\n configMap:\n name: node-dns\n items:\n - key: unbound.conf\n path: unbound.conf\n - name: var-run-unbound\n emptyDir: {}\n"), }, { Key: "Deployment/kube-system/cluster-dns", @@ -69,8 +69,8 @@ var Resources = []cke.ResourceDefinition{ Namespace: "kube-system", Name: "cluster-dns", Revision: 5, - Image: "ghcr.io/cybozu/coredns:1.14.2.1", - Definition: []byte("\nkind: Deployment\napiVersion: apps/v1\nmetadata:\n name: cluster-dns\n namespace: kube-system\n annotations:\n cke.cybozu.com/image: \"ghcr.io/cybozu/coredns:1.14.2.1\"\n cke.cybozu.com/revision: \"5\"\nspec:\n replicas: 2\n strategy:\n type: RollingUpdate\n rollingUpdate:\n maxUnavailable: 1\n selector:\n matchLabels:\n cke.cybozu.com/appname: cluster-dns\n template:\n metadata:\n labels:\n cke.cybozu.com/appname: cluster-dns\n k8s-app: coredns # sonobuoy requires\n annotations:\n prometheus.io/port: \"9153\"\n spec:\n priorityClassName: system-cluster-critical\n serviceAccountName: cke-cluster-dns\n tolerations:\n - key: node-role.kubernetes.io/master\n effect: NoSchedule\n - key: \"CriticalAddonsOnly\"\n operator: \"Exists\"\n - key: kubernetes.io/e2e-evict-taint-key\n operator: Exists\n # for sonobuoy https://github.com/vmware-tanzu/sonobuoy/pull/878\n affinity:\n podAntiAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - labelSelector:\n matchLabels:\n cke.cybozu.com/appname: cluster-dns\n topologyKey: kubernetes.io/hostname\n topologySpreadConstraints:\n - labelSelector:\n matchLabels:\n cke.cybozu.com/appname: cluster-dns\n maxSkew: 1\n topologyKey: topology.kubernetes.io/zone\n whenUnsatisfiable: ScheduleAnyway\n containers:\n - name: coredns\n image: ghcr.io/cybozu/coredns:1.14.2.1\n imagePullPolicy: IfNotPresent\n resources:\n requests:\n cpu: 50m\n memory: 250Mi\n args: [ \"-conf\", \"/etc/coredns/Corefile\" ]\n lifecycle:\n preStop:\n exec:\n command: [\"sh\", \"-c\", \"sleep 5\"]\n volumeMounts:\n - name: config-volume\n mountPath: /etc/coredns\n readOnly: true\n ports:\n - containerPort: 1053\n name: dns\n protocol: UDP\n - containerPort: 1053\n name: dns-tcp\n protocol: TCP\n - containerPort: 9153\n name: metrics\n protocol: TCP\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - all\n readOnlyRootFilesystem: true\n readinessProbe:\n httpGet:\n path: /ready\n port: 8181\n scheme: HTTP\n livenessProbe:\n httpGet:\n path: /health\n port: 8080\n scheme: HTTP\n initialDelaySeconds: 60\n timeoutSeconds: 5\n successThreshold: 1\n failureThreshold: 5\n dnsPolicy: Default\n volumes:\n - name: config-volume\n configMap:\n name: cluster-dns\n items:\n - key: Corefile\n path: Corefile\n"), + Image: "ghcr.io/cybozu/coredns:1.14.2.1@sha256:e29066d55b9455fdba973ceb6f77232677de15263cc1c78f63707935adc01a14", + Definition: []byte("\nkind: Deployment\napiVersion: apps/v1\nmetadata:\n name: cluster-dns\n namespace: kube-system\n annotations:\n cke.cybozu.com/image: \"ghcr.io/cybozu/coredns:1.14.2.1@sha256:e29066d55b9455fdba973ceb6f77232677de15263cc1c78f63707935adc01a14\"\n cke.cybozu.com/revision: \"5\"\nspec:\n replicas: 2\n strategy:\n type: RollingUpdate\n rollingUpdate:\n maxUnavailable: 1\n selector:\n matchLabels:\n cke.cybozu.com/appname: cluster-dns\n template:\n metadata:\n labels:\n cke.cybozu.com/appname: cluster-dns\n k8s-app: coredns # sonobuoy requires\n annotations:\n prometheus.io/port: \"9153\"\n spec:\n priorityClassName: system-cluster-critical\n serviceAccountName: cke-cluster-dns\n tolerations:\n - key: node-role.kubernetes.io/master\n effect: NoSchedule\n - key: \"CriticalAddonsOnly\"\n operator: \"Exists\"\n - key: kubernetes.io/e2e-evict-taint-key\n operator: Exists\n # for sonobuoy https://github.com/vmware-tanzu/sonobuoy/pull/878\n affinity:\n podAntiAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - labelSelector:\n matchLabels:\n cke.cybozu.com/appname: cluster-dns\n topologyKey: kubernetes.io/hostname\n topologySpreadConstraints:\n - labelSelector:\n matchLabels:\n cke.cybozu.com/appname: cluster-dns\n maxSkew: 1\n topologyKey: topology.kubernetes.io/zone\n whenUnsatisfiable: ScheduleAnyway\n containers:\n - name: coredns\n image: ghcr.io/cybozu/coredns:1.14.2.1@sha256:e29066d55b9455fdba973ceb6f77232677de15263cc1c78f63707935adc01a14\n imagePullPolicy: IfNotPresent\n resources:\n requests:\n cpu: 50m\n memory: 250Mi\n args: [ \"-conf\", \"/etc/coredns/Corefile\" ]\n lifecycle:\n preStop:\n exec:\n command: [\"sh\", \"-c\", \"sleep 5\"]\n volumeMounts:\n - name: config-volume\n mountPath: /etc/coredns\n readOnly: true\n ports:\n - containerPort: 1053\n name: dns\n protocol: UDP\n - containerPort: 1053\n name: dns-tcp\n protocol: TCP\n - containerPort: 9153\n name: metrics\n protocol: TCP\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - all\n readOnlyRootFilesystem: true\n readinessProbe:\n httpGet:\n path: /ready\n port: 8181\n scheme: HTTP\n livenessProbe:\n httpGet:\n path: /health\n port: 8080\n scheme: HTTP\n initialDelaySeconds: 60\n timeoutSeconds: 5\n successThreshold: 1\n failureThreshold: 5\n dnsPolicy: Default\n volumes:\n - name: config-volume\n configMap:\n name: cluster-dns\n items:\n - key: Corefile\n path: Corefile\n"), }, { Key: "PodDisruptionBudget/kube-system/cluster-dns-pdb", From 3daf3510781f5c155284bb3d1be5a90cc66e33eb Mon Sep 17 00:00:00 2001 From: Masayuki Ishii Date: Wed, 10 Jun 2026 13:13:07 +0900 Subject: [PATCH 02/13] mtest: verify container images are digest-pinned --- mtest/localproxy_test.go | 20 ++++++++++++++++++++ mtest/operators_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/mtest/localproxy_test.go b/mtest/localproxy_test.go index 347bc7fb2..d496c1bda 100644 --- a/mtest/localproxy_test.go +++ b/mtest/localproxy_test.go @@ -2,6 +2,7 @@ package mtest import ( "bytes" + "encoding/json" "errors" "fmt" @@ -42,4 +43,23 @@ func testLocalProxy() { return errors.New("cke-unbound service is not running") }, 5, 0.1).Should(Succeed()) }) + + It("should use digest-pinned images", func() { + out := execSafeAt(host1, "docker", "inspect", "kube-proxy", "cke-unbound") + var inspects []struct { + Config struct { + Image string `json:"Image"` + } `json:"Config"` + Name string `json:"Name"` + } + err := json.Unmarshal(out, &inspects) + Expect(err).NotTo(HaveOccurred()) + Expect(inspects).To(HaveLen(2)) + + for _, inspect := range inspects { + fmt.Fprintf(GinkgoWriter, "container=%s image=%s\n", inspect.Name, inspect.Config.Image) + Expect(inspect.Config.Image).To(ContainSubstring("@sha256:"), + "container %s uses non-digest image: %s", inspect.Name, inspect.Config.Image) + } + }) } diff --git a/mtest/operators_test.go b/mtest/operators_test.go index 0d8f1cb87..11ad6e10d 100644 --- a/mtest/operators_test.go +++ b/mtest/operators_test.go @@ -90,6 +90,30 @@ func testOperators() { }, )) + By("Checking container images") + for _, n := range []string{node1, node2, node3, node4, node5} { + out := execSafeAt(n, "docker", "ps", "-aq") + containerIDs := strings.Fields(strings.TrimSpace(string(out))) + Expect(containerIDs).NotTo(BeEmpty(), "node %s has no containers", n) + + out = execSafeAt(n, append([]string{"docker", "inspect"}, containerIDs...)...) + var inspects []struct { + Config struct { + Image string `json:"Image"` + } `json:"Config"` + Name string `json:"Name"` + } + err := json.Unmarshal(out, &inspects) + Expect(err).NotTo(HaveOccurred()) + Expect(inspects).To(HaveLen(len(containerIDs))) + + for _, inspect := range inspects { + fmt.Fprintf(GinkgoWriter, "node=%s container=%s image=%s\n", n, inspect.Name, inspect.Config.Image) + Expect(inspect.Config.Image).To(ContainSubstring("@sha256:"), + "container %s on node %s uses non-digest image: %s", inspect.Name, n, inspect.Config.Image) + } + } + By("Stopping etcd servers") // this will run: // - EtcdStartOp From 9d49a50dec05616bba9cb800c46a1e1577e5b310 Mon Sep 17 00:00:00 2001 From: Masayuki Ishii Date: Thu, 18 Jun 2026 13:36:46 +0900 Subject: [PATCH 03/13] Fix PullImage to accept docker-load images without RepoDigest Images loaded via docker load from a tar archive have a tag but no RepoDigest, causing the digest-only check to fail and falling through to a pull that also fails in air-gapped environments. Add Image.Tag() and fall back to tag-based matching so tar-loaded images are recognised as already present without attempting a pull. Co-Authored-By: Claude Sonnet 4.6 --- container.go | 10 ++++++---- images.go | 12 ++++++++++++ localproxy/infrastructure.go | 10 ++++++---- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/container.go b/container.go index c44bdd074..63536c35c 100644 --- a/container.go +++ b/container.go @@ -63,14 +63,16 @@ type docker struct { } func (c docker) PullImage(img Image) error { - stdout, stderr, err := c.agent.Run("docker image list --digests --format '{{.Repository}}@{{.Digest}}'") + stdout, stderr, err := c.agent.Run("docker image list --digests --format '{{.Repository}}:{{.Tag}} {{.Repository}}@{{.Digest}}'") if err != nil { return fmt.Errorf("%w, stdout: %s, stderr: %s", err, stdout, stderr) } - ref := img.Repository() + "@" + img.Digest() - for _, i := range strings.Split(string(stdout), "\n") { - if ref == i { + // Match by digest (pulled from registry) or by tag (loaded from tar, in this case the image has no digest). + tagRef := img.Repository() + ":" + img.Tag() + digestRef := img.Repository() + "@" + img.Digest() + for _, field := range strings.Fields(string(stdout)) { + if field == tagRef || field == digestRef { return nil } } diff --git a/images.go b/images.go index 958b8dc0a..8619bb6ac 100644 --- a/images.go +++ b/images.go @@ -24,6 +24,18 @@ func (i Image) Repository() string { return name } +// Tag returns the tag part of the image reference. +func (i Image) Tag() string { + name := string(i) + if idx := strings.Index(name, "@"); idx >= 0 { + name = name[:idx] + } + if idx := strings.LastIndex(name, ":"); idx >= 0 && !strings.Contains(name[idx:], "/") { + return name[idx+1:] + } + return "" +} + // Digest returns the digest part of the image reference (e.g. "sha256:..."). func (i Image) Digest() string { name := string(i) diff --git a/localproxy/infrastructure.go b/localproxy/infrastructure.go index ff05a2e28..38fec4f33 100644 --- a/localproxy/infrastructure.go +++ b/localproxy/infrastructure.go @@ -113,15 +113,17 @@ var _ cke.ContainerEngine = localDocker{} // PullImage pulls an image. func (l localDocker) PullImage(img cke.Image) error { - cmd := exec.Command("docker", "image", "list", "--digests", "--format={{.Repository}}@{{.Digest}}") + cmd := exec.Command("docker", "image", "list", "--digests", "--format={{.Repository}}:{{.Tag}} {{.Repository}}@{{.Digest}}") stdout, err := cmd.Output() if err != nil { return fmt.Errorf("failed to execute docker image list: %w", err) } - ref := img.Repository() + "@" + img.Digest() - for _, i := range strings.Fields(string(stdout)) { - if ref == i { + // Match by digest (pulled from registry) or by tag (loaded from tar, in this case the image has no digest). + tagRef := img.Repository() + ":" + img.Tag() + digestRef := img.Repository() + "@" + img.Digest() + for _, field := range strings.Fields(string(stdout)) { + if field == tagRef || field == digestRef { return nil } } From 5a0240f62fd3dd9f6c08fbfa333d1ca78a6fcb2c Mon Sep 17 00:00:00 2001 From: Masayuki Ishii Date: Thu, 18 Jun 2026 15:44:18 +0900 Subject: [PATCH 04/13] Use tag reference and --pull=never in docker run for air-gap compatibility docker run with a digest reference fails for images loaded via docker load because tar-loaded images have no RepoDigest. Since PullImage already verifies image presence (by digest or tag), docker run can safely use repo:tag with --pull=never to prevent unintended pulls. Add Image.TagRef() and Image.DigestRef() helpers and use them throughout. Co-Authored-By: Claude Sonnet 4.6 --- container.go | 16 ++++++++++------ images.go | 10 ++++++++++ localproxy/infrastructure.go | 20 ++++++++++++-------- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/container.go b/container.go index 63536c35c..b9978e446 100644 --- a/container.go +++ b/container.go @@ -69,8 +69,8 @@ func (c docker) PullImage(img Image) error { } // Match by digest (pulled from registry) or by tag (loaded from tar, in this case the image has no digest). - tagRef := img.Repository() + ":" + img.Tag() - digestRef := img.Repository() + "@" + img.Digest() + tagRef := img.TagRef() + digestRef := img.DigestRef() for _, field := range strings.Fields(string(stdout)) { if field == tagRef || field == digestRef { return nil @@ -89,6 +89,7 @@ func (c docker) Run(img Image, binds []Mount, command string, args ...string) er "docker", "run", "--log-driver=journald", + "--pull=never", "--rm", "--network=host", "--uts=host", @@ -101,7 +102,7 @@ func (c docker) Run(img Image, binds []Mount, command string, args ...string) er } runArgs = append(runArgs, fmt.Sprintf("--volume=%s:%s:%s", m.Source, m.Destination, o)) } - runArgs = append(runArgs, img.Name(), command) + runArgs = append(runArgs, img.TagRef(), command) runArgs = append(runArgs, args...) _, _, err := c.agent.Run(strings.Join(runArgs, " ")) @@ -113,6 +114,7 @@ func (c docker) RunWithInput(img Image, binds []Mount, command, input string, ar "docker", "run", "--log-driver=journald", + "--pull=never", "--rm", "-i", "--network=host", @@ -126,7 +128,7 @@ func (c docker) RunWithInput(img Image, binds []Mount, command, input string, ar } runArgs = append(runArgs, fmt.Sprintf("--volume=%s:%s:%s", m.Source, m.Destination, o)) } - runArgs = append(runArgs, img.Name(), command) + runArgs = append(runArgs, img.TagRef(), command) runArgs = append(runArgs, args...) return c.agent.RunWithInput(strings.Join(runArgs, " "), input) @@ -137,6 +139,7 @@ func (c docker) RunWithOutput(img Image, binds []Mount, command string, args ... "docker", "run", "--log-driver=journald", + "--pull=never", "--rm", "--network=host", "--uts=host", @@ -149,7 +152,7 @@ func (c docker) RunWithOutput(img Image, binds []Mount, command string, args ... } runArgs = append(runArgs, fmt.Sprintf("--volume=%s:%s:%s", m.Source, m.Destination, o)) } - runArgs = append(runArgs, img.Name(), command) + runArgs = append(runArgs, img.TagRef(), command) runArgs = append(runArgs, args...) stdout, stderr, err := c.agent.Run(strings.Join(runArgs, " ")) @@ -174,6 +177,7 @@ func (c docker) RunSystem(name string, img Image, opts []string, params, extra S "docker", "run", "--log-driver=journald", + "--pull=never", "-d", "--name=" + name, "--read-only", @@ -221,7 +225,7 @@ func (c docker) RunSystem(name string, img Image, opts []string, params, extra S } args = append(args, "--label-file="+labelFile) - args = append(args, img.Name()) + args = append(args, img.TagRef()) args = append(args, params.ExtraArguments...) args = append(args, extra.ExtraArguments...) diff --git a/images.go b/images.go index 8619bb6ac..e41bbd6a1 100644 --- a/images.go +++ b/images.go @@ -45,6 +45,16 @@ func (i Image) Digest() string { return "" } +// TagRef returns the repository:tag reference without the digest. +func (i Image) TagRef() string { + return i.Repository() + ":" + i.Tag() +} + +// DigestRef returns the repository@digest reference without the tag. +func (i Image) DigestRef() string { + return i.Repository() + "@" + i.Digest() +} + // AllImages return container images list used by CKE func AllImages() []string { return []string{ diff --git a/localproxy/infrastructure.go b/localproxy/infrastructure.go index 38fec4f33..bf65b83ed 100644 --- a/localproxy/infrastructure.go +++ b/localproxy/infrastructure.go @@ -120,8 +120,8 @@ func (l localDocker) PullImage(img cke.Image) error { } // Match by digest (pulled from registry) or by tag (loaded from tar, in this case the image has no digest). - tagRef := img.Repository() + ":" + img.Tag() - digestRef := img.Repository() + "@" + img.Digest() + tagRef := img.TagRef() + digestRef := img.DigestRef() for _, field := range strings.Fields(string(stdout)) { if field == tagRef || field == digestRef { return nil @@ -136,6 +136,7 @@ func (l localDocker) Run(img cke.Image, binds []cke.Mount, command string, args runArgs := []string{ "run", "--log-driver=journald", + "--pull=never", "--rm", "--network=host", "--uts=host", @@ -148,12 +149,12 @@ func (l localDocker) Run(img cke.Image, binds []cke.Mount, command string, args } runArgs = append(runArgs, fmt.Sprintf("--volume=%s:%s:%s", m.Source, m.Destination, o)) } - runArgs = append(runArgs, img.Name(), command) + runArgs = append(runArgs, img.TagRef(), command) runArgs = append(runArgs, args...) out, err := exec.Command("docker", runArgs...).CombinedOutput() if err != nil { - return fmt.Errorf("failed to run %s: %s: %w", img.Name(), out, err) + return fmt.Errorf("failed to run %s: %s: %w", img.TagRef(), out, err) } return nil } @@ -163,6 +164,7 @@ func (l localDocker) RunWithInput(img cke.Image, binds []cke.Mount, command, inp runArgs := []string{ "run", "--log-driver=journald", + "--pull=never", "--rm", "-i", "--network=host", @@ -176,7 +178,7 @@ func (l localDocker) RunWithInput(img cke.Image, binds []cke.Mount, command, inp } runArgs = append(runArgs, fmt.Sprintf("--volume=%s:%s:%s", m.Source, m.Destination, o)) } - runArgs = append(runArgs, img.Name(), command) + runArgs = append(runArgs, img.TagRef(), command) runArgs = append(runArgs, args...) cmd := exec.Command("docker", runArgs...) @@ -184,7 +186,7 @@ func (l localDocker) RunWithInput(img cke.Image, binds []cke.Mount, command, inp out, err := cmd.CombinedOutput() if err != nil { - return fmt.Errorf("failed to run %s: %s: %w", img.Name(), out, err) + return fmt.Errorf("failed to run %s: %s: %w", img.TagRef(), out, err) } return nil } @@ -194,6 +196,7 @@ func (l localDocker) RunWithOutput(img cke.Image, binds []cke.Mount, command str runArgs := []string{ "run", "--log-driver=journald", + "--pull=never", "--rm", "--network=host", "--uts=host", @@ -206,7 +209,7 @@ func (l localDocker) RunWithOutput(img cke.Image, binds []cke.Mount, command str } runArgs = append(runArgs, fmt.Sprintf("--volume=%s:%s:%s", m.Source, m.Destination, o)) } - runArgs = append(runArgs, img.Name(), command) + runArgs = append(runArgs, img.TagRef(), command) runArgs = append(runArgs, args...) stdout := new(bytes.Buffer) @@ -224,6 +227,7 @@ func (l localDocker) RunSystem(name string, img cke.Image, opts []string, params "run", "--rm", "--log-driver=journald", + "--pull=never", "-d", "--name=" + name, "--read-only", @@ -280,7 +284,7 @@ func (l localDocker) RunSystem(name string, img cke.Image, opts []string, params } args = append(args, "--label-file="+labelFile.Name()) - args = append(args, img.Name()) + args = append(args, img.TagRef()) args = append(args, params.ExtraArguments...) args = append(args, extra.ExtraArguments...) From cd10269df5b6caa41110006c16fc773631aacc52 Mon Sep 17 00:00:00 2001 From: Masayuki Ishii Date: Thu, 18 Jun 2026 16:18:38 +0900 Subject: [PATCH 05/13] Refactor Image type from string to struct with precomputed refs Replace `type Image string` with a struct holding fullRef, tagRef, and digestRef as precomputed fields, eliminating repeated string parsing and concatenation on every call. Add newImage(repository, tag, digest) constructor used by generated code. Rename Name() to FullRef() and align TagRef()/DigestRef() as the complete set of reference accessors. Update all call sites: ServiceStatus.Image comparisons use TagRef() to match the tag-based docker run, resource image annotations use FullRef() for digest-pinned references. Co-Authored-By: Claude Sonnet 4.6 --- container.go | 2 +- images.go | 67 ++++++++++++------------------------ images_gen.go | 16 ++++----- localproxy/infrastructure.go | 4 +-- localproxy/strategy.go | 4 +-- op/common/image_pull.go | 4 +-- pkg/update-images/main.go | 4 +-- server/node_filter.go | 16 ++++----- server/strategy_test.go | 20 +++++------ 9 files changed, 57 insertions(+), 80 deletions(-) diff --git a/container.go b/container.go index b9978e446..b7d40b6d0 100644 --- a/container.go +++ b/container.go @@ -77,7 +77,7 @@ func (c docker) PullImage(img Image) error { } } - stdout, stderr, err = c.agent.Run("docker image pull " + img.Name()) + stdout, stderr, err = c.agent.Run("docker image pull " + img.FullRef()) if err != nil { return fmt.Errorf("%w, stdout: %s, stderr: %s", err, stdout, stderr) } diff --git a/images.go b/images.go index e41bbd6a1..245486695 100644 --- a/images.go +++ b/images.go @@ -1,69 +1,46 @@ package cke -import "strings" - //go:generate go run ./pkg/update-images/ -// Image is the type of container images. -type Image string - -// Name returns the full image reference. -func (i Image) Name() string { - return string(i) -} - -// Repository returns the repository part of the image reference (without tag and digest). -func (i Image) Repository() string { - name := string(i) - if idx := strings.Index(name, "@"); idx >= 0 { - name = name[:idx] - } - if idx := strings.LastIndex(name, ":"); idx >= 0 && !strings.Contains(name[idx:], "/") { - name = name[:idx] - } - return name +// Image represents a container image reference. +type Image struct { + fullRef string + tagRef string + digestRef string } -// Tag returns the tag part of the image reference. -func (i Image) Tag() string { - name := string(i) - if idx := strings.Index(name, "@"); idx >= 0 { - name = name[:idx] +func newImage(repository, tag, digest string) Image { + return Image{ + fullRef: repository + ":" + tag + "@" + digest, + tagRef: repository + ":" + tag, + digestRef: repository + "@" + digest, } - if idx := strings.LastIndex(name, ":"); idx >= 0 && !strings.Contains(name[idx:], "/") { - return name[idx+1:] - } - return "" } -// Digest returns the digest part of the image reference (e.g. "sha256:..."). -func (i Image) Digest() string { - name := string(i) - if idx := strings.Index(name, "@"); idx >= 0 { - return name[idx+1:] - } - return "" +// FullRef returns the full image reference (repository:tag@digest). +func (i Image) FullRef() string { + return i.fullRef } // TagRef returns the repository:tag reference without the digest. func (i Image) TagRef() string { - return i.Repository() + ":" + i.Tag() + return i.tagRef } // DigestRef returns the repository@digest reference without the tag. func (i Image) DigestRef() string { - return i.Repository() + "@" + i.Digest() + return i.digestRef } // AllImages return container images list used by CKE func AllImages() []string { return []string{ - EtcdImage.Name(), - ToolsImage.Name(), - KubernetesImage.Name(), - PauseImage.Name(), - CoreDNSImage.Name(), - UnboundImage.Name(), - UnboundExporterImage.Name(), + EtcdImage.FullRef(), + ToolsImage.FullRef(), + KubernetesImage.FullRef(), + PauseImage.FullRef(), + CoreDNSImage.FullRef(), + UnboundImage.FullRef(), + UnboundExporterImage.FullRef(), } } diff --git a/images_gen.go b/images_gen.go index 653d7861a..5b859b42f 100644 --- a/images_gen.go +++ b/images_gen.go @@ -2,12 +2,12 @@ package cke -const ( - EtcdImage = Image("ghcr.io/cybozu/etcd:3.6.11.1@sha256:c2e3075893ca9264773d022c4789e401c6ff7346ffd4101a4d79f3bdf37f5637") - KubernetesImage = Image("ghcr.io/cybozu/kubernetes:1.35.5.1@sha256:1051a9fba32c5a095c3895cb1b215d55454ee662d53147dcba56fb1cbda431ea") - ToolsImage = Image("ghcr.io/cybozu-go/cke-tools:1.35.0@sha256:3f0365cf68834dc9e2e082ec901495cae41c8cfc5dcaa9c3a2062d4b50a66f82") - PauseImage = Image("ghcr.io/cybozu/pause:3.10.1.5@sha256:3565a5a085d941c10ef86aaf2bfdaabc8c0f8677d55b2371dce472eb79a79d08") - CoreDNSImage = Image("ghcr.io/cybozu/coredns:1.14.2.1@sha256:e29066d55b9455fdba973ceb6f77232677de15263cc1c78f63707935adc01a14") - UnboundImage = Image("ghcr.io/cybozu/unbound:1.25.1.1@sha256:b84ce87568ed11859891747650a46d8431bac6d1ddda38b542e4d16cd36e0b86") - UnboundExporterImage = Image("ghcr.io/cybozu/unbound_exporter:0.5.0.4@sha256:a7d46220f43bf2ae0c81bbcb0b68801de2d60e87df298db4691ea0dd39b25593") +var ( + EtcdImage = newImage("ghcr.io/cybozu/etcd", "3.6.11.1", "sha256:c2e3075893ca9264773d022c4789e401c6ff7346ffd4101a4d79f3bdf37f5637") + KubernetesImage = newImage("ghcr.io/cybozu/kubernetes", "1.35.5.1", "sha256:1051a9fba32c5a095c3895cb1b215d55454ee662d53147dcba56fb1cbda431ea") + ToolsImage = newImage("ghcr.io/cybozu-go/cke-tools", "1.35.0", "sha256:3f0365cf68834dc9e2e082ec901495cae41c8cfc5dcaa9c3a2062d4b50a66f82") + PauseImage = newImage("ghcr.io/cybozu/pause", "3.10.1.5", "sha256:3565a5a085d941c10ef86aaf2bfdaabc8c0f8677d55b2371dce472eb79a79d08") + CoreDNSImage = newImage("ghcr.io/cybozu/coredns", "1.14.2.1", "sha256:e29066d55b9455fdba973ceb6f77232677de15263cc1c78f63707935adc01a14") + UnboundImage = newImage("ghcr.io/cybozu/unbound", "1.25.1.1", "sha256:b84ce87568ed11859891747650a46d8431bac6d1ddda38b542e4d16cd36e0b86") + UnboundExporterImage = newImage("ghcr.io/cybozu/unbound_exporter", "0.5.0.4", "sha256:a7d46220f43bf2ae0c81bbcb0b68801de2d60e87df298db4691ea0dd39b25593") ) diff --git a/localproxy/infrastructure.go b/localproxy/infrastructure.go index bf65b83ed..0c34038dd 100644 --- a/localproxy/infrastructure.go +++ b/localproxy/infrastructure.go @@ -128,7 +128,7 @@ func (l localDocker) PullImage(img cke.Image) error { } } - return exec.Command("docker", "image", "pull", img.Name()).Run() + return exec.Command("docker", "image", "pull", img.FullRef()).Run() } // Run runs a container as a foreground process. @@ -291,7 +291,7 @@ func (l localDocker) RunSystem(name string, img cke.Image, opts []string, params out, err := exec.Command("docker", args...).CombinedOutput() if err != nil { - return fmt.Errorf("failed to docker run %s: %s: %w", img.Name(), out, err) + return fmt.Errorf("failed to docker run %s: %s: %w", img.TagRef(), out, err) } return nil } diff --git a/localproxy/strategy.go b/localproxy/strategy.go index f3a66cb57..6cf1319d2 100644 --- a/localproxy/strategy.go +++ b/localproxy/strategy.go @@ -26,7 +26,7 @@ func decideOps(c *cke.Cluster, currentAP string, st *status) (newAP string, ops if !st.proxyRunning { ops = append(ops, k8s.KubeProxyBootOp(ckeNodes, c.Name, apURL, c.Options.Proxy)) } else { - if newAP != currentAP || st.proxyImage != cke.KubernetesImage.Name() { + if newAP != currentAP || st.proxyImage != cke.KubernetesImage.TagRef() { ops = append(ops, k8s.KubeProxyRestartOp(ckeNodes, c.Name, apURL, c.Options.Proxy)) } } @@ -36,7 +36,7 @@ func decideOps(c *cke.Cluster, currentAP string, st *status) (newAP string, ops return } - if !bytes.Equal(st.unboundConf, st.desiredUnboundConf) || st.unboundImage != cke.UnboundImage.Name() { + if !bytes.Equal(st.unboundConf, st.desiredUnboundConf) || st.unboundImage != cke.UnboundImage.TagRef() { ops = append(ops, &unboundRestartOp{conf: st.desiredUnboundConf}) } diff --git a/op/common/image_pull.go b/op/common/image_pull.go index 900cd573f..5f49b3799 100644 --- a/op/common/image_pull.go +++ b/op/common/image_pull.go @@ -37,7 +37,7 @@ func (c imagePullCommand) Run(ctx context.Context, inf cke.Infrastructure, _ str } log.Warn("failed to pull image", map[string]interface{}{ - "image": c.img.Name(), + "image": c.img.FullRef(), log.FnError: err, }) select { @@ -56,6 +56,6 @@ func (c imagePullCommand) Run(ctx context.Context, inf cke.Infrastructure, _ str func (c imagePullCommand) Command() cke.Command { return cke.Command{ Name: "image-pull", - Target: c.img.Name(), + Target: c.img.FullRef(), } } diff --git a/pkg/update-images/main.go b/pkg/update-images/main.go index 993fcf038..528b13061 100644 --- a/pkg/update-images/main.go +++ b/pkg/update-images/main.go @@ -34,9 +34,9 @@ const tmpl = `// Code generated by pkg/update-images. DO NOT EDIT. package cke -const ( +var ( {{- range .}} - {{.VarName}} = Image("{{.Repository}}:{{.Tag}}@{{.Digest}}") + {{.VarName}} = newImage("{{.Repository}}", "{{.Tag}}", "{{.Digest}}") {{- end}} ) ` diff --git a/server/node_filter.go b/server/node_filter.go index fc4ad93e9..19c538e4e 100644 --- a/server/node_filter.go +++ b/server/node_filter.go @@ -89,7 +89,7 @@ func (nf *NodeFilter) RiversOutdated(targets []*cke.Node) (nodes []*cke.Node) { switch { case !st.Running: // stopped nodes are excluded - case cke.ToolsImage.Name() != st.Image: + case cke.ToolsImage.TagRef() != st.Image: fallthrough case !currentBuiltIn.Equal(st.BuiltInParams): fallthrough @@ -120,7 +120,7 @@ func (nf *NodeFilter) EtcdRiversOutdated(targets []*cke.Node) (nodes []*cke.Node switch { case !st.Running: // stopped nodes are excluded - case cke.ToolsImage.Name() != st.Image: + case cke.ToolsImage.TagRef() != st.Image: fallthrough case !currentBuiltIn.Equal(st.BuiltInParams): fallthrough @@ -303,7 +303,7 @@ func (nf *NodeFilter) EtcdOutdatedMembers() (nodes []*cke.Node) { } currentBuiltIn := etcd.BuiltInParams(n, []string{}, "new") switch { - case cke.EtcdImage.Name() != st.Image: + case cke.EtcdImage.TagRef() != st.Image: fallthrough case !etcdEqualParams(st.BuiltInParams, currentBuiltIn): fallthrough @@ -368,7 +368,7 @@ func (nf *NodeFilter) APIServerOutdated(targets []*cke.Node) (nodes []*cke.Node) switch { case !st.Running: // stopped nodes are excluded - case cke.KubernetesImage.Name() != st.Image: + case cke.KubernetesImage.TagRef() != st.Image: fallthrough case !currentBuiltIn.Equal(st.BuiltInParams): fallthrough @@ -399,7 +399,7 @@ func (nf *NodeFilter) ControllerManagerOutdated(targets []*cke.Node) (nodes []*c switch { case !st.Running: // stopped nodes are excluded - case cke.KubernetesImage.Name() != st.Image: + case cke.KubernetesImage.TagRef() != st.Image: fallthrough case !currentBuiltIn.Equal(st.BuiltInParams): fallthrough @@ -433,7 +433,7 @@ func (nf *NodeFilter) SchedulerOutdated(targets []*cke.Node, params cke.Schedule switch { case !st.Running: // stopped nodes are excluded - case cke.KubernetesImage.Name() != st.Image: + case cke.KubernetesImage.TagRef() != st.Image: fallthrough case !currentBuiltIn.Equal(st.BuiltInParams): fallthrough @@ -500,7 +500,7 @@ func (nf *NodeFilter) KubeletOutdated(targets []*cke.Node) (nodes []*cke.Node) { // stopped nodes are excluded case kubeletRuntimeChanged(st.BuiltInParams, currentBuiltIn): log.Warn("kubelet's container runtime cannot be changed", nil) - case cke.KubernetesImage.Name() != st.Image: + case cke.KubernetesImage.TagRef() != st.Image: fallthrough case !currentBuiltIn.Equal(st.BuiltInParams): fallthrough @@ -624,7 +624,7 @@ func (nf *NodeFilter) ProxyOutdated(targets []*cke.Node, params cke.ProxyParams) switch { case !st.Running: // stopped nodes are excluded - case cke.KubernetesImage.Name() != st.Image: + case cke.KubernetesImage.TagRef() != st.Image: fallthrough case !currentBuiltIn.Equal(st.BuiltInParams): fallthrough diff --git a/server/strategy_test.go b/server/strategy_test.go index bc2f08de8..9f6851987 100644 --- a/server/strategy_test.go +++ b/server/strategy_test.go @@ -191,7 +191,7 @@ func (d testData) withResources(res []cke.ResourceDefinition) testData { func (d testData) withRivers() testData { for _, v := range d.Status.NodeStatuses { v.Rivers.Running = true - v.Rivers.Image = cke.ToolsImage.Name() + v.Rivers.Image = cke.ToolsImage.TagRef() v.Rivers.BuiltInParams = op.RiversParams(d.ControlPlane(), op.RiversUpstreamPort, op.RiversListenPort) } return d @@ -201,7 +201,7 @@ func (d testData) withEtcdRivers() testData { for _, n := range d.ControlPlane() { st := &d.NodeStatus(n).EtcdRivers st.Running = true - st.Image = cke.ToolsImage.Name() + st.Image = cke.ToolsImage.TagRef() st.BuiltInParams = op.RiversParams(d.ControlPlane(), op.EtcdRiversUpstreamPort, op.EtcdRiversListenPort) } return d @@ -238,7 +238,7 @@ func (d testData) withUnhealthyEtcd() testData { for _, n := range d.ControlPlane() { st := &d.NodeStatus(n).Etcd st.Running = true - st.Image = cke.EtcdImage.Name() + st.Image = cke.EtcdImage.TagRef() st.BuiltInParams = etcd.BuiltInParams(n, nil, "") } return d @@ -265,7 +265,7 @@ func (d testData) withAPIServer(serviceSubnet, domain string) testData { st := &d.NodeStatus(n).APIServer st.Running = true st.IsHealthy = true - st.Image = cke.KubernetesImage.Name() + st.Image = cke.KubernetesImage.TagRef() st.BuiltInParams = k8s.APIServerParams(n.Address, serviceSubnet, false, "", "", domain) } return d @@ -282,7 +282,7 @@ func (d testData) withControllerManager(name, serviceSubnet string) testData { st := &d.NodeStatus(n).ControllerManager st.Running = true st.IsHealthy = true - st.Image = cke.KubernetesImage.Name() + st.Image = cke.KubernetesImage.TagRef() st.BuiltInParams = k8s.ControllerManagerParams(name, serviceSubnet) } return d @@ -293,7 +293,7 @@ func (d testData) withScheduler() testData { st := &d.NodeStatus(n).Scheduler st.Running = true st.IsHealthy = true - st.Image = cke.KubernetesImage.Name() + st.Image = cke.KubernetesImage.TagRef() st.BuiltInParams = k8s.SchedulerParams() st.Config = &schedulerv1.KubeSchedulerConfiguration{} @@ -309,7 +309,7 @@ func (d testData) withKubelet(domain, dns string, allowSwap bool) testData { st := &d.NodeStatus(n).Kubelet st.Running = true st.IsHealthy = true - st.Image = cke.KubernetesImage.Name() + st.Image = cke.KubernetesImage.TagRef() st.BuiltInParams = k8s.KubeletServiceParams(n, cke.KubeletParams{ CRIEndpoint: "/var/run/k8s-containerd.sock", }) @@ -371,7 +371,7 @@ func (d testData) withProxy() testData { st := &d.NodeStatus(n).Proxy st.Running = true st.IsHealthy = true - st.Image = cke.KubernetesImage.Name() + st.Image = cke.KubernetesImage.TagRef() st.BuiltInParams = k8s.ProxyParams() st.Config = &proxyv1alpha1.KubeProxyConfiguration{} st.Config.HostnameOverride = n.Nodename() @@ -550,9 +550,9 @@ func (d testData) withK8sResourceReady() testData { } ks.ResourceStatuses["ClusterRole/system:kube-apiserver-to-kubelet"].Annotations[cke.AnnotationResourceRevision] = "2" ks.ResourceStatuses["ClusterRole/system:cluster-dns"].Annotations[cke.AnnotationResourceRevision] = "2" - ks.ResourceStatuses["Deployment/kube-system/cluster-dns"].Annotations[cke.AnnotationResourceImage] = cke.CoreDNSImage.Name() + ks.ResourceStatuses["Deployment/kube-system/cluster-dns"].Annotations[cke.AnnotationResourceImage] = cke.CoreDNSImage.FullRef() ks.ResourceStatuses["Deployment/kube-system/cluster-dns"].Annotations[cke.AnnotationResourceRevision] = "5" - ks.ResourceStatuses["DaemonSet/kube-system/node-dns"].Annotations[cke.AnnotationResourceImage] = cke.UnboundImage.Name() + "," + cke.UnboundExporterImage.Name() + ks.ResourceStatuses["DaemonSet/kube-system/node-dns"].Annotations[cke.AnnotationResourceImage] = cke.UnboundImage.FullRef() + "," + cke.UnboundExporterImage.FullRef() ks.ResourceStatuses["DaemonSet/kube-system/node-dns"].Annotations[cke.AnnotationResourceRevision] = "4" ks.ClusterDNS.ConfigMap = clusterdns.ConfigMap(testDefaultDNSDomain, testDefaultDNSServers) ks.ClusterDNS.ClusterIP = testDefaultDNSAddr From 3cb70138cb1fd283a952c84dfd73f0ac4b72c638 Mon Sep 17 00:00:00 2001 From: Masayuki Ishii Date: Thu, 18 Jun 2026 16:31:03 +0900 Subject: [PATCH 06/13] Simplify PullImage and remove DigestRef MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change docker image list format to FullRef style (repo:tag@digest) so each output line can be compared directly against img.FullRef() or img.TagRef()+"@", removing the need to split lines into fields. This also correctly rejects images where the tag matches but the digest differs — the previous field-based approach would have accepted them. Remove DigestRef() method and digestRef field from Image as they are no longer used anywhere. Co-Authored-By: Claude Sonnet 4.6 --- container.go | 11 +++++------ images.go | 15 ++++----------- localproxy/infrastructure.go | 11 +++++------ 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/container.go b/container.go index b7d40b6d0..3db76513f 100644 --- a/container.go +++ b/container.go @@ -63,16 +63,15 @@ type docker struct { } func (c docker) PullImage(img Image) error { - stdout, stderr, err := c.agent.Run("docker image list --digests --format '{{.Repository}}:{{.Tag}} {{.Repository}}@{{.Digest}}'") + stdout, stderr, err := c.agent.Run("docker image list --digests --format '{{.Repository}}:{{.Tag}}@{{.Digest}}'") if err != nil { return fmt.Errorf("%w, stdout: %s, stderr: %s", err, stdout, stderr) } - // Match by digest (pulled from registry) or by tag (loaded from tar, in this case the image has no digest). - tagRef := img.TagRef() - digestRef := img.DigestRef() - for _, field := range strings.Fields(string(stdout)) { - if field == tagRef || field == digestRef { + noDigest := img.TagRef() + "@" + for _, line := range strings.Split(strings.TrimSpace(string(stdout)), "\n") { + // Accept if FullRef matches (registry pull) or image has no digest (docker load). + if line == img.FullRef() || line == noDigest { return nil } } diff --git a/images.go b/images.go index 245486695..d7e54950c 100644 --- a/images.go +++ b/images.go @@ -4,16 +4,14 @@ package cke // Image represents a container image reference. type Image struct { - fullRef string - tagRef string - digestRef string + fullRef string + tagRef string } func newImage(repository, tag, digest string) Image { return Image{ - fullRef: repository + ":" + tag + "@" + digest, - tagRef: repository + ":" + tag, - digestRef: repository + "@" + digest, + fullRef: repository + ":" + tag + "@" + digest, + tagRef: repository + ":" + tag, } } @@ -27,11 +25,6 @@ func (i Image) TagRef() string { return i.tagRef } -// DigestRef returns the repository@digest reference without the tag. -func (i Image) DigestRef() string { - return i.digestRef -} - // AllImages return container images list used by CKE func AllImages() []string { return []string{ diff --git a/localproxy/infrastructure.go b/localproxy/infrastructure.go index 0c34038dd..d24377b90 100644 --- a/localproxy/infrastructure.go +++ b/localproxy/infrastructure.go @@ -113,17 +113,16 @@ var _ cke.ContainerEngine = localDocker{} // PullImage pulls an image. func (l localDocker) PullImage(img cke.Image) error { - cmd := exec.Command("docker", "image", "list", "--digests", "--format={{.Repository}}:{{.Tag}} {{.Repository}}@{{.Digest}}") + cmd := exec.Command("docker", "image", "list", "--digests", "--format={{.Repository}}:{{.Tag}}@{{.Digest}}") stdout, err := cmd.Output() if err != nil { return fmt.Errorf("failed to execute docker image list: %w", err) } - // Match by digest (pulled from registry) or by tag (loaded from tar, in this case the image has no digest). - tagRef := img.TagRef() - digestRef := img.DigestRef() - for _, field := range strings.Fields(string(stdout)) { - if field == tagRef || field == digestRef { + noDigest := img.TagRef() + "@" + for _, line := range strings.Split(strings.TrimSpace(string(stdout)), "\n") { + // Accept if FullRef matches (registry pull) or image has no digest (docker load). + if line == img.FullRef() || line == noDigest { return nil } } From 67dbc1ad8bd3d790ba9487e52ae50129573465b9 Mon Sep 17 00:00:00 2001 From: Masayuki Ishii Date: Thu, 18 Jun 2026 16:41:03 +0900 Subject: [PATCH 07/13] docs: add image pull specification Co-Authored-By: Claude Sonnet 4.6 --- docs/image-pull.md | 51 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 docs/image-pull.md diff --git a/docs/image-pull.md b/docs/image-pull.md new file mode 100644 index 000000000..670151ec1 --- /dev/null +++ b/docs/image-pull.md @@ -0,0 +1,51 @@ +Image Pull Specification +======================== + +Image reference format +---------------------- + +CKE manages container images using digest-pinned references in the form: + +``` +repository:tag@sha256: +``` + +All image constants (e.g. `EtcdImage`, `KubernetesImage`) are defined in this format and provide three accessors: + +| Method | Returns | +|--------|---------| +| `FullRef()` | `repository:tag@sha256:` | +| `TagRef()` | `repository:tag` | + +PullImage behaviour +------------------- + +Before pulling an image, CKE checks whether a suitable image is already present on the node using `docker image list --format '{{.Repository}}:{{.Tag}}@{{.Digest}}'`. + +Each line of the output is compared against two conditions: + +1. **FullRef match** — the line equals `img.FullRef()` (e.g. `ghcr.io/cybozu/etcd:3.6.11.1@sha256:...`). + This is the normal case after an image has been pulled from a registry. + +2. **No-digest match** — the line equals `img.TagRef()+"@"` (e.g. `ghcr.io/cybozu/etcd:3.6.11.1@`). + This covers images loaded via `docker load` from a tar archive, which have a tag but no RepoDigest. + +If neither condition is met (including when the tag matches but the digest differs), the image is considered absent and `docker image pull ` is executed. + +Running containers +------------------ + +All `docker run` invocations use: + +- `--pull=never` — prevents Docker from attempting a pull at run time; the image must already be present from `PullImage`. +- `TagRef` as the image argument — works for both registry-pulled images (which have the tag) and `docker load` images (which lack a RepoDigest and cannot be addressed by digest). + +Air-gap environments +-------------------- + +In air-gapped environments, images are pre-loaded onto nodes via `docker load` from a tar archive. These images have a tag but no RepoDigest. + +CKE handles this as follows: + +1. `PullImage` detects the no-digest match and skips the pull. +2. `docker run` addresses the image by `TagRef`, which succeeds because the tag is present. From 8ffaf1d7215bc74c801923643f30a81d7a98bf1f Mon Sep 17 00:00:00 2001 From: Masayuki Ishii Date: Thu, 18 Jun 2026 16:44:55 +0900 Subject: [PATCH 08/13] docs: add link to image pull specification in design.md Co-Authored-By: Claude Sonnet 4.6 --- docs/design.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/design.md b/docs/design.md index c19b40657..57f96910b 100644 --- a/docs/design.md +++ b/docs/design.md @@ -85,6 +85,7 @@ Implementation policies * CKE does not install any tools onto node OS other than containers. * `kubelet` or other system services run by `docker run`. + * Images are identified by digest-pinned references (`repository:tag@sha256:...`). Before running a container, CKE pulls the image if not already present. See [image pull specification](image-pull.md) for details. * CKE employs CNI network plugins. From f70d40bbfab45bed162c651768e8492f2db760d5 Mon Sep 17 00:00:00 2001 From: Masayuki Ishii Date: Thu, 18 Jun 2026 17:13:26 +0900 Subject: [PATCH 09/13] Fix PullImage to tag image after pull and improve error messages Pulling by FullRef (repo:tag@sha256:...) causes Docker to store the image with a tag, making docker run by TagRef fail. Add a docker image tag step after pull to assign the tag. Prefix error messages with the failing command and its arguments so pull and tag failures can be distinguished in logs. Update image-pull.md to document the two-step pull+tag behaviour. Co-Authored-By: Claude Sonnet 4.6 --- container.go | 6 +++++- docs/image-pull.md | 5 ++++- localproxy/infrastructure.go | 8 +++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/container.go b/container.go index 3db76513f..de9fb5b1c 100644 --- a/container.go +++ b/container.go @@ -78,7 +78,11 @@ func (c docker) PullImage(img Image) error { stdout, stderr, err = c.agent.Run("docker image pull " + img.FullRef()) if err != nil { - return fmt.Errorf("%w, stdout: %s, stderr: %s", err, stdout, stderr) + return fmt.Errorf("docker image pull %s: %w, stdout: %s, stderr: %s", img.FullRef(), err, stdout, stderr) + } + stdout, stderr, err = c.agent.Run("docker image tag " + img.FullRef() + " " + img.TagRef()) + if err != nil { + return fmt.Errorf("docker image tag %s %s: %w, stdout: %s, stderr: %s", img.FullRef(), img.TagRef(), err, stdout, stderr) } return nil } diff --git a/docs/image-pull.md b/docs/image-pull.md index 670151ec1..65ce00973 100644 --- a/docs/image-pull.md +++ b/docs/image-pull.md @@ -30,7 +30,10 @@ Each line of the output is compared against two conditions: 2. **No-digest match** — the line equals `img.TagRef()+"@"` (e.g. `ghcr.io/cybozu/etcd:3.6.11.1@`). This covers images loaded via `docker load` from a tar archive, which have a tag but no RepoDigest. -If neither condition is met (including when the tag matches but the digest differs), the image is considered absent and `docker image pull ` is executed. +If neither condition is met (including when the tag matches but the digest differs), the image is considered absent and the following steps are executed: + +1. `docker image pull ` — pulls the image by digest. Docker stores it with `` as the tag. +2. `docker image tag ` — assigns the tag so the image can be addressed by `TagRef` in subsequent `docker run` calls. Running containers ------------------ diff --git a/localproxy/infrastructure.go b/localproxy/infrastructure.go index d24377b90..ff2b3100e 100644 --- a/localproxy/infrastructure.go +++ b/localproxy/infrastructure.go @@ -127,7 +127,13 @@ func (l localDocker) PullImage(img cke.Image) error { } } - return exec.Command("docker", "image", "pull", img.FullRef()).Run() + if err := exec.Command("docker", "image", "pull", img.FullRef()).Run(); err != nil { + return fmt.Errorf("docker image pull %s: %w", img.FullRef(), err) + } + if err := exec.Command("docker", "image", "tag", img.FullRef(), img.TagRef()).Run(); err != nil { + return fmt.Errorf("docker image tag %s %s: %w", img.FullRef(), img.TagRef(), err) + } + return nil } // Run runs a container as a foreground process. From 25fcbd48090a02d9666cc39cc63831aca2899076 Mon Sep 17 00:00:00 2001 From: Masayuki Ishii Date: Thu, 18 Jun 2026 18:44:42 +0900 Subject: [PATCH 10/13] Improve image management: add DigestRef, move AllImages to generated code, verify pull/tag events in mtest - Add DigestRef() (repo@sha256:digest) to Image type for digest-only pulls - Use DigestRef in PullImage instead of FullRef so docker pull does not set a tag - Move AllImages from images.go to images_gen.go as a var, auto-generated by update-images - mtest: split image check into two By() blocks; verify pull events use known DigestRef, tag events map each digest to the expected TagRef, and running containers use TagRef format Co-Authored-By: Claude Sonnet 4.6 --- container.go | 8 ++--- images.go | 25 ++++++--------- images_gen.go | 11 +++++++ localproxy/infrastructure.go | 8 ++--- mtest/operators_test.go | 58 +++++++++++++++++++++++++++++++++-- pkg/ckecli/cmd/images.go | 4 +-- pkg/compile_resources/main.go | 6 ++-- pkg/update-images/main.go | 7 +++++ 8 files changed, 96 insertions(+), 31 deletions(-) diff --git a/container.go b/container.go index de9fb5b1c..6fd0ad91c 100644 --- a/container.go +++ b/container.go @@ -76,13 +76,13 @@ func (c docker) PullImage(img Image) error { } } - stdout, stderr, err = c.agent.Run("docker image pull " + img.FullRef()) + stdout, stderr, err = c.agent.Run("docker image pull " + img.DigestRef()) if err != nil { - return fmt.Errorf("docker image pull %s: %w, stdout: %s, stderr: %s", img.FullRef(), err, stdout, stderr) + return fmt.Errorf("docker image pull %s: %w, stdout: %s, stderr: %s", img.DigestRef(), err, stdout, stderr) } - stdout, stderr, err = c.agent.Run("docker image tag " + img.FullRef() + " " + img.TagRef()) + stdout, stderr, err = c.agent.Run("docker image tag " + img.DigestRef() + " " + img.TagRef()) if err != nil { - return fmt.Errorf("docker image tag %s %s: %w, stdout: %s, stderr: %s", img.FullRef(), img.TagRef(), err, stdout, stderr) + return fmt.Errorf("docker image tag %s %s: %w, stdout: %s, stderr: %s", img.DigestRef(), img.TagRef(), err, stdout, stderr) } return nil } diff --git a/images.go b/images.go index d7e54950c..5a1423817 100644 --- a/images.go +++ b/images.go @@ -4,14 +4,16 @@ package cke // Image represents a container image reference. type Image struct { - fullRef string - tagRef string + fullRef string + tagRef string + digestRef string } func newImage(repository, tag, digest string) Image { return Image{ - fullRef: repository + ":" + tag + "@" + digest, - tagRef: repository + ":" + tag, + fullRef: repository + ":" + tag + "@" + digest, + tagRef: repository + ":" + tag, + digestRef: repository + "@" + digest, } } @@ -25,15 +27,8 @@ func (i Image) TagRef() string { return i.tagRef } -// AllImages return container images list used by CKE -func AllImages() []string { - return []string{ - EtcdImage.FullRef(), - ToolsImage.FullRef(), - KubernetesImage.FullRef(), - PauseImage.FullRef(), - CoreDNSImage.FullRef(), - UnboundImage.FullRef(), - UnboundExporterImage.FullRef(), - } +// DigestRef returns the repository@digest reference without the tag. +func (i Image) DigestRef() string { + return i.digestRef } + diff --git a/images_gen.go b/images_gen.go index 5b859b42f..4ee206fe6 100644 --- a/images_gen.go +++ b/images_gen.go @@ -11,3 +11,14 @@ var ( UnboundImage = newImage("ghcr.io/cybozu/unbound", "1.25.1.1", "sha256:b84ce87568ed11859891747650a46d8431bac6d1ddda38b542e4d16cd36e0b86") UnboundExporterImage = newImage("ghcr.io/cybozu/unbound_exporter", "0.5.0.4", "sha256:a7d46220f43bf2ae0c81bbcb0b68801de2d60e87df298db4691ea0dd39b25593") ) + +// AllImages is the list of all container images used by CKE. +var AllImages = []Image{ + EtcdImage, + KubernetesImage, + ToolsImage, + PauseImage, + CoreDNSImage, + UnboundImage, + UnboundExporterImage, +} diff --git a/localproxy/infrastructure.go b/localproxy/infrastructure.go index ff2b3100e..c4a26eeae 100644 --- a/localproxy/infrastructure.go +++ b/localproxy/infrastructure.go @@ -127,11 +127,11 @@ func (l localDocker) PullImage(img cke.Image) error { } } - if err := exec.Command("docker", "image", "pull", img.FullRef()).Run(); err != nil { - return fmt.Errorf("docker image pull %s: %w", img.FullRef(), err) + if err := exec.Command("docker", "image", "pull", img.DigestRef()).Run(); err != nil { + return fmt.Errorf("docker image pull %s: %w", img.DigestRef(), err) } - if err := exec.Command("docker", "image", "tag", img.FullRef(), img.TagRef()).Run(); err != nil { - return fmt.Errorf("docker image tag %s %s: %w", img.FullRef(), img.TagRef(), err) + if err := exec.Command("docker", "image", "tag", img.DigestRef(), img.TagRef()).Run(); err != nil { + return fmt.Errorf("docker image tag %s %s: %w", img.DigestRef(), img.TagRef(), err) } return nil } diff --git a/mtest/operators_test.go b/mtest/operators_test.go index 11ad6e10d..39b7f3252 100644 --- a/mtest/operators_test.go +++ b/mtest/operators_test.go @@ -90,7 +90,57 @@ func testOperators() { }, )) - By("Checking container images") + By("Checking images are tagged with expected digest") + fullRefs := make([]string, 0, len(cke.AllImages)) + digestRefs := make([]string, 0, len(cke.AllImages)) + expectedTags := make(map[string]string, len(cke.AllImages)) // digest -> tagRef + for _, img := range cke.AllImages { + fullRefs = append(fullRefs, img.FullRef()) + digestRefs = append(digestRefs, img.DigestRef()) + digest := strings.SplitN(img.DigestRef(), "@", 2)[1] + expectedTags[digest] = img.TagRef() + } + for _, n := range []string{node1, node2, node3, node4, node5} { + out := execSafeAt(n, "docker", "image", "list", "--digests", "--format={{.Repository}}:{{.Tag}}@{{.Digest}}") + images := strings.Split(strings.TrimSpace(string(out)), "\n") + Expect(images).To(ContainElements(fullRefs), + "node %s: some CKE images are missing or not tagged with expected digest", n) + + // All image pull events should reference a CKE-defined image digest + out = execSafeAt(n, "docker", "system", "events", + "--since", "2020-01-01T00:00:00Z", + "--until", time.Now().UTC().Format(time.RFC3339), + "--filter", "type=image", + "--filter", "event=pull", + "--format", "{{.Actor.ID}}") // @ + for _, pulled := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if pulled == "" { + continue + } + Expect(pulled).To(BeElementOf(digestRefs), + "node %s: unexpected image pull: %s", n, pulled) + } + + // All CKE images should have been tagged from the correct digest + out = execSafeAt(n, "docker", "system", "events", + "--since", "2020-01-01T00:00:00Z", + "--until", time.Now().UTC().Format(time.RFC3339), + "--filter", "type=image", + "--filter", "event=tag", + "--format", "{{.Actor.ID}} {{.Actor.Attributes.name}}") // + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if line == "" { + continue + } + parts := strings.SplitN(line, " ", 2) + Expect(parts).To(HaveLen(2)) + digest := parts[0] + actualTag := parts[1] + Expect(actualTag).To(Equal(expectedTags[digest])) + } + } + + By("Checking container image references and pull policy") for _, n := range []string{node1, node2, node3, node4, node5} { out := execSafeAt(n, "docker", "ps", "-aq") containerIDs := strings.Fields(strings.TrimSpace(string(out))) @@ -109,8 +159,10 @@ func testOperators() { for _, inspect := range inspects { fmt.Fprintf(GinkgoWriter, "node=%s container=%s image=%s\n", n, inspect.Name, inspect.Config.Image) - Expect(inspect.Config.Image).To(ContainSubstring("@sha256:"), - "container %s on node %s uses non-digest image: %s", inspect.Name, n, inspect.Config.Image) + + // Container's image should be TagRef format (repo:tag, no @sha256: digest) + Expect(inspect.Config.Image).NotTo(ContainSubstring("@"), + "container %s on node %s: image should be TagRef format (no digest): %s", inspect.Name, n, inspect.Config.Image) } } diff --git a/pkg/ckecli/cmd/images.go b/pkg/ckecli/cmd/images.go index e7832e612..fb7954ed5 100644 --- a/pkg/ckecli/cmd/images.go +++ b/pkg/ckecli/cmd/images.go @@ -16,8 +16,8 @@ var imagesCmd = &cobra.Command{ // Override rootCmd.PersistentPreRunE. PersistentPreRun: func(cmd *cobra.Command, args []string) {}, Run: func(cmd *cobra.Command, args []string) { - for _, img := range cke.AllImages() { - fmt.Println(img) + for _, img := range cke.AllImages { + fmt.Println(img.FullRef()) } }, } diff --git a/pkg/compile_resources/main.go b/pkg/compile_resources/main.go index 88e2da33c..8d9fbd0bc 100644 --- a/pkg/compile_resources/main.go +++ b/pkg/compile_resources/main.go @@ -33,9 +33,9 @@ func subMain() error { } images := make(map[string]string) - for _, img := range cke.AllImages() { - id := strings.SplitN(path.Base(img), ":", 2)[0] - images[id] = img + for _, img := range cke.AllImages { + id := strings.SplitN(path.Base(img.FullRef()), ":", 2)[0] + images[id] = img.FullRef() } var allResources []cke.ResourceDefinition diff --git a/pkg/update-images/main.go b/pkg/update-images/main.go index 528b13061..9fc50ea85 100644 --- a/pkg/update-images/main.go +++ b/pkg/update-images/main.go @@ -39,6 +39,13 @@ var ( {{.VarName}} = newImage("{{.Repository}}", "{{.Tag}}", "{{.Digest}}") {{- end}} ) + +// AllImages is the list of all container images used by CKE. +var AllImages = []Image{ +{{- range .}} + {{.VarName}}, +{{- end}} +} ` type packageVersion struct { From c3470a600db73b51b64751aac34ac0a1e5ac1dcf Mon Sep 17 00:00:00 2001 From: Masayuki Ishii Date: Thu, 18 Jun 2026 18:47:17 +0900 Subject: [PATCH 11/13] mtest: fix localproxy_test to expect TagRef format in container image Running containers use TagRef (repo:tag, no digest) due to --pull=never, consistent with operators_test.go. Co-Authored-By: Claude Sonnet 4.6 --- mtest/localproxy_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mtest/localproxy_test.go b/mtest/localproxy_test.go index d496c1bda..37231450f 100644 --- a/mtest/localproxy_test.go +++ b/mtest/localproxy_test.go @@ -58,8 +58,8 @@ func testLocalProxy() { for _, inspect := range inspects { fmt.Fprintf(GinkgoWriter, "container=%s image=%s\n", inspect.Name, inspect.Config.Image) - Expect(inspect.Config.Image).To(ContainSubstring("@sha256:"), - "container %s uses non-digest image: %s", inspect.Name, inspect.Config.Image) + Expect(inspect.Config.Image).NotTo(ContainSubstring("@"), + "container %s: image should be TagRef format (no digest): %s", inspect.Name, inspect.Config.Image) } }) } From 803c1eb421d226b3f8105787df515e3b4e4de46f Mon Sep 17 00:00:00 2001 From: Masayuki Ishii Date: Thu, 18 Jun 2026 18:47:43 +0900 Subject: [PATCH 12/13] fix --- mtest/operators_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mtest/operators_test.go b/mtest/operators_test.go index 39b7f3252..8d1fc2906 100644 --- a/mtest/operators_test.go +++ b/mtest/operators_test.go @@ -140,7 +140,7 @@ func testOperators() { } } - By("Checking container image references and pull policy") + By("Checking container image references") for _, n := range []string{node1, node2, node3, node4, node5} { out := execSafeAt(n, "docker", "ps", "-aq") containerIDs := strings.Fields(strings.TrimSpace(string(out))) From f79029d449e54546afac3c8452252b4f0da20952 Mon Sep 17 00:00:00 2001 From: Masayuki Ishii Date: Thu, 18 Jun 2026 19:02:30 +0900 Subject: [PATCH 13/13] fix --- mtest/operators_test.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mtest/operators_test.go b/mtest/operators_test.go index 8d1fc2906..cf5976058 100644 --- a/mtest/operators_test.go +++ b/mtest/operators_test.go @@ -102,9 +102,13 @@ func testOperators() { } for _, n := range []string{node1, node2, node3, node4, node5} { out := execSafeAt(n, "docker", "image", "list", "--digests", "--format={{.Repository}}:{{.Tag}}@{{.Digest}}") - images := strings.Split(strings.TrimSpace(string(out)), "\n") - Expect(images).To(ContainElements(fullRefs), - "node %s: some CKE images are missing or not tagged with expected digest", n) + for _, image := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if image == "" { + continue + } + Expect(image).To(BeElementOf(fullRefs), + "node %s: image %s is not a known CKE image", n, image) + } // All image pull events should reference a CKE-defined image digest out = execSafeAt(n, "docker", "system", "events",