diff --git a/DEVELOP.md b/DEVELOP.md index c8b761dd4..7ad79e8bf 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -37,6 +37,25 @@ $ go get -d k8s.io/client-go@${VERSION} k8s.io/api@${VERSION} k8s.io/apimachiner k8s.io/kube-proxy@${VERSION} ``` +### Update container image digests + +`images_gen.go` contains the container image references (tag and digest) used by CKE. +It is generated automatically by fetching the latest version of each image from the GitHub Packages API. + +Prerequisites: the `gh` CLI must be installed and authenticated with the `read:packages` scope. + +```console +$ gh auth login -s "read:packages" +``` + +Then run: + +```console +$ make images +``` + +This fetches the latest tagged version of each image and rewrites `images_gen.go`. + ### Update the Kubernetes resource definitions embedded in CKE The Kubernetes resource definitions embedded in CKE is defined in `./static/resource.go`. diff --git a/Makefile b/Makefile index b7427a981..5e1c61abc 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,11 @@ test: test-tools install: go install ./pkg/... +.PHONY: images +images: + go generate ./ + $(MAKE) static + .PHONY: static static: goimports go generate ./static diff --git a/container.go b/container.go index d02928734..6fd0ad91c 100644 --- a/container.go +++ b/container.go @@ -63,20 +63,26 @@ type docker struct { } func (c docker) PullImage(img Image) error { - stdout, stderr, err := c.agent.Run("docker image list --format '{{.Repository}}:{{.Tag}}'") + stdout, stderr, err := c.agent.Run("docker image list --digests --format '{{.Repository}}:{{.Tag}}@{{.Digest}}'") if err != nil { return fmt.Errorf("%w, stdout: %s, stderr: %s", err, stdout, stderr) } - for _, i := range strings.Split(string(stdout), "\n") { - if img.Name() == i { + noDigest := img.TagRef() + "@" + for _, line := range strings.Split(strings.TrimSpace(string(stdout)), "\n") { + // Accept if FullRef matches (registry pull) or image has no digest (docker load). + if line == img.FullRef() || line == noDigest { return nil } } - stdout, stderr, err = c.agent.Run("docker image pull " + img.Name()) + stdout, stderr, err = c.agent.Run("docker image pull " + img.DigestRef()) if err != nil { - return fmt.Errorf("%w, stdout: %s, stderr: %s", err, stdout, stderr) + return fmt.Errorf("docker image pull %s: %w, stdout: %s, stderr: %s", img.DigestRef(), err, stdout, stderr) + } + stdout, stderr, err = c.agent.Run("docker image tag " + img.DigestRef() + " " + img.TagRef()) + if err != nil { + return fmt.Errorf("docker image tag %s %s: %w, stdout: %s, stderr: %s", img.DigestRef(), img.TagRef(), err, stdout, stderr) } return nil } @@ -86,6 +92,7 @@ func (c docker) Run(img Image, binds []Mount, command string, args ...string) er "docker", "run", "--log-driver=journald", + "--pull=never", "--rm", "--network=host", "--uts=host", @@ -98,7 +105,7 @@ func (c docker) Run(img Image, binds []Mount, command string, args ...string) er } runArgs = append(runArgs, fmt.Sprintf("--volume=%s:%s:%s", m.Source, m.Destination, o)) } - runArgs = append(runArgs, img.Name(), command) + runArgs = append(runArgs, img.TagRef(), command) runArgs = append(runArgs, args...) _, _, err := c.agent.Run(strings.Join(runArgs, " ")) @@ -110,6 +117,7 @@ func (c docker) RunWithInput(img Image, binds []Mount, command, input string, ar "docker", "run", "--log-driver=journald", + "--pull=never", "--rm", "-i", "--network=host", @@ -123,7 +131,7 @@ func (c docker) RunWithInput(img Image, binds []Mount, command, input string, ar } runArgs = append(runArgs, fmt.Sprintf("--volume=%s:%s:%s", m.Source, m.Destination, o)) } - runArgs = append(runArgs, img.Name(), command) + runArgs = append(runArgs, img.TagRef(), command) runArgs = append(runArgs, args...) return c.agent.RunWithInput(strings.Join(runArgs, " "), input) @@ -134,6 +142,7 @@ func (c docker) RunWithOutput(img Image, binds []Mount, command string, args ... "docker", "run", "--log-driver=journald", + "--pull=never", "--rm", "--network=host", "--uts=host", @@ -146,7 +155,7 @@ func (c docker) RunWithOutput(img Image, binds []Mount, command string, args ... } runArgs = append(runArgs, fmt.Sprintf("--volume=%s:%s:%s", m.Source, m.Destination, o)) } - runArgs = append(runArgs, img.Name(), command) + runArgs = append(runArgs, img.TagRef(), command) runArgs = append(runArgs, args...) stdout, stderr, err := c.agent.Run(strings.Join(runArgs, " ")) @@ -171,6 +180,7 @@ func (c docker) RunSystem(name string, img Image, opts []string, params, extra S "docker", "run", "--log-driver=journald", + "--pull=never", "-d", "--name=" + name, "--read-only", @@ -218,7 +228,7 @@ func (c docker) RunSystem(name string, img Image, opts []string, params, extra S } args = append(args, "--label-file="+labelFile) - args = append(args, img.Name()) + args = append(args, img.TagRef()) args = append(args, params.ExtraArguments...) args = append(args, extra.ExtraArguments...) diff --git a/docs/design.md b/docs/design.md index c19b40657..57f96910b 100644 --- a/docs/design.md +++ b/docs/design.md @@ -85,6 +85,7 @@ Implementation policies * CKE does not install any tools onto node OS other than containers. * `kubelet` or other system services run by `docker run`. + * Images are identified by digest-pinned references (`repository:tag@sha256:...`). Before running a container, CKE pulls the image if not already present. See [image pull specification](image-pull.md) for details. * CKE employs CNI network plugins. diff --git a/docs/image-pull.md b/docs/image-pull.md new file mode 100644 index 000000000..65ce00973 --- /dev/null +++ b/docs/image-pull.md @@ -0,0 +1,54 @@ +Image Pull Specification +======================== + +Image reference format +---------------------- + +CKE manages container images using digest-pinned references in the form: + +``` +repository:tag@sha256: +``` + +All image constants (e.g. `EtcdImage`, `KubernetesImage`) are defined in this format and provide three accessors: + +| Method | Returns | +|--------|---------| +| `FullRef()` | `repository:tag@sha256:` | +| `TagRef()` | `repository:tag` | + +PullImage behaviour +------------------- + +Before pulling an image, CKE checks whether a suitable image is already present on the node using `docker image list --format '{{.Repository}}:{{.Tag}}@{{.Digest}}'`. + +Each line of the output is compared against two conditions: + +1. **FullRef match** — the line equals `img.FullRef()` (e.g. `ghcr.io/cybozu/etcd:3.6.11.1@sha256:...`). + This is the normal case after an image has been pulled from a registry. + +2. **No-digest match** — the line equals `img.TagRef()+"@"` (e.g. `ghcr.io/cybozu/etcd:3.6.11.1@`). + This covers images loaded via `docker load` from a tar archive, which have a tag but no RepoDigest. + +If neither condition is met (including when the tag matches but the digest differs), the image is considered absent and the following steps are executed: + +1. `docker image pull ` — pulls the image by digest. Docker stores it with `` as the tag. +2. `docker image tag ` — assigns the tag so the image can be addressed by `TagRef` in subsequent `docker run` calls. + +Running containers +------------------ + +All `docker run` invocations use: + +- `--pull=never` — prevents Docker from attempting a pull at run time; the image must already be present from `PullImage`. +- `TagRef` as the image argument — works for both registry-pulled images (which have the tag) and `docker load` images (which lack a RepoDigest and cannot be addressed by digest). + +Air-gap environments +-------------------- + +In air-gapped environments, images are pre-loaded onto nodes via `docker load` from a tar archive. These images have a tag but no RepoDigest. + +CKE handles this as follows: + +1. `PullImage` detects the no-digest match and skips the pull. +2. `docker run` addresses the image by `TagRef`, which succeeds because the tag is present. diff --git a/images.go b/images.go index facdb7507..5a1423817 100644 --- a/images.go +++ b/images.go @@ -1,33 +1,34 @@ package cke -// Image is the type of container images. -type Image string +//go:generate go run ./pkg/update-images/ -// Name returns docker image name. -func (i Image) Name() string { - return string(i) +// Image represents a container image reference. +type Image struct { + fullRef string + tagRef string + digestRef string } -// Container image definitions -const ( - EtcdImage = Image("ghcr.io/cybozu/etcd:3.6.11.1") - KubernetesImage = Image("ghcr.io/cybozu/kubernetes:1.35.5.1") - ToolsImage = Image("ghcr.io/cybozu-go/cke-tools:1.35.0") - PauseImage = Image("ghcr.io/cybozu/pause:3.10.1.5") - CoreDNSImage = Image("ghcr.io/cybozu/coredns:1.14.2.1") - UnboundImage = Image("ghcr.io/cybozu/unbound:1.25.1.1") - UnboundExporterImage = Image("ghcr.io/cybozu/unbound_exporter:0.5.0.4") -) - -// AllImages return container images list used by CKE -func AllImages() []string { - return []string{ - EtcdImage.Name(), - ToolsImage.Name(), - KubernetesImage.Name(), - PauseImage.Name(), - CoreDNSImage.Name(), - UnboundImage.Name(), - UnboundExporterImage.Name(), +func newImage(repository, tag, digest string) Image { + return Image{ + fullRef: repository + ":" + tag + "@" + digest, + tagRef: repository + ":" + tag, + digestRef: repository + "@" + digest, } } + +// FullRef returns the full image reference (repository:tag@digest). +func (i Image) FullRef() string { + return i.fullRef +} + +// TagRef returns the repository:tag reference without the digest. +func (i Image) TagRef() string { + return i.tagRef +} + +// DigestRef returns the repository@digest reference without the tag. +func (i Image) DigestRef() string { + return i.digestRef +} + diff --git a/images_gen.go b/images_gen.go new file mode 100644 index 000000000..4ee206fe6 --- /dev/null +++ b/images_gen.go @@ -0,0 +1,24 @@ +// Code generated by pkg/update-images. DO NOT EDIT. + +package cke + +var ( + EtcdImage = newImage("ghcr.io/cybozu/etcd", "3.6.11.1", "sha256:c2e3075893ca9264773d022c4789e401c6ff7346ffd4101a4d79f3bdf37f5637") + KubernetesImage = newImage("ghcr.io/cybozu/kubernetes", "1.35.5.1", "sha256:1051a9fba32c5a095c3895cb1b215d55454ee662d53147dcba56fb1cbda431ea") + ToolsImage = newImage("ghcr.io/cybozu-go/cke-tools", "1.35.0", "sha256:3f0365cf68834dc9e2e082ec901495cae41c8cfc5dcaa9c3a2062d4b50a66f82") + PauseImage = newImage("ghcr.io/cybozu/pause", "3.10.1.5", "sha256:3565a5a085d941c10ef86aaf2bfdaabc8c0f8677d55b2371dce472eb79a79d08") + CoreDNSImage = newImage("ghcr.io/cybozu/coredns", "1.14.2.1", "sha256:e29066d55b9455fdba973ceb6f77232677de15263cc1c78f63707935adc01a14") + UnboundImage = newImage("ghcr.io/cybozu/unbound", "1.25.1.1", "sha256:b84ce87568ed11859891747650a46d8431bac6d1ddda38b542e4d16cd36e0b86") + UnboundExporterImage = newImage("ghcr.io/cybozu/unbound_exporter", "0.5.0.4", "sha256:a7d46220f43bf2ae0c81bbcb0b68801de2d60e87df298db4691ea0dd39b25593") +) + +// AllImages is the list of all container images used by CKE. +var AllImages = []Image{ + EtcdImage, + KubernetesImage, + ToolsImage, + PauseImage, + CoreDNSImage, + UnboundImage, + UnboundExporterImage, +} diff --git a/localproxy/infrastructure.go b/localproxy/infrastructure.go index 23f583686..c4a26eeae 100644 --- a/localproxy/infrastructure.go +++ b/localproxy/infrastructure.go @@ -113,19 +113,27 @@ var _ cke.ContainerEngine = localDocker{} // PullImage pulls an image. func (l localDocker) PullImage(img cke.Image) error { - cmd := exec.Command("docker", "image", "list", "--format={{.Repository}}:{{.Tag}}") + cmd := exec.Command("docker", "image", "list", "--digests", "--format={{.Repository}}:{{.Tag}}@{{.Digest}}") stdout, err := cmd.Output() if err != nil { return fmt.Errorf("failed to execute docker image list: %w", err) } - for _, i := range strings.Fields(string(stdout)) { - if img.Name() == i { + noDigest := img.TagRef() + "@" + for _, line := range strings.Split(strings.TrimSpace(string(stdout)), "\n") { + // Accept if FullRef matches (registry pull) or image has no digest (docker load). + if line == img.FullRef() || line == noDigest { return nil } } - return exec.Command("docker", "image", "pull", img.Name()).Run() + if err := exec.Command("docker", "image", "pull", img.DigestRef()).Run(); err != nil { + return fmt.Errorf("docker image pull %s: %w", img.DigestRef(), err) + } + if err := exec.Command("docker", "image", "tag", img.DigestRef(), img.TagRef()).Run(); err != nil { + return fmt.Errorf("docker image tag %s %s: %w", img.DigestRef(), img.TagRef(), err) + } + return nil } // Run runs a container as a foreground process. @@ -133,6 +141,7 @@ func (l localDocker) Run(img cke.Image, binds []cke.Mount, command string, args runArgs := []string{ "run", "--log-driver=journald", + "--pull=never", "--rm", "--network=host", "--uts=host", @@ -145,12 +154,12 @@ func (l localDocker) Run(img cke.Image, binds []cke.Mount, command string, args } runArgs = append(runArgs, fmt.Sprintf("--volume=%s:%s:%s", m.Source, m.Destination, o)) } - runArgs = append(runArgs, img.Name(), command) + runArgs = append(runArgs, img.TagRef(), command) runArgs = append(runArgs, args...) out, err := exec.Command("docker", runArgs...).CombinedOutput() if err != nil { - return fmt.Errorf("failed to run %s: %s: %w", img.Name(), out, err) + return fmt.Errorf("failed to run %s: %s: %w", img.TagRef(), out, err) } return nil } @@ -160,6 +169,7 @@ func (l localDocker) RunWithInput(img cke.Image, binds []cke.Mount, command, inp runArgs := []string{ "run", "--log-driver=journald", + "--pull=never", "--rm", "-i", "--network=host", @@ -173,7 +183,7 @@ func (l localDocker) RunWithInput(img cke.Image, binds []cke.Mount, command, inp } runArgs = append(runArgs, fmt.Sprintf("--volume=%s:%s:%s", m.Source, m.Destination, o)) } - runArgs = append(runArgs, img.Name(), command) + runArgs = append(runArgs, img.TagRef(), command) runArgs = append(runArgs, args...) cmd := exec.Command("docker", runArgs...) @@ -181,7 +191,7 @@ func (l localDocker) RunWithInput(img cke.Image, binds []cke.Mount, command, inp out, err := cmd.CombinedOutput() if err != nil { - return fmt.Errorf("failed to run %s: %s: %w", img.Name(), out, err) + return fmt.Errorf("failed to run %s: %s: %w", img.TagRef(), out, err) } return nil } @@ -191,6 +201,7 @@ func (l localDocker) RunWithOutput(img cke.Image, binds []cke.Mount, command str runArgs := []string{ "run", "--log-driver=journald", + "--pull=never", "--rm", "--network=host", "--uts=host", @@ -203,7 +214,7 @@ func (l localDocker) RunWithOutput(img cke.Image, binds []cke.Mount, command str } runArgs = append(runArgs, fmt.Sprintf("--volume=%s:%s:%s", m.Source, m.Destination, o)) } - runArgs = append(runArgs, img.Name(), command) + runArgs = append(runArgs, img.TagRef(), command) runArgs = append(runArgs, args...) stdout := new(bytes.Buffer) @@ -221,6 +232,7 @@ func (l localDocker) RunSystem(name string, img cke.Image, opts []string, params "run", "--rm", "--log-driver=journald", + "--pull=never", "-d", "--name=" + name, "--read-only", @@ -277,14 +289,14 @@ func (l localDocker) RunSystem(name string, img cke.Image, opts []string, params } args = append(args, "--label-file="+labelFile.Name()) - args = append(args, img.Name()) + args = append(args, img.TagRef()) args = append(args, params.ExtraArguments...) args = append(args, extra.ExtraArguments...) out, err := exec.Command("docker", args...).CombinedOutput() if err != nil { - return fmt.Errorf("failed to docker run %s: %s: %w", img.Name(), out, err) + return fmt.Errorf("failed to docker run %s: %s: %w", img.TagRef(), out, err) } return nil } diff --git a/localproxy/status.go b/localproxy/status.go index 96918a745..ba7ebee1f 100644 --- a/localproxy/status.go +++ b/localproxy/status.go @@ -33,22 +33,16 @@ var dialer = &net.Dialer{ } func isRunning(name string) (bool, string, error) { - stdout, err := exec.Command("docker", "ps", "--format={{.Names}} {{.Image}}").Output() + out, err := exec.Command("docker", "container", "inspect", "--format={{.State.Running}} {{.Config.Image}}", name).Output() if err != nil { - return false, "", fmt.Errorf("failed to run docker ps: %w", err) + // Container does not exist + return false, "", nil } - - for _, line := range strings.Split(string(stdout), "\n") { - fields := strings.Fields(line) - if len(fields) != 2 { - continue - } - if fields[0] != name { - continue - } - return true, fields[1], nil + fields := strings.Fields(string(out)) + if len(fields) != 2 { + return false, "", fmt.Errorf("unexpected docker inspect output: %s", out) } - return false, "", nil + return fields[0] == "true", fields[1], nil } func getStatus(ctx context.Context, inf cke.Infrastructure) (*status, error) { diff --git a/localproxy/strategy.go b/localproxy/strategy.go index f3a66cb57..6cf1319d2 100644 --- a/localproxy/strategy.go +++ b/localproxy/strategy.go @@ -26,7 +26,7 @@ func decideOps(c *cke.Cluster, currentAP string, st *status) (newAP string, ops if !st.proxyRunning { ops = append(ops, k8s.KubeProxyBootOp(ckeNodes, c.Name, apURL, c.Options.Proxy)) } else { - if newAP != currentAP || st.proxyImage != cke.KubernetesImage.Name() { + if newAP != currentAP || st.proxyImage != cke.KubernetesImage.TagRef() { ops = append(ops, k8s.KubeProxyRestartOp(ckeNodes, c.Name, apURL, c.Options.Proxy)) } } @@ -36,7 +36,7 @@ func decideOps(c *cke.Cluster, currentAP string, st *status) (newAP string, ops return } - if !bytes.Equal(st.unboundConf, st.desiredUnboundConf) || st.unboundImage != cke.UnboundImage.Name() { + if !bytes.Equal(st.unboundConf, st.desiredUnboundConf) || st.unboundImage != cke.UnboundImage.TagRef() { ops = append(ops, &unboundRestartOp{conf: st.desiredUnboundConf}) } diff --git a/mtest/localproxy_test.go b/mtest/localproxy_test.go index 347bc7fb2..37231450f 100644 --- a/mtest/localproxy_test.go +++ b/mtest/localproxy_test.go @@ -2,6 +2,7 @@ package mtest import ( "bytes" + "encoding/json" "errors" "fmt" @@ -42,4 +43,23 @@ func testLocalProxy() { return errors.New("cke-unbound service is not running") }, 5, 0.1).Should(Succeed()) }) + + It("should use digest-pinned images", func() { + out := execSafeAt(host1, "docker", "inspect", "kube-proxy", "cke-unbound") + var inspects []struct { + Config struct { + Image string `json:"Image"` + } `json:"Config"` + Name string `json:"Name"` + } + err := json.Unmarshal(out, &inspects) + Expect(err).NotTo(HaveOccurred()) + Expect(inspects).To(HaveLen(2)) + + for _, inspect := range inspects { + fmt.Fprintf(GinkgoWriter, "container=%s image=%s\n", inspect.Name, inspect.Config.Image) + Expect(inspect.Config.Image).NotTo(ContainSubstring("@"), + "container %s: image should be TagRef format (no digest): %s", inspect.Name, inspect.Config.Image) + } + }) } diff --git a/mtest/operators_test.go b/mtest/operators_test.go index 0d8f1cb87..cf5976058 100644 --- a/mtest/operators_test.go +++ b/mtest/operators_test.go @@ -90,6 +90,86 @@ func testOperators() { }, )) + By("Checking images are tagged with expected digest") + fullRefs := make([]string, 0, len(cke.AllImages)) + digestRefs := make([]string, 0, len(cke.AllImages)) + expectedTags := make(map[string]string, len(cke.AllImages)) // digest -> tagRef + for _, img := range cke.AllImages { + fullRefs = append(fullRefs, img.FullRef()) + digestRefs = append(digestRefs, img.DigestRef()) + digest := strings.SplitN(img.DigestRef(), "@", 2)[1] + expectedTags[digest] = img.TagRef() + } + for _, n := range []string{node1, node2, node3, node4, node5} { + out := execSafeAt(n, "docker", "image", "list", "--digests", "--format={{.Repository}}:{{.Tag}}@{{.Digest}}") + for _, image := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if image == "" { + continue + } + Expect(image).To(BeElementOf(fullRefs), + "node %s: image %s is not a known CKE image", n, image) + } + + // All image pull events should reference a CKE-defined image digest + out = execSafeAt(n, "docker", "system", "events", + "--since", "2020-01-01T00:00:00Z", + "--until", time.Now().UTC().Format(time.RFC3339), + "--filter", "type=image", + "--filter", "event=pull", + "--format", "{{.Actor.ID}}") // @ + for _, pulled := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if pulled == "" { + continue + } + Expect(pulled).To(BeElementOf(digestRefs), + "node %s: unexpected image pull: %s", n, pulled) + } + + // All CKE images should have been tagged from the correct digest + out = execSafeAt(n, "docker", "system", "events", + "--since", "2020-01-01T00:00:00Z", + "--until", time.Now().UTC().Format(time.RFC3339), + "--filter", "type=image", + "--filter", "event=tag", + "--format", "{{.Actor.ID}} {{.Actor.Attributes.name}}") // + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if line == "" { + continue + } + parts := strings.SplitN(line, " ", 2) + Expect(parts).To(HaveLen(2)) + digest := parts[0] + actualTag := parts[1] + Expect(actualTag).To(Equal(expectedTags[digest])) + } + } + + By("Checking container image references") + for _, n := range []string{node1, node2, node3, node4, node5} { + out := execSafeAt(n, "docker", "ps", "-aq") + containerIDs := strings.Fields(strings.TrimSpace(string(out))) + Expect(containerIDs).NotTo(BeEmpty(), "node %s has no containers", n) + + out = execSafeAt(n, append([]string{"docker", "inspect"}, containerIDs...)...) + var inspects []struct { + Config struct { + Image string `json:"Image"` + } `json:"Config"` + Name string `json:"Name"` + } + err := json.Unmarshal(out, &inspects) + Expect(err).NotTo(HaveOccurred()) + Expect(inspects).To(HaveLen(len(containerIDs))) + + for _, inspect := range inspects { + fmt.Fprintf(GinkgoWriter, "node=%s container=%s image=%s\n", n, inspect.Name, inspect.Config.Image) + + // Container's image should be TagRef format (repo:tag, no @sha256: digest) + Expect(inspect.Config.Image).NotTo(ContainSubstring("@"), + "container %s on node %s: image should be TagRef format (no digest): %s", inspect.Name, n, inspect.Config.Image) + } + } + By("Stopping etcd servers") // this will run: // - EtcdStartOp diff --git a/op/common/image_pull.go b/op/common/image_pull.go index 900cd573f..5f49b3799 100644 --- a/op/common/image_pull.go +++ b/op/common/image_pull.go @@ -37,7 +37,7 @@ func (c imagePullCommand) Run(ctx context.Context, inf cke.Infrastructure, _ str } log.Warn("failed to pull image", map[string]interface{}{ - "image": c.img.Name(), + "image": c.img.FullRef(), log.FnError: err, }) select { @@ -56,6 +56,6 @@ func (c imagePullCommand) Run(ctx context.Context, inf cke.Infrastructure, _ str func (c imagePullCommand) Command() cke.Command { return cke.Command{ Name: "image-pull", - Target: c.img.Name(), + Target: c.img.FullRef(), } } diff --git a/pkg/ckecli/cmd/images.go b/pkg/ckecli/cmd/images.go index e7832e612..fb7954ed5 100644 --- a/pkg/ckecli/cmd/images.go +++ b/pkg/ckecli/cmd/images.go @@ -16,8 +16,8 @@ var imagesCmd = &cobra.Command{ // Override rootCmd.PersistentPreRunE. PersistentPreRun: func(cmd *cobra.Command, args []string) {}, Run: func(cmd *cobra.Command, args []string) { - for _, img := range cke.AllImages() { - fmt.Println(img) + for _, img := range cke.AllImages { + fmt.Println(img.FullRef()) } }, } diff --git a/pkg/compile_resources/main.go b/pkg/compile_resources/main.go index 88e2da33c..8d9fbd0bc 100644 --- a/pkg/compile_resources/main.go +++ b/pkg/compile_resources/main.go @@ -33,9 +33,9 @@ func subMain() error { } images := make(map[string]string) - for _, img := range cke.AllImages() { - id := strings.SplitN(path.Base(img), ":", 2)[0] - images[id] = img + for _, img := range cke.AllImages { + id := strings.SplitN(path.Base(img.FullRef()), ":", 2)[0] + images[id] = img.FullRef() } var allResources []cke.ResourceDefinition diff --git a/pkg/update-images/main.go b/pkg/update-images/main.go new file mode 100644 index 000000000..9fc50ea85 --- /dev/null +++ b/pkg/update-images/main.go @@ -0,0 +1,159 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "go/format" + "net/url" + "os" + "os/exec" + "strings" + "text/template" +) + +type imageEntry struct { + VarName string + Repository string +} + +// Update Repository here when adding or removing images. +var images = []imageEntry{ + {"EtcdImage", "ghcr.io/cybozu/etcd"}, + {"KubernetesImage", "ghcr.io/cybozu/kubernetes"}, + {"ToolsImage", "ghcr.io/cybozu-go/cke-tools"}, + {"PauseImage", "ghcr.io/cybozu/pause"}, + {"CoreDNSImage", "ghcr.io/cybozu/coredns"}, + {"UnboundImage", "ghcr.io/cybozu/unbound"}, + {"UnboundExporterImage", "ghcr.io/cybozu/unbound_exporter"}, +} + +const outputFile = "images_gen.go" + +const tmpl = `// Code generated by pkg/update-images. DO NOT EDIT. + +package cke + +var ( +{{- range .}} + {{.VarName}} = newImage("{{.Repository}}", "{{.Tag}}", "{{.Digest}}") +{{- end}} +) + +// AllImages is the list of all container images used by CKE. +var AllImages = []Image{ +{{- range .}} + {{.VarName}}, +{{- end}} +} +` + +type packageVersion struct { + Name string `json:"name"` // digest e.g. "sha256:abc..." + CreatedAt string `json:"created_at"` + Metadata struct { + Container struct { + Tags []string `json:"tags"` + } `json:"container"` + } `json:"metadata"` +} + +type imageResult struct { + VarName string + Repository string + Tag string + Digest string +} + +// fetchLatest queries the GitHub Packages API (via gh CLI) for the latest +// version of repository, returning its tag and digest. +func fetchLatest(repository string) (tag, digest string, err error) { + withoutRegistry, _ := strings.CutPrefix(repository, "ghcr.io/") + org, pkg, ok := strings.Cut(withoutRegistry, "/") + if !ok { + return "", "", fmt.Errorf("invalid repository %q: expected ghcr.io/org/package", repository) + } + + apiPath := fmt.Sprintf("/orgs/%s/packages/container/%s/versions?per_page=100", + org, url.PathEscape(pkg)) + + out, err := exec.Command("gh", "api", apiPath, "--paginate").Output() + if err != nil { + return "", "", fmt.Errorf("gh api failed for %s: %w", repository, err) + } + + var versions []packageVersion + dec := json.NewDecoder(bytes.NewReader(out)) + for dec.More() { + var page []packageVersion + if err := dec.Decode(&page); err != nil { + return "", "", fmt.Errorf("failed to decode response for %s: %w", repository, err) + } + versions = append(versions, page...) + } + + // Pick the latest version that has at least one tag. + var latest *packageVersion + for i := range versions { + v := &versions[i] + if len(v.Metadata.Container.Tags) == 0 { + continue + } + if latest == nil || v.CreatedAt > latest.CreatedAt { + latest = v + } + } + if latest == nil { + return "", "", fmt.Errorf("no tagged version found for %s", repository) + } + + // Pick the longest tag excluding "latest" + for _, t := range latest.Metadata.Container.Tags { + if t != "latest" && len(t) > len(tag) { + tag = t + } + } + if tag == "" { + return "", "", fmt.Errorf("no non-latest tag found for %s", repository) + } + + return tag, latest.Name, nil +} + +func main() { + results := make([]imageResult, len(images)) + for i, img := range images { + tag, digest, err := fetchLatest(img.Repository) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to fetch latest for %s: %v\n", img.Repository, err) + os.Exit(1) + } + fmt.Printf("%s %s:%s -> %s\n", img.VarName, img.Repository, tag, digest) + results[i] = imageResult{ + VarName: img.VarName, + Repository: img.Repository, + Tag: tag, + Digest: digest, + } + } + + t := template.Must(template.New("").Parse(tmpl)) + var buf bytes.Buffer + if err := t.Execute(&buf, results); err != nil { + fmt.Fprintf(os.Stderr, "failed to render template: %v\n", err) + os.Exit(1) + } + + formatted, err := format.Source(buf.Bytes()) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to format source: %v\n", err) + os.Exit(1) + } + + if err := os.WriteFile(outputFile, formatted, 0644); err != nil { + fmt.Fprintf(os.Stderr, "failed to write %s: %v\n", outputFile, err) + os.Exit(1) + } + + fmt.Printf("Updated %s\n", outputFile) +} diff --git a/server/node_filter.go b/server/node_filter.go index fc4ad93e9..19c538e4e 100644 --- a/server/node_filter.go +++ b/server/node_filter.go @@ -89,7 +89,7 @@ func (nf *NodeFilter) RiversOutdated(targets []*cke.Node) (nodes []*cke.Node) { switch { case !st.Running: // stopped nodes are excluded - case cke.ToolsImage.Name() != st.Image: + case cke.ToolsImage.TagRef() != st.Image: fallthrough case !currentBuiltIn.Equal(st.BuiltInParams): fallthrough @@ -120,7 +120,7 @@ func (nf *NodeFilter) EtcdRiversOutdated(targets []*cke.Node) (nodes []*cke.Node switch { case !st.Running: // stopped nodes are excluded - case cke.ToolsImage.Name() != st.Image: + case cke.ToolsImage.TagRef() != st.Image: fallthrough case !currentBuiltIn.Equal(st.BuiltInParams): fallthrough @@ -303,7 +303,7 @@ func (nf *NodeFilter) EtcdOutdatedMembers() (nodes []*cke.Node) { } currentBuiltIn := etcd.BuiltInParams(n, []string{}, "new") switch { - case cke.EtcdImage.Name() != st.Image: + case cke.EtcdImage.TagRef() != st.Image: fallthrough case !etcdEqualParams(st.BuiltInParams, currentBuiltIn): fallthrough @@ -368,7 +368,7 @@ func (nf *NodeFilter) APIServerOutdated(targets []*cke.Node) (nodes []*cke.Node) switch { case !st.Running: // stopped nodes are excluded - case cke.KubernetesImage.Name() != st.Image: + case cke.KubernetesImage.TagRef() != st.Image: fallthrough case !currentBuiltIn.Equal(st.BuiltInParams): fallthrough @@ -399,7 +399,7 @@ func (nf *NodeFilter) ControllerManagerOutdated(targets []*cke.Node) (nodes []*c switch { case !st.Running: // stopped nodes are excluded - case cke.KubernetesImage.Name() != st.Image: + case cke.KubernetesImage.TagRef() != st.Image: fallthrough case !currentBuiltIn.Equal(st.BuiltInParams): fallthrough @@ -433,7 +433,7 @@ func (nf *NodeFilter) SchedulerOutdated(targets []*cke.Node, params cke.Schedule switch { case !st.Running: // stopped nodes are excluded - case cke.KubernetesImage.Name() != st.Image: + case cke.KubernetesImage.TagRef() != st.Image: fallthrough case !currentBuiltIn.Equal(st.BuiltInParams): fallthrough @@ -500,7 +500,7 @@ func (nf *NodeFilter) KubeletOutdated(targets []*cke.Node) (nodes []*cke.Node) { // stopped nodes are excluded case kubeletRuntimeChanged(st.BuiltInParams, currentBuiltIn): log.Warn("kubelet's container runtime cannot be changed", nil) - case cke.KubernetesImage.Name() != st.Image: + case cke.KubernetesImage.TagRef() != st.Image: fallthrough case !currentBuiltIn.Equal(st.BuiltInParams): fallthrough @@ -624,7 +624,7 @@ func (nf *NodeFilter) ProxyOutdated(targets []*cke.Node, params cke.ProxyParams) switch { case !st.Running: // stopped nodes are excluded - case cke.KubernetesImage.Name() != st.Image: + case cke.KubernetesImage.TagRef() != st.Image: fallthrough case !currentBuiltIn.Equal(st.BuiltInParams): fallthrough diff --git a/server/strategy_test.go b/server/strategy_test.go index bc2f08de8..9f6851987 100644 --- a/server/strategy_test.go +++ b/server/strategy_test.go @@ -191,7 +191,7 @@ func (d testData) withResources(res []cke.ResourceDefinition) testData { func (d testData) withRivers() testData { for _, v := range d.Status.NodeStatuses { v.Rivers.Running = true - v.Rivers.Image = cke.ToolsImage.Name() + v.Rivers.Image = cke.ToolsImage.TagRef() v.Rivers.BuiltInParams = op.RiversParams(d.ControlPlane(), op.RiversUpstreamPort, op.RiversListenPort) } return d @@ -201,7 +201,7 @@ func (d testData) withEtcdRivers() testData { for _, n := range d.ControlPlane() { st := &d.NodeStatus(n).EtcdRivers st.Running = true - st.Image = cke.ToolsImage.Name() + st.Image = cke.ToolsImage.TagRef() st.BuiltInParams = op.RiversParams(d.ControlPlane(), op.EtcdRiversUpstreamPort, op.EtcdRiversListenPort) } return d @@ -238,7 +238,7 @@ func (d testData) withUnhealthyEtcd() testData { for _, n := range d.ControlPlane() { st := &d.NodeStatus(n).Etcd st.Running = true - st.Image = cke.EtcdImage.Name() + st.Image = cke.EtcdImage.TagRef() st.BuiltInParams = etcd.BuiltInParams(n, nil, "") } return d @@ -265,7 +265,7 @@ func (d testData) withAPIServer(serviceSubnet, domain string) testData { st := &d.NodeStatus(n).APIServer st.Running = true st.IsHealthy = true - st.Image = cke.KubernetesImage.Name() + st.Image = cke.KubernetesImage.TagRef() st.BuiltInParams = k8s.APIServerParams(n.Address, serviceSubnet, false, "", "", domain) } return d @@ -282,7 +282,7 @@ func (d testData) withControllerManager(name, serviceSubnet string) testData { st := &d.NodeStatus(n).ControllerManager st.Running = true st.IsHealthy = true - st.Image = cke.KubernetesImage.Name() + st.Image = cke.KubernetesImage.TagRef() st.BuiltInParams = k8s.ControllerManagerParams(name, serviceSubnet) } return d @@ -293,7 +293,7 @@ func (d testData) withScheduler() testData { st := &d.NodeStatus(n).Scheduler st.Running = true st.IsHealthy = true - st.Image = cke.KubernetesImage.Name() + st.Image = cke.KubernetesImage.TagRef() st.BuiltInParams = k8s.SchedulerParams() st.Config = &schedulerv1.KubeSchedulerConfiguration{} @@ -309,7 +309,7 @@ func (d testData) withKubelet(domain, dns string, allowSwap bool) testData { st := &d.NodeStatus(n).Kubelet st.Running = true st.IsHealthy = true - st.Image = cke.KubernetesImage.Name() + st.Image = cke.KubernetesImage.TagRef() st.BuiltInParams = k8s.KubeletServiceParams(n, cke.KubeletParams{ CRIEndpoint: "/var/run/k8s-containerd.sock", }) @@ -371,7 +371,7 @@ func (d testData) withProxy() testData { st := &d.NodeStatus(n).Proxy st.Running = true st.IsHealthy = true - st.Image = cke.KubernetesImage.Name() + st.Image = cke.KubernetesImage.TagRef() st.BuiltInParams = k8s.ProxyParams() st.Config = &proxyv1alpha1.KubeProxyConfiguration{} st.Config.HostnameOverride = n.Nodename() @@ -550,9 +550,9 @@ func (d testData) withK8sResourceReady() testData { } ks.ResourceStatuses["ClusterRole/system:kube-apiserver-to-kubelet"].Annotations[cke.AnnotationResourceRevision] = "2" ks.ResourceStatuses["ClusterRole/system:cluster-dns"].Annotations[cke.AnnotationResourceRevision] = "2" - ks.ResourceStatuses["Deployment/kube-system/cluster-dns"].Annotations[cke.AnnotationResourceImage] = cke.CoreDNSImage.Name() + ks.ResourceStatuses["Deployment/kube-system/cluster-dns"].Annotations[cke.AnnotationResourceImage] = cke.CoreDNSImage.FullRef() ks.ResourceStatuses["Deployment/kube-system/cluster-dns"].Annotations[cke.AnnotationResourceRevision] = "5" - ks.ResourceStatuses["DaemonSet/kube-system/node-dns"].Annotations[cke.AnnotationResourceImage] = cke.UnboundImage.Name() + "," + cke.UnboundExporterImage.Name() + ks.ResourceStatuses["DaemonSet/kube-system/node-dns"].Annotations[cke.AnnotationResourceImage] = cke.UnboundImage.FullRef() + "," + cke.UnboundExporterImage.FullRef() ks.ResourceStatuses["DaemonSet/kube-system/node-dns"].Annotations[cke.AnnotationResourceRevision] = "4" ks.ClusterDNS.ConfigMap = clusterdns.ConfigMap(testDefaultDNSDomain, testDefaultDNSServers) ks.ClusterDNS.ClusterIP = testDefaultDNSAddr diff --git a/static/resources.go b/static/resources.go index 2f8f6c466..dd1869ad8 100644 --- a/static/resources.go +++ b/static/resources.go @@ -60,8 +60,8 @@ var Resources = []cke.ResourceDefinition{ Namespace: "kube-system", Name: "node-dns", Revision: 4, - Image: "ghcr.io/cybozu/unbound:1.25.1.1,ghcr.io/cybozu/unbound_exporter:0.5.0.4", - Definition: []byte("kind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: node-dns\n namespace: kube-system\n annotations:\n cke.cybozu.com/image: \"ghcr.io/cybozu/unbound:1.25.1.1,ghcr.io/cybozu/unbound_exporter:0.5.0.4\"\n cke.cybozu.com/revision: \"4\"\nspec:\n selector:\n matchLabels:\n cke.cybozu.com/appname: node-dns\n updateStrategy:\n type: RollingUpdate\n rollingUpdate:\n maxSurge: 35%\n maxUnavailable: 0\n template:\n metadata:\n labels:\n cke.cybozu.com/appname: node-dns\n spec:\n priorityClassName: system-node-critical\n nodeSelector:\n kubernetes.io/os: linux\n hostNetwork: true\n tolerations:\n - operator: Exists\n terminationGracePeriodSeconds: 1\n containers:\n - name: unbound\n image: ghcr.io/cybozu/unbound:1.25.1.1\n args:\n - -c\n - /etc/unbound/unbound.conf\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n add:\n - NET_BIND_SERVICE\n drop:\n - all\n readOnlyRootFilesystem: true\n readinessProbe:\n tcpSocket:\n port: 53\n host: localhost\n periodSeconds: 1\n livenessProbe:\n tcpSocket:\n port: 53\n host: localhost\n periodSeconds: 1\n initialDelaySeconds: 1\n failureThreshold: 6\n volumeMounts:\n - name: config-volume\n mountPath: /etc/unbound\n - name: var-run-unbound\n mountPath: /var/run/unbound\n resources:\n requests:\n cpu: 50m\n memory: 250Mi\n - name: reload\n image: ghcr.io/cybozu/unbound:1.25.1.1\n command:\n - /usr/local/bin/reload-unbound\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - all\n readOnlyRootFilesystem: true\n volumeMounts:\n - name: config-volume\n mountPath: /etc/unbound\n - name: var-run-unbound\n mountPath: /var/run/unbound\n - name: exporter\n image: ghcr.io/cybozu/unbound_exporter:0.5.0.4\n args:\n # must be same with the path written in /op/nodedns/nodedns.go\n - --unbound.host=unix:///var/run/unbound/unbound.sock\n - --web.reuse-port=true\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - all\n readOnlyRootFilesystem: true\n volumeMounts:\n - name: var-run-unbound\n mountPath: /var/run/unbound\n volumes:\n - name: config-volume\n configMap:\n name: node-dns\n items:\n - key: unbound.conf\n path: unbound.conf\n - name: var-run-unbound\n emptyDir: {}\n"), + Image: "ghcr.io/cybozu/unbound:1.25.1.1@sha256:b84ce87568ed11859891747650a46d8431bac6d1ddda38b542e4d16cd36e0b86,ghcr.io/cybozu/unbound_exporter:0.5.0.4@sha256:a7d46220f43bf2ae0c81bbcb0b68801de2d60e87df298db4691ea0dd39b25593", + Definition: []byte("kind: DaemonSet\napiVersion: apps/v1\nmetadata:\n name: node-dns\n namespace: kube-system\n annotations:\n cke.cybozu.com/image: \"ghcr.io/cybozu/unbound:1.25.1.1@sha256:b84ce87568ed11859891747650a46d8431bac6d1ddda38b542e4d16cd36e0b86,ghcr.io/cybozu/unbound_exporter:0.5.0.4@sha256:a7d46220f43bf2ae0c81bbcb0b68801de2d60e87df298db4691ea0dd39b25593\"\n cke.cybozu.com/revision: \"4\"\nspec:\n selector:\n matchLabels:\n cke.cybozu.com/appname: node-dns\n updateStrategy:\n type: RollingUpdate\n rollingUpdate:\n maxSurge: 35%\n maxUnavailable: 0\n template:\n metadata:\n labels:\n cke.cybozu.com/appname: node-dns\n spec:\n priorityClassName: system-node-critical\n nodeSelector:\n kubernetes.io/os: linux\n hostNetwork: true\n tolerations:\n - operator: Exists\n terminationGracePeriodSeconds: 1\n containers:\n - name: unbound\n image: ghcr.io/cybozu/unbound:1.25.1.1@sha256:b84ce87568ed11859891747650a46d8431bac6d1ddda38b542e4d16cd36e0b86\n args:\n - -c\n - /etc/unbound/unbound.conf\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n add:\n - NET_BIND_SERVICE\n drop:\n - all\n readOnlyRootFilesystem: true\n readinessProbe:\n tcpSocket:\n port: 53\n host: localhost\n periodSeconds: 1\n livenessProbe:\n tcpSocket:\n port: 53\n host: localhost\n periodSeconds: 1\n initialDelaySeconds: 1\n failureThreshold: 6\n volumeMounts:\n - name: config-volume\n mountPath: /etc/unbound\n - name: var-run-unbound\n mountPath: /var/run/unbound\n resources:\n requests:\n cpu: 50m\n memory: 250Mi\n - name: reload\n image: ghcr.io/cybozu/unbound:1.25.1.1@sha256:b84ce87568ed11859891747650a46d8431bac6d1ddda38b542e4d16cd36e0b86\n command:\n - /usr/local/bin/reload-unbound\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - all\n readOnlyRootFilesystem: true\n volumeMounts:\n - name: config-volume\n mountPath: /etc/unbound\n - name: var-run-unbound\n mountPath: /var/run/unbound\n - name: exporter\n image: ghcr.io/cybozu/unbound_exporter:0.5.0.4@sha256:a7d46220f43bf2ae0c81bbcb0b68801de2d60e87df298db4691ea0dd39b25593\n args:\n # must be same with the path written in /op/nodedns/nodedns.go\n - --unbound.host=unix:///var/run/unbound/unbound.sock\n - --web.reuse-port=true\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - all\n readOnlyRootFilesystem: true\n volumeMounts:\n - name: var-run-unbound\n mountPath: /var/run/unbound\n volumes:\n - name: config-volume\n configMap:\n name: node-dns\n items:\n - key: unbound.conf\n path: unbound.conf\n - name: var-run-unbound\n emptyDir: {}\n"), }, { Key: "Deployment/kube-system/cluster-dns", @@ -69,8 +69,8 @@ var Resources = []cke.ResourceDefinition{ Namespace: "kube-system", Name: "cluster-dns", Revision: 5, - Image: "ghcr.io/cybozu/coredns:1.14.2.1", - Definition: []byte("\nkind: Deployment\napiVersion: apps/v1\nmetadata:\n name: cluster-dns\n namespace: kube-system\n annotations:\n cke.cybozu.com/image: \"ghcr.io/cybozu/coredns:1.14.2.1\"\n cke.cybozu.com/revision: \"5\"\nspec:\n replicas: 2\n strategy:\n type: RollingUpdate\n rollingUpdate:\n maxUnavailable: 1\n selector:\n matchLabels:\n cke.cybozu.com/appname: cluster-dns\n template:\n metadata:\n labels:\n cke.cybozu.com/appname: cluster-dns\n k8s-app: coredns # sonobuoy requires\n annotations:\n prometheus.io/port: \"9153\"\n spec:\n priorityClassName: system-cluster-critical\n serviceAccountName: cke-cluster-dns\n tolerations:\n - key: node-role.kubernetes.io/master\n effect: NoSchedule\n - key: \"CriticalAddonsOnly\"\n operator: \"Exists\"\n - key: kubernetes.io/e2e-evict-taint-key\n operator: Exists\n # for sonobuoy https://github.com/vmware-tanzu/sonobuoy/pull/878\n affinity:\n podAntiAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - labelSelector:\n matchLabels:\n cke.cybozu.com/appname: cluster-dns\n topologyKey: kubernetes.io/hostname\n topologySpreadConstraints:\n - labelSelector:\n matchLabels:\n cke.cybozu.com/appname: cluster-dns\n maxSkew: 1\n topologyKey: topology.kubernetes.io/zone\n whenUnsatisfiable: ScheduleAnyway\n containers:\n - name: coredns\n image: ghcr.io/cybozu/coredns:1.14.2.1\n imagePullPolicy: IfNotPresent\n resources:\n requests:\n cpu: 50m\n memory: 250Mi\n args: [ \"-conf\", \"/etc/coredns/Corefile\" ]\n lifecycle:\n preStop:\n exec:\n command: [\"sh\", \"-c\", \"sleep 5\"]\n volumeMounts:\n - name: config-volume\n mountPath: /etc/coredns\n readOnly: true\n ports:\n - containerPort: 1053\n name: dns\n protocol: UDP\n - containerPort: 1053\n name: dns-tcp\n protocol: TCP\n - containerPort: 9153\n name: metrics\n protocol: TCP\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - all\n readOnlyRootFilesystem: true\n readinessProbe:\n httpGet:\n path: /ready\n port: 8181\n scheme: HTTP\n livenessProbe:\n httpGet:\n path: /health\n port: 8080\n scheme: HTTP\n initialDelaySeconds: 60\n timeoutSeconds: 5\n successThreshold: 1\n failureThreshold: 5\n dnsPolicy: Default\n volumes:\n - name: config-volume\n configMap:\n name: cluster-dns\n items:\n - key: Corefile\n path: Corefile\n"), + Image: "ghcr.io/cybozu/coredns:1.14.2.1@sha256:e29066d55b9455fdba973ceb6f77232677de15263cc1c78f63707935adc01a14", + Definition: []byte("\nkind: Deployment\napiVersion: apps/v1\nmetadata:\n name: cluster-dns\n namespace: kube-system\n annotations:\n cke.cybozu.com/image: \"ghcr.io/cybozu/coredns:1.14.2.1@sha256:e29066d55b9455fdba973ceb6f77232677de15263cc1c78f63707935adc01a14\"\n cke.cybozu.com/revision: \"5\"\nspec:\n replicas: 2\n strategy:\n type: RollingUpdate\n rollingUpdate:\n maxUnavailable: 1\n selector:\n matchLabels:\n cke.cybozu.com/appname: cluster-dns\n template:\n metadata:\n labels:\n cke.cybozu.com/appname: cluster-dns\n k8s-app: coredns # sonobuoy requires\n annotations:\n prometheus.io/port: \"9153\"\n spec:\n priorityClassName: system-cluster-critical\n serviceAccountName: cke-cluster-dns\n tolerations:\n - key: node-role.kubernetes.io/master\n effect: NoSchedule\n - key: \"CriticalAddonsOnly\"\n operator: \"Exists\"\n - key: kubernetes.io/e2e-evict-taint-key\n operator: Exists\n # for sonobuoy https://github.com/vmware-tanzu/sonobuoy/pull/878\n affinity:\n podAntiAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - labelSelector:\n matchLabels:\n cke.cybozu.com/appname: cluster-dns\n topologyKey: kubernetes.io/hostname\n topologySpreadConstraints:\n - labelSelector:\n matchLabels:\n cke.cybozu.com/appname: cluster-dns\n maxSkew: 1\n topologyKey: topology.kubernetes.io/zone\n whenUnsatisfiable: ScheduleAnyway\n containers:\n - name: coredns\n image: ghcr.io/cybozu/coredns:1.14.2.1@sha256:e29066d55b9455fdba973ceb6f77232677de15263cc1c78f63707935adc01a14\n imagePullPolicy: IfNotPresent\n resources:\n requests:\n cpu: 50m\n memory: 250Mi\n args: [ \"-conf\", \"/etc/coredns/Corefile\" ]\n lifecycle:\n preStop:\n exec:\n command: [\"sh\", \"-c\", \"sleep 5\"]\n volumeMounts:\n - name: config-volume\n mountPath: /etc/coredns\n readOnly: true\n ports:\n - containerPort: 1053\n name: dns\n protocol: UDP\n - containerPort: 1053\n name: dns-tcp\n protocol: TCP\n - containerPort: 9153\n name: metrics\n protocol: TCP\n securityContext:\n allowPrivilegeEscalation: false\n capabilities:\n drop:\n - all\n readOnlyRootFilesystem: true\n readinessProbe:\n httpGet:\n path: /ready\n port: 8181\n scheme: HTTP\n livenessProbe:\n httpGet:\n path: /health\n port: 8080\n scheme: HTTP\n initialDelaySeconds: 60\n timeoutSeconds: 5\n successThreshold: 1\n failureThreshold: 5\n dnsPolicy: Default\n volumes:\n - name: config-volume\n configMap:\n name: cluster-dns\n items:\n - key: Corefile\n path: Corefile\n"), }, { Key: "PodDisruptionBudget/kube-system/cluster-dns-pdb",