From c261804ea19cda6b95f6cc18f7672146c6df4a5b Mon Sep 17 00:00:00 2001 From: Steven Davidovitz Date: Tue, 14 Jul 2026 13:39:05 -0700 Subject: [PATCH] chore: test grove operator upgrades --- .github/workflows/build-check-test.yaml | 13 +- Makefile | 6 + operator/Makefile | 12 + operator/e2e/setup/grove.go | 6 +- operator/e2e/setup/helm.go | 3 + operator/e2e/tests/suite_test.go | 2 +- operator/e2e/tests/upgrade/upgrade_test.go | 607 +++++++++++++++++++++ operator/go.mod | 2 + operator/go.sum | 5 + 9 files changed, 651 insertions(+), 5 deletions(-) create mode 100644 operator/e2e/tests/upgrade/upgrade_test.go diff --git a/.github/workflows/build-check-test.yaml b/.github/workflows/build-check-test.yaml index 3a3fb4237..1f3b3cb4d 100644 --- a/.github/workflows/build-check-test.yaml +++ b/.github/workflows/build-check-test.yaml @@ -111,6 +111,8 @@ jobs: # use NVIDIA self-hosted runner setting is on Velonix repository runs-on: prod-grove-e2e-v1 timeout-minutes: 60 + env: + GROVE_UPGRADE_E2E_CLUSTER_NAME: grove-upgrade-e2e-${{ github.run_id }}-${{ github.run_attempt }} strategy: fail-fast: false matrix: @@ -137,6 +139,8 @@ jobs: test_pattern: "^Test_CRD_Installer" - test_name: resource_sharing test_pattern: "^Test_RS" + - test_name: upgrade_from_latest_release + make_target: "run-upgrade-e2e" name: E2E - ${{ matrix.test_name }} steps: # print runner specs so we have a record in case of failures @@ -157,6 +161,8 @@ jobs: uses: ./.github/actions/e2e-setup - name: Run e2e tests - ${{ matrix.test_name }} + env: + GITHUB_TOKEN: ${{ matrix.test_name == 'upgrade_from_latest_release' && secrets.GITHUB_TOKEN || '' }} run: | make ${{ matrix.make_target || 'run-e2e-full' }} TEST_PATTERN='${{ matrix.test_pattern }}' E2E_CREATE_FLAGS='--dind-memory-mode' working-directory: operator @@ -168,7 +174,11 @@ jobs: if: always() working-directory: operator run: | - make e2e-cluster-down || true + if [[ "${{ matrix.test_name }}" == "upgrade_from_latest_release" ]]; then + k3d cluster delete "$GROVE_UPGRADE_E2E_CLUSTER_NAME" || true + else + make e2e-cluster-down || true + fi # Upload diagnostic logs collected by debug_utils.go on test failure. # Uses 'warn' since this only runs on failure and missing files indicates a problem. @@ -204,6 +214,7 @@ jobs: - test_name: auto_mnnvl - test_name: crd_installer - test_name: resource_sharing + - test_name: upgrade_from_latest_release name: E2E - ${{ matrix.test_name }} steps: - name: Skip E2E (no relevant changes) diff --git a/Makefile b/Makefile index fc3cccd84..eec37431e 100644 --- a/Makefile +++ b/Makefile @@ -124,6 +124,12 @@ run-e2e: @echo "> Running e2e tests for operator" @make --directory=operator run-e2e +# Runs the standalone latest-release-to-current operator upgrade test. +.PHONY: run-upgrade-e2e +run-upgrade-e2e: + @echo "> Running operator upgrade e2e test" + @make --directory=operator run-upgrade-e2e + # Runs all tests .PHONY: test test: test-unit diff --git a/operator/Makefile b/operator/Makefile index 6d8d73d46..2c0e407ba 100644 --- a/operator/Makefile +++ b/operator/Makefile @@ -27,6 +27,9 @@ INSTALL_CRDS_NAME := "grove-install-crds" K3S_IMAGE ?= rancher/k3s:v1.35.5-k3s1 E2E_REGISTRY_PORT ?= 5001 E2E_CREATE_FLAGS ?= +ifndef DIAG_DIR +DIAG_DIR := $(if $(GROVE_E2E_DIAG_DIR),$(GROVE_E2E_DIAG_DIR),$(MODULE_ROOT)/e2e-diagnostics) +endif # Include tools targets include $(REPO_HACK_DIR)/tools.mk @@ -131,6 +134,15 @@ run-e2e-verbose: $(GOTESTSUM) @echo "> Running e2e tests (verbose)..." @cd e2e && $(GOTESTSUM) --format short-verbose -- -count=1 -tags=e2e ./tests/... -timeout 45m $(if $(TEST_PATTERN),-run '$(TEST_PATTERN)') +# Run the standalone upgrade test from the latest published GitHub release to +# the operator built from the current checkout. The test owns its k3d cluster. +.PHONY: run-upgrade-e2e +run-upgrade-e2e: export GROVE_E2E_DIAG_DIR = $(DIAG_DIR) +run-upgrade-e2e: export GROVE_E2E_DIAG_MODE = $(DIAG_MODE) +run-upgrade-e2e: $(GOTESTSUM) + @echo "> Running e2e upgrade tests (verbose)..." + $(GOTESTSUM) --format short-verbose -- -count=1 -tags=e2e,e2eupgrade ./e2e/tests/upgrade/ -timeout 45m $(if $(TEST_PATTERN),-run '$(TEST_PATTERN)') + # Create a k3d cluster for e2e testing (with Grove and Kai scheduler deployed) # This is optional - tests work with any Kubernetes cluster that has the required components .PHONY: e2e-cluster-up diff --git a/operator/e2e/setup/grove.go b/operator/e2e/setup/grove.go index 8b20f469c..084f50fad 100644 --- a/operator/e2e/setup/grove.go +++ b/operator/e2e/setup/grove.go @@ -133,7 +133,7 @@ func (c *GroveConfig) toHelmValues() (map[string]interface{}, error) { // // This approach avoids wasteful rebuilds while staying compatible with the Skaffold installation. func UpdateGroveConfiguration(ctx context.Context, restConfig *rest.Config, chartDir string, config *GroveConfig, logger *log.Logger) error { - chartVersion, err := getChartVersion(chartDir) + chartVersion, err := GetGroveChartVersion(chartDir) if err != nil { return fmt.Errorf("failed to get chart version: %w", err) } @@ -184,12 +184,12 @@ type chartYAML struct { Version string `yaml:"version"` } -// getChartVersion reads the version from Chart.yaml in the given chart directory. +// GetGroveChartVersion reads the version from Chart.yaml in the given chart directory. // The chartDir parameter should be the path to a Helm chart directory. Chart.yaml is // a required file per the Helm chart specification and will always exist for valid charts. // We read from Chart.yaml rather than hardcoding the version to maintain a single source // of truth, avoiding configuration drift between the chart definition and the e2e test code. -func getChartVersion(chartDir string) (string, error) { +func GetGroveChartVersion(chartDir string) (string, error) { chartFile := filepath.Join(chartDir, "Chart.yaml") data, err := os.ReadFile(chartFile) if err != nil { diff --git a/operator/e2e/setup/helm.go b/operator/e2e/setup/helm.go index 64118e162..e940fdfb8 100644 --- a/operator/e2e/setup/helm.go +++ b/operator/e2e/setup/helm.go @@ -65,6 +65,8 @@ type HelmInstallConfig struct { RepoURL string // ReuseValues reuses the last release's values and merges in the new values. ReuseValues bool + // ResetValues resets to the values built into the new chart before applying Values. + ResetValues bool // Timeout is the time to wait for Kubernetes operations (default: 5 minutes). Timeout time.Duration } @@ -323,6 +325,7 @@ func newUpgradeClient(actionConfig *action.Configuration, config *HelmInstallCon client.Wait = config.Wait client.Version = config.ChartVersion client.ReuseValues = config.ReuseValues + client.ResetValues = config.ResetValues // Set timeout if config.Timeout > 0 { diff --git a/operator/e2e/tests/suite_test.go b/operator/e2e/tests/suite_test.go index a9a2e4847..eb642708f 100644 --- a/operator/e2e/tests/suite_test.go +++ b/operator/e2e/tests/suite_test.go @@ -21,7 +21,7 @@ // These tests are disabled by default due to the 'e2e' build tag above. // To run these tests, use: // -// go test -tags=e2e ./e2e_testing/tests/... +// go test -tags=e2e ./e2e/tests/... // // Without the -tags=e2e flag, these tests will be skipped entirely. package tests diff --git a/operator/e2e/tests/upgrade/upgrade_test.go b/operator/e2e/tests/upgrade/upgrade_test.go new file mode 100644 index 000000000..29c5d36dc --- /dev/null +++ b/operator/e2e/tests/upgrade/upgrade_test.go @@ -0,0 +1,607 @@ +//go:build e2e && e2eupgrade + +// /* +// Copyright 2026 The Grove Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// */ + +// Package upgrade contains an end-to-end test for upgrading the Grove operator. +// +// These tests are disabled by default due to the 'e2e' and 'e2eupgrade' build tags above. +// To run these tests, use: +// +// go test -tags=e2e,e2eupgrade ./e2e/tests/upgrade/... +// +// Without both build tags, these tests will be skipped entirely. + +package upgrade + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + "time" + + "github.com/ai-dynamo/grove/operator/e2e/diagnostics" + "github.com/ai-dynamo/grove/operator/e2e/k8s/k8sclient" + e2elog "github.com/ai-dynamo/grove/operator/e2e/log" + "github.com/ai-dynamo/grove/operator/e2e/setup" + "github.com/google/go-github/v86/github" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + githubOwner = "ai-dynamo" + githubRepository = "grove" + releasedChart = "oci://ghcr.io/ai-dynamo/grove/grove-charts" + releasedImageRegistry = "ghcr.io/ai-dynamo/grove" + releaseName = "grove" + operatorNamespace = setup.OperatorNamespace + workloadNamespace = "grove-upgrade-e2e" + workloadName = "upgrade-survivor" + currentImageTag = "upgrade-e2e-current" +) + +var ( + podsGVR = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"} + deploymentsGVR = schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} + podCliqueSetsGVR = schema.GroupVersionResource{Group: "grove.io", Version: "v1alpha1", Resource: "podcliquesets"} +) + +type commandRunner struct { + t *testing.T + workingDir string + environment []string +} + +// TestUpgradeFromLatestGitHubRelease owns the complete upgrade lifecycle. It deliberately +// does not reuse the shared E2E cluster lifecycle: upgrade coverage should begin with the +// published chart and images, not with a cluster that already contains checkout resources. +func TestUpgradeFromLatestGitHubRelease(t *testing.T) { + if testing.Short() { + t.Skip("upgrade E2E test is disabled by -short") + } + + operatorDir := findOperatorDir(t) + runner := &commandRunner{ + t: t, + workingDir: operatorDir, + environment: os.Environ(), + } + requireCommands(t, "docker", "k3d", "make") + + testTimeout := durationFromEnv(t, "GROVE_UPGRADE_E2E_TIMEOUT", 30*time.Minute) + ctx, cancel := context.WithTimeout(t.Context(), testTimeout) + defer cancel() + + fromVersion := strings.TrimSpace(os.Getenv("GROVE_UPGRADE_FROM_VERSION")) + if fromVersion == "" { + fromVersion = latestGitHubRelease(ctx, t, strings.TrimSpace(os.Getenv("GITHUB_TOKEN"))) + } + t.Logf("testing Grove upgrade from %s to the current checkout", fromVersion) + + runner.mustRun(ctx, "docker", "info") + runner.mustRun(ctx, "make", + "docker-build", + "PLATFORM=linux/"+runtime.GOARCH, + "VERSION="+currentImageTag, + "DOCKER_BUILD_ADDITIONAL_ARGS=--load", + ) + + clusterName := firstNonEmpty( + os.Getenv("GROVE_UPGRADE_E2E_CLUSTER_NAME"), + fmt.Sprintf("grove-upgrade-e2e-%d", os.Getpid()), + ) + keepCluster := envBool("GROVE_UPGRADE_E2E_KEEP_CLUSTER") + t.Cleanup(func() { + if keepCluster { + t.Logf("preserving k3d cluster %q", clusterName) + return + } + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cleanupCancel() + if output, err := runner.run(cleanupCtx, "", "k3d", "cluster", "delete", clusterName); err != nil { + t.Logf("failed to delete k3d cluster %q: %v\n%s", clusterName, err, output) + } + }) + + k3sImage := firstNonEmpty(os.Getenv("GROVE_UPGRADE_E2E_K3S_IMAGE"), "rancher/k3s:v1.35.5-k3s1") + runner.mustRun(ctx, "k3d", "cluster", "create", clusterName, + "--servers", "1", + "--agents", "0", + "--image", k3sImage, + "--no-lb", + "--wait", + "--timeout", "3m", + "--k3s-arg", "--disable=traefik@server:0", + "--kubeconfig-update-default=false", + "--kubeconfig-switch-context=false", + ) + + kubeconfig := runner.mustRun(ctx, "k3d", "kubeconfig", "get", clusterName) + restConfig, dynamicClient, diagnosticClient := newKubernetesClients(t, []byte(kubeconfig)) + diagnosticLogger := e2elog.NewTestLogger(e2elog.InfoLevel) + + t.Cleanup(func() { + if !t.Failed() { + return + } + t.Log("collecting upgrade E2E diagnostics") + diagnosticCtx, diagnosticCancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer diagnosticCancel() + diagMode := os.Getenv(diagnostics.ModeEnvVar) + if diagMode == "" { + diagMode = diagnostics.ModeFile + } + diagDir := os.Getenv(diagnostics.DirEnvVar) + diagnostics.NewDiagCollector( + diagnosticClient, workloadNamespace, diagMode, diagDir, diagnosticLogger, + ).CollectAll(diagnosticCtx, t.Name()) + }) + + runner.mustRun(ctx, "k3d", "image", "import", + "grove-operator:"+currentImageTag, + "grove-initc:"+currentImageTag, + "grove-install-crds:"+currentImageTag, + "--cluster", clusterName, + ) + + if !t.Run("install released operator", func(t *testing.T) { + _, err := setup.InstallHelmChart(&setup.HelmInstallConfig{ + RestConfig: restConfig, + ReleaseName: releaseName, + ChartRef: releasedChart, + ChartVersion: fromVersion, + Namespace: operatorNamespace, + CreateNamespace: true, + Wait: true, + Timeout: 6 * time.Minute, + HelmLoggerFunc: t.Logf, + }) + require.NoError(t, err, "install released Grove chart") + + releasedOperatorImage := releasedImageRegistry + "/grove-operator:" + fromVersion + assertDeployment(t, ctx, dynamicClient, releasedOperatorImage, "", "") + }) { + t.FailNow() + } + + var originalPodUIDs []string + if !t.Run("create workload before upgrade", func(t *testing.T) { + applyManifestEventually(t, ctx, dynamicClient, diagnosticClient.RESTMapper(), compatibilityWorkload) + pods := waitForWorkload(t, ctx, dynamicClient, 1, 5*time.Minute) + for _, pod := range pods.Items { + originalPodUIDs = append(originalPodUIDs, string(pod.GetUID())) + } + + releasedInitImage := releasedImageRegistry + "/grove-initc:" + fromVersion + require.Truef(t, podUsesInitImage(pods, releasedInitImage), + "pre-upgrade workload does not use released init image %q", releasedInitImage) + }) { + t.FailNow() + } + + if !t.Run("upgrade to current checkout", func(t *testing.T) { + chartDir, err := setup.GetGroveChartDir() + require.NoError(t, err, "locate current Grove chart") + chartVersion, err := setup.GetGroveChartVersion(chartDir) + require.NoError(t, err, "read current Grove chart version") + _, err = setup.UpgradeHelmChart(&setup.HelmInstallConfig{ + RestConfig: restConfig, + ReleaseName: releaseName, + ChartRef: chartDir, + ChartVersion: chartVersion, + Namespace: operatorNamespace, + Wait: true, + ResetValues: true, + Timeout: 6 * time.Minute, + Values: map[string]any{ + "image": map[string]any{ + "repository": "grove-operator", + "tag": currentImageTag, + "pullPolicy": "IfNotPresent", + }, + "deployment": map[string]any{ + "env": []any{ + map[string]any{ + "name": "GROVE_INIT_CONTAINER_IMAGE", + "value": "grove-initc", + }, + }, + }, + "crdInstaller": map[string]any{ + "enabled": true, + "image": map[string]any{ + "repository": "grove-install-crds", + "tag": currentImageTag, + "pullPolicy": "IfNotPresent", + }, + }, + }, + HelmLoggerFunc: t.Logf, + }) + require.NoError(t, err, "upgrade Grove chart to current checkout") + + assertDeployment(t, ctx, dynamicClient, + "grove-operator:"+currentImageTag, + "grove-install-crds:"+currentImageTag, + "grove-initc", + ) + }) { + t.FailNow() + } + + if !t.Run("reconcile pre-upgrade workload", func(t *testing.T) { + patchPodCliqueSetEventually(t, ctx, dynamicClient, workloadName, []byte(`{"spec":{"replicas":2}}`)) + + pods := waitForWorkload(t, ctx, dynamicClient, 2, 5*time.Minute) + currentPodUIDs := make([]string, 0, len(pods.Items)) + for _, pod := range pods.Items { + currentPodUIDs = append(currentPodUIDs, string(pod.GetUID())) + } + require.Subsetf(t, currentPodUIDs, originalPodUIDs, + "pods created by %s were replaced during the operator upgrade", fromVersion) + expectedInitImage := "grove-initc:" + currentImageTag + require.Truef(t, podUsesInitImage(pods, expectedInitImage), + "scaled workload does not contain a pod created with current init image %q", expectedInitImage) + }) { + t.FailNow() + } +} + +func findOperatorDir(t *testing.T) string { + t.Helper() + _, filename, _, ok := runtime.Caller(0) + require.True(t, ok, "locate upgrade E2E source file") + return filepath.Clean(filepath.Join(filepath.Dir(filename), "..", "..", "..")) +} + +func requireCommands(t *testing.T, names ...string) { + t.Helper() + for _, name := range names { + _, err := exec.LookPath(name) + require.NoErrorf(t, err, "required command %q was not found in PATH", name) + } +} + +func durationFromEnv(t *testing.T, name string, fallback time.Duration) time.Duration { + t.Helper() + raw := strings.TrimSpace(os.Getenv(name)) + if raw == "" { + return fallback + } + value, err := time.ParseDuration(raw) + require.NoErrorf(t, err, "parse %s=%q as duration", name, raw) + return value +} + +func envBool(name string) bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv(name))) { + case "1", "true", "yes": + return true + default: + return false + } +} + +func latestGitHubRelease(ctx context.Context, t *testing.T, token string) string { + t.Helper() + client := github.NewClient(nil) + if token != "" { + client = client.WithAuthToken(token) + } + + release, _, err := client.Repositories.GetLatestRelease(ctx, githubOwner, githubRepository) + require.NoError(t, err, "get latest Grove release from GitHub") + tagName := strings.TrimSpace(release.GetTagName()) + require.NotEmpty(t, tagName, "latest Grove GitHub release did not contain tag_name") + return tagName +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value = strings.TrimSpace(value); value != "" { + return value + } + } + return "" +} + +func newKubernetesClients(t *testing.T, kubeconfig []byte) (*rest.Config, dynamic.Interface, *k8sclient.Client) { + t.Helper() + config, err := clientcmd.RESTConfigFromKubeConfig(kubeconfig) + require.NoError(t, err, "build Kubernetes client config") + config.UserAgent = "grove-upgrade-e2e" + + dynamicClient, err := dynamic.NewForConfig(config) + require.NoError(t, err, "create dynamic Kubernetes client") + diagnosticClient, err := k8sclient.New(config) + require.NoError(t, err, "create E2E Kubernetes client") + return config, dynamicClient, diagnosticClient +} + +func (r *commandRunner) mustRun(ctx context.Context, name string, args ...string) string { + r.t.Helper() + output, err := r.run(ctx, "", name, args...) + require.NoErrorf(r.t, err, "run %s %s\n%s", name, strings.Join(args, " "), output) + return output +} + +func (r *commandRunner) run(ctx context.Context, stdin, name string, args ...string) (string, error) { + r.t.Helper() + r.t.Logf("$ %s %s", name, strings.Join(args, " ")) + return r.runQuiet(ctx, stdin, name, args...) +} + +func (r *commandRunner) runQuiet(ctx context.Context, stdin, name string, args ...string) (string, error) { + r.t.Helper() + cmd := exec.CommandContext(ctx, name, args...) + cmd.Dir = r.workingDir + cmd.Env = r.environment + if stdin != "" { + cmd.Stdin = strings.NewReader(stdin) + } + var output bytes.Buffer + cmd.Stdout = &output + cmd.Stderr = &output + err := cmd.Run() + if ctxErr := ctx.Err(); ctxErr != nil { + return output.String(), ctxErr + } + return output.String(), err +} + +func applyManifestEventually(t *testing.T, ctx context.Context, client dynamic.Interface, mapper meta.RESTMapper, manifest string) { + t.Helper() + decoder := yaml.NewYAMLOrJSONDecoder(strings.NewReader(manifest), 4096) + for { + object := &unstructured.Unstructured{} + if err := decoder.Decode(object); err != nil { + if err == io.EOF { + return + } + require.NoError(t, err, "decode compatibility workload") + } + if len(object.Object) == 0 { + continue + } + applyObjectEventually(t, ctx, client, mapper, object, 2*time.Minute) + } +} + +func applyObjectEventually(t *testing.T, ctx context.Context, client dynamic.Interface, mapper meta.RESTMapper, object *unstructured.Unstructured, timeout time.Duration) { + t.Helper() + gvk := object.GroupVersionKind() + mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + require.NoErrorf(t, err, "resolve REST mapping for %s", gvk) + namespace := "" + if mapping.Scope.Name() == meta.RESTScopeNameNamespace { + namespace = object.GetNamespace() + require.NotEmptyf(t, namespace, "%s %q is namespaced but metadata.namespace is empty", gvk.Kind, object.GetName()) + } + payload, err := json.Marshal(object) + require.NoErrorf(t, err, "encode %s %s/%s", object.GetKind(), namespace, object.GetName()) + + pollCtx, cancel := context.WithCancel(ctx) + defer cancel() + require.EventuallyWithT(t, func(collect *assert.CollectT) { + _, err := client.Resource(mapping.Resource).Namespace(namespace).Patch(pollCtx, object.GetName(), types.ApplyPatchType, payload, metav1.PatchOptions{ + FieldManager: "grove-upgrade-e2e", + Force: new(true), + }) + assert.NoErrorf(collect, err, "apply %s %s/%s", object.GetKind(), namespace, object.GetName()) + }, timeout, 2*time.Second) +} + +func patchPodCliqueSetEventually(t *testing.T, ctx context.Context, client dynamic.Interface, name string, patch []byte) { + t.Helper() + pollCtx, cancel := context.WithCancel(ctx) + defer cancel() + require.EventuallyWithT(t, func(collect *assert.CollectT) { + _, err := client.Resource(podCliqueSetsGVR).Namespace(workloadNamespace).Patch( + pollCtx, name, types.MergePatchType, patch, metav1.PatchOptions{}, + ) + assert.NoErrorf(collect, err, "patch PodCliqueSet %s/%s", workloadNamespace, name) + }, 2*time.Minute, 2*time.Second) +} + +func waitForWorkload(t *testing.T, ctx context.Context, client dynamic.Interface, expectedReplicas int64, timeout time.Duration) unstructured.UnstructuredList { + t.Helper() + pollCtx, cancel := context.WithCancel(ctx) + defer cancel() + + var pods unstructured.UnstructuredList + require.EventuallyWithT(t, func(collect *assert.CollectT) { + pcs, err := client.Resource(podCliqueSetsGVR).Namespace(workloadNamespace).Get( + pollCtx, workloadName, metav1.GetOptions{}, + ) + require.NoError(collect, err, "get PodCliqueSet") + + podList, err := client.Resource(podsGVR).Namespace(workloadNamespace).List(pollCtx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/part-of=" + workloadName, + }) + require.NoError(collect, err, "list workload pods") + pods = *podList + + observedGeneration, found, err := unstructured.NestedInt64(pcs.Object, "status", "observedGeneration") + require.NoError(collect, err, "read status.observedGeneration") + require.True(collect, found, "status.observedGeneration is unset") + assert.Equal(collect, pcs.GetGeneration(), observedGeneration, "observed generation") + + specReplicas, found, err := unstructured.NestedInt64(pcs.Object, "spec", "replicas") + require.NoError(collect, err, "read spec.replicas") + require.True(collect, found, "spec.replicas is unset") + assert.Equal(collect, expectedReplicas, specReplicas, "spec.replicas") + + statusReplicas, found, err := unstructured.NestedInt64(pcs.Object, "status", "replicas") + require.NoError(collect, err, "read status.replicas") + require.True(collect, found, "status.replicas is unset") + assert.Equal(collect, expectedReplicas, statusReplicas, "status.replicas") + + availableReplicas, found, err := unstructured.NestedInt64(pcs.Object, "status", "availableReplicas") + require.NoError(collect, err, "read status.availableReplicas") + require.True(collect, found, "status.availableReplicas is unset") + assert.Equal(collect, expectedReplicas, availableReplicas, "status.availableReplicas") + + expectedPods := int(expectedReplicas) * 2 // compatibilityWorkload has two one-pod cliques per PCS replica. + assert.Len(collect, pods.Items, expectedPods, "workload pods") + for i := range pods.Items { + assert.Truef(collect, podContainersReady(&pods.Items[i]), "pod %q containers are not ready", pods.Items[i].GetName()) + } + }, timeout, 2*time.Second, "wait for workload to reach %d available replicas", expectedReplicas) + return pods +} + +func podContainersReady(pod *unstructured.Unstructured) bool { + statuses, found, err := unstructured.NestedSlice(pod.Object, "status", "containerStatuses") + if err != nil || !found || len(statuses) == 0 { + return false + } + for _, rawStatus := range statuses { + status, ok := rawStatus.(map[string]any) + if !ok { + return false + } + ready, found, err := unstructured.NestedBool(status, "ready") + if err != nil || !found || !ready { + return false + } + } + return true +} + +func assertDeployment(t *testing.T, ctx context.Context, client dynamic.Interface, operatorImage, crdInstallerImage, initRepository string) { + t.Helper() + deploy, err := client.Resource(deploymentsGVR).Namespace(operatorNamespace).Get( + ctx, "grove-operator", metav1.GetOptions{}, + ) + require.NoError(t, err, "get Grove deployment") + + operator := findContainer(deploy.Object, "grove-operator", "spec", "template", "spec", "containers") + require.Equal(t, operatorImage, imageOf(operator), "operator deployment image") + if crdInstallerImage != "" { + installer := findContainer(deploy.Object, "crd-installer", "spec", "template", "spec", "initContainers") + require.Equal(t, crdInstallerImage, imageOf(installer), "CRD installer image") + } + if initRepository != "" { + actual := containerEnvValue(operator, "GROVE_INIT_CONTAINER_IMAGE") + require.Equal(t, initRepository, actual, "GROVE_INIT_CONTAINER_IMAGE") + } +} + +func findContainer(object map[string]any, name string, fields ...string) map[string]any { + containers, found, err := unstructured.NestedSlice(object, fields...) + if err != nil || !found { + return nil + } + for _, rawContainer := range containers { + container, ok := rawContainer.(map[string]any) + if !ok { + continue + } + containerName, found, err := unstructured.NestedString(container, "name") + if err == nil && found && containerName == name { + return container + } + } + return nil +} + +func imageOf(container map[string]any) string { + image, _, _ := unstructured.NestedString(container, "image") + return image +} + +func containerEnvValue(container map[string]any, name string) string { + environment, found, err := unstructured.NestedSlice(container, "env") + if err != nil || !found { + return "" + } + for _, rawVariable := range environment { + variable, ok := rawVariable.(map[string]any) + if !ok { + continue + } + variableName, nameFound, nameErr := unstructured.NestedString(variable, "name") + value, valueFound, valueErr := unstructured.NestedString(variable, "value") + if nameErr == nil && valueErr == nil && nameFound && valueFound && variableName == name { + return value + } + } + return "" +} + +func podUsesInitImage(pods unstructured.UnstructuredList, image string) bool { + return slices.ContainsFunc(pods.Items, func(pod unstructured.Unstructured) bool { + initContainer := findContainer(pod.Object, "grove-initc", "spec", "initContainers") + return imageOf(initContainer) == image + }) +} + +const compatibilityWorkload = ` +apiVersion: v1 +kind: Namespace +metadata: + name: grove-upgrade-e2e +--- +apiVersion: grove.io/v1alpha1 +kind: PodCliqueSet +metadata: + name: upgrade-survivor + namespace: grove-upgrade-e2e +spec: + replicas: 1 + template: + cliqueStartupType: CliqueStartupTypeExplicit + cliques: + - name: bootstrap + spec: + roleName: bootstrap + replicas: 1 + minAvailable: 1 + podSpec: + containers: + - name: bootstrap + image: registry.k8s.io/pause:3.10 + - name: worker + spec: + roleName: worker + startsAfter: + - bootstrap + replicas: 1 + minAvailable: 1 + podSpec: + containers: + - name: worker + image: registry.k8s.io/pause:3.10 +` diff --git a/operator/go.mod b/operator/go.mod index d89b9a0be..ac8c834d4 100644 --- a/operator/go.mod +++ b/operator/go.mod @@ -8,6 +8,7 @@ require ( github.com/docker/docker v28.3.3+incompatible github.com/go-logr/logr v1.4.3 github.com/go-logr/zapr v1.3.0 + github.com/google/go-github/v86 v86.0.0 github.com/kai-scheduler/KAI-scheduler v0.15.2 github.com/open-policy-agent/cert-controller v0.14.0 github.com/samber/lo v1.52.0 @@ -75,6 +76,7 @@ require ( github.com/google/btree v1.1.3 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect + github.com/google/go-querystring v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/gosuri/uitable v0.0.4 // indirect diff --git a/operator/go.sum b/operator/go.sum index 3595b39a5..3301f4947 100644 --- a/operator/go.sum +++ b/operator/go.sum @@ -132,8 +132,13 @@ github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-github/v86 v86.0.0 h1:S/6aANJhwRm8EQmGKVML3j41yq0h2BsTP8FnDkO7kcA= +github.com/google/go-github/v86 v86.0.0/go.mod h1:zKv1l4SwDXNFMGByi2FWkq71KwSXqj/eQRZuqtmcot8= +github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0= +github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=