ai-dynamo · steved · Jul 14, 2026
@@ -111,6 +111,8 @@ jobs:
     # use NVIDIA self-hosted runner setting is on Velonix repository
     runs-on: prod-grove-e2e-v1
     timeout-minutes: 60
+    env:
+      GROVE_UPGRADE_E2E_CLUSTER_NAME: grove-upgrade-e2e-${{ github.run_id }}-${{ github.run_attempt }}
     strategy:
       fail-fast: false
       matrix:
@@ -137,6 +139,8 @@ jobs:
             test_pattern: "^Test_CRD_Installer"
           - test_name: resource_sharing
             test_pattern: "^Test_RS"
+          - test_name: upgrade_from_latest_release
+            make_target: "run-upgrade-e2e"
     name: E2E - ${{ matrix.test_name }}
     steps:
       # print runner specs so we have a record in case of failures
@@ -157,6 +161,8 @@ jobs:
         uses: ./.github/actions/e2e-setup
 
       - name: Run e2e tests - ${{ matrix.test_name }}
+        env:
+          GITHUB_TOKEN: ${{ matrix.test_name == 'upgrade_from_latest_release' && secrets.GITHUB_TOKEN || '' }}
         run: |
           make ${{ matrix.make_target || 'run-e2e-full' }} TEST_PATTERN='${{ matrix.test_pattern }}' E2E_CREATE_FLAGS='--dind-memory-mode'
         working-directory: operator
@@ -168,7 +174,11 @@ jobs:
         if: always()
         working-directory: operator
         run: |
-          make e2e-cluster-down || true
+          if [[ "${{ matrix.test_name }}" == "upgrade_from_latest_release" ]]; then
+            k3d cluster delete "$GROVE_UPGRADE_E2E_CLUSTER_NAME" || true
+          else
+            make e2e-cluster-down || true
+          fi
 
       # Upload diagnostic logs collected by debug_utils.go on test failure.
       # Uses 'warn' since this only runs on failure and missing files indicates a problem.
@@ -204,6 +214,7 @@ jobs:
           - test_name: auto_mnnvl
           - test_name: crd_installer
           - test_name: resource_sharing
+          - test_name: upgrade_from_latest_release
     name: E2E - ${{ matrix.test_name }}
     steps:
       - name: Skip E2E (no relevant changes)

@@ -124,6 +124,12 @@ run-e2e:
 	@echo "> Running e2e tests for operator"
 	@make --directory=operator run-e2e
 
+# Runs the standalone latest-release-to-current operator upgrade test.
+.PHONY: run-upgrade-e2e
+run-upgrade-e2e:
+	@echo "> Running operator upgrade e2e test"
+	@make --directory=operator run-upgrade-e2e
+
 # Runs all tests
 .PHONY: test
 test: test-unit

@@ -27,6 +27,9 @@ INSTALL_CRDS_NAME    := "grove-install-crds"
 K3S_IMAGE       ?= rancher/k3s:v1.35.5-k3s1
 E2E_REGISTRY_PORT ?= 5001
 E2E_CREATE_FLAGS  ?=
+ifndef DIAG_DIR
+DIAG_DIR := $(if $(GROVE_E2E_DIAG_DIR),$(GROVE_E2E_DIAG_DIR),$(MODULE_ROOT)/e2e-diagnostics)
+endif
 
 # Include tools targets
 include $(REPO_HACK_DIR)/tools.mk
@@ -131,6 +134,15 @@ run-e2e-verbose: $(GOTESTSUM)
 	@echo "> Running e2e tests (verbose)..."
 	@cd e2e && $(GOTESTSUM) --format short-verbose -- -count=1 -tags=e2e ./tests/... -timeout 45m $(if $(TEST_PATTERN),-run '$(TEST_PATTERN)')
 
+# Run the standalone upgrade test from the latest published GitHub release to
+# the operator built from the current checkout. The test owns its k3d cluster.
+.PHONY: run-upgrade-e2e
+run-upgrade-e2e: export GROVE_E2E_DIAG_DIR = $(DIAG_DIR)
+run-upgrade-e2e: export GROVE_E2E_DIAG_MODE = $(DIAG_MODE)
+run-upgrade-e2e: $(GOTESTSUM)
+	@echo "> Running e2e upgrade tests (verbose)..."
+	$(GOTESTSUM) --format short-verbose -- -count=1 -tags=e2e,e2eupgrade ./e2e/tests/upgrade/ -timeout 45m $(if $(TEST_PATTERN),-run '$(TEST_PATTERN)')
+
 # Create a k3d cluster for e2e testing (with Grove and Kai scheduler deployed)
 # This is optional - tests work with any Kubernetes cluster that has the required components
 .PHONY: e2e-cluster-up

@@ -133,7 +133,7 @@ func (c *GroveConfig) toHelmValues() (map[string]interface{}, error) {
 //
 // This approach avoids wasteful rebuilds while staying compatible with the Skaffold installation.
 func UpdateGroveConfiguration(ctx context.Context, restConfig *rest.Config, chartDir string, config *GroveConfig, logger *log.Logger) error {
-	chartVersion, err := getChartVersion(chartDir)
+	chartVersion, err := GetGroveChartVersion(chartDir)
 	if err != nil {
 		return fmt.Errorf("failed to get chart version: %w", err)
 	}
@@ -184,12 +184,12 @@ type chartYAML struct {
 	Version string `yaml:"version"`
 }
 
-// getChartVersion reads the version from Chart.yaml in the given chart directory.
+// GetGroveChartVersion reads the version from Chart.yaml in the given chart directory.
 // The chartDir parameter should be the path to a Helm chart directory. Chart.yaml is
 // a required file per the Helm chart specification and will always exist for valid charts.
 // We read from Chart.yaml rather than hardcoding the version to maintain a single source
 // of truth, avoiding configuration drift between the chart definition and the e2e test code.
-func getChartVersion(chartDir string) (string, error) {
+func GetGroveChartVersion(chartDir string) (string, error) {
 	chartFile := filepath.Join(chartDir, "Chart.yaml")
 	data, err := os.ReadFile(chartFile)
 	if err != nil {

@@ -65,6 +65,8 @@ type HelmInstallConfig struct {
 	RepoURL string
 	// ReuseValues reuses the last release's values and merges in the new values.
 	ReuseValues bool
+	// ResetValues resets to the values built into the new chart before applying Values.
+	ResetValues bool
 	// Timeout is the time to wait for Kubernetes operations (default: 5 minutes).
 	Timeout time.Duration
 }
@@ -323,6 +325,7 @@ func newUpgradeClient(actionConfig *action.Configuration, config *HelmInstallCon
 	client.Wait = config.Wait
 	client.Version = config.ChartVersion
 	client.ReuseValues = config.ReuseValues
+	client.ResetValues = config.ResetValues
 
 	// Set timeout
 	if config.Timeout > 0 {

@@ -21,7 +21,7 @@
 // These tests are disabled by default due to the 'e2e' build tag above.
 // To run these tests, use:
 //
-//	go test -tags=e2e ./e2e_testing/tests/...
+//	go test -tags=e2e ./e2e/tests/...
 //
 // Without the -tags=e2e flag, these tests will be skipped entirely.
 package tests