diff --git a/README.md b/README.md index a4212e7..5b69933 100644 --- a/README.md +++ b/README.md @@ -520,11 +520,9 @@ Put `e2e.yaml` in `scenarios//` to run that spec on Kind. Generate a skeleton with: ```sh -go test ./internal/e2e/ -tags e2e -run TestE2EFixtures/ -e2e.scaffold +go test -tags e2e ./internal/e2e -run TestE2EFixtures/ -e2e.scaffold ``` -`-e2e.preserve` keeps the namespace after a failure. - ### Build and run ```sh diff --git a/internal/e2e/cluster_plan.go b/internal/e2e/cluster_plan.go new file mode 100644 index 0000000..86de8a0 --- /dev/null +++ b/internal/e2e/cluster_plan.go @@ -0,0 +1,103 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "errors" + "sync" +) + +var ( + errClusterExists = errors.New("cluster already exists") + errClusterUnknown = errors.New("cluster lookup failed") +) + +// ClusterIntent is what this process may do with a named Kind cluster. +type ClusterIntent int + +const ( + // ClusterCreate means the name is free. Register cleanup, then create. + ClusterCreate ClusterIntent = iota + // ClusterReplace means force was set. Delete the old cluster, then create. + ClusterReplace + // ClusterRefuse means the name is taken and this run must not delete it. + ClusterRefuse + // ClusterUncertain means the lookup failed. Leave the name alone. + ClusterUncertain +) + +// PlanCluster decides whether this run may create or delete a Kind cluster. +func PlanCluster(found bool, getErr error, force bool) ClusterIntent { + // A lookup error stays uncertain even when found is also set. + if getErr != nil { + return ClusterUncertain + } + if found && !force { + return ClusterRefuse + } + if found && force { + return ClusterReplace + } + return ClusterCreate +} + +// testCleanup is the part of a test used to register teardown. +type testCleanup interface { + Helper() + Cleanup(func()) + Errorf(string, ...any) +} + +// OwnCluster registers down once, before create. +func OwnCluster(tb testCleanup, down func() error) { + tb.Helper() + // down still runs when the test stops early. + var once sync.Once + tb.Cleanup(func() { + once.Do(func() { + if down == nil { + return + } + if err := down(); err != nil { + tb.Errorf("cluster down: %v", err) + } + }) + }) +} + +// PrepareCluster runs a PlanCluster result. +func PrepareCluster(tb testCleanup, intent ClusterIntent, down, up func() error) error { + tb.Helper() + // Cleanup is registered before create. Refuse and uncertain do not delete. + switch intent { + case ClusterRefuse: + return errClusterExists + case ClusterUncertain: + return errClusterUnknown + case ClusterReplace: + if err := down(); err != nil { + return err + } + fallthrough + case ClusterCreate: + OwnCluster(tb, down) + if up == nil { + return nil + } + return up() + default: + return errClusterUnknown + } +} diff --git a/internal/e2e/cluster_plan_test.go b/internal/e2e/cluster_plan_test.go new file mode 100644 index 0000000..4fd2e7e --- /dev/null +++ b/internal/e2e/cluster_plan_test.go @@ -0,0 +1,107 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPlanCluster(t *testing.T) { + t.Parallel() + lookupErr := errors.New("timeout") + tests := []struct { + name string + found bool + err error + force bool + want ClusterIntent + }{ + {name: "absent", want: ClusterCreate}, + {name: "exists without force", found: true, want: ClusterRefuse}, + {name: "exists with force", found: true, force: true, want: ClusterReplace}, + {name: "lookup error", err: lookupErr, want: ClusterUncertain}, + {name: "lookup error ignores force", err: lookupErr, force: true, want: ClusterUncertain}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, PlanCluster(tt.found, tt.err, tt.force)) + }) + } +} + +type fakeCleanup struct { + fns []func() + errors []string +} + +func (f *fakeCleanup) Helper() {} +func (f *fakeCleanup) Cleanup(fn func()) { f.fns = append(f.fns, fn) } +func (f *fakeCleanup) Errorf(format string, args ...any) { + f.errors = append(f.errors, fmt.Sprintf(format, args...)) +} + +func TestPrepareCluster(t *testing.T) { + t.Parallel() + boom := errors.New("boom") + tests := []struct { + name string + intent ClusterIntent + upErr error + downErr error + wantErr string + wantDown string + wantLog []string + cleanups int + }{ + {name: "create then clean up", intent: ClusterCreate, wantLog: []string{"up", "down"}, cleanups: 1}, + {name: "create failure still cleans up", intent: ClusterCreate, upErr: errors.New("status"), wantErr: "status", wantLog: []string{"up", "down"}, cleanups: 1}, + {name: "cleanup reports a down error", intent: ClusterCreate, downErr: boom, wantDown: "cluster down: boom", wantLog: []string{"up", "down"}, cleanups: 1}, + {name: "replace deletes before create", intent: ClusterReplace, wantLog: []string{"down", "up", "down"}, cleanups: 1}, + {name: "refuse does not delete", intent: ClusterRefuse, wantErr: "cluster already exists", cleanups: 0}, + {name: "uncertain does not delete", intent: ClusterUncertain, wantErr: "cluster lookup failed", cleanups: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + var log []string + fake := &fakeCleanup{} + err := PrepareCluster(fake, tt.intent, + func() error { log = append(log, "down"); return tt.downErr }, + func() error { log = append(log, "up"); return tt.upErr }, + ) + gotErr := "" + if err != nil { + gotErr = err.Error() + } + require.Len(t, fake.fns, tt.cleanups) + for _, fn := range fake.fns { + fn() + } + for _, fn := range fake.fns { + fn() + } + assert.Equal(t, tt.wantErr, gotErr) + assert.Equal(t, tt.wantLog, log) + assert.Equal(t, tt.wantDown, strings.Join(fake.errors, "\n")) + }) + } +} diff --git a/internal/e2e/cluster_test.go b/internal/e2e/cluster_test.go new file mode 100644 index 0000000..f02080a --- /dev/null +++ b/internal/e2e/cluster_test.go @@ -0,0 +1,412 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build e2e + +package e2e_test + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "testing" + "time" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/e2e-framework/klient/k8s/resources" + "sigs.k8s.io/e2e-framework/klient/wait" + + "deployah.dev/deployah/internal/testing/e2e" + + planCmd "deployah.dev/deployah/internal/cmd/plan" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// releaseTarget is one Helm release the scenario cleanup uninstalls. +type releaseTarget struct { + Project string + Env string +} + +func execCommand(t *testing.T) e2e.ExecFunc { + t.Helper() + return func(ctx context.Context, dir string, args []string) e2e.CommandResult { + stdout, stderr, err := runInErrContext(t, ctx, dir, args...) + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return e2e.CommandResult{Stdout: stdout, Stderr: stderr, Interrupted: true} + } + code := 0 + if err != nil { + code = 1 + if errors.Is(err, planCmd.ErrChangesPresent) { + code = 2 + } + } + retryable := len(args) > 0 && args[0] == "logs" && code != 0 && isRetryableAPIError(err) + return e2e.CommandResult{Stdout: stdout, Stderr: stderr, ExitCode: code, Retryable: retryable} + } +} + +func (env *e2eEnv) claimNamespace(ctx context.Context, name string) (e2e.Claim, error) { + runID := rand.Text() + ns := &corev1.Namespace{} + ns.Name = name + ns.Annotations = map[string]string{e2e.RunAnnotation: runID} + res, err := env.newResources("") + if err != nil { + return e2e.Claim{}, err + } + createErr := res.Create(ctx, ns) + class := classifyCreate(createErr) + if class == e2e.CreateRejected { + return e2e.Claim{}, fmt.Errorf("create namespace %s: %w", name, createErr) + } + var got *e2e.NamespaceView + var getErr error + if class == e2e.CreateSucceeded && ns.UID != "" { + run := "" + if ns.Annotations != nil { + run = ns.Annotations[e2e.RunAnnotation] + } + got = &e2e.NamespaceView{Name: ns.Name, UID: string(ns.UID), Run: run} + } + for attempt := range e2e.NamespaceReconcileAttempts { + if attempt > 0 || got == nil { + got, getErr = env.namespaceView(ctx, name) + } + decision := e2e.ResolveNamespaceCreate(runID, name, class, got, getErr) + if decision.Owned { + return decision.Claim, nil + } + if !decision.Retry { + if decision.Err != nil { + return e2e.Claim{}, decision.Err + } + return e2e.Claim{}, fmt.Errorf("namespace %s: not owned", name) + } + if attempt == e2e.NamespaceReconcileAttempts-1 { + return e2e.Claim{}, e2e.FinishUncertain(name, createErr, getErr) + } + select { + case <-ctx.Done(): + return e2e.Claim{}, e2e.FinishUncertain(name, createErr, ctx.Err()) + case <-time.After(e2e.NamespaceReconcileInterval): + } + } + return e2e.Claim{}, e2e.FinishUncertain(name, createErr, getErr) +} + +func classifyCreate(err error) e2e.CreateClass { + switch { + case err == nil: + return e2e.CreateSucceeded + case apierrors.IsAlreadyExists(err): + return e2e.CreateAlreadyExists + case apierrors.IsInvalid(err), apierrors.IsForbidden(err), apierrors.IsBadRequest(err): + return e2e.CreateRejected + default: + return e2e.CreateAmbiguous + } +} + +func (env *e2eEnv) namespaceView(ctx context.Context, name string) (*e2e.NamespaceView, error) { + res, err := env.newResources("") + if err != nil { + return nil, err + } + ns := &corev1.Namespace{} + if err = res.Get(ctx, name, "", ns); err != nil { + if apierrors.IsNotFound(err) { + return nil, e2e.ErrNotFound + } + return nil, err + } + return &e2e.NamespaceView{ + Name: ns.Name, + UID: string(ns.UID), + Run: ns.Annotations[e2e.RunAnnotation], + }, nil +} + +func (env *e2eEnv) cleanupClaim(t *testing.T, dir string, claim e2e.Claim, releases []releaseTarget) error { + t.Helper() + phases := make([]e2e.CleanupPhase, 0, len(releases)+3) + for _, rel := range releases { + phases = append(phases, e2e.CleanupPhase{ + Name: "delete " + rel.Project + "/" + rel.Env, + Run: env.gated(claim, env.deleteRelease(t, dir, claim.Name, rel)), + }) + } + phases = append(phases, + e2e.CleanupPhase{Name: "workloads", Run: env.gated(claim, func(ctx context.Context) error { + return env.deleteWorkloads(ctx, claim.Name) + })}, + e2e.CleanupPhase{Name: "pvcs", Run: env.gated(claim, func(ctx context.Context) error { + return env.deletePVCs(ctx, claim.Name) + })}, + e2e.CleanupPhase{Name: "namespace", Run: env.gated(claim, func(ctx context.Context) error { + return env.deleteNamespaceObject(ctx, claim) + })}, + ) + return e2e.RunCleanup(context.Background(), phases, e2e.WithPhaseTimeout(namespaceWaitTimeout)) +} + +func (env *e2eEnv) gated(claim e2e.Claim, fn func(context.Context) error) func(context.Context) error { + return func(ctx context.Context) error { + view, err := env.namespaceView(ctx, claim.Name) + if confirmErr := e2e.ConfirmNamespaceOwner(claim, view, err); confirmErr != nil { + if errors.Is(confirmErr, e2e.ErrNamespaceGone) { + return nil + } + return confirmErr + } + return fn(ctx) + } +} + +func (env *e2eEnv) deleteRelease(t *testing.T, dir, namespace string, rel releaseTarget) func(context.Context) error { + t.Helper() + return func(ctx context.Context) error { + _, _, err := runInErrContext(t, ctx, dir, "delete", rel.Project, rel.Env, + "--yes", "--wait", "--allow-missing-platform", + "--context", kindContext, "--namespace", namespace) + return err + } +} + +func (env *e2eEnv) deleteNamespaceObject(ctx context.Context, claim e2e.Claim) error { + pre, err := e2e.UIDPrecondition(claim.UID) + if err != nil { + return err + } + res, err := env.newResources("") + if err != nil { + return err + } + ns := &corev1.Namespace{} + ns.Name = claim.Name + err = res.Delete(ctx, ns, func(opts *metav1.DeleteOptions) { + opts.Preconditions = &pre + }) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + return wait.For(func(waitCtx context.Context) (bool, error) { + getErr := res.Get(waitCtx, claim.Name, "", ns) + if apierrors.IsNotFound(getErr) { + return true, nil + } + if getErr != nil && !isRetryableAPIError(getErr) { + return false, getErr + } + return false, nil + }, wait.WithTimeout(namespaceWaitTimeout), wait.WithInterval(resourcePollInterval), + wait.WithContext(ctx), wait.WithImmediate()) +} + +func (env *e2eEnv) deleteWorkloads(ctx context.Context, namespace string) error { + res, err := env.newResources(namespace) + if err != nil { + return err + } + var errs []error + var cronjobs batchv1.CronJobList + if listErr := res.List(ctx, &cronjobs); listErr != nil && !apierrors.IsNotFound(listErr) { + errs = append(errs, fmt.Errorf("list cronjobs: %w", listErr)) + } + for i := range cronjobs.Items { + if delErr := res.Delete(ctx, &cronjobs.Items[i]); delErr != nil && !apierrors.IsNotFound(delErr) { + errs = append(errs, fmt.Errorf("delete cronjob %s: %w", cronjobs.Items[i].Name, delErr)) + } + } + var jobs batchv1.JobList + if listErr := res.List(ctx, &jobs); listErr != nil && !apierrors.IsNotFound(listErr) { + errs = append(errs, fmt.Errorf("list jobs: %w", listErr)) + } + for i := range jobs.Items { + if delErr := res.Delete(ctx, &jobs.Items[i]); delErr != nil && !apierrors.IsNotFound(delErr) { + errs = append(errs, fmt.Errorf("delete job %s: %w", jobs.Items[i].Name, delErr)) + } + } + var sets appsv1.StatefulSetList + if listErr := res.List(ctx, &sets); listErr != nil && !apierrors.IsNotFound(listErr) { + errs = append(errs, fmt.Errorf("list statefulsets: %w", listErr)) + } + for i := range sets.Items { + if delErr := res.Delete(ctx, &sets.Items[i]); delErr != nil && !apierrors.IsNotFound(delErr) { + errs = append(errs, fmt.Errorf("delete statefulset %s: %w", sets.Items[i].Name, delErr)) + } + } + var deploys appsv1.DeploymentList + if listErr := res.List(ctx, &deploys); listErr != nil && !apierrors.IsNotFound(listErr) { + errs = append(errs, fmt.Errorf("list deployments: %w", listErr)) + } + for i := range deploys.Items { + if delErr := res.Delete(ctx, &deploys.Items[i]); delErr != nil && !apierrors.IsNotFound(delErr) { + errs = append(errs, fmt.Errorf("delete deployment %s: %w", deploys.Items[i].Name, delErr)) + } + } + waitErr := wait.For(func(waitCtx context.Context) (bool, error) { + var pods corev1.PodList + if listErr := res.List(waitCtx, &pods); listErr != nil { + if apierrors.IsNotFound(listErr) { + return true, nil + } + if isRetryableAPIError(listErr) { + return false, nil + } + return false, listErr + } + return len(pods.Items) == 0, nil + }, wait.WithTimeout(namespaceWaitTimeout), wait.WithInterval(resourcePollInterval), + wait.WithContext(ctx), wait.WithImmediate()) + if waitErr != nil { + errs = append(errs, fmt.Errorf("wait for pods: %w", waitErr)) + } + return errors.Join(errs...) +} + +func (env *e2eEnv) deletePVCs(ctx context.Context, namespace string) error { + res, err := env.newResources(namespace) + if err != nil { + return err + } + var pvcs corev1.PersistentVolumeClaimList + if listErr := res.List(ctx, &pvcs); listErr != nil { + if apierrors.IsNotFound(listErr) { + return nil + } + return fmt.Errorf("list pvcs: %w", listErr) + } + var errs []error + for i := range pvcs.Items { + if delErr := res.Delete(ctx, &pvcs.Items[i]); delErr != nil && !apierrors.IsNotFound(delErr) { + errs = append(errs, fmt.Errorf("delete pvc %s: %w", pvcs.Items[i].Name, delErr)) + } + } + waitErr := wait.For(func(waitCtx context.Context) (bool, error) { + var remaining corev1.PersistentVolumeClaimList + if listErr := res.List(waitCtx, &remaining); listErr != nil { + if apierrors.IsNotFound(listErr) { + return true, nil + } + if isRetryableAPIError(listErr) { + return false, nil + } + return false, listErr + } + return len(remaining.Items) == 0, nil + }, wait.WithTimeout(namespaceWaitTimeout), wait.WithInterval(resourcePollInterval), + wait.WithContext(ctx), wait.WithImmediate()) + if waitErr != nil { + errs = append(errs, fmt.Errorf("wait for pvcs: %w", waitErr)) + } + return errors.Join(errs...) +} + +func (env *e2eEnv) observe(ctx context.Context, sel e2e.ResourceSelect, scenarioNS string) ([]map[string]any, error) { + gvk, err := parseSelectGVK(sel) + if err != nil { + return nil, fmt.Errorf("%w: %w", e2e.ErrObserveTerminal, err) + } + namespaced, err := env.isNamespaced(gvk) + if err != nil { + return nil, wrapObserve(err) + } + lookupNS := "" + if namespaced { + lookupNS = scenarioNS + if sel.Namespace != "" { + lookupNS = sel.Namespace + } + } + res, err := env.newResources(lookupNS) + if err != nil { + return nil, fmt.Errorf("%w: %w", e2e.ErrObserveTerminal, err) + } + if sel.Name != "" { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(gvk) + getErr := res.Get(ctx, sel.Name, lookupNS, obj) + if apierrors.IsNotFound(getErr) { + return []map[string]any{}, nil + } + if getErr != nil { + return nil, wrapObserve(getErr) + } + return []map[string]any{obj.Object}, nil + } + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(schema.GroupVersionKind{ + Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind + "List", + }) + var opts []resources.ListOption + if len(sel.Labels) > 0 { + opts = append(opts, resources.WithLabelSelector(labels.Set(sel.Labels).String())) + } + if listErr := res.List(ctx, list, opts...); listErr != nil { + if apierrors.IsNotFound(listErr) { + return []map[string]any{}, nil + } + return nil, wrapObserve(listErr) + } + out := make([]map[string]any, 0, len(list.Items)) + for i := range list.Items { + out = append(out, list.Items[i].Object) + } + return out, nil +} + +func wrapObserve(err error) error { + if isRetryableAPIError(err) { + return fmt.Errorf("%w: %w", e2e.ErrObserveRetry, err) + } + return fmt.Errorf("%w: %w", e2e.ErrObserveTerminal, err) +} + +func parseSelectGVK(sel e2e.ResourceSelect) (schema.GroupVersionKind, error) { + if sel.Kind == "" || sel.APIVersion == "" { + return schema.GroupVersionKind{}, errors.New("select requires apiVersion and kind") + } + gv, err := schema.ParseGroupVersion(sel.APIVersion) + if err != nil { + return schema.GroupVersionKind{}, fmt.Errorf("parse apiVersion %q: %w", sel.APIVersion, err) + } + return gv.WithKind(sel.Kind), nil +} + +func (env *e2eEnv) isNamespaced(gvk schema.GroupVersionKind) (bool, error) { + env.mapperMu.Lock() + defer env.mapperMu.Unlock() + mapping, err := env.mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if meta.IsNoMatchError(err) { + env.mapper.Reset() + mapping, err = env.mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + } + if err != nil { + return false, fmt.Errorf("RESTMapping %s: %w", gvk.String(), err) + } + return mapping.Scope.Name() == meta.RESTScopeNameNamespace, nil +} diff --git a/internal/e2e/e2e_crd_test.go b/internal/e2e/crd_test.go similarity index 71% rename from internal/e2e/e2e_crd_test.go rename to internal/e2e/crd_test.go index 6e9119b..81496d9 100644 --- a/internal/e2e/e2e_crd_test.go +++ b/internal/e2e/crd_test.go @@ -19,8 +19,11 @@ package e2e_test import ( "context" "encoding/json" + "errors" + "fmt" "os" "path/filepath" + "runtime" "strings" "testing" "time" @@ -28,6 +31,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -55,40 +59,45 @@ var crdLifecycleGVR = schema.GroupVersionResource{ // TestCRDLifecycle checks Helm-native chart CRD install, upgrade, and // uninstall. File mutation cannot be expressed in e2e.yaml. -func (s *E2ESuite) TestCRDLifecycle() { - t := s.T() - src := filepath.Join(s.scenariosDir, "crd-lifecycle") +// Failures in this test should point here, not at the suite caller. +// +//nolint:thelper +func runCRDLifecycle(t *testing.T, env *e2eEnv) { + src := filepath.Join(env.scenariosDir, "crd-lifecycle") require.DirExists(t, src) dir, err := inttest.NewScenarioWorkspace(t, src) require.NoError(t, err) - ns := fixtureNamespace("crd-lifecycle") - s.createNamespace(t, ns) - - restCfg := kubeRESTConfig(t, s.kcPath, kindContext) + restCfg := kubeRESTConfig(t, env.kcPath, kindContext) ext := newApiextensionsClient(t, restCfg) dyn := newDynamicClient(t, restCfg) + + ns := fixtureNamespace("crd-lifecycle") + var crdUID types.UID + claim, err := env.claimNamespace(t.Context(), ns) + require.NoError(t, err) t.Cleanup(func() { - // t.Context() is canceled before Cleanup; teardown needs its own. - cleanupCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), namespaceWaitTimeout) defer cancel() - if delCRDErr := ext.ApiextensionsV1().CustomResourceDefinitions().Delete( - cleanupCtx, crdLifecycleName, metav1.DeleteOptions{}); delCRDErr != nil { - t.Logf("cleanup CRD delete failed (non-fatal): %v", delCRDErr) + var crdErr error + if crdUID != "" { + crdErr = deleteOwnedCRD(ctx, ext, crdLifecycleName, crdUID) } - if _, _, delErr := runInErrContext(t, cleanupCtx, dir, "delete", "crd-lifecycle", "dev", - "--yes", "--wait", "--allow-missing-platform", - "--context", kindContext, "--namespace", ns); delErr != nil { - t.Logf("cleanup delete failed (non-fatal): %v", delErr) + nsErr := env.cleanupClaim(t, dir, claim, []releaseTarget{{ + Project: "crd-lifecycle", Env: "dev", + }}) + if joined := errors.Join(crdErr, nsErr); joined != nil { + t.Errorf("cleanup: %v", joined) } - s.deleteNamespace(t, ns) }) + require.NoError(t, requireCRDAbsent(t.Context(), ext, crdLifecycleName)) widget := hideExtrasManifest(t, dir, "widget.yaml") runIn(t, dir, "deploy", "dev", "--context", kindContext, "--namespace", ns) - crd := waitCRDEstablished(t, ext, crdLifecycleName) + crd := waitCRDEstablished(t, ext, crdLifecycleName, &crdUID) + require.NotEmpty(t, crdUID) assertCRDUserMetadata(t, crd) require.NoError(t, os.WriteFile( @@ -122,70 +131,58 @@ func (s *E2ESuite) TestCRDLifecycle() { require.NoError(t, err, "CRD must survive deployah delete") } -const ( - crdAddedName = "addedwidgets.example.com" - crdAddedYAML = `apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: addedwidgets.example.com -spec: - group: example.com - scope: Namespaced - names: - kind: AddedWidget - plural: addedwidgets - singular: addedwidget - versions: - - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object -` -) +const crdAddedName = "addedwidgets.example.com" // TestCRDNewlyAddedOnUpgrade checks that a CRD file added after the // first install is not installed by an ordinary Helm Upgrade. -func (s *E2ESuite) TestCRDNewlyAddedOnUpgrade() { - t := s.T() - src := filepath.Join(s.scenariosDir, "crd-lifecycle") +// Failures in this test should point here, not at the suite caller. +// +//nolint:thelper +func runCRDNewlyAddedOnUpgrade(t *testing.T, env *e2eEnv) { + src := filepath.Join(env.scenariosDir, "crd-lifecycle") require.DirExists(t, src) dir, err := inttest.NewScenarioWorkspace(t, src) require.NoError(t, err) - ns := fixtureNamespace("crd-added") - s.createNamespace(t, ns) - - restCfg := kubeRESTConfig(t, s.kcPath, kindContext) + restCfg := kubeRESTConfig(t, env.kcPath, kindContext) ext := newApiextensionsClient(t, restCfg) + + ns := fixtureNamespace("crd-added") + var crdUID types.UID + claim, err := env.claimNamespace(t.Context(), ns) + require.NoError(t, err) t.Cleanup(func() { - cleanupCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), namespaceWaitTimeout) defer cancel() - for _, name := range []string{crdLifecycleName, crdAddedName} { - if delCRDErr := ext.ApiextensionsV1().CustomResourceDefinitions().Delete( - cleanupCtx, name, metav1.DeleteOptions{}); delCRDErr != nil { - t.Logf("cleanup CRD delete failed (non-fatal): %v", delCRDErr) - } + var crdErr error + if crdUID != "" { + crdErr = deleteOwnedCRD(ctx, ext, crdLifecycleName, crdUID) } - if _, _, delErr := runInErrContext(t, cleanupCtx, dir, "delete", "crd-lifecycle", "dev", - "--yes", "--wait", "--allow-missing-platform", - "--context", kindContext, "--namespace", ns); delErr != nil { - t.Logf("cleanup delete failed (non-fatal): %v", delErr) + nsErr := env.cleanupClaim(t, dir, claim, []releaseTarget{{ + Project: "crd-lifecycle", Env: "dev", + }}) + if joined := errors.Join(crdErr, nsErr); joined != nil { + t.Errorf("cleanup: %v", joined) } - s.deleteNamespace(t, ns) }) + require.NoError(t, requireCRDAbsent(t.Context(), ext, crdLifecycleName)) + require.NoError(t, requireCRDAbsent(t.Context(), ext, crdAddedName)) hideExtrasManifest(t, dir, "widget.yaml") runIn(t, dir, "deploy", "dev", "--context", kindContext, "--namespace", ns) - waitCRDEstablished(t, ext, crdLifecycleName) + waitCRDEstablished(t, ext, crdLifecycleName, &crdUID) + require.NotEmpty(t, crdUID) assertReleaseExists(t, dir, ns, "crd-lifecycle", "dev") - require.NoError(t, os.WriteFile( + _, sourceFile, _, ok := runtime.Caller(0) + require.True(t, ok) + added, err := os.ReadFile(filepath.Join(filepath.Dir(sourceFile), "testdata", "added-widget-crd.yaml")) // #nosec G304 -- package testdata + require.NoError(t, err) + require.NoError(t, os.WriteFile( // #nosec G703 -- path under the copied workspace filepath.Join(dir, ".deployah", "crds", "addedwidget.yaml"), - []byte(crdAddedYAML), 0o600)) + added, 0o600)) runIn(t, dir, "deploy", "dev", "--context", kindContext, "--namespace", ns) @@ -195,6 +192,30 @@ func (s *E2ESuite) TestCRDNewlyAddedOnUpgrade() { "Helm upgrade must not install a CRD added after first install, got: %v", err) } +func requireCRDAbsent(ctx context.Context, ext apiextensionsclient.Interface, name string) error { + _, err := ext.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf("crd %s: ownership uncertain: %w", name, err) + } + return fmt.Errorf("crd %s already exists, not owned by this run", name) +} + +func deleteOwnedCRD(ctx context.Context, ext apiextensionsclient.Interface, name string, uid types.UID) error { + if uid == "" { + return fmt.Errorf("crd %s: ownership uncertain, not deleting", name) + } + err := ext.ApiextensionsV1().CustomResourceDefinitions().Delete(ctx, name, metav1.DeleteOptions{ + Preconditions: &metav1.Preconditions{UID: &uid}, + }) + if apierrors.IsNotFound(err) { + return nil + } + return err +} + func assertReleaseExists(t *testing.T, dir, ns, project, env string) { t.Helper() stdout, _ := runIn(t, dir, "status", project, @@ -268,7 +289,7 @@ func getCRD(tb testing.TB, ext apiextensionsclient.Interface, name string) *apie return crd } -func waitCRDEstablished(tb testing.TB, ext apiextensionsclient.Interface, name string) *apiextensionsv1.CustomResourceDefinition { +func waitCRDEstablished(tb testing.TB, ext apiextensionsclient.Interface, name string, uid *types.UID) *apiextensionsv1.CustomResourceDefinition { tb.Helper() var latest *apiextensionsv1.CustomResourceDefinition require.NoError(tb, wait.For(func(ctx context.Context) (bool, error) { @@ -281,6 +302,9 @@ func waitCRDEstablished(tb testing.TB, ext apiextensionsclient.Interface, name s return false, err } latest = crd + if uid != nil && *uid == "" && crd.UID != "" { + *uid = crd.UID + } for _, cond := range crd.Status.Conditions { if cond.Type == apiextensionsv1.Established && cond.Status == apiextensionsv1.ConditionTrue { diff --git a/internal/e2e/doc.go b/internal/e2e/doc.go index cf7d3be..bf99c8b 100644 --- a/internal/e2e/doc.go +++ b/internal/e2e/doc.go @@ -1,4 +1,4 @@ -// Copyright 2025 The Deployah Authors +// Copyright 2026 The Deployah Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package e2e drives the Deployah CLI against a live Kind cluster. +// Package e2e runs Deployah scenarios on a Kind cluster. // -// Scenarios run from a private directory copy. Use `nix run .#test-e2e`. -// This file has no build tag so go list can still see the package. +// Step execution lives in internal/testing/e2e. This package wires the +// in-process CLI and the Kubernetes client. Use `nix run .#test-e2e`. package e2e diff --git a/internal/e2e/e2e_assert_test.go b/internal/e2e/e2e_assert_test.go deleted file mode 100644 index 96dfc4d..0000000 --- a/internal/e2e/e2e_assert_test.go +++ /dev/null @@ -1,704 +0,0 @@ -// Copyright 2025 The Deployah Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build e2e - -package e2e_test - -import ( - "context" - "errors" - "fmt" - "maps" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/require" - "go.yaml.in/yaml/v3" - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/e2e-framework/klient/k8s/resources" - "sigs.k8s.io/e2e-framework/klient/wait" - - inttest "deployah.dev/deployah/internal/testing" - appsv1 "k8s.io/api/apps/v1" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - netutil "k8s.io/apimachinery/pkg/util/net" -) - -const ( - resourcePollInterval = 2 * time.Second - logsRetryInterval = 5 * time.Second - namespaceWaitTimeout = 2 * time.Minute - e2eSchemaModeLine = "# $schema: ../../internal/testing/e2e.schema.json" -) - -func (s *E2ESuite) assertE2EFixture(t *testing.T, dir, project, ns string, fx inttest.E2EFixture) { - t.Helper() - if len(fx.Steps) > 0 { - for i, step := range fx.Steps { - s.executeStep(t, dir, project, ns, fx, i, step) - } - return - } - s.executeStep(t, dir, project, ns, fx, 0, inttest.Step{ - Deploy: &inttest.DeployOp{}, - Resources: fx.Resources, - }) - if *flagScaffold { - s.scaffoldSimple(t, dir, ns, fx) - } -} - -func (s *E2ESuite) executeStep(t *testing.T, dir, project, ns string, fx inttest.E2EFixture, index int, step inttest.Step) { - t.Helper() - timeout, err := fx.StepTimeout(step) - require.NoErrorf(t, err, "steps[%d] timeout", index) - args := stepArgs(project, fx.Env, ns, step) - - if step.Logs != nil { - s.retryLogs(t, dir, args, step.Logs.Contains, timeout) - } else { - stdout, stderr := runIn(t, dir, args...) - if step.StdoutContains != "" { - require.Containsf(t, stdout, step.StdoutContains, "steps[%d] stdout", index) - } - if step.StderrContains != "" { - require.Containsf(t, stderr, step.StderrContains, "steps[%d] stderr", index) - } - } - - if len(step.Resources) == 0 { - return - } - s.waitForResources(t, ns, step.Resources, timeout) -} - -func (s *E2ESuite) retryLogs(t *testing.T, dir string, args []string, contains string, timeout time.Duration) { - t.Helper() - deadline := time.Now().Add(timeout) - var stdout, stderr string - var runErr error - for { - stdout, stderr, runErr = runInErr(t, dir, args...) - if runErr == nil && strings.Contains(stdout, contains) { - return - } - if time.Now().After(deadline) { - break - } - select { - case <-t.Context().Done(): - require.Fail(t, "logs retry canceled", "want %q in stdout\nstdout:\n%s\nstderr:\n%s\nerr: %v", - contains, stdout, stderr, runErr) - case <-time.After(logsRetryInterval): - } - } - require.Fail(t, "logs never matched", "want %q in stdout within %s\nstdout:\n%s\nstderr:\n%s\nerr: %v", - contains, timeout, stdout, stderr, runErr) -} - -func stepArgs(project, env, ns string, step inttest.Step) []string { - switch step.OpName() { - case "deploy": - args := []string{"deploy", env, "--context", kindContext, "--namespace", ns} - if step.Deploy != nil && step.Deploy.Spec != "" { - args = append(args, "--spec", step.Deploy.Spec) - } - if step.Deploy != nil { - args = append(args, step.Deploy.Args...) - } - return args - case "run": - return []string{"run", step.Run.Task, env, "--context", kindContext, "--yes", "--namespace", ns} - case "logs": - return []string{ - "logs", project, - "--component=" + step.Logs.Component, - "--environment=" + env, - "--no-follow", - "--context", kindContext, - "--namespace", ns, - } - case "delete": - return []string{ - "delete", project, env, - "--yes", "--wait", "--allow-missing-platform", - "--context", kindContext, - "--namespace", ns, - } - default: - return nil - } -} - -func (s *E2ESuite) waitForResources(t *testing.T, ns string, assertions []inttest.ResourceAssertion, timeout time.Duration) { - t.Helper() - var last []string - err := wait.For(func(ctx context.Context) (bool, error) { - var all []string - for i, ra := range assertions { - diffs, checkErr := s.checkAssertion(ctx, ns, ra) - if checkErr != nil { - if isRetryableAPIError(checkErr) { - all = append(all, fmt.Sprintf("resources[%d]: %v", i, checkErr)) - continue - } - return false, fmt.Errorf("resources[%d]: %w", i, checkErr) - } - for _, d := range diffs { - all = append(all, fmt.Sprintf("resources[%d]: %s", i, d)) - } - } - last = all - return len(all) == 0, nil - }, wait.WithTimeout(timeout), wait.WithInterval(resourcePollInterval), - wait.WithContext(t.Context()), wait.WithImmediate()) - if err == nil { - return - } - s.dumpDiagnostics(t, ns, assertions) - require.NoErrorf(t, err, "resource assertions failed:\n%s", strings.Join(last, "\n")) -} - -// isRetryableAPIError reports whether resource polling should retry err. -// It returns true for NotFound, conflict, timeout, and transport failures. -// REST mapping misses, forbidden requests, and canceled contexts fail -// immediately. -func isRetryableAPIError(err error) bool { - if err == nil { - return false - } - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return false - } - switch { - case apierrors.IsNotFound(err), - apierrors.IsConflict(err), - apierrors.IsServerTimeout(err), - apierrors.IsTooManyRequests(err), - apierrors.IsServiceUnavailable(err), - apierrors.IsTimeout(err), - apierrors.IsInternalError(err): - return true - } - return netutil.IsProbableEOF(err) || - netutil.IsConnectionReset(err) || - netutil.IsConnectionRefused(err) || - netutil.IsTimeout(err) -} - -func (s *E2ESuite) checkAssertion(ctx context.Context, ns string, ra inttest.ResourceAssertion) ([]string, error) { - gvk, err := gvkFromMatch(ra.Match) - if err != nil { - return nil, err - } - namespaced, err := s.isNamespaced(gvk) - if err != nil { - return nil, err - } - lookupNS := "" - if namespaced { - lookupNS = ns - } - name, labelSet := matchMeta(ra.Match) - want := ra.Count() - res, err := s.newResources(lookupNS) - if err != nil { - return nil, err - } - - if name != "" { - u := &unstructured.Unstructured{} - u.SetGroupVersionKind(gvk) - getErr := res.Get(ctx, name, lookupNS, u) - if apierrors.IsNotFound(getErr) { - if want == 0 { - return nil, nil - } - return []string{fmt.Sprintf("%s %s not found", gvk.Kind, name)}, nil - } - if getErr != nil { - return nil, getErr - } - if want == 0 { - return []string{fmt.Sprintf("%s %s still exists", gvk.Kind, name)}, nil - } - return inttest.DiffSubset("$", ra.Match, u.Object), nil - } - - list := &unstructured.UnstructuredList{} - list.SetGroupVersionKind(schema.GroupVersionKind{ - Group: gvk.Group, - Version: gvk.Version, - Kind: gvk.Kind + "List", - }) - var opts []resources.ListOption - if len(labelSet) > 0 { - opts = append(opts, resources.WithLabelSelector(labels.Set(labelSet).String())) - } - if listErr := res.List(ctx, list, opts...); listErr != nil { - return nil, listErr - } - - matched := 0 - var sample []string - for i := range list.Items { - diffs := inttest.DiffSubset("$", ra.Match, list.Items[i].Object) - if len(diffs) == 0 { - matched++ - continue - } - if sample == nil { - sample = diffs - } - } - if want == 0 { - if matched == 0 { - return nil, nil - } - return []string{fmt.Sprintf("want 0 %s matching, got %d", gvk.Kind, matched)}, nil - } - if matched >= want { - return nil, nil - } - msg := fmt.Sprintf("want >= %d %s matching, got %d", want, gvk.Kind, matched) - if sample != nil { - return append([]string{msg}, sample...), nil - } - return []string{msg}, nil -} - -func (s *E2ESuite) isNamespaced(gvk schema.GroupVersionKind) (bool, error) { - s.mapperMu.Lock() - defer s.mapperMu.Unlock() - mapping, err := s.mapper.RESTMapping(gvk.GroupKind(), gvk.Version) - if meta.IsNoMatchError(err) { - s.mapper.Reset() - mapping, err = s.mapper.RESTMapping(gvk.GroupKind(), gvk.Version) - } - if err != nil { - return false, fmt.Errorf("RESTMapping %s: %w", gvk.String(), err) - } - return mapping.Scope.Name() == meta.RESTScopeNameNamespace, nil -} - -func (s *E2ESuite) dumpDiagnostics(t *testing.T, ns string, assertions []inttest.ResourceAssertion) { - t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - for i, ra := range assertions { - gvk, err := gvkFromMatch(ra.Match) - if err != nil { - t.Logf("resources[%d]: %v", i, err) - continue - } - name, labelSet := matchMeta(ra.Match) - namespaced, nsErr := s.isNamespaced(gvk) - lookupNS := "" - if nsErr == nil && namespaced { - lookupNS = ns - } - res, resErr := s.newResources(lookupNS) - if resErr != nil { - t.Logf("resources client: %v", resErr) - continue - } - if name != "" { - u := &unstructured.Unstructured{} - u.SetGroupVersionKind(gvk) - if getErr := res.Get(ctx, name, lookupNS, u); getErr != nil { - t.Logf("describe %s/%s: %v", gvk.Kind, name, getErr) - continue - } - logLiveYAML(t, gvk.Kind, name, u.Object) - continue - } - list := &unstructured.UnstructuredList{} - list.SetGroupVersionKind(schema.GroupVersionKind{ - Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind + "List", - }) - var opts []resources.ListOption - if len(labelSet) > 0 { - opts = append(opts, resources.WithLabelSelector(labels.Set(labelSet).String())) - } - if listErr := res.List(ctx, list, opts...); listErr != nil { - t.Logf("list %s: %v", gvk.Kind, listErr) - continue - } - for j := range list.Items { - logLiveYAML(t, gvk.Kind, list.Items[j].GetName(), list.Items[j].Object) - } - } - var events corev1.EventList - eventRes, eventResErr := s.newResources(ns) - if eventResErr != nil { - t.Logf("events client: %v", eventResErr) - return - } - if listErr := eventRes.List(ctx, &events); listErr != nil { - t.Logf("list events: %v", listErr) - return - } - for i := range events.Items { - ev := events.Items[i] - t.Logf("event %s %s/%s: %s", ev.Type, ev.InvolvedObject.Kind, ev.InvolvedObject.Name, ev.Message) - } -} - -func (s *E2ESuite) createNamespace(t *testing.T, name string) { - t.Helper() - ns := &corev1.Namespace{Name: name} - res, err := s.newResources("") - require.NoError(t, err) - err = res.Create(t.Context(), ns) - if apierrors.IsAlreadyExists(err) { - return - } - require.NoErrorf(t, err, "create namespace %s", name) -} - -func (s *E2ESuite) deleteNamespace(t *testing.T, name string) { - t.Helper() - res, err := s.newResources(name) - require.NoError(t, err) - - // Each phase gets its own deadline. A shared ctx that deletePVCs - // exhausted used to make namespace Delete fail with rate limiter Wait. - workCtx, workCancel := context.WithTimeout(context.Background(), namespaceWaitTimeout) - defer workCancel() - if workErr := s.deleteWorkloads(workCtx, t, res); workErr != nil { - t.Logf("delete workloads in %s: %v", name, workErr) - } - - pvcCtx, pvcCancel := context.WithTimeout(context.Background(), namespaceWaitTimeout) - defer pvcCancel() - if pvcErr := s.deletePVCs(pvcCtx, t, res); pvcErr != nil { - t.Errorf("delete pvcs in %s: %v", name, pvcErr) - } - - ns := &corev1.Namespace{Name: name} - clusterRes, clusterErr := s.newResources("") - require.NoError(t, clusterErr) - nsCtx, nsCancel := context.WithTimeout(context.Background(), namespaceWaitTimeout) - defer nsCancel() - if delErr := clusterRes.Delete(nsCtx, ns); delErr != nil && !apierrors.IsNotFound(delErr) { - t.Errorf("delete namespace %s: %v", name, delErr) - return - } - waitErr := wait.For(func(ctx context.Context) (bool, error) { - getErr := clusterRes.Get(ctx, name, "", ns) - if apierrors.IsNotFound(getErr) { - return true, nil - } - if getErr != nil && !isRetryableAPIError(getErr) { - return false, getErr - } - return false, nil - }, wait.WithTimeout(namespaceWaitTimeout), wait.WithInterval(resourcePollInterval), - wait.WithContext(nsCtx), wait.WithImmediate()) - if waitErr != nil { - t.Errorf("wait for namespace %s deletion: %v", name, waitErr) - } -} - -func (s *E2ESuite) deleteWorkloads(ctx context.Context, t *testing.T, res *resources.Resources) error { - t.Helper() - var cronjobs batchv1.CronJobList - if listErr := res.List(ctx, &cronjobs); listErr != nil && !apierrors.IsNotFound(listErr) { - return fmt.Errorf("list cronjobs: %w", listErr) - } - for i := range cronjobs.Items { - if delErr := res.Delete(ctx, &cronjobs.Items[i]); delErr != nil && !apierrors.IsNotFound(delErr) { - t.Logf("delete cronjob %s: %v", cronjobs.Items[i].Name, delErr) - } - } - var jobs batchv1.JobList - if listErr := res.List(ctx, &jobs); listErr != nil && !apierrors.IsNotFound(listErr) { - return fmt.Errorf("list jobs: %w", listErr) - } - for i := range jobs.Items { - if delErr := res.Delete(ctx, &jobs.Items[i]); delErr != nil && !apierrors.IsNotFound(delErr) { - t.Logf("delete job %s: %v", jobs.Items[i].Name, delErr) - } - } - var sts appsv1.StatefulSetList - if listErr := res.List(ctx, &sts); listErr != nil && !apierrors.IsNotFound(listErr) { - return fmt.Errorf("list statefulsets: %w", listErr) - } - for i := range sts.Items { - if delErr := res.Delete(ctx, &sts.Items[i]); delErr != nil && !apierrors.IsNotFound(delErr) { - t.Logf("delete statefulset %s: %v", sts.Items[i].Name, delErr) - } - } - var deploys appsv1.DeploymentList - if listErr := res.List(ctx, &deploys); listErr != nil && !apierrors.IsNotFound(listErr) { - return fmt.Errorf("list deployments: %w", listErr) - } - for i := range deploys.Items { - if delErr := res.Delete(ctx, &deploys.Items[i]); delErr != nil && !apierrors.IsNotFound(delErr) { - t.Logf("delete deployment %s: %v", deploys.Items[i].Name, delErr) - } - } - return wait.For(func(pollCtx context.Context) (bool, error) { - var pods corev1.PodList - if listErr := res.List(pollCtx, &pods); listErr != nil { - if apierrors.IsNotFound(listErr) { - return true, nil - } - if isRetryableAPIError(listErr) { - return false, nil - } - return false, listErr - } - return len(pods.Items) == 0, nil - }, wait.WithTimeout(namespaceWaitTimeout), wait.WithInterval(resourcePollInterval), - wait.WithContext(ctx), wait.WithImmediate()) -} - -func (s *E2ESuite) deletePVCs(ctx context.Context, t *testing.T, res *resources.Resources) error { - t.Helper() - var pvcs corev1.PersistentVolumeClaimList - if listErr := res.List(ctx, &pvcs); listErr != nil { - if apierrors.IsNotFound(listErr) { - return nil - } - return fmt.Errorf("list pvcs: %w", listErr) - } - for i := range pvcs.Items { - if delErr := res.Delete(ctx, &pvcs.Items[i]); delErr != nil && !apierrors.IsNotFound(delErr) { - t.Logf("delete pvc %s: %v", pvcs.Items[i].Name, delErr) - } - } - return wait.For(func(ctx context.Context) (bool, error) { - var remaining corev1.PersistentVolumeClaimList - if listErr := res.List(ctx, &remaining); listErr != nil { - if apierrors.IsNotFound(listErr) { - return true, nil - } - if isRetryableAPIError(listErr) { - return false, nil - } - return false, listErr - } - return len(remaining.Items) == 0, nil - }, wait.WithTimeout(namespaceWaitTimeout), wait.WithInterval(resourcePollInterval), - wait.WithContext(ctx), wait.WithImmediate()) -} - -func logLiveYAML(t *testing.T, kind, name string, obj map[string]any) { - t.Helper() - raw, err := yaml.Marshal(obj) - if err != nil { - t.Logf("marshal %s/%s: %v", kind, name, err) - return - } - t.Logf("live %s/%s:\n%s", kind, name, raw) -} - -func (s *E2ESuite) scaffoldSimple(t *testing.T, dir, ns string, fx inttest.E2EFixture) { - t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - kinds := []schema.GroupVersionKind{ - {Group: "apps", Version: "v1", Kind: "Deployment"}, - {Group: "apps", Version: "v1", Kind: "StatefulSet"}, - {Group: "", Version: "v1", Kind: "Service"}, - {Group: "", Version: "v1", Kind: "PersistentVolumeClaim"}, - {Group: "batch", Version: "v1", Kind: "Job"}, - {Group: "batch", Version: "v1", Kind: "CronJob"}, - } - var resourcesOut []map[string]any - for _, gvk := range kinds { - list := &unstructured.UnstructuredList{} - list.SetGroupVersionKind(schema.GroupVersionKind{ - Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind + "List", - }) - if listErr := s.listKindInNamespace(ctx, ns, list); listErr != nil { - t.Logf("scaffold list %s: %v", gvk.Kind, listErr) - continue - } - for i := range list.Items { - resourcesOut = append(resourcesOut, map[string]any{ - "match": scaffoldObject(list.Items[i].Object), - }) - } - } - doc := map[string]any{ - "env": fx.Env, - "resources": resourcesOut, - } - body, err := yaml.Marshal(doc) - require.NoError(t, err) - out := e2eSchemaModeLine + "\n" + string(body) - - path := filepath.Join(dir, inttest.E2EFixtureFile) - if _, statErr := os.Stat(path); statErr == nil { - t.Logf("-e2e.scaffold: %s already exists; skeleton follows\n%s", path, out) - return - } - require.NoError(t, os.WriteFile(path, []byte(out), 0o600)) - t.Logf("wrote scaffold %s", path) -} - -func (s *E2ESuite) listKindInNamespace(ctx context.Context, ns string, list *unstructured.UnstructuredList) error { - res, err := s.newResources(ns) - if err != nil { - return err - } - return res.List(ctx, list) -} - -func scaffoldObject(obj map[string]any) map[string]any { - kind := stringField(obj, "kind") - out := map[string]any{ - "apiVersion": obj["apiVersion"], - "kind": kind, - } - if metaMap := mapField(obj, "metadata"); metaMap != nil { - m := map[string]any{} - if name, hasName := metaMap["name"]; hasName { - m["name"] = name - } - if rawLabels := mapField(metaMap, "labels"); rawLabels != nil { - cleaned := maps.Clone(rawLabels) - delete(cleaned, "helm.sh/chart") - if len(cleaned) > 0 { - m["labels"] = cleaned - } - } - out["metadata"] = m - } - if specMap := mapField(obj, "spec"); specMap != nil { - out["spec"] = stripGeneratedSpec(deepCopyMap(specMap)) - } - if st := mapField(obj, "status"); st != nil { - ready := map[string]any{} - switch kind { - case "Deployment", "StatefulSet": - if v, hasReady := st["readyReplicas"]; hasReady { - ready["readyReplicas"] = v - } - case "Job": - if v, hasSucceeded := st["succeeded"]; hasSucceeded { - ready["succeeded"] = v - } - case "PersistentVolumeClaim": - if v, hasPhase := st["phase"]; hasPhase { - ready["phase"] = v - } - } - if len(ready) > 0 { - out["status"] = ready - } - } - return out -} - -func stripGeneratedSpec(spec map[string]any) map[string]any { - delete(spec, "clusterIP") - delete(spec, "clusterIPs") - ports, isPorts := spec["ports"].([]any) - if !isPorts { - return spec - } - for i := range ports { - pm, isMap := ports[i].(map[string]any) - if !isMap { - continue - } - delete(pm, "nodePort") - } - return spec -} - -func deepCopyMap(in map[string]any) map[string]any { - out := maps.Clone(in) - for k, v := range out { - switch n := v.(type) { - case map[string]any: - out[k] = deepCopyMap(n) - case []any: - cp := make([]any, 0, len(n)) - for _, item := range n { - m, isMap := item.(map[string]any) - if isMap { - cp = append(cp, deepCopyMap(m)) - continue - } - cp = append(cp, item) - } - out[k] = cp - } - } - return out -} - -func gvkFromMatch(match map[string]any) (schema.GroupVersionKind, error) { - kind := stringField(match, "kind") - apiVersion := stringField(match, "apiVersion") - if kind == "" || apiVersion == "" { - return schema.GroupVersionKind{}, fmt.Errorf("match requires apiVersion and kind") - } - gv, err := schema.ParseGroupVersion(apiVersion) - if err != nil { - return schema.GroupVersionKind{}, fmt.Errorf("parse apiVersion %q: %w", apiVersion, err) - } - return gv.WithKind(kind), nil -} - -func matchMeta(match map[string]any) (name string, labelSet map[string]string) { - metaMap := mapField(match, "metadata") - if metaMap == nil { - return "", nil - } - name = stringField(metaMap, "name") - raw := mapField(metaMap, "labels") - if raw == nil { - return name, nil - } - labelSet = make(map[string]string, len(raw)) - for k, v := range raw { - s, isString := v.(string) - if !isString { - continue - } - labelSet[k] = s - } - return name, labelSet -} - -func stringField(m map[string]any, key string) string { - s, ok := m[key].(string) - if !ok { - return "" - } - return s -} - -func mapField(m map[string]any, key string) map[string]any { - child, ok := m[key].(map[string]any) - if !ok { - return nil - } - return child -} diff --git a/internal/e2e/groups.go b/internal/e2e/groups.go new file mode 100644 index 0000000..f9780ac --- /dev/null +++ b/internal/e2e/groups.go @@ -0,0 +1,39 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import "testing" + +// RunGroups runs parallel items together, then sequential items. +func RunGroups[T any](t *testing.T, parallel, sequential []T, name func(T) string, run func(*testing.T, T)) { + t.Helper() + // Sequential items start after the parallel group finishes. + // Only the parallel items call t.Parallel. + t.Run("parallel", func(t *testing.T) { + for _, item := range parallel { + t.Run(name(item), func(t *testing.T) { + t.Parallel() + run(t, item) + }) + } + }) + t.Run("sequential", func(t *testing.T) { + for _, item := range sequential { + t.Run(name(item), func(t *testing.T) { + run(t, item) + }) + } + }) +} diff --git a/internal/e2e/groups_test.go b/internal/e2e/groups_test.go new file mode 100644 index 0000000..b3cecfa --- /dev/null +++ b/internal/e2e/groups_test.go @@ -0,0 +1,92 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "flag" + "strconv" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRunGroups(t *testing.T) { + // Overlap is only possible when go test may run two tests at once. + // With -parallel=1 the scenarios still finish in group order. + overlap := false + if f := flag.Lookup("test.parallel"); f != nil { + if n, err := strconv.Atoi(f.Value.String()); err == nil && n >= 2 { + overlap = true + } + } + release := make(chan struct{}) + entered := make(chan struct{}, 2) + if overlap { + go func() { + <-entered + <-entered + close(release) + }() + } + + var mu sync.Mutex + var order []string + activeSeq := 0 + inParallel := 0 + maxParallel := 0 + + RunGroups(t, []string{"a", "b"}, []string{"c", "d"}, + func(name string) string { return name }, + func(t *testing.T, name string) { //nolint:thelper // failures should stay in this test + if name == "a" || name == "b" { + mu.Lock() + inParallel++ + if inParallel > maxParallel { + maxParallel = inParallel + } + mu.Unlock() + if overlap { + entered <- struct{}{} + <-release + } + mu.Lock() + inParallel-- + order = append(order, name) + mu.Unlock() + return + } + mu.Lock() + defer mu.Unlock() + if activeSeq != 0 { + t.Errorf("sequential %s overlapped another sequential scenario", name) + } + activeSeq++ + order = append(order, name) + activeSeq-- + }) + + mu.Lock() + got := append([]string(nil), order...) + peak := maxParallel + mu.Unlock() + if overlap { + require.GreaterOrEqual(t, peak, 2) + } + require.Len(t, got, 4) + assert.ElementsMatch(t, []string{"a", "b"}, got[:2]) + assert.Equal(t, []string{"c", "d"}, got[2:]) +} diff --git a/internal/e2e/e2e_helpers_test.go b/internal/e2e/helpers_test.go similarity index 95% rename from internal/e2e/e2e_helpers_test.go rename to internal/e2e/helpers_test.go index 297c748..7feb6c3 100644 --- a/internal/e2e/e2e_helpers_test.go +++ b/internal/e2e/helpers_test.go @@ -35,11 +35,11 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" ) -func (s *E2ESuite) kubeClients(t *testing.T) (*kubernetes.Clientset, dynamic.Interface) { +func (env *e2eEnv) kubeClients(t *testing.T) (*kubernetes.Clientset, dynamic.Interface) { t.Helper() - cs, err := kubernetes.NewForConfig(s.client.RESTConfig()) + cs, err := kubernetes.NewForConfig(env.client.RESTConfig()) require.NoError(t, err) - dyn, err := dynamic.NewForConfig(s.client.RESTConfig()) + dyn, err := dynamic.NewForConfig(env.client.RESTConfig()) require.NoError(t, err) return cs, dyn } diff --git a/internal/e2e/image_pull_test.go b/internal/e2e/image_pull_test.go new file mode 100644 index 0000000..0ca90cb --- /dev/null +++ b/internal/e2e/image_pull_test.go @@ -0,0 +1,99 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build e2e + +package e2e_test + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "sigs.k8s.io/e2e-framework/klient/wait" + + "deployah.dev/deployah/internal/testing/e2e" + + inttest "deployah.dev/deployah/internal/testing" +) + +// TestPublicImagePull deploys an image that is not special-cased by the +// suite. Kind pulls it from the public registry. +// Failures in this test should point here, not at the suite caller. +// +//nolint:thelper +func runPublicImagePull(t *testing.T, env *e2eEnv) { + src := filepath.Join(env.scenariosDir, "basic-web-service") + require.DirExists(t, src) + dir, err := inttest.NewScenarioWorkspace(t, src) + require.NoError(t, err) + + specPath := filepath.Join(dir, "deployah.yaml") + raw, err := os.ReadFile(specPath) // #nosec G304 -- copied scenario workspace + require.NoError(t, err) + updated := strings.ReplaceAll(string(raw), "image: nginx:latest", "image: nginx:1.27") + require.NotEqual(t, string(raw), updated) + require.NoError(t, os.WriteFile(specPath, []byte(updated), 0o600)) // #nosec G703 -- path under the copied workspace + + ns := fixtureNamespace("public-image") + claim, err := env.claimNamespace(t.Context(), ns) + require.NoError(t, err) + t.Cleanup(func() { + if cleanupErr := env.cleanupClaim(t, dir, claim, []releaseTarget{{ + Project: "basic-web-service", Env: "dev", + }}); cleanupErr != nil { + t.Errorf("cleanup: %v", cleanupErr) + } + }) + + runIn(t, dir, "deploy", "dev", "--context", kindContext, "--namespace", ns) + require.NoError(t, wait.For(func(ctx context.Context) (bool, error) { + objects, obsErr := env.observe(ctx, e2e.ResourceSelect{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "basic-web-service-dev", + }, ns) + if obsErr != nil { + if errors.Is(obsErr, e2e.ErrObserveRetry) { + return false, nil + } + return false, obsErr + } + if len(objects) != 1 { + return false, nil + } + diffs := inttest.DiffSubset("$", map[string]any{ + "spec": map[string]any{ + "template": map[string]any{ + "spec": map[string]any{ + "containers": []any{ + map[string]any{ + "name": "web", + "image": "docker.io/library/nginx:1.27", + }, + }, + }, + }, + }, + "status": map[string]any{"readyReplicas": int64(1)}, + }, objects[0]) + return len(diffs) == 0, nil + }, wait.WithTimeout(4*time.Minute), wait.WithInterval(resourcePollInterval), + wait.WithContext(t.Context()), wait.WithImmediate())) +} diff --git a/internal/e2e/namespace_test.go b/internal/e2e/namespace_test.go new file mode 100644 index 0000000..0016e37 --- /dev/null +++ b/internal/e2e/namespace_test.go @@ -0,0 +1,187 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build e2e + +package e2e_test + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + "sigs.k8s.io/e2e-framework/klient/wait" + + "deployah.dev/deployah/internal/testing/e2e" + + inttest "deployah.dev/deployah/internal/testing" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" +) + +// TestDeployCreatesNamespace checks that deployah deploy creates a namespace +// that the suite did not create first. +// Failures in this test should point here, not at the suite caller. +// +//nolint:thelper +func runDeployCreatesNamespace(t *testing.T, env *e2eEnv) { + src := filepath.Join(env.scenariosDir, "basic-web-service") + require.DirExists(t, src) + dir, err := inttest.NewScenarioWorkspace(t, src) + require.NoError(t, err) + + var buf [8]byte + _, err = rand.Read(buf[:]) + require.NoError(t, err) + ns := "e2e-ns-" + hex.EncodeToString(buf[:]) + _, getErr := env.namespaceView(t.Context(), ns) + require.ErrorIs(t, getErr, e2e.ErrNotFound) + + var uid string + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), namespaceWaitTimeout) + defer cancel() + if cleanupErr := env.cleanupCreatedNamespace(ctx, t, dir, ns, uid, "basic-web-service", "dev"); cleanupErr != nil { + t.Errorf("cleanup: %v", cleanupErr) + } + }) + + _, stderr, deployErr := runInErrContext(t, t.Context(), dir, "deploy", "dev", + "--context", kindContext, "--namespace", ns) + view, viewErr := env.namespaceView(context.Background(), ns) + if viewErr == nil && view != nil { + uid = view.UID + markCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + marked := &corev1.Namespace{} + res, markErr := env.newResources("") + if markErr == nil { + markErr = res.Get(markCtx, ns, "", marked) + } + if markErr == nil && (marked.Name != ns || string(marked.UID) != uid) { + markErr = fmt.Errorf("namespace %s: identity changed before marking", ns) + } + if markErr == nil { + if marked.Annotations == nil { + marked.Annotations = map[string]string{} + } + marked.Annotations[e2e.RunAnnotation] = "deploy-creates-namespace" + markErr = res.Update(markCtx, marked) + } + cancel() + if markErr != nil { + t.Errorf("mark namespace: %v", markErr) + } + } + require.NoError(t, deployErr, stderr) + require.NotEmpty(t, uid) + assertReleaseExists(t, dir, ns, "basic-web-service", "dev") + + require.NoError(t, wait.For(func(ctx context.Context) (bool, error) { + objects, obsErr := env.observe(ctx, e2e.ResourceSelect{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "basic-web-service-dev", + }, ns) + if obsErr != nil { + if errors.Is(obsErr, e2e.ErrObserveRetry) { + return false, nil + } + return false, obsErr + } + if len(objects) != 1 { + return false, nil + } + diffs := inttest.DiffSubset("$", map[string]any{ + "status": map[string]any{"readyReplicas": int64(1)}, + }, objects[0]) + return len(diffs) == 0, nil + }, wait.WithTimeout(3*time.Minute), wait.WithInterval(resourcePollInterval), + wait.WithContext(t.Context()), wait.WithImmediate())) +} + +func (env *e2eEnv) cleanupCreatedNamespace(ctx context.Context, t *testing.T, dir, name, uid, project, environment string) error { + t.Helper() + if uid == "" { + view, err := env.namespaceView(ctx, name) + if errors.Is(err, e2e.ErrNotFound) || apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf("namespace %s: ownership uncertain: %w", name, err) + } + if view == nil { + return fmt.Errorf("namespace %s: ownership uncertain, not deleting", name) + } + return fmt.Errorf("namespace %s exists without a recorded uid, not deleting", name) + } + phases := []e2e.CleanupPhase{ + {Name: "delete release", Run: func(phaseCtx context.Context) error { + if err := env.confirmCreated(phaseCtx, name, uid); err != nil { + if errors.Is(err, e2e.ErrNamespaceGone) { + return nil + } + return err + } + return env.deleteRelease(t, dir, name, releaseTarget{Project: project, Env: environment})(phaseCtx) + }}, + {Name: "workloads", Run: func(phaseCtx context.Context) error { + if err := env.confirmCreated(phaseCtx, name, uid); err != nil { + if errors.Is(err, e2e.ErrNamespaceGone) { + return nil + } + return err + } + return env.deleteWorkloads(phaseCtx, name) + }}, + {Name: "pvcs", Run: func(phaseCtx context.Context) error { + if err := env.confirmCreated(phaseCtx, name, uid); err != nil { + if errors.Is(err, e2e.ErrNamespaceGone) { + return nil + } + return err + } + return env.deletePVCs(phaseCtx, name) + }}, + {Name: "namespace", Run: func(phaseCtx context.Context) error { + if err := env.confirmCreated(phaseCtx, name, uid); err != nil { + if errors.Is(err, e2e.ErrNamespaceGone) { + return nil + } + return err + } + return env.deleteNamespaceObject(phaseCtx, e2e.Claim{Name: name, UID: uid, RunID: "recorded"}) + }}, + } + return e2e.RunCleanup(ctx, phases, e2e.WithPhaseTimeout(namespaceWaitTimeout)) +} + +func (env *e2eEnv) confirmCreated(ctx context.Context, name, uid string) error { + view, err := env.namespaceView(ctx, name) + if errors.Is(err, e2e.ErrNotFound) { + return e2e.ErrNamespaceGone + } + if err != nil { + return fmt.Errorf("namespace %s: %w", name, err) + } + if view == nil || view.Name != name || view.UID != uid { + return fmt.Errorf("namespace %s: %w", name, e2e.ErrNotOwner) + } + return nil +} diff --git a/internal/e2e/e2e_plan_cli_test.go b/internal/e2e/plan_cli_test.go similarity index 73% rename from internal/e2e/e2e_plan_cli_test.go rename to internal/e2e/plan_cli_test.go index f925103..75c46e8 100644 --- a/internal/e2e/e2e_plan_cli_test.go +++ b/internal/e2e/plan_cli_test.go @@ -18,54 +18,61 @@ package e2e_test import ( "bytes" - "context" "encoding/json" "os" "path/filepath" - "time" + "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "deployah.dev/deployah/internal/testing/e2e" + planCmd "deployah.dev/deployah/internal/cmd/plan" inttest "deployah.dev/deployah/internal/testing" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func (s *E2ESuite) TestPlanCLI() { - t := s.T() - src := filepath.Join(s.scenariosDir, "basic-web-service") +// Failures in this test should point here, not at the suite caller. +// +//nolint:thelper +func runPlanCLI(t *testing.T, env *e2eEnv) { + src := filepath.Join(env.scenariosDir, "basic-web-service") require.DirExists(t, src) dir, err := inttest.NewScenarioWorkspace(t, src) require.NoError(t, err) ns := fixtureNamespace("plan-cli") + var claim e2e.Claim t.Cleanup(func() { - cleanupCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - if _, _, delErr := runInErrContext(t, cleanupCtx, dir, "delete", "basic-web-service", "dev", - "--yes", "--wait", "--allow-missing-platform", - "--context", kindContext, "--namespace", ns); delErr != nil { - t.Logf("cleanup delete failed (non-fatal): %v", delErr) + if claim.UID == "" { + return + } + if cleanupErr := env.cleanupClaim(t, dir, claim, []releaseTarget{{ + Project: "basic-web-service", + Env: "dev", + }}); cleanupErr != nil { + t.Errorf("cleanup: %v", cleanupErr) } - s.deleteNamespace(t, ns) }) - stdout, _, err := runInErr(t, dir, "plan", "dev", "-o", "json", + stdout, _, err := runInErrContext(t, t.Context(), dir, "plan", "dev", "-o", "json", "--context", kindContext, "--namespace", ns) require.NoError(t, err) var fresh map[string]any require.NoError(t, json.Unmarshal([]byte(stdout), &fresh)) assert.Empty(t, fresh["drift"]) - cs, _ := s.kubeClients(t) + cs, _ := env.kubeClients(t) _, err = cs.CoreV1().Namespaces().Get(t.Context(), ns, metav1.GetOptions{}) require.True(t, apierrors.IsNotFound(err), "fresh plan must not create namespace %s: %v", ns, err) - s.createNamespace(t, ns) + got, err := env.claimNamespace(t.Context(), ns) + require.NoError(t, err) + claim = got runIn(t, dir, "deploy", "dev", "--context", kindContext, "--namespace", ns) - stdout, _, err = runInErr(t, dir, "plan", "dev", "-o", "json", + stdout, _, err = runInErrContext(t, t.Context(), dir, "plan", "dev", "-o", "json", "--context", kindContext, "--namespace", ns) require.NoError(t, err) var same map[string]any @@ -73,7 +80,7 @@ func (s *E2ESuite) TestPlanCLI() { assert.Empty(t, same["changes"]) assert.Empty(t, same["drift"]) - _, _, err = runInErr(t, dir, "plan", "dev", "--detailed-exitcode", + _, _, err = runInErrContext(t, t.Context(), dir, "plan", "dev", "--detailed-exitcode", "--context", kindContext, "--namespace", ns) require.NoError(t, err) @@ -84,7 +91,7 @@ func (s *E2ESuite) TestPlanCLI() { _, err = cs.AppsV1().Deployments(ns).Update(t.Context(), dep, metav1.UpdateOptions{}) require.NoError(t, err) - stdout, _, err = runInErr(t, dir, "plan", "dev", "-o", "json", + stdout, _, err = runInErrContext(t, t.Context(), dir, "plan", "dev", "-o", "json", "--context", kindContext, "--namespace", ns) require.NoError(t, err) var drifted map[string]any @@ -101,11 +108,11 @@ func (s *E2ESuite) TestPlanCLI() { } assert.Equal(t, 1, modified) - _, _, err = runInErr(t, dir, "plan", "dev", "--detailed-exitcode", + _, _, err = runInErrContext(t, t.Context(), dir, "plan", "dev", "--detailed-exitcode", "--context", kindContext, "--namespace", ns) require.NoError(t, err, "drift alone must not exit 2") - human, _, err := runInErr(t, dir, "plan", "dev", + human, _, err := runInErrContext(t, t.Context(), dir, "plan", "dev", "--context", kindContext, "--namespace", ns) require.NoError(t, err) assert.Contains(t, human, "Drift") @@ -117,7 +124,7 @@ func (s *E2ESuite) TestPlanCLI() { require.Equal(t, 1, bytes.Count(raw, old)) updated := bytes.Replace(raw, old, []byte("port: 80\n replicas: 2\n"), 1) require.NoError(t, os.WriteFile(specPath, updated, 0o600)) // #nosec G703 -- path under test-controlled temp dir - stdout, _, err = runInErr(t, dir, "plan", "dev", "--detailed-exitcode", + stdout, _, err = runInErrContext(t, t.Context(), dir, "plan", "dev", "--detailed-exitcode", "--context", kindContext, "--namespace", ns) require.ErrorIs(t, err, planCmd.ErrChangesPresent) assert.NotEmpty(t, stdout) diff --git a/internal/e2e/scaffold_test.go b/internal/e2e/scaffold_test.go new file mode 100644 index 0000000..7796d24 --- /dev/null +++ b/internal/e2e/scaffold_test.go @@ -0,0 +1,229 @@ +// Copyright 2025 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build e2e + +package e2e_test + +import ( + "context" + "errors" + "maps" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + + "deployah.dev/deployah/internal/testing/e2e" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + netutil "k8s.io/apimachinery/pkg/util/net" +) + +const ( + resourcePollInterval = 2 * time.Second + namespaceWaitTimeout = 2 * time.Minute + e2eSchemaModeLine = "# $schema: ../../internal/testing/e2e/e2e.schema.json" +) + +// isRetryableAPIError reports whether resource polling should retry err. +// It returns true for NotFound, conflict, timeout, and transport failures. +// REST mapping misses, forbidden requests, and canceled contexts fail +// immediately. +func isRetryableAPIError(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + switch { + case apierrors.IsNotFound(err), + apierrors.IsConflict(err), + apierrors.IsServerTimeout(err), + apierrors.IsTooManyRequests(err), + apierrors.IsServiceUnavailable(err), + apierrors.IsTimeout(err), + apierrors.IsInternalError(err): + return true + } + return netutil.IsProbableEOF(err) || + netutil.IsConnectionReset(err) || + netutil.IsConnectionRefused(err) || + netutil.IsTimeout(err) +} + +func (env *e2eEnv) scaffoldSimple(t *testing.T, dir, ns string, fx e2e.E2EFixture) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + kinds := []schema.GroupVersionKind{ + {Group: "apps", Version: "v1", Kind: "Deployment"}, + {Group: "apps", Version: "v1", Kind: "StatefulSet"}, + {Group: "", Version: "v1", Kind: "Service"}, + {Group: "", Version: "v1", Kind: "PersistentVolumeClaim"}, + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "batch", Version: "v1", Kind: "CronJob"}, + } + var assertions []map[string]any + for _, gvk := range kinds { + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(schema.GroupVersionKind{ + Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind + "List", + }) + if listErr := env.listKindInNamespace(ctx, ns, list); listErr != nil { + t.Logf("scaffold list %s: %v", gvk.Kind, listErr) + continue + } + for i := range list.Items { + assertions = append(assertions, scaffoldAssertion(list.Items[i].Object)) + } + } + doc := map[string]any{ + "env": fx.Env, + "steps": []any{ + map[string]any{"deploy": map[string]any{}}, + map[string]any{"assert": assertions}, + }, + } + body, err := yaml.Marshal(doc) + require.NoError(t, err) + out := e2eSchemaModeLine + "\n" + string(body) + + path := filepath.Join(dir, e2e.FixtureFile) + if _, statErr := os.Stat(path); statErr == nil { + t.Logf("-e2e.scaffold: %s already exists; skeleton follows\n%s", path, out) + return + } + require.NoError(t, os.WriteFile(path, []byte(out), 0o600)) + t.Logf("wrote scaffold %s", path) +} + +func (env *e2eEnv) listKindInNamespace(ctx context.Context, ns string, list *unstructured.UnstructuredList) error { + res, err := env.newResources(ns) + if err != nil { + return err + } + return res.List(ctx, list) +} + +func scaffoldAssertion(obj map[string]any) map[string]any { + kind := stringField(obj, "kind") + sel := map[string]any{ + "apiVersion": obj["apiVersion"], + "kind": kind, + } + expect := map[string]any{} + if metaMap := mapField(obj, "metadata"); metaMap != nil { + if name, hasName := metaMap["name"].(string); hasName && name != "" { + sel["name"] = name + } + if rawLabels := mapField(metaMap, "labels"); rawLabels != nil { + cleaned := maps.Clone(rawLabels) + delete(cleaned, "helm.sh/chart") + if len(cleaned) > 0 { + sel["labels"] = cleaned + } + } + } + if specMap := mapField(obj, "spec"); specMap != nil { + expect["spec"] = stripGeneratedSpec(deepCopyMap(specMap)) + } + if st := mapField(obj, "status"); st != nil { + ready := map[string]any{} + switch kind { + case "Deployment", "StatefulSet": + if v, hasReady := st["readyReplicas"]; hasReady { + ready["readyReplicas"] = v + } + case "Job": + if v, hasSucceeded := st["succeeded"]; hasSucceeded { + ready["succeeded"] = v + } + case "PersistentVolumeClaim": + if v, hasPhase := st["phase"]; hasPhase { + ready["phase"] = v + } + } + if len(ready) > 0 { + expect["status"] = ready + } + } + item := map[string]any{"select": sel} + if len(expect) > 0 { + item["expect"] = expect + } + return item +} + +func stripGeneratedSpec(spec map[string]any) map[string]any { + delete(spec, "clusterIP") + delete(spec, "clusterIPs") + ports, isPorts := spec["ports"].([]any) + if !isPorts { + return spec + } + for i := range ports { + pm, isMap := ports[i].(map[string]any) + if !isMap { + continue + } + delete(pm, "nodePort") + } + return spec +} + +func deepCopyMap(in map[string]any) map[string]any { + out := maps.Clone(in) + for k, v := range out { + switch n := v.(type) { + case map[string]any: + out[k] = deepCopyMap(n) + case []any: + cp := make([]any, 0, len(n)) + for _, item := range n { + m, isMap := item.(map[string]any) + if isMap { + cp = append(cp, deepCopyMap(m)) + continue + } + cp = append(cp, item) + } + out[k] = cp + } + } + return out +} + +func stringField(m map[string]any, key string) string { + s, ok := m[key].(string) + if !ok { + return "" + } + return s +} + +func mapField(m map[string]any, key string) map[string]any { + child, ok := m[key].(map[string]any) + if !ok { + return nil + } + return child +} diff --git a/internal/e2e/e2e_semantic_plan_test.go b/internal/e2e/semantic_plan_test.go similarity index 93% rename from internal/e2e/e2e_semantic_plan_test.go rename to internal/e2e/semantic_plan_test.go index 459217e..665af4f 100644 --- a/internal/e2e/e2e_semantic_plan_test.go +++ b/internal/e2e/semantic_plan_test.go @@ -51,8 +51,10 @@ func (c *semanticPlanClient) RenderManifestsWithPrep( return c.result, c.prep, func() {}, nil } -func (s *E2ESuite) TestSemanticPlanDoesNotMutateLive() { - t := s.T() +// Failures in this test should point here, not at the suite caller. +// +//nolint:thelper +func runSemanticPlanDoesNotMutateLive(t *testing.T, env *e2eEnv) { resolved := &spec.ResolvedSpec{ Spec: &spec.Spec{Project: "shop"}, Env: spec.EnvIdentity{Original: "dev"}, @@ -70,7 +72,7 @@ func (s *E2ESuite) TestSemanticPlanDoesNotMutateLive() { }, prep: helm.ReleasePrep{Operation: helm.OperationInstall, NextRevision: 1}, } - p, _, cleanup, err := plan.BuildSemanticPlan(t.Context(), client, s.mapper, nil, plan.SemanticBuildInput{ + p, _, cleanup, err := plan.BuildSemanticPlan(t.Context(), client, env.mapper, nil, plan.SemanticBuildInput{ ClusterContext: kindContext, Resolved: resolved, }) @@ -86,7 +88,7 @@ func (s *E2ESuite) TestSemanticPlanDoesNotMutateLive() { assert.NotEqual(t, "Namespace", c.Resource.Kind) } - cs, _ := s.kubeClients(t) + cs, _ := env.kubeClients(t) _, err = cs.CoreV1().Namespaces().Get(t.Context(), ns, metav1.GetOptions{}) require.True(t, apierrors.IsNotFound(err), "namespace %s should still be absent: %v", ns, err) _, err = cs.CoreV1().ConfigMaps(ns).Get(t.Context(), "app", metav1.GetOptions{}) @@ -97,10 +99,14 @@ func (s *E2ESuite) TestSemanticPlanDoesNotMutateLive() { t.Run("upgrade reads live and does not mutate", func(t *testing.T) { ns := fixtureNamespace("semantic-plan-upgrade") manifest := configMapManifest(ns, "app", "same") + "---\n" + configMapManifest(ns, "other", "same") - cs, dyn := s.kubeClients(t) - _, err := cs.CoreV1().Namespaces().Create(t.Context(), &corev1.Namespace{Name: ns}, metav1.CreateOptions{}) + cs, dyn := env.kubeClients(t) + claim, err := env.claimNamespace(t.Context(), ns) require.NoError(t, err) - t.Cleanup(func() { s.deleteNamespace(t, ns) }) + t.Cleanup(func() { + if cleanupErr := env.cleanupClaim(t, "", claim, nil); cleanupErr != nil { + t.Errorf("cleanup: %v", cleanupErr) + } + }) for _, name := range []string{"app", "other"} { _, err = cs.CoreV1().ConfigMaps(ns).Create(t.Context(), &corev1.ConfigMap{ Name: name, Namespace: ns, @@ -177,7 +183,7 @@ func (s *E2ESuite) TestSemanticPlanDoesNotMutateLive() { NextRevision: 2, }, } - p, _, cleanup, err := plan.BuildSemanticPlan(t.Context(), client, s.mapper, plan.NewDynamicLiveReader(dyn), plan.SemanticBuildInput{ + p, _, cleanup, err := plan.BuildSemanticPlan(t.Context(), client, env.mapper, plan.NewDynamicLiveReader(dyn), plan.SemanticBuildInput{ ClusterContext: kindContext, Resolved: resolved, }) diff --git a/internal/e2e/e2e_test.go b/internal/e2e/suite_test.go similarity index 53% rename from internal/e2e/e2e_test.go rename to internal/e2e/suite_test.go index 8d4e034..55b169e 100644 --- a/internal/e2e/e2e_test.go +++ b/internal/e2e/suite_test.go @@ -29,10 +29,10 @@ import ( "strings" "sync" "testing" + "time" "unicode" "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" "go.yaml.in/yaml/v3" "k8s.io/client-go/discovery" "k8s.io/client-go/discovery/cached/memory" @@ -45,7 +45,9 @@ import ( "deployah.dev/deployah/internal/cmd" "deployah.dev/deployah/internal/localkube" + "deployah.dev/deployah/internal/testing/e2e" + live "deployah.dev/deployah/internal/e2e" inttest "deployah.dev/deployah/internal/testing" ) @@ -55,29 +57,24 @@ const ( maxDNSLabel = 63 ) -var ( - flagScaffold = flag.Bool("e2e.scaffold", false, "generate e2e.yaml skeleton from live cluster state") - flagPreserve = flag.Bool("e2e.preserve", false, "preserve namespace on test failure for debugging") -) +var flagScaffold = flag.Bool("e2e.scaffold", false, "generate e2e.yaml skeleton from live cluster state") -// E2ESuite drives Deployah against a live Kind cluster created by -// `deployah cluster up`. -type E2ESuite struct { - suite.Suite +// e2eEnv is the shared Kind cluster for one TestE2E run. +type e2eEnv struct { kcPath string client klient.Client mapper *restmapper.DeferredDiscoveryRESTMapper mapperMu sync.Mutex scenariosDir string - created bool // TearDownSuite runs cluster down when true } type e2eCase struct { Name string Dir string Project string - Fixture inttest.E2EFixture + Fixture e2e.E2EFixture Parallel bool + LoadErr error } // clusterStatusView mirrors the JSON tags on the unexported status view in @@ -92,62 +89,87 @@ type clusterStatusView struct { // TestE2E runs the Kind-based end-to-end suite. func TestE2E(t *testing.T) { - suite.Run(t, new(E2ESuite)) + env := startCluster(t) + t.Run("TestCRDLifecycle", func(t *testing.T) { runCRDLifecycle(t, env) }) + t.Run("TestCRDNewlyAddedOnUpgrade", func(t *testing.T) { runCRDNewlyAddedOnUpgrade(t, env) }) + t.Run("TestDeployCreatesNamespace", func(t *testing.T) { runDeployCreatesNamespace(t, env) }) + t.Run("TestE2EFixtures", func(t *testing.T) { runE2EFixtures(t, env) }) + t.Run("TestPlanCLI", func(t *testing.T) { runPlanCLI(t, env) }) + t.Run("TestPublicImagePull", func(t *testing.T) { runPublicImagePull(t, env) }) + t.Run("TestSemanticPlanDoesNotMutateLive", func(t *testing.T) { runSemanticPlanDoesNotMutateLive(t, env) }) + t.Run("TestWildcardInstances", func(t *testing.T) { runWildcardInstances(t, env) }) } -// SetupSuite creates the Kind cluster, preloads allowlisted images, and -// builds a discovery RESTMapper. -func (s *E2ESuite) SetupSuite() { - t := s.T() +// startCluster creates the Kind cluster owned by this test. +func startCluster(t *testing.T) *e2eEnv { + t.Helper() + // Cleanup is registered before create. A pre-existing cluster is left + // alone unless DEPLOYAH_E2E_FORCE=1. requireEngine(t) - - scenariosDir, err := filepath.Abs(inttest.TestScenariosDir) - s.Require().NoError(err) - s.scenariosDir = scenariosDir - - // Run from a temp dir; cluster up writes deployah.platform.yaml into cwd. - // t.Chdir restores cwd when the suite ends. + // cluster up writes deployah.platform.yaml into the process directory. + // Child scenarios pass their own directory and do not change it. t.Chdir(t.TempDir()) - requireNoCollision(t) - // Mark before up so TearDownSuite cleans up if create succeeds but a - // later step in cluster up fails. down --force is a no-op when missing. - s.created = true - run(t, "cluster", "up") - - raw := run(t, "cluster", "status", "--output", "json") - var status clusterStatusView - s.Require().NoError(json.Unmarshal([]byte(raw), &status)) - s.Require().Equal(clusterName, status.Name) - s.Require().Equal("running", status.Status) - s.Require().Equal(kindContext, status.Context) - s.Require().True(status.CloudProviderRunning) - - s.kcPath = status.Kubeconfig - s.Require().FileExists(s.kcPath) - - s.client = newKlient(t, s.kcPath, kindContext) - s.preloadImages(t) - s.mapper = newRESTMapper(t, s.client) -} - -// TearDownSuite destroys the Kind cluster only when this suite created it. -func (s *E2ESuite) TearDownSuite() { - if !s.created { - return + scenariosDir, err := filepath.Abs(inttest.TestScenariosDir) + require.NoError(t, err) + found, getErr := lookupCluster(t) + force := os.Getenv("DEPLOYAH_E2E_FORCE") == "1" + intent := live.PlanCluster(found, getErr, force) + if intent == live.ClusterReplace { + t.Log("DEPLOYAH_E2E_FORCE=1: destroying the existing cluster") } - if err := runErr(s.T(), "cluster", "down", "--force"); err != nil { - s.T().Errorf("cluster down failed: %v", err) + switch intent { + case live.ClusterRefuse: + t.Fatal("cluster 'deployah' already exists; " + + "set DEPLOYAH_E2E_FORCE=1 to destroy and recreate it") + case live.ClusterUncertain: + t.Fatalf("cluster lookup: %v", getErr) } + var env e2eEnv + env.scenariosDir = scenariosDir + err = live.PrepareCluster(t, intent, func() error { + // Cleanup runs after t.Context is canceled, so down needs its own deadline. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + _, stderr, downErr := runInErrContext(t, ctx, "", "cluster", "down", "--force") + if downErr == nil { + return nil + } + if trimmed := strings.TrimSpace(stderr); trimmed != "" { + return fmt.Errorf("%w\nstderr:\n%s", downErr, trimmed) + } + return downErr + }, func() error { + run(t, "cluster", "up") + raw := run(t, "cluster", "status", "--output", "json") + var status clusterStatusView + if unmarshalErr := json.Unmarshal([]byte(raw), &status); unmarshalErr != nil { + return unmarshalErr + } + if status.Name != clusterName || status.Status != "running" || status.Context != kindContext || !status.CloudProviderRunning { + return fmt.Errorf("cluster status %+v", status) + } + if _, statErr := os.Stat(status.Kubeconfig); statErr != nil { + return fmt.Errorf("kubeconfig %s: %w", status.Kubeconfig, statErr) + } + env.kcPath = status.Kubeconfig + env.client = newKlient(t, env.kcPath, kindContext) + env.mapper = newRESTMapper(t, env.client) + return nil + }) + require.NoError(t, err) + return &env } -// TestE2EFixtures runs every scenarios/*/e2e.yaml. Parallel fixtures run -// first (bounded by go test -parallel); sequential fixtures run after, -// alphabetically by directory name. -func (s *E2ESuite) TestE2EFixtures() { - t := s.T() - cases := s.loadE2ECases(t) - require.NotEmpty(t, cases, "no e2e.yaml fixtures under %s", s.scenariosDir) +// runE2EFixtures runs every scenarios/*/e2e.yaml. +// Failures in this test should point here, not at the suite caller. +// +//nolint:thelper +func runE2EFixtures(t *testing.T, env *e2eEnv) { + // Parallel fixtures share the go test -parallel limit. Sequential + // fixtures run after them, in directory name order. + cases := env.loadE2ECases(t) + require.NotEmpty(t, cases, "no e2e.yaml fixtures under %s", env.scenariosDir) var parallel, sequential []e2eCase for _, c := range cases { @@ -161,26 +183,12 @@ func (s *E2ESuite) TestE2EFixtures() { return strings.Compare(a.Name, b.Name) }) - t.Run("parallel", func(t *testing.T) { - for _, c := range parallel { - t.Run(c.Name, func(t *testing.T) { - t.Parallel() - s.runE2ECase(t, c) - }) - } - }) - t.Run("sequential", func(t *testing.T) { - for _, c := range sequential { - t.Run(c.Name, func(t *testing.T) { - s.runE2ECase(t, c) - }) - } - }) + live.RunGroups(t, parallel, sequential, func(c e2eCase) string { return c.Name }, env.runE2ECase) } -func (s *E2ESuite) loadE2ECases(t *testing.T) []e2eCase { +func (env *e2eEnv) loadE2ECases(t *testing.T) []e2eCase { t.Helper() - scenarios, err := inttest.DiscoverScenarios(s.scenariosDir) + scenarios, err := inttest.DiscoverScenarios(env.scenariosDir) require.NoError(t, err) seen := map[string]struct{}{} @@ -194,64 +202,90 @@ func (s *E2ESuite) loadE2ECases(t *testing.T) []e2eCase { } seen[sc.ScenarioDir] = struct{}{} - dir := filepath.Join(s.scenariosDir, sc.ScenarioDir) - fx, loadErr := inttest.LoadE2EFixture(sc.E2EFixturePath, dir) - require.NoErrorf(t, loadErr, "load %s", sc.E2EFixturePath) + dir := filepath.Join(env.scenariosDir, sc.ScenarioDir) + fx, loadErr := e2e.LoadE2EFixture(sc.E2EFixturePath, dir) project, projErr := projectName(dir) - require.NoError(t, projErr) - + if projErr != nil && loadErr == nil { + loadErr = projErr + } + parallel := false + if loadErr == nil { + parallel = fx.RunParallel() + } cases = append(cases, e2eCase{ Name: sc.ScenarioDir, Dir: dir, Project: project, Fixture: fx, - Parallel: fx.RunParallel(), + Parallel: parallel, + LoadErr: loadErr, }) } return cases } -func (s *E2ESuite) runE2ECase(t *testing.T, c e2eCase) { - t.Helper() - // Copy before createNamespace so a bad scenario fails before the - // cluster is touched. TempDir cleanup is registered first, so it runs - // after the Kubernetes cleanup below. +// Step lines stay here. t.Helper would point them at the group loop. +// +//nolint:thelper +func (env *e2eEnv) runE2ECase(t *testing.T, c e2eCase) { + if c.LoadErr != nil { + t.Errorf("load %s: %v", c.Name, c.LoadErr) + return + } + // The workspace is local. Preflight must finish before any namespace + // create or other Kubernetes write. dir, err := inttest.NewScenarioWorkspace(t, c.Dir) require.NoError(t, err) + fixturePath := filepath.Join(dir, e2e.FixtureFile) + fx, err := e2e.LoadE2EFixture(fixturePath, dir) + if err != nil { + t.Errorf("load %s: %v", fixturePath, err) + return + } + prepared, err := e2e.PrepareE2EFixture(fixturePath, fx) + if err != nil { + t.Errorf("prepare %s: %v", fixturePath, err) + return + } ns := fixtureNamespace(c.Name) - s.createNamespace(t, ns) + claim, err := env.claimNamespace(t.Context(), ns) + if err != nil { + t.Errorf("namespace %s: %v", ns, err) + return + } t.Cleanup(func() { - if t.Failed() && *flagPreserve { - t.Logf("preserving namespace %s (-e2e.preserve)", ns) + started := time.Now() + cleanupErr := env.cleanupClaim(t, dir, claim, []releaseTarget{{ + Project: c.Project, + Env: fx.Env, + }}) + if cleanupErr != nil { + t.Errorf("cleanup: %v", cleanupErr) return } - // Helm uninstall drops pods so PVC protection finalizers can - // clear. t.Context() is canceled before Cleanup. - cleanupCtx, cancel := context.WithTimeout(context.Background(), namespaceWaitTimeout) - defer cancel() - if _, _, delErr := runInErrContext(t, cleanupCtx, dir, "delete", c.Project, c.Fixture.Env, - "--yes", "--wait", "--allow-missing-platform", - "--context", kindContext, "--namespace", ns); delErr != nil { - t.Logf("cleanup deployah delete failed (non-fatal): %v", delErr) + if testing.Verbose() { + t.Logf("PASS cleanup (%s)", time.Since(started).Round(time.Millisecond)) } - s.deleteNamespace(t, ns) }) - s.assertE2EFixture(t, dir, c.Project, ns, c.Fixture) -} - -func (s *E2ESuite) preloadImages(t *testing.T) { - t.Helper() - m, err := localkube.New() - require.NoError(t, err) - t.Cleanup(func() { - if closeErr := m.Close(); closeErr != nil { - t.Logf("close localkube manager: %v", closeErr) - } + result := e2e.Run(t.Context(), e2e.RunInput{ + Dir: dir, + Namespace: ns, + Project: c.Project, + Env: fx.Env, + Fixture: prepared, + Exec: execCommand(t), + Observe: env.observe, }) - for _, img := range inttest.AllowedE2EImages { - t.Logf("preloading image %s", img) - require.NoErrorf(t, m.LoadImage(t.Context(), clusterName, img), "load %s", img) + info, fails := e2e.StepReports(testing.Verbose(), c.Name, result.Steps) + for _, line := range info { + t.Log(line) + } + for _, line := range fails { + t.Error(line) + } + if *flagScaffold { + env.scaffoldSimple(t, dir, ns, fx) } } @@ -280,8 +314,8 @@ func newKlient(tb testing.TB, kubeconfigPath, contextName string) klient.Client // newResources returns a dedicated Resources client. klient.Client.Resources // mutates a shared namespace field, which is not safe under t.Parallel. -func (s *E2ESuite) newResources(ns string) (*resources.Resources, error) { - res, err := resources.New(s.client.RESTConfig()) +func (env *e2eEnv) newResources(ns string) (*resources.Resources, error) { + res, err := resources.New(env.client.RESTConfig()) if err != nil { return nil, err } @@ -323,35 +357,23 @@ func fixtureNamespace(name string) string { } func run(tb testing.TB, args ...string) string { - tb.Helper() - stdout, _ := runCapture(tb, args...) - return stdout -} - -// runCapture runs deployah in the process cwd and returns stdout and stderr. -func runCapture(tb testing.TB, args ...string) (stdout, stderr string) { tb.Helper() appIO, _, out, errOut := nabattest.NewIO() app := cmd.NewApp(nabat.WithIO(appIO)) err := nabattest.Run(tb, app, args) require.NoErrorf(tb, err, "deployah %s\nstderr:\n%s", strings.Join(args, " "), errOut.String()) - return out.String(), errOut.String() + return out.String() } func runIn(tb testing.TB, dir string, args ...string) (stdout, stderr string) { tb.Helper() - stdout, stderr, err := runInErr(tb, dir, args...) + stdout, stderr, err := runInErrContext(tb, tb.Context(), dir, args...) require.NoErrorf(tb, err, "deployah %s\nstderr:\n%s", strings.Join(args, " "), stderr) return stdout, stderr } -func runInErr(tb testing.TB, dir string, args ...string) (stdout, stderr string, err error) { - tb.Helper() - return runInErrContext(tb, tb.Context(), dir, args...) -} - func runInErrContext(tb testing.TB, ctx context.Context, dir string, args ...string) (stdout, stderr string, err error) { tb.Helper() appIO, _, out, errOut := nabattest.NewIO() @@ -364,18 +386,6 @@ func runInErrContext(tb testing.TB, ctx context.Context, dir string, args ...str return out.String(), errOut.String(), err } -func runErr(tb testing.TB, args ...string) error { - tb.Helper() - _, stderr, err := runInErr(tb, "", args...) - if err == nil { - return nil - } - if trimmed := strings.TrimSpace(stderr); trimmed != "" { - return fmt.Errorf("%w\nstderr:\n%s", err, trimmed) - } - return err -} - func requireEngine(tb testing.TB) { tb.Helper() if err := exec.Command("docker", "info").Run(); err != nil { @@ -386,7 +396,7 @@ func requireEngine(tb testing.TB) { } } -func requireNoCollision(tb testing.TB) { +func lookupCluster(tb testing.TB) (bool, error) { tb.Helper() m, err := localkube.New() require.NoError(tb, err) @@ -395,17 +405,12 @@ func requireNoCollision(tb testing.TB) { tb.Logf("close localkube manager: %v", closeErr) } }) - _, getErr := m.Get(tb.Context(), clusterName) if errors.Is(getErr, localkube.ErrNotFound) { - return + return false, nil } - require.NoError(tb, getErr) - - if os.Getenv("DEPLOYAH_E2E_FORCE") != "1" { - tb.Fatal("cluster 'deployah' already exists; " + - "set DEPLOYAH_E2E_FORCE=1 to destroy and recreate it") + if getErr != nil { + return false, getErr } - tb.Log("DEPLOYAH_E2E_FORCE=1: destroying the existing cluster") - require.NoError(tb, runErr(tb, "cluster", "down", "--force")) + return true, nil } diff --git a/internal/e2e/testdata/added-widget-crd.yaml b/internal/e2e/testdata/added-widget-crd.yaml new file mode 100644 index 0000000..272a004 --- /dev/null +++ b/internal/e2e/testdata/added-widget-crd.yaml @@ -0,0 +1,18 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: addedwidgets.example.com +spec: + group: example.com + scope: Namespaced + names: + kind: AddedWidget + plural: addedwidgets + singular: addedwidget + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object diff --git a/internal/e2e/e2e_wildcard_test.go b/internal/e2e/wildcard_test.go similarity index 61% rename from internal/e2e/e2e_wildcard_test.go rename to internal/e2e/wildcard_test.go index d18acd2..ddd74ee 100644 --- a/internal/e2e/e2e_wildcard_test.go +++ b/internal/e2e/wildcard_test.go @@ -16,8 +16,8 @@ package e2e_test import ( - "context" "encoding/json" + "fmt" "path/filepath" "strings" "testing" @@ -32,6 +32,7 @@ import ( "deployah.dev/deployah/internal/helm" "deployah.dev/deployah/internal/spec" + "deployah.dev/deployah/internal/testing/e2e" inttest "deployah.dev/deployah/internal/testing" ) @@ -44,95 +45,132 @@ const ( // TestWildcardInstances deploys two review/* siblings and checks identity, // list/status isolation, independent runtime env, a manual run, and delete. -func (s *E2ESuite) TestWildcardInstances() { - t := s.T() - src := filepath.Join(s.scenariosDir, "wildcard-review") +// Failures in this test should point here, not at the suite caller. +// +//nolint:thelper +func runWildcardInstances(t *testing.T, env *e2eEnv) { + src := filepath.Join(env.scenariosDir, "wildcard-review") require.DirExists(t, src) dir, err := inttest.NewScenarioWorkspace(t, src) require.NoError(t, err) ns := fixtureNamespace("wildcard-review") - s.createNamespace(t, ns) + claim, err := env.claimNamespace(t.Context(), ns) + require.NoError(t, err) t.Cleanup(func() { - cleanupCtx, cancel := context.WithTimeout(context.Background(), namespaceWaitTimeout) - defer cancel() - for _, env := range []string{wildcardEnvA, wildcardEnvB} { - if _, _, delErr := runInErrContext(t, cleanupCtx, dir, "delete", wildcardProject, env, - "--yes", "--wait", "--allow-missing-platform", - "--context", kindContext, "--namespace", ns); delErr != nil { - t.Logf("cleanup delete %s failed (non-fatal): %v", env, delErr) - } + if cleanupErr := env.cleanupClaim(t, dir, claim, []releaseTarget{ + {Project: wildcardProject, Env: wildcardEnvA}, + {Project: wildcardProject, Env: wildcardEnvB}, + }); cleanupErr != nil { + t.Errorf("cleanup: %v", cleanupErr) } - s.deleteNamespace(t, ns) }) releaseA := helm.GenerateReleaseName(wildcardProject, wildcardEnvA) releaseB := helm.GenerateReleaseName(wildcardProject, wildcardEnvB) require.NotEqual(t, releaseA, releaseB) - s.deployWildcard(t, dir, ns, wildcardEnvA) - s.waitWildcardReady(t, ns, wildcardEnvA, releaseA, "pr-123") + env.deployWildcard(t, dir, ns, wildcardEnvA) + env.waitWildcardReady(t, ns, wildcardEnvA, releaseA, "pr-123") - s.deployWildcard(t, dir, ns, wildcardEnvB) - s.waitWildcardReady(t, ns, wildcardEnvB, releaseB, "pr-456") + env.deployWildcard(t, dir, ns, wildcardEnvB) + env.waitWildcardReady(t, ns, wildcardEnvB, releaseB, "pr-456") - s.assertListJSON(t, dir, ns, "review", []string{wildcardEnvA, wildcardEnvB}) - s.assertListJSON(t, dir, ns, wildcardEnvA, []string{wildcardEnvA}) - s.assertStatusJSON(t, dir, ns, wildcardEnvA, releaseA) + env.assertListJSON(t, dir, ns, "review", []string{wildcardEnvA, wildcardEnvB}) + env.assertListJSON(t, dir, ns, wildcardEnvA, []string{wildcardEnvA}) + env.assertStatusJSON(t, dir, ns, wildcardEnvA, releaseA) - s.assertRunIsolated(t, dir, ns, releaseA) + env.assertRunIsolated(t, dir, ns, releaseA) runIn(t, dir, "delete", wildcardProject, wildcardEnvA, "--yes", "--wait", "--allow-missing-platform", "--context", kindContext, "--namespace", ns) - zero := 0 - s.waitForResources(t, ns, []inttest.ResourceAssertion{ - { - MinCount: &zero, - Match: map[string]any{ - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": map[string]any{ - "labels": map[string]any{ - spec.LabelInstance: releaseA, - }, - }, - }, - }, + env.waitWild(t, ns, []wildCheck{ + {sel: e2e.ResourceSelect{ + APIVersion: "apps/v1", + Kind: "Deployment", + Labels: map[string]string{spec.LabelInstance: releaseA}, + }, empty: true}, wildcardDeployment(wildcardEnvB, releaseB), wildcardEnvConfigMap(releaseB, "pr-456"), }, 3*time.Minute) - s.assertStatusJSON(t, dir, ns, wildcardEnvB, releaseB) + env.assertStatusJSON(t, dir, ns, wildcardEnvB, releaseB) } -func (s *E2ESuite) deployWildcard(t *testing.T, dir, ns, env string) { +func (env *e2eEnv) deployWildcard(t *testing.T, dir, ns, environment string) { t.Helper() - runIn(t, dir, "deploy", env, "--context", kindContext, "--namespace", ns) + runIn(t, dir, "deploy", environment, "--context", kindContext, "--namespace", ns) } -func (s *E2ESuite) waitWildcardReady(t *testing.T, ns, original, release, slot string) { +func (env *e2eEnv) waitWildcardReady(t *testing.T, ns, original, release, slot string) { t.Helper() - s.waitForResources(t, ns, []inttest.ResourceAssertion{ + env.waitWild(t, ns, []wildCheck{ wildcardDeployment(original, release), wildcardEnvConfigMap(release, slot), }, 3*time.Minute) } -func wildcardDeployment(original, release string) inttest.ResourceAssertion { - return inttest.ResourceAssertion{ - Match: map[string]any{ - "apiVersion": "apps/v1", - "kind": "Deployment", +type wildCheck struct { + sel e2e.ResourceSelect + expect map[string]any + empty bool +} + +func (env *e2eEnv) waitWild(t *testing.T, ns string, checks []wildCheck, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + var last []string + for { + last = nil + for i, check := range checks { + objects, err := env.observe(t.Context(), check.sel, ns) + if err != nil { + last = append(last, fmt.Sprintf("check[%d]: %v", i, err)) + continue + } + if check.empty { + if len(objects) != 0 { + last = append(last, fmt.Sprintf("check[%d]: want 0, got %d", i, len(objects))) + } + continue + } + if len(objects) == 0 { + last = append(last, fmt.Sprintf("check[%d]: no resources selected", i)) + continue + } + for _, obj := range objects { + for _, diff := range inttest.DiffSubset("$", check.expect, obj) { + last = append(last, fmt.Sprintf("check[%d]: %s", i, diff)) + } + } + } + if len(last) == 0 { + return + } + if time.Now().After(deadline) { + t.Fatalf("wildcard resources not ready:\n%s", strings.Join(last, "\n")) + } + time.Sleep(resourcePollInterval) + } +} + +func wildcardDeployment(original, release string) wildCheck { + return wildCheck{ + sel: e2e.ResourceSelect{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: release + "-web", + Labels: map[string]string{ + spec.LabelProject: wildcardProject, + spec.LabelEnvironment: "review", + spec.LabelInstance: release, + }, + }, + expect: map[string]any{ "metadata": map[string]any{ - "name": release + "-web", - "labels": map[string]any{ - spec.LabelProject: wildcardProject, - spec.LabelEnvironment: "review", - spec.LabelInstance: release, - }, "annotations": map[string]any{ spec.AnnotationEnvironmentInstance: original, }, @@ -142,28 +180,28 @@ func wildcardDeployment(original, release string) inttest.ResourceAssertion { } } -func wildcardEnvConfigMap(release, slot string) inttest.ResourceAssertion { - return inttest.ResourceAssertion{ - Match: map[string]any{ - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": map[string]any{ - "name": release + "-web-env", - "labels": map[string]any{ - spec.LabelEnvironment: "review", - spec.LabelInstance: release, - }, +func wildcardEnvConfigMap(release, slot string) wildCheck { + return wildCheck{ + sel: e2e.ResourceSelect{ + APIVersion: "v1", + Kind: "ConfigMap", + Name: release + "-web-env", + Labels: map[string]string{ + spec.LabelEnvironment: "review", + spec.LabelInstance: release, }, + }, + expect: map[string]any{ "data": map[string]any{"PR_SLOT": slot}, }, } } -func (s *E2ESuite) assertListJSON(t *testing.T, dir, ns, env string, wantInstances []string) { +func (env *e2eEnv) assertListJSON(t *testing.T, dir, ns, environment string, wantInstances []string) { t.Helper() stdout, _ := runIn(t, dir, "list", "--project", wildcardProject, - "--environment", env, + "--environment", environment, "--output", "json", "--context", kindContext, "--namespace", ns) @@ -182,31 +220,31 @@ func (s *E2ESuite) assertListJSON(t *testing.T, dir, ns, env string, wantInstanc assert.ElementsMatch(t, wantInstances, got) } -func (s *E2ESuite) assertStatusJSON(t *testing.T, dir, ns, env, release string) { +func (env *e2eEnv) assertStatusJSON(t *testing.T, dir, ns, environment, release string) { t.Helper() stdout, _ := runIn(t, dir, "status", wildcardProject, - "--environment", env, + "--environment", environment, "--output", "json", "--context", kindContext, "--namespace", ns) var rows []map[string]any require.NoError(t, json.Unmarshal([]byte(stdout), &rows), stdout) require.Len(t, rows, 1, stdout) - assert.Equal(t, env, rows[0]["instance"]) + assert.Equal(t, environment, rows[0]["instance"]) assert.Equal(t, release, rows[0]["release"]) sibling := wildcardEnvB - if env == wildcardEnvB { + if environment == wildcardEnvB { sibling = wildcardEnvA } assert.NotContains(t, stdout, sibling) } -func (s *E2ESuite) assertRunIsolated(t *testing.T, dir, ns, releaseA string) { +func (env *e2eEnv) assertRunIsolated(t *testing.T, dir, ns, releaseA string) { t.Helper() runIn(t, dir, "run", "ping", wildcardEnvA, "--context", kindContext, "--yes", "--namespace", ns) - res, err := s.newResources(ns) + res, err := env.newResources(ns) require.NoError(t, err) selector := labels.Set{ spec.LabelProject: wildcardProject, diff --git a/internal/testing/celassert/diagnose.go b/internal/testing/celassert/diagnose.go new file mode 100644 index 0000000..e7ce9d5 --- /dev/null +++ b/internal/testing/celassert/diagnose.go @@ -0,0 +1,145 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package celassert + +import ( + "context" + "fmt" + "strings" + + "cel.dev/cel-go/common/ast" + "cel.dev/cel-go/common/types/ref" + "cel.dev/cel-go/interpreter" +) + +// comparisonOps maps a CEL overload to the operator written in the source. +// A literal on the left is supported. Other shapes stay empty. +var comparisonOps = map[string]string{ + "_==_": "==", + "_!=_": "!=", + "_<_": "<", + "_<=_": "<=", + "_>_": ">", + "_>=_": ">=", +} + +// Diagnose explains a false field comparison when the actual value is known. +// The source operator is kept. Other expressions return an empty string. +func (p *Program) Diagnose(ctx context.Context, vars map[string]any) string { + // A field compared with a literal, in either order, includes the observed + // value. Eval itself is unchanged. + if p == nil || p.tracked == nil || p.root == nil || ctx == nil { + return "" + } + p.mu.Lock() + defer p.mu.Unlock() + prepared, err := prepare(vars) + if err != nil { + return "" + } + _, details, err := p.tracked.ContextEval(ctx, prepared) + if err != nil || details == nil { + return "" + } + return comparisonDetail(p.root, details.State()) +} + +func comparisonDetail(root ast.Expr, state interpreter.EvalState) string { + if root == nil || root.Kind() != ast.CallKind || state == nil { + return "" + } + call := root.AsCall() + op, ok := comparisonOps[call.FunctionName()] + if !ok || call.IsMemberFunction() { + return "" + } + args := call.Args() + if len(args) != 2 { + return "" + } + if text := fieldLiteral(op, args[0], args[1], true, state); text != "" { + return text + } + return fieldLiteral(op, args[1], args[0], false, state) +} + +func fieldLiteral(op string, field, lit ast.Expr, fieldOnLeft bool, state interpreter.EvalState) string { + path, pathOK := selectorPath(field) + text, litOK := literalText(lit) + if !pathOK || !litOK { + return "" + } + actual, ok := state.Value(field.ID()) + if !ok { + return "" + } + expected := path + " " + op + " " + text + if !fieldOnLeft { + expected = text + " " + op + " " + path + } + return fmt.Sprintf("expected: %s\nactual: %s = %s", expected, path, formatCEL(actual)) +} + +func selectorPath(e ast.Expr) (string, bool) { + switch e.Kind() { + case ast.IdentKind: + name := e.AsIdent() + if name == "" { + return "", false + } + return name, true + case ast.SelectKind: + sel := e.AsSelect() + if sel.IsTestOnly() { + return "", false + } + base, ok := selectorPath(sel.Operand()) + if !ok || sel.FieldName() == "" { + return "", false + } + return base + "." + sel.FieldName(), true + default: + return "", false + } +} + +func literalText(e ast.Expr) (string, bool) { + if e.Kind() != ast.LiteralKind { + return "", false + } + val := e.AsLiteral() + if val == nil { + return "", false + } + return formatAny(val.Value()), true +} + +func formatCEL(val ref.Val) string { + if val == nil { + return "null" + } + return formatAny(val.Value()) +} + +func formatAny(v any) string { + switch n := v.(type) { + case string: + return fmt.Sprintf("%q", n) + case nil: + return "null" + default: + return strings.TrimSpace(fmt.Sprint(n)) + } +} diff --git a/internal/testing/celassert/diagnose_test.go b/internal/testing/celassert/diagnose_test.go new file mode 100644 index 0000000..10a2ba6 --- /dev/null +++ b/internal/testing/celassert/diagnose_test.go @@ -0,0 +1,133 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package celassert + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDiagnose(t *testing.T) { + t.Parallel() + doc := map[string]any{ + "helmAction": "upgrade", + "summary": map[string]any{"total": int64(0), "create": int64(0)}, + } + vars := map[string]any{"plan": doc} + tests := []struct { + name string + expr string + want string + evalFalse bool + }{ + { + name: "equal", + expr: "plan.summary.total == 1", + want: "expected: plan.summary.total == 1\nactual: plan.summary.total = 0", + evalFalse: true, + }, + { + name: "not equal", + expr: "plan.summary.total != 0", + want: "expected: plan.summary.total != 0\nactual: plan.summary.total = 0", + evalFalse: true, + }, + { + name: "greater than", + expr: "plan.summary.create > 0", + want: "expected: plan.summary.create > 0\nactual: plan.summary.create = 0", + evalFalse: true, + }, + { + name: "greater or equal", + expr: "plan.summary.create >= 1", + want: "expected: plan.summary.create >= 1\nactual: plan.summary.create = 0", + evalFalse: true, + }, + { + name: "less than", + expr: "plan.summary.total < 0", + want: "expected: plan.summary.total < 0\nactual: plan.summary.total = 0", + evalFalse: true, + }, + { + name: "less or equal", + expr: "plan.summary.total <= -1", + want: "expected: plan.summary.total <= -1\nactual: plan.summary.total = 0", + evalFalse: true, + }, + { + name: "string", + expr: `plan.helmAction == "install"`, + want: "expected: plan.helmAction == \"install\"\nactual: plan.helmAction = \"upgrade\"", + evalFalse: true, + }, + { + name: "nested field", + expr: "plan.summary.total == 2", + want: "expected: plan.summary.total == 2\nactual: plan.summary.total = 0", + evalFalse: true, + }, + { + name: "reversed literal", + expr: "0 < plan.summary.create", + want: "expected: 0 < plan.summary.create\nactual: plan.summary.create = 0", + evalFalse: true, + }, + { + name: "reversed equality", + expr: "1 == plan.summary.total", + want: "expected: 1 == plan.summary.total\nactual: plan.summary.total = 0", + evalFalse: true, + }, + { + name: "reversed string", + expr: `"install" == plan.helmAction`, + want: "expected: \"install\" == plan.helmAction\nactual: plan.helmAction = \"upgrade\"", + evalFalse: true, + }, + { + name: "logical expression", + expr: "plan.summary.total == 1 || plan.summary.total == 2", + evalFalse: true, + }, + { + name: "call is not a field comparison", + expr: "plan.summary.total == size(plan.summary)", + evalFalse: true, + }, + { + name: "missing field", + expr: "plan.missing == 1", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + prog, err := Compile(tt.expr, WithVariable("plan")) + require.NoError(t, err) + evalErr := prog.Eval(t.Context(), vars) + require.Error(t, evalErr) + assert.Equal(t, tt.evalFalse, errors.Is(evalErr, ErrFalse)) + assert.Equal(t, tt.want, prog.Diagnose(t.Context(), vars)) + again := prog.Eval(t.Context(), vars) + require.Error(t, again) + assert.Equal(t, tt.evalFalse, errors.Is(again, ErrFalse)) + }) + } +} diff --git a/internal/testing/celassert/program.go b/internal/testing/celassert/program.go index 693c42a..65360e8 100644 --- a/internal/testing/celassert/program.go +++ b/internal/testing/celassert/program.go @@ -23,9 +23,12 @@ import ( "io" "math/big" "strings" + "sync" "cel.dev/cel-go/cel" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + celast "cel.dev/cel-go/common/ast" ) const ( @@ -59,6 +62,9 @@ var ( type Program struct { expression string prog cel.Program + tracked cel.Program + root celast.Expr + mu sync.Mutex } // Option configures Compile. @@ -142,7 +148,20 @@ func Compile(expression string, opts ...Option) (*Program, error) { if err != nil { return nil, fmt.Errorf("compiling expression: %w", err) } - return &Program{expression: expression, prog: prog}, nil + tracked, err := env.Program(ast, + cel.CostLimit(cfg.cost), + cel.InterruptCheckFrequency(DefaultInterruptCheckFrequency), + cel.EvalOptions(cel.OptTrackState), + ) + if err != nil { + return nil, fmt.Errorf("compiling expression: %w", err) + } + return &Program{ + expression: expression, + prog: prog, + tracked: tracked, + root: ast.NativeRep().Expr(), + }, nil } func (cfg config) validate(expression string) error { diff --git a/internal/testing/celassert/program_test.go b/internal/testing/celassert/program_test.go index 5a71bf6..b1855c7 100644 --- a/internal/testing/celassert/program_test.go +++ b/internal/testing/celassert/program_test.go @@ -496,8 +496,6 @@ spec: func TestEval_JobConfigMapOwnership(t *testing.T) { t.Parallel() - // Name link from internal/helm/generate_env_test.go. - // Owner link from internal/e2e/e2e_wildcard_test.go. // UIDs are fixed literals, not client-generated values. migrateCM := "shop-dev-migrate-env" resources := []interface{}{ diff --git a/internal/testing/doc.go b/internal/testing/doc.go index e45d524..79d25f8 100644 --- a/internal/testing/doc.go +++ b/internal/testing/doc.go @@ -14,10 +14,6 @@ // Package testing provides integration test helpers for Deployah scenarios. // -// Scenario directories under scenarios/ hold sample specs, environment -// files, expected Kubernetes output, and optional e2e.yaml Kind fixtures. -// [DiscoverScenarios] finds them; [IntegrationTestSuite] loads a spec, -// generates a chart, renders templates, and compares results to golden -// files. [LoadE2EFixture] decodes e2e.yaml. [NewScenarioWorkspace] copies a -// scenario into a temporary directory for end-to-end tests. +// Scenario directories under scenarios/ hold specs, environment files, +// expected Kubernetes output, and optional e2e.yaml Kind fixtures. package testing diff --git a/internal/testing/e2e.schema.json b/internal/testing/e2e.schema.json deleted file mode 100644 index c1e4814..0000000 --- a/internal/testing/e2e.schema.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://deployah.dev/schemas/e2e.schema.json", - "title": "Deployah e2e fixture", - "description": "Kind cluster assertions for a scenarios/*/e2e.yaml file. Exactly one of resources (simple deploy-then-assert) or steps (explicit CLI sequence) must be set.", - "type": "object", - "additionalProperties": false, - "required": ["env"], - "oneOf": [ - { - "required": ["resources"], - "not": { "required": ["steps"] } - }, - { - "required": ["steps"], - "not": { "required": ["resources"] } - } - ], - "properties": { - "env": { - "type": "string", - "minLength": 1, - "description": "Positional environment argument passed to deploy/run/delete." - }, - "parallel": { - "type": "boolean", - "description": "When false, the fixture runs sequentially after parallel fixtures. Default true." - }, - "timeout": { - "type": "string", - "description": "Default per-step wait (Go duration, e.g. 3m). Default 3m." - }, - "resources": { - "type": "array", - "minItems": 1, - "items": { "$ref": "#/$defs/resourceAssertion" } - }, - "steps": { - "type": "array", - "minItems": 1, - "items": { "$ref": "#/$defs/step" } - } - }, - "$defs": { - "resourceAssertion": { - "type": "object", - "additionalProperties": false, - "required": ["match"], - "properties": { - "match": { - "type": "object", - "description": "Partial Kubernetes object compared as a subset of the live object.", - "required": ["apiVersion", "kind"], - "properties": { - "apiVersion": { "type": "string", "minLength": 1 }, - "kind": { "type": "string", "minLength": 1 } - } - }, - "minCount": { - "type": "integer", - "minimum": 0, - "description": "Minimum matching resources when selecting by labels. Default 1. 0 asserts absence. Must be 0 or 1 when match.metadata.name is set." - } - } - }, - "step": { - "type": "object", - "additionalProperties": false, - "oneOf": [ - { "required": ["deploy"], "not": { "anyOf": [{ "required": ["run"] }, { "required": ["logs"] }, { "required": ["delete"] }] } }, - { "required": ["run"], "not": { "anyOf": [{ "required": ["deploy"] }, { "required": ["logs"] }, { "required": ["delete"] }] } }, - { "required": ["logs"], "not": { "anyOf": [{ "required": ["deploy"] }, { "required": ["run"] }, { "required": ["delete"] }] } }, - { "required": ["delete"], "not": { "anyOf": [{ "required": ["deploy"] }, { "required": ["run"] }, { "required": ["logs"] }] } } - ], - "properties": { - "deploy": { "$ref": "#/$defs/deployOp" }, - "run": { "$ref": "#/$defs/runOp" }, - "logs": { "$ref": "#/$defs/logsOp" }, - "delete": { "$ref": "#/$defs/deleteOp" }, - "timeout": { "type": "string" }, - "stderrContains": { "type": "string" }, - "stdoutContains": { "type": "string" }, - "resources": { - "type": "array", - "items": { "$ref": "#/$defs/resourceAssertion" } - } - } - }, - "deployOp": { - "type": "object", - "additionalProperties": false, - "properties": { - "spec": { "type": "string" }, - "args": { - "type": "array", - "items": { "type": "string" } - } - } - }, - "runOp": { - "type": "object", - "additionalProperties": false, - "required": ["task"], - "properties": { - "task": { "type": "string", "minLength": 1 } - } - }, - "logsOp": { - "type": "object", - "additionalProperties": false, - "required": ["component", "contains"], - "properties": { - "component": { "type": "string", "minLength": 1 }, - "contains": { "type": "string", "minLength": 1 } - } - }, - "deleteOp": { - "type": "object", - "additionalProperties": false - } - } -} diff --git a/internal/testing/e2e/cleanup.go b/internal/testing/e2e/cleanup.go new file mode 100644 index 0000000..ea24186 --- /dev/null +++ b/internal/testing/e2e/cleanup.go @@ -0,0 +1,106 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "errors" + "fmt" + "time" +) + +// DefaultPhaseTimeout is the budget for one cleanup phase. +const DefaultPhaseTimeout = 2 * time.Minute + +// CleanupPhase is one cleanup action with its own deadline. +type CleanupPhase struct { + Name string + Run func(context.Context) error +} + +// CleanupOption configures [RunCleanup]. +type CleanupOption func(*cleanupConfig) + +type cleanupConfig struct { + phaseTimeout time.Duration + budget time.Duration +} + +// WithPhaseTimeout sets how long each phase may run. +func WithPhaseTimeout(d time.Duration) CleanupOption { + return func(cfg *cleanupConfig) { cfg.phaseTimeout = d } +} + +// WithBudget sets the outer cleanup deadline explicitly. +func WithBudget(d time.Duration) CleanupOption { + return func(cfg *cleanupConfig) { cfg.budget = d } +} + +// RunCleanup runs phases under one overall deadline. +func RunCleanup(ctx context.Context, phases []CleanupPhase, opts ...CleanupOption) error { + // Each phase takes its own timeout from ctx, so one slow phase cannot + // eat the time of the next. A lost or uncertain namespace claim stops + // the rest. Other errors are kept, and the next phase still runs. + // Pass a live context. The Kind suite uses a fresh one because cleanup + // runs after the test context is canceled. + if ctx == nil { + return errors.New("cleanup context is nil") + } + cfg := cleanupConfig{phaseTimeout: DefaultPhaseTimeout} + for _, opt := range opts { + opt(&cfg) + } + if cfg.phaseTimeout <= 0 { + return errors.New("cleanup phase timeout must be positive") + } + if len(phases) == 0 { + return nil + } + budget := cfg.budget + if budget <= 0 { + budget = cfg.phaseTimeout * time.Duration(len(phases)) + } + outer, cancelOuter := context.WithTimeout(ctx, budget) + defer cancelOuter() + + var errs []error + for _, phase := range phases { + if outer.Err() != nil { + errs = append(errs, fmt.Errorf("%s: %w", phase.Name, context.DeadlineExceeded)) + break + } + err := runPhase(outer, cfg.phaseTimeout, phase) + if err == nil { + continue + } + errs = append(errs, err) + if errors.Is(err, ErrNotOwner) || errors.Is(err, ErrOwnershipUncertain) { + break + } + } + return errors.Join(errs...) +} + +func runPhase(parent context.Context, timeout time.Duration, phase CleanupPhase) error { + phaseCtx, cancel := context.WithTimeout(parent, timeout) + defer cancel() + if phase.Run == nil { + return fmt.Errorf("%s: nil cleanup function", phase.Name) + } + if err := phase.Run(phaseCtx); err != nil { + return fmt.Errorf("%s: %w", phase.Name, err) + } + return nil +} diff --git a/internal/testing/e2e/doc.go b/internal/testing/e2e/doc.go new file mode 100644 index 0000000..d7e2759 --- /dev/null +++ b/internal/testing/e2e/doc.go @@ -0,0 +1,18 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package e2e loads scenario YAML, checks it, and runs the steps. +// +// The Kind suite calls this package. Nothing here starts a cluster. +package e2e diff --git a/internal/testing/e2e/e2e.schema.json b/internal/testing/e2e/e2e.schema.json new file mode 100644 index 0000000..faaca7e --- /dev/null +++ b/internal/testing/e2e/e2e.schema.json @@ -0,0 +1,163 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://deployah.dev/schemas/e2e.schema.json", + "title": "Deployah e2e fixture", + "description": "Declarative Kind scenario for scenarios/*/e2e.yaml. steps is a non-empty list. Each step has exactly one operation.", + "type": "object", + "additionalProperties": false, + "required": ["env", "steps"], + "properties": { + "env": { + "type": "string", + "minLength": 1, + "description": "Positional environment argument passed to deploy, plan, run, and delete." + }, + "parallel": { + "type": "boolean", + "description": "When false, the fixture runs after parallel fixtures. Default true." + }, + "timeout": { + "type": "string", + "minLength": 1, + "description": "Default per-step wait as a Go duration, for example 3m. Default 3m." + }, + "steps": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/step" } + } + }, + "$defs": { + "step": { + "type": "object", + "additionalProperties": false, + "properties": { + "deploy": { "$ref": "#/$defs/deployOp" }, + "plan": { "$ref": "#/$defs/planOp" }, + "run": { "$ref": "#/$defs/runOp" }, + "logs": { "$ref": "#/$defs/logsOp" }, + "delete": { "$ref": "#/$defs/deleteOp" }, + "assert": { "$ref": "#/$defs/assertList" }, + "timeout": { "type": "string", "minLength": 1 }, + "stopOnFailure": { "type": "boolean" } + }, + "oneOf": [ + { "required": ["deploy"], "not": { "anyOf": [{ "required": ["plan"] }, { "required": ["run"] }, { "required": ["logs"] }, { "required": ["delete"] }, { "required": ["assert"] }] } }, + { "required": ["plan"], "not": { "anyOf": [{ "required": ["deploy"] }, { "required": ["run"] }, { "required": ["logs"] }, { "required": ["delete"] }, { "required": ["assert"] }] } }, + { "required": ["run"], "not": { "anyOf": [{ "required": ["deploy"] }, { "required": ["plan"] }, { "required": ["logs"] }, { "required": ["delete"] }, { "required": ["assert"] }] } }, + { "required": ["logs"], "not": { "anyOf": [{ "required": ["deploy"] }, { "required": ["plan"] }, { "required": ["run"] }, { "required": ["delete"] }, { "required": ["assert"] }] } }, + { "required": ["delete"], "not": { "anyOf": [{ "required": ["deploy"] }, { "required": ["plan"] }, { "required": ["run"] }, { "required": ["logs"] }, { "required": ["assert"] }] } }, + { "required": ["assert"], "not": { "anyOf": [{ "required": ["deploy"] }, { "required": ["plan"] }, { "required": ["run"] }, { "required": ["logs"] }, { "required": ["delete"] }] } } + ] + }, + "outputExpectation": { + "type": "object", + "additionalProperties": false, + "properties": { + "exitCode": { "type": "integer", "minimum": 0 }, + "stdout": { "$ref": "#/$defs/streamMatch" }, + "stderr": { "$ref": "#/$defs/streamMatch" } + } + }, + "streamMatch": { + "type": "object", + "additionalProperties": false, + "required": ["contains"], + "properties": { + "contains": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + } + } + }, + "exprList": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + }, + "deployOp": { + "type": "object", + "additionalProperties": false, + "properties": { + "spec": { "type": "string", "minLength": 1 }, + "args": { "type": "array", "items": { "type": "string" } }, + "output": { "$ref": "#/$defs/outputExpectation" } + } + }, + "planOp": { + "type": "object", + "additionalProperties": false, + "properties": { + "detailedExitCode": { "type": "boolean" }, + "output": { "$ref": "#/$defs/outputExpectation" }, + "expr": { "$ref": "#/$defs/exprList" } + } + }, + "runOp": { + "type": "object", + "additionalProperties": false, + "required": ["task"], + "properties": { + "task": { "type": "string", "minLength": 1 }, + "output": { "$ref": "#/$defs/outputExpectation" } + } + }, + "logsOp": { + "type": "object", + "additionalProperties": false, + "required": ["component", "contains"], + "properties": { + "component": { "type": "string", "minLength": 1 }, + "contains": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + } + } + }, + "deleteOp": { + "type": "object", + "additionalProperties": false, + "properties": { + "output": { "$ref": "#/$defs/outputExpectation" } + } + }, + "assertList": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/resourceAssertion" } + }, + "resourceAssertion": { + "type": "object", + "additionalProperties": false, + "required": ["select"], + "anyOf": [ + { "required": ["expect"] }, + { "required": ["expr"] } + ], + "properties": { + "select": { "$ref": "#/$defs/resourceSelect" }, + "expect": { "type": "object" }, + "expr": { "$ref": "#/$defs/exprList" }, + "stopOnFailure": { "type": "boolean" } + } + }, + "resourceSelect": { + "type": "object", + "additionalProperties": false, + "required": ["apiVersion", "kind"], + "properties": { + "apiVersion": { "type": "string", "minLength": 1 }, + "kind": { "type": "string", "minLength": 1 }, + "name": { "type": "string", "minLength": 1 }, + "namespace": { "type": "string", "minLength": 1 }, + "labels": { + "type": "object", + "minProperties": 1, + "additionalProperties": { "type": "string" } + } + } + } + } +} diff --git a/internal/testing/e2e/fixture.go b/internal/testing/e2e/fixture.go new file mode 100644 index 0000000..60b9a92 --- /dev/null +++ b/internal/testing/e2e/fixture.go @@ -0,0 +1,266 @@ +// Copyright 2025 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "sigs.k8s.io/yaml" + + k8sjson "k8s.io/apimachinery/pkg/util/json" +) + +// FixtureFile is the Kind fixture filename in a scenario directory. +const FixtureFile = "e2e.yaml" + +// DefaultE2EStepTimeout is the wait applied when e2e.yaml omits timeout. +const DefaultE2EStepTimeout = 3 * time.Minute + +// E2EFixture is the decoded form of scenarios/*/e2e.yaml. +type E2EFixture struct { + Env string `json:"env"` + Parallel *bool `json:"parallel,omitempty"` + Timeout string `json:"timeout,omitempty"` + Steps []Step `json:"steps"` +} + +// Step is one operation plus optional timeout and stopOnFailure. +// Exactly one operation is set. +type Step struct { + Deploy *DeployOp `json:"deploy,omitempty"` + Plan *PlanOp `json:"plan,omitempty"` + Run *RunOp `json:"run,omitempty"` + Logs *LogsOp `json:"logs,omitempty"` + Delete *DeleteOp `json:"delete,omitempty"` + Assert []ResourceAssertion `json:"assert,omitempty"` + Timeout string `json:"timeout,omitempty"` + StopOnFailure *bool `json:"stopOnFailure,omitempty"` +} + +// OutputExpectation checks CLI output for deploy, plan, run, and delete. +// A missing exit code means 0. +type OutputExpectation struct { + ExitCode *int `json:"exitCode,omitempty"` + Stdout *StreamMatch `json:"stdout,omitempty"` + Stderr *StreamMatch `json:"stderr,omitempty"` +} + +// StreamMatch lists Go regular expressions. Every pattern must match. +type StreamMatch struct { + Contains []string `json:"contains"` +} + +// DeployOp maps to `deployah deploy`. +type DeployOp struct { + Spec string `json:"spec,omitempty"` + Args []string `json:"args,omitempty"` + Output *OutputExpectation `json:"output,omitempty"` +} + +// PlanOp maps to `deployah plan` with JSON output. +type PlanOp struct { + DetailedExitCode bool `json:"detailedExitCode,omitempty"` + Output *OutputExpectation `json:"output,omitempty"` + Expr []string `json:"expr,omitempty"` +} + +// RunOp maps to `deployah run`. +type RunOp struct { + Task string `json:"task"` + Output *OutputExpectation `json:"output,omitempty"` +} + +// LogsOp maps to `deployah logs`. +type LogsOp struct { + Component string `json:"component"` + Contains []string `json:"contains"` +} + +// DeleteOp maps to `deployah delete`. +type DeleteOp struct { + Output *OutputExpectation `json:"output,omitempty"` +} + +// ResourceAssertion selects Kubernetes objects and checks them. +type ResourceAssertion struct { + Select ResourceSelect `json:"select"` + Expect map[string]any `json:"expect,omitempty"` + Expr []string `json:"expr,omitempty"` + StopOnFailure *bool `json:"stopOnFailure,omitempty"` +} + +// ResourceSelect names the objects to read. Fields are combined with AND. +type ResourceSelect struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Name string `json:"name,omitempty"` + Namespace string `json:"namespace,omitempty"` + Labels map[string]string `json:"labels,omitempty"` +} + +// ConfigError is one preflight failure with a YAML field path. +type ConfigError struct { + File string + Step int + Assertion int + Path string + Err error +} + +// Error reports the scenario file, step, assertion, field path, and cause. +func (e *ConfigError) Error() string { + var b strings.Builder + if e.File != "" { + b.WriteString(e.File) + b.WriteString(": ") + } + if e.Step >= 0 { + fmt.Fprintf(&b, "steps[%d]", e.Step) + if e.Assertion >= 0 { + fmt.Fprintf(&b, " assert[%d]", e.Assertion) + } + if e.Path != "" { + b.WriteString(".") + b.WriteString(e.Path) + } + b.WriteString(": ") + } else if e.Path != "" { + b.WriteString(e.Path) + b.WriteString(": ") + } + if e.Err != nil { + b.WriteString(e.Err.Error()) + } + return b.String() +} + +// Unwrap returns the underlying error. +func (e *ConfigError) Unwrap() error { return e.Err } + +// RunParallel reports whether this fixture may run beside other fixtures. +// A missing value means yes. +func (f E2EFixture) RunParallel() bool { + return f.Parallel == nil || *f.Parallel +} + +// StopOnFailureEnabled reports the step failure policy. Nil means true. +func (s Step) StopOnFailureEnabled() bool { + return s.StopOnFailure == nil || *s.StopOnFailure +} + +// StopOnFailureEnabled reports the assertion failure policy. Nil means false. +func (a ResourceAssertion) StopOnFailureEnabled() bool { + return a.StopOnFailure != nil && *a.StopOnFailure +} + +// OpName returns the operation name, or "" if none or more than one is set. +func (s Step) OpName() string { + n := 0 + name := "" + if s.Deploy != nil { + n++ + name = "deploy" + } + if s.Plan != nil { + n++ + name = "plan" + } + if s.Run != nil { + n++ + name = "run" + } + if s.Logs != nil { + n++ + name = "logs" + } + if s.Delete != nil { + n++ + name = "delete" + } + if len(s.Assert) > 0 { + n++ + name = "assert" + } + if n != 1 { + return "" + } + return name +} + +// Output returns the operation's output expectation, or nil. +func (s Step) Output() *OutputExpectation { + switch { + case s.Deploy != nil: + return s.Deploy.Output + case s.Plan != nil: + return s.Plan.Output + case s.Run != nil: + return s.Run.Output + case s.Delete != nil: + return s.Delete.Output + default: + return nil + } +} + +// LoadE2EFixture reads path, checks the schema, then decodes it. +// A deploy.spec path that does not name a file is rejected. +func LoadE2EFixture(path, scenarioDir string) (E2EFixture, error) { + raw, err := os.ReadFile(path) // #nosec G304 -- scenario e2e.yaml under test + if err != nil { + return E2EFixture{}, fmt.Errorf("read e2e.yaml: %w", err) + } + jsonBytes, err := yaml.YAMLToJSON(raw) + if err != nil { + return E2EFixture{}, &ConfigError{File: path, Step: -1, Assertion: -1, Path: "", Err: fmt.Errorf("parse e2e.yaml: %w", err)} + } + var doc any + if err = json.Unmarshal(jsonBytes, &doc); err != nil { + return E2EFixture{}, &ConfigError{File: path, Step: -1, Assertion: -1, Err: fmt.Errorf("parse e2e.yaml: %w", err)} + } + if err = ValidateE2EDocument(doc); err != nil { + return E2EFixture{}, &ConfigError{File: path, Step: -1, Assertion: -1, Err: err} + } + var fx E2EFixture + if err = k8sjson.Unmarshal(jsonBytes, &fx); err != nil { + return E2EFixture{}, &ConfigError{File: path, Step: -1, Assertion: -1, Err: fmt.Errorf("decode e2e.yaml: %w", err)} + } + if err = validateDeploySpecs(scenarioDir, fx); err != nil { + return E2EFixture{}, &ConfigError{File: path, Step: -1, Assertion: -1, Err: err} + } + return fx, nil +} + +func validateDeploySpecs(scenarioDir string, fx E2EFixture) error { + for i, step := range fx.Steps { + if step.Deploy == nil || step.Deploy.Spec == "" { + continue + } + specPath := filepath.Join(scenarioDir, step.Deploy.Spec) + info, err := os.Stat(specPath) + if err != nil { + return fmt.Errorf("steps[%d].deploy.spec %q: %w", i, step.Deploy.Spec, err) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("steps[%d].deploy.spec %q is not a file", i, step.Deploy.Spec) + } + } + return nil +} diff --git a/internal/testing/e2e/namespace.go b/internal/testing/e2e/namespace.go new file mode 100644 index 0000000..79cee34 --- /dev/null +++ b/internal/testing/e2e/namespace.go @@ -0,0 +1,150 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "errors" + "fmt" + "time" + + "k8s.io/apimachinery/pkg/types" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // KindContext is the kube context of the suite's Kind cluster. + KindContext = "kind-deployah" + + // RunAnnotation stores this execution's id on the namespace. + RunAnnotation = "deployah.dev/e2e-run" + + // NamespaceReconcileAttempts is how many Gets follow an ambiguous create. + NamespaceReconcileAttempts = 4 + + // NamespaceReconcileInterval is the pause between those Gets. + NamespaceReconcileInterval = 500 * time.Millisecond +) + +// ErrNotFound means a Get found no object. +var ErrNotFound = errors.New("not found") + +// ErrNotOwner means the live object is not this execution's claim. +var ErrNotOwner = errors.New("namespace is not owned by this run") + +// ErrOwnershipUncertain means ownership was not proved, so nothing is deleted. +var ErrOwnershipUncertain = errors.New("namespace ownership uncertain") + +// ErrNamespaceGone means a previously claimed namespace is already absent. +var ErrNamespaceGone = errors.New("namespace already deleted") + +// CreateClass is the caller's classification of a namespace Create error. +type CreateClass int + +const ( + // CreateSucceeded means Create returned nil. + CreateSucceeded CreateClass = iota + // CreateAlreadyExists means the API reported the name is taken. + CreateAlreadyExists + // CreateAmbiguous means the call failed and the object might still exist. + CreateAmbiguous + // CreateRejected means the API refused the object and did not store it. + CreateRejected +) + +// NamespaceView is the ownership data from one Get or Create response. +type NamespaceView struct { + Name string + UID string + Run string +} + +// Claim is one execution's proof that it created a namespace. +type Claim struct { + Name string + RunID string + UID string +} + +// CreateDecision is the result of comparing a Create outcome with one Get. +type CreateDecision struct { + Owned bool + Claim Claim + Retry bool + Err error +} + +// ResolveNamespaceCreate decides whether this execution owns the namespace. +func ResolveNamespaceCreate(runID, name string, class CreateClass, got *NamespaceView, getErr error) CreateDecision { + // Retry means Get again. This function never deletes. + switch class { + case CreateRejected: + return CreateDecision{Err: fmt.Errorf("namespace %s: create rejected", name)} + case CreateSucceeded, CreateAlreadyExists, CreateAmbiguous: + if claim, ok := ownedView(runID, name, got); ok && getErr == nil { + return CreateDecision{Owned: true, Claim: claim} + } + if class != CreateSucceeded && getErr == nil && got != nil && !sameRun(runID, name, got) { + return CreateDecision{Err: fmt.Errorf("namespace %s: %w", name, ErrNotOwner)} + } + return CreateDecision{Retry: true} + default: + return CreateDecision{Err: fmt.Errorf("namespace %s: unknown create result", name)} + } +} + +// FinishUncertain reports that reconciliation ended without proof of ownership. +func FinishUncertain(name string, createErr, getErr error) error { + return fmt.Errorf("namespace %s: ownership uncertain, not deleting: %w", + name, errors.Join(ErrOwnershipUncertain, createErr, getErr)) +} + +// ConfirmNamespaceOwner checks that got is still the claimed namespace. +func ConfirmNamespaceOwner(claim Claim, got *NamespaceView, getErr error) error { + if claim.Name == "" || claim.RunID == "" || claim.UID == "" { + return fmt.Errorf("namespace %s: %w", claim.Name, ErrOwnershipUncertain) + } + if errors.Is(getErr, ErrNotFound) { + return ErrNamespaceGone + } + if getErr != nil { + return fmt.Errorf("namespace %s: %w: %w", claim.Name, ErrOwnershipUncertain, getErr) + } + if got == nil || got.Name != claim.Name || got.Run != claim.RunID || got.UID != claim.UID { + return fmt.Errorf("namespace %s: %w", claim.Name, ErrNotOwner) + } + return nil +} + +// UIDPrecondition returns the UID a delete must send. +// A replaced object with a different UID is left alone. +func UIDPrecondition(uid string) (metav1.Preconditions, error) { + if uid == "" { + return metav1.Preconditions{}, errors.New("uid precondition requires a recorded uid") + } + parsed := types.UID(uid) + return metav1.Preconditions{UID: &parsed}, nil +} + +func ownedView(runID, name string, got *NamespaceView) (Claim, bool) { + if !sameRun(runID, name, got) || got.UID == "" { + return Claim{}, false + } + return Claim{Name: name, RunID: runID, UID: got.UID}, true +} + +func sameRun(runID, name string, got *NamespaceView) bool { + return got != nil && got.Name == name && got.Run == runID && runID != "" +} diff --git a/internal/testing/e2e/output.go b/internal/testing/e2e/output.go new file mode 100644 index 0000000..957ce46 --- /dev/null +++ b/internal/testing/e2e/output.go @@ -0,0 +1,76 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "errors" + "fmt" + "regexp" + "strings" +) + +const outputExcerptLimit = 2000 + +// CheckOutput reports whether stdout, stderr, and exitCode satisfy exp. +func CheckOutput(stdout, stderr string, exitCode int, exp *PreparedOutput) error { + // A nil exp means exit code 0. Extra text is fine. Every pattern must match. There is no retry. + want := 0 + if exp != nil { + want = exp.ExitCode + } + var problems []string + if exitCode != want { + problems = append(problems, fmt.Sprintf("exit %d, want %d", exitCode, want)) + } + if exp != nil { + problems = append(problems, missingPatterns("stdout", exp.Stdout, exp.StdoutPat, stdout)...) + problems = append(problems, missingPatterns("stderr", exp.Stderr, exp.StderrPat, stderr)...) + } + if len(problems) == 0 { + return nil + } + var b strings.Builder + b.WriteString(strings.Join(problems, "; ")) + if stdout != "" { + fmt.Fprintf(&b, "\nstdout:\n%s", excerpt(stdout)) + } + if stderr != "" { + fmt.Fprintf(&b, "\nstderr:\n%s", excerpt(stderr)) + } + return errors.New(b.String()) +} + +func missingPatterns(stream string, compiled []*regexp.Regexp, patterns []string, text string) []string { + var missing []string + for i, re := range compiled { + if re.MatchString(text) { + continue + } + pattern := re.String() + if i < len(patterns) { + pattern = patterns[i] + } + missing = append(missing, fmt.Sprintf("%s missing pattern %q", stream, pattern)) + } + return missing +} + +func excerpt(s string) string { + s = strings.TrimSpace(s) + if len(s) <= outputExcerptLimit { + return s + } + return s[:outputExcerptLimit] + "\n..." +} diff --git a/internal/testing/e2e/prepare.go b/internal/testing/e2e/prepare.go new file mode 100644 index 0000000..11763e7 --- /dev/null +++ b/internal/testing/e2e/prepare.go @@ -0,0 +1,292 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "errors" + "fmt" + "regexp" + "time" + + "deployah.dev/deployah/internal/testing/celassert" +) + +// PreparedFixture is an [E2EFixture] with durations, CEL programs, and +// regular expressions compiled once. +type PreparedFixture struct { + Fixture E2EFixture + Steps []PreparedStep +} + +// PreparedStep is one compiled scenario step. +type PreparedStep struct { + Op string + Timeout time.Duration + StopOnFailure bool + Output *PreparedOutput + Deploy *DeployOp + Plan *PreparedPlan + Run *RunOp + Logs *PreparedLogs + Delete *DeleteOp + Assert []PreparedAssert +} + +// PreparedOutput is a compiled [OutputExpectation]. A missing exit code is 0. +type PreparedOutput struct { + ExitCode int + Stdout []*regexp.Regexp + Stderr []*regexp.Regexp + StdoutPat []string + StderrPat []string +} + +// PreparedPlan is a compiled [PlanOp]. +type PreparedPlan struct { + DetailedExitCode bool + Programs []*celassert.Program + Expr []string +} + +// PreparedLogs is a compiled [LogsOp]. +type PreparedLogs struct { + Component string + Patterns []*regexp.Regexp + Raw []string +} + +// PreparedAssert is a compiled [ResourceAssertion]. +type PreparedAssert struct { + Select ResourceSelect + HasExpect bool + Expect map[string]any + Programs []*celassert.Program + Expr []string + StopOnFailure bool +} + +// PrepareE2EFixture compiles durations, CEL, and regular expressions. +// It does not touch Kubernetes. +func PrepareE2EFixture(file string, fx E2EFixture) (PreparedFixture, error) { + var errs []error + if _, err := parseTimeout(fx.Timeout, ""); err != nil { + errs = append(errs, &ConfigError{File: file, Step: -1, Assertion: -1, Path: "timeout", Err: err}) + } + steps := make([]PreparedStep, 0, len(fx.Steps)) + for i, step := range fx.Steps { + prepared, err := prepareStep(file, fx.Timeout, i, step) + if err != nil { + errs = append(errs, err) + continue + } + steps = append(steps, prepared) + } + if err := errors.Join(errs...); err != nil { + return PreparedFixture{}, err + } + return PreparedFixture{Fixture: fx, Steps: steps}, nil +} + +func prepareStep(file, fixtureTimeout string, index int, step Step) (PreparedStep, error) { + timeout, err := parseTimeout(step.Timeout, fixtureTimeout) + op := step.OpName() + if op == "" { + return PreparedStep{}, &ConfigError{ + File: file, Step: index, Assertion: -1, + Err: errors.New("exactly one operation is required"), + } + } + prepared := PreparedStep{ + Op: op, + Timeout: timeout, + StopOnFailure: step.StopOnFailureEnabled(), + Deploy: step.Deploy, + Run: step.Run, + Delete: step.Delete, + } + var errs []error + if err != nil { + errs = append(errs, &ConfigError{File: file, Step: index, Assertion: -1, Path: "timeout", Err: err}) + prepared.Timeout = DefaultE2EStepTimeout + } + if out := step.Output(); out != nil { + compiled, outErr := compileOutput(file, index, out) + if outErr != nil { + errs = append(errs, outErr) + } else { + prepared.Output = compiled + } + } + switch op { + case "plan": + plan, planErr := compilePlan(file, index, step.Plan) + if planErr != nil { + errs = append(errs, planErr) + } else { + prepared.Plan = plan + } + case "logs": + logs, logsErr := compileLogs(file, index, step.Logs) + if logsErr != nil { + errs = append(errs, logsErr) + } else { + prepared.Logs = logs + } + case "assert": + asserts, assertErrs := compileAsserts(file, index, step.Assert) + errs = append(errs, assertErrs...) + prepared.Assert = asserts + } + if joinErr := errors.Join(errs...); joinErr != nil { + return PreparedStep{}, joinErr + } + return prepared, nil +} + +func parseTimeout(raw, fallback string) (time.Duration, error) { + if raw == "" { + raw = fallback + } + if raw == "" { + return DefaultE2EStepTimeout, nil + } + d, err := time.ParseDuration(raw) + if err != nil { + return 0, fmt.Errorf("timeout %q: %w", raw, err) + } + if d <= 0 { + return 0, fmt.Errorf("timeout %q must be positive", raw) + } + return d, nil +} + +func compileOutput(file string, step int, out *OutputExpectation) (*PreparedOutput, error) { + prepared := &PreparedOutput{ExitCode: 0} + if out.ExitCode != nil { + prepared.ExitCode = *out.ExitCode + } + var errs []error + if out.Stdout != nil { + compiled, raw, err := compilePatterns(file, step, -1, "output.stdout.contains", out.Stdout.Contains) + if err != nil { + errs = append(errs, err) + } + prepared.Stdout = compiled + prepared.StdoutPat = raw + } + if out.Stderr != nil { + compiled, raw, err := compilePatterns(file, step, -1, "output.stderr.contains", out.Stderr.Contains) + if err != nil { + errs = append(errs, err) + } + prepared.Stderr = compiled + prepared.StderrPat = raw + } + if err := errors.Join(errs...); err != nil { + return nil, err + } + return prepared, nil +} + +func compilePlan(file string, step int, op *PlanOp) (*PreparedPlan, error) { + plan := &PreparedPlan{DetailedExitCode: op.DetailedExitCode, Expr: op.Expr} + if len(op.Expr) == 0 { + return plan, nil + } + programs := make([]*celassert.Program, 0, len(op.Expr)) + var errs []error + for i, expr := range op.Expr { + prog, err := celassert.Compile(expr, celassert.WithVariable("plan")) + if err != nil { + errs = append(errs, &ConfigError{ + File: file, Step: step, Assertion: -1, + Path: fmt.Sprintf("plan.expr[%d]", i), + Err: err, + }) + continue + } + programs = append(programs, prog) + } + if err := errors.Join(errs...); err != nil { + return nil, err + } + plan.Programs = programs + return plan, nil +} + +func compileLogs(file string, step int, op *LogsOp) (*PreparedLogs, error) { + compiled, raw, err := compilePatterns(file, step, -1, "logs.contains", op.Contains) + if err != nil { + return nil, err + } + return &PreparedLogs{Component: op.Component, Patterns: compiled, Raw: raw}, nil +} + +func compileAsserts(file string, step int, assertions []ResourceAssertion) ([]PreparedAssert, []error) { + out := make([]PreparedAssert, 0, len(assertions)) + var errs []error + for j, assertion := range assertions { + prepared := PreparedAssert{ + Select: assertion.Select, + HasExpect: assertion.Expect != nil, + Expect: assertion.Expect, + Expr: assertion.Expr, + StopOnFailure: assertion.StopOnFailureEnabled(), + } + programs := make([]*celassert.Program, 0, len(assertion.Expr)) + ok := true + for i, expr := range assertion.Expr { + prog, err := celassert.Compile(expr, celassert.WithVariable("objects")) + if err != nil { + ok = false + errs = append(errs, &ConfigError{ + File: file, Step: step, Assertion: j, + Path: fmt.Sprintf("expr[%d]", i), + Err: err, + }) + continue + } + programs = append(programs, prog) + } + if ok { + prepared.Programs = programs + out = append(out, prepared) + } + } + return out, errs +} + +func compilePatterns(file string, step, assertion int, path string, patterns []string) ([]*regexp.Regexp, []string, error) { + compiled := make([]*regexp.Regexp, 0, len(patterns)) + raw := make([]string, 0, len(patterns)) + var errs []error + for i, pattern := range patterns { + re, err := regexp.Compile(pattern) + if err != nil { + errs = append(errs, &ConfigError{ + File: file, Step: step, Assertion: assertion, + Path: fmt.Sprintf("%s[%d]", path, i), + Err: err, + }) + continue + } + compiled = append(compiled, re) + raw = append(raw, pattern) + } + if err := errors.Join(errs...); err != nil { + return nil, nil, err + } + return compiled, raw, nil +} diff --git a/internal/testing/e2e/prepare_test.go b/internal/testing/e2e/prepare_test.go new file mode 100644 index 0000000..32f1ee5 --- /dev/null +++ b/internal/testing/e2e/prepare_test.go @@ -0,0 +1,207 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPrepareE2EFixture(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "e2e.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +env: dev +timeout: 1m +steps: + - deploy: + output: + stdout: + contains: ['ok'] + stderr: + contains: ['warn'] + - plan: + expr: + - 'plan.changes.size() == 0' + - assert: + - select: + apiVersion: v1 + kind: ConfigMap + expr: + - 'objects.size() == 0' + stopOnFailure: true + - logs: + component: web + contains: ['ready'] +`), 0o600)) + fx, err := LoadE2EFixture(path, dir) + require.NoError(t, err) + prepared, err := PrepareE2EFixture("e2e.yaml", fx) + require.NoError(t, err) + require.Len(t, prepared.Steps, 4) + assert.Equal(t, "1m0s", prepared.Steps[0].Timeout.String()) + require.NotNil(t, prepared.Steps[0].Output) + assert.Len(t, prepared.Steps[0].Output.Stdout, 1) + require.NotNil(t, prepared.Steps[1].Plan) + assert.Len(t, prepared.Steps[1].Plan.Programs, 1) + assert.True(t, prepared.Steps[2].Assert[0].StopOnFailure) + assert.Len(t, prepared.Steps[3].Logs.Patterns, 1) +} + +func TestPrepareE2EFixture_errors(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "e2e.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +env: dev +timeout: nope +steps: + - plan: + expr: + - '1' + - 'missing.field' + timeout: -1s + - logs: + component: web + contains: ['['] + - deploy: + output: + stdout: + contains: ['('] +`), 0o600)) + fx, err := LoadE2EFixture(path, dir) + require.NoError(t, err) + before := fx + _, err = PrepareE2EFixture("scenario/e2e.yaml", fx) + require.Error(t, err) + assert.Equal(t, before, fx) + for _, want := range []string{ + "timeout", + "not bool", + "undeclared reference", + "logs.contains", + "output.stdout.contains", + } { + assert.ErrorContains(t, err, want) + } + var cfg *ConfigError + require.ErrorAs(t, err, &cfg) + assert.Equal(t, "scenario/e2e.yaml", cfg.File) +} + +func TestCheckOutput(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "e2e.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +env: dev +steps: + - run: + task: migrate + output: + exitCode: 1 + stdout: + contains: ['done', 'ok'] + stderr: + contains: ['task.*not found'] +`), 0o600)) + fx, err := LoadE2EFixture(path, dir) + require.NoError(t, err) + prepared, err := PrepareE2EFixture("e2e.yaml", fx) + require.NoError(t, err) + require.NoError(t, CheckOutput("done and ok extra", "task foo not found", 1, prepared.Steps[0].Output)) +} + +func TestCheckOutput_excerpt(t *testing.T) { + t.Parallel() + err := CheckOutput(strings.Repeat("a", outputExcerptLimit+10), "", 1, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "...") +} + +func TestCheckOutput_nilExitZero(t *testing.T) { + t.Parallel() + require.NoError(t, CheckOutput("", "", 0, nil)) +} + +func TestCheckOutput_errors(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "e2e.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +env: dev +steps: + - run: + task: migrate + output: + exitCode: 1 + stdout: + contains: ['done', 'ok'] + stderr: + contains: ['task.*not found'] +`), 0o600)) + fx, err := LoadE2EFixture(path, dir) + require.NoError(t, err) + prepared, err := PrepareE2EFixture("e2e.yaml", fx) + require.NoError(t, err) + exp := prepared.Steps[0].Output + + tests := []struct { + name string + stdout string + stderr string + code int + exp *PreparedOutput + wantErr string + }{ + {name: "missing stdout pattern", stdout: "done", stderr: "task foo not found", code: 1, exp: exp, wantErr: "stdout missing pattern"}, + {name: "missing stderr pattern", stdout: "done and ok", stderr: "nope", code: 1, exp: exp, wantErr: "stderr missing pattern"}, + {name: "unexpected exit", stdout: "done and ok", stderr: "task foo not found", code: 0, exp: exp, wantErr: "exit 0, want 1"}, + {name: "nil expectation rejects nonzero", code: 2, wantErr: "exit 2, want 0"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + gotErr := CheckOutput(tt.stdout, tt.stderr, tt.code, tt.exp) + require.ErrorContains(t, gotErr, tt.wantErr) + }) + } +} + +func TestPrepareDoesNotWriteSource(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "e2e.yaml") + body := []byte("env: dev\nsteps:\n - deploy: {}\n") + require.NoError(t, os.WriteFile(path, body, 0o600)) + fx, err := LoadE2EFixture(path, dir) + require.NoError(t, err) + _, err = PrepareE2EFixture(path, fx) + require.NoError(t, err) + got, err := os.ReadFile(path) // #nosec G304 -- path is the temp file this test wrote + require.NoError(t, err) + assert.Equal(t, string(body), string(got)) + entries, err := os.ReadDir(dir) + require.NoError(t, err) + assert.Len(t, entries, 1) + require.False(t, errors.Is(err, os.ErrNotExist)) +} diff --git a/internal/testing/e2e/report.go b/internal/testing/e2e/report.go new file mode 100644 index 0000000..50bbeea --- /dev/null +++ b/internal/testing/e2e/report.go @@ -0,0 +1,98 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "fmt" + "strings" + "time" +) + +// StepReports splits steps into log lines and failures. +func StepReports(verbose bool, scenario string, steps []StepResult) (info, failures []string) { + // Pass and skip are logged only when verbose is true. + // Fail and error always fail the test. Skip does not. + for _, step := range steps { + switch step.Status { + case StatusFail, StatusError: + failures = append(failures, FormatFailure(scenario, step)) + case StatusPass, StatusSkip: + if verbose { + info = append(info, FormatStep(step)) + } + } + } + return info, failures +} + +// FormatFailure is the multi-line diagnostic for one failed step. +func FormatFailure(scenario string, step StepResult) string { + var b strings.Builder + fmt.Fprintf(&b, "%s %s\n", step.Status, scenario) + fmt.Fprintf(&b, " Step: [%d] %s\n", step.Index, step.Op) + writeTiming(&b, step) + if step.Detail != "" { + b.WriteString(indent(step.Detail)) + } + for _, assertion := range step.Assertions { + if assertion.Status == StatusPass || assertion.Status == "" { + continue + } + fmt.Fprintf(&b, " Assertion: [%d] %s\n", assertion.Index, assertion.Status) + if assertion.Detail != "" { + b.WriteString(indent(assertion.Detail)) + } + } + return strings.TrimRight(b.String(), "\n") +} + +// FormatStep is one verbose progress line, including skipped steps. +func FormatStep(step StepResult) string { + var b strings.Builder + fmt.Fprintf(&b, "%s [%d] %s", step.Status, step.Index, step.Op) + if step.Elapsed > 0 { + fmt.Fprintf(&b, " (%s)", step.Elapsed.Round(time.Millisecond)) + } + if step.Status == StatusPass && len(step.Assertions) > 0 { + passed := 0 + for _, assertion := range step.Assertions { + if assertion.Status == StatusPass { + passed++ + } + } + fmt.Fprintf(&b, "\n %d/%d assertions passed", passed, len(step.Assertions)) + } + if step.Detail != "" && step.Status != StatusPass { + fmt.Fprintf(&b, "\n %s", step.Detail) + } + return b.String() +} + +func writeTiming(b *strings.Builder, step StepResult) { + if step.Elapsed > 0 { + fmt.Fprintf(b, " Elapsed: %s\n", step.Elapsed.Round(time.Millisecond)) + } + if step.Timeout > 0 { + fmt.Fprintf(b, " Timeout: %s\n", step.Timeout) + } +} + +func indent(text string) string { + var b strings.Builder + for line := range strings.SplitSeq(strings.TrimRight(text, "\n"), "\n") { + fmt.Fprintf(&b, " %s\n", line) + } + return b.String() +} diff --git a/internal/testing/e2e/report_test.go b/internal/testing/e2e/report_test.go new file mode 100644 index 0000000..51149ac --- /dev/null +++ b/internal/testing/e2e/report_test.go @@ -0,0 +1,80 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestStepReports(t *testing.T) { + t.Parallel() + steps := []StepResult{ + {Index: 0, Op: "plan", Status: StatusPass, Elapsed: 1200 * time.Millisecond}, + { + Index: 1, Op: "assert", Status: StatusPass, Elapsed: 2100 * time.Millisecond, + Assertions: []AssertResult{{Status: StatusPass}, {Status: StatusPass}, {Status: StatusPass}}, + }, + {Index: 2, Op: "deploy", Status: StatusPass, Elapsed: 800 * time.Millisecond}, + { + Index: 3, Op: "assert", Status: StatusFail, Elapsed: time.Second, Timeout: 3 * time.Minute, + Detail: "expr[0] objects.size() == 1: false\nexpected: objects.size() = 1\nactual: objects.size() = 0", + Assertions: []AssertResult{ + {Index: 0, Status: StatusFail, Detail: "no resources selected: apps/v1 Deployment/web observed 0"}, + {Index: 1, Status: StatusPass}, + {Index: 2, Status: StatusSkip, Detail: "skipped after terminal assertion failure"}, + }, + }, + {Index: 4, Op: "delete", Status: StatusSkip, Detail: "skipped after steps[3] assert"}, + {Index: 5, Op: "logs", Status: StatusError, Detail: "logs exit 1\nstderr:\ncomponent not found", Timeout: time.Minute}, + } + tests := []struct { + name string + verbose bool + wantInfo []string + }{ + {name: "default hides success and skip"}, + { + name: "verbose shows success and skip", + verbose: true, + wantInfo: []string{ + "PASS [0] plan (1.2s)", + "PASS [1] assert (2.1s)\n 3/3 assertions passed", + "PASS [2] deploy (800ms)", + "SKIP [4] delete\n skipped after steps[3] assert", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + info, fails := StepReports(tt.verbose, "basic-web-service", steps) + assert.Equal(t, tt.wantInfo, info) + require.Len(t, fails, 2) + assert.Contains(t, fails[0], "FAIL basic-web-service") + assert.Contains(t, fails[0], "Step: [3] assert") + assert.Contains(t, fails[0], "expected: objects.size() = 1") + assert.Contains(t, fails[0], "actual: objects.size() = 0") + assert.Contains(t, fails[0], "no resources selected") + assert.Contains(t, fails[0], "Timeout: 3m0s") + assert.NotContains(t, fails[0], "Assertion: [1]") + assert.Contains(t, fails[1], "ERROR basic-web-service") + assert.Contains(t, fails[1], "component not found") + }) + } +} diff --git a/internal/testing/e2e/result.go b/internal/testing/e2e/result.go new file mode 100644 index 0000000..f5cc9bf --- /dev/null +++ b/internal/testing/e2e/result.go @@ -0,0 +1,93 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "fmt" + "strings" + "time" +) + +// Status is the outcome of one step or assertion. +type Status string + +const ( + // StatusPass means the operation and its expectations succeeded. + StatusPass Status = "PASS" + // StatusFail means an expectation was not met. + StatusFail Status = "FAIL" + // StatusError means execution or infrastructure failed. + StatusError Status = "ERROR" + // StatusSkip means the step did not run because an earlier step stopped + // the scenario. + StatusSkip Status = "SKIP" +) + +// AssertResult is one resource assertion inside an assert step. +type AssertResult struct { + Index int + Status Status + Detail string + Unsafe bool +} + +// StepResult is one scenario step. +type StepResult struct { + Index int + Op string + Status Status + Detail string + Unsafe bool + Assertions []AssertResult + StoppedHere bool + Elapsed time.Duration + Timeout time.Duration +} + +// String is the go test line for a step that did not pass. +func (s StepResult) String() string { + var b strings.Builder + fmt.Fprintf(&b, "%s steps[%d] %s", s.Status, s.Index, s.Op) + if s.Detail != "" { + b.WriteString(": ") + b.WriteString(s.Detail) + } + for _, assertion := range s.Assertions { + if assertion.Status == StatusPass || assertion.Status == "" { + continue + } + fmt.Fprintf(&b, "\n%s steps[%d] assert[%d]", assertion.Status, s.Index, assertion.Index) + if assertion.Detail != "" { + b.WriteString(": ") + b.WriteString(assertion.Detail) + } + } + return b.String() +} + +// RunResult is every step of one scenario, in order. +type RunResult struct { + Steps []StepResult +} + +// Failed reports whether any step is FAIL or ERROR. +func (r RunResult) Failed() bool { + for _, step := range r.Steps { + if step.Status == StatusFail || step.Status == StatusError { + return true + } + } + return false +} diff --git a/internal/testing/e2e/result_test.go b/internal/testing/e2e/result_test.go new file mode 100644 index 0000000..be21c1b --- /dev/null +++ b/internal/testing/e2e/result_test.go @@ -0,0 +1,92 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStepResultString(t *testing.T) { + t.Parallel() + tests := []struct { + name string + step StepResult + want string + }{ + { + name: "pass is one line", + step: StepResult{Index: 0, Op: "deploy", Status: StatusPass}, + want: "PASS steps[0] deploy", + }, + { + name: "fail with detail", + step: StepResult{Index: 1, Op: "plan", Status: StatusFail, Detail: "exit 1"}, + want: "FAIL steps[1] plan: exit 1", + }, + { + name: "error keeps assertion lines", + step: StepResult{ + Index: 2, Op: "assert", Status: StatusError, Detail: "forbidden", + Assertions: []AssertResult{ + {Index: 0, Status: StatusPass, Detail: "hidden"}, + {Index: 1, Status: StatusFail, Detail: "replicas"}, + {Index: 2, Status: StatusError, Detail: "timeout"}, + {Index: 3, Status: StatusSkip}, + {Index: 4}, + }, + }, + want: "ERROR steps[2] assert: forbidden\nFAIL steps[2] assert[1]: replicas\nERROR steps[2] assert[2]: timeout\nSKIP steps[2] assert[3]", + }, + { + name: "empty detail", + step: StepResult{Index: 0, Op: "delete", Status: StatusSkip}, + want: "SKIP steps[0] delete", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, tt.step.String()) + }) + } +} + +func TestRunResultFailed(t *testing.T) { + t.Parallel() + tests := []struct { + name string + run RunResult + want bool + }{ + {name: "empty", run: RunResult{}, want: false}, + {name: "pass and skip", run: RunResult{Steps: []StepResult{ + {Status: StatusPass}, {Status: StatusSkip}, + }}, want: false}, + {name: "fail", run: RunResult{Steps: []StepResult{ + {Status: StatusPass}, {Status: StatusFail}, + }}, want: true}, + {name: "error", run: RunResult{Steps: []StepResult{ + {Status: StatusError}, + }}, want: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, tt.run.Failed()) + }) + } +} diff --git a/internal/testing/e2e/run.go b/internal/testing/e2e/run.go new file mode 100644 index 0000000..5d340a0 --- /dev/null +++ b/internal/testing/e2e/run.go @@ -0,0 +1,593 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "regexp" + "slices" + "strings" + "time" + + "deployah.dev/deployah/internal/testing/celassert" + + inttest "deployah.dev/deployah/internal/testing" +) + +const ( + // ResourcePollInterval is the pause between Kubernetes assertion reads. + ResourcePollInterval = 2 * time.Second + // LogsPollInterval is the pause between log reads. + LogsPollInterval = 5 * time.Second +) + +// CommandResult is one finished CLI invocation. +type CommandResult struct { + Stdout string + Stderr string + ExitCode int + NotExecuted bool + Interrupted bool + // Retryable marks a nonzero exit that may succeed on a later poll. + // The caller sets it. A nonzero exit alone is terminal. + Retryable bool +} + +// ExecFunc runs one deployah invocation. dir may be empty. +type ExecFunc func(ctx context.Context, dir string, args []string) CommandResult + +// ObserveFunc reads the objects for sel. An empty slice means nothing matched. +type ObserveFunc func(ctx context.Context, sel ResourceSelect, scenarioNS string) ([]map[string]any, error) + +// ErrObserveRetry means the resource read should be tried again. +var ErrObserveRetry = errors.New("resource observation should retry") + +// ErrObserveTerminal means the resource read will not succeed later. +var ErrObserveTerminal = errors.New("resource observation failed") + +// RunInput is one prepared scenario. +// Exec and Observe talk to the CLI and the cluster. +type RunInput struct { + Dir string + Namespace string + Project string + Env string + Fixture PreparedFixture + Exec ExecFunc + Observe ObserveFunc + ResourcePoll time.Duration + LogsPoll time.Duration +} + +// Run executes every step on that step's deadline. +// It does not create or delete namespaces. +func Run(ctx context.Context, in RunInput) RunResult { + if ctx == nil { + return RunResult{Steps: []StepResult{{ + Op: "scenario", Status: StatusError, Detail: "context is nil", Unsafe: true, + }}} + } + if in.ResourcePoll <= 0 { + in.ResourcePoll = ResourcePollInterval + } + if in.LogsPoll <= 0 { + in.LogsPoll = LogsPollInterval + } + steps := make([]StepResult, 0, len(in.Fixture.Steps)) + stop := false + for i, step := range in.Fixture.Steps { + if stop { + steps = append(steps, StepResult{ + Index: i, Op: step.Op, Status: StatusSkip, Detail: skipReason(steps), + }) + continue + } + result := runOne(ctx, in, step, i) + steps = append(steps, result) + if result.Unsafe || result.StoppedHere { + stop = true + continue + } + if (result.Status == StatusFail || result.Status == StatusError) && step.StopOnFailure { + stop = true + } + } + return RunResult{Steps: steps} +} + +func runOne(parent context.Context, in RunInput, step PreparedStep, index int) StepResult { + start := time.Now() + ctx, cancel := context.WithTimeout(parent, step.Timeout) + defer cancel() + var result StepResult + switch step.Op { + case "assert": + result = runAssert(ctx, in, step, index) + case "logs": + result = runLogs(ctx, in, step, index) + default: + result = runCommand(ctx, in, step, index) + } + result.Elapsed = time.Since(start) + result.Timeout = step.Timeout + return result +} + +func selectLabel(sel ResourceSelect) string { + label := sel.APIVersion + " " + sel.Kind + if sel.Name != "" { + label += "/" + sel.Name + } + return label +} + +func celFalseDetail(index int, expr, diagnosis string) string { + detail := fmt.Sprintf("expr[%d] %s: false", index, expr) + if diagnosis == "" { + return detail + } + return detail + "\n" + diagnosis +} + +func runCommand(ctx context.Context, in RunInput, step PreparedStep, index int) StepResult { + base := StepResult{Index: index, Op: step.Op} + args, err := stepArgs(in, step) + if err != nil { + base.Status = StatusError + base.Detail = err.Error() + base.Unsafe = true + return base + } + if in.Exec == nil { + base.Status = StatusError + base.Detail = "command executor is nil" + base.Unsafe = true + return base + } + got := in.Exec(ctx, in.Dir, args) + if got.NotExecuted { + base.Status = StatusError + base.Detail = "command was not executed" + base.Unsafe = true + return base + } + if got.Interrupted || ctx.Err() != nil { + base.Status = StatusError + base.Detail = interruptedDetail(ctx, got) + base.Unsafe = parentCanceled(ctx) + return base + } + if err = CheckOutput(got.Stdout, got.Stderr, got.ExitCode, step.Output); err != nil { + base.Status = StatusFail + base.Detail = err.Error() + return base + } + if step.Op == "plan" && step.Plan != nil && len(step.Plan.Programs) > 0 { + return evalPlan(ctx, base, step.Plan, got.Stdout) + } + base.Status = StatusPass + return base +} + +func evalPlan(ctx context.Context, base StepResult, plan *PreparedPlan, stdout string) StepResult { + var doc any + if err := json.Unmarshal([]byte(stdout), &doc); err != nil { + base.Status = StatusError + base.Detail = fmt.Sprintf("plan JSON: %v", err) + return base + } + var problems []string + for i, prog := range plan.Programs { + expr := plan.Expr[i] + err := prog.Eval(ctx, map[string]any{"plan": doc}) + if err == nil { + continue + } + if errors.Is(err, celassert.ErrFalse) { + problems = append(problems, celFalseDetail(i, expr, prog.Diagnose(ctx, map[string]any{"plan": doc}))) + continue + } + base.Status = StatusError + base.Detail = fmt.Sprintf("expr[%d] %s: %v", i, expr, err) + base.Unsafe = parentCanceled(ctx) + return base + } + if len(problems) > 0 { + base.Status = StatusFail + base.Detail = strings.Join(problems, "; ") + return base + } + base.Status = StatusPass + return base +} + +func runLogs(ctx context.Context, in RunInput, step PreparedStep, index int) StepResult { + base := StepResult{Index: index, Op: step.Op} + args, err := stepArgs(in, step) + if err != nil { + base.Status = StatusError + base.Detail = err.Error() + base.Unsafe = true + return base + } + if in.Exec == nil || step.Logs == nil { + base.Status = StatusError + base.Detail = "logs executor is nil" + base.Unsafe = true + return base + } + var last string + sawOutput := false + for { + if ctx.Err() != nil { + return logsTimeout(base, sawOutput, last, ctx) + } + got := in.Exec(ctx, in.Dir, args) + if got.NotExecuted { + base.Status = StatusError + base.Detail = "command was not executed" + base.Unsafe = true + return base + } + if got.Interrupted { + if ctx.Err() != nil { + return logsTimeout(base, sawOutput, last, ctx) + } + base.Status = StatusError + base.Detail = "logs interrupted" + base.Unsafe = true + return base + } + if got.ExitCode != 0 { + last = logsExitDetail(got) + if !got.Retryable { + base.Status = StatusError + base.Detail = last + return base + } + if ctx.Err() != nil { + return logsTimeout(base, false, last, ctx) + } + if !waitPoll(ctx, in.LogsPoll) { + continue + } + continue + } + sawOutput = true + missing := unmatched(step.Logs.Patterns, step.Logs.Raw, got.Stdout) + if len(missing) == 0 { + base.Status = StatusPass + return base + } + last = strings.Join(missing, "; ") + if ctx.Err() != nil { + return logsTimeout(base, true, last, ctx) + } + if !waitPoll(ctx, in.LogsPoll) { + continue + } + } +} + +func logsExitDetail(got CommandResult) string { + detail := fmt.Sprintf("logs exit %d", got.ExitCode) + if stderr := strings.TrimSpace(got.Stderr); stderr != "" { + detail += "\nstderr:\n" + stderr + } + return detail +} + +func logsTimeout(base StepResult, sawOutput bool, last string, ctx context.Context) StepResult { + cause := ctx.Err() + if cause == nil { + cause = context.DeadlineExceeded + } + if last != "" { + base.Detail = last + "; timeout " + cause.Error() + } else { + base.Detail = "timeout " + cause.Error() + } + base.Unsafe = parentCanceled(ctx) + if sawOutput && !base.Unsafe { + base.Status = StatusFail + return base + } + base.Status = StatusError + return base +} + +func runAssert(ctx context.Context, in RunInput, step PreparedStep, index int) StepResult { + base := StepResult{Index: index, Op: step.Op, Assertions: make([]AssertResult, len(step.Assert))} + pending := make([]int, 0, len(step.Assert)) + for i := range step.Assert { + pending = append(pending, i) + } + for { + next, fatal := evalPending(ctx, in, step, base.Assertions, pending) + if fatal { + skipPending(base.Assertions, next) + base.StoppedHere = true + base.Status = aggregateAssert(base.Assertions) + base.Unsafe = assertUnsafe(base.Assertions) + return base + } + pending = next + if len(pending) == 0 { + base.Status = aggregateAssert(base.Assertions) + base.Unsafe = assertUnsafe(base.Assertions) + return base + } + if ctx.Err() != nil { + failPending(base.Assertions, pending, ctx.Err()) + for _, i := range pending { + if step.Assert[i].StopOnFailure { + base.StoppedHere = true + } + } + base.Status = aggregateAssert(base.Assertions) + base.Unsafe = assertUnsafe(base.Assertions) + return base + } + if !waitPoll(ctx, in.ResourcePoll) { + continue + } + } +} + +func evalPending(ctx context.Context, in RunInput, step PreparedStep, results []AssertResult, pending []int) ([]int, bool) { + next := make([]int, 0, len(pending)) + for n, i := range pending { + status, detail, terminal, unsafe := evalAssert(ctx, in, step.Assert[i]) + if !terminal { + results[i] = AssertResult{Index: i, Detail: detail} + next = append(next, i) + continue + } + results[i] = AssertResult{Index: i, Status: status, Detail: detail, Unsafe: unsafe} + if status != StatusPass && (assertionStop(step.Assert[i], status) || unsafe) { + skipPending(results, pending[n+1:]) + return next, true + } + } + return next, false +} + +func assertionStop(assertion PreparedAssert, status Status) bool { + return assertion.StopOnFailure && status != StatusPass +} + +func evalAssert(ctx context.Context, in RunInput, assertion PreparedAssert) (Status, string, bool, bool) { + if in.Observe == nil { + return StatusError, "resource observer is nil", true, true + } + objects, err := in.Observe(ctx, assertion.Select, in.Namespace) + if err != nil { + if errors.Is(err, ErrObserveRetry) && ctx.Err() == nil { + return StatusFail, err.Error(), false, false + } + if ctx.Err() != nil { + return StatusFail, err.Error() + "; timeout " + ctx.Err().Error(), true, parentCanceled(ctx) + } + unsafe := errors.Is(err, ErrObserveTerminal) + return StatusError, err.Error(), true, unsafe + } + if objects == nil { + objects = []map[string]any{} + } + problems := expectProblems(assertion, objects) + exprProblems, evalErr := exprProblems(ctx, assertion, objects) + if evalErr != nil { + return StatusError, evalErr.Error(), true, false + } + problems = append(problems, exprProblems...) + if len(problems) == 0 { + return StatusPass, "", true, false + } + detail := strings.Join(problems, "; ") + if ctx.Err() != nil { + return StatusFail, detail + "; timeout " + ctx.Err().Error(), true, parentCanceled(ctx) + } + return StatusFail, detail, false, false +} + +func expectProblems(assertion PreparedAssert, objects []map[string]any) []string { + if !assertion.HasExpect { + return nil + } + if len(objects) == 0 { + return []string{"no resources selected: " + selectLabel(assertion.Select) + " observed 0"} + } + var problems []string + for i, obj := range objects { + for _, diff := range inttest.DiffSubset("$", assertion.Expect, obj) { + problems = append(problems, fmt.Sprintf("object[%d] %s", i, diff)) + } + } + return problems +} + +func exprProblems(ctx context.Context, assertion PreparedAssert, objects []map[string]any) ([]string, error) { + list := make([]any, 0, len(objects)) + for i := range objects { + list = append(list, objects[i]) + } + var problems []string + for i, prog := range assertion.Programs { + expr := "" + if i < len(assertion.Expr) { + expr = assertion.Expr[i] + } + err := prog.Eval(ctx, map[string]any{"objects": list}) + if err == nil { + continue + } + if errors.Is(err, celassert.ErrFalse) || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + if errors.Is(err, celassert.ErrFalse) { + problems = append(problems, celFalseDetail(i, expr, prog.Diagnose(ctx, map[string]any{"objects": list}))) + continue + } + problems = append(problems, fmt.Sprintf("expr[%d] %s: %v", i, expr, err)) + continue + } + return nil, fmt.Errorf("expr[%d] %s: %w", i, expr, err) + } + return problems, nil +} + +func skipPending(results []AssertResult, pending []int) { + for _, i := range pending { + results[i] = AssertResult{ + Index: i, Status: StatusSkip, Detail: "skipped after terminal assertion failure", + } + } +} + +func failPending(results []AssertResult, pending []int, cause error) { + for _, i := range pending { + detail := results[i].Detail + if detail == "" { + detail = "deadline exceeded" + } + detail += "; timeout " + cause.Error() + results[i] = AssertResult{Index: i, Status: StatusFail, Detail: detail} + } +} + +func aggregateAssert(results []AssertResult) Status { + status := StatusPass + for _, result := range results { + switch result.Status { + case StatusError: + return StatusError + case StatusFail: + status = StatusFail + } + } + return status +} + +func assertUnsafe(results []AssertResult) bool { + for _, result := range results { + if result.Unsafe { + return true + } + } + return false +} + +func stepArgs(in RunInput, step PreparedStep) ([]string, error) { + nsFlag := []string{"--context", KindContext, "--namespace", in.Namespace} + switch step.Op { + case "deploy": + args := []string{"deploy", in.Env} + args = append(args, nsFlag...) + if step.Deploy != nil && step.Deploy.Spec != "" { + args = append(args, "--spec", step.Deploy.Spec) + } + if step.Deploy != nil { + args = append(args, step.Deploy.Args...) + } + return args, nil + case "plan": + args := []string{"plan", in.Env, "-o", "json"} + args = append(args, nsFlag...) + if step.Plan != nil && step.Plan.DetailedExitCode { + args = append(args, "--detailed-exitcode") + } + return args, nil + case "run": + if step.Run == nil || step.Run.Task == "" { + return nil, errors.New("run.task is required") + } + args := []string{"run", step.Run.Task, in.Env, "--yes"} + return append(args, nsFlag...), nil + case "logs": + if step.Logs == nil || step.Logs.Component == "" { + return nil, errors.New("logs.component is required") + } + return []string{ + "logs", in.Project, + "--component=" + step.Logs.Component, + "--environment=" + in.Env, + "--no-follow", + "--context", KindContext, + "--namespace", in.Namespace, + }, nil + case "delete": + args := []string{"delete", in.Project, in.Env, "--yes", "--wait", "--allow-missing-platform"} + return append(args, nsFlag...), nil + case "assert": + return nil, nil + default: + return nil, fmt.Errorf("unknown operation %q", step.Op) + } +} + +func unmatched(patterns []*regexp.Regexp, raw []string, text string) []string { + var missing []string + for i, re := range patterns { + if re.MatchString(text) { + continue + } + pattern := re.String() + if i < len(raw) { + pattern = raw[i] + } + missing = append(missing, fmt.Sprintf("logs missing pattern %q", pattern)) + } + return missing +} + +func waitPoll(ctx context.Context, d time.Duration) bool { + if d <= 0 { + d = time.Millisecond + } + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +func interruptedDetail(ctx context.Context, got CommandResult) string { + detail := "command interrupted" + if ctx.Err() != nil { + detail = "timeout " + ctx.Err().Error() + } + if strings.TrimSpace(got.Stderr) != "" { + detail += "\nstderr:\n" + strings.TrimSpace(got.Stderr) + } + return detail +} + +func parentCanceled(ctx context.Context) bool { + return ctx.Err() != nil && errors.Is(ctx.Err(), context.Canceled) && !errors.Is(ctx.Err(), context.DeadlineExceeded) +} + +func skipReason(prev []StepResult) string { + for _, step := range slices.Backward(prev) { + if step.Status == StatusSkip { + continue + } + return fmt.Sprintf("skipped after steps[%d] %s", step.Index, step.Op) + } + return "skipped" +} diff --git a/internal/testing/e2e/run_test.go b/internal/testing/e2e/run_test.go new file mode 100644 index 0000000..94a2d2f --- /dev/null +++ b/internal/testing/e2e/run_test.go @@ -0,0 +1,1034 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "errors" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + inttest "deployah.dev/deployah/internal/testing" +) + +func TestResolveNamespaceCreate(t *testing.T) { + t.Parallel() + const runID = "run-1" + const name = "e2e-demo" + own := &NamespaceView{Name: name, UID: "uid-1", Run: runID} + + tests := []struct { + name string + class CreateClass + got *NamespaceView + getErr error + wantOwned bool + wantRetry bool + wantUID string + }{ + {name: "create succeeded", class: CreateSucceeded, got: own, wantOwned: true, wantUID: "uid-1"}, + {name: "already exists same run", class: CreateAlreadyExists, got: own, wantOwned: true, wantUID: "uid-1"}, + {name: "ambiguous same run", class: CreateAmbiguous, got: own, wantOwned: true, wantUID: "uid-1"}, + {name: "ambiguous not found", class: CreateAmbiguous, getErr: ErrNotFound, wantRetry: true}, + {name: "ambiguous get failed", class: CreateAmbiguous, getErr: errors.New("timeout"), wantRetry: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := ResolveNamespaceCreate(runID, name, tt.class, tt.got, tt.getErr) + require.NoError(t, got.Err) + assert.Equal(t, tt.wantOwned, got.Owned) + assert.Equal(t, tt.wantRetry, got.Retry) + assert.Equal(t, tt.wantUID, got.Claim.UID) + }) + } +} + +func TestResolveNamespaceCreate_notOwner(t *testing.T) { + t.Parallel() + const runID = "run-1" + const name = "e2e-demo" + other := &NamespaceView{Name: name, UID: "uid-other", Run: "run-2"} + got := ResolveNamespaceCreate(runID, name, CreateAlreadyExists, other, nil) + require.ErrorIs(t, got.Err, ErrNotOwner) + assert.False(t, got.Owned) + assert.False(t, got.Retry) +} + +func TestResolveNamespaceCreate_rejected(t *testing.T) { + t.Parallel() + got := ResolveNamespaceCreate("run-1", "e2e-demo", CreateRejected, nil, nil) + require.ErrorContains(t, got.Err, "create rejected") + assert.False(t, got.Owned) + assert.False(t, got.Retry) +} + +func TestFinishUncertain(t *testing.T) { + t.Parallel() + err := FinishUncertain("e2e-demo", errors.New("create timeout"), ErrNotFound) + require.ErrorIs(t, err, ErrOwnershipUncertain) + assert.ErrorContains(t, err, "not deleting") + assert.ErrorContains(t, err, "e2e-demo") +} + +func TestConfirmNamespaceOwner(t *testing.T) { + t.Parallel() + claim := Claim{Name: "e2e-demo", RunID: "run-1", UID: "uid-1"} + err := ConfirmNamespaceOwner(claim, &NamespaceView{Name: claim.Name, UID: claim.UID, Run: claim.RunID}, nil) + require.NoError(t, err) +} + +func TestConfirmNamespaceOwner_errors(t *testing.T) { + t.Parallel() + claim := Claim{Name: "e2e-demo", RunID: "run-1", UID: "uid-1"} + tests := []struct { + name string + claim Claim + got *NamespaceView + getErr error + want error + }{ + {name: "already gone", claim: claim, getErr: ErrNotFound, want: ErrNamespaceGone}, + {name: "different uid", claim: claim, got: &NamespaceView{Name: claim.Name, UID: "other", Run: claim.RunID}, want: ErrNotOwner}, + {name: "get failed", claim: claim, getErr: errors.New("boom"), want: ErrOwnershipUncertain}, + {name: "incomplete claim", claim: Claim{Name: claim.Name}, want: ErrOwnershipUncertain}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := ConfirmNamespaceOwner(tt.claim, tt.got, tt.getErr) + require.ErrorIs(t, err, tt.want) + }) + } +} + +func TestUIDPrecondition(t *testing.T) { + t.Parallel() + pre, err := UIDPrecondition("uid-1") + require.NoError(t, err) + require.NotNil(t, pre.UID) + assert.Equal(t, "uid-1", string(*pre.UID)) +} + +func TestUIDPrecondition_empty(t *testing.T) { + t.Parallel() + _, err := UIDPrecondition("") + require.Error(t, err) +} + +func TestRunCleanup_phaseIsolationAndBudget(t *testing.T) { + t.Parallel() + var started [2]bool + err := RunCleanup(context.Background(), []CleanupPhase{ + {Name: "first", Run: func(ctx context.Context) error { + started[0] = true + <-ctx.Done() + return ctx.Err() + }}, + {Name: "second", Run: func(ctx context.Context) error { + started[1] = true + require.NoError(t, ctx.Err()) + return errors.New("pvc failed") + }}, + }, WithPhaseTimeout(20*time.Millisecond)) + require.Error(t, err) + assert.True(t, started[0]) + assert.True(t, started[1]) + assert.ErrorContains(t, err, "first") + assert.ErrorContains(t, err, "pvc failed") + + called := false + err = RunCleanup(context.Background(), []CleanupPhase{ + {Name: "slow", Run: func(ctx context.Context) error { + <-ctx.Done() + return ctx.Err() + }}, + {Name: "late", Run: func(context.Context) error { + called = true + return nil + }}, + }, WithPhaseTimeout(time.Second), WithBudget(15*time.Millisecond)) + require.Error(t, err) + assert.False(t, called) + assert.ErrorContains(t, err, "deadline") +} + +func TestRunCleanup_joinsAndStopsOnLostClaim(t *testing.T) { + t.Parallel() + var deleted bool + err := RunCleanup(context.Background(), []CleanupPhase{ + {Name: "release", Run: func(context.Context) error { return errors.New("release failed") }}, + {Name: "workloads", Run: func(context.Context) error { return errors.New("workloads failed") }}, + {Name: "namespace", Run: func(context.Context) error { return errors.New("namespace failed") }}, + }, WithPhaseTimeout(time.Second)) + require.Error(t, err) + assert.ErrorContains(t, err, "release failed") + assert.ErrorContains(t, err, "workloads failed") + assert.ErrorContains(t, err, "namespace failed") + + err = RunCleanup(context.Background(), []CleanupPhase{ + {Name: "confirm", Run: func(context.Context) error { return ErrNotOwner }}, + {Name: "namespace", Run: func(context.Context) error { + deleted = true + return nil + }}, + }, WithPhaseTimeout(time.Second)) + require.ErrorIs(t, err, ErrNotOwner) + assert.False(t, deleted) +} + +func TestRun_policies(t *testing.T) { + t.Parallel() + t.Run("default exit and extra output", func(t *testing.T) { + t.Parallel() + prepared := prepareRun(t, "env: dev\nsteps:\n - deploy:\n output:\n stdout:\n contains: ['Deployed']\n") + var calls int + result := Run(context.Background(), RunInput{ + Env: "dev", Namespace: "ns", Project: "p", Fixture: prepared, + Exec: func(context.Context, string, []string) CommandResult { + calls++ + return CommandResult{Stdout: "Deployed extra", ExitCode: 0} + }, + }) + require.Len(t, result.Steps, 1) + assert.Equal(t, StatusPass, result.Steps[0].Status) + assert.Equal(t, 1, calls) + }) + + t.Run("expected and unexpected exit", func(t *testing.T) { + t.Parallel() + prepared := prepareRun(t, ` +env: dev +steps: + - run: + task: missing + output: + exitCode: 1 + stderr: + contains: ['not found'] +`) + ok := Run(context.Background(), RunInput{ + Env: "dev", Fixture: prepared, + Exec: func(context.Context, string, []string) CommandResult { + return CommandResult{Stderr: "task not found", ExitCode: 1} + }, + }) + assert.Equal(t, StatusPass, ok.Steps[0].Status) + bad := Run(context.Background(), RunInput{ + Env: "dev", Fixture: prepared, + Exec: func(context.Context, string, []string) CommandResult { + return CommandResult{ExitCode: 2} + }, + }) + assert.Equal(t, StatusFail, bad.Steps[0].Status) + }) + + t.Run("not executed", func(t *testing.T) { + t.Parallel() + prepared := prepareRun(t, "env: dev\nsteps:\n - deploy: {}\n - delete: {}\n") + result := Run(context.Background(), RunInput{ + Env: "dev", Fixture: prepared, + Exec: func(context.Context, string, []string) CommandResult { + return CommandResult{NotExecuted: true} + }, + }) + assert.Equal(t, StatusError, result.Steps[0].Status) + assert.True(t, result.Steps[0].Unsafe) + assert.Equal(t, StatusSkip, result.Steps[1].Status) + }) + + t.Run("plan expr uses one execution", func(t *testing.T) { + t.Parallel() + prepared := prepareRun(t, ` +env: dev +steps: + - plan: + expr: + - 'plan.changes.size() == 0' + - 'plan.tasks.size() > 0' +`) + var calls int + result := Run(context.Background(), RunInput{ + Env: "dev", Fixture: prepared, + Exec: func(context.Context, string, []string) CommandResult { + calls++ + return CommandResult{Stdout: `{"changes":[],"tasks":[{"name":"migrate"}]}`, ExitCode: 0} + }, + }) + assert.Equal(t, StatusPass, result.Steps[0].Status) + assert.Equal(t, 1, calls) + + falseResult := Run(context.Background(), RunInput{ + Env: "dev", Fixture: prepared, + Exec: func(context.Context, string, []string) CommandResult { + return CommandResult{Stdout: `{"changes":[{}],"tasks":[]}`, ExitCode: 0} + }, + }) + assert.Equal(t, StatusFail, falseResult.Steps[0].Status) + + badJSON := Run(context.Background(), RunInput{ + Env: "dev", Fixture: prepared, + Exec: func(context.Context, string, []string) CommandResult { + return CommandResult{Stdout: "not-json", ExitCode: 1} + }, + }) + assert.Equal(t, StatusFail, badJSON.Steps[0].Status, "exit mismatch is an expectation failure before JSON is required") + }) + + t.Run("plan output only allows bad json", func(t *testing.T) { + t.Parallel() + prepared := prepareRun(t, ` +env: dev +steps: + - plan: + output: + exitCode: 1 + stderr: + contains: ['failed'] +`) + result := Run(context.Background(), RunInput{ + Env: "dev", Fixture: prepared, + Exec: func(context.Context, string, []string) CommandResult { + return CommandResult{Stdout: "not-json", Stderr: "failed", ExitCode: 1} + }, + }) + assert.Equal(t, StatusPass, result.Steps[0].Status) + }) + + t.Run("malformed plan json with expr is error", func(t *testing.T) { + t.Parallel() + prepared := prepareRun(t, ` +env: dev +steps: + - plan: + output: + exitCode: 0 + expr: + - 'plan.changes.size() == 0' +`) + result := Run(context.Background(), RunInput{ + Env: "dev", Fixture: prepared, + Exec: func(context.Context, string, []string) CommandResult { + return CommandResult{Stdout: "{", ExitCode: 0} + }, + }) + assert.Equal(t, StatusError, result.Steps[0].Status) + assert.Contains(t, result.Steps[0].Detail, "plan JSON") + }) + + t.Run("step stop skips later steps", func(t *testing.T) { + t.Parallel() + prepared := prepareRun(t, ` +env: dev +steps: + - deploy: {} + stopOnFailure: true + - delete: {} + stopOnFailure: false +`) + result := Run(context.Background(), RunInput{ + Env: "dev", Fixture: prepared, + Exec: func(context.Context, string, []string) CommandResult { + return CommandResult{ExitCode: 1} + }, + }) + assert.Equal(t, StatusFail, result.Steps[0].Status) + assert.Equal(t, StatusSkip, result.Steps[1].Status) + }) + + t.Run("shared deadline records both assertions", func(t *testing.T) { + t.Parallel() + prepared := prepareRun(t, ` +env: dev +timeout: 25ms +steps: + - assert: + - select: + apiVersion: v1 + kind: ConfigMap + name: a + expr: ['objects.size() == 1'] + - select: + apiVersion: v1 + kind: ConfigMap + name: b + expr: ['objects.size() == 1'] +`) + seen := map[string]int{} + result := Run(context.Background(), RunInput{ + Env: "dev", Fixture: prepared, ResourcePoll: time.Millisecond, + Observe: func(_ context.Context, sel ResourceSelect, _ string) ([]map[string]any, error) { + seen[sel.Name]++ + return []map[string]any{}, nil + }, + }) + assert.Equal(t, StatusFail, result.Steps[0].Status) + assert.Equal(t, StatusFail, result.Steps[0].Assertions[0].Status) + assert.Equal(t, StatusFail, result.Steps[0].Assertions[1].Status) + assert.Greater(t, seen["a"], 0) + assert.Greater(t, seen["b"], 0) + }) +} + +func TestRun_assertions(t *testing.T) { + t.Parallel() + object := []map[string]any{{"kind": "ConfigMap"}} + terminal := errors.Join(ErrObserveTerminal, errors.New("forbidden")) + tests := []struct { + name string + doc string + observe func(name string, call int) ([]map[string]any, error) + wantSeen []string + want []Status + step Status + next Status + hasNext bool + stopped bool + minCalls int + }{ + { + name: "first terminal stop skips later assertions", + doc: ` +env: dev +timeout: 40ms +steps: + - assert: + - select: + apiVersion: v1 + kind: ConfigMap + name: a + expr: ['false'] + stopOnFailure: true + - select: + apiVersion: v1 + kind: ConfigMap + name: b + expr: ['objects.size() == 1'] + stopOnFailure: false + - deploy: {} +`, + observe: func(name string, _ int) ([]map[string]any, error) { + if name == "a" { + return nil, terminal + } + return object, nil + }, + wantSeen: []string{"a"}, + want: []Status{StatusError, StatusSkip}, + step: StatusError, + next: StatusSkip, + hasNext: true, + stopped: true, + }, + { + name: "middle terminal stop skips the rest of the tick", + doc: ` +env: dev +timeout: 1s +steps: + - assert: + - select: + apiVersion: v1 + kind: ConfigMap + name: a + expr: ['objects.size() == 1'] + - select: + apiVersion: v1 + kind: ConfigMap + name: b + expr: ['false'] + stopOnFailure: true + - select: + apiVersion: v1 + kind: ConfigMap + name: c + expr: ['objects.size() == 1'] + stopOnFailure: false + - deploy: {} +`, + observe: func(name string, _ int) ([]map[string]any, error) { + if name == "b" { + return nil, terminal + } + return object, nil + }, + wantSeen: []string{"a", "b"}, + want: []Status{StatusPass, StatusError, StatusSkip}, + step: StatusError, + next: StatusSkip, + hasNext: true, + stopped: true, + }, + { + name: "last assertion stop", + doc: ` +env: dev +timeout: 1s +steps: + - assert: + - select: + apiVersion: v1 + kind: ConfigMap + name: a + expr: ['objects.size() == 1'] + - select: + apiVersion: v1 + kind: ConfigMap + name: b + expr: ['false'] + stopOnFailure: true +`, + observe: func(name string, _ int) ([]map[string]any, error) { + if name == "b" { + return []map[string]any{}, nil + } + return object, nil + }, + want: []Status{StatusPass, StatusFail}, + step: StatusFail, + stopped: true, + }, + { + name: "nonfatal failures stay inside the step", + doc: ` +env: dev +timeout: 30ms +steps: + - assert: + - select: + apiVersion: v1 + kind: ConfigMap + name: a + expr: ['objects.size() == 1'] + - select: + apiVersion: v1 + kind: ConfigMap + name: b + expr: ['objects.size() == 1'] + stopOnFailure: false + - deploy: {} +`, + observe: func(string, int) ([]map[string]any, error) { + return []map[string]any{}, nil + }, + want: []Status{StatusFail, StatusFail}, + step: StatusFail, + next: StatusPass, + hasNext: true, + }, + { + name: "nonfatal failures then step stop", + doc: ` +env: dev +timeout: 40ms +steps: + - assert: + - select: + apiVersion: v1 + kind: ConfigMap + name: a + expr: ['objects.size() == 1'] + stopOnFailure: false + - select: + apiVersion: v1 + kind: ConfigMap + name: b + expr: ['objects.size() == 1'] + stopOnFailure: false + stopOnFailure: true + - deploy: {} +`, + observe: func(string, int) ([]map[string]any, error) { + return []map[string]any{}, nil + }, + want: []Status{StatusFail, StatusFail}, + step: StatusFail, + next: StatusSkip, + hasNext: true, + }, + { + name: "transient mismatch then success", + doc: ` +env: dev +timeout: 1s +steps: + - assert: + - select: + apiVersion: v1 + kind: ConfigMap + name: a + expr: ['objects.size() == 1'] +`, + observe: func(_ string, call int) ([]map[string]any, error) { + if call == 1 { + return []map[string]any{}, nil + } + return object, nil + }, + want: []Status{StatusPass}, + step: StatusPass, + minCalls: 2, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + prepared := prepareRun(t, tt.doc) + var seen []string + var calls int + result := Run(context.Background(), RunInput{ + Env: "dev", Fixture: prepared, ResourcePoll: time.Millisecond, + Observe: func(_ context.Context, sel ResourceSelect, _ string) ([]map[string]any, error) { + calls++ + seen = append(seen, sel.Name) + return tt.observe(sel.Name, calls) + }, + Exec: func(context.Context, string, []string) CommandResult { + return CommandResult{ExitCode: 0} + }, + }) + if tt.wantSeen != nil { + assert.Equal(t, tt.wantSeen, seen) + } + require.Len(t, result.Steps[0].Assertions, len(tt.want)) + for i, status := range tt.want { + assert.Equal(t, status, result.Steps[0].Assertions[i].Status) + } + assert.Equal(t, tt.step, result.Steps[0].Status) + assert.Equal(t, tt.stopped, result.Steps[0].StoppedHere) + if tt.hasNext { + require.Len(t, result.Steps, 2) + assert.Equal(t, tt.next, result.Steps[1].Status) + } + if tt.minCalls > 0 { + assert.GreaterOrEqual(t, calls, tt.minCalls) + } + assert.Equal(t, tt.step == StatusFail || tt.step == StatusError, result.Failed()) + }) + } +} + +func TestRun_logs(t *testing.T) { + t.Parallel() + tests := []struct { + name string + doc string + cancel bool + reply func(call int) CommandResult + want Status + detail string + notDetail string + calls int + minCalls int + next Status + hasNext bool + unsafe bool + }{ + { + name: "content appears on a later poll", + doc: ` +env: dev +timeout: 200ms +steps: + - logs: + component: web + contains: ['started'] +`, + reply: func(call int) CommandResult { + if call < 2 { + return CommandResult{Stdout: "waiting", ExitCode: 0} + } + return CommandResult{Stdout: "started", ExitCode: 0} + }, + want: StatusPass, + minCalls: 2, + }, + { + name: "terminal exit stops", + doc: ` +env: dev +timeout: 1s +steps: + - logs: + component: missing + contains: ['started'] + - deploy: {} +`, + reply: func(int) CommandResult { + return CommandResult{Stderr: "component not found", ExitCode: 1} + }, + want: StatusError, + detail: "component not found", + calls: 1, + next: StatusSkip, + hasNext: true, + }, + { + name: "retryable failure then a match", + doc: ` +env: dev +timeout: 1s +steps: + - logs: + component: web + contains: ['ready'] +`, + reply: func(call int) CommandResult { + if call == 1 { + return CommandResult{Stderr: "connection reset", ExitCode: 1, Retryable: true} + } + return CommandResult{Stdout: "ready", ExitCode: 0} + }, + want: StatusPass, + calls: 2, + }, + { + name: "retryable failure exhausts the deadline", + doc: ` +env: dev +timeout: 30ms +steps: + - logs: + component: web + contains: ['ready'] + stopOnFailure: false + - deploy: {} +`, + reply: func(int) CommandResult { + return CommandResult{Stderr: "timeout talking to apiserver", ExitCode: 1, Retryable: true} + }, + want: StatusError, + notDetail: "missing pattern", + next: StatusPass, + hasNext: true, + }, + { + name: "retrieved logs that never match", + doc: ` +env: dev +timeout: 30ms +steps: + - logs: + component: web + contains: ['ready'] +`, + reply: func(int) CommandResult { + return CommandResult{Stdout: "still starting", ExitCode: 0} + }, + want: StatusFail, + detail: "ready", + }, + { + name: "command was not executed", + doc: ` +env: dev +timeout: 1s +steps: + - logs: + component: web + contains: ['x'] +`, + reply: func(int) CommandResult { + return CommandResult{NotExecuted: true} + }, + want: StatusError, + detail: "not executed", + unsafe: true, + }, + { + name: "interrupted retrieval", + doc: ` +env: dev +timeout: 1s +steps: + - logs: + component: web + contains: ['x'] +`, + reply: func(int) CommandResult { + return CommandResult{Interrupted: true} + }, + want: StatusError, + detail: "interrupted", + unsafe: true, + }, + { + name: "canceled parent", + cancel: true, + doc: ` +env: dev +timeout: 1s +steps: + - logs: + component: web + contains: ['ready'] + stopOnFailure: false + - deploy: {} +`, + reply: func(int) CommandResult { + return CommandResult{Stdout: "ready", ExitCode: 0} + }, + want: StatusError, + unsafe: true, + next: StatusSkip, + hasNext: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + prepared := prepareRun(t, tt.doc) + ctx := context.Background() + if tt.cancel { + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(context.Background()) + cancel() + } + var calls int + result := Run(ctx, RunInput{ + Env: "dev", Project: "p", Fixture: prepared, LogsPoll: time.Millisecond, + Exec: func(_ context.Context, _ string, args []string) CommandResult { + if len(args) == 0 || args[0] != "logs" { + return CommandResult{ExitCode: 0} + } + calls++ + return tt.reply(calls) + }, + }) + assert.Equal(t, tt.want, result.Steps[0].Status) + assert.Equal(t, tt.unsafe, result.Steps[0].Unsafe) + if tt.detail != "" { + assert.Contains(t, result.Steps[0].Detail, tt.detail) + } + if tt.notDetail != "" { + assert.NotContains(t, result.Steps[0].Detail, tt.notDetail) + } + if tt.calls > 0 { + assert.Equal(t, tt.calls, calls) + } + if tt.minCalls > 0 { + assert.GreaterOrEqual(t, calls, tt.minCalls) + } + if tt.hasNext { + require.Len(t, result.Steps, 2) + assert.Equal(t, tt.next, result.Steps[1].Status) + } + }) + } +} + +func TestRun_expect(t *testing.T) { + t.Parallel() + doc := ` +env: dev +timeout: 20ms +steps: + - assert: + - select: + apiVersion: v1 + kind: ConfigMap + name: a + expect: + data: + ok: "yes" +` + tests := []struct { + name string + objects []map[string]any + detail string + }{ + {name: "empty selection", objects: []map[string]any{}, detail: "no resources selected"}, + {name: "field mismatch", objects: []map[string]any{{"data": map[string]any{"ok": "no"}}}, detail: "ok"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + prepared := prepareRun(t, doc) + result := Run(context.Background(), RunInput{ + Env: "dev", Fixture: prepared, ResourcePoll: time.Millisecond, + Observe: func(context.Context, ResourceSelect, string) ([]map[string]any, error) { + return tt.objects, nil + }, + }) + assert.Equal(t, StatusFail, result.Steps[0].Status) + assert.Contains(t, result.Steps[0].Assertions[0].Detail, tt.detail) + }) + } +} + +func TestRun_edges(t *testing.T) { + t.Parallel() + + t.Run("nil context", func(t *testing.T) { + t.Parallel() + // Run rejects a missing context instead of panicking. + var missing context.Context + result := Run(missing, RunInput{}) + require.Len(t, result.Steps, 1) + assert.Equal(t, StatusError, result.Steps[0].Status) + assert.True(t, result.Steps[0].Unsafe) + assert.True(t, result.Failed()) + }) + + t.Run("parent cancel during deploy", func(t *testing.T) { + t.Parallel() + prepared := prepareRun(t, "env: dev\ntimeout: 1s\nsteps:\n - deploy: {}\n stopOnFailure: false\n - delete: {}\n") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + result := Run(ctx, RunInput{ + Env: "dev", Namespace: "ns", Fixture: prepared, + Exec: func(context.Context, string, []string) CommandResult { + return CommandResult{Stderr: "canceled", Interrupted: true} + }, + }) + assert.Equal(t, StatusError, result.Steps[0].Status) + assert.True(t, result.Steps[0].Unsafe) + assert.Contains(t, result.Steps[0].Detail, "canceled") + assert.Equal(t, StatusSkip, result.Steps[1].Status) + assert.Contains(t, result.Steps[1].Detail, "steps[0] deploy") + }) + + t.Run("nil observer is unsafe", func(t *testing.T) { + t.Parallel() + prepared := prepareRun(t, ` +env: dev +timeout: 1s +steps: + - assert: + - select: + apiVersion: v1 + kind: ConfigMap + expr: ['true'] + - deploy: {} +`) + result := Run(context.Background(), RunInput{Env: "dev", Fixture: prepared}) + assert.Equal(t, StatusError, result.Steps[0].Status) + assert.True(t, result.Steps[0].Unsafe) + assert.Equal(t, StatusSkip, result.Steps[1].Status) + }) + + t.Run("command flags", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + require.NoError(t, os.WriteFile(dir+"/deployah.yaml", []byte("apiVersion: v1-alpha.5\n"), 0o600)) + require.NoError(t, os.WriteFile(dir+"/e2e.yaml", []byte(` +env: dev +steps: + - deploy: + spec: deployah.yaml + args: ['--atomic'] + - plan: + detailedExitCode: true + - run: + task: migrate + - delete: {} +`), 0o600)) + fx, err := LoadE2EFixture(dir+"/e2e.yaml", dir) + require.NoError(t, err) + prepared, err := PrepareE2EFixture(dir+"/e2e.yaml", fx) + require.NoError(t, err) + var got [][]string + result := Run(context.Background(), RunInput{ + Env: "dev", Namespace: "ns", Project: "demo", Fixture: prepared, + Exec: func(_ context.Context, _ string, args []string) CommandResult { + got = append(got, append([]string(nil), args...)) + return CommandResult{ExitCode: 0} + }, + }) + require.False(t, result.Failed(), result.Steps) + require.Len(t, got, 4) + assert.Equal(t, []string{"deploy", "dev", "--context", KindContext, "--namespace", "ns", "--spec", "deployah.yaml", "--atomic"}, got[0]) + assert.Contains(t, got[1], "--detailed-exitcode") + assert.Equal(t, "run", got[2][0]) + assert.Equal(t, "migrate", got[2][1]) + assert.Equal(t, "delete", got[3][0]) + }) + + t.Run("missing run task", func(t *testing.T) { + t.Parallel() + result := Run(context.Background(), RunInput{ + Fixture: PreparedFixture{Steps: []PreparedStep{{Op: "run", Timeout: time.Second}}}, + Exec: func(context.Context, string, []string) CommandResult { return CommandResult{} }, + }) + assert.Equal(t, StatusError, result.Steps[0].Status) + assert.Contains(t, result.Steps[0].Detail, "run.task") + assert.True(t, result.Steps[0].Unsafe) + }) + + t.Run("plan expression error", func(t *testing.T) { + t.Parallel() + prepared := prepareRun(t, ` +env: dev +steps: + - plan: + expr: + - 'plan' +`) + result := Run(context.Background(), RunInput{ + Env: "dev", Fixture: prepared, + Exec: func(context.Context, string, []string) CommandResult { + return CommandResult{Stdout: `{"changes":[]}`, ExitCode: 0} + }, + }) + assert.Equal(t, StatusError, result.Steps[0].Status) + assert.Contains(t, result.Steps[0].Detail, "expr[0]") + }) + + t.Run("false comparison shows the observed value", func(t *testing.T) { + t.Parallel() + prepared := prepareRun(t, ` +env: dev +steps: + - plan: + expr: + - 'plan.summary.total == 1' + - 'plan.summary.total == 1 || plan.summary.total == 2' +`) + result := Run(context.Background(), RunInput{ + Env: "dev", Fixture: prepared, + Exec: func(context.Context, string, []string) CommandResult { + return CommandResult{Stdout: `{"summary":{"total":0}}`, ExitCode: 0} + }, + }) + assert.Equal(t, StatusFail, result.Steps[0].Status) + assert.Contains(t, result.Steps[0].Detail, "expected: plan.summary.total == 1") + assert.Contains(t, result.Steps[0].Detail, "actual: plan.summary.total = 0") + assert.Contains(t, result.Steps[0].Detail, "expr[1]") + assert.NotContains(t, result.Steps[0].Detail, "expected: plan.summary.total == 1\nactual: plan.summary.total = 0\nexpected:") + }) +} + +func TestScenariosDirResolves(t *testing.T) { + t.Parallel() + info, err := os.Stat(inttest.TestScenariosDir) + require.NoError(t, err) + assert.True(t, info.IsDir()) +} + +func prepareRun(t *testing.T, body string) PreparedFixture { + t.Helper() + dir := t.TempDir() + path := dir + "/e2e.yaml" + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + fx, err := LoadE2EFixture(path, dir) + require.NoError(t, err) + prepared, err := PrepareE2EFixture(path, fx) + require.NoError(t, err) + return prepared +} diff --git a/internal/testing/e2e/schema.go b/internal/testing/e2e/schema.go new file mode 100644 index 0000000..a2bb16c --- /dev/null +++ b/internal/testing/e2e/schema.go @@ -0,0 +1,64 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "fmt" + "sync" + + _ "embed" + + jsonschema "github.com/santhosh-tekuri/jsonschema/v6" +) + +//go:embed e2e.schema.json +var e2eSchemaJSON []byte + +// E2ESchemaJSON returns a copy of the embedded Draft 2020-12 schema for +// scenarios/*/e2e.yaml. Editors still read the source file on disk. +func E2ESchemaJSON() []byte { + return bytes.Clone(e2eSchemaJSON) +} + +var compileE2ESchema = sync.OnceValues(func() (*jsonschema.Schema, error) { + compiler := jsonschema.NewCompiler() + doc, err := jsonschema.UnmarshalJSON(bytes.NewReader(e2eSchemaJSON)) + if err != nil { + return nil, fmt.Errorf("parse embedded e2e schema: %w", err) + } + const id = "e2e.schema.json" + if err = compiler.AddResource(id, doc); err != nil { + return nil, fmt.Errorf("load embedded e2e schema: %w", err) + } + compiled, err := compiler.Compile(id) + if err != nil { + return nil, fmt.Errorf("compile embedded e2e schema: %w", err) + } + return compiled, nil +}) + +// ValidateE2EDocument checks doc against the embedded e2e JSON Schema. +// doc is the generic JSON value, not a decoded fixture struct. +func ValidateE2EDocument(doc any) error { + compiled, err := compileE2ESchema() + if err != nil { + return err + } + if err = compiled.Validate(doc); err != nil { + return fmt.Errorf("e2e schema: %w", err) + } + return nil +} diff --git a/internal/testing/e2e/schema_test.go b/internal/testing/e2e/schema_test.go new file mode 100644 index 0000000..b6d08e3 --- /dev/null +++ b/internal/testing/e2e/schema_test.go @@ -0,0 +1,290 @@ +// Copyright 2026 The Deployah Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/yaml" + + inttest "deployah.dev/deployah/internal/testing" +) + +const specNginxLatest = ` +apiVersion: v1-alpha.5 +project: demo +components: + web: + image: nginx:latest +` + +func TestE2ESchemaJSON(t *testing.T) { + t.Parallel() + raw := E2ESchemaJSON() + require.NotEmpty(t, raw) + raw[0] = 'x' + assert.NotEqual(t, raw[0], E2ESchemaJSON()[0]) +} + +func TestConfigErrorUnwrap(t *testing.T) { + t.Parallel() + base := errors.New("boom") + err := &ConfigError{File: "e2e.yaml", Step: 1, Assertion: 2, Path: "expr", Err: base} + assert.ErrorIs(t, err, base) + assert.Contains(t, err.Error(), "e2e.yaml") + assert.Contains(t, err.Error(), "steps[1]") + assert.Contains(t, err.Error(), "expr") +} + +func TestE2ESchema_compiles(t *testing.T) { + t.Parallel() + require.NoError(t, ValidateE2EDocument(map[string]any{ + "env": "dev", + "steps": []any{ + map[string]any{"deploy": map[string]any{}}, + }, + })) +} + +func TestE2ESchema_documents(t *testing.T) { + t.Parallel() + tests := []struct { + name string + doc string + }{ + {name: "deploy", doc: "env: dev\nsteps:\n - deploy: {}\n"}, + {name: "plan", doc: ` +env: dev +steps: + - plan: + detailedExitCode: true + output: + exitCode: 2 + expr: + - 'plan.summary.create > 0' +`}, + {name: "run", doc: "env: dev\nsteps:\n - run:\n task: migrate\n"}, + {name: "logs", doc: "env: dev\nsteps:\n - logs:\n component: web\n contains: ['ready']\n"}, + {name: "delete", doc: "env: dev\nsteps:\n - delete: {}\n"}, + {name: "assert", doc: ` +env: dev +steps: + - assert: + - select: + apiVersion: apps/v1 + kind: Deployment + name: web + expect: + spec: + replicas: 2 + template: + metadata: + annotations: + example.com/unlisted: kept + expr: + - 'objects.size() == 1' +`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + require.NoError(t, ValidateE2EDocument(mustYAML(t, tt.doc))) + }) + } +} + +func TestE2ESchema_rejects(t *testing.T) { + t.Parallel() + tests := []struct { + name string + doc string + wantErr string + }{ + {name: "unknown property", doc: "env: dev\nextra: true\nsteps:\n - deploy: {}\n", wantErr: "additional"}, + {name: "unknown operation", doc: "env: dev\nsteps:\n - bounce: {}\n", wantErr: "additional"}, + {name: "two operations", doc: "env: dev\nsteps:\n - deploy: {}\n run:\n task: x\n", wantErr: "'oneOf' failed"}, + {name: "missing env", doc: "steps:\n - deploy: {}\n", wantErr: "env"}, + {name: "empty steps", doc: "env: dev\nsteps: []\n", wantErr: "minItems"}, + {name: "empty assert", doc: "env: dev\nsteps:\n - assert: []\n", wantErr: "minItems"}, + {name: "selector missing kind", doc: ` +env: dev +steps: + - assert: + - select: + apiVersion: v1 + expr: ['objects.size() == 0'] +`, wantErr: "kind"}, + {name: "missing expect and expr", doc: ` +env: dev +steps: + - assert: + - select: + apiVersion: v1 + kind: ConfigMap +`, wantErr: "missing property 'expect'"}, + {name: "negative exit", doc: ` +env: dev +steps: + - deploy: + output: + exitCode: -1 +`, wantErr: "minimum"}, + {name: "stopOnFailure type", doc: "env: dev\nsteps:\n - deploy: {}\n stopOnFailure: yes-please\n", wantErr: "stopOnFailure"}, + {name: "misspelled stopOnFailure", doc: "env: dev\nsteps:\n - deploy: {}\n stopOnFailur: true\n", wantErr: "additional"}, + {name: "unknown assertion key", doc: ` +env: dev +steps: + - assert: + - select: + apiVersion: v1 + kind: ConfigMap + expr: ['true'] + match: {} +`, wantErr: "additional"}, + {name: "old stdoutContains", doc: "env: dev\nsteps:\n - deploy: {}\n stdoutContains: Deployed\n", wantErr: "additional"}, + {name: "empty expr", doc: ` +env: dev +steps: + - plan: + expr: [] +`, wantErr: "minItems"}, + {name: "old resources shorthand", doc: ` +env: dev +resources: + - match: + apiVersion: v1 + kind: ConfigMap +`, wantErr: "additional"}, + {name: "old stderrContains", doc: "env: dev\nsteps:\n - deploy: {}\n stderrContains: Deployed\n", wantErr: "additional"}, + {name: "old minCount", doc: ` +env: dev +steps: + - assert: + - minCount: 0 + select: + apiVersion: v1 + kind: ConfigMap + expr: ['true'] +`, wantErr: "additional"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := ValidateE2EDocument(mustYAML(t, tt.doc)) + require.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestE2ESchema_scenarioFiles(t *testing.T) { + t.Parallel() + if _, err := os.Stat(inttest.TestScenariosDir); err != nil { + t.Skip("scenarios directory not found") + } + scenarios, err := inttest.DiscoverScenarios(inttest.TestScenariosDir) + require.NoError(t, err) + found := 0 + for _, sc := range scenarios { + if !sc.HasE2EFixture { + continue + } + found++ + t.Run(sc.Name, func(t *testing.T) { + t.Parallel() + raw, readErr := os.ReadFile(sc.E2EFixturePath) + require.NoError(t, readErr) + require.NoError(t, ValidateE2EDocument(mustYAML(t, string(raw)))) + }) + } + require.Greater(t, found, 0) +} + +func mustYAML(t *testing.T, doc string) any { + t.Helper() + raw, err := yaml.YAMLToJSON([]byte(doc)) + require.NoError(t, err) + var out any + require.NoError(t, json.Unmarshal(raw, &out)) + return out +} + +func TestLoadE2EFixture(t *testing.T) { + t.Parallel() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "deployah.yaml"), []byte(specNginxLatest), 0o600)) + body := []byte(` +env: dev +steps: + - deploy: + spec: deployah.yaml + - assert: + - select: + apiVersion: apps/v1 + kind: Deployment + name: demo-dev + expr: + - 'objects.size() == 1' +`) + path := filepath.Join(dir, "e2e.yaml") + require.NoError(t, os.WriteFile(path, body, 0o600)) + fx, err := LoadE2EFixture(path, dir) + require.NoError(t, err) + assert.Equal(t, "dev", fx.Env) + assert.True(t, fx.RunParallel()) + require.Len(t, fx.Steps, 2) + assert.Equal(t, "deploy", fx.Steps[0].OpName()) + assert.Equal(t, "assert", fx.Steps[1].OpName()) + assert.False(t, fx.Steps[1].Assert[0].StopOnFailureEnabled()) + assert.True(t, fx.Steps[0].StopOnFailureEnabled()) +} + +func TestLoadE2EFixture_specFile(t *testing.T) { + t.Parallel() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "deployah.yaml"), []byte(` +apiVersion: v1-alpha.5 +project: demo +components: + web: + image: ghcr.io/acme/api:1.2.3 +`), 0o600)) + path := filepath.Join(dir, "e2e.yaml") + require.NoError(t, os.WriteFile(path, []byte("env: dev\nsteps:\n - deploy:\n spec: deployah.yaml\n"), 0o600)) + fx, err := LoadE2EFixture(path, dir) + require.NoError(t, err) + require.NotNil(t, fx.Steps[0].Deploy) + + require.NoError(t, os.WriteFile(path, []byte("env: dev\nsteps:\n - deploy:\n spec: missing.yaml\n"), 0o600)) + _, err = LoadE2EFixture(path, dir) + require.ErrorContains(t, err, "missing.yaml") +} + +func TestLoadE2EFixture_invalidYAML(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "e2e.yaml") + require.NoError(t, os.WriteFile(path, []byte(":\n - ["), 0o600)) + _, err := LoadE2EFixture(path, dir) + require.Error(t, err) + var cfg *ConfigError + require.ErrorAs(t, err, &cfg) + assert.Equal(t, path, cfg.File) +} diff --git a/internal/testing/e2e_fixture.go b/internal/testing/e2e_fixture.go deleted file mode 100644 index 1be4355..0000000 --- a/internal/testing/e2e_fixture.go +++ /dev/null @@ -1,314 +0,0 @@ -// Copyright 2025 The Deployah Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package testing - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "time" - - "sigs.k8s.io/yaml" - - k8sjson "k8s.io/apimachinery/pkg/util/json" -) - -// E2EFixtureFile is the Kind fixture filename in a scenario directory. -const E2EFixtureFile = "e2e.yaml" - -// DefaultE2EStepTimeout is the wait applied when e2e.yaml omits timeout. -const DefaultE2EStepTimeout = 3 * time.Minute - -// AllowedE2EImages is the Kind preload list. [LoadE2EFixture] rejects -// scenario specs that reference any other image. -var AllowedE2EImages = []string{ - "nginx:latest", - "nginx:1.26", - "busybox:1.36", - "redis:7-alpine", - "redis:7", - "postgres:16", -} - -// E2EFixture is the decoded form of scenarios/*/e2e.yaml, requiring -// exactly one of Resources or Steps. -type E2EFixture struct { - Env string `json:"env"` - Parallel *bool `json:"parallel,omitempty"` - Timeout string `json:"timeout,omitempty"` - Resources []ResourceAssertion `json:"resources,omitempty"` - Steps []Step `json:"steps,omitempty"` -} - -// ResourceAssertion is one cluster assertion. -// Match is a partial Kubernetes object compared with [DiffSubset]. -// MinCount defaults to 1; 0 requires the list to be empty. -// When Match sets metadata.name, MinCount may only be 0 or 1 because a -// named Get cannot count list items. -type ResourceAssertion struct { - Match map[string]any `json:"match"` - MinCount *int `json:"minCount,omitempty"` -} - -// Step is one CLI operation plus optional output and resource assertions. -// Exactly one of Deploy, Run, Logs, or Delete must be set. -type Step struct { - Deploy *DeployOp `json:"deploy,omitempty"` - Run *RunOp `json:"run,omitempty"` - Logs *LogsOp `json:"logs,omitempty"` - Delete *DeleteOp `json:"delete,omitempty"` - Timeout string `json:"timeout,omitempty"` - StderrContains string `json:"stderrContains,omitempty"` - StdoutContains string `json:"stdoutContains,omitempty"` - Resources []ResourceAssertion `json:"resources,omitempty"` -} - -// DeployOp maps to `deployah deploy`. -type DeployOp struct { - Spec string `json:"spec,omitempty"` - Args []string `json:"args,omitempty"` -} - -// RunOp maps to `deployah run`. -type RunOp struct { - Task string `json:"task"` -} - -// LogsOp maps to `deployah logs`. -type LogsOp struct { - Component string `json:"component"` - Contains string `json:"contains"` -} - -// DeleteOp maps to `deployah delete`. -type DeleteOp struct{} - -// RunParallel reports whether the fixture may run concurrently. -// A nil Parallel field means true. -func (f E2EFixture) RunParallel() bool { - return f.Parallel == nil || *f.Parallel -} - -// StepTimeout is the per-step wait: step override, then fixture default, -// then [DefaultE2EStepTimeout]. -func (f E2EFixture) StepTimeout(step Step) (time.Duration, error) { - raw := step.Timeout - if raw == "" { - raw = f.Timeout - } - if raw == "" { - return DefaultE2EStepTimeout, nil - } - d, err := time.ParseDuration(raw) - if err != nil { - return 0, fmt.Errorf("timeout %q: %w", raw, err) - } - if d <= 0 { - return 0, fmt.Errorf("timeout %q must be positive", raw) - } - return d, nil -} - -// Count returns the minimum matching resources, or 1 when MinCount is unset. -func (r ResourceAssertion) Count() int { - if r.MinCount == nil { - return 1 - } - return *r.MinCount -} - -// OpName returns the operation name, or "" if none or more than one is set. -func (s Step) OpName() string { - n := 0 - name := "" - if s.Deploy != nil { - n++ - name = "deploy" - } - if s.Run != nil { - n++ - name = "run" - } - if s.Logs != nil { - n++ - name = "logs" - } - if s.Delete != nil { - n++ - name = "delete" - } - if n != 1 { - return "" - } - return name -} - -// LoadE2EFixture reads path, validates document shape, and checks that -// images in scenarioDir specs are on [AllowedE2EImages]. -func LoadE2EFixture(path, scenarioDir string) (E2EFixture, error) { - raw, err := os.ReadFile(path) // #nosec G304 -- scenario e2e.yaml under test - if err != nil { - return E2EFixture{}, fmt.Errorf("read e2e.yaml: %w", err) - } - jsonBytes, err := yaml.YAMLToJSON(raw) - if err != nil { - return E2EFixture{}, fmt.Errorf("parse e2e.yaml: %w", err) - } - var fx E2EFixture - if decErr := k8sjson.Unmarshal(jsonBytes, &fx); decErr != nil { - return E2EFixture{}, fmt.Errorf("decode e2e.yaml: %w", decErr) - } - if valErr := fx.validate(); valErr != nil { - return E2EFixture{}, valErr - } - if imgErr := validateScenarioImages(scenarioDir, fx); imgErr != nil { - return E2EFixture{}, imgErr - } - return fx, nil -} - -func (f E2EFixture) validate() error { - if f.Env == "" { - return fmt.Errorf("e2e.yaml: env is required") - } - hasRes := len(f.Resources) > 0 - hasSteps := len(f.Steps) > 0 - if hasRes == hasSteps { - return fmt.Errorf("e2e.yaml: exactly one of resources or steps is required") - } - if _, err := f.StepTimeout(Step{}); err != nil { - return fmt.Errorf("e2e.yaml: %w", err) - } - for i, ra := range f.Resources { - if err := ra.validate(); err != nil { - return fmt.Errorf("e2e.yaml resources[%d]: %w", i, err) - } - } - for i, step := range f.Steps { - if step.OpName() == "" { - return fmt.Errorf("e2e.yaml steps[%d]: exactly one of deploy, run, logs, delete is required", i) - } - if step.Run != nil && step.Run.Task == "" { - return fmt.Errorf("e2e.yaml steps[%d]: run.task is required", i) - } - if step.Logs != nil && (step.Logs.Component == "" || step.Logs.Contains == "") { - return fmt.Errorf("e2e.yaml steps[%d]: logs.component and logs.contains are required", i) - } - if _, err := f.StepTimeout(step); err != nil { - return fmt.Errorf("e2e.yaml steps[%d]: %w", i, err) - } - for j, ra := range step.Resources { - if err := ra.validate(); err != nil { - return fmt.Errorf("e2e.yaml steps[%d].resources[%d]: %w", i, j, err) - } - } - } - return nil -} - -func (r ResourceAssertion) validate() error { - if len(r.Match) == 0 { - return fmt.Errorf("match is required") - } - kind, hasKind := r.Match["kind"].(string) - apiVersion, hasAPI := r.Match["apiVersion"].(string) - if !hasKind || kind == "" || !hasAPI || apiVersion == "" { - return fmt.Errorf("match requires apiVersion and kind") - } - if r.Count() < 0 { - return fmt.Errorf("minCount must be >= 0") - } - if r.Count() > 1 { - metaMap, hasMeta := r.Match["metadata"].(map[string]any) - if hasMeta { - name, hasName := metaMap["name"].(string) - if hasName && name != "" { - return fmt.Errorf("minCount > 1 cannot be used with metadata.name") - } - } - } - return nil -} - -func validateScenarioImages(scenarioDir string, fx E2EFixture) error { - specs := []string{filepath.Join(scenarioDir, "deployah.yaml")} - for _, step := range fx.Steps { - if step.Deploy != nil && step.Deploy.Spec != "" { - specs = append(specs, filepath.Join(scenarioDir, step.Deploy.Spec)) - } - } - seen := map[string]struct{}{} - for _, specPath := range specs { - if _, err := os.Stat(specPath); err != nil { - continue - } - raw, err := os.ReadFile(specPath) // #nosec G304 -- scenario spec under test - if err != nil { - return fmt.Errorf("read spec %s: %w", specPath, err) - } - var doc any - if parseErr := yaml.Unmarshal(raw, &doc); parseErr != nil { - return fmt.Errorf("parse spec %s: %w", specPath, parseErr) - } - for _, img := range collectImages(doc) { - if _, ok := seen[img]; ok { - continue - } - seen[img] = struct{}{} - if !imageAllowed(img) { - return fmt.Errorf("image %q is not on the e2e allowlist", img) - } - } - } - return nil -} - -func collectImages(v any) []string { - var out []string - switch n := v.(type) { - case map[string]any: - for k, child := range n { - if k == "image" { - if s, ok := child.(string); ok && s != "" && !strings.Contains(s, "${") { - out = append(out, s) - } - continue - } - out = append(out, collectImages(child)...) - } - case []any: - for _, child := range n { - out = append(out, collectImages(child)...) - } - } - return out -} - -func imageAllowed(img string) bool { - norm := normalizeImageRef(img) - for _, allowed := range AllowedE2EImages { - if normalizeImageRef(allowed) == norm { - return true - } - } - return false -} - -func normalizeImageRef(img string) string { - img = strings.TrimPrefix(img, "docker.io/library/") - img = strings.TrimPrefix(img, "docker.io/") - return img -} diff --git a/internal/testing/e2e_fixture_test.go b/internal/testing/e2e_fixture_test.go deleted file mode 100644 index a410b50..0000000 --- a/internal/testing/e2e_fixture_test.go +++ /dev/null @@ -1,333 +0,0 @@ -// Copyright 2025 The Deployah Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package testing - -import ( - "bytes" - "encoding/json" - "os" - "path/filepath" - "testing" - - "github.com/santhosh-tekuri/jsonschema/v6" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "sigs.k8s.io/yaml" -) - -const specNginxLatest = ` -apiVersion: v1-alpha.5 -project: demo -components: - web: - image: nginx:latest -` - -func TestLoadE2EFixture(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - spec string - e2e string - wantEnv string - wantParallel bool - wantNRes int - wantCount int - }{ - { - name: "simple deployment", - spec: specNginxLatest, - e2e: ` -env: dev -resources: - - match: - apiVersion: apps/v1 - kind: Deployment - metadata: - name: demo-dev -`, - wantEnv: "dev", - wantParallel: true, - wantNRes: 1, - wantCount: 1, - }, - { - name: "docker.io library prefix allowed", - spec: ` -apiVersion: v1-alpha.5 -project: demo -components: - web: - image: docker.io/library/nginx:latest -`, - e2e: ` -env: dev -resources: - - match: - apiVersion: apps/v1 - kind: Deployment - metadata: {name: x} -`, - wantEnv: "dev", - wantParallel: true, - wantNRes: 1, - wantCount: 1, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "deployah.yaml"), []byte(tt.spec), 0o600)) - path := filepath.Join(dir, "e2e.yaml") - require.NoError(t, os.WriteFile(path, []byte(tt.e2e), 0o600)) - - fx, err := LoadE2EFixture(path, dir) - require.NoError(t, err) - assert.Equal(t, tt.wantEnv, fx.Env) - assert.Equal(t, tt.wantParallel, fx.RunParallel()) - require.Len(t, fx.Resources, tt.wantNRes) - assert.Equal(t, tt.wantCount, fx.Resources[0].Count()) - }) - } -} - -func TestLoadE2EFixture_Stepped(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - spec string - e2e string - wantEnv string - wantParallel bool - wantOps []string - wantCount int - }{ - { - name: "four closed ops", - spec: specNginxLatest, - e2e: ` -env: staging -parallel: false -steps: - - deploy: - spec: deployah.yaml - args: [--skip-crds] - stderrContains: Deployed - - run: - task: backfill - - logs: - component: web - contains: listening - - delete: {} - resources: - - minCount: 0 - match: - apiVersion: batch/v1 - kind: Job - metadata: - labels: - app: demo -`, - wantEnv: "staging", - wantParallel: false, - wantOps: []string{"deploy", "run", "logs", "delete"}, - wantCount: 0, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "deployah.yaml"), []byte(tt.spec), 0o600)) - path := filepath.Join(dir, "e2e.yaml") - require.NoError(t, os.WriteFile(path, []byte(tt.e2e), 0o600)) - - fx, err := LoadE2EFixture(path, dir) - require.NoError(t, err) - assert.Equal(t, tt.wantEnv, fx.Env) - assert.Equal(t, tt.wantParallel, fx.RunParallel()) - require.Len(t, fx.Steps, len(tt.wantOps)) - for i, op := range tt.wantOps { - assert.Equal(t, op, fx.Steps[i].OpName()) - } - require.NotEmpty(t, fx.Steps[len(fx.Steps)-1].Resources) - assert.Equal(t, tt.wantCount, fx.Steps[len(fx.Steps)-1].Resources[0].Count()) - }) - } -} - -func TestLoadE2EFixture_Error(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - spec string - e2e string - wantErr string - }{ - { - name: "unknown image", - spec: ` -apiVersion: v1-alpha.5 -project: demo -components: - web: - image: ghcr.io/acme/not-allowed:1 -`, - e2e: ` -env: dev -resources: - - match: - kind: Deployment - apiVersion: apps/v1 - metadata: - name: x -`, - wantErr: "allowlist", - }, - { - name: "mixed resources and steps", - e2e: ` -env: dev -resources: - - match: - kind: Deployment - apiVersion: apps/v1 - metadata: {name: x} -steps: - - deploy: {} -`, - wantErr: "exactly one of resources or steps", - }, - { - name: "missing env", - e2e: ` -resources: - - match: - apiVersion: apps/v1 - kind: Deployment - metadata: {name: x} -`, - wantErr: "env is required", - }, - { - name: "match missing kind", - e2e: ` -env: dev -resources: - - match: - apiVersion: apps/v1 - metadata: {name: x} -`, - wantErr: "match requires apiVersion and kind", - }, - { - name: "named match with minCount 2", - e2e: ` -env: dev -resources: - - minCount: 2 - match: - apiVersion: apps/v1 - kind: Deployment - metadata: {name: x} -`, - wantErr: "minCount > 1 cannot be used with metadata.name", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - dir := t.TempDir() - if tt.spec != "" { - require.NoError(t, os.WriteFile(filepath.Join(dir, "deployah.yaml"), []byte(tt.spec), 0o600)) - } - path := filepath.Join(dir, "e2e.yaml") - require.NoError(t, os.WriteFile(path, []byte(tt.e2e), 0o600)) - - _, err := LoadE2EFixture(path, dir) - require.Error(t, err) - assert.ErrorContains(t, err, tt.wantErr) - }) - } -} - -func TestImageAllowed(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - img string - want bool - }{ - {name: "nginx latest", img: "nginx:latest", want: true}, - {name: "docker.io library prefix", img: "docker.io/library/nginx:latest", want: true}, - {name: "docker.io prefix", img: "docker.io/nginx:latest", want: true}, - {name: "redis alpine", img: "redis:7-alpine", want: true}, - {name: "unknown registry", img: "ghcr.io/acme/not-allowed:1", want: false}, - {name: "unknown tag", img: "nginx:not-a-real-tag", want: false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - assert.Equal(t, tt.want, imageAllowed(tt.img)) - }) - } -} - -func TestE2ESchema_validatesScenarioFiles(t *testing.T) { - t.Parallel() - - schemaBytes, err := os.ReadFile("e2e.schema.json") - require.NoError(t, err) - - compiler := jsonschema.NewCompiler() - doc, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaBytes)) - require.NoError(t, err) - require.NoError(t, compiler.AddResource("e2e.schema.json", doc)) - compiled, err := compiler.Compile("e2e.schema.json") - require.NoError(t, err) - - if _, statErr := os.Stat(TestScenariosDir); statErr != nil { - t.Skip("scenarios directory not found") - } - scenarios, err := DiscoverScenarios(TestScenariosDir) - require.NoError(t, err) - found := 0 - for _, sc := range scenarios { - if !sc.HasE2EFixture { - continue - } - found++ - t.Run(sc.Name, func(t *testing.T) { - t.Parallel() - raw, readErr := os.ReadFile(sc.E2EFixturePath) - require.NoError(t, readErr) - jsonBytes, yamlErr := yaml.YAMLToJSON(raw) - require.NoError(t, yamlErr) - var obj any - require.NoError(t, json.Unmarshal(jsonBytes, &obj)) - require.NoError(t, compiled.Validate(obj)) - }) - } - require.Greater(t, found, 0, "expected at least one scenarios/*/e2e.yaml") -} diff --git a/internal/testing/scenario_discovery.go b/internal/testing/scenario_discovery.go index 94bad19..1d44723 100644 --- a/internal/testing/scenario_discovery.go +++ b/internal/testing/scenario_discovery.go @@ -40,6 +40,10 @@ var errorScenarioIndicators = []string{ "malformed-", } +// scenarioE2EFile is the Kind fixture filename. It stays in this package so +// discovery does not import the e2e runner. +const scenarioE2EFile = "e2e.yaml" + // DiscoverScenarios discovers test scenarios from the directory structure. func DiscoverScenarios(scenariosDir string) ([]TestScenario, error) { var scenarios []TestScenario @@ -83,7 +87,7 @@ func DiscoverScenarios(scenariosDir string) ([]TestScenario, error) { base.PlatformFile = "deployah.platform.yaml" } - e2ePath := filepath.Join(path, E2EFixtureFile) + e2ePath := filepath.Join(path, scenarioE2EFile) if e2eInfo, statErr := os.Stat(e2ePath); statErr == nil && e2eInfo.Mode().IsRegular() { abs, absErr := filepath.Abs(e2ePath) if absErr != nil { diff --git a/internal/testing/types.go b/internal/testing/types.go index d0188fc..062b505 100644 --- a/internal/testing/types.go +++ b/internal/testing/types.go @@ -48,6 +48,7 @@ func getTestScenariosDir() string { // Try different possible locations relative to the test file possiblePaths := []string{ "../../scenarios", + "../../../scenarios", "../scenarios", "./scenarios", "scenarios", diff --git a/scenarios/basic-web-service/e2e.yaml b/scenarios/basic-web-service/e2e.yaml index 1178352..54acb63 100644 --- a/scenarios/basic-web-service/e2e.yaml +++ b/scenarios/basic-web-service/e2e.yaml @@ -1,46 +1,65 @@ -# $schema: ../../internal/testing/e2e.schema.json +# $schema: ../../internal/testing/e2e/e2e.schema.json env: dev -resources: - - match: - apiVersion: apps/v1 - kind: Deployment - metadata: - name: basic-web-service-dev - labels: - deployah.dev/project: basic-web-service - deployah.dev/environment: dev - deployah.dev/component: web - spec: - replicas: 1 - template: +steps: + - plan: + detailedExitCode: true + output: + exitCode: 2 + expr: + - 'plan.helmAction == "install"' + - 'plan.summary.create > 0' + stopOnFailure: true + - deploy: {} + stopOnFailure: true + - assert: + - select: + apiVersion: apps/v1 + kind: Deployment + name: basic-web-service-dev + labels: + deployah.dev/project: basic-web-service + deployah.dev/environment: dev + deployah.dev/component: web + expect: spec: - containers: - - name: web - image: docker.io/library/nginx:latest - ports: - - name: http - containerPort: 80 - status: - readyReplicas: 1 - - match: - apiVersion: v1 - kind: Service - metadata: - name: basic-web-service-dev - spec: - ports: - - name: http - port: 80 - targetPort: http - selector: - app.kubernetes.io/instance: basic-web-service-dev - app.kubernetes.io/name: web - - match: - apiVersion: v1 - kind: Pod - metadata: - labels: - deployah.dev/project: basic-web-service - deployah.dev/environment: dev - status: - phase: Running + replicas: 1 + template: + spec: + containers: + - name: web + image: docker.io/library/nginx:latest + ports: + - name: http + containerPort: 80 + status: + readyReplicas: 1 + - select: + apiVersion: v1 + kind: Service + name: basic-web-service-dev + expect: + spec: + ports: + - name: http + port: 80 + targetPort: http + selector: + app.kubernetes.io/instance: basic-web-service-dev + app.kubernetes.io/name: web + - select: + apiVersion: v1 + kind: Pod + labels: + deployah.dev/project: basic-web-service + deployah.dev/environment: dev + expect: + status: + phase: Running + - plan: + detailedExitCode: true + output: + exitCode: 0 + expr: + - 'plan.changes.size() == 0' + - 'plan.summary.total == 0' + stopOnFailure: true diff --git a/scenarios/crd-create-idempotent/e2e.yaml b/scenarios/crd-create-idempotent/e2e.yaml index bf092a1..ef96d4c 100644 --- a/scenarios/crd-create-idempotent/e2e.yaml +++ b/scenarios/crd-create-idempotent/e2e.yaml @@ -1,33 +1,38 @@ -# $schema: ../../internal/testing/e2e.schema.json +# $schema: ../../internal/testing/e2e/e2e.schema.json env: dev parallel: false steps: - deploy: {} - resources: - - match: + stopOnFailure: true + - assert: + - select: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition - metadata: - name: idempotentwidgets.example.com + name: idempotentwidgets.example.com + expect: status: conditions: - type: Established status: "True" - - match: + - select: apiVersion: apps/v1 kind: Deployment - metadata: - name: crd-create-idempotent-dev-web + name: crd-create-idempotent-dev-web + expect: status: readyReplicas: 1 - - deploy: {} - stderrContains: Deployed - resources: - - match: + - deploy: + output: + stderr: + contains: + - Deployed + stopOnFailure: true + - assert: + - select: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition - metadata: - name: idempotentwidgets.example.com + name: idempotentwidgets.example.com + expect: status: conditions: - type: Established diff --git a/scenarios/crd-skip-then-upgrade/e2e.yaml b/scenarios/crd-skip-then-upgrade/e2e.yaml index e80127c..a9b5ed1 100644 --- a/scenarios/crd-skip-then-upgrade/e2e.yaml +++ b/scenarios/crd-skip-then-upgrade/e2e.yaml @@ -1,35 +1,37 @@ -# $schema: ../../internal/testing/e2e.schema.json +# $schema: ../../internal/testing/e2e/e2e.schema.json env: dev parallel: false steps: - deploy: - args: [--skip-crds] - resources: - - match: + args: ["--skip-crds"] + stopOnFailure: true + - assert: + - select: apiVersion: apps/v1 kind: Deployment - metadata: - name: crd-skip-then-upgrade-dev-web + name: crd-skip-then-upgrade-dev-web + expect: status: readyReplicas: 1 - - minCount: 0 - match: + - select: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition - metadata: - name: skipwidgets.example.com + name: skipwidgets.example.com + expr: + - 'objects.size() == 0' - deploy: {} - resources: - - match: + stopOnFailure: true + - assert: + - select: apiVersion: apps/v1 kind: Deployment - metadata: - name: crd-skip-then-upgrade-dev-web + name: crd-skip-then-upgrade-dev-web + expect: status: readyReplicas: 1 - - minCount: 0 - match: + - select: apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition - metadata: - name: skipwidgets.example.com + name: skipwidgets.example.com + expr: + - 'objects.size() == 0' diff --git a/scenarios/extras-env-and-crds/e2e.yaml b/scenarios/extras-env-and-crds/e2e.yaml index 8bd1b7b..8f8f91a 100644 --- a/scenarios/extras-env-and-crds/e2e.yaml +++ b/scenarios/extras-env-and-crds/e2e.yaml @@ -1,25 +1,29 @@ -# $schema: ../../internal/testing/e2e.schema.json +# $schema: ../../internal/testing/e2e/e2e.schema.json env: prod parallel: false -resources: - - match: - apiVersion: apiextensions.k8s.io/v1 - kind: CustomResourceDefinition - metadata: - name: clusterwidgets.example.com - status: - conditions: - - type: Established - status: "True" - - match: - apiVersion: v1 - kind: ConfigMap - metadata: - name: shared-widget - - match: - apiVersion: apps/v1 - kind: Deployment - metadata: - name: extras-env-and-crds-prod-web - status: - readyReplicas: 1 +steps: + - deploy: {} + stopOnFailure: true + - assert: + - select: + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + name: clusterwidgets.example.com + expect: + status: + conditions: + - type: Established + status: "True" + - select: + apiVersion: v1 + kind: ConfigMap + name: shared-widget + expr: + - 'objects.size() == 1' + - select: + apiVersion: apps/v1 + kind: Deployment + name: extras-env-and-crds-prod-web + expect: + status: + readyReplicas: 1 diff --git a/scenarios/stateful-basic/e2e.yaml b/scenarios/stateful-basic/e2e.yaml index f7d55ea..b682ab2 100644 --- a/scenarios/stateful-basic/e2e.yaml +++ b/scenarios/stateful-basic/e2e.yaml @@ -1,59 +1,62 @@ -# $schema: ../../internal/testing/e2e.schema.json +# $schema: ../../internal/testing/e2e/e2e.schema.json env: dev -resources: - - match: - apiVersion: apps/v1 - kind: StatefulSet - metadata: - name: stateful-basic-dev-db - labels: - deployah.dev/project: stateful-basic - deployah.dev/environment: dev - deployah.dev/component: db - spec: - replicas: 1 - template: +steps: + - deploy: {} + stopOnFailure: true + - assert: + - select: + apiVersion: apps/v1 + kind: StatefulSet + name: stateful-basic-dev-db + labels: + deployah.dev/project: stateful-basic + deployah.dev/environment: dev + deployah.dev/component: db + expect: spec: - containers: - - name: db - image: docker.io/library/redis:7-alpine - ports: - - name: http - containerPort: 6379 - status: - readyReplicas: 1 - - match: - apiVersion: v1 - kind: Service - metadata: - name: stateful-basic-dev-db - spec: - ports: - - name: http - port: 80 - targetPort: http - selector: - app.kubernetes.io/instance: stateful-basic-dev - app.kubernetes.io/name: db - - match: - apiVersion: v1 - kind: Service - metadata: - name: stateful-basic-dev-db-headless - spec: - clusterIP: None - selector: - app.kubernetes.io/instance: stateful-basic-dev - app.kubernetes.io/name: db - - match: - apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - labels: - deployah.dev/component: db - spec: - resources: - requests: - storage: 20Gi - status: - phase: Bound + replicas: 1 + template: + spec: + containers: + - name: db + image: docker.io/library/redis:7-alpine + ports: + - name: http + containerPort: 6379 + status: + readyReplicas: 1 + - select: + apiVersion: v1 + kind: Service + name: stateful-basic-dev-db + expect: + spec: + ports: + - name: http + port: 80 + targetPort: http + selector: + app.kubernetes.io/instance: stateful-basic-dev + app.kubernetes.io/name: db + - select: + apiVersion: v1 + kind: Service + name: stateful-basic-dev-db-headless + expect: + spec: + clusterIP: None + selector: + app.kubernetes.io/instance: stateful-basic-dev + app.kubernetes.io/name: db + - select: + apiVersion: v1 + kind: PersistentVolumeClaim + labels: + deployah.dev/component: db + expect: + spec: + resources: + requests: + storage: 20Gi + status: + phase: Bound diff --git a/scenarios/stateful-scale/e2e.yaml b/scenarios/stateful-scale/e2e.yaml index 9a82d38..3f1fa3f 100644 --- a/scenarios/stateful-scale/e2e.yaml +++ b/scenarios/stateful-scale/e2e.yaml @@ -1,34 +1,38 @@ -# $schema: ../../internal/testing/e2e.schema.json +# $schema: ../../internal/testing/e2e/e2e.schema.json env: dev steps: - deploy: {} - resources: - - match: + stopOnFailure: true + - assert: + - select: apiVersion: apps/v1 kind: StatefulSet - metadata: - name: stateful-scale-dev-cache + name: stateful-scale-dev-cache + expect: status: readyReplicas: 1 - deploy: spec: deployah-replicas-2.yaml timeout: 5m - resources: - - match: + stopOnFailure: true + - assert: + - select: apiVersion: apps/v1 kind: StatefulSet - metadata: - name: stateful-scale-dev-cache + name: stateful-scale-dev-cache + expect: spec: replicas: 2 status: readyReplicas: 2 - - minCount: 2 - match: + - select: apiVersion: v1 kind: PersistentVolumeClaim - metadata: - labels: - deployah.dev/component: cache + labels: + deployah.dev/component: cache + expect: status: phase: Bound + expr: + - 'objects.size() >= 2' + timeout: 5m diff --git a/scenarios/task-migrate-smoke/e2e.yaml b/scenarios/task-migrate-smoke/e2e.yaml index 06e2288..8d11b41 100644 --- a/scenarios/task-migrate-smoke/e2e.yaml +++ b/scenarios/task-migrate-smoke/e2e.yaml @@ -1,17 +1,18 @@ -# $schema: ../../internal/testing/e2e.schema.json +# $schema: ../../internal/testing/e2e/e2e.schema.json env: dev steps: - deploy: {} - resources: - - match: + stopOnFailure: true + - assert: + - select: apiVersion: apps/v1 kind: Deployment - metadata: - name: taskdemo-dev-api - labels: - deployah.dev/project: taskdemo - deployah.dev/environment: dev - deployah.dev/component: api + name: taskdemo-dev-api + labels: + deployah.dev/project: taskdemo + deployah.dev/environment: dev + deployah.dev/component: api + expect: spec: replicas: 1 template: @@ -21,11 +22,11 @@ steps: image: docker.io/library/nginx:latest status: readyReplicas: 1 - - match: + - select: apiVersion: v1 kind: Service - metadata: - name: taskdemo-dev-api + name: taskdemo-dev-api + expect: spec: ports: - name: http @@ -35,27 +36,31 @@ steps: task: backfill - run: task: backfill - resources: - - minCount: 2 - match: + stopOnFailure: true + - assert: + - select: apiVersion: batch/v1 kind: Job - metadata: - labels: - deployah.dev/project: taskdemo - deployah.dev/component: backfill + labels: + deployah.dev/project: taskdemo + deployah.dev/component: backfill + expect: status: succeeded: 1 + expr: + - 'objects.size() >= 2' - logs: component: backfill - contains: backfill-ok + contains: + - backfill-ok - delete: {} - resources: - - minCount: 0 - match: + stopOnFailure: true + - assert: + - select: apiVersion: batch/v1 kind: Job - metadata: - labels: - deployah.dev/project: taskdemo - deployah.dev/environment: dev + labels: + deployah.dev/project: taskdemo + deployah.dev/environment: dev + expr: + - 'objects.size() == 0' diff --git a/scenarios/task-schedule/e2e.yaml b/scenarios/task-schedule/e2e.yaml index 73b6cf9..322f6c5 100644 --- a/scenarios/task-schedule/e2e.yaml +++ b/scenarios/task-schedule/e2e.yaml @@ -1,16 +1,18 @@ -# $schema: ../../internal/testing/e2e.schema.json +# $schema: ../../internal/testing/e2e/e2e.schema.json env: dev steps: - deploy: {} - resources: - - match: + stopOnFailure: true + - assert: + - select: apiVersion: batch/v1 kind: CronJob + name: taskcron-dev-cleanup + labels: + deployah.dev/project: taskcron + deployah.dev/component: cleanup + expect: metadata: - name: taskcron-dev-cleanup - labels: - deployah.dev/project: taskcron - deployah.dev/component: cleanup annotations: helm.sh/hook: null spec: @@ -30,17 +32,28 @@ steps: containers: - name: cleanup command: ["echo", "cleanup-ok"] + - plan: + detailedExitCode: true + output: + exitCode: 2 + expr: + - 'plan.helmAction == "none"' + - 'plan.changes.size() == 0' + - 'plan.summary.total == 0' + - 'plan.tasks.exists(t, t.name == "cleanup" && t.phase == "schedule" && t.action == "create" && !t.willRun)' + stopOnFailure: true - run: task: cleanup - resources: - - match: + stopOnFailure: true + - assert: + - select: apiVersion: batch/v1 kind: Job - metadata: - labels: - deployah.dev/project: taskcron - deployah.dev/component: cleanup - deployah.dev/managed-by: deployah + labels: + deployah.dev/project: taskcron + deployah.dev/component: cleanup + deployah.dev/managed-by: deployah + expect: spec: activeDeadlineSeconds: null status: diff --git a/scenarios/worker-basic/e2e.yaml b/scenarios/worker-basic/e2e.yaml index 348b3d0..e078b94 100644 --- a/scenarios/worker-basic/e2e.yaml +++ b/scenarios/worker-basic/e2e.yaml @@ -1,32 +1,35 @@ -# $schema: ../../internal/testing/e2e.schema.json +# $schema: ../../internal/testing/e2e/e2e.schema.json env: dev -resources: - - match: - apiVersion: apps/v1 - kind: Deployment - metadata: - name: worker-basic-dev - labels: - deployah.dev/project: worker-basic - deployah.dev/environment: dev - deployah.dev/component: worker - spec: - replicas: 1 - template: +steps: + - deploy: {} + stopOnFailure: true + - assert: + - select: + apiVersion: apps/v1 + kind: Deployment + name: worker-basic-dev + labels: + deployah.dev/project: worker-basic + deployah.dev/environment: dev + deployah.dev/component: worker + expect: spec: - containers: - - name: worker - image: docker.io/library/busybox:1.36 - command: ["sleep"] - args: ["infinity"] - status: - readyReplicas: 1 - - match: - apiVersion: v1 - kind: Pod - metadata: - labels: - deployah.dev/project: worker-basic - deployah.dev/environment: dev - status: - phase: Running + replicas: 1 + template: + spec: + containers: + - name: worker + image: docker.io/library/busybox:1.36 + command: ["sleep"] + args: ["infinity"] + status: + readyReplicas: 1 + - select: + apiVersion: v1 + kind: Pod + labels: + deployah.dev/project: worker-basic + deployah.dev/environment: dev + expect: + status: + phase: Running