test(e2e): E2E test improvements

.github/workflows/e2e-nightly.yaml

@@ -0,0 +1,91 @@
+name: E2E (nightly)
+
+on:
+  # schedule:
+  # Nightly at 03:00 UTC.
+  # - cron: "0 3 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    # Test provisions kind cluster, cert-manager, Argo CD and Kargo, so the full run takes a while.
+    timeout-minutes: 180
+    steps:
+    - name: Harden the runner (Audit all outbound calls)
+      uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1
+      with:
+        egress-policy: audit
+
+    - name: Checkout code
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+    - name: Install Go
+      uses: actions/setup-go@v5
+      with:
+        # Use the e2e module's toolchain version.
+        go-version-file: hack/test/e2e/go.mod
+        cache-dependency-path: |
+          go.sum
+          hack/test/e2e/go.sum
+
+    # The test shells out to helm, kind, kargo and argocd; install them all into
+    # hack/bin, which is added to PATH for the subsequent steps.
+    - name: Add hack/bin to PATH
+      run: echo "${GITHUB_WORKSPACE}/hack/bin" >> "${GITHUB_PATH}"
+
+    - name: Install helm
+      run: make install-helm
+
+    - name: Build kind
+      run: go build -o hack/bin/kind sigs.k8s.io/kind
+
+    - name: Build kargo CLI
+      run: go build -o hack/bin/kargo ./cmd/cli
+
+    - name: Install argocd CLI
+      run: |
+        curl -fsSL -o hack/bin/argocd \
+          https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64
+        chmod +x hack/bin/argocd
+
+    - name: Verify tools
+      run: |
+        go version
+        helm version
+        kind version
+        kargo version --client || true
+        argocd version --client
+
+    # nightly_config.yaml ships with an empty `context` section. Populate it from
+    # repository secrets so the git- and http-dependent suites can run. Each key
+    # is only written when its secret is set; suites whose context is missing
+    # skip or fail on their own.
+    - name: Configure e2e context
+      env:
+        E2E_KARGO_DEMO_GITOPS_REPO: ${{ secrets.E2E_KARGO_DEMO_GITOPS_REPO }}
+        E2E_GIT_PAT: ${{ secrets.E2E_GIT_PAT }}
+        ## TODO: We need to set up the endpoint to ebable that
+        # E2E_HTTP_ENDPOINT: ${{ secrets.E2E_HTTP_ENDPOINT }}
+      run: |
+        cfg=hack/test/e2e/envs/nightly_config.yaml
+        # Drop the existing (last) context section and re-add it from secrets.
+        sed -i '/^context:/,$d' "${cfg}"
+        {
+          echo "context:"
+          [ -n "${E2E_KARGO_DEMO_GITOPS_REPO}" ] && echo "  kargo_demo_gitops_repo: ${E2E_KARGO_DEMO_GITOPS_REPO}"
+          [ -n "${E2E_GIT_PAT}" ] && echo "  git_pat: ${E2E_GIT_PAT}"
+          [ -n "${E2E_HTTP_ENDPOINT}" ] && echo "  http_endpoint: ${E2E_HTTP_ENDPOINT}"
+        } >> "${cfg}"
+
+    - name: Build kargo image
+      run: |
+        make hack-build
+
+    - name: Run e2e tests
+      working-directory: hack/test/e2e
+      run: |
+        go test -tags=e2e,shared ./suites/shared -timeout 120m -args -env-file=nightly_config.yaml

hack/test/e2e/README.md

@@ -12,6 +12,12 @@ To run tests using your home-directory kargo config, aquired with `kargo login`
 go test <package_name> -args -env-file=home_config.yaml
 ```
 
+## Prerequisites
+
+- `go` - tests are run with `go test`
+- `kubectl`, `kargo` and `argocd` CLI tools installed - some tests and helper functions require those CLIs
+- `kind` and `helm` CLI tools installed to run in local `kind` cluster
+
 ## Test framework used
 
 This project uses the https://github.com/kubernetes-sigs/e2e-framework to set up environment and test runners.
@@ -76,9 +82,30 @@ Please see `kargo_promotion_fail` as an example for running promotions in a proj
 
 `envs` folder contains YAML files with configurations accessible to the tests, each `go test` run can be run with a particular environment. Environment configurations mainly used by the environment configuration functions in the `envfuncs` package.
 
+## Provisioning a local cluster
+
+By default the tests run against an already-running Kargo instance (see `home_config.yaml`). The `envfuncs` package can instead stand up a self-contained environment on a local [kind](https://kind.sigs.k8s.io/) cluster using the `sigs.k8s.io/e2e-framework` kind and Helm helpers:
+
+- `CreateKindCluster` / `DestroyKindCluster` -- create and delete the cluster
+- `InstallCertManager` -- installs cert-manager (a prerequisite for Kargo)
+- `InstallArgoCD` -- installs the Argo CD Helm chart
+- `InstallKargo` -- installs the Kargo Helm chart (optionally loading a locally built image into the cluster first)
+
+`ClusterSetupFuncs()` and `ClusterTeardownFuncs()` return these as ordered `env.Func` slices. They are driven entirely by the env file: they run only when it contains a `cluster` section and otherwise no-op, so they are safe to add to a `funcsloader` unconditionally. See `envs/kind_config.yaml` for all configurable fields (cluster name/image, chart repos, versions, values files, `--set` overrides, and an optional Kargo image to load).
+
+`prerequisites`: the `kind` and `helm` binaries must be available on `PATH`.
+
+These env functions will be called for each test, but require env configuration with `cluster`, `cert_manager`, `argocd` and `kargo` keys to instruct the functions to actually set up envorinments.
+**See envs/kind_config.yaml file for configuration reference**
+
 ## Dependency modules
 
 This folder contains the main e2e test module `github.com/akuity/kargo/hack/test/e2e` and a few helper packages, each in its own module: `envs`, `envfuncs`, `funcsloader`.
 
 The purpose of that is to be able to override `envs` and `funcsloader` in dependent `e2e` test modules to provide different environment configuration and setup/teardown functions to test in more environments than this package provides.
 
+## TODO
+
+- Allow disabling teardown on errors.
+- Allow separate context values file from init congigurations
+- Surface errors better

hack/test/e2e/envs/home_config.yaml

@@ -8,4 +8,7 @@ argocd_cli:
   config_file: ~/.config/argocd/config
 
 context:
-  kargo_demo_gitops_repo: "https://github.com/hairyhum/kargo-demo-gitops.git"
+  # kargo_demo_gitops_repo: <specify a repo here>
+  # git_pat: <specify a PAT here>
+  # http_endpoint: <URL of an HTTP endpoint that returns 2xx to a POST>
+

hack/test/e2e/envs/kind_config.yaml

@@ -0,0 +1,85 @@
+## Example environment for running e2e tests against a self-managed kind cluster
+## with Kargo and Argo CD installed via Helm.
+##
+## The `cluster` section opts the run into managing its own cluster: the
+## envfuncs.ClusterSetupFuncs (CreateKindCluster -> InstallCertManager ->
+## InstallArgoCD -> InstallKargo) run only when it is present, and
+## envfuncs.ClusterTeardownFuncs (DestroyKindCluster) tears it down afterwards.
+## Wire these into a funcsloader (see framework/funcsloader) to use them.
+##
+## Every field below is optional; the values shown are the built-in defaults.
+description: "Provision a kind cluster and install Kargo + Argo CD via Helm"
+
+cluster:
+  name: kargo-e2e
+  ## Optional kind node image, e.g. kindest/node:v1.31.2
+  image: ""
+  ## Optional path to a kind cluster config file (supports a leading ~).
+  config_file: ""
+
+## cert-manager is a prerequisite for Kargo's self-signed certificates.
+cert_manager:
+  release_name: cert-manager
+  namespace: cert-manager
+  chart: jetstack/cert-manager
+  chart_repo_name: jetstack
+  chart_repo_url: https://charts.jetstack.io
+  version: ""
+  timeout: 10m
+  ## Defaults to ["crds.enabled=true"] when omitted.
+  set: []
+  values_files: []
+
+## argo-rollouts are required for AnalysisTemplate in kargo
+argo-rollouts:
+  namespace: argo-rollouts
+  timeout: 10m
+
+argocd:
+  release_name: argocd
+  namespace: argocd
+  chart: argo/argo-cd
+  chart_repo_name: argo
+  chart_repo_url: https://argoproj.github.io/argo-helm
+  version: ""
+  timeout: 10m
+  set: []
+  values_files: [
+    ../../values.argocd.test.yaml
+  ]
+
+kargo:
+  release_name: kargo
+  namespace: kargo
+  ## Defaults to the published OCI chart. Point at a local path (e.g.
+  ## ../../../charts/kargo) to test this checkout's chart.
+  chart: oci://ghcr.io/akuity/kargo-charts/kargo
+  ## For a classic (non-OCI) repo, set chart_repo_name + chart_repo_url instead.
+  chart_repo_name: ""
+  chart_repo_url: ""
+  version: ""
+  timeout: 10m
+  ## Optional locally built image to load into the kind cluster before install,
+  ## e.g. kargo:dev (pair with `set` entries pointing the chart at it).
+  image: ""
+  set: []
+  values_files: [
+    ../../values.test.yaml
+  ]
+
+kargo_cli:
+  ## Indicate that we need to run `kargo login` to login into a kargo instance in `kind` cluster
+  kargo_login:
+    ## Use tmp directory to store kargo config
+    use_tmp_config_home: true
+
+argocd_cli:
+  ## Using system-local argocd config created by `argocd login`
+  config_file: ~/.config/argocd/config
+  ## Indicate that we need to run `argocd login` to login to an argocd instance in `kind` cluster
+  argocd_login: true
+
+context:
+  # kargo_demo_gitops_repo: <specify a repo here>
+  # git_pat: <specify a PAT here>
+  # http_endpoint: <URL of an HTTP endpoint that returns 2xx to a POST>

hack/test/e2e/envs/nightly_config.yaml

@@ -0,0 +1,51 @@
+## Environment template used to configure nightly e2e runs.
+
+## It is based on kind_config.yaml and is going to use `kind` and `helm` to set up kargo and argocd instances.
+description: "Nightly test wth kind cluster"
+
+cluster:
+  name: kargo-e2e
+
+## cert-manager is a prerequisite for Kargo's self-signed certificates.
+cert_manager:
+  namespace: cert-manager
+  timeout: 10m
+
+argocd:
+  namespace: argocd
+  timeout: 10m
+  values_files: [
+    ../../values.argocd.test.yaml
+  ]
+
+argo-rollouts:
+  namespace: argo-rollouts
+  timeout: 10m
+
+kargo:
+  namespace: kargo
+  ## Using a chart directly from the repo
+  chart: ../../../../../charts/kargo
+  ## Using an image built with make hack-build
+  image: docker.io/library/kargo:dev
+  timeout: 10m
+  values_files: [
+    ../../values.test.yaml
+  ]
+
+kargo_cli:
+  ## Indicate that we need to run `kargo login` to login into a kargo instance in `kind` cluster
+  kargo_login:
+    ## Use tmp directory to store kargo config
+    use_tmp_config_home: true
+
+argocd_cli:
+  ## Using system-local argocd config created by `argocd login`
+  config_file: ~/.config/argocd/config
+  ## Indicate that we need to run `argocd login` to login to an argocd instance in `kind` cluster
+  argocd_login: true
+
+context:
+  # kargo_demo_gitops_repo: <specify a repo here>
+  # git_pat: <specify a PAT here>
+  # http_endpoint: <URL of an HTTP endpoint that returns 2xx to a POST>

hack/test/e2e/framework/envfuncs/argocd_cli.go

@@ -3,16 +3,26 @@ package envfuncs
 import (
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"strings"
 
 	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/utils"
 )
 
 const ArgoCDConfigFile ContextKey = "argocd_config_file"
+const ArgocdHostKey ContextKey = "argocd_host"
+const ArgocdPasswordKey ContextKey = "argocd_password"
+const ArgocdUsernameKey ContextKey = "argocd_username"
 
 func LoadArgocdConfig(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
-	// TODO: other ways to discover/setup argocd config
+	if argocdConfigFileVal := ctx.Value(ArgoCDConfigFile); argocdConfigFileVal != nil {
+		// Config file already set, noop
+		return ctx, nil
+	}
+
+	// TODO: other ways to discover/setup argocd config (such as using tempdir)
 	if argocdEnvConfig, err := GetEnv(ctx, []string{"argocd_cli", "config_file"}); err == nil {
 		fileName := argocdEnvConfig.(string)
 		if strings.HasPrefix(fileName, "~") {
@@ -25,3 +35,52 @@ func LoadArgocdConfig(ctx context.Context, cfg *envconf.Config) (context.Context
 	// Argocd config is optional. Do not fail here
 	return ctx, nil
 }
+
+
+func ArgocdLogin(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
+	_, err := GetEnv(ctx, []string{"argocd_cli", "argocd_login"})
+	if err != nil {
+		fmt.Printf("Argocd login disabled, skipping \n")
+		return ctx, nil
+	}
+
+	argocdHost, err := GetValueOrEnv(ctx, ArgocdHostKey, []string{"argocd_cli", "argocd_login", "argocd_host"})
+	if err != nil {
+		fmt.Printf("%v is not set, skipping argocd login \n", ArgocdHostKey)
+		return ctx, nil
+	}
+
+	argocdPassword, err := GetValueOrEnv(ctx, ArgocdPasswordKey, []string{"argocd_cli", "argocd_login", "argocd_password"})
+	if err != nil {
+		fmt.Printf("%v is not set, skipping argocd login \n", ArgocdPasswordKey)
+		return ctx, nil
+	}
+
+	argocdUsername, err := GetValueOrEnv(ctx, ArgocdUsernameKey, []string{"argocd_cli", "argocd_login", "argocd_username"})
+	if err != nil {
+		fmt.Printf("%v is not set, skipping argocd login \n", ArgocdUsernameKey)
+		return ctx, nil
+	}
+
+	argocdConfigFile, ok := ctx.Value(ArgoCDConfigFile).(string)
+	if !ok {
+		fmt.Printf("%v is not set, skipping argocd login \n", ArgoCDConfigFile)
+		return ctx, nil
+	}
+
+	fmt.Printf("Argocd login \n")
+
+	cmd := fmt.Sprintf("argocd login --insecure %s --username %s --password %s --config %s", 
+		argocdHost, argocdUsername, argocdPassword, argocdConfigFile)
+	p := utils.RunCommandContext(ctx, cmd)
+	if p.Err() != nil {
+		outBytes, outErr := io.ReadAll(p.Out())
+		if outErr != nil {
+			return ctx, fmt.Errorf("argocd login failed: %w %w", p.Err(), outErr)
+		}
+		return ctx, fmt.Errorf("argocd login failed: %w : %s", p.Err(), outBytes)
+	}
+
+	return ctx, nil
+
+}