diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml deleted file mode 100644 index f53e7cdc..00000000 --- a/.github/workflows/tags.yaml +++ /dev/null @@ -1,32 +0,0 @@ -on: - push: - tags: - - "v*" -name: Create release -jobs: - release: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Generate Release Notes - run: | - release_notes=$(gh api repos/{owner}/{repo}/releases/generate-notes -F tag_name=${{ github.ref }} --jq .body) - echo 'RELEASE_NOTES<> $GITHUB_ENV - echo "${release_notes}" >> $GITHUB_ENV - echo 'EOF' >> $GITHUB_ENV - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OWNER: ${{ github.repository_owner }} - REPO: ${{ github.event.repository.name }} - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} - body: ${{ env.RELEASE_NOTES }} - draft: false - prerelease: false diff --git a/.gitignore b/.gitignore index 309d821f..37a774dd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,11 @@ error.log .task .state capt/output/ +capt/bin/ .vscode/ sushy.cert sushy.key -htpasswd \ No newline at end of file +htpasswd +.validation-success +capt/e2e/artifacts/* +out diff --git a/capt/.env b/capt/.env new file mode 100644 index 00000000..943e5332 --- /dev/null +++ b/capt/.env @@ -0,0 +1 @@ +TASK_X_ENV_PRECEDENCE=1 diff --git a/capt/README.md b/capt/README.md index 6406af9d..65dce6b5 100644 --- a/capt/README.md +++ b/capt/README.md @@ -16,15 +16,22 @@ This playground has only been tested on Ubuntu 22.04 LTS. If you are using a vir ### Binaries +The following must be installed system-wide: + - [Libvirtd](https://wiki.debian.org/KVM) >= libvirtd (libvirt) 8.0.0 - [Docker](https://docs.docker.com/engine/install/) >= 24.0.7 -- [Helm](https://helm.sh/docs/intro/install/) >= v3.13.1 -- [KinD](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) >= v0.20.0 -- [clusterctl](https://cluster-api.sigs.k8s.io/user/quick-start#install-clusterctl) >= v1.6.0 -- [kubectl](https://www.downloadkubernetes.com/) >= v1.28.2 - [virt-install](https://virt-manager.org/) >= 4.0.0 -- [yq](https://github.com/mikefarah/yq/#install) >= v4.44.2 - [task](https://taskfile.dev/installation/) >= 3.37.2 +- `curl`, `tar`, `ssh-keygen` (from `openssh-client`) + +The following are downloaded automatically into `./bin/` by `task install-binaries` (invoked as part of `task create-playground`); pinned versions live near the top of [Taskfile.yaml](./Taskfile.yaml): + +- `cue` — workload-manifest renderer +- `helm` +- `kind` +- `kubectl` +- `clusterctl` +- `yq` ### Packages @@ -57,6 +64,29 @@ Delete the CAPT playground: task delete-playground ``` +### External Tinkerbell mode + +When `externalTinkerbell: true` is set in [`config.yaml`](./config.yaml), the +playground spins up a **second** KinD cluster (named `-tinkerbell`) +on the same `kind` docker network and deploys the Tinkerbell stack there +instead of into the management cluster. Hardware, Machine (BMC), and Workflow +CRs all live in this second cluster. CAPT, running in the management cluster, +talks to it via the `external-tinkerbell-kubeconfig` Secret in the +`capt-system` namespace (created by +[scripts/create_external_kubeconfig_secret.sh](./scripts/create_external_kubeconfig_secret.sh) +and labeled for `clusterctl move`). + +Two kubeconfigs are produced under `output/`: + +- `kind.kubeconfig` — the management cluster (CAPI/CAPT components live here) +- `tinkerbell-kind.kubeconfig` — the Tinkerbell cluster (Hardware/BMC/Workflows live here) + +Caveat: after `task pivot`, the workload cluster receives the secret via +`clusterctl move`, but it must still be able to reach the Tinkerbell KinD +container's API server at the IP embedded in the kubeconfig — that IP is only +reachable from containers on the host's `kind` docker network, so cross-host +pivots are not supported in this mode. + ## Next Steps With the playground up and running and a workload cluster created, you can run through a few CAPI lifecycle operations. @@ -81,6 +111,77 @@ To be written. To be written. +## Running E2E Tests + +The `e2e/run.sh` script orchestrates a matrix of provisioning combos +(topology × bootmode × mirror) defined in [`e2e/cue/matrix.cue`](e2e/cue/matrix.cue). +Each combo renders its own `config.yaml` from CUE, runs +`task create-playground`, executes Ginkgo specs against the resulting +clusters, then tears the playground down. + +List available combos: + +```bash +./e2e/run.sh --list +``` + +Run a single combo: + +```bash +./e2e/run.sh single-nomirror-netboot +``` + +Run the whole matrix (combos that use the registry mirror require +`--mirror-host`): + +```bash +./e2e/run.sh --mirror-host reg.example.com all +``` + +Useful flags: + +- `--no-teardown` — keep the playground running after tests so you can + poke at it. **Resources persist; clean up with `./e2e/run.sh --cleanup`.** +- `--dry-run` — render configs and print what would run, but skip + `task` and `ginkgo` invocations. +- `--labels FILTER` — Ginkgo `--label-filter` (default `provisioning`). +- `--artifacts DIR` — where per-combo logs and JUnit reports are written + (default `e2e/artifacts/`). + +The Ginkgo suite under [`e2e/test/`](e2e/test/) can also be run directly +against an already-provisioned playground by exporting the kubeconfig +paths: + +```bash +E2E_MGMT_KUBECONFIG="$(yq .kind.kubeconfig .state)" \ +E2E_WORKLOAD_KUBECONFIG="$(yq .outputDir .state)/$(yq .clusterName .state).kubeconfig" \ +E2E_NAMESPACE="$(yq .namespace .state)" \ +ginkgo -v --label-filter=provisioning ./e2e/test/... +``` + +## How CUE renders the playground + +`config.yaml` is the only file most users touch. Everything else +(`.state`, generated CAPI manifests, kind config, hardware/BMC YAML, +`hosts.toml` mirror drop-ins) is derived by CUE packages under +[`cue/`](cue/): + +- [`cue/state`](cue/state/state.cue) — reads `config.yaml`, computes + derived names/IPs/MACs, writes `.state` (the source of truth for + every downstream renderer). +- [`cue/values`](cue/values/values.cue) — the `#Config` schema for + `.state`. Inner structs are closed so typos fail `cue vet`. +- [`cue/capi`](cue/capi/render.cue), [`cue/infra`](cue/infra/render.cue), + [`cue/clusterctl`](cue/clusterctl/clusterctl.cue), [`cue/kind`](cue/kind/kind.cue) + — render Kubernetes resources from `.state`. +- [`cue/mirror`](cue/mirror/schema.cue) — optional pull-through OCI + registry mirror. Disabled by default; the wiring sentinel in + [`cue/wiring`](cue/wiring/wiring.cue) ensures the feature can't be + half-removed by accident. + +`task generate-state` runs `cue vet` before `cue export`, so schema +errors surface with line numbers before any other task runs. + ## Known Issues ### DNS issue diff --git a/capt/Taskfile.yaml b/capt/Taskfile.yaml index 67a04ef5..5fddab5a 100644 --- a/capt/Taskfile.yaml +++ b/capt/Taskfile.yaml @@ -1,22 +1,52 @@ version: "3" +# Apply bash-style strict mode to every `cmds:` block in this Taskfile. +# Replaces the per-task `set -euo pipefail` heredoc preamble. +# https://taskfile.dev/docs/reference/schema#set-options +set: [errexit, nounset, pipefail] + includes: create: ./tasks/Taskfile-create.yaml delete: ./tasks/Taskfile-delete.yaml vbmc: ./tasks/Taskfile-vbmc.yaml capi: ./tasks/Taskfile-capi.yaml capi-pivot: ./tasks/Taskfile-capi-pivot.yaml + helm: ./tasks/Taskfile-helm.yaml + tools: ./tasks/Taskfile-tools.yaml + +env: + # Prepend ./bin to PATH so all pinned tools (yq, helm, kind, kubectl, + # clusterctl, cue) resolve as bare command names — both in `cmds:` and + # in `vars: sh:` blocks. Requires the TASK_X_ENV_PRECEDENCE=1 experiment + # (set in .env) so the Task `env:` value wins over the OS PATH. + PATH: '{{.ROOT_DIR}}/bin:{{env "PATH"}}' vars: OUTPUT_DIR: - sh: echo $(yq eval '.outputDir' config.yaml) - CURR_DIR: - sh: pwd + sh: | + awk -F: '/^outputDir:/ {gsub(/^[ "]+|[ "]+$/, "", $2); print $2}' {{.ROOT_DIR}}/config.yaml STATE_FILE: ".state" STATE_FILE_FQ_PATH: - sh: echo {{joinPath .CURR_DIR .STATE_FILE}} + sh: echo {{joinPath .ROOT_DIR .STATE_FILE}} CONFIG_FILE_FQ_PATH: - sh: echo {{joinPath .CURR_DIR "config.yaml"}} + sh: echo {{joinPath .ROOT_DIR "config.yaml"}} + # Pinned tool versions for binaries downloaded into ./bin from each upstream + # project's release artifacts. Bump these to upgrade tooling; the + # install-binaries task is idempotent on the version string. + CUE_VERSION: v0.16.1 + HELM_VERSION: v4.1.4 + KIND_VERSION: v0.31.0 + # clusterctl version pins to the CAPI core release; must be compatible with + # the providers `clusterctl init` will install. + CLUSTERCTL_VERSION: v1.12.5 + KUBECTL_VERSION: v1.35.4 + YQ_VERSION: v4.53.2 + # Host OS/arch used to construct release download URLs. Override on the + # command line if cross-installing (e.g. `task install-binaries OS=darwin`). + OS: + sh: uname -s | tr '[:upper:]' '[:lower:]' + ARCH: + sh: case "$(uname -m)" in x86_64) echo amd64;; aarch64|arm64) echo arm64;; *) uname -m;; esac tasks: create-playground: @@ -25,7 +55,8 @@ tasks: Create the CAPT playground. Use the config.yaml file to define things like cluster size and Kubernetes version. cmds: - task: system-deps-warnings - - task: validate-binaries + - task: tools:validate-binaries + - task: tools:install-binaries - task: ensure-output-dir - task: generate-state - task: create:playground-ordered @@ -36,29 +67,10 @@ tasks: summary: | Delete the CAPT playground. cmds: - - task: validate-binaries + - task: tools:validate-binaries + - task: tools:install-binaries - task: delete:playground - validate-binaries: - silent: true - summary: | - Validate all required dependencies for the CAPT playground. - cmds: - - for: - [ - "virsh", - "docker", - "helm", - "kind", - "kubectl", - "clusterctl", - "virt-install", - "yq", - ] - cmd: command -v {{ .ITEM }} >/dev/null || echo "'{{ .ITEM }}' was not found in the \$PATH, please ensure it is installed." - # sudo apt install virtinst # for virt-install - # sudo apt install bridge-utils # for brctl - system-deps-warnings: summary: | Run CAPT playground system warnings. @@ -79,17 +91,46 @@ tasks: - echo ;[ -d {{.OUTPUT_DIR}} ] - echo ;[ -d {{.OUTPUT_DIR}}/xdg ] + generate-ssh-key: + summary: | + Generate an ed25519 SSH keypair in {{.OUTPUT_DIR}} for cloud-init + authorized_keys injection. + deps: [ensure-output-dir] + generates: + - "{{.OUTPUT_DIR}}/capt-ssh-key" + - "{{.OUTPUT_DIR}}/capt-ssh-key.pub" + cmds: + - ssh-keygen -t ed25519 -f {{.OUTPUT_DIR}}/capt-ssh-key -N "" -C capt-playground >/dev/null + status: + - test -f {{.OUTPUT_DIR}}/capt-ssh-key + - test -f {{.OUTPUT_DIR}}/capt-ssh-key.pub + generate-state: summary: | - Populate the state file. + Populate the state file from config.yaml using the cue/state package. + Reads the SSH public key generated by generate-ssh-key. + deps: [generate-ssh-key] sources: - config.yaml + - cue/state/*.cue + - cue/values/*.cue + - cue/mirror/*.cue + - "{{.OUTPUT_DIR}}/capt-ssh-key.pub" generates: - .state cmds: - - ./scripts/generate_state.sh config.yaml .state - status: - - if [[ -n "$(yq '.os.sshKey' {{.STATE_FILE_FQ_PATH}})" ]]; then true; else false; fi + # Validate config.yaml against the cue/state schema (and transitively + # cue/values + cue/mirror) before exporting. Surfaces schema errors + # before they appear as confusing yaml-export failures. + # Folded scalar (>-) is required because the bare `yaml:` token would + # otherwise be parsed as a YAML mapping key. + - >- + cue vet ./cue/state yaml: config.yaml -l 'config:' + - >- + cue export ./cue/state yaml: config.yaml -l 'config:' + -t cwd="$PWD" + -t sshPubKey="$(tr -d '\n' < {{.OUTPUT_DIR}}/capt-ssh-key.pub)" + -e out --out yaml > .state next-steps: silent: true @@ -104,29 +145,41 @@ tasks: sh: yq eval '.clusterName' {{.STATE_FILE_FQ_PATH}} KIND_KUBECONFIG: sh: yq eval '.kind.kubeconfig' {{.STATE_FILE_FQ_PATH}} + EXTERNAL_TINKERBELL: + sh: yq eval '.externalTinkerbell // false' {{.STATE_FILE_FQ_PATH}} + TINK_KUBECONFIG: + sh: yq eval '.kind.tinkerbell.kubeconfig // ""' {{.STATE_FILE_FQ_PATH}} cmds: - | echo echo The workload cluster is now being created. echo Once the cluster nodes are up and running, you will need to deploy a CNI for the cluster to be fully functional. echo The management cluster kubeconfig is located at: {{.KIND_KUBECONFIG}} + {{- if eq .EXTERNAL_TINKERBELL "true" }} + echo The Tinkerbell cluster kubeconfig is located at: {{.TINK_KUBECONFIG}} + echo "NOTE: External Tinkerbell mode is enabled. Tinkerbell resources (Hardware, Workflows, BMC) are in the Tinkerbell cluster." + {{- end }} echo The workload cluster kubeconfig is located at: {{.OUTPUT_DIR}}/{{.CLUSTER_NAME}}.kubeconfig echo echo 1. Watch and wait for the first control plane node to be provisioned successfully: STATE_SUCCESS + {{- if eq .EXTERNAL_TINKERBELL "true" }} + echo "KUBECONFIG={{.TINK_KUBECONFIG}} kubectl get workflows -n {{.NAMESPACE}} -o wide -w" + {{- else }} echo "KUBECONFIG={{.KIND_KUBECONFIG}} kubectl get workflows -n {{.NAMESPACE}} -o wide -w" + {{- end }} echo echo echo 2. Watch and wait for the Kubernetes API server to be ready and responding: - echo "until KUBECONFIG={{.CURR_DIR}}/{{.OUTPUT_DIR}}/{{.CLUSTER_NAME}}.kubeconfig kubectl get node; do echo 'Waiting for Kube API server to respond...'; sleep 5; done" + echo "until KUBECONFIG={{.ROOT_DIR}}/{{.OUTPUT_DIR}}/{{.CLUSTER_NAME}}.kubeconfig kubectl get node; do echo 'Waiting for Kube API server to respond...'; sleep 5; done" echo echo 3. Deploy a CNI echo Cilium - echo "KUBECONFIG={{.CURR_DIR}}/{{.OUTPUT_DIR}}/{{.CLUSTER_NAME}}.kubeconfig cilium install" + echo "KUBECONFIG={{.ROOT_DIR}}/{{.OUTPUT_DIR}}/{{.CLUSTER_NAME}}.kubeconfig cilium install" echo or KUBEROUTER - echo "KUBECONFIG={{.CURR_DIR}}/{{.OUTPUT_DIR}}/{{.CLUSTER_NAME}}.kubeconfig kubectl apply -f https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/kubeadm-kuberouter.yaml" + echo "KUBECONFIG={{.ROOT_DIR}}/{{.OUTPUT_DIR}}/{{.CLUSTER_NAME}}.kubeconfig kubectl apply -f https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/kubeadm-kuberouter.yaml" echo echo 4. Watch and wait for all nodes to join the cluster and be ready: - echo "KUBECONFIG={{.CURR_DIR}}/{{.OUTPUT_DIR}}/{{.CLUSTER_NAME}}.kubeconfig kubectl get nodes -w" + echo "KUBECONFIG={{.ROOT_DIR}}/{{.OUTPUT_DIR}}/{{.CLUSTER_NAME}}.kubeconfig kubectl get nodes -w" - touch {{.OUTPUT_DIR}}/.next-steps-displayed status: - echo ;[ -f {{.OUTPUT_DIR}}/.next-steps-displayed ] diff --git a/capt/config.yaml b/capt/config.yaml index 49cb5b91..745acef7 100644 --- a/capt/config.yaml +++ b/capt/config.yaml @@ -7,11 +7,11 @@ counts: workers: 1 spares: 1 versions: - capt: v0.6.7 - chart: v0.23.0 - kube: v1.31.3 - os: 20.04 - kubevip: 0.9.1 + capt: v0.7.0 + chart: v0.23.1-23da0880 + kube: v1.35.2 + os: 2404 + kubevip: 1.1.2 capt: providerRepository: "https://github.com/tinkerbell/cluster-api-provider-tinkerbell/releases" #providerRepository: "/home/tink/repos/tinkerbell/cluster-api-provider-tinkerbell/out/release/infrastructure-tinkerbell" @@ -19,13 +19,13 @@ chart: location: "oci://ghcr.io/tinkerbell/charts/tinkerbell" #location: "/home/tink/repos/tinkerbell/tinkerbell/helm/tinkerbell" extraVars: - # - deployment.imagePullPolicy=Always - - optional.hookos.downloadURL=https://github.com/tinkerbell/hook/releases/download/latest + - optional.captainos.enabled=true + - optional.captainos.image=ghcr.io/tinkerbell/captain/artifacts:v0.0.0-a4be23c - deployment.envs.ui.enableAutoLogin=true os: - registry: ghcr.io/tinkerbell/cluster-api-provider-tinkerbell - distro: ubuntu - sshKey: "" + # the image must have a tag in the format of --.gz, for example: 2404-v1.36.0-arm64.gz + # ghcr.io/tinkerbell/cluster-api-provider-tinkerbell/ubuntu:2404-v1.36.0-arm64.gz + registry: ghcr.io/tinkerbell/cluster-api-provider-tinkerbell/ubuntu vm: baseName: "node" cpusPerVM: 2 @@ -38,3 +38,34 @@ virtualBMC: user: "root" pass: "calvin" bootMode: netboot +externalTinkerbell: false +arch: amd64 +captainos: + kernelVersion: "6.18.16" +# Pull-through OCI registry mirror. When enabled, every image pulled from +# this directory is routed through `host` (HTTPS, publicly-trusted CA only): +# - Host-side image strings (helm chart.location, os.registry, +# virtualBMC.image) are rewritten to `/...` in cue/state by +# cue/mirror/#rewrite — they reach helm/docker/crane already pointing +# at the mirror. +# - kind nodes get a rendered `certs.d` tree bind-mounted into each +# node at `/etc/containerd/certs.d` via `extraMounts` in the kind +# cluster config. The certs.d tree and the kind-config YAML are +# rendered by mirror:render-kind-config and consumed by both +# kind-cluster and tinkerbell-kind-cluster, which pass the config +# via --config only when this block is enabled. +# - Workload nodes get the same drop-in via KubeadmConfigSpec.files +# (cue/capi/resources.cue + cue/mirror/#cloudInitFiles). +# Everything else (workflow actions, kube-vip ctr pulls, all helm-chart +# Pod images including captainos, kubeadm pulls) keeps its original +# upstream hostname — the containerd mirror redirects at pull time. +# Default-disabled. +registryMirror: + enabled: false + host: "" + upstreams: + - "ghcr.io" + - "quay.io" + - "registry.k8s.io" + - "gcr.io" + - "docker.io" diff --git a/capt/cue.mod/module.cue b/capt/cue.mod/module.cue new file mode 100644 index 00000000..34ed0152 --- /dev/null +++ b/capt/cue.mod/module.cue @@ -0,0 +1,4 @@ +module: "tinkerbell.org/capt-playground" +language: { + version: "v0.16.1" +} diff --git a/capt/cue/capi/bootmode.cue b/capt/cue/capi/bootmode.cue new file mode 100644 index 00000000..6b03fd9b --- /dev/null +++ b/capt/cue/capi/bootmode.cue @@ -0,0 +1,75 @@ +// Bootmode discriminator. _mode (in resources.cue) selects netboot vs isoboot; +// this file produces the bootOptions struct embedded in each TMT and the +// extra workflow actions that only iso boot needs (disable cloud-init +// networking + install a static netplan, since iso boot has no DHCP). +// +// The interface is marked `optional: true` so systemd-networkd-wait-online +// does not gate boot on it (a static-only config can otherwise stall boot +// at the 2-minute wait-online default timeout). +package capi + +_bootOptions: { + if _mode == "netboot" { + bootMode: "netboot" + } + if _mode == "isoboot" { + bootMode: "isoboot" + isoURL: "http://\(values.tinkerbell.vip):7080/iso/hook.iso" + } +} + +_isoExtraActions: [ + { + name: "disable cloud-init networking" + image: "quay.io/tinkerbell/actions/writefile" + timeout: 90 + environment: { + CONTENTS: "network: {config: disabled}" + DEST_DISK: "{{ formatPartition ( index .Hardware.Disks 0 ) 3 }}" + DEST_PATH: "/etc/cloud/cloud.cfg.d/99-disable-network-config.cfg" + DIRMODE: "0700" + FS_TYPE: "ext4" + GID: "0" + MODE: "0600" + UID: "0" + } + }, + { + name: "create static netplan" + image: "quay.io/tinkerbell/actions/writefile" + timeout: 90 + environment: { + CONTENTS: """ + network: + version: 2 + renderer: networkd + ethernets: + id0: + match: + macaddress: {{ (index .Hardware.Interfaces 0).DHCP.MAC }} + addresses: + - {{ (index .Hardware.Interfaces 0).DHCP.IP.Address }}/16 + nameservers: + addresses: [{{ (index .Hardware.Interfaces 0).DHCP.NameServers | join \",\"}}] + routes: + - to: default + via: {{ (index .Hardware.Interfaces 0).DHCP.IP.Gateway }} + optional: true + + """ + DEST_DISK: "{{ formatPartition ( index .Hardware.Disks 0 ) 3 }}" + DEST_PATH: "/etc/netplan/config.yaml" + DIRMODE: "0755" + FS_TYPE: "ext4" + GID: "0" + MODE: "0600" + UID: "0" + } + }, +] + +// Empty for netboot, two extra writefile actions for isoboot. +_extraActions: [...] +_extraActions: [ + if _mode == "isoboot" for a in _isoExtraActions {a}, +] diff --git a/capt/cue/capi/render.cue b/capt/cue/capi/render.cue new file mode 100644 index 00000000..dd096305 --- /dev/null +++ b/capt/cue/capi/render.cue @@ -0,0 +1,19 @@ +// Render entrypoint for the CAPI subpackage. The pipeline runs: +// cue export ./cue/capi yaml: .state -l 'values:' -t mode= -e out --out text +// `out` is a multi-document YAML string produced by yaml.MarshalStream so each +// resource is separated by `---` and kubectl apply -f can ingest the result. +package capi + +import "encoding/yaml" + +_resources: [ + _cluster, + _tinkerbellCluster, + _kcp, + _md, + _kct, + _tmtCp, + _tmtWorker, +] + +out: yaml.MarshalStream(_resources) diff --git a/capt/cue/capi/resources.cue b/capt/cue/capi/resources.cue new file mode 100644 index 00000000..10feba2a --- /dev/null +++ b/capt/cue/capi/resources.cue @@ -0,0 +1,198 @@ +// CAPI/CAPT resources composed for the playground. +// +// Names and references are computed in the shared values package so that +// bumping values.versions.kube changes the TMT names (with version suffix) +// and the references from KCP/MD in lock-step. Drift is impossible by +// construction. +package capi + +import ( + "encoding/yaml" + "list" + "tinkerbell.org/capt-playground/cue/mirror" + v "tinkerbell.org/capt-playground/cue/values" +) + +// Mirror cloud-init files + preKubeadmCommands. cue/mirror is the single +// source of truth shared with cue/kind. When the mirror is disabled the +// `_mirrorFiles` list is empty and `_mirrorPreCmds` is empty, so the KCP +// and KCT below render identically to the no-mirror case. +_mirrorRender: (mirror & {"values": values}).cloudInitFiles +_mirrorFiles: _mirrorRender +_mirrorPreCmds: [if len(_mirrorRender) > 0 {"systemctl restart containerd"}] + +// Top-level injected by the task pipeline: +// cue export ./cue/capi yaml: .state -l 'values:' -t mode= -e out --out text +values: v.#Config +_mode: *"netboot" | "isoboot" @tag(mode) + +// Single instantiation of the shared computed locals; reference c. +// from every resource below. Keeps this file's local namespace clean. +c: v.#Computed & {"values": values, mode: _mode} + +_cluster: { + apiVersion: "cluster.x-k8s.io/v1beta2" + kind: "Cluster" + metadata: { + name: c.clusterName + namespace: c.namespace + } + spec: { + clusterNetwork: { + pods: cidrBlocks: [values.cluster.podCIDR] + services: cidrBlocks: ["172.26.0.0/16"] + } + controlPlaneEndpoint: { + host: values.cluster.controlPlane.vip + port: 6443 + } + // v1beta2 refs use apiGroup + kind + name (no apiVersion); CAPI looks + // up the served version from CRD contract labels. + controlPlaneRef: { + apiGroup: "controlplane.cluster.x-k8s.io" + kind: "KubeadmControlPlane" + name: c.kcpName + } + infrastructureRef: { + apiGroup: "infrastructure.cluster.x-k8s.io" + kind: "TinkerbellCluster" + name: c.tinkClusterName + } + } +} + +_tinkerbellCluster: { + apiVersion: "infrastructure.cluster.x-k8s.io/v1beta1" + kind: "TinkerbellCluster" + metadata: { + name: c.tinkClusterName + namespace: c.namespace + } + spec: imageLookupBaseRegistry: "" +} + +_kcp: { + apiVersion: "controlplane.cluster.x-k8s.io/v1beta2" + kind: "KubeadmControlPlane" + metadata: { + name: c.kcpName + namespace: c.namespace + } + spec: { + replicas: values.counts.controlPlanes + version: values.versions.kube + machineTemplate: spec: infrastructureRef: { + apiGroup: "infrastructure.cluster.x-k8s.io" + kind: "TinkerbellMachineTemplate" + name: c.tmtCpName + } + kubeadmConfigSpec: { + // v1beta2: kubeletExtraArgs is a list of {name,value} objects (was a + // map[string]string in v1beta1). + initConfiguration: nodeRegistration: kubeletExtraArgs: [{name: "provider-id", value: "PROVIDER_ID"}] + joinConfiguration: nodeRegistration: { + ignorePreflightErrors: ["DirAvailable--etc-kubernetes-manifests"] + kubeletExtraArgs: [{name: "provider-id", value: "PROVIDER_ID"}] + } + // Mirror restart must run before kubeadm so the drop-in is loaded. + preKubeadmCommands: list.Concat([_mirrorPreCmds, [c.kubeVipCmd]]) + if len(_mirrorFiles) > 0 { + files: _mirrorFiles + } + users: c.users + } + } +} + +_md: { + apiVersion: "cluster.x-k8s.io/v1beta2" + kind: "MachineDeployment" + metadata: { + name: c.mdName + namespace: c.namespace + labels: { + "cluster.x-k8s.io/cluster-name": c.clusterName + pool: "worker-a" + } + } + spec: { + clusterName: c.clusterName + replicas: values.counts.workers + selector: matchLabels: { + "cluster.x-k8s.io/cluster-name": c.clusterName + pool: "worker-a" + } + template: { + metadata: labels: { + "cluster.x-k8s.io/cluster-name": c.clusterName + pool: "worker-a" + } + spec: { + clusterName: c.clusterName + version: values.versions.kube + bootstrap: configRef: { + apiGroup: "bootstrap.cluster.x-k8s.io" + kind: "KubeadmConfigTemplate" + name: c.kctName + } + infrastructureRef: { + apiGroup: "infrastructure.cluster.x-k8s.io" + kind: "TinkerbellMachineTemplate" + name: c.tmtWorkerName + } + } + } + } +} + +_kct: { + apiVersion: "bootstrap.cluster.x-k8s.io/v1beta2" + kind: "KubeadmConfigTemplate" + metadata: { + name: c.kctName + namespace: c.namespace + } + spec: template: spec: { + joinConfiguration: nodeRegistration: kubeletExtraArgs: [{name: "provider-id", value: "PROVIDER_ID"}] + if len(_mirrorFiles) > 0 { + files: _mirrorFiles + preKubeadmCommands: _mirrorPreCmds + } + users: c.users + } +} + +// Helper: a TMT for a given role (control-plane or worker). +#tmt: { + _name: string + _role: "control-plane" | "worker" + apiVersion: "infrastructure.cluster.x-k8s.io/v1beta1" + kind: "TinkerbellMachineTemplate" + metadata: { + name: _name + namespace: c.namespace + } + spec: template: spec: { + bootOptions: _bootOptions + hardwareAffinity: required: [{ + labelSelector: matchLabels: "tinkerbell.org/role": _role + }] + templateOverride: _workflowYAML + } +} + +// Marshal the workflow once at package scope so `cue vet ./cue/capi` +// evaluates it eagerly (rather than only when a TMT is materialised). +// Any structural error in _workflow surfaces during validation, not at +// kubectl-apply time. +_workflowYAML: yaml.Marshal(_workflow) + +_tmtCp: #tmt & { + _name: c.tmtCpName + _role: "control-plane" +} + +_tmtWorker: #tmt & { + _name: c.tmtWorkerName + _role: "worker" +} diff --git a/capt/cue/capi/workflow.cue b/capt/cue/capi/workflow.cue new file mode 100644 index 00000000..e7b04130 --- /dev/null +++ b/capt/cue/capi/workflow.cue @@ -0,0 +1,96 @@ +// Workflow templateOverride. Authored as a structured CUE value, then marshaled +// to YAML by resources.cue and embedded as a string in TMT.spec.template.spec.templateOverride. +// +// CAPT reads templateOverride as text, template-expands `{{ ... }}` Go template +// expressions against per-machine Hardware data, then parses the result as YAML +// to produce the Workflow CR. +package capi + +import "list" + +_workflow: { + version: "0.1" + name: "playground-template" + global_timeout: 6000 + tasks: [{ + name: "playground-template" + worker: "{{.device_1}}" + volumes: ["/dev:/dev"] + actions: list.Concat([_commonActionsHead, _extraActions, _commonActionsTail]) + }] +} + +_commonActionsHead: [ + { + name: "stream image" + image: "quay.io/tinkerbell/actions/oci2disk" + timeout: 1200 + environment: { + IMG_URL: c.imgURL + DEST_DISK: "{{ index .Hardware.Disks 0 }}" + COMPRESSED: true + } + }, + { + name: "add tink cloud-init config" + image: "quay.io/tinkerbell/actions/writefile" + timeout: 90 + environment: { + DEST_DISK: "{{ formatPartition ( index .Hardware.Disks 0 ) 3 }}" + FS_TYPE: "ext4" + DEST_PATH: "/etc/cloud/cloud.cfg.d/10_tinkerbell.cfg" + UID: 0 + GID: 0 + MODE: "0600" + DIRMODE: "0700" + CONTENTS: """ + datasource: + Ec2: + metadata_urls: [\"\(c.metadataURL)\"] + strict_id: false + system_info: + default_user: + name: tink + groups: [wheel, adm] + sudo: [\"ALL=(ALL) NOPASSWD:ALL\"] + shell: /bin/bash + manage_etc_hosts: localhost + warnings: + dsid_missing_source: off + + """ + } + }, +] + +_commonActionsTail: [ + { + name: "add tink cloud-init ds-config" + image: "quay.io/tinkerbell/actions/writefile" + timeout: 90 + environment: { + DEST_DISK: "{{ formatPartition ( index .Hardware.Disks 0 ) 3 }}" + FS_TYPE: "ext4" + DEST_PATH: "/etc/cloud/ds-identify.cfg" + UID: 0 + GID: 0 + MODE: "0600" + DIRMODE: "0700" + CONTENTS: "datasource: Ec2\n" + } + }, + { + name: "kexec image" + image: "ghcr.io/jacobweinstock/waitdaemon:latest" + timeout: 90 + pid: "host" + environment: { + BLOCK_DEVICE: "{{ formatPartition ( index .Hardware.Disks 0 ) 1 }}" + FS_TYPE: "vfat" + IMAGE: "quay.io/tinkerbell/actions/kexec" + GRUBCFG_PATH: "/grub/grub.cfg" + WAIT_SECONDS: 10 + } + volumes: ["/var/run/docker.sock:/var/run/docker.sock"] + }, +] diff --git a/capt/cue/clusterctl/clusterctl.cue b/capt/cue/clusterctl/clusterctl.cue new file mode 100644 index 00000000..3ae5a075 --- /dev/null +++ b/capt/cue/clusterctl/clusterctl.cue @@ -0,0 +1,17 @@ +// clusterctl config (replaces templates/clusterctl.tmpl). Rendered by: +// cue export ./cue/clusterctl yaml: .state -l 'values:' -e out --out yaml +// > output/clusterctl.yaml +package clusterctl + +import v "tinkerbell.org/capt-playground/cue/values" + +values: v.#Config + +out: { + providers: [{ + name: "tinkerbell" + url: "\(values.capt.providerRepository)/\(values.versions.capt)/infrastructure-components.yaml" + type: "InfrastructureProvider" + }] + images: "infrastructure-tinkerbell": tag: values.versions.capt +} diff --git a/capt/cue/infra/bmc.cue b/capt/cue/infra/bmc.cue new file mode 100644 index 00000000..f5a8fa3d --- /dev/null +++ b/capt/cue/infra/bmc.cue @@ -0,0 +1,61 @@ +// BMC Machine CRs (one per VM) and the single shared BMC credentials Secret. +package infra + +import "encoding/base64" + +#bmcMachine: { + _name: string + _port: int + apiVersion: "bmc.tinkerbell.org/v1alpha1" + kind: "Machine" + metadata: { + name: _name + namespace: c.namespace + } + spec: connection: { + authSecretRef: { + name: "bmc-creds" + namespace: c.namespace + } + host: values.virtualBMC.ip + insecureTLS: true + port: _port + providerOptions: { + preferredOrder: ["ipmitool"] + ipmitool: port: _port + redfish: { + useBasicAuth: true + systemName: _name + } + } + } +} + +outBmcMachines: { + for name, vm in values.vm.details { + "\(name)": #bmcMachine & { + _name: name + _port: vm.bmc.port + } + } +} + +// All machines share a single Secret; only namespace, user, and password +// vary. base64.Encode happens in CUE so no shell base64 step is needed. +outBmcSecret: { + apiVersion: "v1" + kind: "Secret" + metadata: { + name: "bmc-creds" + namespace: c.namespace + labels: { + "clusterctl.cluster.x-k8s.io/move": "" + "clusterctl.cluster.x-k8s.io": "" + } + } + data: { + password: base64.Encode(null, values.virtualBMC.pass) + username: base64.Encode(null, values.virtualBMC.user) + } + type: "kubernetes.io/basic-auth" +} diff --git a/capt/cue/infra/hardware.cue b/capt/cue/infra/hardware.cue new file mode 100644 index 00000000..512fc0a8 --- /dev/null +++ b/capt/cue/infra/hardware.cue @@ -0,0 +1,71 @@ +// Hardware CRs, one per VM in values.vm.details. +// +// outHardware is a map keyed by node name; the Taskfile iterates over the +// keys and runs `cue export -e outHardware. --out yaml` once per node +// so each Hardware CR lands in its own output/hardware-.yaml file. +package infra + +#hardware: { + _name: string + _mac: string + _role: "control-plane" | "worker" | "spare" + _ip: string + _gateway: string + + apiVersion: "tinkerbell.org/v1alpha1" + kind: "Hardware" + metadata: { + labels: { + "tinkerbell.org/role": _role + "node.cluster.x-k8s.io/rack": "test-\(_role)" + } + name: _name + namespace: c.namespace + } + spec: { + bmcRef: { + apiGroup: "bmc.tinkerbell.org" + kind: "Machine" + name: _name + } + disks: [{device: "/dev/vda"}] + interfaces: [{ + dhcp: { + arch: c.dhcpArch + hostname: _name + ip: { + address: _ip + gateway: _gateway + netmask: "255.255.0.0" + } + // As DHCP addresses are Reservations in Tinkerbell it doesn't + // matter much the value here. We don't want the dhcp client constantly + // renewing its address though, so we just use the Max DHCP lease time (uint32 max - 1). + lease_time: 4294967294 + mac: _mac + uefi: true + name_servers: ["8.8.8.8", "1.1.1.1"] + } + netboot: { + allowPXE: true + allowWorkflow: true + } + }] + metadata: instance: { + hostname: _name + id: _mac + } + } +} + +outHardware: { + for name, vm in values.vm.details { + "\(name)": #hardware & { + _name: name + _mac: vm.mac + _role: vm.role + _ip: vm.ip + _gateway: vm.gateway + } + } +} diff --git a/capt/cue/infra/render.cue b/capt/cue/infra/render.cue new file mode 100644 index 00000000..5780258a --- /dev/null +++ b/capt/cue/infra/render.cue @@ -0,0 +1,20 @@ +// Render entrypoint for the infra subpackage. Three exported symbols, each +// fetched independently by the Taskfile via `cue export -e --out yaml`: +// +// outHardware map[string]Hardware — one file per VM +// outBmcMachines map[string]Machine — one file per VM +// outBmcSecret Secret — single shared secret +package infra + +import v "tinkerbell.org/capt-playground/cue/values" + +// Top-level injected by the task pipeline: +// cue export ./cue/infra yaml: .state -l 'values:' -e --out yaml +values: v.#Config + +// _mode is unused by the infra resources but the shared #Computed struct +// requires it; default to netboot to keep the unification valid. The choice +// has no effect on any field referenced by infra (clusterName, namespace). +_mode: "netboot" + +c: v.#Computed & {"values": values, mode: _mode} diff --git a/capt/cue/kind/kind.cue b/capt/cue/kind/kind.cue new file mode 100644 index 00000000..503318a1 --- /dev/null +++ b/capt/cue/kind/kind.cue @@ -0,0 +1,60 @@ +// Renders a kind v1alpha4 cluster configuration. +// +// When the registry mirror is enabled, the kind config bind-mounts the +// host-side certs.d tree (rendered to /certs.d//hosts.toml +// by tasks/Taskfile-mirror.yaml#render-kind-config) into every node at +// /etc/containerd/certs.d. kind's default containerd config already sets +// `config_path = "/etc/containerd/certs.d"`, so no containerdConfigPatches +// are needed — and indeed must not be added: containerd v2 (kindest/node +// v1.35+) refuses any inline `[plugins."...".registry.mirrors.]` block +// when `config_path` is also set, with: +// +// `mirrors` cannot be set when `config_path` is provided +// +// The same hosts.toml bodies are emitted as cloud-init files on workload +// nodes by cue/mirror/files.cue#cloudInitFiles, so kind and CAPT-VM nodes +// see identical mirror configuration. +// +// Pipeline: +// cue export ./cue/kind yaml: .state -l 'values:' -e out --out yaml +// +// Output is consumed by tasks/Taskfile-create.yaml#kind-cluster, which +// passes it to `kind create cluster --config`. +package kind + +import ( + "tinkerbell.org/capt-playground/cue/mirror" + v "tinkerbell.org/capt-playground/cue/values" +) + +values: v.#Config + +// Adapt values into the shape mirror.values expects. +_mirrorValues: { + registryMirror: mirror.#Spec + if values.registryMirror != _|_ { + registryMirror: values.registryMirror + } +} + +_mirrorEnabled: _mirrorValues.registryMirror.enabled && + len(_mirrorValues.registryMirror.upstreams) > 0 + +// Absolute host path to the rendered certs.d tree. Bind-mounted into +// each kind node. Rendered by tasks/Taskfile-mirror.yaml#render-kind-config. +_hostCertsDir: "\(values.outputDir)/certs.d" + +out: { + kind: "Cluster" + apiVersion: "kind.x-k8s.io/v1alpha4" + if _mirrorEnabled { + nodes: [{ + role: "control-plane" + extraMounts: [{ + hostPath: _hostCertsDir + containerPath: "/etc/containerd/certs.d" + readOnly: true + }] + }] + } +} diff --git a/capt/cue/mirror/files.cue b/capt/cue/mirror/files.cue new file mode 100644 index 00000000..29599b9c --- /dev/null +++ b/capt/cue/mirror/files.cue @@ -0,0 +1,83 @@ +// Renders containerd registry-mirror configuration in the modern +// hosts.toml drop-in style — required for containerd v2.x (kindest/node +// v1.35+) which rejects the legacy `[plugins."...".registry.mirrors.]` +// inline form with: +// +// `mirrors` cannot be set when `config_path` is provided +// +// Single source of truth (`HostsTomlByUpstream`) consumed by: +// - cue/kind/kind.cue: rendered to host-side files under +// /certs.d//hosts.toml and bind-mounted into kind nodes +// via nodes[].extraMounts -> /etc/containerd/certs.d +// - cue/capi/resources.cue: emitted as cloud-init `files:` entries on +// workload nodes at /etc/containerd/certs.d//hosts.toml +// +// Pipelines: +// cue export ./cue/mirror yaml: .state -l 'values:' -e hostsTomlByUpstream --out json +// cue export ./cue/mirror yaml: .state -l 'values:' -e cloudInitFiles --out json +package mirror + +values: { + registryMirror: #Spec + ... +} + +// Per-upstream hosts.toml body. Map shape: { "": "" }. +// Empty when the feature is off or no upstreams are configured. +// +// hosts.toml schema reference: +// https://github.com/containerd/containerd/blob/main/docs/hosts.md +HostsTomlByUpstream: { + if values.registryMirror.enabled for u in values.registryMirror.upstreams { + "\(u)": """ + server = "https://\(u)" + + [host."https://\(values.registryMirror.host)"] + capabilities = ["pull", "resolve"] + """ + } +} + +// Flat list of {path, content} for the host-side certs.d tree consumed by +// tasks/Taskfile-mirror.yaml#render-kind-config. `path` is relative to +// the chosen output directory (e.g. "ghcr.io/hosts.toml"). Empty list +// when the feature is off — the task wipes the directory regardless so +// disabled-state leaves no stale files. +hostCertsdFiles: [ + if values.registryMirror.enabled for u in values.registryMirror.upstreams { + { + path: "\(u)/hosts.toml" + content: HostsTomlByUpstream[u] + } + }, +] + +// Cloud-init `files:` entries for workload nodes. One file per upstream +// at /etc/containerd/certs.d//hosts.toml plus a tiny conf.d drop-in +// that sets `config_path` (Ubuntu's containerd default doesn't set it; +// kind's does, so kind nodes don't get this drop-in — they get a host +// bind-mount instead, see cue/kind/kind.cue). +// +// Ubuntu's default containerd config has `imports = ["/etc/containerd/conf.d/*.toml"]`. +cloudInitFiles: [ + if values.registryMirror.enabled for u in values.registryMirror.upstreams { + { + path: "/etc/containerd/certs.d/\(u)/hosts.toml" + owner: "root:root" + permissions: "0644" + content: HostsTomlByUpstream[u] + } + }, + if values.registryMirror.enabled && len(values.registryMirror.upstreams) > 0 { + { + path: "/etc/containerd/conf.d/registry-config-path.toml" + owner: "root:root" + permissions: "0644" + content: """ + version = 2 + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" + """ + } + }, +] diff --git a/capt/cue/mirror/rewrite.cue b/capt/cue/mirror/rewrite.cue new file mode 100644 index 00000000..453d39fc --- /dev/null +++ b/capt/cue/mirror/rewrite.cue @@ -0,0 +1,42 @@ +// URL rewrite helper. Replaces the first matching `/` (or +// `oci:///`) prefix in `in` with `cfg.host/`. When the mirror is +// disabled, no upstream is configured, or no upstream matches, returns +// `in` unchanged. +// +// Used by cue/state/state.cue to rewrite the small set of image strings +// that are pulled OUTSIDE any containerd (and therefore can't benefit from +// the containerd registry-mirror config baked in by cue/kind and +// cue/capi/resources.cue): +// - chart.location (helm OCI client on the host) +// - os.registry (crane inside the oci2disk action container) +// - virtualBMC.image (docker run on the host) +// +// Everything else flows through a containerd that has the mirror baked +// in — do NOT rewrite those, because doing so hides the original upstream +// from the operator and breaks debuggability. +package mirror + +import "strings" + +#rewrite: { + in: string + cfg: #Spec + // First-match wins. Each candidate prefix is tried in declaration order; + // the first one that matches `in` produces the rewritten string. If none + // match (or cfg.enabled is false), `in` flows through unchanged. + _candidates: [ + if cfg.enabled for u in cfg.upstreams { + {prefix: "oci://\(u)/", replacement: "oci://\(cfg.host)/"} + }, + if cfg.enabled for u in cfg.upstreams { + {prefix: "\(u)/", replacement: "\(cfg.host)/"} + }, + ] + _matches: [for c in _candidates if strings.HasPrefix(in, c.prefix) { + c.replacement + strings.TrimPrefix(in, c.prefix) + }] + out: [ + if len(_matches) > 0 {_matches[0]}, + in, + ][0] +} diff --git a/capt/cue/mirror/schema.cue b/capt/cue/mirror/schema.cue new file mode 100644 index 00000000..04f97753 --- /dev/null +++ b/capt/cue/mirror/schema.cue @@ -0,0 +1,30 @@ +// Schema for the optional registry-mirror feature. Self-contained CUE package +// so the entire feature can be removed by deleting cue/mirror/, the sibling +// *_extension.cue files in cue/values and cue/state, the cue/kind/ package, +// tasks/Taskfile-mirror.yaml, the `mirror:` include, the `task: mirror:...` +// lines in tasks/Taskfile-create.yaml, and the `registryMirror:` block in +// config.yaml. +// +// HTTPS only. The mirror host must present a publicly-trusted CA cert. There +// are no insecure knobs (no skip_verify, no custom CA). +package mirror + +// Public spec consumed by the values and state packages via additive +// extension files. enabled=false makes every consumer a no-op. +// +// When enabled=true, both `host` and `upstreams` must be non-empty (and +// every upstream entry must itself be non-empty). The conditional block +// below tightens the otherwise-defaulted fields so an empty host or +// empty upstreams list fails `cue vet`/`cue export` with a line-numbered +// error before any task runs, instead of silently rendering broken +// `https:///...` URLs and an empty certs.d tree. +#Spec: { + enabled: bool | *false + host: string | *"" + upstreams: [...string] | *[] + + if enabled == true { + host: string & !="" + upstreams: [string & !="", ...string & !=""] + } +} diff --git a/capt/cue/state/mirror_extension.cue b/capt/cue/state/mirror_extension.cue new file mode 100644 index 00000000..a66f5cf8 --- /dev/null +++ b/capt/cue/state/mirror_extension.cue @@ -0,0 +1,41 @@ +// Additive extension of the state package: accepts an optional +// `registryMirror` block in config.yaml and passes it through to .state so +// downstream consumers (Taskfile-mirror.yaml, the cue/mirror package) can +// read it via yq or cue. +// +// Also rewrites the small set of image strings that are pulled OUTSIDE any +// containerd (helm-on-host, crane-in-action-container, docker-on-host) to +// route through the mirror host. Everything else is left as-is because +// the kind containerd mirror (cue/kind) and the workload-node containerd +// mirror drop-in (cue/capi/resources.cue) handle redirection at runtime. +// +// CUE merges sibling files in the same package, so state.cue is not +// touched. Delete this file to remove the field and the rewrites. +package state + +import "tinkerbell.org/capt-playground/cue/mirror" + +#ConfigInput: { + registryMirror?: mirror.#Spec + ... +} + +// Effective mirror spec, with a disabled-default when the field is absent +// from config.yaml. +_mirrorCfg: [ + if config.registryMirror != _|_ {config.registryMirror}, + {enabled: false, host: "", upstreams: []}, +][0] + +out: { + registryMirror: _mirrorCfg + + // helm OCI client direct HTTPS (no containerd in the pull path). + chart: location: (mirror.#rewrite & {in: config.chart.location, cfg: _mirrorCfg}).out + + // crane inside the oci2disk action container, direct HTTPS. + os: registry: (mirror.#rewrite & {in: config.os.registry, cfg: _mirrorCfg}).out + + // `docker run` on the host. + virtualBMC: image: (mirror.#rewrite & {in: config.virtualBMC.image, cfg: _mirrorCfg}).out +} diff --git a/capt/cue/state/state.cue b/capt/cue/state/state.cue new file mode 100644 index 00000000..e17dfed6 --- /dev/null +++ b/capt/cue/state/state.cue @@ -0,0 +1,201 @@ +package state + +import ( + "crypto/md5" + "encoding/hex" + "list" + "strings" + "path" +) + +#ConfigInput: { + clusterName: string & !="" + outputDir: string & !="" + namespace: string & !="" + arch: "amd64" | "arm64" + bootMode: "netboot" | "isoboot" + externalTinkerbell: bool | *false + counts: { + controlPlanes: int & >=1 + workers: int & >=0 + spares: int & >=0 + } + versions: { + capt: string & !="" + chart: string & !="" + kube: =~"^v[0-9]+\\.[0-9]+\\.[0-9]+$" + os: string | int + kubevip: string | number + } + capt: providerRepository: string & !="" + chart: { + location: string & !="" + extraVars?: [...string] + } + os: { + registry: string & !="" + } + vm: { + baseName: string & !="" + cpusPerVM: int & >0 + memInMBPerVM: int & >0 + diskSizeInGBPerVM: int & >0 + diskPath: string & !="" + } + virtualBMC: { + containerName: string & !="" + image: string & !="" + user: string & !="" + pass: string & !="" + } + captainos?: { + kernelVersion: string & !="" + } +} + +config: #ConfigInput + +cwd: string | *"" @tag(cwd) +sshPubKey: string | *"" @tag(sshPubKey) +_gatewayIP: string | *"" @tag(gatewayIP) +_bridge: string | *"" @tag(bridgeName) + +_outputDir: [ + if path.IsAbs(config.outputDir, path.Unix) {config.outputDir}, + if cwd != "" {path.Join([cwd, config.outputDir], path.Unix)}, + config.outputDir, +][0] + +_totalNodes: config.counts.controlPlanes + config.counts.workers + config.counts.spares + +_osVersion: strings.Replace("\(config.versions.os)", ".", "", -1) + +_indexes: list.Range(1, _totalNodes+1, 1) + +#mac: { + _input: string + _sum: md5.Sum(_input + "\n") + _hex: strings.Split(hex.Encode(_sum), "") + out: "02:" + strings.Join([ + _hex[0] + _hex[1], + _hex[2] + _hex[3], + _hex[4] + _hex[5], + _hex[6] + _hex[7], + _hex[8] + _hex[9], + ], ":") +} + +#role: { + _idx: int + out: [ + if _idx <= config.counts.controlPlanes {"control-plane"}, + if _idx <= config.counts.controlPlanes+config.counts.workers {"worker"}, + "spare", + ][0] +} + +_gwParts: strings.Split(_gatewayIP, ".") +_nodeIPBase: [ + if _gatewayIP == "" {""}, + "\(_gwParts[0]).\(_gwParts[1]).10.20", +][0] +_baseLastOctet: 20 + +#offsetIP: { + _offset: int + out: [ + if _gatewayIP == "" {""}, + "\(_gwParts[0]).\(_gwParts[1]).10.\(_baseLastOctet+_offset)", + ][0] +} + +_podCIDR: [ + if _gatewayIP == "" {""}, + "\(_gwParts[0]).100.0.0/16", +][0] + +_details: { + for i in _indexes { + "\(config.vm.baseName)\(i)": { + mac: (#mac & {_input: "\(config.vm.baseName)\(i)"}).out + bmc: port: 6230 + i + role: (#role & {_idx: i}).out + if _gatewayIP != "" { + ip: (#offsetIP & {_offset: i}).out + gateway: _gatewayIP + } + } + } +} + +out: { + clusterName: config.clusterName + outputDir: _outputDir + namespace: config.namespace + arch: config.arch + bootMode: config.bootMode + externalTinkerbell: config.externalTinkerbell + counts: config.counts + versions: config.versions + capt: config.capt + // chart.location is supplied by cue/state/mirror_extension.cue (so the + // optional registry mirror can rewrite it). Pass through everything else. + chart: { + if config.chart.extraVars != _|_ { + extraVars: config.chart.extraVars + } + } + os: { + // os.registry is supplied by cue/state/mirror_extension.cue. + sshKey: sshPubKey + version: _osVersion + } + vm: { + baseName: config.vm.baseName + cpusPerVM: config.vm.cpusPerVM + memInMBPerVM: config.vm.memInMBPerVM + diskSizeInGBPerVM: config.vm.diskSizeInGBPerVM + diskPath: config.vm.diskPath + details: _details + } + virtualBMC: { + containerName: config.virtualBMC.containerName + // virtualBMC.image is supplied by cue/state/mirror_extension.cue. + user: config.virtualBMC.user + pass: config.virtualBMC.pass + } + if config.captainos != _|_ { + captainos: config.captainos + } + totalNodes: _totalNodes + kind: { + kubeconfig: "\(_outputDir)/kind.kubeconfig" + if _gatewayIP != "" { + gatewayIP: _gatewayIP + nodeIPBase: _nodeIPBase + } + if _bridge != "" { + bridgeName: _bridge + } + // Second KinD cluster used as the Tinkerbell stack target when + // `externalTinkerbell: true`. Same docker network ("kind") so pods + // in the management cluster can reach the Tinkerbell API server via + // the container IP (see scripts/create_external_kubeconfig_secret.sh). + if config.externalTinkerbell { + tinkerbell: { + clusterName: "\(config.clusterName)-tinkerbell" + kubeconfig: "\(_outputDir)/tinkerbell-kind.kubeconfig" + } + } + } + if _gatewayIP != "" { + tinkerbell: { + vip: (#offsetIP & {_offset: _totalNodes + 51}).out + hookosVip: (#offsetIP & {_offset: _totalNodes + 50}).out + } + cluster: { + controlPlane: vip: (#offsetIP & {_offset: _totalNodes + 52}).out + podCIDR: _podCIDR + } + } +} diff --git a/capt/cue/values/mirror_extension.cue b/capt/cue/values/mirror_extension.cue new file mode 100644 index 00000000..9f514453 --- /dev/null +++ b/capt/cue/values/mirror_extension.cue @@ -0,0 +1,13 @@ +// Additive extension of the values package: declares the optional +// `registryMirror` field on #Config so consumers can read it. +// +// CUE merges sibling files in the same package, so values.cue is not +// touched. Delete this file to remove the field from the schema. +package values + +import "tinkerbell.org/capt-playground/cue/mirror" + +#Config: { + registryMirror?: mirror.#Spec + ... +} diff --git a/capt/cue/values/values.cue b/capt/cue/values/values.cue new file mode 100644 index 00000000..24e2d9bf --- /dev/null +++ b/capt/cue/values/values.cue @@ -0,0 +1,170 @@ +// Shared input schema (#Config) and shared computed locals (#Computed) used +// by every render subpackage (capi, infra, clusterctl). +// +// Each consumer subpackage declares its own top-level `values: #Config` +// field so that `cue export ... yaml: .state -l 'values:'` can inject the +// .state file. Consumers then instantiate `#Computed & {values: values, mode: _mode}` +// and reference the public (non-underscore) fields exposed below. +// +// Constraints in #Config catch typos before render: a missing field or a +// kube version without a leading `v` will fail `cue vet` with a line-numbered +// error. +package values + +import "strings" + +// Inner structs are closed to catch field-name typos at vet time. The +// outermost #Config keeps an open `...` so that sibling files (e.g. +// mirror_extension.cue) can additively extend it via the extension +// pattern documented at the top of those files. +#Config: { + clusterName: string & !="" + namespace: string & !="" + outputDir: string & !="" + arch: "amd64" | "arm64" + bootMode: "netboot" | "isoboot" + + // externalTinkerbell + totalNodes are added by cue/state/state.cue. + externalTinkerbell?: bool + totalNodes?: int & >0 + + versions: close({ + kube: =~"^v[0-9]+\\.[0-9]+\\.[0-9]+$" + os: string | int + kubevip: string | number + capt: string & !="" + chart: string & !="" + }) + capt: close({ + providerRepository: string & !="" + }) + chart: close({ + // chart.location is injected by cue/state/mirror_extension.cue; + // it is always present in .state. + location: string & !="" + extraVars?: [...string] + }) + os: close({ + // os.registry is injected by cue/state/mirror_extension.cue. + registry: string & !="" + sshKey: string & !="" + version: string | int + }) + // tinkerbell + cluster are populated by state.cue once the kind + // gateway IP is known. Required here because every capi consumer + // references them unconditionally; cue/values is only vetted via + // .state generated AFTER the kind cluster is up (Taskfile-create's + // `render-state` task supplies -t gatewayIP). The partial .state + // produced by the root generate-state task is never vetted by capi. + tinkerbell: close({ + vip: string & !="" + hookosVip: string & !="" + }) + cluster: close({ + controlPlane: close({ + vip: string & !="" + }) + podCIDR?: string + }) + counts: close({ + controlPlanes: int & >=1 + workers: int & >=0 + spares: int & >=0 + }) + vm: close({ + baseName: string & !="" + cpusPerVM: int & >0 + memInMBPerVM: int & >0 + diskSizeInGBPerVM: int & >0 + diskPath: string & !="" + details: [string]: close({ + mac: string & !="" + role: "control-plane" | "worker" | "spare" + ip?: string & !="" + gateway?: string & !="" + bmc: close({ + port: int & >0 + }) + }) + }) + virtualBMC: close({ + // containerName + image come from config.yaml; ip is injected by + // tasks/Taskfile-vbmc.yaml after the vbmc container starts (its + // IP is only known then). Required here because cue/infra/bmc.cue + // references it unconditionally; cue/infra is only vetted via a + // .state regenerated AFTER vbmc:update-state has run. + containerName: string & !="" + image: string & !="" + ip: string & !="" + user: string & !="" + pass: string & !="" + }) + captainos?: close({ + kernelVersion: string & !="" + }) + // kind block is populated by state.cue. + kind?: close({ + kubeconfig: string & !="" + gatewayIP?: string & !="" + nodeIPBase?: string & !="" + bridgeName?: string & !="" + tinkerbell?: close({ + clusterName: string & !="" + kubeconfig: string & !="" + }) + }) + // registryMirror is added by cue/values/mirror_extension.cue. + ... +} + +// All values-derived names and snippets shared across subpackages. Public +// (non-underscore) fields so that consumers in other packages can reference +// them after instantiating: `c: values.#Computed & {values: values, mode: _mode}`. +#Computed: { + values: #Config + mode: "netboot" | "isoboot" + + // Replace dots in the kube version so it can suffix CR names without + // violating DNS subdomain rules. e.g. v1.35.2 -> v1-35-2. + dashedKubeVersion: strings.Replace(values.versions.kube, ".", "-", -1) + + // Map OCI arch names (amd64/arm64) to the Linux kernel / DHCP arch + // names used in Hardware CRs and HookOS ISO filenames. + dhcpArch: { + if values.arch == "amd64" {"x86_64"} + if values.arch == "arm64" {"aarch64"} + } + + // Normalize dotted Ubuntu versions (e.g. 20.04 -> 2004) so image + // tags match the published CAPT naming scheme. + normalizedOSVersion: strings.Replace("\(values.os.version)", ".", "", -1) + + clusterName: values.clusterName + namespace: values.namespace + kcpName: "\(clusterName)-control-plane" + mdName: "\(clusterName)-worker-a" + kctName: "\(clusterName)-worker-a" + tinkClusterName: clusterName + tmtCpName: "\(clusterName)-control-plane-\(dashedKubeVersion)" + tmtWorkerName: "\(clusterName)-worker-a-\(dashedKubeVersion)" + + // Shared list of cloud-init users injected into KCP and KCT. + users: [{ + name: "tink" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: [values.os.sshKey] + }] + + imgURL: "\(values.os.registry):\(normalizedOSVersion)-\(values.versions.kube)-\(values.arch).gz" + + metadataURL: "http://\(values.tinkerbell.vip):7080" + + // kube-vip preKubeadmCommand. Single string (one entry in preKubeadmCommands). + // k8s v1.29+ uses super-admin.conf; the if/else guards older clusters. + kubeVipCmd: strings.Join([ + "if [ $(cat /etc/kubernetes-version | awk -F. '{print $2}') -ge 29 ] && [ -f /run/kubeadm/kubeadm.yaml ]; then export KUBE_FILE=/etc/kubernetes/super-admin.conf; else export KUBE_FILE=/etc/kubernetes/admin.conf; fi", + "mkdir -p /etc/kubernetes/manifests", + "ctr images pull ghcr.io/kube-vip/kube-vip:v\(values.versions.kubevip)", + "ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:v\(values.versions.kubevip) vip /kube-vip manifest pod --arp --interface $(ip -4 -j route list default | jq -r .[0].dev) --address \(values.cluster.controlPlane.vip) --controlplane --leaderElection --k8sConfigPath $KUBE_FILE > /etc/kubernetes/manifests/kube-vip.yaml", + ], " && ") +} diff --git a/capt/cue/wiring/wiring.cue b/capt/cue/wiring/wiring.cue new file mode 100644 index 00000000..368fd731 --- /dev/null +++ b/capt/cue/wiring/wiring.cue @@ -0,0 +1,40 @@ +// Wiring sentinel for the optional registry-mirror feature. +// +// The feature is intentionally split across three locations so the whole +// thing can be removed by deleting the cue/mirror/ directory plus the two +// *_extension.cue files. That makes it easy to remove — and equally easy +// to half-remove. This file is a leaf package that imports all three +// pieces; `cue vet ./cue/...` fails loudly the moment one of them +// disappears, instead of silently rendering an empty mirror config. +// +// To remove the feature cleanly, delete (in any order): +// - cue/mirror/ (whole package) +// - cue/values/mirror_extension.cue +// - cue/state/mirror_extension.cue +// - cue/wiring/ (this whole package) +// +// This package is intentionally a leaf — nothing imports it — so adding +// the imports below cannot create an import cycle. +package wiring + +import ( + "tinkerbell.org/capt-playground/cue/mirror" + "tinkerbell.org/capt-playground/cue/state" + "tinkerbell.org/capt-playground/cue/values" +) + +// Each line will fail with a clear "undefined field" error if the +// corresponding piece of the mirror feature is removed without removing +// this wiring file. +_sentinel: { + // From cue/values/mirror_extension.cue: + _valuesHasRegistryMirror: values.#Config.registryMirror | *null + // From cue/state/mirror_extension.cue: + _stateHasRegistryMirror: state.#ConfigInput.registryMirror | *null + // From cue/mirror/schema.cue: + _mirrorSpec: mirror.#Spec + // From cue/mirror/files.cue: + _mirrorHostsToml: mirror.HostsTomlByUpstream + // From cue/mirror/rewrite.cue: + _mirrorRewrite: mirror.#rewrite +} diff --git a/capt/e2e/cue/base.cue b/capt/e2e/cue/base.cue new file mode 100644 index 00000000..0a497731 --- /dev/null +++ b/capt/e2e/cue/base.cue @@ -0,0 +1,63 @@ +// Shared test defaults for all matrix combos. +// Imports #ConfigInput from the capt CUE module directly — no schema duplication. +package e2e + +import ( + "tinkerbell.org/capt-playground/cue/state" +) + +_spares: int | *0 @tag(spares,type=int) + +base: state.#ConfigInput & { + clusterName: "e2e-test" + outputDir: "output" + namespace: "tinkerbell" + arch: "amd64" + + counts: { + controlPlanes: 1 + workers: 1 + spares: _spares + } + + versions: { + capt: "v0.7.0" + chart: "v0.23.1-23da0880" + kube: "v1.35.2" + os: 2404 + kubevip: "1.1.2" + } + + capt: providerRepository: "https://github.com/tinkerbell/cluster-api-provider-tinkerbell/releases" + + chart: { + location: "oci://ghcr.io/tinkerbell/charts/tinkerbell" + extraVars: [ + "optional.captainos.enabled=true", + "optional.captainos.image=ghcr.io/tinkerbell/captain/artifacts:v0.0.0-a4be23c", + "deployment.envs.ui.enableAutoLogin=true", + ] + } + + os: registry: "ghcr.io/tinkerbell/cluster-api-provider-tinkerbell/ubuntu" + + vm: { + baseName: "node" + cpusPerVM: 2 + memInMBPerVM: 2048 + diskSizeInGBPerVM: 4 + diskPath: "/tmp" + } + + virtualBMC: { + containerName: "virtualbmc" + image: "ghcr.io/jacobweinstock/virtualbmc:latest" + user: "root" + pass: "calvin" + } + + captainos: kernelVersion: "6.18.16" + + // registryMirror is set per-combo in matrix.cue (enabled or disabled). + // Not defaulted here to avoid CUE unification conflicts. +} diff --git a/capt/e2e/cue/matrix.cue b/capt/e2e/cue/matrix.cue new file mode 100644 index 00000000..306d63a4 --- /dev/null +++ b/capt/e2e/cue/matrix.cue @@ -0,0 +1,93 @@ +// Test matrix: 3 binary axes → 8 combos. +// +// Axes: +// 1. Cluster topology: single (co-located) vs external Tinkerbell +// 2. Registry mirrors: disabled vs enabled (pull-through cache) +// 3. Boot method: netboot (iPXE) vs isoboot (ISO) +// +// Usage (from the capt/ directory): +// # List all combo names +// cue eval ./e2e/cue -e comboNames --out json +// +// # Render a single combo as config.yaml +// cue export ./e2e/cue -e 'combos["single-nomirror-netboot"]' --out yaml +// +// # Render all combos (struct keyed by name) +// cue export ./e2e/cue -e combos -t mirrorHost=reg.example.com --out yaml +package e2e + +import ( + "list" + "tinkerbell.org/capt-playground/cue/mirror" +) + +_mirrorHost: string | *"" @tag(mirrorHost) + +// Mirror config used by all mirror-enabled combos. +_mirrorConfig: mirror.#Spec & { + enabled: true + host: _mirrorHost + upstreams: [ + "ghcr.io", + "quay.io", + "registry.k8s.io", + "gcr.io", + "docker.io", + ] +} + +// Disabled mirror config for non-mirror combos. +_noMirror: mirror.#Spec & { + enabled: false +} + +// Per-combo overrides. Each is merged with `base` to produce a full config. +_overrides: { + "single-nomirror-netboot": { + bootMode: "netboot" + externalTinkerbell: false + registryMirror: _noMirror + } + "single-nomirror-isoboot": { + bootMode: "isoboot" + externalTinkerbell: false + registryMirror: _noMirror + } + "single-mirror-netboot": { + bootMode: "netboot" + externalTinkerbell: false + registryMirror: _mirrorConfig + } + "single-mirror-isoboot": { + bootMode: "isoboot" + externalTinkerbell: false + registryMirror: _mirrorConfig + } + "external-nomirror-netboot": { + bootMode: "netboot" + externalTinkerbell: true + registryMirror: _noMirror + } + "external-nomirror-isoboot": { + bootMode: "isoboot" + externalTinkerbell: true + registryMirror: _noMirror + } + "external-mirror-netboot": { + bootMode: "netboot" + externalTinkerbell: true + registryMirror: _mirrorConfig + } + "external-mirror-isoboot": { + bootMode: "isoboot" + externalTinkerbell: true + registryMirror: _mirrorConfig + } +} + +// All combos: base merged with per-combo overrides. Each value is a +// complete config ready to be written as config.yaml for capt. +combos: {for name, ovr in _overrides {(name): base & ovr}} + +// Sorted list of combo names for scripting. +comboNames: list.SortStrings([for name, _ in _overrides {name}]) diff --git a/capt/e2e/run.sh b/capt/e2e/run.sh new file mode 100755 index 00000000..6572ecdf --- /dev/null +++ b/capt/e2e/run.sh @@ -0,0 +1,342 @@ +#!/usr/bin/env bash +# Matrix test orchestrator for the CAPT playground. +# +# Loops over user-selected combos, generates config.yaml via CUE, +# creates/validates/deletes the playground for each combo, and prints +# a summary table. +# +# Usage: +# ./e2e/run.sh [FLAGS] [COMBO ...| all] +# ./e2e/run.sh --cleanup +# +# Flags: +# --list List available combos and exit +# --cleanup Delete any active playground and restore config.yaml +# --labels FILTER Ginkgo --label-filter (default: "provisioning") +# --artifacts DIR Artifact directory (default: e2e/artifacts) +# --spares N Override spare VM count (default: 0) +# --mirror-host HOST Registry mirror hostname (required for mirror combos) +# --no-teardown Keep the playground running after tests complete +# --dry-run Render configs and print what would run, but skip task/ginkgo +# +# Examples: +# ./e2e/run.sh --list +# ./e2e/run.sh single-nomirror-netboot +# ./e2e/run.sh --labels "provisioning" --mirror-host reg.example.com all + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CAPT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +E2E_DIR="$SCRIPT_DIR" +BIN_DIR="$CAPT_DIR/bin" + +# Ginkgo CLI — installed to the same bin/ as other tools +GINKGO_VERSION="v2.28.1" +GINKGO="$BIN_DIR/ginkgo" + +ensure_ginkgo() { + local versioned="$BIN_DIR/ginkgo-$GINKGO_VERSION" + if [[ ! -x $versioned ]]; then + echo "Installing ginkgo $GINKGO_VERSION to $BIN_DIR" + GOBIN="$BIN_DIR" go install "github.com/onsi/ginkgo/v2/ginkgo@$GINKGO_VERSION" + mv "$BIN_DIR/ginkgo" "$versioned" + fi + ln -sf "ginkgo-$GINKGO_VERSION" "$GINKGO" +} + +# Defaults +LABELS="provisioning" +ARTIFACTS_DIR="$E2E_DIR/artifacts" +SPARES=0 +MIRROR_HOST="" +DRY_RUN=false +NO_TEARDOWN=false +COMBOS=() + +# --- Argument parsing --- + +while [[ $# -gt 0 ]]; do + case "$1" in + -h | --help) + cat <<'EOF' +Usage: ./e2e/run.sh [FLAGS] [COMBO ...| all] + ./e2e/run.sh --cleanup + ./e2e/run.sh --list + +Flags: + -h, --help Show this help and exit + --list List available combos and exit + --cleanup Delete any active playground and restore config.yaml + --labels FILTER Ginkgo --label-filter (default: "provisioning") + --artifacts DIR Artifact directory (default: e2e/artifacts) + --spares N Override spare VM count (default: 0) + --mirror-host HOST Registry mirror hostname (required for mirror combos) + --no-teardown Keep the playground running after tests complete + --dry-run Render configs and print what would run, but skip task/ginkgo + +Examples: + ./e2e/run.sh --list + ./e2e/run.sh single-nomirror-netboot + ./e2e/run.sh --no-teardown single-nomirror-netboot + ./e2e/run.sh --labels "provisioning" --mirror-host reg.example.com all + ./e2e/run.sh --cleanup +EOF + exit 0 + ;; + --list) + cd "$CAPT_DIR" + cue eval ./e2e/cue -e comboNames --out json | jq -r '.[]' + exit 0 + ;; + --cleanup) + echo "Cleaning up playground and restoring config..." + cd "$CAPT_DIR" + if ! task delete-playground 2>&1; then + echo "WARN: 'task delete-playground' exited non-zero ($?); continuing cleanup." >&2 + fi + if [[ -f "$CAPT_DIR/config.yaml.e2e-backup" ]]; then + cp "$CAPT_DIR/config.yaml.e2e-backup" "$CAPT_DIR/config.yaml" + rm -f "$CAPT_DIR/config.yaml.e2e-backup" + echo "Restored config.yaml from backup." + fi + echo "Cleanup complete." + exit 0 + ;; + --labels) + LABELS="$2" + shift 2 + ;; + --artifacts) + ARTIFACTS_DIR="$2" + shift 2 + ;; + --spares) + SPARES="$2" + shift 2 + ;; + --mirror-host) + MIRROR_HOST="$2" + shift 2 + ;; + --dry-run) + DRY_RUN=true + shift + ;; + --no-teardown) + NO_TEARDOWN=true + shift + ;; + -*) + echo "Unknown flag: $1" >&2 + exit 1 + ;; + all) + mapfile -t COMBOS < <(cd "$CAPT_DIR" && cue eval ./e2e/cue -e comboNames --out json | jq -r '.[]') + shift + ;; + *) + COMBOS+=("$1") + shift + ;; + esac +done + +if [[ ${#COMBOS[@]} -eq 0 ]]; then + echo "Usage: $0 [FLAGS] [COMBO ...| all]" >&2 + echo "Run '$0 --list' to see available combos." >&2 + exit 1 +fi + +# --- Setup --- + +mkdir -p "$ARTIFACTS_DIR" +ensure_ginkgo + +# Back up original config.yaml to a known location so --cleanup can restore it +if [[ -f "$CAPT_DIR/config.yaml" && ! -f "$CAPT_DIR/config.yaml.e2e-backup" ]]; then + cp "$CAPT_DIR/config.yaml" "$CAPT_DIR/config.yaml.e2e-backup" +fi + +# --- Results tracking --- + +declare -a RESULT_NAMES=() +declare -a RESULT_STATUS=() +declare -a RESULT_DURATION=() +declare -a RESULT_DETAIL=() + +# --- Run combos --- + +for COMBO in "${COMBOS[@]}"; do + echo "" + echo "================================================================" + echo " COMBO: $COMBO" + echo "================================================================" + echo "" + + COMBO_ARTIFACTS="$ARTIFACTS_DIR/$COMBO" + mkdir -p "$COMBO_ARTIFACTS" + COMBO_START=$(date +%s) + + # --- Step 1: Render config.yaml from CUE --- + + CUE_TAGS=("-t" "spares=$SPARES") + if [[ -n $MIRROR_HOST ]]; then + CUE_TAGS+=("-t" "mirrorHost=$MIRROR_HOST") + fi + + echo "Rendering config for combo: $COMBO" + cd "$CAPT_DIR" + if ! cue export ./e2e/cue -e "combos[\"$COMBO\"]" "${CUE_TAGS[@]}" --out yaml >"$COMBO_ARTIFACTS/config.yaml" 2>&1; then + echo "FAIL: CUE render failed for $COMBO" >&2 + cat "$COMBO_ARTIFACTS/config.yaml" >&2 + RESULT_NAMES+=("$COMBO") + RESULT_STATUS+=("FAIL") + RESULT_DURATION+=("$(($(date +%s) - COMBO_START))s") + RESULT_DETAIL+=("CUE render failed") + continue + fi + + cp "$COMBO_ARTIFACTS/config.yaml" "$CAPT_DIR/config.yaml" + + if $DRY_RUN; then + echo "[dry-run] Would run: task --yes create-playground" + echo "[dry-run] Config:" + cat "$CAPT_DIR/config.yaml" + RESULT_NAMES+=("$COMBO") + RESULT_STATUS+=("DRY-RUN") + RESULT_DURATION+=("0s") + RESULT_DETAIL+=("-") + continue + fi + + # --- Step 2: Create playground --- + + COMBO_STATUS="FAIL" + COMBO_DETAIL="" + + create_ok=false + echo "Creating playground for combo: $COMBO" + if task --yes create-playground >"$COMBO_ARTIFACTS/create.log" 2>&1; then + create_ok=true + else + COMBO_DETAIL="playground creation failed" + cp "$COMBO_ARTIFACTS/create.log" "$COMBO_ARTIFACTS/create-error.log" + fi + + if $create_ok; then + # --- Step 3: Extract kubeconfigs from .state --- + + STATE_FILE="$CAPT_DIR/.state" + MGMT_KC="$(yq eval '.kind.kubeconfig' "$STATE_FILE")" + TINK_KC="" + if [[ "$(yq eval '.externalTinkerbell' "$STATE_FILE")" == "true" ]]; then + TINK_KC="$(yq eval '.kind.tinkerbell.kubeconfig' "$STATE_FILE")" + fi + CLUSTER_NAME="$(yq eval '.clusterName' "$STATE_FILE")" + OUTPUT_DIR="$(yq eval '.outputDir' "$STATE_FILE")" + WORKLOAD_KC="$OUTPUT_DIR/$CLUSTER_NAME.kubeconfig" + NS="$(yq eval '.namespace' "$STATE_FILE")" + + # --- Step 4: Wait for workload kubeconfig --- + + echo "Waiting for workload kubeconfig: $WORKLOAD_KC" + wk_timeout=300 + wk_elapsed=0 + while [[ ! -f $WORKLOAD_KC ]] && ((wk_elapsed < wk_timeout)); do + sleep 5 + wk_elapsed=$((wk_elapsed + 5)) + done + + if [[ ! -f $WORKLOAD_KC ]]; then + COMBO_DETAIL="workload kubeconfig not found after ${wk_timeout}s" + else + # --- Step 5: Run Ginkgo tests --- + + echo "Running Ginkgo tests (labels: $LABELS)" + GINKGO_ARGS=( + -v + --label-filter="$LABELS" + --output-dir="$COMBO_ARTIFACTS" + --junit-report=junit.xml + ) + + E2E_FLAGS=( + -- + -e2e.config="$E2E_DIR/test/config/e2e.yaml" + -e2e.artifacts="$COMBO_ARTIFACTS" + -e2e.mgmt-kubeconfig="$MGMT_KC" + -e2e.workload-kubeconfig="$WORKLOAD_KC" + -e2e.namespace="$NS" + ) + if [[ -n $TINK_KC ]]; then + E2E_FLAGS+=(-e2e.tink-kubeconfig="$TINK_KC") + fi + + if (cd "$E2E_DIR/test" && "$GINKGO" "${GINKGO_ARGS[@]}" ./... "${E2E_FLAGS[@]}") >"$COMBO_ARTIFACTS/ginkgo.log" 2>&1; then + COMBO_STATUS="PASS" + COMBO_DETAIL="" + else + COMBO_DETAIL="ginkgo tests failed" + fi + fi + fi + + # --- Step 7: Delete playground (unless --no-teardown) --- + + if $NO_TEARDOWN; then + echo "" + echo "################################################################" + echo "# --no-teardown: leaving playground RUNNING for combo: $COMBO" + echo "# Resources (VMs, kind clusters, BMC container) will persist." + echo "# Clean up manually with: ./e2e/run.sh --cleanup" + echo "################################################################" + else + echo "Deleting playground for combo: $COMBO" + if ! task delete-playground >"$COMBO_ARTIFACTS/delete.log" 2>&1; then + rc=$? + echo "WARN: teardown failed for $COMBO (exit $rc); see $COMBO_ARTIFACTS/delete.log" >&2 + COMBO_DETAIL="${COMBO_DETAIL:+$COMBO_DETAIL; }teardown failed (exit $rc)" + fi + fi + + # --- Record result --- + + COMBO_END=$(date +%s) + COMBO_ELAPSED="$((COMBO_END - COMBO_START))s" + RESULT_NAMES+=("$COMBO") + RESULT_STATUS+=("$COMBO_STATUS") + RESULT_DURATION+=("$COMBO_ELAPSED") + RESULT_DETAIL+=("${COMBO_DETAIL:-OK}") +done + +# --- Summary --- + +echo "" +echo "================================================================" +echo " MATRIX RESULTS" +echo "================================================================" +echo "" +printf "%-35s %-8s %-10s %s\n" "COMBO" "RESULT" "DURATION" "DETAIL" +printf "%-35s %-8s %-10s %s\n" "-----" "------" "--------" "------" + +EXIT_CODE=0 +for i in "${!RESULT_NAMES[@]}"; do + printf "%-35s %-8s %-10s %s\n" \ + "${RESULT_NAMES[$i]}" \ + "${RESULT_STATUS[$i]}" \ + "${RESULT_DURATION[$i]}" \ + "${RESULT_DETAIL[$i]}" + if [[ ${RESULT_STATUS[$i]} == "FAIL" ]]; then + EXIT_CODE=1 + fi +done + +# Restore original config.yaml and remove backup after a full run +if [[ -f "$CAPT_DIR/config.yaml.e2e-backup" ]]; then + cp "$CAPT_DIR/config.yaml.e2e-backup" "$CAPT_DIR/config.yaml" + rm -f "$CAPT_DIR/config.yaml.e2e-backup" +fi + +echo "" +exit "$EXIT_CODE" diff --git a/capt/e2e/test/config/e2e.yaml b/capt/e2e/test/config/e2e.yaml new file mode 100644 index 00000000..44c49eb4 --- /dev/null +++ b/capt/e2e/test/config/e2e.yaml @@ -0,0 +1,26 @@ +# CAPI E2E test configuration for CAPT playground matrix testing. +# Intervals define timeout and polling periods for test operations. +# Format: ["timeout", "interval"] — both as Go duration strings. + +intervals: + # Provisioning + default/wait-workflow: ["25m", "10s"] + default/wait-nodes: ["10m", "10s"] + + # Scale (future) + default/wait-scale: ["15m", "30s"] + + # Upgrades (future) + default/wait-upgrade: ["30m", "30s"] + + # Deletion (future) + default/wait-delete: ["10m", "10s"] + + # Pivot (future) + default/wait-pivot: ["15m", "30s"] + +variables: + KUBERNETES_VERSION: "v1.35.2" + KUBERNETES_VERSION_UPGRADE_TO: "v1.35.3" + CONTROL_PLANE_MACHINE_COUNT: "1" + WORKER_MACHINE_COUNT: "1" diff --git a/capt/e2e/test/e2e_suite_test.go b/capt/e2e/test/e2e_suite_test.go new file mode 100644 index 00000000..431a4567 --- /dev/null +++ b/capt/e2e/test/e2e_suite_test.go @@ -0,0 +1,141 @@ +// Package e2e provides end-to-end tests for CAPT playground provisioning. +package e2e + +import ( + "flag" + "fmt" + "os" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + tinkv1 "github.com/tinkerbell/tinkerbell/api/v1alpha1/tinkerbell" + "gopkg.in/yaml.v3" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +// E2EConfig holds test intervals and variables loaded from the e2e config YAML. +type E2EConfig struct { + Intervals map[string][]string `yaml:"intervals"` + Variables map[string]string `yaml:"variables"` +} + +// GetInterval returns parsed timeout and polling durations for a key. +// Returns zero values if the key is missing or unparseable. +func (c *E2EConfig) GetInterval(key string) (timeout, interval time.Duration) { + if c == nil { + return 0, 0 + } + vals, ok := c.Intervals[key] + if !ok || len(vals) < 2 { + return 0, 0 + } + t, err1 := time.ParseDuration(vals[0]) + i, err2 := time.ParseDuration(vals[1]) + if err1 != nil || err2 != nil { + return 0, 0 + } + return t, i +} + +var ( + e2eConfigPath string + artifactsDir string + + mgmtKubeconfig string + tinkKubeconfig string + workloadKubeconfig string + namespace string + + e2eConfig *E2EConfig + + mgmtClient client.Client + tinkClient client.Client + workloadClient client.Client +) + +func init() { + flag.StringVar(&e2eConfigPath, "e2e.config", "", "Path to the E2E config YAML") + flag.StringVar(&artifactsDir, "e2e.artifacts", os.TempDir(), "Directory for test artifacts") + flag.StringVar(&mgmtKubeconfig, "e2e.mgmt-kubeconfig", os.Getenv("E2E_MGMT_KUBECONFIG"), "Management cluster kubeconfig") + flag.StringVar(&tinkKubeconfig, "e2e.tink-kubeconfig", os.Getenv("E2E_TINK_KUBECONFIG"), "Tinkerbell cluster kubeconfig (external mode)") + flag.StringVar(&workloadKubeconfig, "e2e.workload-kubeconfig", os.Getenv("E2E_WORKLOAD_KUBECONFIG"), "Workload cluster kubeconfig") + flag.StringVar(&namespace, "e2e.namespace", os.Getenv("E2E_NAMESPACE"), "Namespace for Tinkerbell resources") +} + +func TestE2E(t *testing.T) { + if mgmtKubeconfig == "" { + t.Skip("set E2E_MGMT_KUBECONFIG or pass -e2e.mgmt-kubeconfig to run E2E tests") + } + RegisterFailHandler(Fail) + RunSpecs(t, "CAPT Playground E2E Suite") +} + +func newScheme() *runtime.Scheme { + s := runtime.NewScheme() + Expect(clientgoscheme.AddToScheme(s)).To(Succeed()) + + // Register Tinkerbell v1alpha1 types (Workflow, Hardware, Template, etc.) + sb := &scheme.Builder{GroupVersion: schema.GroupVersion{Group: "tinkerbell.org", Version: "v1alpha1"}} + sb.Register(&tinkv1.Workflow{}, &tinkv1.WorkflowList{}) + sb.Register(&tinkv1.Hardware{}, &tinkv1.HardwareList{}) + sb.Register(&tinkv1.Template{}, &tinkv1.TemplateList{}) + Expect(sb.AddToScheme(s)).To(Succeed()) + + return s +} + +func clientFromKubeconfig(kubeconfigPath string, s *runtime.Scheme) (client.Client, error) { + cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + if err != nil { + return nil, fmt.Errorf("building config from %s: %w", kubeconfigPath, err) + } + return client.New(cfg, client.Options{Scheme: s}) +} + +var _ = SynchronizedBeforeSuite(func() []byte { + Expect(mgmtKubeconfig).ToNot(BeEmpty(), "management kubeconfig is required (--e2e.mgmt-kubeconfig or E2E_MGMT_KUBECONFIG)") + Expect(workloadKubeconfig).ToNot(BeEmpty(), "workload kubeconfig is required (--e2e.workload-kubeconfig or E2E_WORKLOAD_KUBECONFIG)") + + if namespace == "" { + namespace = "tinkerbell" + } + + if e2eConfigPath != "" { + data, err := os.ReadFile(e2eConfigPath) + Expect(err).ToNot(HaveOccurred(), "failed to read e2e config %s", e2eConfigPath) + e2eConfig = &E2EConfig{} + Expect(yaml.Unmarshal(data, e2eConfig)).To(Succeed(), "failed to parse e2e config") + } + + s := newScheme() + + var err error + mgmtClient, err = clientFromKubeconfig(mgmtKubeconfig, s) + Expect(err).ToNot(HaveOccurred(), "failed to create management cluster client") + + // Tinkerbell client: use separate kubeconfig in external mode, else same as mgmt. + if tinkKubeconfig != "" { + tinkClient, err = clientFromKubeconfig(tinkKubeconfig, s) + Expect(err).ToNot(HaveOccurred(), "failed to create tinkerbell cluster client") + } else { + tinkClient = mgmtClient + } + + workloadClient, err = clientFromKubeconfig(workloadKubeconfig, s) + Expect(err).ToNot(HaveOccurred(), "failed to create workload cluster client") + + return nil +}, func(_ []byte) {}) + +var _ = SynchronizedAfterSuite(func() {}, func() { + if artifactsDir != "" { + By(fmt.Sprintf("Test artifacts saved to %s", artifactsDir)) + } +}) diff --git a/capt/e2e/test/go.mod b/capt/e2e/test/go.mod new file mode 100644 index 00000000..1fde29ac --- /dev/null +++ b/capt/e2e/test/go.mod @@ -0,0 +1,61 @@ +module github.com/tinkerbell/playground/e2e + +go 1.26.1 + +require ( + github.com/onsi/ginkgo/v2 v2.28.1 + github.com/onsi/gomega v1.39.1 + github.com/tinkerbell/tinkerbell/api v0.23.0 + gopkg.in/yaml.v3 v3.0.1 + k8s.io/api v0.35.4 + k8s.io/apimachinery v0.35.4 + k8s.io/client-go v0.35.4 + sigs.k8s.io/controller-runtime v0.23.3 +) + +require ( + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.uber.org/zap v1.27.1 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/mod v0.34.0 // indirect + golang.org/x/net v0.53.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/term v0.42.0 // indirect + golang.org/x/text v0.36.0 // indirect + golang.org/x/time v0.11.0 // indirect + golang.org/x/tools v0.43.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/apiextensions-apiserver v0.35.4 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.4.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) diff --git a/capt/e2e/test/go.sum b/capt/e2e/test/go.sum new file mode 100644 index 00000000..959d0907 --- /dev/null +++ b/capt/e2e/test/go.sum @@ -0,0 +1,178 @@ +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= +github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= +github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= +github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/tinkerbell/tinkerbell/api v0.23.0 h1:Au7qacPsSViZtPYU4Q0H41UhtUQYmrZse549vDpNI4M= +github.com/tinkerbell/tinkerbell/api v0.23.0/go.mod h1:S1DVHOyJ3FjthCv9iDURC3rVrRgh4jOLbohocJUbxMU= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= +golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= +golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.35.4 h1:P7nFYKl5vo9AGUp1Z+Pmd3p2tA7bX2wbFWCvDeRv988= +k8s.io/api v0.35.4/go.mod h1:yl4lqySWOgYJJf9RERXKUwE9g2y+CkuwG+xmcOK8wXU= +k8s.io/apiextensions-apiserver v0.35.4 h1:HeP+Upp7ItdvnyGmub0yoix+2z5+ev4M5cE5TCgtOUU= +k8s.io/apiextensions-apiserver v0.35.4/go.mod h1:ogQlk+stIE8mnoRthSYCwlOS12fVqgWFiErMwPaXA7c= +k8s.io/apimachinery v0.35.4 h1:xtdom9RG7e+yDp71uoXoJDWEE2eOiHgeO4GdBzwWpds= +k8s.io/apimachinery v0.35.4/go.mod h1:NNi1taPOpep0jOj+oRha3mBJPqvi0hGdaV8TCqGQ+cc= +k8s.io/client-go v0.35.4 h1:DN6fyaGuzK64UvnKO5fOA6ymSjvfGAnCAHAR0C66kD8= +k8s.io/client-go v0.35.4/go.mod h1:2Pg9WpsS4NeOpoYTfHHfMxBG8zFMSAUi4O/qoiJC3nY= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.23.3 h1:VjB/vhoPoA9l1kEKZHBMnQF33tdCLQKJtydy4iqwZ80= +sigs.k8s.io/controller-runtime v0.23.3/go.mod h1:B6COOxKptp+YaUT5q4l6LqUJTRpizbgf9KSRNdQGns0= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.4.0 h1:qmp2e3ZfFi1/jJbDGpD4mt3wyp6PE1NfKHCYLqgNQJo= +sigs.k8s.io/structured-merge-diff/v6 v6.4.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/capt/e2e/test/helpers.go b/capt/e2e/test/helpers.go new file mode 100644 index 00000000..9f9498bc --- /dev/null +++ b/capt/e2e/test/helpers.go @@ -0,0 +1,117 @@ +package e2e + +import ( + "context" + "fmt" + "os/exec" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + tinkv1 "github.com/tinkerbell/tinkerbell/api/v1alpha1/tinkerbell" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// WaitForWorkflowsSuccess polls Workflow CRs until all reach SUCCESS state. +// Fails fast if any workflow reaches FAILED or TIMEOUT. +func WaitForWorkflowsSuccess(ctx context.Context, c client.Client, ns string, expectedCount int, timeout, interval time.Duration) { + By(fmt.Sprintf("Waiting for %d workflow(s) in %s to reach SUCCESS (timeout %s)", expectedCount, ns, timeout)) + + Eventually(func(g Gomega) { + wfList := &tinkv1.WorkflowList{} + g.Expect(c.List(ctx, wfList, client.InNamespace(ns))).To(Succeed()) + + // Workflows may not exist yet while CAPT reconciles. Keep polling. + g.Expect(len(wfList.Items)).To(BeNumerically(">=", expectedCount), + "waiting for %d workflows, currently %d", expectedCount, len(wfList.Items)) + + for _, wf := range wfList.Items { + state := wf.Status.State + GinkgoWriter.Printf(" workflow/%s state=%s\n", wf.Name, state) + g.Expect(state).ToNot(Equal(tinkv1.WorkflowStateFailed), + "workflow %s failed", wf.Name) + g.Expect(state).ToNot(Equal(tinkv1.WorkflowStateTimeout), + "workflow %s timed out", wf.Name) + g.Expect(state).To(Equal(tinkv1.WorkflowStateSuccess), + "workflow %s in state %s, want SUCCESS", wf.Name, state) + } + }).WithTimeout(timeout).WithPolling(interval).Should(Succeed()) +} + +// WaitForAllNodesReady waits for the expected number of nodes to be Ready +// on the workload cluster. +func WaitForAllNodesReady(ctx context.Context, c client.Client, expectedCount int, timeout, interval time.Duration) { + By(fmt.Sprintf("Waiting for %d node(s) to be Ready (timeout %s)", expectedCount, timeout)) + + Eventually(func(g Gomega) { + nodeList := &corev1.NodeList{} + g.Expect(c.List(ctx, nodeList)).To(Succeed()) + g.Expect(nodeList.Items).To(HaveLen(expectedCount), "expected %d nodes, got %d", expectedCount, len(nodeList.Items)) + + for _, node := range nodeList.Items { + ready := false + for _, cond := range node.Status.Conditions { + if cond.Type == corev1.NodeReady && cond.Status == corev1.ConditionTrue { + ready = true + break + } + } + g.Expect(ready).To(BeTrue(), "node %s is not Ready", node.Name) + } + }).WithTimeout(timeout).WithPolling(interval).Should(Succeed()) +} + +// WaitForAPIServerReady polls the workload cluster API server until it responds. +// The provided ctx bounds each kubectl invocation so a hung connection cannot +// outlive the spec. +func WaitForAPIServerReady(ctx context.Context, kubeconfigPath string, timeout, interval time.Duration) { + By(fmt.Sprintf("Waiting for workload API server to be reachable (timeout %s)", timeout)) + + Eventually(func(g Gomega) { + cmd := exec.CommandContext(ctx, + "kubectl", "--kubeconfig", kubeconfigPath, + "get", "--raw", "/readyz", + ) + out, err := cmd.CombinedOutput() + g.Expect(err).ToNot(HaveOccurred(), "API server not ready: %s", string(out)) + }).WithTimeout(timeout).WithPolling(interval).Should(Succeed()) +} + +// DeployCNI applies kube-router to the workload cluster so nodes can reach Ready. +func DeployCNI(ctx context.Context, kubeconfigPath string) { + By("Deploying kube-router CNI") + cmd := exec.CommandContext(ctx, + "kubectl", "--kubeconfig", kubeconfigPath, + "apply", "-f", "https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/kubeadm-kuberouter.yaml", + ) + out, err := cmd.CombinedOutput() + if err != nil { + GinkgoWriter.Printf("CNI apply output:\n%s\n", string(out)) + } + Expect(err).ToNot(HaveOccurred(), "failed to deploy kube-router CNI") +} + +// ListWorkflows prints all workflows in the namespace for debugging. +func ListWorkflows(ctx context.Context, c client.Client, ns string) { + wfList := &tinkv1.WorkflowList{} + if err := c.List(ctx, wfList, client.InNamespace(ns)); err != nil { + GinkgoWriter.Printf("Failed to list workflows: %v\n", err) + return + } + for _, wf := range wfList.Items { + GinkgoWriter.Printf(" workflow/%s state=%s\n", wf.Name, wf.Status.State) + } +} + +// ListHardware prints all hardware in the namespace for debugging. +func ListHardware(ctx context.Context, c client.Client, ns string) { + hwList := &tinkv1.HardwareList{} + if err := c.List(ctx, hwList, client.InNamespace(ns)); err != nil { + GinkgoWriter.Printf("Failed to list hardware: %v\n", err) + return + } + for _, hw := range hwList.Items { + GinkgoWriter.Printf(" hardware/%s\n", hw.Name) + } +} diff --git a/capt/e2e/test/provisioning_test.go b/capt/e2e/test/provisioning_test.go new file mode 100644 index 00000000..989fbe81 --- /dev/null +++ b/capt/e2e/test/provisioning_test.go @@ -0,0 +1,51 @@ +package e2e + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" +) + +var _ = Describe("Workload cluster provisioning", Label("provisioning"), Ordered, func() { + var ( + workflowTimeout = 25 * time.Minute + workflowInterval = 10 * time.Second + nodeTimeout = 10 * time.Minute + nodeInterval = 10 * time.Second + expectedNodes = 2 // 1 control plane + 1 worker + ) + + // Override from E2E config if loaded. + BeforeAll(func() { + if t, i := e2eConfig.GetInterval("default/wait-workflow"); t > 0 { + workflowTimeout = t + workflowInterval = i + } + if t, i := e2eConfig.GetInterval("default/wait-nodes"); t > 0 { + nodeTimeout = t + nodeInterval = i + } + }) + + AfterAll(func(ctx SpecContext) { + By("Dumping workflows and hardware for debugging") + ListWorkflows(ctx, tinkClient, namespace) + ListHardware(ctx, tinkClient, namespace) + }) + + It("completes all workflows successfully", func(ctx SpecContext) { + WaitForWorkflowsSuccess(ctx, tinkClient, namespace, expectedNodes, workflowTimeout, workflowInterval) + }, SpecTimeout(30*time.Minute)) + + It("has a reachable workload API server", func(ctx SpecContext) { + WaitForAPIServerReady(ctx, workloadKubeconfig, nodeTimeout, 10*time.Second) + }, SpecTimeout(15*time.Minute)) + + It("deploys CNI to the workload cluster", func(ctx SpecContext) { + DeployCNI(ctx, workloadKubeconfig) + }, SpecTimeout(5*time.Minute)) + + It("has all nodes Ready after CNI deployment", func(ctx SpecContext) { + WaitForAllNodesReady(ctx, workloadClient, expectedNodes, nodeTimeout, nodeInterval) + }, SpecTimeout(15*time.Minute)) +}) diff --git a/capt/scripts/create_external_kubeconfig_secret.sh b/capt/scripts/create_external_kubeconfig_secret.sh new file mode 100755 index 00000000..aee69f57 --- /dev/null +++ b/capt/scripts/create_external_kubeconfig_secret.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Create the external-tinkerbell-kubeconfig secret in the management cluster +# so CAPT can connect to the Tinkerbell cluster. +# +# Usage: create_external_kubeconfig_secret.sh + +set -euo pipefail + +TINK_CLUSTER_NAME="${1}" +MGMT_KUBECONFIG="${2}" + +# Get the Tinkerbell KinD cluster's container IP (reachable from pods on the shared Docker "kind" network) +TINK_CONTAINER_IP=$(docker inspect -f '{{.NetworkSettings.Networks.kind.IPAddress}}' "${TINK_CLUSTER_NAME}-control-plane") + +# Get the kubeconfig and patch the server URL to use the container IP +TINK_KUBECONFIG=$(kind get kubeconfig --name "${TINK_CLUSTER_NAME}") +TINK_KUBECONFIG=$(echo "$TINK_KUBECONFIG" | sed "s|server: https://.*|server: https://${TINK_CONTAINER_IP}:6443|g") + +# Ensure the capt-system namespace exists in the management cluster +KUBECONFIG="${MGMT_KUBECONFIG}" kubectl create namespace capt-system --dry-run=client -o yaml | KUBECONFIG="${MGMT_KUBECONFIG}" kubectl apply -f - + +# Create the secret with key "kubeconfig" (matches CAPT's mount path /var/run/secrets/external-tinkerbell/kubeconfig) +KUBECONFIG="${MGMT_KUBECONFIG}" kubectl create secret generic external-tinkerbell-kubeconfig \ + --namespace capt-system \ + --from-literal=kubeconfig="$TINK_KUBECONFIG" \ + --dry-run=client -o yaml | KUBECONFIG="${MGMT_KUBECONFIG}" kubectl apply -f - + +# Label the secret so clusterctl move includes it during a CAPI pivot. +KUBECONFIG="${MGMT_KUBECONFIG}" kubectl label secret external-tinkerbell-kubeconfig \ + --namespace capt-system \ + clusterctl.cluster.x-k8s.io/move="" \ + clusterctl.cluster.x-k8s.io="" \ + --overwrite diff --git a/capt/scripts/generate_bmc.sh b/capt/scripts/generate_bmc.sh deleted file mode 100755 index 9a2a2f66..00000000 --- a/capt/scripts/generate_bmc.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# This script creates the BMC machine yaml files needed for the CAPT playground. - -function main() { - declare -r STATE_FILE="$1" - declare -r OUTPUT_DIR=$(yq eval '.outputDir' "$STATE_FILE") - - rm -f "$OUTPUT_DIR"/bmc-machine*.yaml - - namespace=$(yq eval '.namespace' "$STATE_FILE") - bmc_ip=$(yq eval '.virtualBMC.ip' "$STATE_FILE") - - while IFS=$',' read -r name port; do - export NODE_NAME="$name" - export BMC_IP="$bmc_ip" - export BMC_PORT="$port" - export NAMESPACE="$namespace" - envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1))" "$OUTPUT_DIR"/bmc-machine-"$NODE_NAME".yaml - unset NODE_NAME - unset BMC_IP - unset BMC_PORT - unset NAMESPACE - done < <(yq e '.vm.details.[] | [key, .bmc.port] | @csv' "$STATE_FILE") -} - -main "$@" diff --git a/capt/scripts/generate_hardware.sh b/capt/scripts/generate_hardware.sh deleted file mode 100755 index 99a75689..00000000 --- a/capt/scripts/generate_hardware.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# Generate hardware - -set -euo pipefail - -function main() { - # Generate hardware - declare -r STATE_FILE="$1" - declare -r OUTPUT_DIR=$(yq eval '.outputDir' "$STATE_FILE") - declare -r NS=$(yq eval '.namespace' "$STATE_FILE") - - rm -f "$OUTPUT_DIR"/hardware*.yaml - - while IFS=$',' read -r name mac role ip gateway; do - export NODE_NAME="$name" - export NODE_MAC="$mac" - export NODE_ROLE="$role" - export NODE_IP="$ip" - export GATEWAY_IP="$gateway" - export NAMESPACE="$NS" - envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1))" "$OUTPUT_DIR"/hardware-"$NODE_NAME".yaml - unset NODE_ROLE - unset NODE_NAME - unset NODE_IP - unset NODE_MAC - unset GATEWAY_IP - done < <(yq e '.vm.details.[] | [key, .mac, .role, .ip, .gateway] | @csv' "$STATE_FILE") - -} - -main "$@" diff --git a/capt/scripts/generate_secret.sh b/capt/scripts/generate_secret.sh deleted file mode 100755 index a83b1da8..00000000 --- a/capt/scripts/generate_secret.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Generate secret. All machines share the same secret. The only customization is the namespace, user name, and password. - -function main() { - declare -r STATE_FILE="$1" - declare -r OUTPUT_DIR=$(yq eval '.outputDir' "$STATE_FILE") - export NAMESPACE=$(yq eval '.namespace' "$STATE_FILE") - export BMC_USER_BASE64=$(yq eval '.virtualBMC.user' "$STATE_FILE" | tr -d '\n' | base64) - export BMC_PASS_BASE64=$(yq eval '.virtualBMC.pass' "$STATE_FILE" | tr -d '\n' | base64) - - envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1))" "$OUTPUT_DIR"/bmc-secret.yaml - unset BMC_USER_BASE64 - unset BMC_PASS_BASE64 - unset NAMESPACE -} - -main "$@" diff --git a/capt/scripts/generate_state.sh b/capt/scripts/generate_state.sh deleted file mode 100755 index 93405b10..00000000 --- a/capt/scripts/generate_state.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/bin/bash -# This script generates the state data needed for creating the CAPT playground. - -# state file spec -cat </dev/null ---- -clusterName: "capt-playground" -outputDir: "/home/tink/repos/tinkerbell/playground/capt/output" -namespace: "tink" -counts: - controlPlanes: 1 - workers: 1 - spares: 1 -versions: - capt: v0.6.5 - chart: v0.19.0 - kube: v1.29.4 - os: 20.04 - kubevip: 0.9.1 -capt: - providerRepository: "https://github.com/tinkerbell/cluster-api-provider-tinkerbell/releases" -chart: - location: "oci://ghcr.io/tinkerbell/charts/tinkerbell" - extraVars: - - deployment.image=custom.registry/tinkerbell/tinkerbell - - deployment.imageTag=v0.19.1 - - deployment.agentImageTag=latest -os: - registry: ghcr.io/tinkerbell/cluster-api-provider-tinkerbell - distro: ubuntu - sshKey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH9a2GwjgVfnpjOvIqNuJTwazS3tqJ9xzcepXzKMccVf capt-playground" - sshKeyAutoGenerated: true - version: "2004" -vm: - baseName: "node" - cpusPerVM: 2 - memInMBPerVM: 2048 - diskSizeInGBPerVM: 10 - diskPath: "/tmp" - details: - node1: - mac: 02:7f:92:bd:2d:57 - bmc: - port: 6231 - role: control-plane - ip: 172.18.10.21 - gateway: 172.18.0.1 - node2: - mac: 02:f3:eb:c1:aa:2b - bmc: - port: 6232 - role: worker - ip: 172.18.10.22 - gateway: 172.18.0.1 - node3: - mac: 02:3c:e6:70:1b:5e - bmc: - port: 6233 - role: spare - ip: 172.18.10.23 - gateway: 172.18.0.1 -virtualBMC: - containerName: "virtualbmc" - image: ghcr.io/jacobweinstock/virtualbmc:latest - user: "root" - pass: "calvin" - ip: 172.18.0.3 -bootMode: netboot -totalNodes: 3 -kind: - kubeconfig: /home/tink/repos/tinkerbell/playground/capt/output/kind.kubeconfig - gatewayIP: 172.18.0.1 - nodeIPBase: 172.18.10.20 - bridgeName: br-3d1549d4f99f -tinkerbell: - vip: 172.18.10.74 - hookosVip: 172.18.10.73 -cluster: - controlPlane: - vip: 172.18.10.75 - podCIDR: 172.100.0.0/16 -EOF - -set -euo pipefail - -function generate_mac() { - declare NODE_NAME="$1" - - echo "$NODE_NAME" | md5sum | sed 's/^\(..\)\(..\)\(..\)\(..\)\(..\).*$/02:\1:\2:\3:\4:\5/' -} - -function main() { - # read in the config.yaml file and populate the .state file - declare CONFIG_FILE="$1" - declare STATE_FILE="$2" - - # update outputDir to be a fully qualified path - output_dir=$(yq eval '.outputDir' "$CONFIG_FILE") - if [[ $output_dir == /* ]]; then - echo - else - current_dir=$(pwd) - output_dir="$current_dir/$output_dir" - fi - config_file=$(realpath "$CONFIG_FILE") - state_file="$STATE_FILE" - - cp -a "$config_file" "$state_file" - yq e -i '.outputDir = "'$output_dir'"' "$state_file" - - # totalNodes - total_nodes=$(($(yq eval '.counts.controlPlanes' "$state_file") + $(yq eval '.counts.workers' "$state_file") + $(yq eval '.counts.spares' "$state_file"))) - yq e -i ".totalNodes = $total_nodes" "$state_file" - - # populate vmNames - base_name=$(yq eval '.vm.baseName' "$state_file") - base_ipmi_port=6230 - for i in $(seq 1 $total_nodes); do - name="$base_name$i" - mac=$(generate_mac "$name") - yq e -i ".vm.details.$name.mac = \"$mac\"" "$state_file" - yq e -i ".vm.details.$name.bmc.port = $((base_ipmi_port + i))" "$state_file" - # set the node role - if [[ $i -le $(yq eval '.counts.controlPlanes' "$state_file") ]]; then - yq e -i ".vm.details.$name.role = \"control-plane\"" "$state_file" - elif [[ $i -le $(($(yq eval '.counts.controlPlanes' "$state_file") + $(yq eval '.counts.workers' "$state_file"))) ]]; then - yq e -i ".vm.details.$name.role = \"worker\"" "$state_file" - else - yq e -i ".vm.details.$name.role = \"spare\"" "$state_file" - fi - unset name - unset mac - done - - # populate kind.kubeconfig - yq e -i '.kind.kubeconfig = "'$output_dir'/kind.kubeconfig"' "$state_file" - - # populate the expected OS version in the raw image name (22.04 -> 2204) - os_version=$(yq eval '.versions.os' "$state_file") - os_version=$(echo "$os_version" | tr -d '.') - yq e -i '.os.version = "'$os_version'"' "$state_file" - - # if the sshKey is not set, generate a default one - ssh_key=$(yq eval '.os.sshKey' "$config_file") - if [[ -z $ssh_key ]]; then - mkdir -p "$output_dir" - rm -rf "$output_dir"/capt-ssh-key* >>"$output_dir"/error.log 2>&1 - ssh-keygen -t ed25519 -f "$output_dir"/capt-ssh-key -N "" -C "capt-playground" >>"$output_dir"/error.log 2>&1 - if [[ $? -ne 0 ]]; then - echo "Error generating SSH key. Check error.log for details." >>"$output_dir"/error.log 2>&1 - exit 1 - fi - ssh_key=$(cat "$output_dir/capt-ssh-key.pub" | tr -d '\n') - if [[ -z $ssh_key ]]; then - echo "Error reading SSH key from $output_dir/capt-ssh-key.pub" >>"$output_dir"/error.log 2>&1 - exit 1 - fi - yq e -i ".os.sshKey = \"$ssh_key\"" "$state_file" - yq e -i ".os.sshKeyAutoGenerated = true" "$state_file" - else - yq e -i ".os.sshKeyAutoGenerated = false" "$state_file" - fi -} - -main "$@" diff --git a/capt/scripts/update_state.sh b/capt/scripts/update_state.sh deleted file mode 100755 index a58627df..00000000 --- a/capt/scripts/update_state.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# this script updates the state file with the generated hardware data - -function main() { - declare -r STATE_FILE="$1" - declare CLUSTER_NAME=$(yq eval '.clusterName' "$STATE_FILE") - declare GATEWAY_IP=$(docker inspect -f '{{ .NetworkSettings.Networks.kind.Gateway }}' "$CLUSTER_NAME"-control-plane) - declare NODE_IP_BASE=$(awk -F"." '{print $1"."$2".10.20"}' <<<"$GATEWAY_IP") - declare NODE_BASE=$(yq eval '.vm.baseName' "$STATE_FILE") - declare IP_LAST_OCTET=$(echo "$NODE_IP_BASE" | cut -d. -f4) - - yq e -i '.kind.gatewayIP = "'$GATEWAY_IP'"' "$STATE_FILE" - yq e -i '.kind.nodeIPBase = "'$NODE_IP_BASE'"' "$STATE_FILE" - - # set an ip and gateway per node - idx=1 - while IFS=$',' read -r name; do - v=$(echo "$NODE_IP_BASE" | awk -F"." '{print $1"."$2"."$3}').$((IP_LAST_OCTET + idx)) - ((idx++)) - yq e -i ".vm.details.$name.ip = \"$v\"" "$STATE_FILE" - yq e -i ".vm.details.$name.gateway = \"$GATEWAY_IP\"" "$STATE_FILE" - unset v - done < <(yq e '.vm.details.[] | [key] | @csv' "$STATE_FILE") - - # set the Tinkerbell Load Balancer IP (VIP) - offset=50 - t_lb=$(echo "$NODE_IP_BASE" | awk -F"." '{print $1"."$2"."$3}').$((IP_LAST_OCTET + idx + offset)) - yq e -i '.tinkerbell.vip = "'$t_lb'"' "$STATE_FILE" - - # set the Tinkerbell HookOS VIP - hookos_vip=$(echo "$NODE_IP_BASE" | awk -F"." '{print $1"."$2"."$3}').$((IP_LAST_OCTET + idx + offset - 1)) - yq e -i '.tinkerbell.hookosVip = "'$hookos_vip'"' "$STATE_FILE" - - # set the cluster control plane load balancer IP (VIP) - cp_lb=$(echo "$NODE_IP_BASE" | awk -F"." '{print $1"."$2"."$3}').$((IP_LAST_OCTET + idx + offset + 1)) - yq e -i '.cluster.controlPlane.vip = "'$cp_lb'"' "$STATE_FILE" - - # set the cluster pod cidr - POD_CIDR=$(awk -F"." '{print $1".100.0.0/16"}' <<<"$GATEWAY_IP") - yq e -i '.cluster.podCIDR = "'$POD_CIDR'"' "$STATE_FILE" - - # set the KinD bridge name - network_id=$(docker network inspect -f '{{.Id}}' kind) - bridge_name="br-${network_id:0:12}" - yq e -i '.kind.bridgeName = "'$bridge_name'"' "$STATE_FILE" - -} - -main "$@" diff --git a/capt/tasks/Taskfile-capi-pivot.yaml b/capt/tasks/Taskfile-capi-pivot.yaml index 02b05b7b..7af37205 100644 --- a/capt/tasks/Taskfile-capi-pivot.yaml +++ b/capt/tasks/Taskfile-capi-pivot.yaml @@ -2,23 +2,24 @@ version: "3" includes: delete: ./Taskfile-delete.yaml + helm: ./Taskfile-helm.yaml vars: OUTPUT_DIR: - sh: yq eval '.outputDir' config.yaml - CURR_DIR: - sh: pwd + sh: | + awk -F: '/^outputDir:/ {gsub(/^[ "]+|[ "]+$/, "", $2); print $2}' {{.ROOT_DIR}}/config.yaml STATE_FILE: ".state" STATE_FILE_FQ_PATH: - sh: echo {{joinPath .CURR_DIR .STATE_FILE}} + sh: echo {{joinPath .ROOT_DIR .STATE_FILE}} CONFIG_FILE_FQ_PATH: - sh: echo {{joinPath .CURR_DIR "config.yaml"}} + sh: echo {{joinPath .ROOT_DIR "config.yaml"}} CLUSTER_NAME: - sh: yq eval '.clusterName' config.yaml + sh: | + awk -F: '/^clusterName:/ {gsub(/^[ "]+|[ "]+$/, "", $2); print $2}' {{.ROOT_DIR}}/config.yaml MGMT_KUBECONFIG: - sh: echo {{list .CLUSTER_NAME "kubeconfig" | join "." | joinPath .CURR_DIR .OUTPUT_DIR}} + sh: echo {{list .CLUSTER_NAME "kubeconfig" | join "." | joinPath .ROOT_DIR .OUTPUT_DIR}} KIND_KUBECONFIG: - sh: echo {{ joinPath .CURR_DIR .OUTPUT_DIR "kind.kubeconfig"}} + sh: echo {{ joinPath .ROOT_DIR .OUTPUT_DIR "kind.kubeconfig"}} tasks: ordered: @@ -34,39 +35,20 @@ tasks: deploy-tinkerbell-helm-chart: run: once - summary: | - Deploy the Tinkerbell Helm chart. + # Skip when the Tinkerbell stack lives in a separate cluster: in that + # mode CAPT keeps using the external-tinkerbell-kubeconfig secret and + # we must not install a second Tinkerbell release into the pivoted + # workload cluster. + if: '{{ne .EXTERNAL_TINKERBELL "true"}}' vars: - KUBECONFIG: "{{.MGMT_KUBECONFIG}}" - LB_IP: - sh: yq eval '.tinkerbell.vip' {{.STATE_FILE_FQ_PATH}} - LB_IP2: - sh: yq eval '.tinkerbell.hookosVip' {{.STATE_FILE_FQ_PATH}} - TRUSTED_PROXIES: - sh: KUBECONFIG={{.KUBECONFIG}} kubectl get nodes -o jsonpath='{.items[*].spec.podCIDR}' | tr ' ' ',' - STACK_CHART_VERSION: - sh: yq eval '.versions.chart' {{.STATE_FILE_FQ_PATH}} - NAMESPACE: - sh: yq eval '.namespace' {{.STATE_FILE_FQ_PATH}} - LOCATION: - sh: yq eval '.chart.location' {{.STATE_FILE_FQ_PATH}} - CHART_NAME: tinkerbell - BOOTMODE: - sh: yq eval '.bootMode' {{.STATE_FILE_FQ_PATH}} - GLOBAL_VARS: - - trustedProxies={"{{.TRUSTED_PROXIES}}"} - - publicIP={{.LB_IP}} - - artifactsFileServer=http://{{.LB_IP2}}:7173 - ISO_VARS: - - deployment.envs.smee.dhcpEnabled=false - - deployment.envs.smee.isoUpstreamURL=http://{{.LB_IP2}}:7173/hook-latest-lts-x86_64-efi-initrd.iso - - optional.hookos.extension=both - EXTRA_VARS: - sh: yq eval '.chart.extraVars | .[]' {{.STATE_FILE_FQ_PATH}} | xargs + EXTERNAL_TINKERBELL: + sh: "yq eval '.externalTinkerbell // false' {{.STATE_FILE_FQ_PATH}}" + summary: | + Deploy the Tinkerbell Helm chart to the workload cluster after pivot. cmds: - - KUBECONFIG="{{.MGMT_KUBECONFIG}}" helm upgrade --install {{.CHART_NAME}} {{.LOCATION}} --version "{{.STACK_CHART_VERSION}}" --create-namespace --namespace {{.NAMESPACE}} --wait {{range .GLOBAL_VARS}}--set "{{.}}" {{end}} {{- if eq .BOOTMODE "isoboot" }} {{- range .ISO_VARS }}--set "{{.}}" {{ end }} {{ end }} {{- if .EXTRA_VARS }} {{- range (splitList " " .EXTRA_VARS ) }}--set "{{.}}" {{ end }} {{ end }} - status: - - helm_status=$(KUBECONFIG="{{.KUBECONFIG}}" helm status -n {{.NAMESPACE}} {{.CHART_NAME}} -o yaml | yq .info.status); [[ "$helm_status" == "deployed" ]] + - task: helm:deploy-tinkerbell-helm-chart + vars: + KUBECONFIG: "{{.MGMT_KUBECONFIG}}" init: run: once @@ -75,7 +57,7 @@ tasks: Initialize the cluster. env: TINKERBELL_IP: - sh: yq eval '.tinkerbell.vip' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.tinkerbell.vip' {{.STATE_FILE_FQ_PATH}}" CLUSTERCTL_DISABLE_VERSIONCHECK: true XDG_CONFIG_HOME: "{{.OUTPUT_DIR}}/xdg" XDG_CONFIG_DIRS: "{{.OUTPUT_DIR}}/xdg" @@ -86,9 +68,9 @@ tasks: XDG_DATA_DIRS: "{{.OUTPUT_DIR}}/xdg" vars: OUTPUT_DIR: - sh: echo $(yq eval '.outputDir' config.yaml) + sh: "echo $(yq eval '.outputDir' config.yaml)" KIND_GATEWAY_IP: - sh: yq eval '.kind.gatewayIP' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.kind.gatewayIP' {{.STATE_FILE_FQ_PATH}}" KUBECONFIG: "{{.MGMT_KUBECONFIG}}" cmds: - KUBECONFIG="{{.KUBECONFIG}}" clusterctl --config {{.OUTPUT_DIR}}/clusterctl.yaml init --infrastructure tinkerbell @@ -111,9 +93,9 @@ tasks: XDG_DATA_DIRS: "{{.OUTPUT_DIR}}/xdg" vars: OUTPUT_DIR: - sh: echo $(yq eval '.outputDir' config.yaml) + sh: "echo $(yq eval '.outputDir' config.yaml)" NAMESPACE: - sh: yq eval '.namespace' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.namespace' {{.STATE_FILE_FQ_PATH}}" cmds: - KUBECONFIG="{{.KIND_KUBECONFIG}}" clusterctl move --to-kubeconfig="{{.MGMT_KUBECONFIG}}" --config {{.OUTPUT_DIR}}/clusterctl.yaml --kubeconfig "{{.KIND_KUBECONFIG}}" -n {{.NAMESPACE}} status: diff --git a/capt/tasks/Taskfile-capi.yaml b/capt/tasks/Taskfile-capi.yaml index b18f98f9..ee7e2d89 100644 --- a/capt/tasks/Taskfile-capi.yaml +++ b/capt/tasks/Taskfile-capi.yaml @@ -7,26 +7,29 @@ tasks: cmds: - task: create-cluster-yaml - task: init - - task: generate-cluster-yaml - - task: create-kustomize-file - - task: apply-kustomization + - task: render-cluster-yaml create-cluster-yaml: run: once summary: | - Create the cluster yaml. - env: - CAPT_VERSION: - sh: yq eval '.versions.capt' {{.STATE_FILE_FQ_PATH}} - LOCATION: - sh: yq eval '.capt.providerRepository' {{.STATE_FILE_FQ_PATH}} + Render clusterctl config from the cue/clusterctl subpackage. Replaces + the previous envsubst pipeline against templates/clusterctl.tmpl. vars: OUTPUT_DIR: - sh: echo $(yq eval '.outputDir' config.yaml) + sh: "echo $(yq eval '.outputDir' config.yaml)" + CAPT_VERSION: + sh: "yq eval '.versions.capt' {{.STATE_FILE_FQ_PATH}}" + sources: + - "{{.STATE_FILE_FQ_PATH}}" + - cue/clusterctl/*.cue + - cue/values/*.cue + generates: + - "{{.OUTPUT_DIR}}/clusterctl.yaml" cmds: - - envsubst '$CAPT_VERSION,$LOCATION' < templates/clusterctl.tmpl > {{.OUTPUT_DIR}}/clusterctl.yaml + - "cue vet ./cue/clusterctl yaml: {{.STATE_FILE_FQ_PATH}} -l 'values:'" + - "cue export ./cue/clusterctl yaml: {{.STATE_FILE_FQ_PATH}} -l 'values:' -e out --out yaml > {{.OUTPUT_DIR}}/clusterctl.yaml" status: - - grep -q "$CAPT_VERSION" {{.OUTPUT_DIR}}/clusterctl.yaml + - grep -q "{{.CAPT_VERSION}}" {{.OUTPUT_DIR}}/clusterctl.yaml init: run: once @@ -35,7 +38,7 @@ tasks: Initialize the cluster. env: TINKERBELL_IP: - sh: yq eval '.tinkerbell.vip' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.tinkerbell.vip' {{.STATE_FILE_FQ_PATH}}" CLUSTERCTL_DISABLE_VERSIONCHECK: true XDG_CONFIG_HOME: "{{.OUTPUT_DIR}}/xdg" XDG_CONFIG_DIRS: "{{.OUTPUT_DIR}}/xdg" @@ -46,112 +49,41 @@ tasks: XDG_DATA_DIRS: "{{.OUTPUT_DIR}}/xdg" vars: OUTPUT_DIR: - sh: echo $(yq eval '.outputDir' config.yaml) + sh: "echo $(yq eval '.outputDir' config.yaml)" KIND_GATEWAY_IP: - sh: yq eval '.kind.gatewayIP' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.kind.gatewayIP' {{.STATE_FILE_FQ_PATH}}" KUBECONFIG: - sh: yq eval '.kind.kubeconfig' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.kind.kubeconfig' {{.STATE_FILE_FQ_PATH}}" cmds: - KUBECONFIG="{{.KUBECONFIG}}" clusterctl --config {{.OUTPUT_DIR}}/clusterctl.yaml init --infrastructure tinkerbell status: - expected=1; got=$(KUBECONFIG="{{.KUBECONFIG}}" kubectl get pods -n capt-system |grep -ce "capt-controller"); [[ "$got" == "$expected" ]] - generate-cluster-yaml: + render-cluster-yaml: run: once - deps: [init] summary: | - Generate the cluster yaml. - env: - CONTROL_PLANE_VIP: - sh: yq eval '.cluster.controlPlane.vip' {{.STATE_FILE_FQ_PATH}} - POD_CIDR: - sh: yq eval '.cluster.podCIDR' {{.STATE_FILE_FQ_PATH}} - CLUSTERCTL_DISABLE_VERSIONCHECK: true - XDG_CONFIG_HOME: "{{.OUTPUT_DIR}}/xdg" - XDG_CONFIG_DIRS: "{{.OUTPUT_DIR}}/xdg" - XDG_STATE_HOME: "{{.OUTPUT_DIR}}/xdg" - XDG_CACHE_HOME: "{{.OUTPUT_DIR}}/xdg" - XDG_RUNTIME_DIR: "{{.OUTPUT_DIR}}/xdg" - XDG_DATA_HOME: "{{.OUTPUT_DIR}}/xdg" - XDG_DATA_DIRS: "{{.OUTPUT_DIR}}/xdg" + Render the workload cluster manifest from the cue/capi subpackage. + Inputs: capt/.state (read directly as YAML by cue) and the bootMode + tag. Output: {{.OUTPUT_DIR}}/{{.CLUSTER_NAME}}.yaml — a multi-document + YAML applied as-is by create-workload-cluster. vars: CLUSTER_NAME: - sh: yq eval '.clusterName' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.clusterName' {{.STATE_FILE_FQ_PATH}}" OUTPUT_DIR: - sh: yq eval '.outputDir' config.yaml - KUBE_VERSION: - sh: yq eval '.versions.kube' {{.STATE_FILE_FQ_PATH}} - CP_COUNT: - sh: yq eval '.counts.controlPlanes' {{.STATE_FILE_FQ_PATH}} - WORKER_COUNT: - sh: yq eval '.counts.workers' {{.STATE_FILE_FQ_PATH}} - NAMESPACE: - sh: yq eval '.namespace' {{.STATE_FILE_FQ_PATH}} - KUBECONFIG: - sh: yq eval '.kind.kubeconfig' {{.STATE_FILE_FQ_PATH}} - cmds: - - KUBECONFIG="{{.KUBECONFIG}}" clusterctl generate cluster {{.CLUSTER_NAME}} --config {{.OUTPUT_DIR}}/clusterctl.yaml --kubernetes-version "{{.KUBE_VERSION}}" --control-plane-machine-count="{{.CP_COUNT}}" --worker-machine-count="{{.WORKER_COUNT}}" --target-namespace={{.NAMESPACE}} --write-to {{.OUTPUT_DIR}}/prekustomization.yaml - status: - - grep -q "{{.KUBE_VERSION}}" {{.OUTPUT_DIR}}/prekustomization.yaml - - create-kustomize-file: - run: once - summary: | - Kustomize file for the CAPI generated config file (prekustomization.yaml). - env: - NAMESPACE: - sh: yq eval '.namespace' {{.STATE_FILE_FQ_PATH}} - OS_REGISTRY: - sh: yq eval '.os.registry' {{.STATE_FILE_FQ_PATH}} - OS_DISTRO: - sh: yq eval '.os.distro' {{.STATE_FILE_FQ_PATH}} - OS_VERSION: - sh: yq eval '.os.version' {{.STATE_FILE_FQ_PATH}} - VERSIONS_OS: - sh: yq eval '.versions.os' {{.STATE_FILE_FQ_PATH}} - SSH_AUTH_KEY: - sh: yq eval '.os.sshKey' {{.STATE_FILE_FQ_PATH}} - KUBE_VERSION: - sh: yq eval '.versions.kube' {{.STATE_FILE_FQ_PATH}} - TINKERBELL_VIP: - sh: yq eval '.tinkerbell.vip' {{.STATE_FILE_FQ_PATH}} - CLUSTER_NAME: - sh: yq eval '.clusterName' {{.STATE_FILE_FQ_PATH}} - KUBEVIP_VERSION: - sh: yq eval '.versions.kubevip' {{.STATE_FILE_FQ_PATH}} - CONTROL_PLANE_VIP: - sh: yq eval '.cluster.controlPlane.vip' {{.STATE_FILE_FQ_PATH}} - vars: - KUBE_VERSION: - sh: yq eval '.versions.kube' {{.STATE_FILE_FQ_PATH}} - OUTPUT_DIR: - sh: yq eval '.outputDir' config.yaml + sh: "yq eval '.outputDir' config.yaml" BOOTMODE: - sh: yq eval '.bootMode' {{.STATE_FILE_FQ_PATH}} - KUSTOMIZE_FILE: - sh: "[[ {{.BOOTMODE}} == 'isoboot' ]] && echo kustomization-iso.tmpl || echo kustomization-netboot.tmpl" - sources: - - config.yaml - generates: - - "{{.OUTPUT_DIR}}/kustomization.yaml" - cmds: - - envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1))" < templates/{{.KUSTOMIZE_FILE}} > {{.OUTPUT_DIR}}/kustomization.yaml - # FYI, the `preKubeadmCommands` checks for k8s v1.29 because of https://github.com/kube-vip/kube-vip/issues/684 - - apply-kustomization: - run: once - deps: [generate-cluster-yaml, create-kustomize-file] - summary: | - Kustomize the cluster yaml. - vars: - CLUSTER_NAME: - sh: yq eval '.clusterName' {{.STATE_FILE_FQ_PATH}} - KUBECONFIG: - sh: yq eval '.kind.kubeconfig' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.bootMode' {{.STATE_FILE_FQ_PATH}}" sources: - - "{{.OUTPUT_DIR}}/kustomization.yaml" - - "{{.OUTPUT_DIR}}/prekustomization.yaml" + - "{{.STATE_FILE_FQ_PATH}}" + - cue/capi/*.cue + - cue/values/*.cue + - cue/mirror/*.cue generates: - "{{.OUTPUT_DIR}}/{{.CLUSTER_NAME}}.yaml" cmds: - - KUBECONFIG="{{.KUBECONFIG}}" kubectl kustomize {{.OUTPUT_DIR}} -o {{.OUTPUT_DIR}}/{{.CLUSTER_NAME}}.yaml + # Lint the CUE module first; fails with line-numbered errors if any + # constraints in cue/values/values.cue#Config are violated by .state. + # The `yaml:` qualifier tells cue to read .state as YAML directly — no + # intermediate JSON conversion needed. + - "cue vet ./cue/capi yaml: {{.STATE_FILE_FQ_PATH}} -l 'values:'" + - "cue export ./cue/capi yaml: {{.STATE_FILE_FQ_PATH}} -l 'values:' -t mode={{.BOOTMODE}} -e out --out text > {{.OUTPUT_DIR}}/{{.CLUSTER_NAME}}.yaml" diff --git a/capt/tasks/Taskfile-create.yaml b/capt/tasks/Taskfile-create.yaml index e813d7a5..dfc859c1 100644 --- a/capt/tasks/Taskfile-create.yaml +++ b/capt/tasks/Taskfile-create.yaml @@ -3,6 +3,8 @@ version: "3" includes: vbmc: ./Taskfile-vbmc.yaml capi: ./Taskfile-capi.yaml + mirror: ./Taskfile-mirror.yaml + helm: ./Taskfile-helm.yaml tasks: playground-ordered: @@ -11,6 +13,7 @@ tasks: Create the CAPT playground. cmds: - task: kind-cluster + - task: tinkerbell-kind-cluster - task: update-state - task: deploy-tinkerbell-helm-chart - task: vbmc:prepare @@ -25,7 +28,9 @@ tasks: - task: apply-bmc-secret - task: apply-bmc-machines - task: apply-hardware + - task: create-external-kubeconfig-secret - task: capi:ordered + - task: ensure-mgmt-namespace - task: allow-customization - task: create-workload-cluster - task: get-workload-cluster-kubeconfig @@ -37,101 +42,143 @@ tasks: kind-cluster: run: once + deps: [mirror:render-kind-config] summary: | - Install a KinD cluster. + Install the management KinD cluster. vars: + ENABLED: + sh: "yq eval '.registryMirror.enabled // false' {{.STATE_FILE_FQ_PATH}}" CLUSTER_NAME: - sh: yq eval '.clusterName' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.clusterName' {{.STATE_FILE_FQ_PATH}}" KUBECONFIG: - sh: yq eval '.kind.kubeconfig' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.kind.kubeconfig' {{.STATE_FILE_FQ_PATH}}" + KIND_CONFIG: "{{.OUTPUT_DIR}}/kind-config.yaml" + KIND_CONFIG_FLAG: '{{if eq .ENABLED "true"}}--config {{.KIND_CONFIG}}{{end}}' cmds: - - kind create cluster --name {{.CLUSTER_NAME}} --kubeconfig "{{.KUBECONFIG}}" + - kind create cluster --name {{.CLUSTER_NAME}} --kubeconfig "{{.KUBECONFIG}}" {{.KIND_CONFIG_FLAG}} - until KUBECONFIG="{{.KUBECONFIG}}" kubectl wait --for=condition=ready node --all --timeout=5m; do echo "Waiting for nodes to be ready..."; sleep 1; done status: - KUBECONFIG="{{.KUBECONFIG}}" kind get clusters | grep -q {{.CLUSTER_NAME}} + tinkerbell-kind-cluster: + run: once + if: '{{eq .EXTERNAL_TINKERBELL "true"}}' + deps: [mirror:render-kind-config] + summary: | + Install a second KinD cluster that hosts the Tinkerbell stack when + `externalTinkerbell: true`. Shares the default `kind` docker network + with the management cluster so pods in the management cluster can + reach the Tinkerbell API server. + vars: + ENABLED: + sh: "yq eval '.registryMirror.enabled // false' {{.STATE_FILE_FQ_PATH}}" + EXTERNAL_TINKERBELL: + sh: "yq eval '.externalTinkerbell // false' {{.STATE_FILE_FQ_PATH}}" + TINK_CLUSTER_NAME: + sh: 'yq eval ''.kind.tinkerbell.clusterName // ""'' {{.STATE_FILE_FQ_PATH}}' + TINK_KUBECONFIG: + sh: 'yq eval ''.kind.tinkerbell.kubeconfig // ""'' {{.STATE_FILE_FQ_PATH}}' + KIND_CONFIG: "{{.OUTPUT_DIR}}/kind-config.yaml" + KIND_CONFIG_FLAG: '{{if eq .ENABLED "true"}}--config {{.KIND_CONFIG}}{{end}}' + cmds: + - kind create cluster --name "{{.TINK_CLUSTER_NAME}}" --kubeconfig "{{.TINK_KUBECONFIG}}" {{.KIND_CONFIG_FLAG}} + - until KUBECONFIG="{{.TINK_KUBECONFIG}}" kubectl wait --for=condition=ready node --all --timeout=5m; do echo "Waiting for Tinkerbell cluster nodes to be ready..."; sleep 1; done + status: + - kind get clusters 2>/dev/null | grep -q "{{.TINK_CLUSTER_NAME}}" + update-state: - silent: true run: once - deps: [kind-cluster] + deps: [kind-cluster, tinkerbell-kind-cluster] summary: | - Update the state file with the KinD cluster information. Should be run only after the KinD cluster is created. + Re-render .state with KinD-derived values (gateway IP, bridge name) + via the cue/state package. + vars: + CLUSTER_NAME: + sh: "yq eval '.clusterName' {{.STATE_FILE_FQ_PATH}}" + GATEWAY_IP: + sh: "docker inspect -f '{{`{{ .NetworkSettings.Networks.kind.Gateway }}`}}' {{.CLUSTER_NAME}}-control-plane" + NETWORK_ID: + sh: "docker network inspect -f '{{`{{.Id}}`}}' kind" + BRIDGE_NAME: "br-{{substr 0 12 .NETWORK_ID}}" cmds: - - ./scripts/update_state.sh "{{.STATE_FILE_FQ_PATH}}" + - >- + cue export ./cue/state yaml: config.yaml -l 'config:' + -t cwd="$PWD" + -t sshPubKey="$(tr -d '\n' < {{.OUTPUT_DIR}}/capt-ssh-key.pub)" + -t gatewayIP="{{.GATEWAY_IP}}" + -t bridgeName="{{.BRIDGE_NAME}}" + -e out --out yaml > {{.STATE_FILE_FQ_PATH}} + status: + - '[[ "$(yq eval ''.kind.gatewayIP'' {{.STATE_FILE_FQ_PATH}})" == "{{.GATEWAY_IP}}" ]]' hardware-cr: run: once deps: [update-state] summary: | - Create BMC Machine object. + Render Hardware CRs (one per VM) from cue/infra. sources: - "{{.STATE_FILE_FQ_PATH}}" + - cue/infra/*.cue + - cue/values/*.cue generates: - "{{.OUTPUT_DIR}}/hardware-*.yaml" + vars: + NAMES: + sh: "yq e '.vm.details[] | [key] | @csv' {{.STATE_FILE_FQ_PATH}}" cmds: - - ./scripts/generate_hardware.sh {{.STATE_FILE_FQ_PATH}} + - rm -f {{.OUTPUT_DIR}}/hardware-*.yaml + - "cue vet ./cue/infra yaml: {{.STATE_FILE_FQ_PATH}} -l 'values:'" + - for: { var: NAMES } + cmd: "cue export ./cue/infra yaml: {{.STATE_FILE_FQ_PATH}} -l 'values:' -e outHardware.{{.ITEM}} --out yaml > {{.OUTPUT_DIR}}/hardware-{{.ITEM}}.yaml" bmc-machine-cr: run: once deps: [vbmc:update-state] summary: | - Create BMC Machine objects. + Render BMC Machine CRs (one per VM) from cue/infra. sources: - "{{.STATE_FILE_FQ_PATH}}" + - cue/infra/*.cue + - cue/values/*.cue generates: - "{{.OUTPUT_DIR}}/bmc-machine-*.yaml" + vars: + NAMES: + sh: "yq e '.vm.details[] | [key] | @csv' {{.STATE_FILE_FQ_PATH}}" cmds: - - ./scripts/generate_bmc.sh {{.STATE_FILE_FQ_PATH}} + - rm -f {{.OUTPUT_DIR}}/bmc-machine-*.yaml + - "cue vet ./cue/infra yaml: {{.STATE_FILE_FQ_PATH}} -l 'values:'" + - for: { var: NAMES } + cmd: "cue export ./cue/infra yaml: {{.STATE_FILE_FQ_PATH}} -l 'values:' -e outBmcMachines.{{.ITEM}} --out yaml > {{.OUTPUT_DIR}}/bmc-machine-{{.ITEM}}.yaml" bmc-secret: run: once deps: [update-state] summary: | - Create BMC Machine objects. + Render the shared BMC credentials Secret from cue/infra. sources: - "{{.STATE_FILE_FQ_PATH}}" + - cue/infra/*.cue + - cue/values/*.cue generates: - "{{.OUTPUT_DIR}}/bmc-secret.yaml" cmds: - - ./scripts/generate_secret.sh {{.STATE_FILE_FQ_PATH}} + - "cue vet ./cue/infra yaml: {{.STATE_FILE_FQ_PATH}} -l 'values:'" + - "cue export ./cue/infra yaml: {{.STATE_FILE_FQ_PATH}} -l 'values:' -e outBmcSecret --out yaml > {{.OUTPUT_DIR}}/bmc-secret.yaml" deploy-tinkerbell-helm-chart: run: once - deps: [kind-cluster, update-state] + deps: [kind-cluster, tinkerbell-kind-cluster, update-state] summary: | - Deploy the Tinkerbell Helm chart. + Deploy the Tinkerbell Helm chart. Targets the Tinkerbell KinD cluster + when `externalTinkerbell: true`, otherwise the management cluster. vars: KUBECONFIG: - sh: yq eval '.kind.kubeconfig' {{.STATE_FILE_FQ_PATH}} - LB_IP: - sh: yq eval '.tinkerbell.vip' {{.STATE_FILE_FQ_PATH}} - LB_IP2: - sh: yq eval '.tinkerbell.hookosVip' {{.STATE_FILE_FQ_PATH}} - TRUSTED_PROXIES: - sh: KUBECONFIG={{.KUBECONFIG}} kubectl get nodes -o jsonpath='{.items[*].spec.podCIDR}' | tr ' ' ',' - STACK_CHART_VERSION: - sh: yq eval '.versions.chart' {{.STATE_FILE_FQ_PATH}} - NAMESPACE: - sh: yq eval '.namespace' {{.STATE_FILE_FQ_PATH}} - LOCATION: - sh: yq eval '.chart.location' {{.STATE_FILE_FQ_PATH}} - CHART_NAME: tinkerbell - BOOTMODE: - sh: yq eval '.bootMode' {{.STATE_FILE_FQ_PATH}} - GLOBAL_VARS: - - trustedProxies={"{{.TRUSTED_PROXIES}}"} - - publicIP={{.LB_IP}} - - artifactsFileServer=http://{{.LB_IP2}}:7173 - ISO_VARS: - - deployment.envs.smee.dhcpEnabled=false - - deployment.envs.smee.isoUpstreamURL=http://{{.LB_IP2}}:7173/hook-latest-lts-x86_64-efi-initrd.iso - - optional.hookos.extension=both - EXTRA_VARS: - sh: yq eval '.chart.extraVars | .[]' {{.STATE_FILE_FQ_PATH}} | xargs - cmds: - - KUBECONFIG="{{.KUBECONFIG}}" helm upgrade --install {{.CHART_NAME}} {{.LOCATION}} --version "{{.STACK_CHART_VERSION}}" --create-namespace --namespace {{.NAMESPACE}} --wait {{range .GLOBAL_VARS}}--set "{{.}}" {{end}} {{- if eq .BOOTMODE "isoboot" }} {{- range .ISO_VARS }}--set "{{.}}" {{ end }} {{ end }} {{- if .EXTRA_VARS }} {{- range (splitList " " .EXTRA_VARS ) }}--set "{{.}}" {{ end }} {{ end }} - status: - - helm_status=$(KUBECONFIG="{{.KUBECONFIG}}" helm status -n {{.NAMESPACE}} {{.CHART_NAME}} -o yaml | yq .info.status); [[ "$helm_status" == "deployed" ]] + sh: "yq eval '(.kind.tinkerbell.kubeconfig // .kind.kubeconfig)' {{.STATE_FILE_FQ_PATH}}" + cmds: + - task: helm:deploy-tinkerbell-helm-chart + vars: + KUBECONFIG: "{{.KUBECONFIG}}" vms: run: once @@ -140,9 +187,9 @@ tasks: Create Libvirt VMs. vars: TOTAL_HARDWARE: - sh: yq eval '.totalNodes' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.totalNodes' {{.STATE_FILE_FQ_PATH}}" VM_BASE_NAME: - sh: yq eval '.vm.baseName' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.vm.baseName' {{.STATE_FILE_FQ_PATH}}" cmds: - ./scripts/create_vms.sh "{{.STATE_FILE_FQ_PATH}}" status: @@ -150,14 +197,14 @@ tasks: apply-bmc-secret: run: once - deps: [kind-cluster, bmc-secret] + deps: [kind-cluster, tinkerbell-kind-cluster, bmc-secret] summary: | - Apply the BMC secret. + Apply the BMC secret to the cluster that hosts the Tinkerbell stack. vars: NAMESPACE: - sh: yq eval '.namespace' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.namespace' {{.STATE_FILE_FQ_PATH}}" KUBECONFIG: - sh: yq eval '.kind.kubeconfig' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '(.kind.tinkerbell.kubeconfig // .kind.kubeconfig)' {{.STATE_FILE_FQ_PATH}}" cmds: - KUBECONFIG="{{.KUBECONFIG}}" kubectl apply -f {{.OUTPUT_DIR}}/bmc-secret.yaml status: @@ -165,20 +212,20 @@ tasks: apply-bmc-machines: run: once - deps: [kind-cluster, bmc-machine-cr] + deps: [kind-cluster, tinkerbell-kind-cluster, bmc-machine-cr] summary: | - Apply the BMC machines. + Apply the BMC machines to the cluster that hosts the Tinkerbell stack. vars: NAMES: - sh: yq e '.vm.details[] | [key] | @csv' {{.STATE_FILE_FQ_PATH}} + sh: "yq e '.vm.details[] | [key] | @csv' {{.STATE_FILE_FQ_PATH}}" TOTAL_HARDWARE: - sh: yq eval '.totalNodes' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.totalNodes' {{.STATE_FILE_FQ_PATH}}" VM_BASE_NAME: - sh: yq eval '.vm.baseName' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.vm.baseName' {{.STATE_FILE_FQ_PATH}}" NAMESPACE: - sh: yq eval '.namespace' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.namespace' {{.STATE_FILE_FQ_PATH}}" KUBECONFIG: - sh: yq eval '.kind.kubeconfig' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '(.kind.tinkerbell.kubeconfig // .kind.kubeconfig)' {{.STATE_FILE_FQ_PATH}}" cmds: - for: { var: NAMES } cmd: KUBECONFIG="{{.KUBECONFIG}}" kubectl apply -f {{.OUTPUT_DIR}}/bmc-machine-{{.ITEM}}.yaml @@ -187,20 +234,20 @@ tasks: apply-hardware: run: once - deps: [kind-cluster, hardware-cr] + deps: [kind-cluster, tinkerbell-kind-cluster, hardware-cr] summary: | - Apply the hardware. + Apply the hardware to the cluster that hosts the Tinkerbell stack. vars: NAMES: - sh: yq e '.vm.details[] | [key] | @csv' {{.STATE_FILE_FQ_PATH}} + sh: "yq e '.vm.details[] | [key] | @csv' {{.STATE_FILE_FQ_PATH}}" TOTAL_HARDWARE: - sh: yq eval '.totalNodes' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.totalNodes' {{.STATE_FILE_FQ_PATH}}" VM_BASE_NAME: - sh: yq eval '.vm.baseName' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.vm.baseName' {{.STATE_FILE_FQ_PATH}}" NAMESPACE: - sh: yq eval '.namespace' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.namespace' {{.STATE_FILE_FQ_PATH}}" KUBECONFIG: - sh: yq eval '.kind.kubeconfig' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '(.kind.tinkerbell.kubeconfig // .kind.kubeconfig)' {{.STATE_FILE_FQ_PATH}}" cmds: - for: { var: NAMES } cmd: KUBECONFIG="{{.KUBECONFIG}}" kubectl apply -f {{.OUTPUT_DIR}}/hardware-{{.ITEM}}.yaml @@ -209,22 +256,42 @@ tasks: create-workload-cluster: run: once - deps: [kind-cluster, capi:ordered] + deps: [kind-cluster, capi:ordered, ensure-mgmt-namespace] summary: | Create the workload cluster by applying the generated manifest file. vars: CLUSTER_NAME: - sh: yq eval '.clusterName' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.clusterName' {{.STATE_FILE_FQ_PATH}}" KUBECONFIG: - sh: yq eval '.kind.kubeconfig' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.kind.kubeconfig' {{.STATE_FILE_FQ_PATH}}" NAMESPACE: - sh: yq eval '.namespace' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.namespace' {{.STATE_FILE_FQ_PATH}}" cmds: - until KUBECONFIG="{{.KUBECONFIG}}" kubectl apply -f {{.OUTPUT_DIR}}/{{.CLUSTER_NAME}}.yaml >>{{.OUTPUT_DIR}}/error.log 2>&1; do echo "Trying kubectl apply again..."; sleep 3; done - echo "Workload manifest applied to cluster." status: - KUBECONFIG="{{.KUBECONFIG}}" kubectl get -n {{.NAMESPACE}} cluster {{.CLUSTER_NAME}} + ensure-mgmt-namespace: + run: once + deps: [kind-cluster] + summary: | + Ensure the workload-cluster namespace exists in the management cluster. + In single-cluster mode the Tinkerbell helm release creates this + namespace as a side effect of `--create-namespace`, but in external + Tinkerbell mode the chart targets a different cluster so the + management cluster is missing the namespace required by the CAPI + Cluster/KCP/MachineDeployment/etc. CRs. + vars: + NAMESPACE: + sh: "yq eval '.namespace' {{.STATE_FILE_FQ_PATH}}" + KUBECONFIG: + sh: "yq eval '.kind.kubeconfig' {{.STATE_FILE_FQ_PATH}}" + cmds: + - KUBECONFIG="{{.KUBECONFIG}}" kubectl create namespace {{.NAMESPACE}} --dry-run=client -o yaml | KUBECONFIG="{{.KUBECONFIG}}" kubectl apply -f - + status: + - KUBECONFIG="{{.KUBECONFIG}}" kubectl get namespace {{.NAMESPACE}} >/dev/null 2>&1 + get-workload-cluster-kubeconfig: run: once deps: [create-workload-cluster] @@ -232,17 +299,17 @@ tasks: Get the workload cluster's kubeconfig. vars: KUBECONFIG: - sh: yq eval '.kind.kubeconfig' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.kind.kubeconfig' {{.STATE_FILE_FQ_PATH}}" NAMESPACE: - sh: yq eval '.namespace' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.namespace' {{.STATE_FILE_FQ_PATH}}" CLUSTER_NAME: - sh: yq eval '.clusterName' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.clusterName' {{.STATE_FILE_FQ_PATH}}" cmds: - until KUBECONFIG="{{.KUBECONFIG}}" clusterctl get kubeconfig -n {{.NAMESPACE}} {{.CLUSTER_NAME}} >>{{.OUTPUT_DIR}}/error.log 2>&1 ; do echo "Waiting for workload cluster kubeconfig to be available..."; sleep 4; done - KUBECONFIG="{{.KUBECONFIG}}" clusterctl get kubeconfig -n {{.NAMESPACE}} {{.CLUSTER_NAME}} > {{.OUTPUT_DIR}}/{{.CLUSTER_NAME}}.kubeconfig - echo "Workload cluster kubeconfig saved to {{.OUTPUT_DIR}}/{{.CLUSTER_NAME}}.kubeconfig." status: - - echo ; [ -f {{.OUTPUT_DIR}}/{{.CLUSTER_NAME}}.kubeconfig ] + - test -f {{.OUTPUT_DIR}}/{{.CLUSTER_NAME}}.kubeconfig default-storage-pool: summary: | @@ -252,3 +319,26 @@ tasks: - virsh --connect qemu:///system pool-start --build default status: - virsh --connect qemu:///system pool-info default + + create-external-kubeconfig-secret: + if: '{{eq .EXTERNAL_TINKERBELL "true"}}' + deps: [kind-cluster, tinkerbell-kind-cluster, update-state] + summary: | + When `externalTinkerbell: true`, create the + `external-tinkerbell-kubeconfig` Secret in the management cluster's + `capt-system` namespace so the CAPT controller (which mounts it at + /var/run/secrets/external-tinkerbell/kubeconfig) can talk to the + Tinkerbell KinD cluster's API server. The script labels the secret + with clusterctl.cluster.x-k8s.io/move so a `clusterctl move` carries + it during pivot. No-op when external mode is off. + vars: + EXTERNAL_TINKERBELL: + sh: "yq eval '.externalTinkerbell // false' {{.STATE_FILE_FQ_PATH}}" + TINK_CLUSTER_NAME: + sh: 'yq eval ''.kind.tinkerbell.clusterName // ""'' {{.STATE_FILE_FQ_PATH}}' + MGMT_KUBECONFIG: + sh: "yq eval '.kind.kubeconfig' {{.STATE_FILE_FQ_PATH}}" + cmds: + - ./scripts/create_external_kubeconfig_secret.sh "{{.TINK_CLUSTER_NAME}}" "{{.MGMT_KUBECONFIG}}" + status: + - 'KUBECONFIG="{{.MGMT_KUBECONFIG}}" kubectl get secret external-tinkerbell-kubeconfig -n capt-system >/dev/null 2>&1' diff --git a/capt/tasks/Taskfile-delete.yaml b/capt/tasks/Taskfile-delete.yaml index cd9acba6..9ef34486 100644 --- a/capt/tasks/Taskfile-delete.yaml +++ b/capt/tasks/Taskfile-delete.yaml @@ -6,39 +6,58 @@ tasks: Delete the CAPT playground. cmds: - task: kind-cluster + - task: tinkerbell-kind-cluster - task: vbmc-container - - task: vbmc-generated-files - task: vms - task: default-storage-pool - task: output-dir - - task: remove-auto-generated-ssh-key kind-cluster: summary: | Delete the KinD cluster. vars: CLUSTER_NAME: - sh: yq eval '.clusterName' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.clusterName' {{.STATE_FILE_FQ_PATH}}" cmds: - kind delete cluster --name {{.CLUSTER_NAME}} status: - - got=$(kind get clusters | grep -c {{.CLUSTER_NAME}} || :); [[ "$got" == "0" ]] + # Returns 0 (skip) if cluster is missing; 1 (run) if cluster exists + - "kind get clusters 2>/dev/null | grep -qx '{{.CLUSTER_NAME}}' && exit 1 || exit 0" + + tinkerbell-kind-cluster: + if: '{{eq .EXTERNAL_TINKERBELL "true"}}' + summary: | + Delete the Tinkerbell KinD cluster (if it exists). + vars: + TINK_CLUSTER_NAME: + sh: 'yq eval ''.kind.tinkerbell.clusterName // ""'' {{.STATE_FILE_FQ_PATH}}' + EXTERNAL_TINKERBELL: + sh: "yq eval '.externalTinkerbell // false' {{.STATE_FILE_FQ_PATH}}" + cmds: + - kind delete cluster --name "{{.TINK_CLUSTER_NAME}}" + status: + # Returns 0 (skip) if cluster is missing; 1 (run) if cluster exists + - "kind get clusters 2>/dev/null | grep -qx '{{.TINK_CLUSTER_NAME}}' && exit 1 || exit 0" vms: summary: | Delete the VMs. vars: VM_NAMES: - sh: yq e '.vm.details[] | [key] | @csv' {{.STATE_FILE_FQ_PATH}} - VM_BASE_NAME: - sh: yq eval '.vm.baseName' {{.STATE_FILE_FQ_PATH}} + sh: "yq e '.vm.details | keys | .[]' {{.STATE_FILE_FQ_PATH}}" + VM_NAMES_RE: + sh: 'yq e ''.vm.details | keys | join("|")'' {{.STATE_FILE_FQ_PATH}}' cmds: - for: { var: VM_NAMES } - cmd: (virsh --connect qemu:///system destroy {{.ITEM}} || true) ## if the VM is already off, this will fail - - for: { var: VM_NAMES } - cmd: virsh --connect qemu:///system undefine --nvram --remove-all-storage {{.ITEM}} + cmd: | + virsh --connect qemu:///system dominfo {{.ITEM}} >/dev/null 2>&1 || exit 0 + virsh --connect qemu:///system destroy {{.ITEM}} >/dev/null 2>&1 || true + virsh --connect qemu:///system undefine --nvram --remove-all-storage {{.ITEM}} status: - - got=$(virsh --connect qemu:///system list --all --name | grep -ce "{{.VM_BASE_NAME}}*" || :); [[ "$got" == "0" ]] + # Same shape as kind-cluster's status (line 26). grep -xE = whole-line + # ERE match, so 'node1|node2|node3' only matches those exact names — + # not 'nod', not 'nodeeee', not unrelated host VMs like 'virtual'. + - "virsh --connect qemu:///system list --all --name 2>/dev/null | grep -qxE '{{.VM_NAMES_RE}}' && exit 1 || exit 0" default-storage-pool: summary: | @@ -48,28 +67,29 @@ tasks: - virsh --connect qemu:///system pool-destroy default || true - virsh --connect qemu:///system pool-undefine default status: - - (! virsh --connect qemu:///system pool-info default ) + - virsh --connect qemu:///system pool-info default && exit 1 || exit 0 vbmc-container: summary: | Delete the Virtual BMC container. vars: VBMC_CONTAINER_NAME: - sh: yq eval '.virtualBMC.containerName' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.virtualBMC.containerName' {{.STATE_FILE_FQ_PATH}}" cmds: - docker rm -f {{.VBMC_CONTAINER_NAME}} status: - - got=$(docker ps -a | grep -c {{.VBMC_CONTAINER_NAME}} || :); [[ "$got" == "0" ]] + - docker inspect {{.VBMC_CONTAINER_NAME}} >/dev/null 2>&1 && exit 1 || exit 0 vbmc-generated-files: summary: | - Delete the Virtual BMC generated files. + Delete the Virtual BMC generated files. Kept for standalone invocation; + the playground delete flow relies on `output-dir` to wipe these. cmds: - - rm -f {{.CURR_DIR}}/scripts/htpasswd {{.CURR_DIR}}/scripts/sushy.key {{.CURR_DIR}}/scripts/sushy.cert + - rm -f {{.OUTPUT_DIR}}/htpasswd {{.OUTPUT_DIR}}/sushy.key {{.OUTPUT_DIR}}/sushy.cert status: - - test ! -f {{.CURR_DIR}}/scripts/htpasswd - - test ! -f {{.CURR_DIR}}/scripts/sushy.key - - test ! -f {{.CURR_DIR}}/scripts/sushy.cert + - test ! -f {{.OUTPUT_DIR}}/htpasswd + - test ! -f {{.OUTPUT_DIR}}/sushy.key + - test ! -f {{.OUTPUT_DIR}}/sushy.cert output-dir: summary: | @@ -77,13 +97,4 @@ tasks: cmds: - rm -rf {{.OUTPUT_DIR}} status: - - echo ;[ ! -d {{.OUTPUT_DIR}} ] - - remove-auto-generated-ssh-key: - summary: | - Remove the auto generated ssh key from the .state file. - cmds: - - yq e -i '.os.sshKey = ""' {{.STATE_FILE_FQ_PATH}} - - yq e -i '.os.sshKeyAutoGenerated = false' {{.STATE_FILE_FQ_PATH}} - status: - - v=$(yq eval '.os.sshKey' {{.CONFIG_FILE_FQ_PATH}}); [[ "$v" != "" ]] || [[ "$(yq eval '.os.sshKey' {{.STATE_FILE_FQ_PATH}})" == "" ]] + - test ! -d {{.OUTPUT_DIR}} diff --git a/capt/tasks/Taskfile-helm.yaml b/capt/tasks/Taskfile-helm.yaml new file mode 100644 index 00000000..1f5478de --- /dev/null +++ b/capt/tasks/Taskfile-helm.yaml @@ -0,0 +1,63 @@ +version: "3" + +# Shared Tinkerbell Helm chart deployment task. Consumed by both +# Taskfile-create.yaml (initial playground setup) and +# Taskfile-capi-pivot.yaml (re-deploy after pivot). Callers pass +# KUBECONFIG via `vars:` to target the appropriate cluster. + +tasks: + deploy-tinkerbell-helm-chart: + run: once + summary: | + Deploy the Tinkerbell Helm chart to the cluster identified by + KUBECONFIG. When bootMode is isoboot, Captain OS artifact names + (ISO, kernel, initrd) are computed from captainos.kernelVersion + and arch in .state, matching the upstream naming convention: + captainos--.iso + vmlinuz- + initramfs- + vars: + # KUBECONFIG must be supplied by the caller. + LB_IP: + sh: "yq eval '.tinkerbell.vip' {{.STATE_FILE_FQ_PATH}}" + LB_IP2: + sh: "yq eval '.tinkerbell.hookosVip' {{.STATE_FILE_FQ_PATH}}" + TRUSTED_PROXIES: + sh: "KUBECONFIG={{.KUBECONFIG}} kubectl get nodes -o jsonpath='{.items[*].spec.podCIDR}' | tr ' ' ','" + STACK_CHART_VERSION: + sh: "yq eval '.versions.chart' {{.STATE_FILE_FQ_PATH}}" + NAMESPACE: + sh: "yq eval '.namespace' {{.STATE_FILE_FQ_PATH}}" + LOCATION: + sh: "yq eval '.chart.location' {{.STATE_FILE_FQ_PATH}}" + CHART_NAME: tinkerbell + BOOTMODE: + sh: "yq eval '.bootMode' {{.STATE_FILE_FQ_PATH}}" + # Map OCI arch (amd64/arm64) to the kernel arch used in Captain OS + # artifact filenames (x86_64/aarch64). + OSIE_ARCH: + sh: "yq eval '.arch' {{.STATE_FILE_FQ_PATH}} | sed 's/amd64/x86_64/;s/arm64/aarch64/'" + KERNEL_VER: + sh: 'yq eval ''.captainos.kernelVersion // ""'' {{.STATE_FILE_FQ_PATH}}' + GLOBAL_VARS: + - trustedProxies={"{{.TRUSTED_PROXIES}}"} + - publicIP={{.LB_IP}} + - artifactsFileServer=http://{{.LB_IP2}}:7173 + - optional.captainos.enabled=true + ISO_VARS: + - deployment.envs.smee.dhcpEnabled=false + - deployment.envs.smee.isoUpstreamURL=http://{{.LB_IP2}}:7173/captainos-{{.KERNEL_VER}}-{{.OSIE_ARCH}}.iso + NETBOOT_VARS: + - deployment.envs.smee.ipxeHttpScriptKernelName=vmlinuz-{{.KERNEL_VER}} + - deployment.envs.smee.ipxeHttpScriptInitrdName=initramfs-{{.KERNEL_VER}} + EXTRA_VARS: + sh: "yq eval '.chart.extraVars | .[]' {{.STATE_FILE_FQ_PATH}} | xargs" + cmds: + - >- + KUBECONFIG="{{.KUBECONFIG}}" helm upgrade --install {{.CHART_NAME}} {{.LOCATION}} --version "{{.STACK_CHART_VERSION}}" --create-namespace --namespace {{.NAMESPACE}} --wait + {{range .GLOBAL_VARS}}--set "{{.}}" {{end}} + {{- if eq .BOOTMODE "netboot" }} {{- range .NETBOOT_VARS }}--set "{{.}}" {{ end }} {{ end }} + {{- if eq .BOOTMODE "isoboot" }} {{- range .ISO_VARS }}--set "{{.}}" {{ end }} {{ end }} + {{- if .EXTRA_VARS }} {{- range (splitList " " .EXTRA_VARS ) }}--set "{{.}}" {{ end }} {{ end }} + status: + - helm_status=$(KUBECONFIG="{{.KUBECONFIG}}" helm status -n {{.NAMESPACE}} {{.CHART_NAME}} -o yaml | yq .info.status); [[ "$helm_status" == "deployed" ]] diff --git a/capt/tasks/Taskfile-mirror.yaml b/capt/tasks/Taskfile-mirror.yaml new file mode 100644 index 00000000..5250d613 --- /dev/null +++ b/capt/tasks/Taskfile-mirror.yaml @@ -0,0 +1,86 @@ +version: "3" + +# Optional pull-through OCI registry mirror tasks. The single render-only +# task here (render-kind-config) self-gates on `.registryMirror.enabled` +# in .state — when disabled it's a strict no-op via its `status:` guard +# and writes nothing. Both kind-cluster and tinkerbell-kind-cluster in +# Taskfile-create.yaml depend on it and conditionally pass --config to +# `kind create` based on the same enabled flag. +# +# Where the rest of the redirection happens (NOT in this file): +# - Image strings pulled OUTSIDE any containerd (chart.location, +# os.registry, virtualBMC.image) are rewritten in CUE by +# cue/state/mirror_extension.cue using cue/mirror/#rewrite, so they +# land in .state already pointing at the mirror host. +# - kind nodes get the mirror via cue/kind/kind.cue → nodes[].extraMounts, +# bind-mounting the host-rendered /certs.d tree into each node +# at /etc/containerd/certs.d. kind's default containerd config already +# sets `config_path` there, so no containerdConfigPatches are needed. +# - Workload (CAPT) nodes get the same hosts.toml drop-ins via +# KubeadmConfigSpec.files, baked in by cue/capi/resources.cue using +# cue/mirror/#cloudInitFiles. +# Everything pulled by a mirrored containerd (workflow actions, kube-vip +# `ctr` pulls, all helm-chart Pod images including captainos, kubeadm +# pulls) is left referencing the original upstream hostname — the +# containerd registry-mirror config redirects them at pull time. +# +# To remove the entire feature: delete this file, the cue/mirror and +# cue/kind packages, the *_extension.cue files in cue/values and +# cue/state, the `mirror:` line in ../Taskfile.yaml#includes, the +# `deps: [mirror:render-kind-config]` and the `KIND_CONFIG_FLAG` var +# in Taskfile-create.yaml#kind-cluster and #tinkerbell-kind-cluster, the +# mirror references in cue/capi/resources.cue (_mirrorFiles / +# _mirrorPreCmds), and the `registryMirror:` block in ../config.yaml. +# +# Invariants: +# - HTTPS only. No insecure knobs anywhere. +# - When the mirror is disabled, the render task is a strict no-op and +# writes no file. When enabled, the render runs each invocation +# (sub-second cue export); we accept that over the alternative of +# writing a stale file when the feature is off. + +tasks: + render-kind-config: + summary: | + Render output/kind-config.yaml from cue/kind AND the per-upstream + hosts.toml tree under output/certs.d//hosts.toml from cue/mirror. + The kind config bind-mounts that tree into every node at + /etc/containerd/certs.d via nodes[].extraMounts. Both kind-cluster + and tinkerbell-kind-cluster consume the kind-config.yaml file (a + single file serves both — cue/kind has no per-cluster differentiation + today). + + Strict no-op when the mirror is disabled — neither the kind-config + nor the certs.d tree is written, kind-cluster's KIND_CONFIG_FLAG is + empty, and `kind create` runs without `--config`. + + We deliberately use `status:` rather than go-task's + `sources:`/`generates:` fingerprinting: when `status:` returns + non-zero (i.e. mirror enabled) go-task ignores any sources/generates + entries and always runs the cmds, so the two are mutually exclusive. + The cue renders are sub-second, and the user-experience win of + "no files when feature is off" outweighs skipping a fast cue + invocation when it's on. + vars: + ENABLED: + sh: "yq eval '.registryMirror.enabled // false' {{.STATE_FILE_FQ_PATH}}" + KIND_CONFIG: "{{.OUTPUT_DIR}}/kind-config.yaml" + CERTS_DIR: "{{.OUTPUT_DIR}}/certs.d" + cmds: + - "cue export ./cue/kind yaml: {{.STATE_FILE_FQ_PATH}} -l 'values:' -e out --out yaml > {{.KIND_CONFIG}}" + # Render the host-side certs.d tree from cue/mirror#hostCertsdFiles. + # CUE emits [{path, content}, ...]; we iterate by index with yq so + # multi-line `content` values stay intact (no awk/sed escaping). Wipe + # and recreate the dir so removed upstreams don't leave stale entries. + - | + rm -rf "{{.CERTS_DIR}}" + mkdir -p "{{.CERTS_DIR}}" + manifest="$(cue export ./cue/mirror yaml: {{.STATE_FILE_FQ_PATH}} -l 'values:' -e hostCertsdFiles --out json)" + count=$(yq -p json '. | length' <<<"$manifest") + for i in $(seq 0 $((count - 1))); do + path=$(yq -p json ".[$i].path" <<<"$manifest") + mkdir -p "{{.CERTS_DIR}}/$(dirname "$path")" + yq -p json -r ".[$i].content" <<<"$manifest" > "{{.CERTS_DIR}}/$path" + done + status: + - '[[ "{{.ENABLED}}" != "true" ]]' diff --git a/capt/tasks/Taskfile-tools.yaml b/capt/tasks/Taskfile-tools.yaml new file mode 100644 index 00000000..a1572975 --- /dev/null +++ b/capt/tasks/Taskfile-tools.yaml @@ -0,0 +1,139 @@ +version: "3" + +# Apply bash-style strict mode to every `cmds:` block in this Taskfile. +# Replaces the per-task `set -euo pipefail` heredoc preamble. +# https://taskfile.dev/docs/reference/schema#set-options +set: [errexit, nounset, pipefail] + +tasks: + validate-binaries: + summary: | + Validate that the required system dependencies are installed. + Tools that are pinned and installed locally into ./bin are handled by + the install-binaries task. + vars: + REQUIRED_SYSTEM_BINS: "virsh docker virt-install curl tar ssh-keygen" + cmds: + - for: { var: REQUIRED_SYSTEM_BINS, split: " " } + cmd: command -v {{ .ITEM }} >/dev/null || { echo "'{{ .ITEM }}' not found in \$PATH"; exit 1; } + status: + - for bin in {{.REQUIRED_SYSTEM_BINS}}; do command -v "$bin" >/dev/null || exit 1; done + + ensure-bin-dir: + summary: | + Create the local bin/ directory used by install-binaries. + cmds: + - mkdir -p {{.ROOT_DIR}}/bin + status: + - test -d {{.ROOT_DIR}}/bin + + install-binaries: + deps: [ensure-bin-dir] + dir: "{{.ROOT_DIR}}/bin" + summary: | + Download pinned versions of cue, helm, kind, clusterctl, kubectl, and yq + into ./bin from each project's upstream release artifacts. Idempotent: + each tool is reinstalled only if missing or its version output does not + match the pin. + cmds: + - task: install-binary-cue + - task: install-binary-helm + - task: install-binary-kind + - task: install-binary-clusterctl + - task: install-binary-kubectl + - task: install-binary-yq + status: + - test "$(readlink cue)" = "cue-{{.CUE_VERSION}}" && test -x cue-{{.CUE_VERSION}} + - test "$(readlink helm)" = "helm-{{.HELM_VERSION}}" && test -x helm-{{.HELM_VERSION}} + - test "$(readlink kind)" = "kind-{{.KIND_VERSION}}" && test -x kind-{{.KIND_VERSION}} + - test "$(readlink clusterctl)" = "clusterctl-{{.CLUSTERCTL_VERSION}}" && test -x clusterctl-{{.CLUSTERCTL_VERSION}} + - test "$(readlink kubectl)" = "kubectl-{{.KUBECTL_VERSION}}" && test -x kubectl-{{.KUBECTL_VERSION}} + - test "$(readlink yq)" = "yq-{{.YQ_VERSION}}" && test -x yq-{{.YQ_VERSION}} + + install-binary-cue: + internal: true + silent: true + deps: [ensure-bin-dir] + dir: "{{.ROOT_DIR}}/bin" + cmds: + # cue ships a tarball with the binary at the top level. + - | + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + curl -fsSL -o "$tmp/cue.tgz" "https://github.com/cue-lang/cue/releases/download/{{.CUE_VERSION}}/cue_{{.CUE_VERSION}}_{{.OS}}_{{.ARCH}}.tar.gz" + tar -C "$tmp" -xzf "$tmp/cue.tgz" cue + install -m 0755 "$tmp/cue" cue-{{.CUE_VERSION}} + - ln -sfn cue-{{.CUE_VERSION}} cue + status: + - test -x cue-{{.CUE_VERSION}} + - test "$(readlink cue)" = "cue-{{.CUE_VERSION}}" + + install-binary-helm: + internal: true + silent: true + deps: [ensure-bin-dir] + dir: "{{.ROOT_DIR}}/bin" + cmds: + # helm ships a tarball with the binary at -/helm. + - | + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + curl -fsSL -o "$tmp/helm.tgz" "https://get.helm.sh/helm-{{.HELM_VERSION}}-{{.OS}}-{{.ARCH}}.tar.gz" + tar -C "$tmp" -xzf "$tmp/helm.tgz" "{{.OS}}-{{.ARCH}}/helm" + install -m 0755 "$tmp/{{.OS}}-{{.ARCH}}/helm" helm-{{.HELM_VERSION}} + - ln -sfn helm-{{.HELM_VERSION}} helm + status: + - test -x helm-{{.HELM_VERSION}} + - test "$(readlink helm)" = "helm-{{.HELM_VERSION}}" + + install-binary-kind: + internal: true + silent: true + deps: [ensure-bin-dir] + dir: "{{.ROOT_DIR}}/bin" + cmds: + - curl -fsSL -o kind-{{.KIND_VERSION}} "https://github.com/kubernetes-sigs/kind/releases/download/{{.KIND_VERSION}}/kind-{{.OS}}-{{.ARCH}}" + - chmod +x kind-{{.KIND_VERSION}} + - ln -sfn kind-{{.KIND_VERSION}} kind + status: + - test -x kind-{{.KIND_VERSION}} + - test "$(readlink kind)" = "kind-{{.KIND_VERSION}}" + + install-binary-clusterctl: + internal: true + silent: true + deps: [ensure-bin-dir] + dir: "{{.ROOT_DIR}}/bin" + cmds: + - curl -fsSL -o clusterctl-{{.CLUSTERCTL_VERSION}} "https://github.com/kubernetes-sigs/cluster-api/releases/download/{{.CLUSTERCTL_VERSION}}/clusterctl-{{.OS}}-{{.ARCH}}" + - chmod +x clusterctl-{{.CLUSTERCTL_VERSION}} + - ln -sfn clusterctl-{{.CLUSTERCTL_VERSION}} clusterctl + status: + - test -x clusterctl-{{.CLUSTERCTL_VERSION}} + - test "$(readlink clusterctl)" = "clusterctl-{{.CLUSTERCTL_VERSION}}" + + install-binary-kubectl: + internal: true + silent: true + deps: [ensure-bin-dir] + dir: "{{.ROOT_DIR}}/bin" + cmds: + - curl -fsSL -o kubectl-{{.KUBECTL_VERSION}} "https://dl.k8s.io/release/{{.KUBECTL_VERSION}}/bin/{{.OS}}/{{.ARCH}}/kubectl" + - chmod +x kubectl-{{.KUBECTL_VERSION}} + - ln -sfn kubectl-{{.KUBECTL_VERSION}} kubectl + status: + - test -x kubectl-{{.KUBECTL_VERSION}} + - test "$(readlink kubectl)" = "kubectl-{{.KUBECTL_VERSION}}" + + install-binary-yq: + internal: true + silent: true + deps: [ensure-bin-dir] + dir: "{{.ROOT_DIR}}/bin" + cmds: + - curl -fsSL -o yq-{{.YQ_VERSION}} "https://github.com/mikefarah/yq/releases/download/{{.YQ_VERSION}}/yq_{{.OS}}_{{.ARCH}}" + - chmod +x yq-{{.YQ_VERSION}} + - ln -sfn yq-{{.YQ_VERSION}} yq + status: + - test -x yq-{{.YQ_VERSION}} + - test "$(readlink yq)" = "yq-{{.YQ_VERSION}}" diff --git a/capt/tasks/Taskfile-vbmc.yaml b/capt/tasks/Taskfile-vbmc.yaml index 18a7f93a..0fa93654 100644 --- a/capt/tasks/Taskfile-vbmc.yaml +++ b/capt/tasks/Taskfile-vbmc.yaml @@ -7,18 +7,26 @@ tasks: Prepare the virtualbmc server. vars: VBMC_CONTAINER_IMAGE: - sh: yq eval '.virtualBMC.image' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.virtualBMC.image' {{.STATE_FILE_FQ_PATH}}" USERNAME: - sh: yq eval '.virtualBMC.user' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.virtualBMC.user' {{.STATE_FILE_FQ_PATH}}" PASSWORD: - sh: yq eval '.virtualBMC.pass' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.virtualBMC.pass' {{.STATE_FILE_FQ_PATH}}" + # Absolute path is required for `docker -v` bind mounts (relative paths + # are interpreted as named volumes). state.cue resolves outputDir to an + # absolute path when generating .state. + OUTPUT_DIR_ABS: + sh: "yq eval '.outputDir' {{.STATE_FILE_FQ_PATH}}" cmds: - - docker run -it --rm --entrypoint htpasswd {{.VBMC_CONTAINER_IMAGE}} -nbB "{{.USERNAME}}" "{{.PASSWORD}}" > {{.CURR_DIR}}/scripts/htpasswd - - docker run -it --rm --entrypoint openssl -v {{.CURR_DIR}}/scripts:/scripts {{.VBMC_CONTAINER_IMAGE}} req -x509 -newkey rsa:2048 -keyout /scripts/sushy.key -out /scripts/sushy.cert -days 365 -nodes -subj "/C=US/ST=CA/L=Los Angeles/O=Engineering/OU=Engineering/CN=tinkerbell.org" + - mkdir -p {{.OUTPUT_DIR}} + # Pass the password via stdin (-i) instead of argv (-b) so it does not + # appear in `ps`/`docker ps --no-trunc` while htpasswd runs. + - printf '%s' "{{.PASSWORD}}" | docker run -i --rm --entrypoint htpasswd {{.VBMC_CONTAINER_IMAGE}} -niB "{{.USERNAME}}" > {{.OUTPUT_DIR}}/htpasswd + - docker run -it --rm --entrypoint openssl -v {{.OUTPUT_DIR_ABS}}:/scripts {{.VBMC_CONTAINER_IMAGE}} req -x509 -newkey rsa:2048 -keyout /scripts/sushy.key -out /scripts/sushy.cert -days 365 -nodes -subj "/C=US/ST=CA/L=Los Angeles/O=Engineering/OU=Engineering/CN=tinkerbell.org" status: - - test -f {{.CURR_DIR}}/scripts/htpasswd - - test -f {{.CURR_DIR}}/scripts/sushy.key - - test -f {{.CURR_DIR}}/scripts/sushy.cert + - test -f {{.OUTPUT_DIR}}/htpasswd + - test -f {{.OUTPUT_DIR}}/sushy.key + - test -f {{.OUTPUT_DIR}}/sushy.cert start-server: run: once @@ -27,11 +35,14 @@ tasks: Start the virtualbmc server. Requires the "kind" docker network to exist. vars: VBMC_CONTAINER_NAME: - sh: yq eval '.virtualBMC.containerName' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.virtualBMC.containerName' {{.STATE_FILE_FQ_PATH}}" VBMC_CONTAINER_IMAGE: - sh: yq eval '.virtualBMC.image' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.virtualBMC.image' {{.STATE_FILE_FQ_PATH}}" + # Absolute path required for `docker -v` bind mounts (see prepare). + OUTPUT_DIR_ABS: + sh: "yq eval '.outputDir' {{.STATE_FILE_FQ_PATH}}" cmds: - - docker run -d --privileged --rm --network kind -e SUSHY_EMULATOR_CONFIG=/etc/sushy/sushy-emulator.conf -v /var/run/libvirt:/var/run/libvirt -v "${PWD}/scripts/sushy.key:/etc/sushy/sushy.key" -v "${PWD}/scripts/sushy.cert:/etc/sushy/sushy.cert" -v "${PWD}/scripts/sushy-tools.conf:/etc/sushy/sushy-emulator.conf" -v "${PWD}/scripts/htpasswd:/etc/sushy/htpasswd" --name {{.VBMC_CONTAINER_NAME}} {{.VBMC_CONTAINER_IMAGE}} + - docker run -d --privileged --rm --network kind -e SUSHY_EMULATOR_CONFIG=/etc/sushy/sushy-emulator.conf -v /var/run/libvirt:/var/run/libvirt -v "{{.OUTPUT_DIR_ABS}}/sushy.key:/etc/sushy/sushy.key" -v "{{.OUTPUT_DIR_ABS}}/sushy.cert:/etc/sushy/sushy.cert" -v "${PWD}/scripts/sushy-tools.conf:/etc/sushy/sushy-emulator.conf" -v "{{.OUTPUT_DIR_ABS}}/htpasswd:/etc/sushy/htpasswd" --name {{.VBMC_CONTAINER_NAME}} {{.VBMC_CONTAINER_IMAGE}} status: - docker ps | grep -q {{.VBMC_CONTAINER_NAME}} @@ -42,7 +53,7 @@ tasks: Register and start the virtualbmc servers. Requires that the virtual machines exist. vars: VBMC_NAME: - sh: yq e '.virtualBMC.containerName' {{.STATE_FILE_FQ_PATH}} + sh: "yq e '.virtualBMC.containerName' {{.STATE_FILE_FQ_PATH}}" cmds: - ./scripts/virtualbmc.sh {{.STATE_FILE_FQ_PATH}} status: @@ -55,7 +66,7 @@ tasks: Update the state file with the virtual bmc server information. vars: VBMC_CONTAINER_NAME: - sh: yq eval '.virtualBMC.containerName' {{.STATE_FILE_FQ_PATH}} + sh: "yq eval '.virtualBMC.containerName' {{.STATE_FILE_FQ_PATH}}" cmds: - vbmc_ip=$(docker inspect -f '{{`{{ .NetworkSettings.Networks.kind.IPAddress }}`}}' {{.VBMC_CONTAINER_NAME}}); yq e -i '.virtualBMC.ip = "'$vbmc_ip'"' {{.STATE_FILE_FQ_PATH}} status: diff --git a/capt/templates/bmc-machine.tmpl b/capt/templates/bmc-machine.tmpl deleted file mode 100644 index d76dc7fb..00000000 --- a/capt/templates/bmc-machine.tmpl +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: bmc.tinkerbell.org/v1alpha1 -kind: Machine -metadata: - name: $NODE_NAME - namespace: $NAMESPACE -spec: - connection: - authSecretRef: - name: bmc-creds - namespace: $NAMESPACE - host: $BMC_IP - insecureTLS: true - port: $BMC_PORT - providerOptions: - preferredOrder: - - ipmitool - ipmitool: - port: $BMC_PORT - redfish: - useBasicAuth: true - systemName: $NODE_NAME diff --git a/capt/templates/bmc-secret.tmpl b/capt/templates/bmc-secret.tmpl deleted file mode 100644 index 85b62c78..00000000 --- a/capt/templates/bmc-secret.tmpl +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: bmc-creds - namespace: $NAMESPACE - labels: - clusterctl.cluster.x-k8s.io/move: "" - clusterctl.cluster.x-k8s.io: "" -data: - password: $BMC_PASS_BASE64 - username: $BMC_USER_BASE64 -type: kubernetes.io/basic-auth diff --git a/capt/templates/clusterctl.tmpl b/capt/templates/clusterctl.tmpl deleted file mode 100644 index e2519553..00000000 --- a/capt/templates/clusterctl.tmpl +++ /dev/null @@ -1,7 +0,0 @@ -providers: - - name: "tinkerbell" - url: "$LOCATION/$CAPT_VERSION/infrastructure-components.yaml" - type: "InfrastructureProvider" -images: - infrastructure-tinkerbell: - tag: $CAPT_VERSION \ No newline at end of file diff --git a/capt/templates/hardware.tmpl b/capt/templates/hardware.tmpl deleted file mode 100644 index 55a40c1b..00000000 --- a/capt/templates/hardware.tmpl +++ /dev/null @@ -1,35 +0,0 @@ -apiVersion: tinkerbell.org/v1alpha1 -kind: Hardware -metadata: - labels: - tinkerbell.org/role: $NODE_ROLE - name: $NODE_NAME - namespace: $NAMESPACE -spec: - bmcRef: - apiGroup: bmc.tinkerbell.org - kind: Machine - name: $NODE_NAME - disks: - - device: /dev/vda - interfaces: - - dhcp: - arch: x86_64 - hostname: $NODE_NAME - ip: - address: $NODE_IP - gateway: $GATEWAY_IP - netmask: 255.255.0.0 - lease_time: 4294967294 - mac: $NODE_MAC - uefi: true - name_servers: - - 8.8.8.8 - - 1.1.1.1 - netboot: - allowPXE: true - allowWorkflow: true - metadata: - instance: - hostname: $NODE_NAME - id: $NODE_MAC \ No newline at end of file diff --git a/capt/templates/kustomization-iso.tmpl b/capt/templates/kustomization-iso.tmpl deleted file mode 100644 index 88e354a2..00000000 --- a/capt/templates/kustomization-iso.tmpl +++ /dev/null @@ -1,300 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: $NAMESPACE -resources: - - prekustomization.yaml -patches: - - target: - group: infrastructure.cluster.x-k8s.io - kind: TinkerbellMachineTemplate - name: ".*control-plane.*" - version: v1beta1 - patch: |- - - op: add - path: /spec/template/spec - value: - bootOptions: - bootMode: isoboot - isoURL: "http://$TINKERBELL_VIP:7171/iso/hook.iso" - hardwareAffinity: - required: - - labelSelector: - matchLabels: - tinkerbell.org/role: control-plane - templateOverride: | - version: "0.1" - name: playground-template - global_timeout: 6000 - tasks: - - name: "playground-template" - worker: "{{.device_1}}" - volumes: - - /dev:/dev - - /dev/console:/dev/console - - /lib/firmware:/lib/firmware:ro - actions: - - name: "stream image" - image: quay.io/tinkerbell/actions/oci2disk - timeout: 1200 - environment: - IMG_URL: $OS_REGISTRY/$OS_DISTRO-$OS_VERSION:$KUBE_VERSION.gz - DEST_DISK: {{ index .Hardware.Disks 0 }} - COMPRESSED: true - - name: "add tink cloud-init config" - image: quay.io/tinkerbell/actions/writefile - timeout: 90 - environment: - DEST_DISK: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} - FS_TYPE: ext4 - DEST_PATH: /etc/cloud/cloud.cfg.d/10_tinkerbell.cfg - UID: 0 - GID: 0 - MODE: 0600 - DIRMODE: 0700 - CONTENTS: | - datasource: - Ec2: - metadata_urls: ["http://$TINKERBELL_VIP:7172"] - strict_id: false - system_info: - default_user: - name: tink - groups: [wheel, adm] - sudo: ["ALL=(ALL) NOPASSWD:ALL"] - shell: /bin/bash - manage_etc_hosts: localhost - warnings: - dsid_missing_source: off - - name: "disable cloud-init networking" - image: quay.io/tinkerbell/actions/writefile - timeout: 90 - environment: - CONTENTS: 'network: {config: disabled}' - DEST_DISK: '{{ formatPartition ( index .Hardware.Disks 0 ) 1 }}' - DEST_PATH: /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg - DIRMODE: "0700" - FS_TYPE: ext4 - GID: "0" - MODE: "0600" - UID: "0" - - name: "create static netplan" - image: quay.io/tinkerbell/actions/writefile - timeout: 90 - environment: - CONTENTS: | - network: - version: 2 - renderer: networkd - ethernets: - id0: - match: - macaddress: {{ (index .Hardware.Interfaces 0).DHCP.MAC }} - addresses: - - {{ (index .Hardware.Interfaces 0).DHCP.IP.Address }}/16 - nameservers: - addresses: [{{ (index .Hardware.Interfaces 0).DHCP.NameServers | join ","}}] - routes: - - to: default - via: {{ (index .Hardware.Interfaces 0).DHCP.IP.Gateway }} - DEST_DISK: '{{ formatPartition ( index .Hardware.Disks 0 ) 1 }}' - DEST_PATH: /etc/netplan/config.yaml - DIRMODE: "0755" - FS_TYPE: ext4 - GID: "0" - MODE: "0600" - UID: "0" - - name: "add tink cloud-init ds-config" - image: quay.io/tinkerbell/actions/writefile - timeout: 90 - environment: - DEST_DISK: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} - FS_TYPE: ext4 - DEST_PATH: /etc/cloud/ds-identify.cfg - UID: 0 - GID: 0 - MODE: 0600 - DIRMODE: 0700 - CONTENTS: | - datasource: Ec2 - - name: "kexec image" - image: ghcr.io/jacobweinstock/waitdaemon:0.2.2 - timeout: 90 - pid: host - environment: - BLOCK_DEVICE: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} - FS_TYPE: ext4 - IMAGE: quay.io/tinkerbell/actions/kexec - WAIT_SECONDS: 5 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - target: - group: infrastructure.cluster.x-k8s.io - kind: TinkerbellMachineTemplate - name: ".*worker.*" - version: v1beta1 - patch: |- - - op: add - path: /spec/template/spec - value: - bootOptions: - bootMode: isoboot - isoURL: "http://$TINKERBELL_VIP:7171/iso/hook.iso" - hardwareAffinity: - required: - - labelSelector: - matchLabels: - tinkerbell.org/role: worker - templateOverride: | - version: "0.1" - name: playground-template - global_timeout: 6000 - tasks: - - name: "playground-template" - worker: "{{.device_1}}" - volumes: - - /dev:/dev - - /dev/console:/dev/console - - /lib/firmware:/lib/firmware:ro - actions: - - name: "stream image" - image: quay.io/tinkerbell/actions/oci2disk - timeout: 1200 - environment: - IMG_URL: $OS_REGISTRY/$OS_DISTRO-$OS_VERSION:$KUBE_VERSION.gz - DEST_DISK: {{ index .Hardware.Disks 0 }} - COMPRESSED: true - - name: "add tink cloud-init config" - image: quay.io/tinkerbell/actions/writefile - timeout: 90 - environment: - DEST_DISK: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} - FS_TYPE: ext4 - DEST_PATH: /etc/cloud/cloud.cfg.d/10_tinkerbell.cfg - UID: 0 - GID: 0 - MODE: 0600 - DIRMODE: 0700 - CONTENTS: | - datasource: - Ec2: - metadata_urls: ["http://$TINKERBELL_VIP:7172"] - strict_id: false - system_info: - default_user: - name: tink - groups: [wheel, adm] - sudo: ["ALL=(ALL) NOPASSWD:ALL"] - shell: /bin/bash - manage_etc_hosts: localhost - warnings: - dsid_missing_source: off - - name: "disable cloud-init networking" - image: quay.io/tinkerbell/actions/writefile - timeout: 90 - environment: - CONTENTS: 'network: {config: disabled}' - DEST_DISK: '{{ formatPartition ( index .Hardware.Disks 0 ) 1 }}' - DEST_PATH: /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg - DIRMODE: "0700" - FS_TYPE: ext4 - GID: "0" - MODE: "0600" - UID: "0" - - name: "create static netplan" - image: quay.io/tinkerbell/actions/writefile - timeout: 90 - environment: - CONTENTS: | - network: - version: 2 - renderer: networkd - ethernets: - id0: - match: - macaddress: {{ (index .Hardware.Interfaces 0).DHCP.MAC }} - addresses: - - {{ (index .Hardware.Interfaces 0).DHCP.IP.Address }}/16 - nameservers: - addresses: [{{ (index .Hardware.Interfaces 0).DHCP.NameServers | join ","}}] - routes: - - to: default - via: {{ (index .Hardware.Interfaces 0).DHCP.IP.Gateway }} - DEST_DISK: '{{ formatPartition ( index .Hardware.Disks 0 ) 1 }}' - DEST_PATH: /etc/netplan/config.yaml - DIRMODE: "0755" - FS_TYPE: ext4 - GID: "0" - MODE: "0600" - UID: "0" - - name: "add tink cloud-init ds-config" - image: quay.io/tinkerbell/actions/writefile - timeout: 90 - environment: - DEST_DISK: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} - FS_TYPE: ext4 - DEST_PATH: /etc/cloud/ds-identify.cfg - UID: 0 - GID: 0 - MODE: 0600 - DIRMODE: 0700 - CONTENTS: | - datasource: Ec2 - - name: "kexec image" - image: ghcr.io/jacobweinstock/waitdaemon:0.2.2 - timeout: 90 - pid: host - environment: - BLOCK_DEVICE: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} - FS_TYPE: ext4 - IMAGE: quay.io/tinkerbell/actions/kexec - WAIT_SECONDS: 5 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - target: - group: infrastructure.cluster.x-k8s.io - kind: TinkerbellCluster - name: ".*" - version: v1beta1 - patch: |- - - op: add - path: /spec - value: - imageLookupBaseRegistry: "$OS_REGISTRY" - imageLookupOSDistro: "$OS_DISTRO" - imageLookupOSVersion: "$VERSIONS_OS" - - target: - group: bootstrap.cluster.x-k8s.io - kind: KubeadmConfigTemplate - name: "$CLUSTER_NAME-.*" - version: v1beta1 - patch: |- - - op: add - path: /spec/template/spec/users - value: - - name: tink - sudo: ALL=(ALL) NOPASSWD:ALL - sshAuthorizedKeys: - - $SSH_AUTH_KEY - - target: - group: controlplane.cluster.x-k8s.io - kind: KubeadmControlPlane - name: "$CLUSTER_NAME-.*" - version: v1beta1 - patch: |- - - op: add - path: /spec/kubeadmConfigSpec/users - value: - - name: tink - sudo: ALL=(ALL) NOPASSWD:ALL - sshAuthorizedKeys: - - $SSH_AUTH_KEY - - target: - group: controlplane.cluster.x-k8s.io - kind: KubeadmControlPlane - name: "$CLUSTER_NAME-.*" - version: v1beta1 - patch: |- - - op: add - path: /spec/kubeadmConfigSpec/preKubeadmCommands - value: - - if [ $(cat /etc/kubernetes-version | awk -F. '{print $2}') -ge 29 ] && [ -f /run/kubeadm/kubeadm.yaml ]; then export KUBE_FILE=/etc/kubernetes/super-admin.conf; else export KUBE_FILE=/etc/kubernetes/admin.conf; fi && mkdir -p /etc/kubernetes/manifests && ctr images pull ghcr.io/kube-vip/kube-vip:v$KUBEVIP_VERSION && ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:v$KUBEVIP_VERSION vip /kube-vip manifest pod --arp --interface $(ip -4 -j route list default | jq -r .[0].dev) --address $CONTROL_PLANE_VIP --controlplane --leaderElection --k8sConfigPath $KUBE_FILE > /etc/kubernetes/manifests/kube-vip.yaml diff --git a/capt/templates/kustomization-netboot.tmpl b/capt/templates/kustomization-netboot.tmpl deleted file mode 100644 index 44b88dc9..00000000 --- a/capt/templates/kustomization-netboot.tmpl +++ /dev/null @@ -1,222 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: $NAMESPACE -resources: - - prekustomization.yaml -patches: - - target: - group: infrastructure.cluster.x-k8s.io - kind: TinkerbellMachineTemplate - name: ".*control-plane.*" - version: v1beta1 - patch: |- - - op: add - path: /spec/template/spec - value: - bootOptions: - bootMode: netboot - hardwareAffinity: - required: - - labelSelector: - matchLabels: - tinkerbell.org/role: control-plane - templateOverride: | - version: "0.1" - name: playground-template - global_timeout: 6000 - tasks: - - name: "playground-template" - worker: "{{.device_1}}" - volumes: - - /dev:/dev - - /dev/console:/dev/console - - /lib/firmware:/lib/firmware:ro - actions: - - name: "stream image" - image: quay.io/tinkerbell/actions/oci2disk - timeout: 1200 - environment: - IMG_URL: $OS_REGISTRY/$OS_DISTRO-$OS_VERSION:$KUBE_VERSION.gz - DEST_DISK: {{ index .Hardware.Disks 0 }} - COMPRESSED: true - - name: "add tink cloud-init config" - image: quay.io/tinkerbell/actions/writefile - timeout: 90 - environment: - DEST_DISK: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} - FS_TYPE: ext4 - DEST_PATH: /etc/cloud/cloud.cfg.d/10_tinkerbell.cfg - UID: 0 - GID: 0 - MODE: 0600 - DIRMODE: 0700 - CONTENTS: | - datasource: - Ec2: - metadata_urls: ["http://$TINKERBELL_VIP:7172"] - strict_id: false - system_info: - default_user: - name: tink - groups: [wheel, adm] - sudo: ["ALL=(ALL) NOPASSWD:ALL"] - shell: /bin/bash - manage_etc_hosts: localhost - warnings: - dsid_missing_source: off - - name: "add tink cloud-init ds-config" - image: quay.io/tinkerbell/actions/writefile - timeout: 90 - environment: - DEST_DISK: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} - FS_TYPE: ext4 - DEST_PATH: /etc/cloud/ds-identify.cfg - UID: 0 - GID: 0 - MODE: 0600 - DIRMODE: 0700 - CONTENTS: | - datasource: Ec2 - - name: "kexec image" - image: ghcr.io/jacobweinstock/waitdaemon:0.2.2 - timeout: 90 - pid: host - environment: - BLOCK_DEVICE: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} - FS_TYPE: ext4 - IMAGE: quay.io/tinkerbell/actions/kexec - WAIT_SECONDS: 5 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - target: - group: infrastructure.cluster.x-k8s.io - kind: TinkerbellMachineTemplate - name: ".*worker.*" - version: v1beta1 - patch: |- - - op: add - path: /spec/template/spec - value: - bootOptions: - bootMode: netboot - hardwareAffinity: - required: - - labelSelector: - matchLabels: - tinkerbell.org/role: worker - templateOverride: | - version: "0.1" - name: playground-template - global_timeout: 6000 - tasks: - - name: "playground-template" - worker: "{{.device_1}}" - volumes: - - /dev:/dev - - /dev/console:/dev/console - - /lib/firmware:/lib/firmware:ro - actions: - - name: "stream image" - image: quay.io/tinkerbell/actions/oci2disk - timeout: 1200 - environment: - IMG_URL: $OS_REGISTRY/$OS_DISTRO-$OS_VERSION:$KUBE_VERSION.gz - DEST_DISK: {{ index .Hardware.Disks 0 }} - COMPRESSED: true - - name: "add tink cloud-init config" - image: quay.io/tinkerbell/actions/writefile - timeout: 90 - environment: - DEST_DISK: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} - FS_TYPE: ext4 - DEST_PATH: /etc/cloud/cloud.cfg.d/10_tinkerbell.cfg - UID: 0 - GID: 0 - MODE: 0600 - DIRMODE: 0700 - CONTENTS: | - datasource: - Ec2: - metadata_urls: ["http://$TINKERBELL_VIP:7172"] - strict_id: false - system_info: - default_user: - name: tink - groups: [wheel, adm] - sudo: ["ALL=(ALL) NOPASSWD:ALL"] - shell: /bin/bash - manage_etc_hosts: localhost - warnings: - dsid_missing_source: off - - name: "add tink cloud-init ds-config" - image: quay.io/tinkerbell/actions/writefile - timeout: 90 - environment: - DEST_DISK: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} - FS_TYPE: ext4 - DEST_PATH: /etc/cloud/ds-identify.cfg - UID: 0 - GID: 0 - MODE: 0600 - DIRMODE: 0700 - CONTENTS: | - datasource: Ec2 - - name: "kexec image" - image: ghcr.io/jacobweinstock/waitdaemon:0.2.2 - timeout: 90 - pid: host - environment: - BLOCK_DEVICE: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} - FS_TYPE: ext4 - IMAGE: quay.io/tinkerbell/actions/kexec - WAIT_SECONDS: 5 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - target: - group: infrastructure.cluster.x-k8s.io - kind: TinkerbellCluster - name: ".*" - version: v1beta1 - patch: |- - - op: add - path: /spec - value: - imageLookupBaseRegistry: "$OS_REGISTRY" - imageLookupOSDistro: "$OS_DISTRO" - imageLookupOSVersion: "$VERSIONS_OS" - - target: - group: bootstrap.cluster.x-k8s.io - kind: KubeadmConfigTemplate - name: "$CLUSTER_NAME-.*" - version: v1beta1 - patch: |- - - op: add - path: /spec/template/spec/users - value: - - name: tink - sudo: ALL=(ALL) NOPASSWD:ALL - sshAuthorizedKeys: - - $SSH_AUTH_KEY - - target: - group: controlplane.cluster.x-k8s.io - kind: KubeadmControlPlane - name: "$CLUSTER_NAME-.*" - version: v1beta1 - patch: |- - - op: add - path: /spec/kubeadmConfigSpec/users - value: - - name: tink - sudo: ALL=(ALL) NOPASSWD:ALL - sshAuthorizedKeys: - - $SSH_AUTH_KEY - - target: - group: controlplane.cluster.x-k8s.io - kind: KubeadmControlPlane - name: "$CLUSTER_NAME-.*" - version: v1beta1 - patch: |- - - op: add - path: /spec/kubeadmConfigSpec/preKubeadmCommands - value: - - if [ $(cat /etc/kubernetes-version | awk -F. '{print $2}') -ge 29 ] && [ -f /run/kubeadm/kubeadm.yaml ]; then export KUBE_FILE=/etc/kubernetes/super-admin.conf; else export KUBE_FILE=/etc/kubernetes/admin.conf; fi && mkdir -p /etc/kubernetes/manifests && ctr images pull ghcr.io/kube-vip/kube-vip:v$KUBEVIP_VERSION && ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:v$KUBEVIP_VERSION vip /kube-vip manifest pod --arp --interface $(ip -4 -j route list default | jq -r .[0].dev) --address $CONTROL_PLANE_VIP --controlplane --leaderElection --k8sConfigPath $KUBE_FILE > /etc/kubernetes/manifests/kube-vip.yaml diff --git a/contrib/tag-release.sh b/contrib/tag-release.sh deleted file mode 100755 index e689cdab..00000000 --- a/contrib/tag-release.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -o nounset -o pipefail - -if [ -z "${1-}" ]; then - echo "Must specify new tag" - exit 1 -fi - -new_tag=${1-} -[[ $new_tag =~ ^v[0-9]*\.[0-9]*\.[0-9]*$ ]] || ( - echo "Tag must be in the form of vX.Y.Z" - exit 1 -) - -if [[ $(git symbolic-ref HEAD) != refs/heads/main ]] && [[ -z ${ALLOW_NON_MAIN-} ]]; then - echo "Must be on main branch" >&2 - exit 1 -fi -if [[ $(git describe --dirty) != $(git describe) ]]; then - echo "Repo must be in a clean state" >&2 - exit 1 -fi - -git fetch --all - -last_tag=$(git describe --abbrev=0) -last_tag_commit=$(git rev-list -n1 "$last_tag") -last_specific_tag=$(git tag --contains="$last_tag_commit" | grep -E "^v[0-9]*\.[0-9]*\.[0-9]*$" | tail -n 1) -last_specific_tag_commit=$(git rev-list -n1 "$last_specific_tag") -if [[ $last_specific_tag_commit == $(git rev-list -n1 HEAD) ]]; then - echo "No commits since last tag" >&2 - exit 1 -fi - -if [[ -n ${SIGN_TAG-} ]]; then - git tag -s -m "${new_tag}" "${new_tag}" &>/dev/null && echo "created signed tag ${new_tag}" >&2 && exit -else - git tag -a -m "${new_tag}" "${new_tag}" &>/dev/null && echo "created annotated tag ${new_tag}" >&2 && exit -fi diff --git a/stack/docs/quickstarts/VAGRANTLVIRT.md b/stack/docs/quickstarts/VAGRANTLVIRT.md index 4a75e2c7..3913f863 100644 --- a/stack/docs/quickstarts/VAGRANTLVIRT.md +++ b/stack/docs/quickstarts/VAGRANTLVIRT.md @@ -9,6 +9,10 @@ This option will also create a VM and provision an OS onto it. - [Libvirt](https://ubuntu.com/server/docs/virtualization-libvirt) is installed - Vagrant Libvirt plugin is installed: `vagrant plugin install vagrant-libvirt` - A connection to the public internet (air gapped and proxied environments are not supported) +- Tools to build the worker `tinkerbell/pxe` Vagrant box (one-time, no + hosting required): + - Debian/Ubuntu: `sudo apt-get install dosfstools mtools qemu-utils gdisk` + - Fedora: `sudo dnf install dosfstools mtools qemu-img gdisk` ## Steps @@ -19,6 +23,22 @@ This option will also create a VM and provision an OS onto it. cd playground ``` +1. Build and register the `tinkerbell/pxe` worker box (one-time, per host) + + The worker VM (`machine1`) PXE-boots via a tiny iPXE-on-disk box built + from [`stack/pxe-box/`](../../pxe-box/README.md). It's not published to a + registry; build it locally for your host's architecture and register it + with Vagrant: + + ```bash + cd stack/pxe-box + # Pick the arch your host runs (amd64 or arm64). + make ARCHES=amd64 libvirt + vagrant box add --provider libvirt --architecture amd64 \ + --name tinkerbell/pxe out/pxe-amd64-libvirt.box + cd ../.. + ``` + 1. Start the stack ```bash diff --git a/stack/docs/quickstarts/VAGRANTVBOX.md b/stack/docs/quickstarts/VAGRANTVBOX.md index fd6e6664..078e02fd 100644 --- a/stack/docs/quickstarts/VAGRANTVBOX.md +++ b/stack/docs/quickstarts/VAGRANTVBOX.md @@ -8,6 +8,11 @@ This option will also create a VM and provision an OS onto it. - [Vagrant](https://www.vagrantup.com/downloads) is installed - [VirtualBox](https://www.virtualbox.org/) is installed - A connection to the public internet (air gapped and proxied environments are not supported) +- Tools to build the worker `tinkerbell/pxe` Vagrant box (one-time, no + hosting required): + - macOS: `brew install dosfstools mtools qemu coreutils gptfdisk` + - Debian/Ubuntu: `sudo apt-get install dosfstools mtools qemu-utils gdisk` + - Fedora: `sudo dnf install dosfstools mtools qemu-img gdisk` ## Steps @@ -18,6 +23,22 @@ This option will also create a VM and provision an OS onto it. cd playground ``` +1. Build and register the `tinkerbell/pxe` worker box (one-time, per host) + + The worker VM (`machine1`) PXE-boots via a tiny iPXE-on-disk box built + from [`stack/pxe-box/`](../../pxe-box/README.md). It's not published to a + registry; build it locally for your host's architecture and register it + with Vagrant: + + ```bash + cd stack/pxe-box + # Pick the arch your host runs (amd64 or arm64). + make ARCHES=amd64 vbox + vagrant box add --provider virtualbox --architecture amd64 \ + --name tinkerbell/pxe out/pxe-amd64-virtualbox.box + cd ../.. + ``` + 1. Start the stack ```bash diff --git a/stack/pxe-box/scripts/build-disk.sh b/stack/pxe-box/scripts/build-disk.sh index 9b817fdf..9f135fad 100755 --- a/stack/pxe-box/scripts/build-disk.sh +++ b/stack/pxe-box/scripts/build-disk.sh @@ -17,7 +17,7 @@ amd64) loader="BOOTX64.EFI" ;; arm64) - url="https://boot.ipxe.org/arm64-efi/snponly.efi" + url="https://boot.ipxe.org/arm64-efi/ipxe.efi" loader="BOOTAA64.EFI" ;; *) diff --git a/stack/vagrant/.env b/stack/vagrant/.env index dfc1ac4a..34d8225e 100644 --- a/stack/vagrant/.env +++ b/stack/vagrant/.env @@ -11,3 +11,6 @@ KUBECTL_VERSION=1.35.2 K3D_VERSION=v5.8.3 HELM_VERSION=v3.17.0 +# Override to point machine1 at a fork's pxe-box release or a local metadata.json. +#PXE_BOX_URL=https://github.com/tinkerbell/playground/releases/download/pxe-box-latest/metadata.json + diff --git a/stack/vagrant/Vagrantfile b/stack/vagrant/Vagrantfile index 131e30f9..df716d16 100644 --- a/stack/vagrant/Vagrantfile +++ b/stack/vagrant/Vagrantfile @@ -1,6 +1,28 @@ # -*- mode: ruby -*- # vi: set ft=ruby : +require "fileutils" + +OVMF_CODE_PATHS = [ + "/usr/share/OVMF/OVMF_CODE_4M.fd", + "/usr/share/OVMF/OVMF_CODE.fd", + "/usr/share/edk2/ovmf/OVMF_CODE.fd", + "/usr/share/edk2-ovmf/x64/OVMF_CODE.fd", + "/usr/share/qemu/ovmf-x86_64-code.bin", +].freeze + +OVMF_VARS_PATHS = [ + "/usr/share/OVMF/OVMF_VARS_4M.fd", + "/usr/share/OVMF/OVMF_VARS.fd", + "/usr/share/edk2/ovmf/OVMF_VARS.fd", + "/usr/share/edk2-ovmf/x64/OVMF_VARS.fd", + "/usr/share/qemu/ovmf-x86_64-vars.bin", +].freeze + +def find_ovmf(candidates) + candidates.find { |p| File.exist?(p) } +end + open(".env", "r").readlines.each { |l| kv = l.split("=") @@ -13,7 +35,6 @@ LIBVIRT_HOST_IP = ENV["LIBVIRT_HOST_IP"] || "192.168.56.1" PROVISIONER_IP = ENV["PROVISIONER_IP"] || "192.168.56.4" LOADBALANCER_IP = ENV["LOADBALANCER_IP"] || "192.168.56.40" LOADBALANCER_IP_2 = ENV["LOADBALANCER_IP_2"] || "192.168.56.41" -GATEWAY_IP = ENV["GATEWAY_IP"] || "" MACHINE1_IP = ENV["MACHINE1_IP"] || "192.168.56.43" MACHINE1_MAC = (ENV["MACHINE1_MAC"] || "08:00:27:9E:F5:3A").downcase HELM_CHART_VERSION = ENV["HELM_CHART_VERSION"] || "0.19.1" @@ -21,6 +42,7 @@ KUBECTL_VERSION = ENV["KUBECTL_VERSION"] || "1.28.3" K3D_VERSION = ENV["K3D_VERSION"] || "v5.6.0" HELM_VERSION = ENV["HELM_VERSION"] || "v3.14.4" HELM_LOADBALANCER_INTERFACE = ENV["HELM_LOADBALANCER_INTERFACE"] || "eth1" +PXE_BOX_URL = ENV["PXE_BOX_URL"] || "https://tinkerbell.github.io/playground/pxe-box/metadata.json" DEST_DIR_BASE = "/playground/stack/" Vagrant.configure("2") do |config| @@ -29,58 +51,118 @@ Vagrant.configure("2") do |config| end config.vm.define "stack" do |stack| - stack.vm.box = "generic/ubuntu2204" - stack.vm.synced_folder "", DEST_DIR_BASE + stack.vm.box = "bento/ubuntu-22.04" stack.vm.network "private_network", ip: "192.168.56.4", netmask: "255.255.255.0", libvirt__network_name: "tink_network", libvirt__host_ip: LIBVIRT_HOST_IP, libvirt__netmask: "255.255.255.0", - libvirt__dhcp_enabled: false + libvirt__dhcp_enabled: false, + libvirt__forward_mode: "nat" stack.vm.provider "virtualbox" do |v, override| v.memory = 2048 v.cpus = 2 + override.vm.synced_folder "", DEST_DIR_BASE end stack.vm.provider "libvirt" do |l, override| - l.memory = 2048 + l.memory = 4096 l.cpus = 2 + l.machine_type = "q35" + l.driver = "kvm" + l.cpu_mode = "host-passthrough" + ovmf_code = find_ovmf(OVMF_CODE_PATHS) + ovmf_vars = find_ovmf(OVMF_VARS_PATHS) + if ovmf_code && ovmf_vars + l.loader = ovmf_code + l.nvram = ovmf_vars + else + $stderr.puts "WARNING: OVMF not found; install ovmf (Debian/Ubuntu) or edk2-ovmf (Fedora/Arch) for UEFI boot" + end + override.vm.synced_folder ".", "/vagrant", disabled: true override.vm.synced_folder "", DEST_DIR_BASE, type: "rsync" end - stack.vm.provision :shell, path: "setup.sh", args: [PROVISIONER_IP, MACHINE1_IP, MACHINE1_MAC, DEST_DIR_BASE, LOADBALANCER_IP, HELM_CHART_VERSION, HELM_LOADBALANCER_INTERFACE, KUBECTL_VERSION, K3D_VERSION, HELM_VERSION, LOADBALANCER_IP_2, GATEWAY_IP] + stack.vm.provision :shell, path: "setup.sh", args: [PROVISIONER_IP, MACHINE1_IP, MACHINE1_MAC, DEST_DIR_BASE, LOADBALANCER_IP, HELM_CHART_VERSION, HELM_LOADBALANCER_INTERFACE, KUBECTL_VERSION, K3D_VERSION, HELM_VERSION, LOADBALANCER_IP_2] end config.vm.define :machine1, autostart: false do |machine1| - machine1.ssh.insert_key = false - machine1.vm.boot_timeout = 10 + machine1.vm.box = "tinkerbell/pxe" + machine1.vm.box_url = PXE_BOX_URL + machine1.vm.boot_timeout = 1 machine1.vm.synced_folder ".", "/vagrant", disabled: true + # Pin the host-only / tink_network NIC to adapter slot 1. On VirtualBox + # this overrides Vagrant's default (NAT on slot 1) so iPXE picks up the + # Tinkerbell network as net0 and DHCPs from smee immediately instead of + # iterating through NAT's failed IPv4/IPv6/PXE/HTTP timeouts. The + # :adapter key is ignored by the libvirt provider. machine1.vm.network :private_network, ip: MACHINE1_IP, mac: MACHINE1_MAC.gsub(/[:-]/, "").strip, adapter: 1, libvirt__network_name: "tink_network", - libvirt__dhcp_enabled: false, - libvirt__forward_mode: "nat" + libvirt__dhcp_enabled: false - machine1.vm.provider "libvirt" do |v| - v.storage :file, :size => "20G" + machine1.vm.provider "libvirt" do |v, override| + override.vm.box = nil + boot_network = { "network" => "tink_network" } + v.boot boot_network + v.boot "hd" + v.storage :file, :size => "10G" v.memory = 4096 v.cpus = 2 - v.boot "hd" - v.boot "network" + v.machine_type = "q35" + v.driver = "kvm" + v.cpu_mode = "host-passthrough" + ovmf_code = find_ovmf(OVMF_CODE_PATHS) + ovmf_vars = find_ovmf(OVMF_VARS_PATHS) + if ovmf_code && ovmf_vars + v.loader = ovmf_code + v.nvram = ovmf_vars + end v.graphics_ip = "0.0.0.0" - v.machine_arch = "x86_64" end machine1.vm.provider "virtualbox" do |v, override| - override.vm.box = "jtyr/pxe" - v.memory = 2048 + v.memory = 4096 v.cpus = 2 v.gui = true - v.customize ["modifyvm", :id, "--nic1", "hostonlynet", "--nic2", "nat", "--boot1", "disk", "--boot2", "net"] + + # NAT moves to slot 2 so the booted OS still has internet via VBox NAT + # while the host-only NIC (slot 1, set above) drives PXE. + v.network_adapter(2, :nat, auto_config: false) + + # virtio is required for VBox amd64 EFI to expose an SNP handle that + # snponly.efi can use; without it iPXE prints "No more network devices". + v.customize ["modifyvm", :id, "--nictype1", "virtio"] + v.customize ["modifyvm", :id, "--nictype2", "virtio"] + + # The smee-known MAC lives on the host-only NIC, which is now slot 1. v.customize ["modifyvm", :id, "--macaddress1", MACHINE1_MAC.gsub(/[:-]/, "").strip] - v.customize ["setextradata", :id, "GUI/ScaleFactor", "3.0"] v.check_guest_additions = false + + # Attach an extra 10GB disk on the existing NVMe controller (port 1). + # The box ships with a 64MB disk on NVMe port 0; this gives the OS + # install something realistic to land on. + machine1_disk_dir = File.expand_path("../.vagrant/machine1", __FILE__) + machine1_disk_path = File.join(machine1_disk_dir, "extra-disk.vdi") + FileUtils.mkdir_p(machine1_disk_dir) + unless File.exist?(machine1_disk_path) + v.customize ["createmedium", "disk", + "--filename", machine1_disk_path, + "--size", "10240", + "--format", "VDI"] + end + v.customize ["storagectl", :id, + "--name", "VSCSI-extra", + "--add", "virtio-scsi", + "--portcount", "1", + "--bootable", "off"] + v.customize ["storageattach", :id, + "--storagectl", "VSCSI-extra", + "--port", "0", + "--device", "0", + "--type", "hdd", + "--medium", machine1_disk_path] end end end diff --git a/stack/vagrant/hardware.yaml b/stack/vagrant/hardware.yaml index b8a719f8..da9d940f 100644 --- a/stack/vagrant/hardware.yaml +++ b/stack/vagrant/hardware.yaml @@ -17,7 +17,7 @@ spec: version: "20.04" interfaces: - dhcp: - arch: x86_64 + arch: $TINKERBELL_CLIENT_ARCH hostname: machine1 ip: address: $TINKERBELL_CLIENT_IP diff --git a/stack/vagrant/setup.sh b/stack/vagrant/setup.sh index c1fb5a1f..c1f04b67 100755 --- a/stack/vagrant/setup.sh +++ b/stack/vagrant/setup.sh @@ -1,5 +1,23 @@ #!/bin/bash +host_arch() { + uname -m +} + +normalized_platform() { + case "$(host_arch)" in + aarch64 | arm64) + echo arm64 + ;; + x86_64 | amd64) + echo amd64 + ;; + *) + host_arch + ;; + esac +} + install_docker() { curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" @@ -10,8 +28,11 @@ install_docker() { install_kubectl() { local kubectl_version=$1 + local platform - curl -LO https://dl.k8s.io/v"$kubectl_version"/bin/linux/amd64/kubectl + platform=$(normalized_platform) + + curl -LO "https://dl.k8s.io/v$kubectl_version/bin/linux/${platform}/kubectl" chmod +x ./kubectl mv ./kubectl /usr/local/bin/kubectl } @@ -41,7 +62,7 @@ update_apt() { } install_k3d() { - local k3d_Version=$1 + local k3d_version=$1 wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG="$k3d_version" bash } @@ -55,6 +76,73 @@ start_k3d() { mkdir -p ~/.kube/ k3d kubeconfig get -a >~/.kube/config until kubectl wait --for=condition=Ready nodes --all --timeout=600s; do sleep 1; done + + fix_coredns_upstream +} + +# With --network host and K3D_FIX_DNS=false, CoreDNS inherits the node's +# /etc/resolv.conf which on Ubuntu points at the systemd-resolved stub at +# 127.0.0.53. That address is unreachable from inside a pod's network namespace, +# so cluster pods get DNS failures (the host and the k3d container itself work +# because they share the host netns). +# +# The k3s-supported "coredns-custom" ConfigMap can NOT be used here: +# - "*.override" entries are imported INSIDE the bundled ".:53 { ... }" block, +# but "forward" is a single-instance plugin so it can't be redeclared. +# - "*.server" entries are imported OUTSIDE the bundled block, intended for +# additional zones; declaring ". { ... }" there collides with the bundled +# ".:53" server and CoreDNS aborts with +# "cannot serve dns://.:53 - it is already defined". +# +# Instead, patch the bundled "coredns" ConfigMap in place: replace +# "forward . /etc/resolv.conf" with the discovered host upstreams and restart +# the Deployment. This is fine for the single-shot vagrant playground because +# k3s is never restarted within a session (the addon manager would otherwise +# re-apply the bundled manifest). +fix_coredns_upstream() { + local upstreams="" + if [[ -r /run/systemd/resolve/resolv.conf ]]; then + upstreams=$(awk '/^nameserver / {print $2}' /run/systemd/resolve/resolv.conf | grep -v '^127\.' | tr '\n' ' ' | sed 's/ $//') + fi + if [[ -z $upstreams ]]; then + upstreams="1.1.1.1" + fi + echo "==> Patching bundled CoreDNS Corefile to forward to: $upstreams" + + # Wait (bounded) for the CoreDNS ConfigMap and Deployment to exist. k3s + # deploys them asynchronously via the addon manager. + local waited=0 + until kubectl -n kube-system get configmap coredns >/dev/null 2>&1 && + kubectl -n kube-system get deployment coredns >/dev/null 2>&1; do + if ((waited >= 180)); then + echo "!! CoreDNS did not appear within ${waited}s; dumping diagnostics" >&2 + kubectl get all,cm,helmchart -A >&2 || true + docker logs k3d-k3s-default-server-0 2>&1 | tail -200 >&2 || true + return 1 + fi + sleep 2 + waited=$((waited + 2)) + done + + # Replace the "forward . /etc/resolv.conf" line in the Corefile with the + # discovered upstreams. Re-apply the ConfigMap preserving the NodeHosts + # data key (the bundled ConfigMap has both Corefile and NodeHosts). + local corefile new_corefile nodehosts + corefile=$(kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}') + nodehosts=$(kubectl -n kube-system get configmap coredns -o jsonpath='{.data.NodeHosts}') + new_corefile=$(printf '%s\n' "$corefile" | sed -E "s|forward \. /etc/resolv\.conf|forward . ${upstreams}|") + if [[ $new_corefile == "$corefile" ]]; then + echo "!! Could not find 'forward . /etc/resolv.conf' in CoreDNS Corefile; current contents:" >&2 + printf '%s\n' "$corefile" >&2 + return 1 + fi + kubectl -n kube-system create configmap coredns \ + --from-literal=Corefile="$new_corefile" \ + --from-literal=NodeHosts="$nodehosts" \ + --dry-run=client -o yaml | kubectl apply -f - + + kubectl -n kube-system rollout restart deployment coredns + kubectl -n kube-system rollout status deployment coredns --timeout=120s } kubectl_for_vagrant_user() { @@ -108,13 +196,14 @@ apply_manifests() { export TINKERBELL_CLIENT_IP="$worker_ip" export TINKERBELL_CLIENT_MAC="$worker_mac" export TINKERBELL_HOST_IP="$host_ip" + export TINKERBELL_CLIENT_ARCH="$(host_arch)" # (x86_64 | aarch64) + export TINKERBELL_CLIENT_PLATFORM="$(normalized_platform)" # (amd64 | arm64) - for i in "$manifests_dir"/{hardware.yaml,template.yaml,workflow.yaml}; do - envsubst <"$i" + for i in "$manifests_dir"/{hardware.yaml,template.yaml,workflow.yaml,ubuntu-download.yaml}; do + envsubst '$DISK_DEVICE $TINKERBELL_CLIENT_IP $TINKERBELL_CLIENT_MAC $TINKERBELL_HOST_IP $TINKERBELL_CLIENT_ARCH $TINKERBELL_CLIENT_PLATFORM' <"$i" echo -e '---' done >/tmp/manifests.yaml kubectl apply -n "$namespace" -f /tmp/manifests.yaml - kubectl apply -n "$namespace" -f "$manifests_dir"/ubuntu-download.yaml } run_helm() { @@ -150,7 +239,6 @@ main() { local k3d_version="$9" local helm_version="${10}" local loadbalancer_ip_2="${11}" - local gateway_ip="${12}" update_apt install_docker diff --git a/stack/vagrant/template.yaml b/stack/vagrant/template.yaml index 85cbb738..bd064a20 100644 --- a/stack/vagrant/template.yaml +++ b/stack/vagrant/template.yaml @@ -12,30 +12,28 @@ spec: worker: "{{.device_1}}" volumes: - /dev:/dev - - /dev/console:/dev/console - - /lib/firmware:/lib/firmware:ro actions: - name: "stream ubuntu image" image: quay.io/tinkerbell/actions/image2disk:latest timeout: 600 environment: DEST_DISK: {{ index .Hardware.Disks 0 }} - IMG_URL: "http://$TINKERBELL_HOST_IP:7173/jammy-server-cloudimg-amd64.raw.gz" + IMG_URL: "http://$TINKERBELL_HOST_IP:7173/jammy-server-cloudimg-$TINKERBELL_CLIENT_PLATFORM.raw.gz" COMPRESSED: true - name: "grow-partition" image: quay.io/tinkerbell/actions/cexec:latest timeout: 90 environment: - BLOCK_DEVICE: {{ index .Hardware.Disks 0 }}1 + BLOCK_DEVICE: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} FS_TYPE: ext4 CHROOT: y DEFAULT_INTERPRETER: "/bin/sh -c" - CMD_LINE: "growpart {{ index .Hardware.Disks 0 }} 1 && resize2fs {{ index .Hardware.Disks 0 }}1" + CMD_LINE: "growpart {{ index .Hardware.Disks 0 }} 1 && resize2fs {{ formatPartition ( index .Hardware.Disks 0 ) 1 }}" - name: "install openssl" image: quay.io/tinkerbell/actions/cexec:latest timeout: 90 environment: - BLOCK_DEVICE: {{ index .Hardware.Disks 0 }}1 + BLOCK_DEVICE: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} FS_TYPE: ext4 CHROOT: y DEFAULT_INTERPRETER: "/bin/sh -c" @@ -44,7 +42,7 @@ spec: image: quay.io/tinkerbell/actions/cexec:latest timeout: 90 environment: - BLOCK_DEVICE: {{ index .Hardware.Disks 0 }}1 + BLOCK_DEVICE: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} FS_TYPE: ext4 CHROOT: y DEFAULT_INTERPRETER: "/bin/sh -c" @@ -53,7 +51,7 @@ spec: image: quay.io/tinkerbell/actions/cexec:latest timeout: 90 environment: - BLOCK_DEVICE: {{ index .Hardware.Disks 0 }}1 + BLOCK_DEVICE: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} FS_TYPE: ext4 CHROOT: y DEFAULT_INTERPRETER: "/bin/sh -c" @@ -62,7 +60,7 @@ spec: image: quay.io/tinkerbell/actions/cexec:latest timeout: 90 environment: - BLOCK_DEVICE: {{ index .Hardware.Disks 0 }}1 + BLOCK_DEVICE: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} FS_TYPE: ext4 CHROOT: y DEFAULT_INTERPRETER: "/bin/sh -c" @@ -71,7 +69,7 @@ spec: image: quay.io/tinkerbell/actions/writefile:latest timeout: 90 environment: - DEST_DISK: {{ index .Hardware.Disks 0 }}1 + DEST_DISK: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} FS_TYPE: ext4 DEST_PATH: /etc/netplan/config.yaml CONTENTS: | @@ -87,14 +85,25 @@ spec: GID: 0 MODE: 0644 DIRMODE: 0755 + {{- if regexMatch "a(arch|rm)" (lower (index .Hardware.Interfaces 0).DHCP.Arch) }} + - name: "reboot into os" + {{- else }} - name: "kexec into os" + {{- end }} image: ghcr.io/jacobweinstock/waitdaemon:latest timeout: 90 + {{- if regexMatch "a(arch|rm)" (lower (index .Hardware.Interfaces 0).DHCP.Arch) }} + command: ["sh", "-c", "echo 1 > /proc/sys/kernel/sysrq && echo b > /proc/sysrq-trigger"] + {{- end }} pid: host environment: BLOCK_DEVICE: {{ formatPartition ( index .Hardware.Disks 0 ) 1 }} FS_TYPE: ext4 + {{- if regexMatch "a(arch|rm)" (lower (index .Hardware.Interfaces 0).DHCP.Arch) }} + IMAGE: alpine:latest + {{- else }} IMAGE: quay.io/tinkerbell/actions/kexec:latest - WAIT_SECONDS: 10 + {{- end }} + WAIT_SECONDS: 5 volumes: - /var/run/docker.sock:/var/run/docker.sock diff --git a/stack/vagrant/ubuntu-download.yaml b/stack/vagrant/ubuntu-download.yaml index d7fbe351..2d7fa32e 100644 --- a/stack/vagrant/ubuntu-download.yaml +++ b/stack/vagrant/ubuntu-download.yaml @@ -36,7 +36,7 @@ spec: command: ["/script/entrypoint.sh"] args: [ - "https://cloud-images.ubuntu.com/daily/server/jammy/current/jammy-server-cloudimg-amd64.img", + "https://cloud-images.ubuntu.com/daily/server/jammy/current/jammy-server-cloudimg-${TINKERBELL_CLIENT_PLATFORM}.img", "/output", ] volumeMounts: diff --git a/stack/vagrant/workflow.yaml b/stack/vagrant/workflow.yaml index c8f966ef..a6f6e100 100644 --- a/stack/vagrant/workflow.yaml +++ b/stack/vagrant/workflow.yaml @@ -7,3 +7,5 @@ spec: hardwareRef: machine1 hardwareMap: device_1: $TINKERBELL_CLIENT_MAC + bootOptions: + toggleAllowNetboot: true