From fd1d90c3bd587d966268fcf14a449749158e4207 Mon Sep 17 00:00:00 2001 From: Jan Larwig Date: Fri, 24 Jul 2026 11:24:46 +0200 Subject: [PATCH] feat: cluster credential rotator Signed-off-by: Jan Larwig --- .../script/cluster-credential-rotator.tmpl | 386 ++++++++++++++++++ .../templates/cluster-credential-rotator.yaml | 82 ++++ .../helm/argo-cd/values.yaml | 46 +++ 3 files changed, 514 insertions(+) create mode 100644 bootstrap/platform-components/helm/argo-cd/script/cluster-credential-rotator.tmpl create mode 100644 bootstrap/platform-components/helm/argo-cd/templates/cluster-credential-rotator.yaml diff --git a/bootstrap/platform-components/helm/argo-cd/script/cluster-credential-rotator.tmpl b/bootstrap/platform-components/helm/argo-cd/script/cluster-credential-rotator.tmpl new file mode 100644 index 0000000..bc96ca4 --- /dev/null +++ b/bootstrap/platform-components/helm/argo-cd/script/cluster-credential-rotator.tmpl @@ -0,0 +1,386 @@ +#!/bin/sh + +set -u + +RELEASE_NAMESPACE="{{ .Release.Namespace }}" +LABEL_KEY="{{ .Values.clusterCredentialRotator.secret.labelKey }}" +LABEL_VALUE="{{ .Values.clusterCredentialRotator.secret.labelValue }}" +REQUIRED_TYPE="{{ .Values.clusterCredentialRotator.secret.requiredType }}" +BOOTSTRAP_KUBECONFIG_KEY="{{ .Values.clusterCredentialRotator.secret.bootstrapKubeconfigKey }}" +GENERATED_CONFIG_KEY="{{ .Values.clusterCredentialRotator.secret.generatedConfigKey }}" +REMOTE_NAMESPACE="{{ .Values.clusterCredentialRotator.remoteAccess.namespace }}" +REMOTE_SERVICE_ACCOUNT_NAME="{{ .Values.clusterCredentialRotator.remoteAccess.serviceAccountName }}" +REMOTE_CLUSTER_ROLE_BINDING_NAME="{{ .Values.clusterCredentialRotator.remoteAccess.clusterRoleBindingName }}" +REMOTE_CLUSTER_ROLE_NAME="{{ .Values.clusterCredentialRotator.remoteAccess.clusterRoleName }}" +TOKEN_DURATION="{{ .Values.clusterCredentialRotator.token.duration }}" +REFRESH_BEFORE="{{ .Values.clusterCredentialRotator.token.refreshBefore }}" + +ISSUED_AT_ANNOTATION="kubara.io/cluster-credential-issued-at" +EXPIRES_AT_ANNOTATION="kubara.io/cluster-credential-expires-at" +MANAGED_BY_ANNOTATION="kubara.io/cluster-credential-managed-by" +MANAGED_BY_VALUE="cluster-credential-rotator" + +log() { + printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" +} + +normalize_optional_value() { + if [ "${1:-}" = "" ] || [ "${1:-}" = "" ]; then + return 0 + fi + + printf '%s' "${1:-}" +} + +duration_to_seconds() { + awk -v value="$1" ' + function fail() { exit 1 } + BEGIN { + rest = value + total = 0 + + if (rest == "") { + fail() + } + + while (rest != "") { + if (match(rest, /^[0-9]+[smhd]/) == 0) { + fail() + } + + part = substr(rest, RSTART, RLENGTH) + amount = substr(part, 1, length(part) - 1) + 0 + unit = substr(part, length(part), 1) + + if (unit == "s") { + multiplier = 1 + } else if (unit == "m") { + multiplier = 60 + } else if (unit == "h") { + multiplier = 3600 + } else if (unit == "d") { + multiplier = 86400 + } else { + fail() + } + + total += amount * multiplier + rest = substr(rest, RLENGTH + 1) + } + + print total + } + ' /dev/null || true)" +} + +get_secret_data() { + template_string="$(printf '%s%s index .data "%s" %s%s' '{' '{' "$2" '}' '}')" + normalize_optional_value "$(kubectl -n "$RELEASE_NAMESPACE" get secret "$1" -o "go-template=${template_string}" 2>/dev/null || true)" +} + +get_secret_value() { + encoded_value="$(get_secret_data "$1" "$2")" + + if [ -z "$encoded_value" ]; then + return 0 + fi + + printf '%s' "$encoded_value" | base64 -d +} + +get_secret_type() { + template_string="$(printf '%s%s .type %s%s' '{' '{' '}' '}')" + normalize_optional_value "$(kubectl -n "$RELEASE_NAMESPACE" get secret "$1" -o "go-template=${template_string}" 2>/dev/null || true)" +} + +build_generated_kubeconfig() { + kubeconfig_path=$1 + server=$2 + ca_data=$3 + insecure=$4 + token=$5 + + if [ "$insecure" = "true" ]; then + cat >"$kubeconfig_path" <"$kubeconfig_path" </dev/null 2>&1 +} + +ensure_remote_access() { + bootstrap_kubeconfig=$1 + + cat </dev/null +apiVersion: v1 +kind: Namespace +metadata: + name: ${REMOTE_NAMESPACE} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ${REMOTE_SERVICE_ACCOUNT_NAME} + namespace: ${REMOTE_NAMESPACE} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ${REMOTE_CLUSTER_ROLE_BINDING_NAME} +subjects: + - kind: ServiceAccount + name: ${REMOTE_SERVICE_ACCOUNT_NAME} + namespace: ${REMOTE_NAMESPACE} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ${REMOTE_CLUSTER_ROLE_NAME} +EOF +} + +secret_needs_refresh() { + secret_name=$1 + validation_kubeconfig=$2 + server=$3 + ca_data=$4 + insecure=$5 + refresh_after_epoch=$6 + + current_config="$(get_secret_value "$secret_name" "$GENERATED_CONFIG_KEY")" + if [ -z "$current_config" ]; then + return 0 + fi + + expires_at="$(get_secret_annotation "$secret_name" "$EXPIRES_AT_ANNOTATION")" + if [ -z "$expires_at" ]; then + return 0 + fi + + case "$expires_at" in + ''|*[!0-9]*) + return 0 + ;; + esac + + if [ "$refresh_after_epoch" -ge "$expires_at" ]; then + return 0 + fi + + current_token="$(generated_config_token "$current_config")" + if [ -z "$current_token" ]; then + return 0 + fi + + if ! validate_generated_credentials "$validation_kubeconfig" "$server" "$ca_data" "$insecure" "$current_token"; then + return 0 + fi + + return 1 +} + +patch_secret_with_credentials() { + secret_name=$1 + patch_file=$2 + cluster_name=$3 + server=$4 + generated_config=$5 + issued_at=$6 + expires_at=$7 + + escaped_name="$(json_escape "$cluster_name")" + escaped_server="$(json_escape "$server")" + escaped_config="$(json_escape "$generated_config")" + + cat >"$patch_file" </dev/null +} + +reconcile_secret() ( + set -u + + secret_name=$1 + bootstrap_kubeconfig_file=$(mktemp) + validation_kubeconfig_file=$(mktemp) + patch_file=$(mktemp) + trap 'rm -f "$bootstrap_kubeconfig_file" "$validation_kubeconfig_file" "$patch_file"' EXIT INT TERM + + secret_type="$(get_secret_type "$secret_name")" + if [ -n "$REQUIRED_TYPE" ] && [ "$secret_type" != "$REQUIRED_TYPE" ]; then + log "skipping ${secret_name}: expected secret type ${REQUIRED_TYPE}, found ${secret_type:-}" + return 0 + fi + + bootstrap_kubeconfig="$(get_secret_value "$secret_name" "$BOOTSTRAP_KUBECONFIG_KEY")" + if [ -z "$bootstrap_kubeconfig" ]; then + log "secret ${secret_name} is missing required key ${BOOTSTRAP_KUBECONFIG_KEY}" + return 1 + fi + + printf '%s' "$bootstrap_kubeconfig" >"$bootstrap_kubeconfig_file" + + cluster_name="$(get_secret_value "$secret_name" "name")" + if [ -z "$cluster_name" ]; then + cluster_name="$secret_name" + fi + + server="$(get_secret_value "$secret_name" "server")" + if [ -z "$server" ]; then + server="$(kubectl --kubeconfig "$bootstrap_kubeconfig_file" config view --raw --minify -o jsonpath='{.clusters[0].cluster.server}')" + fi + + if [ -z "$server" ]; then + log "secret ${secret_name} did not yield a cluster server endpoint" + return 1 + fi + + ca_data="$(kubectl --kubeconfig "$bootstrap_kubeconfig_file" config view --raw --minify -o jsonpath='{.clusters[0].cluster.certificate-authority-data}')" + insecure="$(kubectl --kubeconfig "$bootstrap_kubeconfig_file" config view --raw --minify -o jsonpath='{.clusters[0].cluster.insecure-skip-tls-verify}')" + if [ -z "$insecure" ]; then + insecure="false" + fi + + current_epoch="$(date +%s)" + refresh_after_epoch=$((current_epoch + REFRESH_BEFORE_SECONDS)) + + if ! secret_needs_refresh "$secret_name" "$validation_kubeconfig_file" "$server" "$ca_data" "$insecure" "$refresh_after_epoch"; then + log "secret ${secret_name} already has valid generated credentials" + return 0 + fi + + log "refreshing credentials for ${secret_name}" + + ensure_remote_access "$bootstrap_kubeconfig_file" + token="$(kubectl --kubeconfig "$bootstrap_kubeconfig_file" -n "$REMOTE_NAMESPACE" create token "$REMOTE_SERVICE_ACCOUNT_NAME" --duration "$TOKEN_DURATION")" + + if [ -z "$token" ]; then + log "failed to mint a service account token for ${secret_name}" + return 1 + fi + + generated_config="$(printf '{"bearerToken":"%s","tlsClientConfig":{"caData":"%s","insecure":%s}}' "$token" "$ca_data" "$insecure")" + issued_at="$current_epoch" + expires_at=$((current_epoch + TOKEN_DURATION_SECONDS)) + + patch_secret_with_credentials "$secret_name" "$patch_file" "$cluster_name" "$server" "$generated_config" "$issued_at" "$expires_at" + + if ! validate_generated_credentials "$validation_kubeconfig_file" "$server" "$ca_data" "$insecure" "$token"; then + log "generated credentials for ${secret_name} failed validation after refresh" + return 1 + fi + + log "successfully refreshed credentials for ${secret_name}" +) + +TOKEN_DURATION_SECONDS="$(duration_to_seconds "$TOKEN_DURATION")" +REFRESH_BEFORE_SECONDS="$(duration_to_seconds "$REFRESH_BEFORE")" + +managed_secrets="$(kubectl -n "$RELEASE_NAMESPACE" get secrets -l "${LABEL_KEY}=${LABEL_VALUE}" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')" + +if [ -z "$managed_secrets" ]; then + log "no rotator-managed cluster secrets found in namespace ${RELEASE_NAMESPACE}" + exit 0 +fi + +failure_count=0 + +for secret_name in $managed_secrets; do + if ! reconcile_secret "$secret_name"; then + failure_count=$((failure_count + 1)) + fi +done + +if [ "$failure_count" -gt 0 ]; then + log "${failure_count} secret reconciliation(s) failed" + exit 1 +fi + +log "cluster credential rotation completed successfully" diff --git a/bootstrap/platform-components/helm/argo-cd/templates/cluster-credential-rotator.yaml b/bootstrap/platform-components/helm/argo-cd/templates/cluster-credential-rotator.yaml new file mode 100644 index 0000000..1706e24 --- /dev/null +++ b/bootstrap/platform-components/helm/argo-cd/templates/cluster-credential-rotator.yaml @@ -0,0 +1,82 @@ +{{- if .Values.clusterCredentialRotator.enabled }} +{{- $serviceAccountName := .Values.clusterCredentialRotator.serviceAccount.name | default "cluster-credential-rotator" }} +{{- $configMapName := printf "%s-script" $serviceAccountName }} +--- +{{- if .Values.clusterCredentialRotator.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +--- +{{- end }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "patch", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} + apiGroup: "" +roleRef: + kind: Role + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $configMapName }} + namespace: {{ .Release.Namespace }} +data: + cluster-credential-rotator.sh: |- + {{- tpl ($.Files.Get "script/cluster-credential-rotator.tmpl") . | nindent 4 }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +spec: + schedule: {{ .Values.clusterCredentialRotator.schedule | quote }} + concurrencyPolicy: {{ .Values.clusterCredentialRotator.concurrencyPolicy }} + failedJobsHistoryLimit: {{ .Values.clusterCredentialRotator.failedJobsHistoryLimit }} + successfulJobsHistoryLimit: {{ .Values.clusterCredentialRotator.successfulJobsHistoryLimit }} + startingDeadlineSeconds: {{ .Values.clusterCredentialRotator.startingDeadlineSeconds }} + jobTemplate: + spec: + backoffLimit: 1 + template: + spec: + restartPolicy: OnFailure + serviceAccountName: {{ $serviceAccountName }} + containers: + - name: cluster-credential-rotator + image: "{{ .Values.clusterCredentialRotator.image.repository }}:{{ .Values.clusterCredentialRotator.image.tag }}" + imagePullPolicy: {{ .Values.clusterCredentialRotator.image.pullPolicy }} + command: ["sh", "-c", "/script/cluster-credential-rotator.sh"] + volumeMounts: + - name: script + mountPath: /script + {{- with .Values.clusterCredentialRotator.resources }} + resources: + {{- toYaml . | nindent 16 }} + {{- end }} + volumes: + - name: script + configMap: + name: {{ $configMapName }} + defaultMode: 0775 +{{- end }} diff --git a/bootstrap/platform-components/helm/argo-cd/values.yaml b/bootstrap/platform-components/helm/argo-cd/values.yaml index 255f69a..ba9a7b4 100644 --- a/bootstrap/platform-components/helm/argo-cd/values.yaml +++ b/bootstrap/platform-components/helm/argo-cd/values.yaml @@ -9,6 +9,52 @@ dexRestarter: maxAttempts: "30" interval: "10" # in seconds secretName: "oauth2-credentials" +clusterCredentialRotator: + enabled: false + schedule: "0 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + startingDeadlineSeconds: 300 + image: + repository: "alpine/kubectl" + tag: "1.36.2" + pullPolicy: IfNotPresent + serviceAccount: + create: true + name: "cluster-credential-rotator" + resources: {} + secret: + labelKey: "kubara.io/cluster-credential-rotator" + labelValue: "true" + requiredType: "kubara.io/argocd-cluster-bootstrap" + bootstrapKubeconfigKey: "bootstrapKubeconfig" + generatedConfigKey: "config" + remoteAccess: + namespace: "kubara" + serviceAccountName: "kubara-argocd-manager" + clusterRoleBindingName: "kubara-argocd-manager" + clusterRoleName: "cluster-admin" + token: + duration: "720h" + refreshBefore: "168h" + # Managed secrets are created separately and discovered by the configured label. + # The minimum supported shape is: + # + # apiVersion: v1 + # kind: Secret + # metadata: + # name: spoke-dev + # namespace: argocd + # labels: + # argocd.argoproj.io/secret-type: cluster + # kubara.io/cluster-credential-rotator: "true" + # type: kubara.io/argocd-cluster-bootstrap + # stringData: + # bootstrapKubeconfig: | + # apiVersion: v1 + # kind: Config + # ... defaultProject: name: default description: |-