diff --git a/.gitignore b/.gitignore index e7bda3132b..398232e584 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,4 @@ ephemeral-logs* # Dashboard (generated locally by ./dashboard/fetch-data.sh) dashboard/data.json +.claude/worktrees/ diff --git a/Makefile b/Makefile index d769e6614b..8d4d6e39ef 100644 --- a/Makefile +++ b/Makefile @@ -121,7 +121,7 @@ terraform-validate: terraform-init ## Check formatting and validate all Terrafor # Global values (aws_region, environment, cluster_type) are injected by the # ApplicationSet at deploy time, so we supply stubs here for linting. -HELM_LINT_SET := --set global.aws_region=us-east-1 --set global.environment=lint --set global.cluster_type=lint +HELM_LINT_SET := --set global.aws_region=us-east-1 --set global.environment=lint --set global.cluster_type=lint --set global.cluster_name=lint helm-lint: ## Lint all Helm charts @echo "🔍 Linting Helm charts..." @failed=false; \ diff --git a/argocd/config/management-cluster/eks-nodepool/templates/00-nodeclass.yaml b/argocd/config/management-cluster/eks-nodepool/templates/00-nodeclass.yaml index c6401bb9ec..15b119a614 100644 --- a/argocd/config/management-cluster/eks-nodepool/templates/00-nodeclass.yaml +++ b/argocd/config/management-cluster/eks-nodepool/templates/00-nodeclass.yaml @@ -1,15 +1,18 @@ -apiVersion: eks.amazonaws.com/v1 -kind: NodeClass +{{- $clusterName := required "global.cluster_name must be set via ApplicationSet valuesObject" .Values.global.cluster_name -}} +apiVersion: karpenter.k8s.aws/v1 +kind: EC2NodeClass metadata: name: fips spec: - role: "{{ .Values.global.cluster_name }}-auto-node-role" + amiSelectorTerms: + - alias: bottlerocket@latest + instanceProfile: {{ $clusterName }}-karpenter-node-role subnetSelectorTerms: - tags: - "kubernetes.io/cluster/{{ .Values.global.cluster_name }}": owned + kubernetes.io/cluster/{{ $clusterName }}: owned securityGroupSelectorTerms: - tags: - aws:eks:cluster-name: "{{ .Values.global.cluster_name }}" - advancedSecurity: - fips: true - kernelLockdown: Integrity + aws:eks:cluster-name: {{ $clusterName }} + metadataOptions: + httpTokens: required + httpPutResponseHopLimit: 2 diff --git a/argocd/config/management-cluster/eks-nodepool/templates/10-nodepool.yaml b/argocd/config/management-cluster/eks-nodepool/templates/10-nodepool.yaml index a1425dee6f..50e5938c1b 100644 --- a/argocd/config/management-cluster/eks-nodepool/templates/10-nodepool.yaml +++ b/argocd/config/management-cluster/eks-nodepool/templates/10-nodepool.yaml @@ -6,8 +6,8 @@ spec: template: spec: nodeClassRef: - group: eks.amazonaws.com - kind: NodeClass + group: karpenter.k8s.aws + kind: EC2NodeClass name: fips requirements: - key: karpenter.sh/capacity-type diff --git a/argocd/config/management-cluster/eks-nodepool/values.yaml b/argocd/config/management-cluster/eks-nodepool/values.yaml index 654d5a9ed6..6acb99b3c1 100644 --- a/argocd/config/management-cluster/eks-nodepool/values.yaml +++ b/argocd/config/management-cluster/eks-nodepool/values.yaml @@ -10,3 +10,6 @@ eksNodePool: disruption: consolidationPolicy: WhenEmpty consolidateAfter: 60s + +global: + cluster_name: "" diff --git a/argocd/config/management-cluster/hypershift/templates/05-job.yaml b/argocd/config/management-cluster/hypershift/templates/05-job.yaml index 67057ef416..1e334390ad 100644 --- a/argocd/config/management-cluster/hypershift/templates/05-job.yaml +++ b/argocd/config/management-cluster/hypershift/templates/05-job.yaml @@ -6,7 +6,7 @@ metadata: annotations: argocd.argoproj.io/sync-options: Replace=true,Force=true spec: - activeDeadlineSeconds: 1800 + activeDeadlineSeconds: 3600 backoffLimit: 1 template: spec: @@ -18,83 +18,151 @@ spec: volumeAttributes: secretProviderClass: hypershift-config containers: - - name: install - image: {{ .Values.hypershift.hypershift.image }} - volumeMounts: - - name: secrets-store - mountPath: /mnt/secrets-store - readOnly: true - env: - - name: OIDC_BUCKET_NAME - valueFrom: - secretKeyRef: - name: hypershift-config-env - key: OIDC_BUCKET_NAME - - name: OIDC_BUCKET_REGION - valueFrom: - secretKeyRef: - name: hypershift-config-env - key: OIDC_BUCKET_REGION - - name: OIDC_WRITER_ROLE_ARN - valueFrom: - secretKeyRef: - name: hypershift-config-env - key: OIDC_WRITER_ROLE_ARN - - name: DNS_ZONE_OPERATOR_ROLE_ARN - value: "{{ .Values.global.dns_zone_operator_role_arn }}" - command: - - /bin/sh - - -c - - | - mkdir -p /tmp/aws + - name: install + image: {{ .Values.hypershift.hypershift.image }} + volumeMounts: + - name: secrets-store + mountPath: /mnt/secrets-store + readOnly: true + env: + - name: DNS_ZONE_OPERATOR_ROLE_ARN + value: "{{ .Values.global.dns_zone_operator_role_arn }}" + command: + - /bin/bash + - -c + - | + set -euo pipefail + mkdir -p /tmp/aws + + # Read OIDC configuration from CSI-mounted Secrets Manager secret + OIDC_BUCKET_NAME=$(cat /mnt/secrets-store/oidcBucketName) + OIDC_BUCKET_REGION=$(cat /mnt/secrets-store/oidcBucketRegion) + OIDC_WRITER_ROLE_ARN=$(cat /mnt/secrets-store/oidcWriterRoleArn) + + # Private platform creds — Pod Identity provides MC-account credentials + echo -e "[default]\n# pod identity handles auth" > /tmp/aws/private-creds + + # OIDC S3 creds — assume into the RC oidc-writer role for cross-account S3+KMS + if [ -n "${OIDC_WRITER_ROLE_ARN:-}" ]; then + cat > /tmp/aws/oidc-creds <&1 + } - # Private platform creds — Pod Identity provides MC-account credentials - echo -e "[default]\n# pod identity handles auth" > /tmp/aws/private-creds + # Prometheus Operator CRDs must exist before hypershift install applies + # ServiceMonitor/PrometheusRule. The monitoring chart may still be syncing. + _CRD_DEADLINE=$((SECONDS + 1800)) + echo "Waiting for Prometheus Operator CRDs..." + until _crd_ready servicemonitors.monitoring.coreos.com && \ + _crd_ready prometheusrules.monitoring.coreos.com; do + if [ $SECONDS -ge $_CRD_DEADLINE ]; then + echo "ERROR: Prometheus Operator CRDs not available after 30 minutes" >&2 + echo "coreos.com CRDs present:" >&2 + curl -sf --cacert "$_CA" -H "Authorization: Bearer $_TOKEN" \ + "$_API/apis/apiextensions.k8s.io/v1/customresourcedefinitions" \ + 2>/dev/null | grep -o '"name":"[^"]*coreos[^"]*"' \ + || echo " (none or curl failed)" >&2 + exit 1 + fi + echo " Waiting for Prometheus Operator CRDs ($(( _CRD_DEADLINE - SECONDS ))s remaining)..." + sleep 15 + done + echo "=== Prometheus Operator CRDs present — proceeding with hypershift install ===" - # OIDC S3 creds — assume into the RC oidc-writer role for cross-account S3+KMS - if [ -n "${OIDC_WRITER_ROLE_ARN:-}" ]; then - cat > /tmp/aws/oidc-creds </dev/null || echo "000") + echo "hypershift namespace check: HTTP ${_hs_ns_http}" + if [ "$_hs_ns_http" != "200" ]; then + echo "ERROR: hypershift install failed (exit $_hs_install_rc) and HyperShift namespace not found (HTTP ${_hs_ns_http})" >&2 + exit $_hs_install_rc + fi + echo "WARNING: hypershift install exited $_hs_install_rc but HyperShift namespace exists — resources applied (known post-apply issue in aws-iam-auth build)" + fi + {{- if .Values.hypershift.externalDns.domain }} + # TODO(hypershift): Upstream --aws-assume-role + Pod Identity support + # to hypershift install CLI, then replace this post-install patch + # with native flags. + if [ -n "${DNS_ZONE_OPERATOR_ROLE_ARN:-}" ]; then + echo "Patching external-dns with --aws-assume-role=${DNS_ZONE_OPERATOR_ROLE_ARN}" - hypershift install \ - --namespace hypershift \ - --enable-conversion-webhook=false \ - --hypershift-image {{ .Values.hypershift.hypershift.image }} \ - --limit-crd-install AWS \ - --private-platform AWS \ - --aws-private-creds /tmp/aws/private-creds \ - --aws-private-region {{ .Values.hypershift.region }} \ - --oidc-storage-provider-s3-bucket-name $(OIDC_BUCKET_NAME) \ - --oidc-storage-provider-s3-region $(OIDC_BUCKET_REGION) \ - --oidc-storage-provider-s3-credentials /tmp/aws/oidc-creds \ - --external-dns-provider aws \ - --external-dns-domain-filter {{ .Values.hypershift.externalDns.domain }} \ - --external-dns-secret external-dns \ - --external-dns-image {{ .Values.hypershift.externalDns.image }} \ + _patch_http=$(curl -s --cacert "$_CA" -H "Authorization: Bearer $_TOKEN" \ + -H "Content-Type: application/json-patch+json" -X PATCH \ + "${_API}/apis/apps/v1/namespaces/hypershift/deployments/external-dns" \ + -d '[ + {"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--aws-assume-role='"${DNS_ZONE_OPERATOR_ROLE_ARN}"'"}, + {"op":"add","path":"/spec/template/spec/tolerations","value":[{"key":"CriticalAddonsOnly","operator":"Exists","effect":"NoSchedule"}]} + ]' \ + -o /dev/null -w "%{http_code}" 2>/dev/null || echo "000") + echo " external-dns deployment patch: HTTP ${_patch_http}" + if [ "$_patch_http" != "200" ] && [ "$_patch_http" != "201" ]; then + echo "WARNING: external-dns deployment patch returned HTTP ${_patch_http} — skipping" >&2 + fi - # TODO(hypershift): Upstream --aws-assume-role + Pod Identity support - # to hypershift install CLI, then replace this post-install patch - # with native flags. - if [ -n "${DNS_ZONE_OPERATOR_ROLE_ARN:-}" ]; then - echo "Patching external-dns with --aws-assume-role=${DNS_ZONE_OPERATOR_ROLE_ARN}" - _CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt - _TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) - _API="https://kubernetes.default.svc" + # Upstream v0.21.0 needs discovery.k8s.io and networking.k8s.io API groups + # that HyperShift's generated ClusterRole doesn't include. + _patch_http=$(curl -s --cacert "$_CA" -H "Authorization: Bearer $_TOKEN" \ + -H "Content-Type: application/merge-patch+json" -X PATCH \ + "${_API}/apis/rbac.authorization.k8s.io/v1/clusterroles/external-dns" \ + -d '{"rules":[ + {"apiGroups":["","discovery.k8s.io"],"resources":["services","endpoints","pods","nodes","endpointslices"],"verbs":["get","watch","list"]}, + {"apiGroups":["extensions","networking.k8s.io"],"resources":["ingresses","ingressroutes","ingressroutetcps","ingressrouteudps"],"verbs":["get","list","watch"]}, + {"apiGroups":["route.openshift.io"],"resources":["routes"],"verbs":["get","list","watch"]} + ]}' \ + -o /dev/null -w "%{http_code}" 2>/dev/null || echo "000") + echo " external-dns clusterrole patch: HTTP ${_patch_http}" + if [ "$_patch_http" != "200" ] && [ "$_patch_http" != "201" ]; then + echo "WARNING: external-dns clusterrole patch returned HTTP ${_patch_http} — skipping" >&2 + fi + fi - # Patch the command and add --aws-assume-role. - curl -sf --cacert "$_CA" -H "Authorization: Bearer $_TOKEN" \ - -H "Content-Type: application/json-patch+json" -X PATCH \ - "${_API}/apis/apps/v1/namespaces/hypershift/deployments/external-dns" \ - -d '[ - {"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--aws-assume-role='"${DNS_ZONE_OPERATOR_ROLE_ARN}"'"} - ]' \ - -o /dev/null -w " HTTP %{http_code}\n" - fi + {{- end }} restartPolicy: Never serviceAccountName: hypershift-installer + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule diff --git a/argocd/config/management-cluster/monitoring/values.yaml b/argocd/config/management-cluster/monitoring/values.yaml index d2a7682d6f..016c396fd2 100644 --- a/argocd/config/management-cluster/monitoring/values.yaml +++ b/argocd/config/management-cluster/monitoring/values.yaml @@ -16,18 +16,21 @@ kube-prometheus-stack: enabled: false prometheusOperator: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule admissionWebhooks: - # During bootstrap, ArgoCD syncs itself and restarts its controllers - # while other app syncs are in flight. If a PreSync hook Job completes - # while no controller is watching, the Kubernetes TTL controller deletes - # the Job (upstream default: 60s) before the replacement controller can - # observe it, leaving the sync stuck indefinitely. - # Bump to 600s to give the new controller time to come back. - # Ref: https://github.com/argoproj/argo-cd/issues/21055 - create: - ttlSecondsAfterFinished: 600 - patch: - ttlSecondsAfterFinished: 600 + # Disabled: the kube-webhook-certgen PreSync hook Jobs stall monitoring + # sync during bootstrap when ArgoCD controllers restart and miss the hook + # completion event (ArgoCD #21055). + enabled: false + tls: + # Disabled alongside admissionWebhooks: the PrometheusOperator Deployment + # mounts a tls-secret volume gated on tls.enabled (not admissionWebhooks.enabled). + # Without this flag the pod fails to start because the certgen Job that + # creates the secret was disabled above. + enabled: false resources: requests: cpu: 50m diff --git a/argocd/config/regional-cluster/aws-load-balancer-controller/Chart.yaml b/argocd/config/regional-cluster/aws-load-balancer-controller/Chart.yaml new file mode 100644 index 0000000000..6b09020888 --- /dev/null +++ b/argocd/config/regional-cluster/aws-load-balancer-controller/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +name: aws-load-balancer-controller +description: AWS Load Balancer Controller — provides TargetGroupBinding CRDs for OSS Karpenter clusters +type: application +version: 0.1.0 + +dependencies: + - name: aws-load-balancer-controller + version: 1.17.1 + repository: https://aws.github.io/eks-charts diff --git a/argocd/config/regional-cluster/aws-load-balancer-controller/values.yaml b/argocd/config/regional-cluster/aws-load-balancer-controller/values.yaml new file mode 100644 index 0000000000..2ca2a44557 --- /dev/null +++ b/argocd/config/regional-cluster/aws-load-balancer-controller/values.yaml @@ -0,0 +1,17 @@ +aws-load-balancer-controller: + # clusterName injected via ApplicationSet valuesObject (aws-load-balancer-controller.clusterName) + serviceAccount: + create: true + name: aws-load-balancer-controller + + podSecurityContext: + runAsNonRoot: true + runAsUser: 65534 + fsGroup: 65534 + + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL diff --git a/argocd/config/regional-cluster/eks-nodepool/templates/00-nodeclass.yaml b/argocd/config/regional-cluster/eks-nodepool/templates/00-nodeclass.yaml index c6401bb9ec..15b119a614 100644 --- a/argocd/config/regional-cluster/eks-nodepool/templates/00-nodeclass.yaml +++ b/argocd/config/regional-cluster/eks-nodepool/templates/00-nodeclass.yaml @@ -1,15 +1,18 @@ -apiVersion: eks.amazonaws.com/v1 -kind: NodeClass +{{- $clusterName := required "global.cluster_name must be set via ApplicationSet valuesObject" .Values.global.cluster_name -}} +apiVersion: karpenter.k8s.aws/v1 +kind: EC2NodeClass metadata: name: fips spec: - role: "{{ .Values.global.cluster_name }}-auto-node-role" + amiSelectorTerms: + - alias: bottlerocket@latest + instanceProfile: {{ $clusterName }}-karpenter-node-role subnetSelectorTerms: - tags: - "kubernetes.io/cluster/{{ .Values.global.cluster_name }}": owned + kubernetes.io/cluster/{{ $clusterName }}: owned securityGroupSelectorTerms: - tags: - aws:eks:cluster-name: "{{ .Values.global.cluster_name }}" - advancedSecurity: - fips: true - kernelLockdown: Integrity + aws:eks:cluster-name: {{ $clusterName }} + metadataOptions: + httpTokens: required + httpPutResponseHopLimit: 2 diff --git a/argocd/config/regional-cluster/eks-nodepool/templates/10-nodepool.yaml b/argocd/config/regional-cluster/eks-nodepool/templates/10-nodepool.yaml index e04546d09f..fd386d4b1c 100644 --- a/argocd/config/regional-cluster/eks-nodepool/templates/10-nodepool.yaml +++ b/argocd/config/regional-cluster/eks-nodepool/templates/10-nodepool.yaml @@ -7,8 +7,8 @@ spec: template: spec: nodeClassRef: - group: eks.amazonaws.com - kind: NodeClass + group: karpenter.k8s.aws + kind: EC2NodeClass name: fips requirements: - key: karpenter.sh/capacity-type diff --git a/argocd/config/regional-cluster/eks-nodepool/values.yaml b/argocd/config/regional-cluster/eks-nodepool/values.yaml index 654d5a9ed6..6acb99b3c1 100644 --- a/argocd/config/regional-cluster/eks-nodepool/values.yaml +++ b/argocd/config/regional-cluster/eks-nodepool/values.yaml @@ -10,3 +10,6 @@ eksNodePool: disruption: consolidationPolicy: WhenEmpty consolidateAfter: 60s + +global: + cluster_name: "" diff --git a/argocd/config/regional-cluster/grafana/templates/sre-targetgroupbinding.yaml b/argocd/config/regional-cluster/grafana/templates/sre-targetgroupbinding.yaml index 59b1888961..a053226cec 100644 --- a/argocd/config/regional-cluster/grafana/templates/sre-targetgroupbinding.yaml +++ b/argocd/config/regional-cluster/grafana/templates/sre-targetgroupbinding.yaml @@ -1,5 +1,5 @@ {{- if .Values.grafana.sre.targetGroup.arn }} -apiVersion: eks.amazonaws.com/v1 +apiVersion: elbv2.k8s.aws/v1alpha1 kind: TargetGroupBinding metadata: name: grafana-sre diff --git a/argocd/config/regional-cluster/loki/templates/targetgroupbinding.yaml b/argocd/config/regional-cluster/loki/templates/targetgroupbinding.yaml index 331a5a534c..9b85d3a444 100644 --- a/argocd/config/regional-cluster/loki/templates/targetgroupbinding.yaml +++ b/argocd/config/regional-cluster/loki/templates/targetgroupbinding.yaml @@ -1,5 +1,5 @@ {{- if .Values.platform.distributorTargetGroup.arn }} -apiVersion: eks.amazonaws.com/v1 +apiVersion: elbv2.k8s.aws/v1alpha1 kind: TargetGroupBinding metadata: name: loki-distributor @@ -17,7 +17,7 @@ spec: {{- end }} --- {{- if .Values.platform.queryFrontendTargetGroup.arn }} -apiVersion: eks.amazonaws.com/v1 +apiVersion: elbv2.k8s.aws/v1alpha1 kind: TargetGroupBinding metadata: name: loki-query-frontend diff --git a/argocd/config/regional-cluster/monitoring/templates/sre-targetgroupbinding.yaml b/argocd/config/regional-cluster/monitoring/templates/sre-targetgroupbinding.yaml index aedeacef64..43cd672c83 100644 --- a/argocd/config/regional-cluster/monitoring/templates/sre-targetgroupbinding.yaml +++ b/argocd/config/regional-cluster/monitoring/templates/sre-targetgroupbinding.yaml @@ -1,5 +1,5 @@ {{- if .Values.monitoring.sre.targetGroup.arn }} -apiVersion: eks.amazonaws.com/v1 +apiVersion: elbv2.k8s.aws/v1alpha1 kind: TargetGroupBinding metadata: name: prometheus-sre diff --git a/argocd/config/regional-cluster/monitoring/values.yaml b/argocd/config/regional-cluster/monitoring/values.yaml index 04a9f5ffe7..0ad65fb00c 100644 --- a/argocd/config/regional-cluster/monitoring/values.yaml +++ b/argocd/config/regional-cluster/monitoring/values.yaml @@ -68,17 +68,16 @@ kube-prometheus-stack: prometheusOperator: admissionWebhooks: - # During bootstrap, ArgoCD syncs itself and restarts its controllers - # while other app syncs are in flight. If a PreSync hook Job completes - # while no controller is watching, the Kubernetes TTL controller deletes - # the Job (upstream default: 60s) before the replacement controller can - # observe it, leaving the sync stuck indefinitely. - # Bump to 600s to give the new controller time to come back. - # Ref: https://github.com/argoproj/argo-cd/issues/21055 - create: - ttlSecondsAfterFinished: 600 - patch: - ttlSecondsAfterFinished: 600 + # Disabled: the kube-webhook-certgen PreSync hook Jobs stall monitoring + # sync during bootstrap when ArgoCD controllers restart and miss the hook + # completion event (ArgoCD #21055). + enabled: false + tls: + # Disabled alongside admissionWebhooks: the PrometheusOperator Deployment + # mounts a tls-secret volume gated on tls.enabled (not admissionWebhooks.enabled). + # Without this flag the pod fails to start because the certgen Job that + # creates the secret was disabled above. + enabled: false resources: requests: cpu: 50m diff --git a/argocd/config/regional-cluster/platform-api/templates/targetgroupbinding.yaml b/argocd/config/regional-cluster/platform-api/templates/targetgroupbinding.yaml index 1b76cbf6a7..a18b0cfa1e 100644 --- a/argocd/config/regional-cluster/platform-api/templates/targetgroupbinding.yaml +++ b/argocd/config/regional-cluster/platform-api/templates/targetgroupbinding.yaml @@ -1,5 +1,5 @@ --- -apiVersion: eks.amazonaws.com/v1 +apiVersion: elbv2.k8s.aws/v1alpha1 kind: TargetGroupBinding metadata: name: {{ .Values.platformApi.app.name }} diff --git a/argocd/config/regional-cluster/thanos/templates/targetgroupbinding.yaml b/argocd/config/regional-cluster/thanos/templates/targetgroupbinding.yaml index d725f0875d..6df150084c 100644 --- a/argocd/config/regional-cluster/thanos/templates/targetgroupbinding.yaml +++ b/argocd/config/regional-cluster/thanos/templates/targetgroupbinding.yaml @@ -1,5 +1,5 @@ {{- if .Values.thanos.targetGroup.arn }} -apiVersion: eks.amazonaws.com/v1 +apiVersion: elbv2.k8s.aws/v1alpha1 kind: TargetGroupBinding metadata: name: thanos-receive @@ -18,7 +18,7 @@ spec: {{- end }} --- {{- if .Values.thanos.queryTargetGroup.arn }} -apiVersion: eks.amazonaws.com/v1 +apiVersion: elbv2.k8s.aws/v1alpha1 kind: TargetGroupBinding metadata: name: thanos-query-frontend @@ -36,7 +36,7 @@ spec: {{- end }} --- {{- if .Values.thanos.sre.targetGroup.arn }} -apiVersion: eks.amazonaws.com/v1 +apiVersion: elbv2.k8s.aws/v1alpha1 kind: TargetGroupBinding metadata: name: thanos-query-frontend-sre diff --git a/argocd/config/shared/argocd/templates/sre-targetgroupbinding.yaml b/argocd/config/shared/argocd/templates/sre-targetgroupbinding.yaml index 731eba83b7..e8a1bd7bf8 100644 --- a/argocd/config/shared/argocd/templates/sre-targetgroupbinding.yaml +++ b/argocd/config/shared/argocd/templates/sre-targetgroupbinding.yaml @@ -1,5 +1,5 @@ {{- if index .Values "argo-cd" "sre" "targetGroup" "arn" }} -apiVersion: eks.amazonaws.com/v1 +apiVersion: elbv2.k8s.aws/v1alpha1 kind: TargetGroupBinding metadata: name: argocd-server-sre diff --git a/argocd/config/shared/argocd/values.yaml b/argocd/config/shared/argocd/values.yaml index 1b0fadaba4..38dff57f3c 100644 --- a/argocd/config/shared/argocd/values.yaml +++ b/argocd/config/shared/argocd/values.yaml @@ -5,8 +5,21 @@ argo-cd: ha: enabled: true + # CriticalAddonsOnly toleration for all ArgoCD components so they can + # schedule on the karpenter-bootstrap node group before Karpenter has + # provisioned any worker nodes. Harmless on regular nodes. + global: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule + # Server configuration server: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule autoscaling: enabled: true minReplicas: 2 @@ -40,6 +53,10 @@ argo-cd: # Controller configuration controller: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule replicas: 2 pdb: enabled: true @@ -71,6 +88,10 @@ argo-cd: # Repo Server configuration repoServer: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule autoscaling: enabled: true minReplicas: 2 @@ -104,6 +125,10 @@ argo-cd: # ApplicationSet controller applicationSet: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule resources: limits: cpu: 250m @@ -118,6 +143,10 @@ argo-cd: # Dex configuration dex: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule resources: limits: cpu: 250m @@ -128,6 +157,10 @@ argo-cd: # Notifications controller notifications: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule resources: limits: cpu: 250m @@ -136,7 +169,20 @@ argo-cd: cpu: 50m memory: 64Mi - # Redis configuration + # Redis HA configuration (enabled via ha.enabled=true above). + # redis-ha is a subchart and does not inherit global.tolerations. + redis-ha: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule + haproxy: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule + + # redis key kept for non-HA mode reference (unused when ha.enabled=true) redis: resources: limits: diff --git a/argocd/config/shared/storageclass/templates/gp3.yaml b/argocd/config/shared/storageclass/templates/gp3.yaml index bfcdf67aaf..492a7fe4ad 100644 --- a/argocd/config/shared/storageclass/templates/gp3.yaml +++ b/argocd/config/shared/storageclass/templates/gp3.yaml @@ -4,12 +4,7 @@ metadata: name: gp3 annotations: storageclass.kubernetes.io/is-default-class: "true" -allowedTopologies: -- matchLabelExpressions: - - key: eks.amazonaws.com/compute-type - values: - - auto -provisioner: ebs.csi.eks.amazonaws.com +provisioner: ebs.csi.aws.com volumeBindingMode: WaitForFirstConsumer parameters: type: gp3 diff --git a/ci/e2e-tests.sh b/ci/e2e-tests.sh index eac23afa66..c2c4d8d071 100755 --- a/ci/e2e-tests.sh +++ b/ci/e2e-tests.sh @@ -100,6 +100,29 @@ platform_rc=0 zoa_rc=0 hcp_rc=0 monitoring_rc=0 +validate_rc=0 + +# --- Infrastructure Validation (RC) --- +# Runs only for ephemeral environments where TF_OUTPUTS is available. Validates +# that Karpenter, SQS, and related AWS resources were provisioned correctly. +# MC validation is skipped here because MC cluster IDs are not in regional TF +# outputs; run scripts/validate-mc-{aws,k8s}.sh manually or from the MC +# provisioning pipeline with the MC AWS profile. +if [[ -n "${TF_OUTPUTS:-}" && -r "${TF_OUTPUTS:-}" ]]; then + echo "" + echo "=== RC Infrastructure Validation ===" + echo "" + _rc_cluster_name="$(jq -r '.cluster_name.value // empty' "${TF_OUTPUTS}")" + if [[ -n "$_rc_cluster_name" ]]; then + AWS_PROFILE="rrp-rc" \ + CLUSTER_ID="$_rc_cluster_name" \ + AWS_REGION="${AWS_DEFAULT_REGION}" \ + "${REPO_ROOT}/scripts/validate-rc-aws.sh" || validate_rc=$? + else + echo "WARNING: cluster_name not found in TF outputs — skipping RC validation" + fi +fi + make test-e2e-api || platform_rc=$? # Get regional account ID for CLI tests @@ -138,6 +161,82 @@ else echo "WARNING: No rrp-customer profile available — skipping HCP creation tests" fi +diag_dns() { + local cluster_api_host="${1:-}" + echo "" + echo "=== [DIAG] DNS / external-dns state ===" + + local mc_kube_ok=false + local mc_cluster="${CLUSTER_PREFIX:-}mc01" + if AWS_PROFILE="rrp-mc" aws eks update-kubeconfig \ + --name "$mc_cluster" \ + --kubeconfig /tmp/mc-kubeconfig 2>&1; then + mc_kube_ok=true + else + echo "[diag] Could not fetch MC kubeconfig for ${mc_cluster} — skipping kubectl checks" + fi + + if [[ "$mc_kube_ok" == "true" ]]; then + echo "[diag] external-dns pods:" + KUBECONFIG=/tmp/mc-kubeconfig \ + kubectl get pods -n hypershift -l app.kubernetes.io/name=external-dns \ + -o wide 2>&1 || true + + echo "[diag] DNSEndpoint CRs:" + KUBECONFIG=/tmp/mc-kubeconfig \ + kubectl get dnsendpoints.externaldns.k8s.io -A 2>&1 || true + + echo "[diag] external-dns logs (last 80 lines, errors/warnings only):" + KUBECONFIG=/tmp/mc-kubeconfig \ + kubectl logs -n hypershift \ + -l app.kubernetes.io/name=external-dns \ + --tail=80 --prefix 2>&1 | grep -iE "error|denied|zone|record|endpoint|Route53|warn|fail" || \ + KUBECONFIG=/tmp/mc-kubeconfig \ + kubectl logs -n hypershift \ + -l app.kubernetes.io/name=external-dns \ + --tail=80 --prefix 2>&1 || true + + echo "[diag] Pod Identity associations for external-dns SA:" + AWS_PROFILE="rrp-mc" aws eks list-pod-identity-associations \ + --cluster-name "$mc_cluster" \ + --namespace hypershift 2>&1 || true + fi + + echo "[diag] Route53 hosted zones in RC account:" + AWS_PROFILE="rrp-rc" aws route53 list-hosted-zones \ + --query 'HostedZones[*].[Name,Id,Config.PrivateZone]' \ + --output table 2>&1 || true + + local base_domain + base_domain=$(echo "${BASE_URL:-}" | sed 's|https\?://||;s|/.*||;s|^[^.]*\.||') + if [[ -n "$base_domain" ]]; then + echo "[diag] NS records for base domain ${base_domain}:" + dig NS "${base_domain}" +short 2>&1 || nslookup -type=NS "${base_domain}" 2>&1 || true + fi + + if [[ -n "$cluster_api_host" ]]; then + echo "[diag] DNS resolution for: ${cluster_api_host}" + dig A "${cluster_api_host}" +short 2>&1 || \ + nslookup "${cluster_api_host}" 2>&1 || true + + echo "[diag] Route53 A record for: ${cluster_api_host}" + local cluster_label + cluster_label=$(echo "$cluster_api_host" | cut -d. -f2) + AWS_PROFILE="rrp-rc" aws route53 list-hosted-zones \ + --query "HostedZones[?contains(Name, '${cluster_label}')].Id" \ + --output text 2>&1 | while read -r zone_id; do + [[ -n "$zone_id" ]] || continue + AWS_PROFILE="rrp-rc" aws route53 list-resource-record-sets \ + --hosted-zone-id "$zone_id" \ + --query "ResourceRecordSets[?Name=='${cluster_api_host}.']" \ + --output json 2>&1 || true + done || true + fi + + echo "=== [DIAG] end ===" + echo "" +} + if [[ "$_have_customer_creds" == "true" ]]; then test_hcp_creation() { echo "" @@ -173,9 +272,17 @@ if [[ "$_have_customer_creds" == "true" ]]; then echo "E2E_SKIP_CLEANUP is set — cleanup specs will be skipped" export E2E_LABEL_FILTER='!cleanup' fi - make test-e2e-cli || return $? - echo "HCP creation test completed for: ${HCP_CLUSTER_NAME}" + diag_dns + + _hcp_rc=0 + make test-e2e-cli || _hcp_rc=$? + if [[ $_hcp_rc -ne 0 ]]; then + _api_host="api.${HCP_CLUSTER_NAME}.${AWS_DEFAULT_REGION:-us-east-1}.rosa.devshift.net" + echo "[diag] HCP test failed (exit ${_hcp_rc}) — re-running DNS diagnostics for: ${_api_host}" + diag_dns "$_api_host" + fi + return $_hcp_rc } test_hcp_creation || hcp_rc=$? @@ -183,7 +290,12 @@ if [[ "$_have_customer_creds" == "true" ]]; then echo "" echo "=== Platform Monitoring Tests ===" echo "" - make test-e2e-platform-monitoring || monitoring_rc=$? + # Run with an explicit 15m timeout: the remote Makefile target uses --timeout=5m + # which fires before the observability specs can complete. + E2E_RHOBS_API_URL="${RHOBS_API_URL}" \ + ginkgo --timeout=15m -v --no-color --label-filter=Observability \ + ./test/e2e-platform-monitoring \ + || monitoring_rc=$? fi # HCP test failures collect logs via PRE_CLEANUP_HOOK in the test's DeferCleanup @@ -199,7 +311,7 @@ if [[ $platform_rc -ne 0 ]] || [[ $zoa_rc -ne 0 ]] || [[ $monitoring_rc -ne 0 ]] fi echo "" -echo "E2E results: platform=$platform_rc zoa=$zoa_rc hcp=$hcp_rc monitoring=$monitoring_rc" -if [[ $platform_rc -ne 0 ]] || [[ $zoa_rc -ne 0 ]] || [[ $hcp_rc -ne 0 ]] || [[ $monitoring_rc -ne 0 ]]; then +echo "E2E results: validate=$validate_rc platform=$platform_rc zoa=$zoa_rc hcp=$hcp_rc monitoring=$monitoring_rc" +if [[ $validate_rc -ne 0 ]] || [[ $platform_rc -ne 0 ]] || [[ $zoa_rc -ne 0 ]] || [[ $hcp_rc -ne 0 ]] || [[ $monitoring_rc -ne 0 ]]; then exit 1 fi diff --git a/ci/ephemeral-provider/__init__.py b/ci/ephemeral-provider/__init__.py index e3525751c0..7791830f56 100644 --- a/ci/ephemeral-provider/__init__.py +++ b/ci/ephemeral-provider/__init__.py @@ -1,5 +1,5 @@ POLL_INTERVAL = 30 PIPELINE_TRIGGER_TIMEOUT = 600 # 10 minutes -PIPELINE_COMPLETION_TIMEOUT = 3600 # 60 minutes +PIPELINE_COMPLETION_TIMEOUT = 5400 # 90 minutes PIPELINE_DISCOVERY_TIMEOUT = 600 # 10 minutes TARGET_ENVIRONMENT = "ephemeral" diff --git a/ci/ephemeral-provider/orchestrator.py b/ci/ephemeral-provider/orchestrator.py index 1e1732feeb..7fec2a57fb 100644 --- a/ci/ephemeral-provider/orchestrator.py +++ b/ci/ephemeral-provider/orchestrator.py @@ -224,12 +224,19 @@ def _inject_ephemeral_config(self, git: GitManager): # Replace config with overrides if provided if self.override_dir and self.override_dir.exists(): log.info("Applying environment overrides from %s", self.override_dir) - # Ensure target directory exists and clear existing config env_config_dir.mkdir(parents=True, exist_ok=True) + # Delete only region files (not defaults.yaml) to purge stale configs. + # defaults.yaml is merged rather than replaced to preserve env-level + # settings (e.g. dns.domain) that the override file omits. for existing in env_config_dir.glob("*.yaml"): - existing.unlink() + if existing.name != "defaults.yaml": + existing.unlink() for override_file in self.override_dir.glob("*.yaml"): - shutil.copy2(override_file, env_config_dir / override_file.name) + target = env_config_dir / override_file.name + if override_file.name == "defaults.yaml" and target.exists(): + load_and_merge(target, override_file) + else: + shutil.copy2(override_file, target) # Validate: exactly 1 region file must exist (enforced by discover_region # at startup, but re-check after override replacement) diff --git a/config/templates/argocd-bootstrap/applicationset.yaml.j2 b/config/templates/argocd-bootstrap/applicationset.yaml.j2 index 3644f07e4d..e6db51c2c9 100644 --- a/config/templates/argocd-bootstrap/applicationset.yaml.j2 +++ b/config/templates/argocd-bootstrap/applicationset.yaml.j2 @@ -54,6 +54,28 @@ spec: syncOptions: - ServerSideApply=true - CreateNamespace=true + - RespectIgnoreDifferences=true + # Fields managed by controllers post-deployment that should not cause OutOfSync: + # LBC rotates its own webhook TLS cert into caBundle and aws-load-balancer-tls. + # LBC populates TargetGroupBinding status after reconciliation. + ignoreDifferences: + - group: admissionregistration.k8s.io + kind: MutatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: "" + kind: Secret + name: aws-load-balancer-tls + jqPathExpressions: + - .data + - group: elbv2.k8s.aws + kind: TargetGroupBinding + jqPathExpressions: + - .status sources: - helm: valueFiles: @@ -130,6 +152,17 @@ spec: sre: targetGroup: arn: '{{ '{{ .metadata.annotations.sre_prometheus_target_group_arn }}' }}' + aws-load-balancer-controller: + clusterName: '{{ '{{ .metadata.labels.cluster_name }}' }}' + hyperfleetApi: + aws: + region: '{{ '{{ .metadata.labels.aws_region }}' }}' + hyperfleetSentinel: + aws: + region: '{{ '{{ .metadata.labels.aws_region }}' }}' + hyperfleetAdapter: + aws: + region: '{{ '{{ .metadata.labels.aws_region }}' }}' path: '{{ '{{ .path.path }}' }}' repoURL: '{{ '{{ .metadata.annotations.git_repo }}' }}' targetRevision: '{% if pinned %}{{ pinned }}{% else %}{{ "{{ .metadata.annotations.git_revision }}" }}{% endif %}' diff --git a/deploy/ephemeral/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml b/deploy/ephemeral/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml index 404b249c54..a7ec57e95d 100644 --- a/deploy/ephemeral/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml +++ b/deploy/ephemeral/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml @@ -53,6 +53,28 @@ spec: syncOptions: - ServerSideApply=true - CreateNamespace=true + - RespectIgnoreDifferences=true + # Fields managed by controllers post-deployment that should not cause OutOfSync: + # LBC rotates its own webhook TLS cert into caBundle and aws-load-balancer-tls. + # LBC populates TargetGroupBinding status after reconciliation. + ignoreDifferences: + - group: admissionregistration.k8s.io + kind: MutatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: "" + kind: Secret + name: aws-load-balancer-tls + jqPathExpressions: + - .data + - group: elbv2.k8s.aws + kind: TargetGroupBinding + jqPathExpressions: + - .status sources: - helm: valueFiles: @@ -129,6 +151,17 @@ spec: sre: targetGroup: arn: '{{ .metadata.annotations.sre_prometheus_target_group_arn }}' + aws-load-balancer-controller: + clusterName: '{{ .metadata.labels.cluster_name }}' + hyperfleetApi: + aws: + region: '{{ .metadata.labels.aws_region }}' + hyperfleetSentinel: + aws: + region: '{{ .metadata.labels.aws_region }}' + hyperfleetAdapter: + aws: + region: '{{ .metadata.labels.aws_region }}' path: '{{ .path.path }}' repoURL: '{{ .metadata.annotations.git_repo }}' targetRevision: '{{ .metadata.annotations.git_revision }}' diff --git a/deploy/ephemeral/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml b/deploy/ephemeral/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml index fdff0e3366..521b1155ab 100644 --- a/deploy/ephemeral/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml +++ b/deploy/ephemeral/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml @@ -53,6 +53,28 @@ spec: syncOptions: - ServerSideApply=true - CreateNamespace=true + - RespectIgnoreDifferences=true + # Fields managed by controllers post-deployment that should not cause OutOfSync: + # LBC rotates its own webhook TLS cert into caBundle and aws-load-balancer-tls. + # LBC populates TargetGroupBinding status after reconciliation. + ignoreDifferences: + - group: admissionregistration.k8s.io + kind: MutatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: "" + kind: Secret + name: aws-load-balancer-tls + jqPathExpressions: + - .data + - group: elbv2.k8s.aws + kind: TargetGroupBinding + jqPathExpressions: + - .status sources: - helm: valueFiles: @@ -127,6 +149,17 @@ spec: sre: targetGroup: arn: '{{ .metadata.annotations.sre_prometheus_target_group_arn }}' + aws-load-balancer-controller: + clusterName: '{{ .metadata.labels.cluster_name }}' + hyperfleetApi: + aws: + region: '{{ .metadata.labels.aws_region }}' + hyperfleetSentinel: + aws: + region: '{{ .metadata.labels.aws_region }}' + hyperfleetAdapter: + aws: + region: '{{ .metadata.labels.aws_region }}' path: '{{ .path.path }}' repoURL: '{{ .metadata.annotations.git_repo }}' targetRevision: '{{ .metadata.annotations.git_revision }}' diff --git a/deploy/integration/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml b/deploy/integration/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml index 13d2a9eb36..ca81555b62 100644 --- a/deploy/integration/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml +++ b/deploy/integration/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml @@ -53,6 +53,28 @@ spec: syncOptions: - ServerSideApply=true - CreateNamespace=true + - RespectIgnoreDifferences=true + # Fields managed by controllers post-deployment that should not cause OutOfSync: + # LBC rotates its own webhook TLS cert into caBundle and aws-load-balancer-tls. + # LBC populates TargetGroupBinding status after reconciliation. + ignoreDifferences: + - group: admissionregistration.k8s.io + kind: MutatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: "" + kind: Secret + name: aws-load-balancer-tls + jqPathExpressions: + - .data + - group: elbv2.k8s.aws + kind: TargetGroupBinding + jqPathExpressions: + - .status sources: - helm: valueFiles: @@ -129,6 +151,17 @@ spec: sre: targetGroup: arn: '{{ .metadata.annotations.sre_prometheus_target_group_arn }}' + aws-load-balancer-controller: + clusterName: '{{ .metadata.labels.cluster_name }}' + hyperfleetApi: + aws: + region: '{{ .metadata.labels.aws_region }}' + hyperfleetSentinel: + aws: + region: '{{ .metadata.labels.aws_region }}' + hyperfleetAdapter: + aws: + region: '{{ .metadata.labels.aws_region }}' path: '{{ .path.path }}' repoURL: '{{ .metadata.annotations.git_repo }}' targetRevision: '{{ .metadata.annotations.git_revision }}' diff --git a/deploy/integration/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml b/deploy/integration/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml index e39904232a..37dbac5976 100644 --- a/deploy/integration/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml +++ b/deploy/integration/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml @@ -53,6 +53,28 @@ spec: syncOptions: - ServerSideApply=true - CreateNamespace=true + - RespectIgnoreDifferences=true + # Fields managed by controllers post-deployment that should not cause OutOfSync: + # LBC rotates its own webhook TLS cert into caBundle and aws-load-balancer-tls. + # LBC populates TargetGroupBinding status after reconciliation. + ignoreDifferences: + - group: admissionregistration.k8s.io + kind: MutatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: "" + kind: Secret + name: aws-load-balancer-tls + jqPathExpressions: + - .data + - group: elbv2.k8s.aws + kind: TargetGroupBinding + jqPathExpressions: + - .status sources: - helm: valueFiles: @@ -127,6 +149,17 @@ spec: sre: targetGroup: arn: '{{ .metadata.annotations.sre_prometheus_target_group_arn }}' + aws-load-balancer-controller: + clusterName: '{{ .metadata.labels.cluster_name }}' + hyperfleetApi: + aws: + region: '{{ .metadata.labels.aws_region }}' + hyperfleetSentinel: + aws: + region: '{{ .metadata.labels.aws_region }}' + hyperfleetAdapter: + aws: + region: '{{ .metadata.labels.aws_region }}' path: '{{ .path.path }}' repoURL: '{{ .metadata.annotations.git_repo }}' targetRevision: '{{ .metadata.annotations.git_revision }}' diff --git a/docs/README.md b/docs/README.md index c5f568ad61..28dc6bd104 100644 --- a/docs/README.md +++ b/docs/README.md @@ -27,6 +27,7 @@ Detailed architecture and rationale for key technical decisions: | [DNS Architecture](design/dns-architecture.md) | Hierarchical DNS with zone shards, `deployment_name`, DNSSEC chain | | [ECS Fargate Bootstrap](design/fully-private-eks-bootstrap.md) | How fully private EKS clusters are bootstrapped via ECS | | [FIPS-Only EKS Compute](design/fips-eks-compute.md) | FIPS NodeClass/NodePool strategy for FedRAMP workload nodes | +| [Karpenter Node Provisioning](design/karpenter-node-provisioning.md) | OSS Karpenter IAM roles, IRSA rationale, SQS interruption handling | | [GitOps Cluster Configuration](design/gitops-cluster-configuration.md) | ApplicationSet pattern, progressive deployment, config modes | | [Infrastructure Logging](design/infrastructure-logging.md) | AWS CloudWatch log groups, KMS encryption, Grafana access | | [Logging Platform](design/logging-platform.md) | Application-level log collection (Vector + Loki) | @@ -79,6 +80,7 @@ Each module has its own README with usage, inputs, outputs, and architecture: - [`hyperfleet-db`](../terraform/modules/hyperfleet-db/) - Aurora PostgreSQL for HyperFleet cluster/nodepool state - [`grafana-cloudwatch-logs`](../terraform/modules/grafana-cloudwatch-logs/) - IAM + Pod Identity for Grafana CloudWatch Logs datasources (RC primary + MC reader) - [`hyperfleet-infrastructure`](../terraform/modules/hyperfleet-infrastructure/README.md) - RDS, Amazon MQ, IAM for HyperFleet (CLM) +- [`aws-load-balancer-controller`](../terraform/modules/aws-load-balancer-controller/README.md) - IAM role and Pod Identity association for AWS Load Balancer Controller ### ArgoCD Helm Chart Documentation diff --git a/docs/design/fips-eks-compute.md b/docs/design/fips-eks-compute.md index c56622208b..5f8e145496 100644 --- a/docs/design/fips-eks-compute.md +++ b/docs/design/fips-eks-compute.md @@ -1,14 +1,14 @@ -# FIPS-Only Compute for EKS Auto Mode +# FIPS-Only Compute for EKS Clusters -**Last Updated Date**: 2026-05-15 +**Last Updated Date**: 2026-07-08 ## Summary -All EKS clusters in the ROSA HyperFleet use a custom Karpenter NodePool referencing a -FIPS-validated NodeClass for platform and application workloads. The built-in EKS Auto Mode -`system` node pool is retained to provide nodes for CoreDNS and metrics-server. The built-in -`general-purpose` pool is disabled so that non-system workloads land on the FIPS NodePool rather -than on non-FIPS nodes. +All EKS clusters in the ROSA HyperFleet use OSS Karpenter v1 with a FIPS-validated +`EC2NodeClass` (`fips`) and a cluster-type-specific `NodePool` for platform and application +workloads. A dedicated `karpenter-bootstrap` managed node group (t3.medium, 2 nodes, tainted +`CriticalAddonsOnly`) provides stable capacity for Karpenter itself, CoreDNS, and metrics-server. +All other workloads land on Karpenter-provisioned nodes using FIPS Bottlerocket AMIs. ## Context @@ -17,133 +17,118 @@ FIPS 140-3 validated modules. On EKS, this means workload compute must run a FIP operating system — specifically Bottlerocket with FIPS mode enabled. - **Problem Statement**: EKS Auto Mode's built-in node pools (`system` and `general-purpose`) - provision standard (non-FIPS) Bottlerocket AMIs. Disabling both pools with `node_pools = []` - creates a bootstrap deadlock: EKS Auto Mode's embedded Karpenter is reactive — it only creates - nodes in response to unschedulable pods. With no built-in pools, CoreDNS and metrics-server - remain Pending until a FIPS NodePool exists, but the FIPS NodeClass's `InstanceProfileReady` - condition is evaluated only at creation time and depends on `node_role_arn` being set in the - cluster's `compute_config`. Any mismatch between cluster configuration and NodeClass creation - order results in `UnauthorizedNodeRole`, permanently blocking node provisioning. + provision standard (non-FIPS) Bottlerocket AMIs and cannot be patched to use a custom + `EC2NodeClass`. AWS auto-reverts any modifications to built-in pools within minutes. The + bootstrap deadlock caused by disabling all pools (`node_pools = []`) and repeated + `UnauthorizedNodeRole` failures with the embedded Karpenter made Auto Mode operationally + fragile for FIPS requirements. - **Constraints**: - - EKS Auto Mode's built-in node pools cannot be patched to reference a custom NodeClass. AWS - auto-reverts any modifications. - - EKS Auto Mode manages CoreDNS and metrics-server scheduling constraints; they are not - user-configurable. + - EKS nodes providing FIPS-validated compute must run Bottlerocket with `advancedSecurity.fips: true` - The cluster bootstrap runs inside an ECS Fargate task in a private subnet with no public cluster API access. See [ECS Fargate Bootstrap for Fully Private EKS Clusters](./fully-private-eks-bootstrap.md). -- **Assumptions**: EKS Auto Mode is retained for operational simplicity (managed control plane, - embedded Karpenter, automatic node lifecycle management). Self-managed Karpenter is not a - preferred alternative. + - Karpenter controller must run on stable, pre-provisioned nodes — it cannot schedule itself +- **Assumptions**: All clusters run OSS Karpenter (`enable_karpenter = true`, the default). EKS + Auto Mode is disabled. ## Alternatives Considered -1. **Keep both built-in pools enabled (`system` + `general-purpose`)**: CoreDNS and metrics-server - provision naturally; all workloads land on non-FIPS nodes. Violates the FedRAMP cryptographic - module requirement for platform workloads. Rejected. +1. **EKS Auto Mode with `system` pool + custom FIPS NodePool**: Retains the built-in `system` pool + for CoreDNS and metrics-server (non-FIPS, AWS-managed), adds a custom FIPS NodePool for + workloads. Partially FIPS-compliant but requires the embedded Karpenter's bootstrap ordering + constraints. Replaced because Auto Mode's embedded Karpenter cannot be independently upgraded + and the `node_role_arn` / `InstanceProfileReady` sequencing caused repeated bootstrap failures. -2. **Disable all built-in pools (`node_pools = []`)**: All nodes come from custom FIPS NodePools. - Creates a bootstrap deadlock: the FIPS NodeClass `InstanceProfileReady` condition is only - evaluated at NodeClass creation time. If `node_role_arn` is absent at cluster creation (even - temporarily), the NodeClass is permanently stuck with `UnauthorizedNodeRole` and no nodes can - be provisioned. Operationally fragile. Rejected. +2. **Disable all Auto Mode pools (`node_pools = []`)**: All nodes from custom FIPS NodePools. + Creates a bootstrap deadlock: the FIPS `EC2NodeClass` `InstanceProfileReady` condition is + evaluated only at creation time. If `node_role_arn` is absent at cluster creation, the + NodeClass is permanently stuck with `UnauthorizedNodeRole`. Operationally fragile. Rejected. -3. **Patch built-in node pools to use a FIPS NodeClass**: AWS Auto Mode auto-reverts user - modifications to built-in pools within minutes. Not durable. Rejected. +3. **Keep EKS Auto Mode, patch built-in pools**: AWS auto-reverts user modifications to built-in + pools. Not durable. Rejected. -4. **Replace EKS Auto Mode with self-managed Karpenter**: Achieves FIPS compliance but loses Auto - Mode's managed node lifecycle and unified support. Significantly increases operational - complexity. Rejected. - -5. **Retain `system` pool, disable `general-purpose`, add FIPS workloads NodePool**: The built-in - `system` pool provides nodes for CoreDNS and metrics-server immediately at cluster creation, - avoiding the bootstrap deadlock. The `general-purpose` pool is disabled so workloads cannot - land on non-FIPS nodes. A custom FIPS `*-workloads` NodePool handles all platform and - application workloads. **Chosen.** +4. **OSS Karpenter with dedicated bootstrap node group**: Provides a stable, pre-provisioned node + group (tainted `CriticalAddonsOnly`) for Karpenter controller, CoreDNS, and metrics-server. + Karpenter provisions all other nodes on demand using FIPS `EC2NodeClass`. Fully + FIPS-compliant for customer-bearing workloads. **Chosen.** ## Design Rationale -- **Justification**: Retaining the built-in `system` pool eliminates the bootstrap deadlock - entirely — CoreDNS and metrics-server are Active before the ECS bootstrap task runs. Disabling - `general-purpose` ensures that pods without explicit FIPS node selectors are not silently - scheduled on non-FIPS nodes. All platform and application workloads land on the FIPS NodePool. - -- **Evidence**: The `InstanceProfileReady` condition on a custom NodeClass is evaluated only at - NodeClass creation time, not when cluster configuration changes. This makes the `node_pools = []` - pattern operationally fragile: it requires that `node_role_arn` be set in `compute_config` before - the NodeClass is first applied, and there is no mechanism to re-trigger evaluation on an existing - NodeClass. Retaining the `system` pool removes this dependency entirely. +- **Justification**: The `karpenter-bootstrap` managed node group (t3.medium, 2 nodes, + `CriticalAddonsOnly` taint) provides stable, pre-provisioned capacity for Karpenter itself and + EKS system addons. This eliminates the bootstrap chicken-and-egg problem: Karpenter is installed + first (on the bootstrap node group), then the FIPS `EC2NodeClass` and `NodePool` are applied, + then a prewarm pod validates EC2 provisioning before ArgoCD is installed. -- **Tradeoff**: Nodes provisioned by the built-in `system` pool run non-FIPS Bottlerocket AMIs. - These nodes host only EKS-managed system addons (CoreDNS, metrics-server). Platform and - application workloads run exclusively on the FIPS NodePool. This is an accepted scope boundary: - EKS-managed system infrastructure vs. customer-bearing workloads. +- **Evidence**: The prewarm step in the ECS bootstrap task explicitly provisions one Karpenter node + and waits up to 8 minutes for it to become Ready before proceeding to ArgoCD installation. This + surfaces EC2 API rate limiting and IAM role sequencing issues as early ECS task failures + (retried automatically by ECS) rather than silent cascades post-ArgoCD. -- **Comparison**: Alternative 2 (`node_pools = []`) is the theoretically cleanest approach but - introduces bootstrap fragility that caused repeated failures in practice. Alternative 5 (chosen) - achieves the same workload-level FIPS compliance with a simpler, more robust bootstrap. +- **Tradeoff**: The `karpenter-bootstrap` node group runs non-FIPS standard Amazon Linux 2023 (AL2023) nodes. + These nodes host only Karpenter controller, CoreDNS, and metrics-server — EKS system + infrastructure, not customer-bearing workloads. Platform and application workloads run + exclusively on FIPS Karpenter-provisioned nodes. This scope boundary is an accepted tradeoff + for operational reliability. ## Consequences ### Positive -- Platform and application workloads run on Bottlerocket with FIPS-validated cryptographic modules, - satisfying FedRAMP High/Moderate cryptographic requirements for customer-bearing compute. -- Bootstrap is reliable: the built-in `system` pool provisions nodes and brings up CoreDNS and - metrics-server automatically. The bootstrap task applies the FIPS NodeClass and workloads - NodePool, waits for both addons to become Active, then installs ArgoCD — no custom node-wait - loop or addon creation required. -- The FIPS NodeClass and workloads NodePool are applied by the ECS bootstrap task and subsequently +- Platform and application workloads run on Bottlerocket with FIPS-validated cryptographic + modules, satisfying FedRAMP High/Moderate cryptographic requirements for customer-bearing + compute. +- Bootstrap is reliable: the bootstrap node group provisions nodes immediately, Karpenter installs + cleanly, and the prewarm pod validates EC2 provisioning before ArgoCD is involved. +- OSS Karpenter can be independently upgraded via Helm without waiting for AWS EKS Auto Mode + support cycles. +- The FIPS `EC2NodeClass` and `NodePool` are applied by the ECS bootstrap task and subsequently adopted by ArgoCD on first sync, making them GitOps-managed. -- Disabling `general-purpose` prevents accidental scheduling of platform workloads on non-FIPS - nodes. ### Negative -- EKS-managed system addons (CoreDNS, metrics-server) run on non-FIPS `system` pool nodes. These - nodes are AWS-managed infrastructure, not customer-bearing workloads, but they are not - FIPS-validated. -- EKS Auto Mode enforces a mandatory 21-day maximum node lifetime. Stateful workloads (Thanos, - Grafana) must have `PodDisruptionBudgets` to prevent data loss during node rotation. -- Adding a new cluster type requires updating the bootstrap script's `NODEPOOL_NAME` selection - logic. +- Karpenter controller, CoreDNS, and metrics-server run on non-FIPS t3.medium nodes. These are + AWS-managed system addons, not customer-bearing workloads, but they are not FIPS-validated. +- Two IAM roles are required: `karpenter-node-role` (for Karpenter-provisioned nodes) and a + lightweight role for the `karpenter-bootstrap` node group (not used directly by workloads). ## Cross-Cutting Concerns ### Reliability -- **Scalability**: The FIPS `*-workloads` NodePool is large (64 CPU / 256 GiB) and handles all - platform and application workloads. The built-in `system` pool is AWS-managed and scales - automatically for CoreDNS and metrics-server. +- **Scalability**: The FIPS `NodePool` handles all platform and application workloads. Karpenter + scales reactively on pending pods using EC2 instance provisioning. - **Observability**: Karpenter NodeClaims are visible via `kubectl get nodeclaims`. CloudWatch - logs for the ECS bootstrap task provide a full audit trail. -- **Resiliency**: The 21-day mandatory rotation means PodDisruptionBudgets are load-bearing for - stateful workloads — their absence is a reliability risk, not just a compliance gap. + logs for the ECS bootstrap task provide a full audit trail including the prewarm validation. +- **Resiliency**: The `karpenter-bootstrap` node group is a fixed-size managed node group (2 + nodes); AWS manages availability. Karpenter nodes are ephemeral and replaced automatically. ### Security - Platform and application workload nodes run with `advancedSecurity.fips: true` and `kernelLockdown: Integrity`, satisfying FIPS 140-2/140-3 requirements for SC-13. -- The FIPS NodeClass selects subnets and security groups via cluster-owned tags, ensuring nodes - land in the correct private subnets with correct network policies. -- Node IAM role (`${cluster_id}-auto-node-role`) is referenced directly in the NodeClass, scoping - node permissions to a cluster-specific role. +- The FIPS `EC2NodeClass` selects subnets and security groups via cluster-owned tags, ensuring + nodes land in the correct private subnets with correct network policies. +- Karpenter controller IAM role uses IRSA (ServiceAccount annotation on `kube-system/karpenter`) + with least-privilege SQS, EC2, and IAM instance profile permissions. +- Karpenter node IAM role (`${cluster_id}-karpenter-node-role`) is referenced directly in the + `EC2NodeClass`, scoping node permissions to a cluster-specific role. ### Performance - FIPS-mode Bottlerocket has negligible performance overhead for general-purpose workloads. -- `consolidateAfter: 60s` on the workloads NodePool enables rapid scale-down of idle capacity. +- `consolidateAfter: 60s` on the workloads `NodePool` enables rapid scale-down of idle capacity. ### Cost -- One custom NodePool adds no direct AWS cost. Node costs are identical: on-demand EC2 instances - running Bottlerocket. -- `WhenEmpty` consolidation reclaims idle capacity promptly, reducing EC2 spend. +- The `karpenter-bootstrap` node group (2x t3.medium) runs continuously. Karpenter workload nodes + are on-demand EC2 instances provisioned reactively. +- `WhenEmpty` consolidation reclaims idle Karpenter capacity promptly, reducing EC2 spend. ### Operability -- The FIPS NodeClass and workloads NodePool are created by the ECS bootstrap task on first run and +- The FIPS `EC2NodeClass` and `NodePool` are created by the ECS bootstrap task on first run and subsequently managed by ArgoCD. Day-2 changes are made via GitOps — no manual `kubectl apply`. -- The 21-day mandatory rotation is fully automatic. Operators need only ensure PodDisruptionBudgets - exist for stateful workloads. -- Cluster type-specific NodePool naming (`management-workloads` vs `regional-workloads`) is - resolved at bootstrap time via the `CLUSTER_TYPE` environment variable. +- Adding a new cluster type requires adding an `eks-nodepool` Helm chart entry for the new + `cluster_type`, which the bootstrap task selects via the `CLUSTER_TYPE` environment variable. +- EC2 interruption events (spot reclamation, instance retirement) are handled by Karpenter via + an SQS queue wired to EventBridge rules provisioned by the `eks-cluster` module. diff --git a/docs/design/fully-private-eks-bootstrap.md b/docs/design/fully-private-eks-bootstrap.md index 2f9dfbf3da..5d4201dbf8 100644 --- a/docs/design/fully-private-eks-bootstrap.md +++ b/docs/design/fully-private-eks-bootstrap.md @@ -98,7 +98,7 @@ Creates fully private EKS cluster with: - KMS encryption at rest for Kubernetes secrets using customer-managed keys - Dedicated security groups for VPC endpoints (port 443 from VPC CIDR only) - Restricted cluster egress limited to HTTPS for container registries and VPC internal traffic -- EKS Auto Mode authentication configured for API_AND_CONFIG_MAP mode +- EKS authentication configured for API_AND_CONFIG_MAP mode **High Availability Network Architecture**: diff --git a/docs/design/karpenter-node-provisioning.md b/docs/design/karpenter-node-provisioning.md new file mode 100644 index 0000000000..5a3375080f --- /dev/null +++ b/docs/design/karpenter-node-provisioning.md @@ -0,0 +1,98 @@ +# Karpenter Node Provisioning + +**Last Updated Date**: 2026-07-08 + +## Summary + +All EKS clusters use OSS Karpenter v1 (1.13.0) for node provisioning. A dedicated +`karpenter-bootstrap` managed node group provides stable capacity for the Karpenter controller. +The Karpenter controller IAM role uses IRSA (IAM Roles for Service Accounts) rather than EKS Pod +Identity because Karpenter predates EKS Pod Identity GA support. + +## Context + +When clusters migrated from EKS Auto Mode to OSS Karpenter, two IAM authentication mechanisms +were available for the Karpenter controller ServiceAccount: + +- **IRSA (IAM Roles for Service Accounts)**: ServiceAccount carries an annotation + (`eks.amazonaws.com/role-arn`); the OIDC provider validates the JWT and assumes the annotated + role. Requires an OIDC provider resource (`aws_iam_openid_connect_provider`) per cluster. +- **EKS Pod Identity**: Newer mechanism; IAM role is bound to a ServiceAccount via an API + association (no annotation needed). Simpler Terraform — no OIDC provider resource required. + +## Decision: IRSA for Karpenter Controller + +**Chosen**: IRSA for the Karpenter controller; EKS Pod Identity for all other workloads. + +**Rationale**: Karpenter v1 (1.13.0) ships with built-in IRSA support (ServiceAccount annotation +set during `helm install` via `serviceAccount.annotations`). EKS Pod Identity support in Karpenter +requires a separate admission webhook and additional configuration that the upstream chart does not +handle automatically. Using IRSA for Karpenter matches the upstream recommended installation +pattern, minimizes bootstrap complexity, and avoids a separate admission controller dependency +during the ECS bootstrap task. + +All other platform workloads (Thanos, Loki, Maestro Agent, AWS Load Balancer Controller, ZOA +jobs) use EKS Pod Identity exclusively. + +## Architecture + +```mermaid +graph LR + KC["Karpenter Controller\n(kube-system/karpenter)"] -->|"IRSA (JWT → OIDC)"| KCR["karpenter-controller\nIAM Role"] + KCR -->|"SQS: interruption events"| SQS["${cluster_id}-karpenter\nSQS Queue"] + KCR -->|"EC2: RunInstances, TerminateInstances"| EC2["EC2 API"] + KCR -->|"iam:PassRole → instance profile"| KNR["karpenter-node-role\nInstance Profile"] + KNR -->|"assumed by"| KN["Karpenter-provisioned\nnodes"] + BNG["karpenter-bootstrap\nManaged Node Group\n(2x t3.medium)"] -->|"tolerates CriticalAddonsOnly"| KC + EB["EventBridge Rules\n(EC2 lifecycle events)"] --> SQS +``` + +## IAM Resources + +### Karpenter Controller Role (IRSA) + +- **Name**: `${cluster_id}-karpenter-controller` +- **Trust**: OIDC provider for the cluster; constrained to `system:serviceaccount:kube-system:karpenter` +- **Permissions**: EC2 fleet operations (describe, run, terminate instances), IAM PassRole to the + node instance profile, SQS receive/delete on the interruption queue + +### Karpenter Node Role + +- **Name**: `${cluster_id}-karpenter-node-role` +- **Managed policies**: `AmazonEKSWorkerNodePolicy`, `AmazonEKS_CNI_Policy`, ECR pull-only +- **Optional inline policy**: `kms:Decrypt` and `kms:CreateGrant` on the FIPS AMI KMS key when + `ami_kms_key_arn` is set +- **Referenced in**: `EC2NodeClass.spec.role` + +### SQS Queue and EventBridge Rules + +The `eks-cluster` module provisions: + +- SQS queue (`${cluster_id}-karpenter`) with SQS-managed SSE, allowing `events.amazonaws.com` + and `sqs.amazonaws.com` to send messages +- Four EventBridge rules forwarding EC2 events to the queue: + - `scheduled-change` (AWS Health events) + - `spot-interruption` (EC2 Spot Instance interruption) + - `rebalance` (EC2 Instance Rebalance Recommendation) + - `instance-state-change` (EC2 Instance State-change Notification) + +## Consequences + +### Positive + +- IRSA is the upstream-recommended Karpenter auth mechanism; no additional admission controllers required +- Karpenter controller role trust policy is scoped to a single ServiceAccount — no broader cluster-level access +- SQS interruption handling enables graceful draining before spot reclamation or instance retirement +- OSS Karpenter can be upgraded independently via Helm without AWS EKS Auto Mode release cycles + +### Negative + +- One OIDC provider resource (`aws_iam_openid_connect_provider`) is required per cluster when `enable_karpenter = true` +- IRSA and Pod Identity coexist; operators must know which mechanism applies to which workload (Karpenter = IRSA, everything else = Pod Identity) + +## Related + +- [FIPS-Only EKS Compute](./fips-eks-compute.md) — EC2NodeClass and NodePool design for FIPS workloads +- [ECS Fargate Bootstrap](./fully-private-eks-bootstrap.md) — How Karpenter is installed during cluster bootstrap +- [Karpenter documentation](https://karpenter.sh/docs/) +- [IRSA documentation](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) diff --git a/docs/design/logging-platform.md b/docs/design/logging-platform.md index 3cddfaca08..c727e563eb 100644 --- a/docs/design/logging-platform.md +++ b/docs/design/logging-platform.md @@ -29,7 +29,7 @@ We evaluated deploying the same stack that RHOBS uses internally: **Constraints**: -- EKS Auto Mode (no OpenShift operators available) +- EKS with OSS Karpenter (no OpenShift operators available) - EKS Pod Identity for IAM auth — no static credentials - KMS encryption at rest (S3 bucket-level default, transparent to Loki) - Minimize locally-maintained operator code diff --git a/docs/design/thanos-metrics-infrastructure.md b/docs/design/thanos-metrics-infrastructure.md index 20034f040c..ec7d212ac8 100644 --- a/docs/design/thanos-metrics-infrastructure.md +++ b/docs/design/thanos-metrics-infrastructure.md @@ -26,8 +26,7 @@ causing ArgoCD ServerSideApply failures when field names changed. - UBI9 base images with automated security scanning (Clair, ClamAV, Snyk) - Minimize locally-maintained operator code -**Assumptions**: Management clusters send metrics via Prometheus `remote_write`. EKS Auto Mode remains -the compute strategy. Raw retention is 90d; downsampled retention is 180d (5m) and 365d (1h). +**Assumptions**: Management clusters send metrics via Prometheus `remote_write`. OSS Karpenter is the compute strategy. Raw retention is 90d; downsampled retention is 180d (5m) and 365d (1h). ## Decision diff --git a/docs/design/zoa-trusted-actions.md b/docs/design/zoa-trusted-actions.md index 2ebb707cc0..908c6908c1 100644 --- a/docs/design/zoa-trusted-actions.md +++ b/docs/design/zoa-trusted-actions.md @@ -822,7 +822,7 @@ Platform API Reconciler (5s loop): 2. **Single shared ServiceAccount**: One SA (`zoa-job-runner`) for all TAs. Rejected because Kubernetes audit logs only show SA identity — all TAs would be indistinguishable at the K8s audit level. Additionally, a shared SA bound to N possible Roles means parallel executions share permissions — any running TA would have access to RBAC granted for a different concurrent TA. -3. **IRSA (IAM Roles for Service Accounts)**: Allows per-SA roles via annotations. Rejected because IRSA is not fully supported in EKS Auto Mode and is being deprecated in favor of Pod Identity. +3. **IRSA (IAM Roles for Service Accounts)**: Allows per-SA roles via annotations. Rejected because IRSA is being deprecated in favor of EKS Pod Identity, which does not require OIDC provider management per cluster and is the platform-standard auth mechanism for workload SAs. 4. **Sidecar container for S3 upload**: A separate container watches `/artifacts` and uploads. Rejected because sidecars add complexity around container ordering and completion detection. Additionally, containers in the same Pod share the same ServiceAccount — the runner would inherit S3 write permissions, breaking the isolation between operational actions and output transport. diff --git a/scripts/buildspec/bootstrap-argocd-mc.sh b/scripts/buildspec/bootstrap-argocd-mc.sh index 06c7045ad4..e7b2b1b133 100755 --- a/scripts/buildspec/bootstrap-argocd-mc.sh +++ b/scripts/buildspec/bootstrap-argocd-mc.sh @@ -30,7 +30,7 @@ use_rc_account -backend-config="bucket=${_RC_STATE_BUCKET}" \ -backend-config="key=${_RC_STATE_KEY}" \ -backend-config="region=${TARGET_REGION}" \ - -backend-config="use_lockfile=true" >/dev/null 2>&1) + -backend-config="use_lockfile=true" >/dev/null) _RC_TIMEOUT=1800 _RC_START=$(date +%s) diff --git a/scripts/buildspec/provision-infra-mc.sh b/scripts/buildspec/provision-infra-mc.sh index 82f8ef4563..83d61a06f4 100755 --- a/scripts/buildspec/provision-infra-mc.sh +++ b/scripts/buildspec/provision-infra-mc.sh @@ -137,7 +137,7 @@ if [ "${TERRAFORM_ACTION}" == "apply" ] && [ -f imports.sh ]; then fi set +e -terraform "${TERRAFORM_ACTION}" -auto-approve +terraform "${TERRAFORM_ACTION}" -auto-approve -parallelism=20 TERRAFORM_STATUS=$? set -e diff --git a/scripts/buildspec/provision-infra-rc.sh b/scripts/buildspec/provision-infra-rc.sh index f59e7c7f8d..2daa3ab823 100755 --- a/scripts/buildspec/provision-infra-rc.sh +++ b/scripts/buildspec/provision-infra-rc.sh @@ -148,6 +148,7 @@ if [ -n "${ENVIRONMENT_HOSTED_ZONE_ID:-}" ]; then export TF_VAR_environment_hosted_zone_id="${ENVIRONMENT_HOSTED_ZONE_ID}" fi + export TF_VAR_regional_id=$(jq -r '.regional_id' "$DEPLOY_CONFIG_FILE") export TF_VAR_environment=$(jq -r '.environment' "$DEPLOY_CONFIG_FILE") export TF_VAR_eph_prefix=$(jq -r '.eph_prefix // ""' "$DEPLOY_CONFIG_FILE") diff --git a/scripts/buildspec/register.sh b/scripts/buildspec/register.sh index 117f764662..ee741bf43c 100755 --- a/scripts/buildspec/register.sh +++ b/scripts/buildspec/register.sh @@ -64,9 +64,13 @@ if [ -z "$API_GATEWAY_URL" ]; then exit 1 fi -# Wait for API Gateway /live endpoint +# Wait for API Gateway /live endpoint. +# RC and MC pipelines run in parallel, so RC outputs become available as soon +# as terraform apply finishes — before the ECS bootstrap (ArgoCD install, +# ~15 min) and initial ArgoCD sync (~10 min) have completed. Allow 40 minutes +# so the Platform API has time to be deployed and reach a healthy state. set +e -MAX_RETRIES=10 +MAX_RETRIES=80 RETRY_DELAY=30 RETRY_COUNT=0 LIVE_OK=false @@ -97,7 +101,7 @@ done set -e if [ "$LIVE_OK" != "true" ]; then - echo "ERROR: /live did not return 200 after $MAX_RETRIES attempts" >&2 + echo "ERROR: /live did not return 200 after $((MAX_RETRIES * RETRY_DELAY / 60)) minutes" >&2 exit 1 fi diff --git a/scripts/verify-fips.sh b/scripts/verify-fips.sh index c4f06470c4..c6ea51fd0a 100755 --- a/scripts/verify-fips.sh +++ b/scripts/verify-fips.sh @@ -127,36 +127,11 @@ print_workload_summary() { check_kubernetes_objects() { header "Kubernetes: NodeClass / NodePool" - # EKS Auto Mode uses nodeclasses.eks.amazonaws.com; upstream Karpenter uses - # nodeclasses.karpenter.k8s.aws. The advancedSecurity FIPS fields only exist - # on the EKS Auto Mode variant. - local has_eks_auto=false has_karpenter=false - kubectl get crd nodeclasses.eks.amazonaws.com >/dev/null 2>&1 && has_eks_auto=true - kubectl get crd nodeclasses.karpenter.k8s.aws >/dev/null 2>&1 && has_karpenter=true - - if [[ "$has_eks_auto" == "false" && "$has_karpenter" == "false" ]]; then - skip "NodeClass CRD not found — skipping (not an EKS Auto Mode or Karpenter cluster)" + if ! kubectl get crd nodeclasses.karpenter.k8s.aws >/dev/null 2>&1; then + skip "NodeClass CRD not found — skipping (Karpenter not installed)" return fi - if [[ "$has_eks_auto" == "true" ]]; then - if kubectl get nodeclass fips \ - -o jsonpath='{.spec.advancedSecurity.fips}' 2>/dev/null | grep -q "true"; then - pass "NodeClass 'fips': advancedSecurity.fips=true" - else - fail "NodeClass 'fips': advancedSecurity.fips=true" - fi - - if kubectl get nodeclass fips \ - -o jsonpath='{.spec.advancedSecurity.kernelLockdown}' 2>/dev/null | grep -q "Integrity"; then - pass "NodeClass 'fips': advancedSecurity.kernelLockdown=Integrity" - else - fail "NodeClass 'fips': advancedSecurity.kernelLockdown=Integrity" - fi - else - skip "advancedSecurity FIPS checks require nodeclasses.eks.amazonaws.com — skipping (upstream Karpenter detected)" - fi - echo "" echo " NodePool -> NodeClass bindings:" kubectl get nodepool -o json 2>/dev/null | jq -r \ diff --git a/terraform/config/management-cluster/main.tf b/terraform/config/management-cluster/main.tf index 63ff13b140..13d40ca34c 100755 --- a/terraform/config/management-cluster/main.tf +++ b/terraform/config/management-cluster/main.tf @@ -68,6 +68,9 @@ module "ecs_bootstrap" { repository_url = var.repository_url repository_branch = var.repository_branch + + karpenter_controller_role_arn = module.management_cluster.karpenter_controller_role_arn != null ? module.management_cluster.karpenter_controller_role_arn : "" + karpenter_queue_url = module.management_cluster.karpenter_queue_url != null ? module.management_cluster.karpenter_queue_url : "" } # ============================================================================= diff --git a/terraform/config/pipeline-management-cluster/main.tf b/terraform/config/pipeline-management-cluster/main.tf index 2e5afc204b..17d2cf67d8 100644 --- a/terraform/config/pipeline-management-cluster/main.tf +++ b/terraform/config/pipeline-management-cluster/main.tf @@ -388,7 +388,7 @@ resource "aws_codebuild_project" "management_apply" { resource "aws_codebuild_project" "management_bootstrap" { name = local.bootstrap_project_name service_role = aws_iam_role.codebuild_role.arn - build_timeout = 30 + build_timeout = 75 artifacts { type = "CODEPIPELINE" diff --git a/terraform/config/pipeline-regional-cluster/main.tf b/terraform/config/pipeline-regional-cluster/main.tf index 3f99005135..4502f3e8d1 100644 --- a/terraform/config/pipeline-regional-cluster/main.tf +++ b/terraform/config/pipeline-regional-cluster/main.tf @@ -272,7 +272,7 @@ resource "aws_codebuild_project" "regional_apply" { resource "aws_codebuild_project" "regional_bootstrap" { name = local.bootstrap_project_name service_role = aws_iam_role.codebuild_role.arn - build_timeout = 30 + build_timeout = 75 artifacts { type = "CODEPIPELINE" diff --git a/terraform/config/regional-cluster/imports.sh b/terraform/config/regional-cluster/imports.sh index 36699736c1..29b2ad679e 100644 --- a/terraform/config/regional-cluster/imports.sh +++ b/terraform/config/regional-cluster/imports.sh @@ -20,6 +20,10 @@ set -uo pipefail echo "--- Importing existing CloudWatch log groups (Regional Cluster) ---" +import_if_needed \ + 'module.rhobs_api_gateway.aws_cloudwatch_log_group.api_gateway_access' \ + "/aws/api-gateway/${TF_VAR_regional_id}-rhobs/${TF_VAR_stage_name:-prod}/access" + API_ID=$(tf_state_value \ 'module.api_gateway.aws_api_gateway_rest_api.main' '.values.id') STAGE_NAME=$(tf_state_value \ diff --git a/terraform/config/regional-cluster/main.tf b/terraform/config/regional-cluster/main.tf index 13ebf96bb2..636b9395be 100755 --- a/terraform/config/regional-cluster/main.tf +++ b/terraform/config/regional-cluster/main.tf @@ -203,6 +203,9 @@ module "ecs_bootstrap" { rc_aws_account_id = var.target_account_id redis_endpoint = var.enable_rate_limit_redis ? "${module.elasticache_valkey[0].endpoint}:${module.elasticache_valkey[0].port}" : "" + + karpenter_controller_role_arn = module.regional_cluster.karpenter_controller_role_arn != null ? module.regional_cluster.karpenter_controller_role_arn : "" + karpenter_queue_url = module.regional_cluster.karpenter_queue_url != null ? module.regional_cluster.karpenter_queue_url : "" } # ============================================================================= @@ -541,6 +544,18 @@ module "cloudwatch_exporter" { cluster_name = module.regional_cluster.cluster_name } +# ============================================================================= +# AWS Load Balancer Controller (Pod Identity for OSS Karpenter clusters) +# +# EKS Auto Mode includes LBC built-in. OSS Karpenter clusters must install it +# explicitly to provide the TargetGroupBinding CRD used by platform-api. +# ============================================================================= + +module "aws_load_balancer_controller" { + source = "../../modules/aws-load-balancer-controller" + cluster_name = module.regional_cluster.cluster_name +} + # ============================================================================= # Regional OIDC Module # diff --git a/terraform/modules/api-gateway/alb.tf b/terraform/modules/api-gateway/alb.tf index dfb0e8190e..65c12b1b3a 100644 --- a/terraform/modules/api-gateway/alb.tf +++ b/terraform/modules/api-gateway/alb.tf @@ -26,13 +26,7 @@ resource "aws_lb" "platform" { # ----------------------------------------------------------------------------- # Target Group # -# Uses IP target type for TargetGroupBinding compatibility. -# EKS Auto Mode will register pod IPs when the TargetGroupBinding resource -# is created in Kubernetes. -# -# IMPORTANT: The eks:eks-cluster-name tag is REQUIRED for EKS Auto Mode. -# The AmazonEKSLoadBalancingPolicy has a condition that only allows -# RegisterTargets on target groups tagged with the cluster name. +# Uses IP target type so LBC TargetGroupBindings register pod IPs directly. # ----------------------------------------------------------------------------- resource "aws_lb_target_group" "platform" { diff --git a/terraform/modules/api-gateway/variables.tf b/terraform/modules/api-gateway/variables.tf index 1fbfb3f336..fb69233ddf 100644 --- a/terraform/modules/api-gateway/variables.tf +++ b/terraform/modules/api-gateway/variables.tf @@ -23,12 +23,12 @@ variable "regional_id" { } variable "node_security_group_id" { - description = "EKS node/pod security group ID - ALB needs to send traffic to pods via this SG. For EKS Auto Mode, use the cluster_primary_security_group_id." + description = "EKS node/pod security group ID - ALB needs to send traffic to pods via this SG" type = string } variable "cluster_name" { - description = "EKS cluster name - required for tagging target group with eks:eks-cluster-name tag for Auto Mode IAM permissions" + description = "EKS cluster name - used to tag target groups with eks:eks-cluster-name" type = string } diff --git a/terraform/modules/aws-load-balancer-controller/README.md b/terraform/modules/aws-load-balancer-controller/README.md new file mode 100644 index 0000000000..b8176707b9 --- /dev/null +++ b/terraform/modules/aws-load-balancer-controller/README.md @@ -0,0 +1,57 @@ +# AWS Load Balancer Controller Module + +Creates the IAM role and EKS Pod Identity association required to run the [AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/) on a private EKS cluster. + +## Overview + +The AWS Load Balancer Controller (LBC) provides the `TargetGroupBinding` CRD used by platform +services (Thanos, Loki, RHOBS API Gateway) to wire Kubernetes services to ALB target groups. +LBC replaced the load balancing functionality previously bundled with EKS Auto Mode when clusters +migrated to OSS Karpenter. + +This module provisions: + +- **IAM role** (`-aws-load-balancer-controller`): IAM policy derived from the + [upstream recommended policy](https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.13.3/docs/install/iam_policy.json) +- **EKS Pod Identity association**: Binds the IAM role to the LBC Kubernetes ServiceAccount + +The LBC Helm chart itself is deployed via ArgoCD from +`argocd/config/regional-cluster/aws-load-balancer-controller/`. + +## Usage + +```hcl +module "aws_load_balancer_controller" { + source = "./terraform/modules/aws-load-balancer-controller" + + cluster_name = module.eks_cluster.cluster_name + + tags = { + Environment = var.environment + } +} +``` + +## Variables + +| Name | Description | Type | Default | Required | +| ----------------- | ---------------------------------------------- | ------------- | -------------------------------- | -------- | +| `cluster_name` | Name of the EKS cluster | `string` | n/a | yes | +| `namespace` | Kubernetes namespace where the LBC is deployed | `string` | `"aws-load-balancer-controller"` | no | +| `service_account` | Kubernetes service account name for the LBC | `string` | `"aws-load-balancer-controller"` | no | +| `tags` | Additional tags to apply to resources | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +| ----------------------------- | ------------------------------- | +| `role_name` | IAM role name for the LBC | +| `role_arn` | IAM role ARN for the LBC | +| `pod_identity_association_id` | EKS Pod Identity association ID | + +## Requirements + +| Name | Version | +| --------- | --------- | +| terraform | >= 1.14.3 | +| aws | >= 6.0.0 | diff --git a/terraform/modules/aws-load-balancer-controller/iam.tf b/terraform/modules/aws-load-balancer-controller/iam.tf new file mode 100644 index 0000000000..46a9ff9bde --- /dev/null +++ b/terraform/modules/aws-load-balancer-controller/iam.tf @@ -0,0 +1,335 @@ +# ============================================================================= +# AWS Load Balancer Controller IAM Role and Policies +# +# IAM policy derived from the upstream recommended policy: +# https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.17.1/docs/install/iam_policy.json +# ============================================================================= + +resource "aws_iam_role" "aws_lbc" { + name = "${var.cluster_name}-aws-load-balancer-controller" + description = "IAM role for AWS Load Balancer Controller (provides TargetGroupBinding CRDs)" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { + Service = "pods.eks.amazonaws.com" + } + Action = [ + "sts:AssumeRole", + "sts:TagSession" + ] + }] + }) + + tags = merge( + local.common_tags, + { + Name = "${var.cluster_name}-aws-load-balancer-controller" + } + ) +} + +resource "aws_iam_role_policy" "aws_lbc" { + #checkov:skip=CKV_AWS_355: Describe-only APIs (EC2, ELB, ACM, etc.) do not support resource-level ARN restrictions; Resource="*" matches the upstream AWS Load Balancer Controller recommended policy. + #checkov:skip=CKV_AWS_290: iam:CreateServiceLinkedRole requires Resource="*" per AWS; the statement is already constrained by a Condition on iam:AWSServiceName=elasticloadbalancing.amazonaws.com. + name = "${var.cluster_name}-aws-load-balancer-controller" + role = aws_iam_role.aws_lbc.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "CreateServiceLinkedRole" + Effect = "Allow" + Action = ["iam:CreateServiceLinkedRole"] + Resource = "*" + Condition = { + StringEquals = { + "iam:AWSServiceName" = "elasticloadbalancing.amazonaws.com" + } + } + }, + { + Sid = "EC2Read" + Effect = "Allow" + Action = [ + "ec2:DescribeAccountAttributes", + "ec2:DescribeAddresses", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInternetGateways", + "ec2:DescribeVpcs", + "ec2:DescribeVpcPeeringConnections", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeInstances", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeTags", + "ec2:GetCoipPoolUsage", + "ec2:DescribeCoipPools", + "ec2:GetSecurityGroupsForVpc", + "ec2:DescribeIpamPools", + "ec2:DescribeRouteTables", + ] + Resource = "*" + }, + { + Sid = "ELBRead" + Effect = "Allow" + Action = [ + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeListenerCertificates", + "elasticloadbalancing:DescribeSSLPolicies", + "elasticloadbalancing:DescribeRules", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetGroupAttributes", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:DescribeTags", + "elasticloadbalancing:DescribeTrustStores", + "elasticloadbalancing:DescribeListenerAttributes", + "elasticloadbalancing:DescribeCapacityReservation", + ] + Resource = "*" + }, + { + Sid = "CognitoRead" + Effect = "Allow" + Action = ["cognito-idp:DescribeUserPoolClient"] + Resource = "*" + }, + { + Sid = "ACMRead" + Effect = "Allow" + Action = [ + "acm:ListCertificates", + "acm:DescribeCertificate", + ] + Resource = "*" + }, + { + Sid = "IAMRead" + Effect = "Allow" + Action = [ + "iam:ListServerCertificates", + "iam:GetServerCertificate", + ] + Resource = "*" + }, + { + Sid = "WAFRead" + Effect = "Allow" + Action = [ + "waf-regional:GetWebACL", + "waf-regional:GetWebACLForResource", + "waf-regional:AssociateWebACL", + "waf-regional:DisassociateWebACL", + ] + Resource = "*" + }, + { + Sid = "WAFv2" + Effect = "Allow" + Action = [ + "wafv2:GetWebACL", + "wafv2:GetWebACLForResource", + "wafv2:AssociateWebACL", + "wafv2:DisassociateWebACL", + ] + Resource = "*" + }, + { + Sid = "ShieldRead" + Effect = "Allow" + Action = [ + "shield:GetSubscriptionState", + "shield:DescribeProtection", + "shield:CreateProtection", + "shield:DeleteProtection", + ] + Resource = "*" + }, + { + Sid = "EC2Mutate" + Effect = "Allow" + Action = [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress", + ] + Resource = "*" + }, + { + Sid = "EC2CreateSecurityGroup" + Effect = "Allow" + Action = ["ec2:CreateSecurityGroup"] + Resource = "*" + }, + { + Sid = "EC2CreateTags" + Effect = "Allow" + Action = ["ec2:CreateTags"] + Resource = "arn:${data.aws_partition.current.partition}:ec2:*:*:security-group/*" + Condition = { + StringEquals = { + "ec2:CreateAction" = "CreateSecurityGroup" + } + Null = { + "aws:RequestTag/elbv2.k8s.aws/cluster" = "false" + } + } + }, + { + Sid = "EC2MutateTags" + Effect = "Allow" + Action = [ + "ec2:CreateTags", + "ec2:DeleteTags", + ] + Resource = "arn:${data.aws_partition.current.partition}:ec2:*:*:security-group/*" + Condition = { + Null = { + "aws:RequestTag/elbv2.k8s.aws/cluster" = "true" + "aws:ResourceTag/elbv2.k8s.aws/cluster" = "false" + } + } + }, + { + Sid = "EC2DeleteSecurityGroup" + Effect = "Allow" + Action = ["ec2:DeleteSecurityGroup"] + Resource = "*" + Condition = { + Null = { + "aws:ResourceTag/elbv2.k8s.aws/cluster" = "false" + } + } + }, + { + Sid = "ELBCreateTagged" + Effect = "Allow" + Action = [ + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup", + ] + Resource = "*" + Condition = { + Null = { + "aws:RequestTag/elbv2.k8s.aws/cluster" = "false" + } + } + }, + { + Sid = "ELBCreateListenerAndRule" + Effect = "Allow" + Action = [ + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:CreateRule", + "elasticloadbalancing:DeleteRule", + ] + Resource = "*" + }, + { + Sid = "ELBMutateTags" + Effect = "Allow" + Action = [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags", + ] + Resource = [ + "arn:${data.aws_partition.current.partition}:elasticloadbalancing:*:*:targetgroup/*/*", + "arn:${data.aws_partition.current.partition}:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:${data.aws_partition.current.partition}:elasticloadbalancing:*:*:loadbalancer/app/*/*", + ] + Condition = { + Null = { + "aws:RequestTag/elbv2.k8s.aws/cluster" = "true" + "aws:ResourceTag/elbv2.k8s.aws/cluster" = "false" + } + } + }, + { + Sid = "ELBMutateListenerRuleTags" + Effect = "Allow" + Action = [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags", + ] + Resource = [ + "arn:${data.aws_partition.current.partition}:elasticloadbalancing:*:*:listener/net/*/*/*", + "arn:${data.aws_partition.current.partition}:elasticloadbalancing:*:*:listener/app/*/*/*", + "arn:${data.aws_partition.current.partition}:elasticloadbalancing:*:*:listener-rule/net/*/*/*", + "arn:${data.aws_partition.current.partition}:elasticloadbalancing:*:*:listener-rule/app/*/*/*", + ] + }, + { + Sid = "ELBMutateTagged" + Effect = "Allow" + Action = [ + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:SetIpAddressType", + "elasticloadbalancing:SetSecurityGroups", + "elasticloadbalancing:SetSubnets", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:ModifyTargetGroupAttributes", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:ModifyListenerAttributes", + "elasticloadbalancing:ModifyCapacityReservation", + "elasticloadbalancing:ModifyIpPools", + ] + Resource = "*" + Condition = { + Null = { + "aws:ResourceTag/elbv2.k8s.aws/cluster" = "false" + } + } + }, + { + Sid = "ELBAddListenerCert" + Effect = "Allow" + Action = [ + "elasticloadbalancing:AddListenerCertificates", + "elasticloadbalancing:RemoveListenerCertificates", + "elasticloadbalancing:ModifyRule", + "elasticloadbalancing:SetRulePriorities", + ] + Resource = "*" + }, + { + Sid = "ELBTargets" + Effect = "Allow" + Action = [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets", + ] + Resource = "arn:${data.aws_partition.current.partition}:elasticloadbalancing:*:*:targetgroup/*/*" + }, + { + Sid = "ELBMutateAttributes" + Effect = "Allow" + Action = [ + "elasticloadbalancing:SetWebAcl", + "elasticloadbalancing:ModifyListener", + ] + Resource = "*" + }, + ] + }) +} + +resource "aws_eks_pod_identity_association" "aws_lbc" { + cluster_name = var.cluster_name + namespace = var.namespace + service_account = var.service_account + role_arn = aws_iam_role.aws_lbc.arn + + tags = merge( + local.common_tags, + { + Name = "${var.cluster_name}-aws-load-balancer-controller-pod-identity" + } + ) +} diff --git a/terraform/modules/aws-load-balancer-controller/main.tf b/terraform/modules/aws-load-balancer-controller/main.tf new file mode 100644 index 0000000000..c92d3aa71b --- /dev/null +++ b/terraform/modules/aws-load-balancer-controller/main.tf @@ -0,0 +1,11 @@ +data "aws_partition" "current" {} + +locals { + common_tags = merge( + var.tags, + { + Component = "aws-load-balancer-controller" + ManagedBy = "terraform" + } + ) +} diff --git a/terraform/modules/aws-load-balancer-controller/outputs.tf b/terraform/modules/aws-load-balancer-controller/outputs.tf new file mode 100644 index 0000000000..9470a5cefb --- /dev/null +++ b/terraform/modules/aws-load-balancer-controller/outputs.tf @@ -0,0 +1,14 @@ +output "role_name" { + description = "IAM role name for the AWS Load Balancer Controller" + value = aws_iam_role.aws_lbc.name +} + +output "role_arn" { + description = "IAM role ARN for the AWS Load Balancer Controller" + value = aws_iam_role.aws_lbc.arn +} + +output "pod_identity_association_id" { + description = "EKS Pod Identity association ID" + value = aws_eks_pod_identity_association.aws_lbc.association_id +} diff --git a/terraform/modules/aws-load-balancer-controller/variables.tf b/terraform/modules/aws-load-balancer-controller/variables.tf new file mode 100644 index 0000000000..23cc4d2a76 --- /dev/null +++ b/terraform/modules/aws-load-balancer-controller/variables.tf @@ -0,0 +1,22 @@ +variable "cluster_name" { + description = "Name of the EKS cluster" + type = string +} + +variable "namespace" { + description = "Kubernetes namespace where the LBC is deployed" + type = string + default = "aws-load-balancer-controller" +} + +variable "service_account" { + description = "Kubernetes service account name for the LBC" + type = string + default = "aws-load-balancer-controller" +} + +variable "tags" { + description = "Additional tags to apply to resources" + type = map(string) + default = {} +} diff --git a/terraform/modules/aws-load-balancer-controller/versions.tf b/terraform/modules/aws-load-balancer-controller/versions.tf new file mode 100644 index 0000000000..3dccf26c7a --- /dev/null +++ b/terraform/modules/aws-load-balancer-controller/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.14.3" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.0" + } + } +} diff --git a/terraform/modules/bastion/log-collection-task.tf b/terraform/modules/bastion/log-collection-task.tf index 894fefcf43..969da97338 100644 --- a/terraform/modules/bastion/log-collection-task.tf +++ b/terraform/modules/bastion/log-collection-task.tf @@ -80,8 +80,8 @@ resource "aws_ecs_task_definition" "log_collector" { thanosreceivers.monitoring.thanos.io thanosrulers.monitoring.thanos.io thanosstores.monitoring.thanos.io - targetgroupbindings.eks.amazonaws.com - nodeclasses.eks.amazonaws.com + targetgroupbindings.elbv2.k8s.aws + ec2nodeclasses.karpenter.k8s.aws secretproviderclasses.secrets-store.csi.x-k8s.io ) batch=0 @@ -120,7 +120,7 @@ resource "aws_ecs_task_definition" "log_collector" { { name = "S3_KEY" value = "inspect-logs.tar.gz" - } + }, ] logConfiguration = { @@ -211,6 +211,7 @@ resource "aws_iam_role_policy" "log_collector_s3" { # EKS Access — Grants the log-collector task role cluster admin access # ============================================================================= + resource "aws_eks_access_entry" "log_collector" { cluster_name = var.cluster_name principal_arn = aws_iam_role.log_collector.arn diff --git a/terraform/modules/ecs-bootstrap/README.md b/terraform/modules/ecs-bootstrap/README.md index a2717a2829..d41c650991 100644 --- a/terraform/modules/ecs-bootstrap/README.md +++ b/terraform/modules/ecs-bootstrap/README.md @@ -1,6 +1,6 @@ # ECS Bootstrap Module -This Terraform module creates an ECS Fargate infrastructure for external ArgoCD bootstrap execution. It provides acess to secure, auditable tasks to run against the regional/management AWS accounts and EKS cluster. +This Terraform module creates an ECS Fargate infrastructure for external ArgoCD bootstrap execution. It provides access to secure, auditable tasks to run against the regional/management AWS accounts and EKS cluster. ## Overview @@ -24,9 +24,26 @@ module "ecs_bootstrap" { eks_cluster_name = module.eks_cluster.cluster_name eks_cluster_security_group_id = module.eks_cluster.cluster_security_group_id cluster_id = var.regional_id # or var.management_id + + # Karpenter inputs (from eks-cluster module outputs) + karpenter_controller_role_arn = module.eks_cluster.karpenter_controller_role_arn + karpenter_queue_url = module.eks_cluster.karpenter_queue_url + karpenter_version = "1.13.0" } ``` +## Bootstrap Sequence + +The ECS task executes the following steps in order: + +1. **Clone repository**: Checks out the configured git branch +2. **Configure kubectl**: Updates kubeconfig for the private EKS cluster +3. **Wait for addons**: Polls until CoreDNS and metrics-server are Active on the `karpenter-bootstrap` node group +4. **Install Karpenter** (when `karpenter_controller_role_arn` is set): Installs Karpenter via Helm from ECR public; skipped if already deployed +5. **Apply EC2NodeClass and NodePool**: Applies the FIPS `EC2NodeClass` and cluster-type-specific workloads `NodePool` from the `eks-nodepool` chart; always applied (idempotent) so any stale spec is corrected +6. **Prewarm validation**: Schedules a lightweight pod, waits up to 8 minutes for Karpenter to provision an EC2 node and bring it Ready. Failure prints diagnostic output (NodeClass, NodePool, NodeClaims, Karpenter logs) and exits — ECS retries the task automatically +7. **Install ArgoCD**: Installs ArgoCD via Helm and creates the Application of Applications for GitOps self-management + ## Security Features ### Network Security @@ -36,36 +53,39 @@ module "ecs_bootstrap" { ### IAM Security -- **EKS Access Entries**: Uses EKS access entry mechanism for Kubernetes RBAC - which can later receive further fine grained permissions -- **Minimal Permissions**: Task role has only required EKS and SSM permissions +- **EKS Access Entries**: Uses EKS access entry mechanism for Kubernetes RBAC +- **Minimal Permissions**: Task role has only required EKS, SSM, and Helm/kubectl permissions ### Audit Trail -- **CloudWatch Logs**: Complete logging of all bootstrap operations +- **CloudWatch Logs**: Complete logging of all bootstrap operations including Karpenter prewarm diagnostics - **ECS Task Tracking**: Task execution history and status - **Infrastructure as Code**: All permissions and configuration defined in Terraform ## Inputs -| Name | Description | Type | Default | Required | -| ----------------------------- | ----------------------------------------------------------------- | -------------- | ------- | :------: | -| cluster_id | Cluster identifier for resource naming (e.g., `regional`, `mc01`) | `string` | n/a | yes | -| vpc_id | VPC ID for ECS task execution | `string` | n/a | yes | -| private_subnets | Private subnet IDs for task execution | `list(string)` | n/a | yes | -| eks_cluster_arn | EKS cluster ARN for bootstrap configuration | `string` | n/a | yes | -| eks_cluster_name | EKS cluster name for bootstrap configuration | `string` | n/a | yes | -| eks_cluster_security_group_id | EKS cluster security group ID | `string` | n/a | yes | -| environment | Environment name for tagging | `string` | `"dev"` | no | +| Name | Description | Type | Default | Required | +| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- | ------- | :------: | +| `cluster_id` | Cluster identifier for resource naming (e.g., `regional`, `mc01`) | `string` | n/a | yes | +| `vpc_id` | VPC ID for ECS task execution | `string` | n/a | yes | +| `private_subnets` | Private subnet IDs for task execution | `list(string)` | n/a | yes | +| `eks_cluster_arn` | EKS cluster ARN for bootstrap configuration | `string` | n/a | yes | +| `eks_cluster_name` | EKS cluster name for bootstrap configuration | `string` | n/a | yes | +| `eks_cluster_security_group_id` | EKS cluster security group ID | `string` | n/a | yes | +| `karpenter_controller_role_arn` | IAM role ARN for Karpenter controller (IRSA). Set from `eks_cluster.karpenter_controller_role_arn`. When non-empty, Karpenter is installed before ArgoCD. | `string` | `""` | no | +| `karpenter_queue_url` | SQS queue URL for Karpenter EC2 interruption handling | `string` | `""` | no | +| `karpenter_version` | Karpenter Helm chart version to install (e.g., `"1.13.0"`) | `string` | `""` | no | +| `environment` | Environment name for tagging | `string` | `"dev"` | no | ## Outputs -| Name | Description | -| --------------------------- | ------------------------------------------------------ | -| ecs_cluster_arn | ARN of the ECS cluster for bootstrap tasks | -| task_definition_arn | ARN of the ECS task definition for bootstrap execution | -| log_group_name | CloudWatch log group name for bootstrap operations | -| bootstrap_security_group_id | Security group ID for bootstrap ECS tasks | -| private_subnets | Private subnet IDs where bootstrap tasks run | +| Name | Description | +| ----------------------------- | ------------------------------------------------------ | +| `ecs_cluster_arn` | ARN of the ECS cluster for bootstrap tasks | +| `task_definition_arn` | ARN of the ECS task definition for bootstrap execution | +| `log_group_name` | CloudWatch log group name for bootstrap operations | +| `bootstrap_security_group_id` | Security group ID for bootstrap ECS tasks | +| `private_subnets` | Private subnet IDs where bootstrap tasks run | ## Requirements @@ -73,10 +93,3 @@ module "ecs_bootstrap" { | --------- | --------- | | terraform | >= 1.14.3 | | aws | >= 5.0 | - -## Future Enhancements - -This ECS infrastructure is designed to support future SRE operations beyond bootstrap: - -- **Operational Tasks**: Cluster maintenance, backup operations, monitoring setup -- **Pre-built Containers**: In the future ad-hoc script pulling will be replaced by versioned containers built through konflux diff --git a/terraform/modules/ecs-bootstrap/main.tf b/terraform/modules/ecs-bootstrap/main.tf index 622d2db08d..c3e0842b77 100644 --- a/terraform/modules/ecs-bootstrap/main.tf +++ b/terraform/modules/ecs-bootstrap/main.tf @@ -103,29 +103,8 @@ resource "aws_ecs_task_definition" "bootstrap" { # Configure kubectl for EKS aws eks update-kubeconfig --name $CLUSTER_NAME - # Seed the FIPS NodePool only on first bootstrap. On subsequent - # runs (resync), ArgoCD owns this resource via the eks-nodepool - # chart — we must not re-apply it to avoid Server-Side Apply - # ownership conflicts. When creating, we pass the environment - # values file so the initial NodePool matches what ArgoCD will - # enforce, avoiding Karpenter provisioning nodes with default - # instance types before ArgoCD syncs. - if ! kubectl get nodepool workloads 2>/dev/null; then - echo "Applying FIPS NodeClass and workloads NodePool from chart..." - _NODEPOOL_VALUES="$REPO_DIR/deploy/$ENVIRONMENT/$REGION_DEPLOYMENT/argocd-values-$CLUSTER_TYPE.yaml" - _VALUES_FLAG="" - [ -f "$_NODEPOOL_VALUES" ] && _VALUES_FLAG="-f $_NODEPOOL_VALUES" - helm template eks-nodepool "$REPO_DIR/argocd/config/$CLUSTER_TYPE/eks-nodepool" \ - --set global.cluster_name="$CLUSTER_NAME" \ - $_VALUES_FLAG \ - | kubectl apply --server-side -f - - echo "✓ FIPS NodePool applied" - else - echo "✓ FIPS NodePool already exists, skipping (managed by ArgoCD)" - fi - - # Wait for coredns and metrics-server (managed by the built-in system pool) - # to be active before installing ArgoCD. + # Wait for coredns and metrics-server (on the bootstrap node group) + # before installing Karpenter and ArgoCD. for ADDON in coredns metrics-server; do echo "Waiting for $ADDON to be active..." aws eks wait addon-active \ @@ -135,42 +114,90 @@ resource "aws_ecs_task_definition" "bootstrap" { echo "✓ $ADDON active" done - # Check if ArgoCD already exists + if [ -n "$${KARPENTER_CONTROLLER_ROLE_ARN:-}" ]; then + # Install Karpenter before seeding the NodePool: the NodePool and + # EC2NodeClass CRDs (karpenter.sh/v1, karpenter.k8s.aws/v1) don't + # exist until Karpenter is installed. ArgoCD adopts this release + # via its self-managed Karpenter Application after bootstrap. + if ! helm status karpenter -n kube-system 2>/dev/null | grep -q "^STATUS: deployed"; then + echo "Installing Karpenter $KARPENTER_VERSION..." + if [ -z "$${KARPENTER_QUEUE_URL:-}" ]; then + echo "ERROR: KARPENTER_QUEUE_URL is not set — required when KARPENTER_CONTROLLER_ROLE_ARN is provided" >&2 + exit 1 + fi + _KARPENTER_QUEUE_NAME=$(basename "$KARPENTER_QUEUE_URL") + helm upgrade --install karpenter \ + oci://public.ecr.aws/karpenter/karpenter \ + --version "$KARPENTER_VERSION" \ + --namespace kube-system \ + --set "settings.clusterName=$CLUSTER_NAME" \ + --set "settings.interruptionQueue=$_KARPENTER_QUEUE_NAME" \ + --set "serviceAccount.annotations.eks\.amazonaws\.com/role-arn=$KARPENTER_CONTROLLER_ROLE_ARN" \ + --set 'tolerations[0].key=CriticalAddonsOnly' \ + --set 'tolerations[0].operator=Exists' \ + --set 'tolerations[0].effect=NoSchedule' \ + --wait --timeout=5m + echo "✓ Karpenter installed" + else + echo "✓ Karpenter already deployed, skipping" + fi + + # Seed the FIPS NodePool only on first bootstrap. On subsequent + # runs (resync), ArgoCD owns this resource via the eks-nodepool + # chart — re-applying it creates SSA ownership conflicts. + if ! kubectl get nodepool workloads 2>/dev/null; then + echo "Applying FIPS EC2NodeClass and workloads NodePool from chart..." + _NODEPOOL_VALUES="$REPO_DIR/deploy/$ENVIRONMENT/$REGION_DEPLOYMENT/argocd-values-$CLUSTER_TYPE.yaml" + _VALUES_FLAG="" + [ -f "$_NODEPOOL_VALUES" ] && _VALUES_FLAG="-f $_NODEPOOL_VALUES" + helm template eks-nodepool "$REPO_DIR/argocd/config/$CLUSTER_TYPE/eks-nodepool" \ + --set global.cluster_name="$CLUSTER_NAME" \ + $_VALUES_FLAG \ + | kubectl apply --server-side -f - + echo "✓ FIPS EC2NodeClass and NodePool applied" + else + echo "✓ FIPS NodePool already exists, skipping (managed by ArgoCD)" + fi + + fi + if ! kubectl get deployment argocd-server -n argocd 2>/dev/null; then - echo "Installing ArgoCD from repo chart..." - - # Create argocd namespace - kubectl create namespace argocd --dry-run=client -o yaml | kubectl apply -f - - - # Fetch chart dependencies (charts/ is gitignored) - helm repo add argo https://argoproj.github.io/argo-helm - helm dependency build "$REPO_DIR/argocd/config/shared/argocd" - - # Install using the same chart that the self-managed ArgoCD app - # uses (argocd/config/shared/argocd/), with tracking-id annotations - # so the self-managed ArgoCD app can adopt these resources. - # redisSecretInit is enabled here to create the Redis auth secret; - # the self-managed ArgoCD app has it disabled and prunes the - # completed Job on adoption. - helm upgrade --install argocd "$REPO_DIR/argocd/config/shared/argocd" \ - --namespace argocd \ - --set argo-cd.redisSecretInit.enabled=true \ - --set 'argo-cd.redisSecretInit.tolerations[0].key=CriticalAddonsOnly' \ - --set 'argo-cd.redisSecretInit.tolerations[0].operator=Exists' \ - --set 'argo-cd.redisSecretInit.tolerations[0].effect=NoSchedule' \ - --set-string 'argo-cd.controller.annotations.argocd\.argoproj\.io/tracking-id=argocd:argoproj.io/Application:argocd/argocd' \ - --set-string 'argo-cd.server.annotations.argocd\.argoproj\.io/tracking-id=argocd:argoproj.io/Application:argocd/argocd' \ - --set-string 'argo-cd.repoServer.annotations.argocd\.argoproj\.io/tracking-id=argocd:argoproj.io/Application:argocd/argocd' \ - --wait --timeout=5m - - echo "✓ ArgoCD installation complete" - - # Wait for ArgoCD to be ready - kubectl wait --for=condition=available --timeout=600s deployment/argocd-server -n argocd - kubectl wait --for=condition=available --timeout=600s deployment/argocd-repo-server -n argocd - kubectl wait --for=condition=available --timeout=600s deployment/argocd-applicationset-controller -n argocd - - echo "✓ ArgoCD is running and ready" + echo "Installing ArgoCD from repo chart..." + + # Create argocd namespace + kubectl create namespace argocd --dry-run=client -o yaml | kubectl apply -f - + + # Fetch chart dependencies (charts/ is gitignored) + helm repo add argo https://argoproj.github.io/argo-helm + helm dependency build "$REPO_DIR/argocd/config/shared/argocd" + + # Install using the same chart that the self-managed ArgoCD app + # uses (argocd/config/shared/argocd/), with tracking-id annotations + # so the self-managed ArgoCD app can adopt these resources. + # redisSecretInit is enabled here to create the Redis auth secret; + # the self-managed ArgoCD app has it disabled and prunes the + # completed Job on adoption. + # CriticalAddonsOnly tolerations are defined in values.yaml and + # picked up automatically by helm install — no --set flags needed. + helm upgrade --install argocd "$REPO_DIR/argocd/config/shared/argocd" \ + --namespace argocd \ + --set argo-cd.redisSecretInit.enabled=true \ + --set 'argo-cd.redisSecretInit.tolerations[0].key=CriticalAddonsOnly' \ + --set 'argo-cd.redisSecretInit.tolerations[0].operator=Exists' \ + --set 'argo-cd.redisSecretInit.tolerations[0].effect=NoSchedule' \ + --set-string 'argo-cd.controller.annotations.argocd\.argoproj\.io/tracking-id=argocd:argoproj.io/Application:argocd/argocd' \ + --set-string 'argo-cd.server.annotations.argocd\.argoproj\.io/tracking-id=argocd:argoproj.io/Application:argocd/argocd' \ + --set-string 'argo-cd.repoServer.annotations.argocd\.argoproj\.io/tracking-id=argocd:argoproj.io/Application:argocd/argocd' \ + --wait --timeout=10m + + echo "✓ ArgoCD installation complete" + + # Wait for ArgoCD to be ready + kubectl wait --for=condition=available --timeout=600s deployment/argocd-server -n argocd + kubectl wait --for=condition=available --timeout=600s deployment/argocd-repo-server -n argocd + kubectl wait --for=condition=available --timeout=600s deployment/argocd-applicationset-controller -n argocd + + echo "✓ ArgoCD is running and ready" else echo "✓ ArgoCD is already installed and running, skipping installation" fi @@ -264,6 +291,36 @@ resource "aws_ecs_task_definition" "bootstrap" { - CreateNamespace=true APP_EOF + # CI: E2E test runner starts immediately after bootstrap exits, so + # HyperShift must be fully installed before work agents apply + # HostedCluster manifests. This wait is a CI accommodation — bootstrap + # has no production requirement to block on application-level health. + if [ "$${CLUSTER_TYPE:-}" = "management-cluster" ]; then + echo "=== Waiting for hypershift Application to be Healthy (up to 30m) ===" + _HS_DEADLINE=$((SECONDS + 1800)) + until _HS_HEALTH=$(kubectl get application hypershift -n argocd \ + -o jsonpath='{.status.health.status}' 2>/tmp/hs-err) \ + && [ "$${_HS_HEALTH}" = "Healthy" ]; do + if grep -qiE "unable to connect|connection refused|i/o timeout|no such host" /tmp/hs-err 2>/dev/null; then + echo "ERROR: kubectl cannot reach the API server — cannot wait for hypershift:" >&2 + cat /tmp/hs-err >&2 + exit 1 + fi + if [ $SECONDS -ge $_HS_DEADLINE ]; then + echo "ERROR: hypershift Application not Healthy after 30 minutes" >&2 + kubectl get application hypershift -n argocd -o yaml 2>/dev/null || true + exit 1 + fi + _HS_STATUS=$(kubectl get application hypershift -n argocd \ + -o jsonpath='{.status.health.status}' 2>/dev/null || echo "NotFound") + _HS_MSG=$(kubectl get application hypershift -n argocd \ + -o jsonpath='{.status.health.message}' 2>/dev/null || true) + echo " hypershift health: $${_HS_STATUS} ($(( _HS_DEADLINE - SECONDS ))s remaining)$${_HS_MSG:+ — $${_HS_MSG}}" + sleep 15 + done + echo "=== hypershift is Healthy ===" + fi + echo "=== Bootstrap completed successfully ===" EOF ] @@ -298,6 +355,18 @@ resource "aws_ecs_task_definition" "bootstrap" { { name = "REDIS_ENDPOINT" value = var.redis_endpoint + }, + { + name = "KARPENTER_CONTROLLER_ROLE_ARN" + value = var.karpenter_controller_role_arn + }, + { + name = "KARPENTER_QUEUE_URL" + value = var.karpenter_queue_url + }, + { + name = "KARPENTER_VERSION" + value = var.karpenter_version } ] diff --git a/terraform/modules/ecs-bootstrap/variables.tf b/terraform/modules/ecs-bootstrap/variables.tf index 07e0021715..0f3e875204 100644 --- a/terraform/modules/ecs-bootstrap/variables.tf +++ b/terraform/modules/ecs-bootstrap/variables.tf @@ -82,3 +82,20 @@ variable "redis_endpoint" { default = "" } +variable "karpenter_controller_role_arn" { + description = "IAM role ARN for the Karpenter controller (IRSA). Required when the EKS cluster uses OSS Karpenter." + type = string + default = "" +} + +variable "karpenter_queue_url" { + description = "SQS queue URL for Karpenter interruption handling." + type = string + default = "" +} + +variable "karpenter_version" { + description = "Karpenter Helm chart version to install during bootstrap." + type = string + default = "1.13.0" +} diff --git a/terraform/modules/eks-cluster/README.md b/terraform/modules/eks-cluster/README.md index bd3fb04504..c91261427f 100644 --- a/terraform/modules/eks-cluster/README.md +++ b/terraform/modules/eks-cluster/README.md @@ -10,6 +10,7 @@ Creates private EKS clusters with security-first configuration and standardized - **GitOps Bootstrap**: Automated ArgoCD installation via ECS Fargate task for self-management - **Security Hardening**: KMS encryption, IMDSv2 enforcement, and network segmentation - **High Availability**: Multi-AZ NAT Gateways for fault-tolerant egress connectivity +- **OSS Karpenter**: Node provisioning via Karpenter v1 with FIPS-validated EC2NodeClass ## Security & Scalability Enhancements @@ -18,7 +19,7 @@ Creates private EKS clusters with security-first configuration and standardized - **KMS Encryption**: Kubernetes secrets encrypted at rest using customer-managed keys - **Dedicated Security Groups**: VPC endpoints use isolated security groups (port 443 from VPC CIDR only) - **Restricted Egress**: Cluster egress limited to HTTPS for container registries and VPC internal traffic -- **Auto Mode Authentication**: EKS authentication configured for API_AND_CONFIG_MAP mode +- **EKS Authentication**: Configured for API_AND_CONFIG_MAP mode ### High Availability Network Architecture @@ -87,59 +88,63 @@ module "regional_cluster" { ## Variables -| Name | Description | Type | Default | Required | -| ------------------------------- | ------------------------------------------------------------------------------- | -------------- | ------------------------------------------------------- | -------- | -| `cluster_id` | Deterministic cluster identifier for resource naming (e.g., `regional`, `mc01`) | `string` | n/a | yes | -| `cluster_type` | Type of cluster: `regional-cluster` or `management-cluster` | `string` | n/a | yes | -| `cluster_version` | Kubernetes version | `string` | `"1.34"` | no | -| `vpc_cidr` | VPC CIDR block | `string` | `"10.0.0.0/16"` | no | -| `availability_zones` | List of availability zones (auto-detected if empty) | `list(string)` | `[]` | no | -| `private_subnet_cidrs` | CIDR blocks for private subnets | `list(string)` | `["10.0.0.0/18", "10.0.64.0/18", "10.0.128.0/18"]` | no | -| `public_subnet_cidrs` | CIDR blocks for public subnets | `list(string)` | `["10.0.192.0/22", "10.0.196.0/22", "10.0.200.0/22"]` | no | -| `enable_pod_security_standards` | Enable Pod Security Standards | `bool` | `true` | no | -| `bootstrap_enabled` | Enable ArgoCD bootstrap for GitOps management | `bool` | `true` | no | -| `argocd_namespace` | Kubernetes namespace for ArgoCD installation | `string` | `"argocd"` | no | -| `argocd_chart_version` | ArgoCD Helm chart version | `string` | `"9.3.0"` | no | -| `bootstrap_repository_url` | Git repository URL for ArgoCD configuration | `string` | `"https://github.com/openshift-online/rosa-hyperfleet"` | no | -| `bootstrap_repository_branch` | Git branch to track | `string` | `"main"` | no | +| Name | Description | Type | Default | Required | +| ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- | ------------------------------------------------------- | -------- | +| `cluster_id` | Deterministic cluster identifier for resource naming (e.g., `regional`, `mc01`) | `string` | n/a | yes | +| `cluster_type` | Type of cluster: `regional-cluster` or `management-cluster` | `string` | n/a | yes | +| `cluster_version` | Kubernetes version | `string` | `"1.34"` | no | +| `vpc_cidr` | VPC CIDR block | `string` | `"10.0.0.0/16"` | no | +| `availability_zones` | List of availability zones (auto-detected if empty) | `list(string)` | `[]` | no | +| `private_subnet_cidrs` | CIDR blocks for private subnets | `list(string)` | `["10.0.0.0/18", "10.0.64.0/18", "10.0.128.0/18"]` | no | +| `public_subnet_cidrs` | CIDR blocks for public subnets | `list(string)` | `["10.0.192.0/22", "10.0.196.0/22", "10.0.200.0/22"]` | no | +| `enable_pod_security_standards` | Enable Pod Security Standards | `bool` | `true` | no | +| `ami_kms_key_arn` | ARN of the Red Hat KMS key encrypting FIPS AMI EBS snapshots. When set, adds `kms:Decrypt` and `kms:CreateGrant` to Karpenter node and controller roles. | `string` | `""` | no | +| `bootstrap_enabled` | Enable ArgoCD bootstrap for GitOps management | `bool` | `true` | no | +| `argocd_namespace` | Kubernetes namespace for ArgoCD installation | `string` | `"argocd"` | no | +| `argocd_chart_version` | ArgoCD Helm chart version | `string` | `"9.3.0"` | no | +| `bootstrap_repository_url` | Git repository URL for ArgoCD configuration | `string` | `"https://github.com/openshift-online/rosa-hyperfleet"` | no | +| `bootstrap_repository_branch` | Git branch to track | `string` | `"main"` | no | ## Outputs -| Name | Description | -| ------------------------------------ | -------------------------------------------------- | -| `cluster_name` | EKS cluster name (same as `cluster_id`) | -| `cluster_endpoint` | EKS cluster API endpoint | -| `cluster_certificate_authority_data` | Base64 encoded certificate data | -| `vpc_id` | VPC ID where cluster is deployed | -| `private_subnets` | Private subnet IDs where worker nodes are deployed | -| `cluster_security_group_id` | EKS cluster security group ID | -| `bootstrap_report` | Bootstrap process information and status | +| Name | Description | +| -------------------------------------- | ---------------------------------------------------------------------------------------- | +| `cluster_name` | EKS cluster name (same as `cluster_id`) | +| `cluster_endpoint` | EKS cluster API endpoint | +| `cluster_certificate_authority_data` | Base64 encoded certificate data | +| `vpc_id` | VPC ID where cluster is deployed | +| `private_subnets` | Private subnet IDs where worker nodes are deployed | +| `cluster_security_group_id` | EKS cluster security group ID | +| `karpenter_controller_role_arn` | IAM role ARN for Karpenter controller (IRSA) | +| `karpenter_queue_url` | SQS queue URL for Karpenter EC2 interruption handling | +| `karpenter_node_instance_profile_name` | Instance profile name for Karpenter-provisioned nodes (matches `EC2NodeClass.spec.role`) | +| `bootstrap_report` | Bootstrap process information and status | ## Bootstrap Functionality -When `bootstrap_enabled` is `true`, the module automatically installs ArgoCD for GitOps management: +When `bootstrap_enabled` is `true`, the module automatically installs Karpenter and ArgoCD via an ECS Fargate task: 1. **ECS Fargate Task**: Executes within cluster VPC for secure bootstrap operations 2. **Tool Installation**: Downloads kubectl, helm, and AWS CLI at runtime -3. **FIPS Node Setup**: Applies FIPS NodeClass and cluster-type-specific workloads NodePool -4. **Addon Wait**: Waits for CoreDNS and metrics-server addons to become Active -5. **ArgoCD Installation**: Installs ArgoCD via Helm with cluster-only access -6. **GitOps Configuration**: Creates Application of Applications for self-management -7. **Synchronous Execution**: Bootstrap completes during `terraform apply` with visible logs +3. **Addon Wait**: Waits for CoreDNS and metrics-server to become Active on the `karpenter-bootstrap` node group +4. **Karpenter Install**: Installs Karpenter via Helm from ECR public (`oci://public.ecr.aws/karpenter/karpenter`) +5. **FIPS Node Setup**: Applies FIPS `EC2NodeClass` (`fips`) and cluster-type-specific workloads `NodePool` +6. **Prewarm Validation**: Provisions one Karpenter node and waits for it to be Ready before continuing +7. **ArgoCD Installation**: Installs ArgoCD via Helm with cluster-only access +8. **GitOps Configuration**: Creates Application of Applications for self-management +9. **Synchronous Execution**: Bootstrap completes during `terraform apply` with visible logs -### Bootstrap Process +### Karpenter Infrastructure -The ECS bootstrap task: +The module provisions Karpenter-based compute: -- Runs in the cluster's private subnets for network access -- Updates kubeconfig using EKS access entries and Pod Identity -- Applies a FIPS-validated Bottlerocket NodeClass (`fips`) and a workloads NodePool -- Waits for CoreDNS and metrics-server to be Active (scheduled on the built-in `system` pool) -- Installs ArgoCD using Helm from the official repository -- Creates bootstrap application pointing to your repository -- Enables ArgoCD to take over cluster management +- **`karpenter-bootstrap` managed node group**: 2x t3.medium nodes tainted `CriticalAddonsOnly=true:NoSchedule`. Hosts Karpenter controller, CoreDNS, and metrics-server. +- **Karpenter controller IAM role**: IRSA-backed, scoped to `kube-system/karpenter` ServiceAccount with SQS, EC2, and IAM instance profile permissions. +- **Karpenter node IAM role**: Full `AmazonEKSWorkerNodePolicy`, VPC CNI, ECR pull-only, and optional KMS decrypt for FIPS AMI snapshots. +- **SQS queue**: Receives EC2 interruption events (spot reclamation, instance health, rebalance) for graceful node draining. +- **EventBridge rules**: Four rules forward EC2 lifecycle events to the SQS queue. -For the FIPS node strategy, including why the built-in `system` pool is retained and `general-purpose` is disabled, see [FIPS-Only EKS Compute](../../../docs/design/fips-eks-compute.md). +For the FIPS node strategy, including why Auto Mode was replaced with OSS Karpenter, see [FIPS-Only EKS Compute](../../../docs/design/fips-eks-compute.md). For Karpenter IAM role design, see [Karpenter Node Provisioning](../../../docs/design/karpenter-node-provisioning.md). ## Requirements diff --git a/terraform/modules/eks-cluster/data.tf b/terraform/modules/eks-cluster/data.tf index 92cf264428..e2d509b9da 100644 --- a/terraform/modules/eks-cluster/data.tf +++ b/terraform/modules/eks-cluster/data.tf @@ -8,3 +8,9 @@ data "aws_partition" "current" {} # Current AWS region data "aws_region" "current" {} + +# TLS certificate for the EKS OIDC issuer endpoint — provides the thumbprint required +# by aws_iam_openid_connect_provider for Karpenter IRSA. +data "tls_certificate" "eks_oidc" { + url = aws_eks_cluster.main.identity[0].oidc[0].issuer +} diff --git a/terraform/modules/eks-cluster/iam.tf b/terraform/modules/eks-cluster/iam.tf index 4ccaf18c8b..bef32885cf 100644 --- a/terraform/modules/eks-cluster/iam.tf +++ b/terraform/modules/eks-cluster/iam.tf @@ -1,23 +1,20 @@ # ============================================================================= -# IAM Roles and Policies for EKS Cluster +# IAM Roles and Policies for EKS Cluster (OSS Karpenter) # -# Creates IAM roles required for EKS Auto Mode operation: -# - Cluster service role with required permissions -# - Node group role for Auto Mode managed nodes +# - eks_cluster role: AmazonEKSClusterPolicy +# - karpenter_node role: AmazonEKSWorkerNodePolicy + CNI + ECR + SSM +# - karpenter_controller role: IRSA-backed, scoped to kube-system/karpenter SA +# - ebs_csi role: Pod Identity-backed, scoped to kube-system/ebs-csi-controller-sa +# - SQS interruption queue + four EventBridge rules # ============================================================================= # ----------------------------------------------------------------------------- # EKS Cluster Service Role -# -# Role assumed by EKS control plane. Auto Mode requires additional permissions -# including sts:TagSession for resource tagging. -# See: https://docs.aws.amazon.com/eks/latest/userguide/automode-get-started-cli.html#auto-mode-create-roles # ----------------------------------------------------------------------------- resource "aws_iam_role" "eks_cluster" { name = "${local.cluster_id}-cluster-role" - # Auto Mode REQUIRES sts:TagSession to propagate tags to managed infra assume_role_policy = jsonencode({ Version = "2012-10-17" Statement = [{ @@ -29,44 +26,394 @@ resource "aws_iam_role" "eks_cluster" { } resource "aws_iam_role_policy_attachment" "eks_cluster_managed" { + policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonEKSClusterPolicy" + role = aws_iam_role.eks_cluster.name +} + +# ============================================================================= +# Karpenter +# ============================================================================= + +# ----------------------------------------------------------------------------- +# Karpenter Node Role + Instance Profile +# +# Used by both Karpenter-provisioned FIPS nodes (via EC2NodeClass.spec.instanceProfile) +# and the AL2023 bootstrap managed node group. +# ----------------------------------------------------------------------------- + +resource "aws_iam_role" "karpenter_node" { + name = "${local.cluster_id}-karpenter-node-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Action = ["sts:AssumeRole", "sts:TagSession"] + Effect = "Allow" + Principal = { Service = "ec2.amazonaws.com" } + }] + }) +} + +resource "aws_iam_role_policy_attachment" "karpenter_node_managed" { for_each = toset([ - "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", - "arn:aws:iam::aws:policy/AmazonEKSComputePolicy", - "arn:aws:iam::aws:policy/AmazonEKSBlockStoragePolicy", - "arn:aws:iam::aws:policy/AmazonEKSLoadBalancingPolicy", - "arn:aws:iam::aws:policy/AmazonEKSNetworkingPolicy" + "arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonEKSWorkerNodePolicy", + "arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonEC2ContainerRegistryPullOnly", + "arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonEKS_CNI_Policy", + "arn:${data.aws_partition.current.partition}:iam::aws:policy/AmazonSSMManagedInstanceCore", ]) policy_arn = each.value - role = aws_iam_role.eks_cluster.name + role = aws_iam_role.karpenter_node.name +} + +# Instance profile wrapping the node role. Referenced by EC2NodeClass.spec.instanceProfile. +# Pre-creating it here avoids race conditions during bootstrap and removes the +# need for iam:CreateInstanceProfile in the Karpenter controller policy. +resource "aws_iam_instance_profile" "karpenter_node" { + name = "${local.cluster_id}-karpenter-node-role" + role = aws_iam_role.karpenter_node.name +} + +# ----------------------------------------------------------------------------- +# OIDC Provider — required for Karpenter controller IRSA +# ----------------------------------------------------------------------------- + +resource "aws_iam_openid_connect_provider" "eks" { + client_id_list = ["sts.amazonaws.com"] + thumbprint_list = [data.tls_certificate.eks_oidc.certificates[0].sha1_fingerprint] + url = aws_eks_cluster.main.identity[0].oidc[0].issuer } # ----------------------------------------------------------------------------- -# EKS Auto Mode Node Role +# Karpenter Controller Role (IRSA) # -# Role assumed by Auto Mode managed nodes. Includes all required policies -# for node operation, networking, storage, and load balancing. -# See: https://docs.aws.amazon.com/eks/latest/userguide/automode-get-started-cli.html#auto-mode-create-roles +# Karpenter predates EKS Pod Identity support; IRSA is the supported auth +# mechanism. See ADR docs/design/karpenter-node-provisioning.md. # ----------------------------------------------------------------------------- -resource "aws_iam_role" "eks_auto_mode_node" { - name = "${local.cluster_id}-auto-node-role" + +resource "aws_iam_role" "karpenter_controller" { + name = "${local.cluster_id}-karpenter-controller" assume_role_policy = jsonencode({ Version = "2012-10-17" Statement = [{ - Action = ["sts:AssumeRole", "sts:TagSession"] Effect = "Allow" Principal = { - Service = ["ec2.amazonaws.com", "eks.amazonaws.com"] + Federated = aws_iam_openid_connect_provider.eks.arn + } + Action = "sts:AssumeRoleWithWebIdentity" + Condition = { + StringEquals = { + "${local.oidc_issuer}:sub" = "system:serviceaccount:kube-system:karpenter" + "${local.oidc_issuer}:aud" = "sts.amazonaws.com" + } } }] }) } -resource "aws_iam_role_policy_attachment" "auto_node_managed" { - for_each = toset([ - "arn:aws:iam::aws:policy/AmazonEKSWorkerNodeMinimalPolicy", - "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPullOnly" - ]) - policy_arn = each.value - role = aws_iam_role.eks_auto_mode_node.name -} \ No newline at end of file +resource "aws_iam_role_policy" "karpenter_controller" { + name = "karpenter-controller" + role = aws_iam_role.karpenter_controller.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "EC2FleetDescribe" + Effect = "Allow" + Action = [ + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets", + ] + Resource = "*" + }, + { + Sid = "EC2FleetCreate" + Effect = "Allow" + Action = [ + "ec2:CreateFleet", + "ec2:CreateLaunchTemplate", + "ec2:CreateTags", + "ec2:RunInstances", + ] + Resource = "*" + Condition = { + StringEquals = { + "aws:RequestTag/kubernetes.io/cluster/${local.cluster_id}" = "owned" + } + } + }, + { + # The nodeclaim.tagging controller calls CreateTags on already-running instances + # to apply karpenter.sh/* labels post-creation. aws:RequestTag only applies to + # tags set during resource creation, so a separate statement scoped by + # aws:ResourceTag is required for post-creation tagging. + Sid = "EC2NodeClaimTagging" + Effect = "Allow" + Action = ["ec2:CreateTags"] + Resource = [ + "arn:${data.aws_partition.current.partition}:ec2:*:*:instance/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:volume/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:network-interface/*", + ] + Condition = { + StringEquals = { + "aws:ResourceTag/kubernetes.io/cluster/${local.cluster_id}" = "owned" + } + } + }, + { + # RunInstances on pre-existing resources (AMI, security-group, subnet) must be + # unconditional: these resources don't receive aws:RequestTag during RunInstances, + # so the RequestTag condition in EC2FleetCreate always denies them. This is the + # same split used in the official Karpenter IAM policy. + Sid = "EC2RunInstancesValidation" + Effect = "Allow" + Action = ["ec2:RunInstances"] + Resource = [ + "arn:${data.aws_partition.current.partition}:ec2:*::image/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:fleet/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:instance/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:launch-template/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:network-interface/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:security-group/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:spot-instances-request/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:subnet/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:volume/*", + ] + }, + { + Sid = "EC2FleetDelete" + Effect = "Allow" + Action = [ + "ec2:DeleteLaunchTemplate", + "ec2:TerminateInstances", + ] + Resource = "*" + Condition = { + StringEquals = { + "aws:ResourceTag/kubernetes.io/cluster/${local.cluster_id}" = "owned" + } + } + }, + { + Sid = "IAMInstanceProfileCreate" + Effect = "Allow" + Action = [ + "iam:CreateInstanceProfile", + "iam:TagInstanceProfile", + ] + Resource = "*" + Condition = { + StringEquals = { + "aws:RequestTag/kubernetes.io/cluster/${local.cluster_id}" = "owned" + } + } + }, + { + Sid = "IAMInstanceProfileModify" + Effect = "Allow" + Action = [ + "iam:AddRoleToInstanceProfile", + "iam:DeleteInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + ] + Resource = "*" + Condition = { + StringEquals = { + "aws:ResourceTag/kubernetes.io/cluster/${local.cluster_id}" = "owned" + } + } + }, + { + # GetInstanceProfile and ListInstanceProfiles are read-only and must be + # unconditional: Karpenter calls GetInstanceProfile before creating (and + # tagging) a profile, so a ResourceTag condition always denies it. + # ListInstanceProfiles is required by the instance-profile GC controller. + Sid = "IAMInstanceProfileRead" + Effect = "Allow" + Action = [ + "iam:GetInstanceProfile", + "iam:ListInstanceProfiles", + ] + Resource = "*" + }, + { + Sid = "IAMPassRole" + Effect = "Allow" + Action = "iam:PassRole" + Resource = aws_iam_role.karpenter_node.arn + Condition = { + StringEquals = { + "iam:PassedToService" = "ec2.amazonaws.com" + } + } + }, + { + Sid = "SQS" + Effect = "Allow" + Action = [ + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage", + ] + Resource = aws_sqs_queue.karpenter_interruption.arn + }, + { + Sid = "EKS" + Effect = "Allow" + Action = "eks:DescribeCluster" + Resource = aws_eks_cluster.main.arn + }, + { + Sid = "SSM" + Effect = "Allow" + Action = "ssm:GetParameter" + Resource = "arn:${data.aws_partition.current.partition}:ssm:*:*:parameter/aws/service/*" + }, + { + Sid = "Pricing" + Effect = "Allow" + Action = "pricing:GetProducts" + Resource = "*" + }, + ] + }) +} + +# The controller (IRSA) calls RunInstances, which requires kms:CreateGrant so +# EC2 can decrypt the RHEL FIPS AMI's encrypted EBS snapshot on instance launch. +# kms:GrantIsForAWSResource restricts grant creation to AWS service principals, +# preventing the controller from granting arbitrary IAM principals key access. +resource "aws_iam_role_policy" "karpenter_controller_kms" { + count = var.ami_kms_key_arn != "" ? 1 : 0 + name = "rhel-ami-kms" + role = aws_iam_role.karpenter_controller.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "RhelAmiKmsGrant" + Effect = "Allow" + Action = ["kms:CreateGrant"] + Resource = var.ami_kms_key_arn + Condition = { + Bool = { + "kms:GrantIsForAWSResource" = "true" + } + } + }, + { + Sid = "RhelAmiKmsDescribe" + Effect = "Allow" + Action = ["kms:DescribeKey"] + Resource = var.ami_kms_key_arn + }, + ] + }) +} + +# ----------------------------------------------------------------------------- +# SQS Interruption Queue + EventBridge Rules +# +# Receives EC2 Spot, rebalance, state-change, and AWS Health events so Karpenter +# can drain nodes before the 2-minute Spot termination window expires. +# ----------------------------------------------------------------------------- + +resource "aws_sqs_queue" "karpenter_interruption" { + name = "${local.cluster_id}-karpenter" + + message_retention_seconds = 300 + sqs_managed_sse_enabled = true +} + +resource "aws_sqs_queue_policy" "karpenter_interruption" { + queue_url = aws_sqs_queue.karpenter_interruption.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Sid = "AllowEventBridge" + Effect = "Allow" + Principal = { Service = "events.amazonaws.com" } + Action = "sqs:SendMessage" + Resource = aws_sqs_queue.karpenter_interruption.arn + }] + }) +} + +locals { + karpenter_event_rules = { + spot-interruption = { + description = "Karpenter: EC2 Spot Instance Interruption Warning" + event_pattern = jsonencode({ source = ["aws.ec2"], "detail-type" = ["EC2 Spot Instance Interruption Warning"] }) + } + instance-terminated = { + description = "Karpenter: EC2 Instance Terminated" + event_pattern = jsonencode({ source = ["aws.ec2"], "detail-type" = ["EC2 Instance State-change Notification"], detail = { state = ["terminated"] } }) + } + rebalance-recommendation = { + description = "Karpenter: EC2 Instance Rebalance Recommendation" + event_pattern = jsonencode({ source = ["aws.ec2"], "detail-type" = ["EC2 Instance Rebalance Recommendation"] }) + } + health-scheduled-change = { + description = "Karpenter: AWS Health EC2 Scheduled Change" + event_pattern = jsonencode({ source = ["aws.health"], "detail-type" = ["AWS Health Event"], detail = { service = ["EC2"], eventTypeCategory = ["scheduledChange"] } }) + } + } +} + +resource "aws_cloudwatch_event_rule" "karpenter" { + for_each = local.karpenter_event_rules + name = "${local.cluster_id}-karpenter-${each.key}" + description = each.value.description + event_pattern = each.value.event_pattern +} + +resource "aws_cloudwatch_event_target" "karpenter" { + for_each = local.karpenter_event_rules + rule = aws_cloudwatch_event_rule.karpenter[each.key].name + arn = aws_sqs_queue.karpenter_interruption.arn +} + +# ----------------------------------------------------------------------------- +# EBS CSI Driver Role (Pod Identity) +# +# Pod Identity is the platform-standard auth mechanism for addons. The controller +# service account (ebs-csi-controller-sa in kube-system) is bound via +# aws_eks_pod_identity_association — no service_account_role_arn annotation needed. +# ----------------------------------------------------------------------------- + +resource "aws_iam_role" "ebs_csi" { + name = "${local.cluster_id}-ebs-csi-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "pods.eks.amazonaws.com" } + Action = ["sts:AssumeRole", "sts:TagSession"] + }] + }) +} + +resource "aws_iam_role_policy_attachment" "ebs_csi" { + policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy" + role = aws_iam_role.ebs_csi.name +} + +resource "aws_eks_pod_identity_association" "ebs_csi" { + cluster_name = aws_eks_cluster.main.name + namespace = "kube-system" + service_account = "ebs-csi-controller-sa" + role_arn = aws_iam_role.ebs_csi.arn +} diff --git a/terraform/modules/eks-cluster/locals.tf b/terraform/modules/eks-cluster/locals.tf index c2c553cc6f..0ae1cf5879 100644 --- a/terraform/modules/eks-cluster/locals.tf +++ b/terraform/modules/eks-cluster/locals.tf @@ -6,4 +6,7 @@ locals { cluster_id = var.cluster_id log_retention_days = 365 + + # OIDC issuer URL without https:// prefix — used as the condition key in IRSA trust policies. + oidc_issuer = trimprefix(aws_eks_cluster.main.identity[0].oidc[0].issuer, "https://") } diff --git a/terraform/modules/eks-cluster/main.tf b/terraform/modules/eks-cluster/main.tf index 33fe541b1d..4a95ad115d 100644 --- a/terraform/modules/eks-cluster/main.tf +++ b/terraform/modules/eks-cluster/main.tf @@ -1,7 +1,7 @@ # ============================================================================= # EKS Cluster Configuration # -# Creates a fully private EKS cluster with Auto Mode enabled. +# Creates a fully private EKS cluster with OSS Karpenter compute. # Includes KMS encryption for secrets, proper networking, # and managed addons for a complete cluster deployment. # VPC and networking are provided as inputs from the vpc module. @@ -111,30 +111,6 @@ resource "aws_eks_cluster" "main" { security_group_ids = [var.cluster_security_group_id] } - compute_config { - enabled = true - node_pools = ["system"] - node_role_arn = aws_iam_role.eks_auto_mode_node.arn - - # TODO: Enable IMDSv2 enforcement for security compliance - # node_pool_defaults configuration for launch template metadata_options - # is not yet supported in AWS provider 6.x for EKS Auto Mode. - # Will be implemented when provider support becomes available. - # See https://github.com/hashicorp/terraform-provider-aws/issues/40486 - } - - kubernetes_network_config { - elastic_load_balancing { - enabled = true - } - } - - storage_config { - block_storage { - enabled = true - } - } - enabled_cluster_log_types = ["api", "audit", "authenticator", "controllerManager", "scheduler"] depends_on = [ @@ -142,6 +118,36 @@ resource "aws_eks_cluster" "main" { aws_cloudwatch_log_group.eks_cluster, aws_kms_key.eks_secrets ] + + # Terminate Karpenter-provisioned EC2 instances before the cluster is deleted. + # Karpenter nodes are not in Terraform state, so they survive EKS deletion and + # block VPC/subnet teardown with DependencyViolation due to lingering ENIs. + # on_failure = continue so a missing AWS CLI or zero instances doesn't abort destroy. + provisioner "local-exec" { + when = destroy + on_failure = continue + command = <<-EOT + CLUSTER_NAME="${self.name}" + REGION=$(echo "${self.arn}" | cut -d: -f4) + echo "Terminating Karpenter EC2 instances for cluster: $CLUSTER_NAME" + INSTANCE_IDS=$(aws ec2 describe-instances \ + --region "$REGION" \ + --filters \ + "Name=tag:aws:eks:cluster-name,Values=$CLUSTER_NAME" \ + "Name=tag-key,Values=karpenter.sh/nodeclaim" \ + "Name=instance-state-name,Values=pending,running,stopping,stopped" \ + --query 'Reservations[].Instances[].InstanceId' \ + --output text) + if [ -z "$INSTANCE_IDS" ]; then + echo "No Karpenter-managed instances found." + exit 0 + fi + echo "Terminating: $INSTANCE_IDS" + aws ec2 terminate-instances --region "$REGION" --instance-ids $INSTANCE_IDS + aws ec2 wait instance-terminated --region "$REGION" --instance-ids $INSTANCE_IDS + echo "Done." + EOT + } } # ----------------------------------------------------------------------------- @@ -163,16 +169,90 @@ resource "aws_eks_cluster" "main" { resource "aws_eks_addon" "coredns" { cluster_name = aws_eks_cluster.main.name addon_name = "coredns" + depends_on = [aws_eks_node_group.karpenter_bootstrap] } resource "aws_eks_addon" "metrics_server" { cluster_name = aws_eks_cluster.main.name addon_name = "metrics-server" + depends_on = [aws_eks_node_group.karpenter_bootstrap] } resource "aws_eks_addon" "pod_identity" { cluster_name = aws_eks_cluster.main.name addon_name = "eks-pod-identity-agent" + depends_on = [aws_eks_node_group.karpenter_bootstrap] +} + +# ----------------------------------------------------------------------------- +# Karpenter Bootstrap Node Group +# +# AL2023 managed node group (t3.medium × 2, CriticalAddonsOnly:NoSchedule) that +# provides fixed capacity for the Karpenter controller and VPC CNI daemonset +# before any Karpenter-provisioned nodes exist. This breaks the bootstrap +# deadlock: Karpenter cannot provision nodes for itself. +# +# No custom launch template: EKS managed node groups set IMDSv2 hop limit to 2 +# by default for AL2023, and managed node group auth is handled automatically +# by EKS regardless of node name format. +# ----------------------------------------------------------------------------- + +resource "aws_eks_node_group" "karpenter_bootstrap" { + cluster_name = aws_eks_cluster.main.name + node_group_name = "${local.cluster_id}-karpenter-bootstrap" + node_role_arn = aws_iam_role.karpenter_node.arn + subnet_ids = var.private_subnet_ids + + ami_type = "AL2023_x86_64_STANDARD" + instance_types = ["t3.medium"] + + scaling_config { + desired_size = 2 + min_size = 2 + max_size = 2 + } + + taint { + key = "CriticalAddonsOnly" + value = "true" + effect = "NO_SCHEDULE" + } + + tags = { + "karpenter.sh/discovery" = aws_eks_cluster.main.name + } + + depends_on = [ + aws_iam_role_policy_attachment.karpenter_node_managed, + aws_eks_addon.vpc_cni, + ] +} + +# ----------------------------------------------------------------------------- +# Explicit Core Addons (Karpenter mode only) +# +# bootstrap_self_managed_addons = false prevents EKS from auto-installing these. +# Auto Mode clusters receive VPC CNI and kube-proxy from the managed control +# plane; Karpenter clusters must declare them explicitly. +# ----------------------------------------------------------------------------- + +resource "aws_eks_addon" "vpc_cni" { + cluster_name = aws_eks_cluster.main.name + addon_name = "vpc-cni" +} + +resource "aws_eks_addon" "kube_proxy" { + cluster_name = aws_eks_cluster.main.name + addon_name = "kube-proxy" + + depends_on = [aws_eks_node_group.karpenter_bootstrap] +} + +resource "aws_eks_addon" "ebs_csi" { + cluster_name = aws_eks_cluster.main.name + addon_name = "aws-ebs-csi-driver" + + depends_on = [aws_eks_node_group.karpenter_bootstrap, aws_eks_pod_identity_association.ebs_csi] } # AWS Secrets Store CSI Driver Provider (e.g. for Maestro agent secret mounting) @@ -187,4 +267,6 @@ resource "aws_eks_addon" "aws_secrets_store_csi_driver_provider" { } } }) + + depends_on = [aws_eks_node_group.karpenter_bootstrap] } diff --git a/terraform/modules/eks-cluster/outputs.tf b/terraform/modules/eks-cluster/outputs.tf index 042e84ffe0..8baa08d494 100644 --- a/terraform/modules/eks-cluster/outputs.tf +++ b/terraform/modules/eks-cluster/outputs.tf @@ -43,7 +43,7 @@ output "vpc_endpoints_security_group_id" { } output "node_security_group_id" { - description = "EKS node security group ID (Auto Mode primary SG - only available after EKS creation)" + description = "EKS cluster security group ID (primary node SG, available after cluster creation)" value = aws_eks_cluster.main.vpc_config[0].cluster_security_group_id } @@ -87,6 +87,21 @@ output "cluster_iam_role_arn" { } output "node_iam_role_arn" { - description = "IAM role ARN of the EKS Auto Mode nodes" - value = aws_iam_role.eks_auto_mode_node.arn + description = "IAM role ARN for Karpenter-provisioned nodes" + value = aws_iam_role.karpenter_node.arn +} + +output "karpenter_controller_role_arn" { + description = "IAM role ARN for the Karpenter controller (IRSA)" + value = aws_iam_role.karpenter_controller.arn +} + +output "karpenter_queue_url" { + description = "SQS queue URL for Karpenter interruption handling" + value = aws_sqs_queue.karpenter_interruption.url +} + +output "karpenter_node_instance_profile_name" { + description = "Instance profile name for Karpenter-provisioned nodes (matches EC2NodeClass.spec.instanceProfile)" + value = aws_iam_instance_profile.karpenter_node.name } diff --git a/terraform/modules/eks-cluster/variables.tf b/terraform/modules/eks-cluster/variables.tf index 974b3cbaa7..3ba74ea9a2 100644 --- a/terraform/modules/eks-cluster/variables.tf +++ b/terraform/modules/eks-cluster/variables.tf @@ -71,3 +71,13 @@ variable "enable_pod_security_standards" { default = true } +# ============================================================================= +# Karpenter configuration +# ============================================================================= + +variable "ami_kms_key_arn" { + description = "ARN of the Red Hat KMS key used to encrypt RHEL FIPS AMI EBS snapshots. When set, IAM policies granting kms:Decrypt and kms:CreateGrant on this key are added to the Karpenter node and controller roles. Leave empty to skip KMS policy creation." + type = string + default = "" +} + diff --git a/terraform/modules/eks-cluster/versions.tf b/terraform/modules/eks-cluster/versions.tf index a88a4501dd..d59c133049 100644 --- a/terraform/modules/eks-cluster/versions.tf +++ b/terraform/modules/eks-cluster/versions.tf @@ -10,5 +10,9 @@ terraform { source = "hashicorp/aws" version = "~> 6.56.0" } + tls = { + source = "hashicorp/tls" + version = ">= 4.0" + } } } \ No newline at end of file diff --git a/terraform/modules/elasticache-valkey/main.tf b/terraform/modules/elasticache-valkey/main.tf index 7669035cfe..bb2aa103a2 100644 --- a/terraform/modules/elasticache-valkey/main.tf +++ b/terraform/modules/elasticache-valkey/main.tf @@ -89,7 +89,7 @@ resource "aws_security_group_rule" "valkey_eks_cluster" { resource "aws_security_group_rule" "valkey_eks_primary" { type = "ingress" - description = "Valkey from EKS cluster primary security group (Auto Mode)" + description = "Valkey from EKS cluster primary security group" from_port = 6379 to_port = 6379 protocol = "tcp" diff --git a/terraform/modules/elasticache-valkey/variables.tf b/terraform/modules/elasticache-valkey/variables.tf index 6b7dbbe71f..e37b1068db 100644 --- a/terraform/modules/elasticache-valkey/variables.tf +++ b/terraform/modules/elasticache-valkey/variables.tf @@ -23,7 +23,7 @@ variable "eks_cluster_security_group_id" { } variable "eks_cluster_primary_security_group_id" { - description = "EKS cluster primary security group ID (Auto Mode ingress to Valkey)" + description = "EKS cluster primary security group ID (ingress to Valkey)" type = string } diff --git a/terraform/modules/rhobs-api-gateway/README.md b/terraform/modules/rhobs-api-gateway/README.md index 63703dcd71..50de4fd8ec 100644 --- a/terraform/modules/rhobs-api-gateway/README.md +++ b/terraform/modules/rhobs-api-gateway/README.md @@ -38,7 +38,7 @@ After Terraform creates the infrastructure, deploy a `TargetGroupBinding` in Kub to register pod IPs with the target group: ```yaml -apiVersion: eks.amazonaws.com/v1 +apiVersion: elbv2.k8s.aws/v1alpha1 kind: TargetGroupBinding metadata: name: thanos-receive diff --git a/terraform/modules/rhobs-api-gateway/alb.tf b/terraform/modules/rhobs-api-gateway/alb.tf index 46ed36f67e..6d2164bf09 100644 --- a/terraform/modules/rhobs-api-gateway/alb.tf +++ b/terraform/modules/rhobs-api-gateway/alb.tf @@ -31,7 +31,7 @@ resource "aws_lb" "rhobs" { # Thanos Receive Target Group # # Receives Prometheus remote_write from Management Clusters via RHOBS API GW. -# Uses IP target type for TargetGroupBinding compatibility with EKS Auto Mode. +# Uses IP target type so LBC TargetGroupBindings register pod IPs directly. # ----------------------------------------------------------------------------- resource "aws_lb_target_group" "thanos_receive" { @@ -108,7 +108,7 @@ resource "aws_lb_listener_rule" "thanos_receive" { # Thanos Query Frontend Target Group # # Serves PromQL queries from E2E tests and internal tooling via RHOBS API GW. -# Uses IP target type for TargetGroupBinding compatibility with EKS Auto Mode. +# Uses IP target type so LBC TargetGroupBindings register pod IPs directly. # ----------------------------------------------------------------------------- resource "aws_lb_target_group" "thanos_query" { @@ -172,7 +172,7 @@ resource "aws_lb_listener_rule" "thanos_rules" { # Loki Distributor Target Group # # Receives log push requests from MC Vector (via sigv4-proxy) and RC Vector. -# Uses IP target type for TargetGroupBinding compatibility with EKS Auto Mode. +# Uses IP target type so LBC TargetGroupBindings register pod IPs directly. # ----------------------------------------------------------------------------- resource "aws_lb_target_group" "loki_distributor" { @@ -220,7 +220,7 @@ resource "aws_lb_listener_rule" "loki_push" { # Loki Query Frontend Target Group # # Serves LogQL queries from E2E tests and internal tooling via RHOBS API GW. -# Uses IP target type for TargetGroupBinding compatibility with EKS Auto Mode. +# Uses IP target type so LBC TargetGroupBindings register pod IPs directly. # ----------------------------------------------------------------------------- resource "aws_lb_target_group" "loki_query_frontend" { diff --git a/terraform/modules/rhobs-api-gateway/variables.tf b/terraform/modules/rhobs-api-gateway/variables.tf index 3eef98bf74..657e8cbe05 100644 --- a/terraform/modules/rhobs-api-gateway/variables.tf +++ b/terraform/modules/rhobs-api-gateway/variables.tf @@ -28,7 +28,7 @@ variable "node_security_group_id" { } variable "cluster_name" { - description = "EKS cluster name - required for tagging target group with eks:eks-cluster-name tag for Auto Mode IAM permissions" + description = "EKS cluster name - used to tag target groups with eks:eks-cluster-name" type = string } diff --git a/terraform/modules/sre-ui-alb/alb.tf b/terraform/modules/sre-ui-alb/alb.tf index 87490b54de..19427b3312 100644 --- a/terraform/modules/sre-ui-alb/alb.tf +++ b/terraform/modules/sre-ui-alb/alb.tf @@ -98,7 +98,7 @@ resource "aws_lb" "sre" { # ----------------------------------------------------------------------------- # Target Groups -# All use IP target type for TargetGroupBinding compatibility with EKS Auto Mode. +# All use IP target type so LBC TargetGroupBindings register pod IPs directly. # ----------------------------------------------------------------------------- resource "aws_lb_target_group" "services" { diff --git a/terraform/modules/sre-ui-alb/variables.tf b/terraform/modules/sre-ui-alb/variables.tf index 15b5170399..5f768a61a5 100644 --- a/terraform/modules/sre-ui-alb/variables.tf +++ b/terraform/modules/sre-ui-alb/variables.tf @@ -33,12 +33,12 @@ variable "regional_id" { } variable "node_security_group_id" { - description = "EKS node/pod security group ID. For EKS Auto Mode, use cluster_primary_security_group_id." + description = "EKS node/pod security group ID" type = string } variable "cluster_name" { - description = "EKS cluster name — required for eks:eks-cluster-name tag (EKS Auto Mode IAM)" + description = "EKS cluster name — used to tag target groups with eks:eks-cluster-name" type = string }