diff --git a/_sub/compute/eks-addons/main.tf b/_sub/compute/eks-addons/main.tf index 9e578b4c0..32b620165 100644 --- a/_sub/compute/eks-addons/main.tf +++ b/_sub/compute/eks-addons/main.tf @@ -4,7 +4,7 @@ resource "aws_eks_addon" "vpc-cni" { addon_version = local.vpccni_version configuration_values = jsonencode({ "env" = { - "ENABLE_PREFIX_DELEGATION" = tostring(var.vpccni_prefix_delegation_enabled) + "ENABLE_PREFIX_DELEGATION" = tostring(true) } }) resolve_conflicts_on_create = "OVERWRITE" diff --git a/_sub/compute/eks-addons/vars.tf b/_sub/compute/eks-addons/vars.tf index 3177cc6ce..9369e13fa 100644 --- a/_sub/compute/eks-addons/vars.tf +++ b/_sub/compute/eks-addons/vars.tf @@ -35,12 +35,6 @@ variable "awsefscsidriver_version_override" { default = "" } -variable "vpccni_prefix_delegation_enabled" { - type = bool - description = "Whether to enable prefix delegation mode on the VPC CNI addon." - default = false -} - variable "most_recent" { type = bool default = false diff --git a/_sub/compute/eks-nodegroup-managed/dependencies.tf b/_sub/compute/eks-nodegroup-managed/dependencies.tf index ed2999a5e..65384e147 100644 --- a/_sub/compute/eks-nodegroup-managed/dependencies.tf +++ b/_sub/compute/eks-nodegroup-managed/dependencies.tf @@ -18,4 +18,30 @@ data "aws_ami" "eks-node" { locals { # Pins AMI to 'ami_id' if it is set, otherwise, sets to the latest AMI. node_ami = var.ami_id != "" ? var.ami_id : data.aws_ami.eks-node.id -} \ No newline at end of file +} + +locals { + max_pods = 110 +} + +data "aws_ec2_instance_type" "this" { + for_each = toset(var.instance_types) + instance_type = each.value +} + +locals { + max_pods_calc = { + for instance_type, instance_data in data.aws_ec2_instance_type.this : instance_type => ( + (instance_data.maximum_network_interfaces * ((instance_data.maximum_ipv4_addresses_per_interface - 1) * 16)) + 2 + ) + } +} + +check "instance_types_support_max_pods" { + assert { + condition = alltrue([for instance_type, instance_data in local.max_pods_calc : ( + instance_data >= local.max_pods + )]) + error_message = "One or more instance types is too small and does not support the required max pods of ${local.max_pods}." + } +} diff --git a/_sub/compute/eks-nodegroup-managed/main.tf b/_sub/compute/eks-nodegroup-managed/main.tf index 782705202..819929e1a 100644 --- a/_sub/compute/eks-nodegroup-managed/main.tf +++ b/_sub/compute/eks-nodegroup-managed/main.tf @@ -3,7 +3,7 @@ resource "aws_launch_template" "eks" { count = signum(var.desired_size_per_subnet) image_id = local.node_ami - instance_type = element(var.instance_types, 0) + instance_type = null # because instance_type is a list set on node groups name_prefix = "eks-${var.cluster_name}-${var.nodegroup_name}-" # Make sure to update the max pod values in the template below using the script # `src/produce-eni-max-pods.sh` when updating the EKS VPC CNI addon. @@ -11,15 +11,10 @@ resource "aws_launch_template" "eks" { eks_endpoint : var.eks_endpoint, eks_certificate_authority : var.eks_certificate_authority, cluster_name : var.cluster_name, - worker_inotify_max_user_watches : var.worker_inotify_max_user_watches, - vpc_cni_prefix_delegation_enabled : var.vpc_cni_prefix_delegation_enabled, cidr : var.eks_service_cidr, - max_pods : var.max_pods, - kube_cpu : var.kube_reserved_cpu, - kube_memory : var.kube_reserved_memory, - sys_cpu : var.system_reserved_cpu, - sys_memory : var.system_reserved_memory, + max_pods : local.max_pods, docker_hub_creds : var.docker_hub_creds_ssm_path, + kubelet_memory_reservation : "${tostring(ceil(255 + (11 * local.max_pods)))}Mi" })) key_name = var.ec2_ssh_key update_default_version = true @@ -54,6 +49,7 @@ resource "aws_eks_node_group" "group" { node_group_name = var.nodegroup_name node_role_arn = var.node_role_arn subnet_ids = var.subnet_ids + instance_types = var.instance_types capacity_type = var.use_spot_instances ? "SPOT" : "ON_DEMAND" dynamic "taint" { diff --git a/_sub/compute/eks-nodegroup-managed/user-data.sh.tftpl b/_sub/compute/eks-nodegroup-managed/user-data.sh.tftpl index 0e9a74a80..c0fe46d4b 100644 --- a/_sub/compute/eks-nodegroup-managed/user-data.sh.tftpl +++ b/_sub/compute/eks-nodegroup-managed/user-data.sh.tftpl @@ -7,9 +7,6 @@ Content-Type: text/x-shellscript; charset="us-ascii" #!/bin/sh set -o xtrace -echo fs.inotify.max_user_watches=${worker_inotify_max_user_watches} | sudo tee -a /etc/sysctl.conf -sudo sysctl -p - DOCKER_HUB_CREDS=$(aws ssm get-parameter --name ${docker_hub_creds} --with-decrypt --query 'Parameter.Value' --output text) DOCKER_HUB_USER=$(echo "$${DOCKER_HUB_CREDS}" | jq -r .username) DOCKER_HUB_PASSWD=$(echo "$${DOCKER_HUB_CREDS}" | jq -r .password) @@ -29,34 +26,18 @@ Content-Type: application/node.eks.aws apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig spec: - cluster: - name: ${cluster_name} - apiServerEndpoint: ${eks_endpoint} - certificateAuthority: ${eks_certificate_authority} - cidr: ${cidr} - kubelet: - config: -%{ if vpc_cni_prefix_delegation_enabled } - maxPods: ${max_pods} -%{ endif ~} - registryPullQPS: 0 -%{ if kube_cpu != null || kube_memory != null } - kubeReserved: -%{ endif ~} -%{ if kube_cpu != null } - cpu: ${kube_cpu} -%{ endif ~} -%{ if kube_memory != null } - memory: ${kube_memory} -%{ endif ~} -%{ if sys_cpu != null || sys_memory != null } - systemReserved: -%{ endif ~} -%{ if sys_cpu != null } - cpu: ${sys_cpu} -%{ endif ~} -%{ if sys_memory != null } - memory: ${sys_memory} -%{ endif ~} + cluster: + name: ${cluster_name} + apiServerEndpoint: ${eks_endpoint} + certificateAuthority: ${eks_certificate_authority} + cidr: ${cidr} + kubelet: + config: + maxPods: ${max_pods} + registryPullQPS: 0 + kubeReserved: + memory: ${kubelet_memory_reservation} + systemReserved: + memory: 200Mi --BOUNDARY-- diff --git a/_sub/compute/eks-nodegroup-managed/vars.tf b/_sub/compute/eks-nodegroup-managed/vars.tf index ee14e25b1..0cff5856f 100644 --- a/_sub/compute/eks-nodegroup-managed/vars.tf +++ b/_sub/compute/eks-nodegroup-managed/vars.tf @@ -79,10 +79,6 @@ variable "eks_service_cidr" { description = "The CIDR block IPv4 used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself." } -variable "worker_inotify_max_user_watches" { - type = number -} - variable "docker_hub_creds_ssm_path" { type = string description = "Docker Hub credentials SSM Parameter Store path" @@ -103,12 +99,6 @@ variable "scale_to_zero_cron" { description = "The time when the ASG will be scaled to zero, specified in Unix cron syntax" } -variable "vpc_cni_prefix_delegation_enabled" { - type = bool - description = "Configures the maximum pods limit on the nodes assuming that the prefix delegation feature is enabled on the VPC CNI addon." - default = false -} - variable "max_unavailable" { type = number description = "Desired max number of unavailable worker nodes during node group update." @@ -119,34 +109,4 @@ variable "max_unavailable_percentage" { type = number description = "Desired max percentage of unavailable worker nodes during node group update." default = null -} - -variable "max_pods" { - type = number - description = "Maximum number of Pods that can run on the Kubelet" - default = 110 -} - -variable "kube_reserved_cpu" { - type = string - description = "CPU reserved for kubernetes system components" - default = null -} - -variable "kube_reserved_memory" { - type = string - description = "Memory reserved for kubernetes system components" - default = null -} - -variable "system_reserved_cpu" { - type = string - description = "CPU reserved for system components" - default = null -} - -variable "system_reserved_memory" { - type = string - description = "Memory reserved for system components" - default = null -} +} \ No newline at end of file diff --git a/compute/eks-ec2/main.tf b/compute/eks-ec2/main.tf index d46179563..6eefac7b6 100644 --- a/compute/eks-ec2/main.tf +++ b/compute/eks-ec2/main.tf @@ -254,8 +254,6 @@ module "eks_managed_workers_node_group" { eks_endpoint = module.eks_cluster.eks_endpoint eks_certificate_authority = module.eks_cluster.eks_certificate_authority eks_service_cidr = module.eks_cluster.eks_service_cidr - vpc_cni_prefix_delegation_enabled = var.eks_addon_vpccni_prefix_delegation_enabled - worker_inotify_max_user_watches = var.eks_worker_inotify_max_user_watches # Node group variations nodegroup_name = each.key @@ -271,11 +269,6 @@ module "eks_managed_workers_node_group" { subnet_ids = length(each.value.availability_zones) == 0 ? module.eks_managed_workers_subnet.subnet_ids : [ for sn in module.eks_managed_workers_subnet.subnets : sn.id if contains(each.value.availability_zones, sn.availability_zone) ] - max_pods = each.value.max_pods - kube_reserved_cpu = each.value.kube_cpu - kube_reserved_memory = each.value.kube_memory - system_reserved_cpu = each.value.sys_cpu - system_reserved_memory = each.value.sys_memory # Docker Hub credentials docker_hub_creds_ssm_path = aws_ssm_parameter.dockerhub.name @@ -323,7 +316,6 @@ module "eks_addons" { kubeproxy_version_override = var.eks_addon_kubeproxy_version_override coredns_version_override = var.eks_addon_coredns_version_override vpccni_version_override = var.eks_addon_vpccni_version_override - vpccni_prefix_delegation_enabled = var.eks_addon_vpccni_prefix_delegation_enabled awsebscsidriver_version_override = var.eks_addon_awsebscsidriver_version_override awsefscsidriver_version_override = var.eks_addon_awsefscsidriver_version_override podidentity_version_override = var.eks_addon_podidentity_version_override diff --git a/compute/eks-ec2/vars.tf b/compute/eks-ec2/vars.tf index f54998d02..a039edef3 100644 --- a/compute/eks-ec2/vars.tf +++ b/compute/eks-ec2/vars.tf @@ -99,11 +99,6 @@ variable "eks_cluster_log_retention_days" { default = 90 } -variable "eks_worker_inotify_max_user_watches" { - type = number - default = 131072 # default t3.large is 8192 which is too low -} - variable "eks_managed_worker_subnets" { type = list(object({ availability_zone = string, @@ -134,12 +129,6 @@ variable "eks_addon_vpccni_version_override" { default = "" } -variable "eks_addon_vpccni_prefix_delegation_enabled" { - type = bool - description = "Whether to enable the prefix delegation mode on the VPC CNI EKS addon." - default = false -} - variable "eks_addon_awsebscsidriver_version_override" { type = string default = "" @@ -199,11 +188,6 @@ variable "eks_managed_nodegroups" { effect = string })), []) labels = optional(map(string), {}) - max_pods = optional(number, 110) - sys_cpu = optional(string, null) - sys_memory = optional(string, null) - kube_cpu = optional(string, null) - kube_memory = optional(string, null) })) default = {} validation { diff --git a/test/integration/eu-west-1/k8s-qa/cluster/terragrunt.hcl b/test/integration/eu-west-1/k8s-qa/cluster/terragrunt.hcl index 455b19eb0..ba008c9b8 100644 --- a/test/integration/eu-west-1/k8s-qa/cluster/terragrunt.hcl +++ b/test/integration/eu-west-1/k8s-qa/cluster/terragrunt.hcl @@ -26,7 +26,6 @@ inputs = { eks_cluster_zones = 2 eks_cluster_log_types = ["api", "authenticator", "scheduler", "controllerManager"] - eks_addon_vpccni_prefix_delegation_enabled = true eks_addon_most_recent = false # renovate: eksAddonsFilter={"kubernetesVersion":"1.34","addonName":"vpc-cni"} eks_addon_vpccni_version_override = "v1.22.1-eksbuild.2" @@ -68,10 +67,6 @@ inputs = { ami_id = "ami-0275aedb4db249683" availability_zones = ["eu-west-1a", "eu-west-1b", "eu-west-1c"] max_unavailable_percentage = 50 - kube_memory = "1024Mi" - kube_cpu = "500m" - sys_memory = "768Mi" - sys_cpu = "300m" labels = { "karpenter.sh/controller" = "true" # required for Karpenter } @@ -94,10 +89,6 @@ inputs = { labels = { dedicated = "observability" } - kube_memory = "585Mi" - kube_cpu = "90m" - sys_memory = "585Mi" - sys_cpu = "90m" } } diff --git a/test/integration/suite/vpc_cni_driver_test.go b/test/integration/suite/vpc_cni_driver_test.go index 74c79efdf..1a9cbb807 100644 --- a/test/integration/suite/vpc_cni_driver_test.go +++ b/test/integration/suite/vpc_cni_driver_test.go @@ -1,9 +1,29 @@ package main -import "testing" +import ( + "testing" + + "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/stretchr/testify/assert" +) func TestVpcCniDriverDaemonSet(t *testing.T) { t.Parallel() clientset := NewK8sClientSet(t) AssertK8sDaemonSet(t, clientset, "kube-system", "aws-node") } + +func TestVpcCniPrefixDelegationEnabled(t *testing.T) { + t.Parallel() + options := k8s.NewKubectlOptions("", "", "kube-system") + ds := k8s.GetDaemonSet(t, options, "aws-node") + + prefixDeletegationEnabled := false + for _, env := range ds.Spec.Template.Spec.Containers[0].Env { + if env.Name == "ENABLE_PREFIX_DELEGATION" { + assert.Equal(t, "true", env.Value, "Expected ENABLE_PREFIX_DELEGATION to be set to 'true'") + prefixDeletegationEnabled = true + } + } + assert.True(t, prefixDeletegationEnabled, "ENABLE_PREFIX_DELEGATION environment variable not found") +}