Remove/eks/managed nodes/inotify max user watches

_sub/compute/eks-addons/main.tf

@@ -4,7 +4,7 @@ resource "aws_eks_addon" "vpc-cni" {
   addon_version = local.vpccni_version
   configuration_values = jsonencode({
     "env" = {
-      "ENABLE_PREFIX_DELEGATION" = tostring(var.vpccni_prefix_delegation_enabled)
+      "ENABLE_PREFIX_DELEGATION" = tostring(true)
     }
   })
   resolve_conflicts_on_create = "OVERWRITE"

_sub/compute/eks-addons/vars.tf

@@ -35,12 +35,6 @@ variable "awsefscsidriver_version_override" {
   default = ""
 }
 
-variable "vpccni_prefix_delegation_enabled" {
-  type        = bool
-  description = "Whether to enable prefix delegation mode on the VPC CNI addon."
-  default     = false
-}
-
 variable "most_recent" {
   type        = bool
   default     = false

_sub/compute/eks-nodegroup-managed/dependencies.tf

@@ -18,4 +18,30 @@ data "aws_ami" "eks-node" {
 locals {
   # Pins AMI to 'ami_id' if it is set, otherwise, sets to the latest AMI.
   node_ami = var.ami_id != "" ? var.ami_id : data.aws_ami.eks-node.id
-}
+}
+
+locals {
+  max_pods = 110
+}
+
+data "aws_ec2_instance_type" "this" {
+  for_each = toset(var.instance_types)
+  instance_type = each.value
+}
+
+locals {
+  max_pods_calc = {
+    for instance_type, instance_data in data.aws_ec2_instance_type.this : instance_type => (
+      (instance_data.maximum_network_interfaces * ((instance_data.maximum_ipv4_addresses_per_interface - 1) * 16)) + 2
+    )
+  }
+}
+
+check "instance_types_support_max_pods" {
+ assert {
+    condition     = alltrue([for instance_type, instance_data in local.max_pods_calc : (
+      instance_data >= local.max_pods
+    )])
+    error_message = "One or more instance types is too small and does not support the required max pods of ${local.max_pods}."
+  }
+}

_sub/compute/eks-nodegroup-managed/main.tf

@@ -3,23 +3,18 @@ resource "aws_launch_template" "eks" {
   count = signum(var.desired_size_per_subnet)
 
   image_id      = local.node_ami
-  instance_type = element(var.instance_types, 0)
+  instance_type = null # because instance_type is a list set on node groups
   name_prefix   = "eks-${var.cluster_name}-${var.nodegroup_name}-"
   # Make sure to update the max pod values in the template below using the script
   # `src/produce-eni-max-pods.sh` when updating the EKS VPC CNI addon.
   user_data = base64encode(templatefile("${path.module}/user-data.sh.tftpl", {
     eks_endpoint : var.eks_endpoint,
     eks_certificate_authority : var.eks_certificate_authority,
     cluster_name : var.cluster_name,
-    worker_inotify_max_user_watches : var.worker_inotify_max_user_watches,
-    vpc_cni_prefix_delegation_enabled : var.vpc_cni_prefix_delegation_enabled,
     cidr : var.eks_service_cidr,
-    max_pods : var.max_pods,
-    kube_cpu : var.kube_reserved_cpu,
-    kube_memory : var.kube_reserved_memory,
-    sys_cpu : var.system_reserved_cpu,
-    sys_memory : var.system_reserved_memory,
+    max_pods : local.max_pods,
     docker_hub_creds : var.docker_hub_creds_ssm_path,
+    kubelet_memory_reservation : "${tostring(ceil(255 + (11 * local.max_pods)))}Mi"
   }))
   key_name               = var.ec2_ssh_key
   update_default_version = true
@@ -54,6 +49,7 @@ resource "aws_eks_node_group" "group" {
   node_group_name = var.nodegroup_name
   node_role_arn   = var.node_role_arn
   subnet_ids      = var.subnet_ids
+  instance_types  = var.instance_types
   capacity_type   = var.use_spot_instances ? "SPOT" : "ON_DEMAND"
 
   dynamic "taint" {

_sub/compute/eks-nodegroup-managed/user-data.sh.tftpl

@@ -7,9 +7,6 @@ Content-Type: text/x-shellscript; charset="us-ascii"
 #!/bin/sh
 set -o xtrace
 
-echo fs.inotify.max_user_watches=${worker_inotify_max_user_watches} | sudo tee -a /etc/sysctl.conf
-sudo sysctl -p
-
 DOCKER_HUB_CREDS=$(aws ssm get-parameter --name ${docker_hub_creds} --with-decrypt --query 'Parameter.Value' --output text)
 DOCKER_HUB_USER=$(echo "$${DOCKER_HUB_CREDS}" | jq -r .username)
 DOCKER_HUB_PASSWD=$(echo "$${DOCKER_HUB_CREDS}" | jq -r .password)
@@ -29,34 +26,18 @@ Content-Type: application/node.eks.aws
 apiVersion: node.eks.aws/v1alpha1
 kind: NodeConfig
 spec:
-    cluster:
-        name: ${cluster_name}
-        apiServerEndpoint: ${eks_endpoint}
-        certificateAuthority: ${eks_certificate_authority}
-        cidr: ${cidr}
-    kubelet:
-        config:
-%{ if vpc_cni_prefix_delegation_enabled }
-            maxPods: ${max_pods}
-%{ endif ~}
-            registryPullQPS: 0
-%{ if kube_cpu != null || kube_memory != null }
-            kubeReserved:
-%{ endif ~}
-%{ if kube_cpu != null }
-                cpu: ${kube_cpu}
-%{ endif ~}
-%{ if kube_memory != null }
-                memory: ${kube_memory}
-%{ endif ~}
-%{ if sys_cpu != null || sys_memory != null }
-            systemReserved:
-%{ endif ~}
-%{ if sys_cpu != null }
-                cpu: ${sys_cpu}
-%{ endif ~}
-%{ if sys_memory != null }
-                memory: ${sys_memory}
-%{ endif ~}
+  cluster:
+    name: ${cluster_name}
+    apiServerEndpoint: ${eks_endpoint}
+    certificateAuthority: ${eks_certificate_authority}
+    cidr: ${cidr}
+  kubelet:
+    config:
+      maxPods: ${max_pods}
+      registryPullQPS: 0
+      kubeReserved:
+        memory: ${kubelet_memory_reservation}
+      systemReserved:
+        memory: 200Mi
 
 --BOUNDARY--

_sub/compute/eks-nodegroup-managed/vars.tf

@@ -79,10 +79,6 @@ variable "eks_service_cidr" {
   description = "The CIDR block IPv4 used by the cluster to assign Kubernetes service IP addresses. This is derived from the cluster itself."
 }
 
-variable "worker_inotify_max_user_watches" {
-  type = number
-}
-
 variable "docker_hub_creds_ssm_path" {
   type        = string
   description = "Docker Hub credentials SSM Parameter Store path"
@@ -103,12 +99,6 @@ variable "scale_to_zero_cron" {
   description = "The time when the ASG will be scaled to zero, specified in Unix cron syntax"
 }
 
-variable "vpc_cni_prefix_delegation_enabled" {
-  type        = bool
-  description = "Configures the maximum pods limit on the nodes assuming that the prefix delegation feature is enabled on the VPC CNI addon."
-  default     = false
-}
-
 variable "max_unavailable" {
   type        = number
   description = "Desired max number of unavailable worker nodes during node group update."
@@ -119,34 +109,4 @@ variable "max_unavailable_percentage" {
   type        = number
   description = "Desired max percentage of unavailable worker nodes during node group update."
   default     = null
-}
-
-variable "max_pods" {
-  type        = number
-  description = "Maximum number of Pods that can run on the Kubelet"
-  default     = 110
-}
-
-variable "kube_reserved_cpu" {
-  type        = string
-  description = "CPU reserved for kubernetes system components"
-  default     = null
-}
-
-variable "kube_reserved_memory" {
-  type        = string
-  description = "Memory reserved for kubernetes system components"
-  default     = null
-}
-
-variable "system_reserved_cpu" {
-  type        = string
-  description = "CPU reserved for system components"
-  default     = null
-}
-
-variable "system_reserved_memory" {
-  type        = string
-  description = "Memory reserved for system components"
-  default     = null
-}
+}

compute/eks-ec2/main.tf

@@ -254,8 +254,6 @@ module "eks_managed_workers_node_group" {
   eks_endpoint                              = module.eks_cluster.eks_endpoint
   eks_certificate_authority                 = module.eks_cluster.eks_certificate_authority
   eks_service_cidr                          = module.eks_cluster.eks_service_cidr
-  vpc_cni_prefix_delegation_enabled         = var.eks_addon_vpccni_prefix_delegation_enabled
-  worker_inotify_max_user_watches           = var.eks_worker_inotify_max_user_watches
 
   # Node group variations
   nodegroup_name             = each.key
@@ -271,11 +269,6 @@ module "eks_managed_workers_node_group" {
   subnet_ids = length(each.value.availability_zones) == 0 ? module.eks_managed_workers_subnet.subnet_ids : [
     for sn in module.eks_managed_workers_subnet.subnets : sn.id if contains(each.value.availability_zones, sn.availability_zone)
   ]
-  max_pods               = each.value.max_pods
-  kube_reserved_cpu      = each.value.kube_cpu
-  kube_reserved_memory   = each.value.kube_memory
-  system_reserved_cpu    = each.value.sys_cpu
-  system_reserved_memory = each.value.sys_memory
 
   # Docker Hub credentials
   docker_hub_creds_ssm_path = aws_ssm_parameter.dockerhub.name
@@ -323,7 +316,6 @@ module "eks_addons" {
   kubeproxy_version_override       = var.eks_addon_kubeproxy_version_override
   coredns_version_override         = var.eks_addon_coredns_version_override
   vpccni_version_override          = var.eks_addon_vpccni_version_override
-  vpccni_prefix_delegation_enabled = var.eks_addon_vpccni_prefix_delegation_enabled
   awsebscsidriver_version_override = var.eks_addon_awsebscsidriver_version_override
   awsefscsidriver_version_override = var.eks_addon_awsefscsidriver_version_override
   podidentity_version_override     = var.eks_addon_podidentity_version_override

compute/eks-ec2/vars.tf

@@ -99,11 +99,6 @@ variable "eks_cluster_log_retention_days" {
   default     = 90
 }
 
-variable "eks_worker_inotify_max_user_watches" {
-  type    = number
-  default = 131072 # default t3.large is 8192 which is too low
-}
-
 variable "eks_managed_worker_subnets" {
   type = list(object({
     availability_zone         = string,
@@ -134,12 +129,6 @@ variable "eks_addon_vpccni_version_override" {
   default = ""
 }
 
-variable "eks_addon_vpccni_prefix_delegation_enabled" {
-  type        = bool
-  description = "Whether to enable the prefix delegation mode on the VPC CNI EKS addon."
-  default     = false
-}
-
 variable "eks_addon_awsebscsidriver_version_override" {
   type    = string
   default = ""
@@ -199,11 +188,6 @@ variable "eks_managed_nodegroups" {
       effect = string
     })), [])
     labels      = optional(map(string), {})
-    max_pods    = optional(number, 110)
-    sys_cpu     = optional(string, null)
-    sys_memory  = optional(string, null)
-    kube_cpu    = optional(string, null)
-    kube_memory = optional(string, null)
   }))
   default = {}
   validation {

test/integration/eu-west-1/k8s-qa/cluster/terragrunt.hcl

@@ -26,7 +26,6 @@ inputs = {
   eks_cluster_zones                          = 2
   eks_cluster_log_types                      = ["api", "authenticator", "scheduler", "controllerManager"]
 
-  eks_addon_vpccni_prefix_delegation_enabled = true
   eks_addon_most_recent                      = false
   # renovate: eksAddonsFilter={"kubernetesVersion":"1.34","addonName":"vpc-cni"}
   eks_addon_vpccni_version_override = "v1.22.1-eksbuild.2"
@@ -68,10 +67,6 @@ inputs = {
       ami_id             = "ami-0275aedb4db249683"
       availability_zones         = ["eu-west-1a", "eu-west-1b", "eu-west-1c"]
       max_unavailable_percentage = 50
-      kube_memory                = "1024Mi"
-      kube_cpu                   = "500m"
-      sys_memory                 = "768Mi"
-      sys_cpu                    = "300m"
       labels = {
         "karpenter.sh/controller" = "true" # required for Karpenter
       }
@@ -94,10 +89,6 @@ inputs = {
       labels = {
         dedicated = "observability"
       }
-      kube_memory   = "585Mi"
-      kube_cpu      = "90m"
-      sys_memory    = "585Mi"
-      sys_cpu       = "90m"
     }
   }
 

test/integration/suite/vpc_cni_driver_test.go

@@ -1,9 +1,29 @@
 package main
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/gruntwork-io/terratest/modules/k8s"
+	"github.com/stretchr/testify/assert"
+)
 
 func TestVpcCniDriverDaemonSet(t *testing.T) {
 	t.Parallel()
 	clientset := NewK8sClientSet(t)
 	AssertK8sDaemonSet(t, clientset, "kube-system", "aws-node")
 }
+
+func TestVpcCniPrefixDelegationEnabled(t *testing.T) {
+	t.Parallel()
+	options := k8s.NewKubectlOptions("", "", "kube-system")
+	ds := k8s.GetDaemonSet(t, options, "aws-node")
+
+	prefixDeletegationEnabled := false
+	for _, env := range ds.Spec.Template.Spec.Containers[0].Env {
+		if env.Name == "ENABLE_PREFIX_DELEGATION" {
+			assert.Equal(t, "true", env.Value, "Expected ENABLE_PREFIX_DELEGATION to be set to 'true'")
+			prefixDeletegationEnabled = true
+		}
+	}
+	assert.True(t, prefixDeletegationEnabled, "ENABLE_PREFIX_DELEGATION environment variable not found")
+}