Refactor/eks/managed nodes/system reserved and kube reserved

_sub/compute/eks-nodegroup-managed/dependencies.tf

@@ -18,4 +18,30 @@ data "aws_ami" "eks-node" {
 locals {
   # Pins AMI to 'ami_id' if it is set, otherwise, sets to the latest AMI.
   node_ami = var.ami_id != "" ? var.ami_id : data.aws_ami.eks-node.id
-}
+}
+
+locals {
+  max_pods = 110
+}
+
+data "aws_ec2_instance_type" "this" {
+  for_each = toset(var.instance_types)
+  instance_type = each.value
+}
+
+locals {
+  max_pods_calc = {
+    for instance_type, instance_data in data.aws_ec2_instance_type.this : instance_type => (
+      (instance_data.maximum_network_interfaces * ((instance_data.maximum_ipv4_addresses_per_interface - 1) * 16)) + 2
+    )
+  }
+}
+
+check "instance_types_support_max_pods" {
+ assert {
+    condition     = alltrue([for instance_type, instance_data in local.max_pods_calc : (
+      instance_data >= local.max_pods
+    )])
+    error_message = "One or more instance types is too small and does not support the required max pods of ${local.max_pods}."
+  }
+}

_sub/compute/eks-nodegroup-managed/main.tf

@@ -3,7 +3,7 @@ resource "aws_launch_template" "eks" {
   count = signum(var.desired_size_per_subnet)
 
   image_id      = local.node_ami
-  instance_type = element(var.instance_types, 0)
+  instance_type = null # because instance_type is a list set on node groups
   name_prefix   = "eks-${var.cluster_name}-${var.nodegroup_name}-"
   # Make sure to update the max pod values in the template below using the script
   # `src/produce-eni-max-pods.sh` when updating the EKS VPC CNI addon.
@@ -13,12 +13,9 @@ resource "aws_launch_template" "eks" {
     cluster_name : var.cluster_name,
     worker_inotify_max_user_watches : var.worker_inotify_max_user_watches,
     cidr : var.eks_service_cidr,
-    max_pods : var.max_pods,
-    kube_cpu : var.kube_reserved_cpu,
-    kube_memory : var.kube_reserved_memory,
-    sys_cpu : var.system_reserved_cpu,
-    sys_memory : var.system_reserved_memory,
+    max_pods : local.max_pods,
     docker_hub_creds : var.docker_hub_creds_ssm_path,
+    kubelet_memory_reservation : "${tostring(ceil(255 + (11 * local.max_pods)))}Mi"
   }))
   key_name               = var.ec2_ssh_key
   update_default_version = true
@@ -45,14 +42,20 @@ resource "aws_launch_template" "eks" {
   monitoring {
     enabled = true
   }
+
+  lifecycle {
+    create_before_destroy = true
+  }
 }
 
 resource "aws_eks_node_group" "group" {
   count           = signum(var.desired_size_per_subnet)
   cluster_name    = var.cluster_name
-  node_group_name = var.nodegroup_name
+  node_group_name = null # conflicts with node_group_name_prefix
+  node_group_name_prefix = "${var.nodegroup_name}-" # needed for create before destroy lifecycle
   node_role_arn   = var.node_role_arn
   subnet_ids      = var.subnet_ids
+  instance_types  = var.instance_types
   capacity_type   = var.use_spot_instances ? "SPOT" : "ON_DEMAND"
 
   dynamic "taint" {
@@ -90,6 +93,10 @@ resource "aws_eks_node_group" "group" {
     max_size     = local.asg_max_size
     min_size     = local.asg_min_size
   }
+
+  lifecycle {
+    create_before_destroy = true
+  }
 }
 
 resource "aws_autoscaling_schedule" "eks" {

_sub/compute/eks-nodegroup-managed/user-data.sh.tftpl

@@ -29,32 +29,18 @@ Content-Type: application/node.eks.aws
 apiVersion: node.eks.aws/v1alpha1
 kind: NodeConfig
 spec:
-    cluster:
-        name: ${cluster_name}
-        apiServerEndpoint: ${eks_endpoint}
-        certificateAuthority: ${eks_certificate_authority}
-        cidr: ${cidr}
-    kubelet:
-        config:
-            maxPods: ${max_pods}
-            registryPullQPS: 0
-%{ if kube_cpu != null || kube_memory != null }
-            kubeReserved:
-%{ endif ~}
-%{ if kube_cpu != null ~}
-                cpu: ${kube_cpu}
-%{ endif ~}
-%{ if kube_memory != null ~}
-                memory: ${kube_memory}
-%{ endif ~}
-%{ if sys_cpu != null || sys_memory != null ~}
-            systemReserved:
-%{ endif ~}
-%{ if sys_cpu != null ~}
-                cpu: ${sys_cpu}
-%{ endif ~}
-%{ if sys_memory != null ~}
-                memory: ${sys_memory}
-%{ endif ~}
+  cluster:
+    name: ${cluster_name}
+    apiServerEndpoint: ${eks_endpoint}
+    certificateAuthority: ${eks_certificate_authority}
+    cidr: ${cidr}
+  kubelet:
+    config:
+      maxPods: ${max_pods}
+      registryPullQPS: 0
+      kubeReserved:
+        memory: ${kubelet_memory_reservation}
+      systemReserved:
+        memory: 200Mi
 
 --BOUNDARY--

_sub/compute/eks-nodegroup-managed/vars.tf

@@ -113,34 +113,4 @@ variable "max_unavailable_percentage" {
   type        = number
   description = "Desired max percentage of unavailable worker nodes during node group update."
   default     = null
-}
-
-variable "max_pods" {
-  type        = number
-  description = "Maximum number of Pods that can run on the Kubelet"
-  default     = 110
-}
-
-variable "kube_reserved_cpu" {
-  type        = string
-  description = "CPU reserved for kubernetes system components"
-  default     = null
-}
-
-variable "kube_reserved_memory" {
-  type        = string
-  description = "Memory reserved for kubernetes system components"
-  default     = null
-}
-
-variable "system_reserved_cpu" {
-  type        = string
-  description = "CPU reserved for system components"
-  default     = null
-}
-
-variable "system_reserved_memory" {
-  type        = string
-  description = "Memory reserved for system components"
-  default     = null
-}
+}

compute/eks-ec2/main.tf

@@ -270,11 +270,6 @@ module "eks_managed_workers_node_group" {
   subnet_ids = length(each.value.availability_zones) == 0 ? module.eks_managed_workers_subnet.subnet_ids : [
     for sn in module.eks_managed_workers_subnet.subnets : sn.id if contains(each.value.availability_zones, sn.availability_zone)
   ]
-  max_pods               = each.value.max_pods
-  kube_reserved_cpu      = each.value.kube_cpu
-  kube_reserved_memory   = each.value.kube_memory
-  system_reserved_cpu    = each.value.sys_cpu
-  system_reserved_memory = each.value.sys_memory
 
   # Docker Hub credentials
   docker_hub_creds_ssm_path = aws_ssm_parameter.dockerhub.name

compute/eks-ec2/vars.tf

@@ -193,11 +193,6 @@ variable "eks_managed_nodegroups" {
       effect = string
     })), [])
     labels      = optional(map(string), {})
-    max_pods    = optional(number, 110)
-    sys_cpu     = optional(string, null)
-    sys_memory  = optional(string, null)
-    kube_cpu    = optional(string, null)
-    kube_memory = optional(string, null)
   }))
   default = {}
   validation {

test/integration/eu-west-1/k8s-qa/cluster/terragrunt.hcl

@@ -67,10 +67,6 @@ inputs = {
       ami_id             = "ami-0275aedb4db249683"
       availability_zones         = ["eu-west-1a", "eu-west-1b", "eu-west-1c"]
       max_unavailable_percentage = 50
-      kube_memory                = "1024Mi"
-      kube_cpu                   = "500m"
-      sys_memory                 = "768Mi"
-      sys_cpu                    = "300m"
       labels = {
         "karpenter.sh/controller" = "true" # required for Karpenter
       }
@@ -93,10 +89,6 @@ inputs = {
       labels = {
         dedicated = "observability"
       }
-      kube_memory   = "585Mi"
-      kube_cpu      = "90m"
-      sys_memory    = "585Mi"
-      sys_cpu       = "90m"
     }
   }