diff --git a/_sub/compute/eks-nodegroup-managed/dependencies.tf b/_sub/compute/eks-nodegroup-managed/dependencies.tf index ed2999a5e..65384e147 100644 --- a/_sub/compute/eks-nodegroup-managed/dependencies.tf +++ b/_sub/compute/eks-nodegroup-managed/dependencies.tf @@ -18,4 +18,30 @@ data "aws_ami" "eks-node" { locals { # Pins AMI to 'ami_id' if it is set, otherwise, sets to the latest AMI. node_ami = var.ami_id != "" ? var.ami_id : data.aws_ami.eks-node.id -} \ No newline at end of file +} + +locals { + max_pods = 110 +} + +data "aws_ec2_instance_type" "this" { + for_each = toset(var.instance_types) + instance_type = each.value +} + +locals { + max_pods_calc = { + for instance_type, instance_data in data.aws_ec2_instance_type.this : instance_type => ( + (instance_data.maximum_network_interfaces * ((instance_data.maximum_ipv4_addresses_per_interface - 1) * 16)) + 2 + ) + } +} + +check "instance_types_support_max_pods" { + assert { + condition = alltrue([for instance_type, instance_data in local.max_pods_calc : ( + instance_data >= local.max_pods + )]) + error_message = "One or more instance types is too small and does not support the required max pods of ${local.max_pods}." + } +} diff --git a/_sub/compute/eks-nodegroup-managed/main.tf b/_sub/compute/eks-nodegroup-managed/main.tf index 6d30585f3..bde7fb1c6 100644 --- a/_sub/compute/eks-nodegroup-managed/main.tf +++ b/_sub/compute/eks-nodegroup-managed/main.tf @@ -3,7 +3,7 @@ resource "aws_launch_template" "eks" { count = signum(var.desired_size_per_subnet) image_id = local.node_ami - instance_type = element(var.instance_types, 0) + instance_type = null # because instance_type is a list set on node groups name_prefix = "eks-${var.cluster_name}-${var.nodegroup_name}-" # Make sure to update the max pod values in the template below using the script # `src/produce-eni-max-pods.sh` when updating the EKS VPC CNI addon. @@ -13,12 +13,9 @@ resource "aws_launch_template" "eks" { cluster_name : var.cluster_name, worker_inotify_max_user_watches : var.worker_inotify_max_user_watches, cidr : var.eks_service_cidr, - max_pods : var.max_pods, - kube_cpu : var.kube_reserved_cpu, - kube_memory : var.kube_reserved_memory, - sys_cpu : var.system_reserved_cpu, - sys_memory : var.system_reserved_memory, + max_pods : local.max_pods, docker_hub_creds : var.docker_hub_creds_ssm_path, + kubelet_memory_reservation : "${tostring(ceil(255 + (11 * local.max_pods)))}Mi" })) key_name = var.ec2_ssh_key update_default_version = true @@ -45,14 +42,20 @@ resource "aws_launch_template" "eks" { monitoring { enabled = true } + + lifecycle { + create_before_destroy = true + } } resource "aws_eks_node_group" "group" { count = signum(var.desired_size_per_subnet) cluster_name = var.cluster_name - node_group_name = var.nodegroup_name + node_group_name = null # conflicts with node_group_name_prefix + node_group_name_prefix = "${var.nodegroup_name}-" # needed for create before destroy lifecycle node_role_arn = var.node_role_arn subnet_ids = var.subnet_ids + instance_types = var.instance_types capacity_type = var.use_spot_instances ? "SPOT" : "ON_DEMAND" dynamic "taint" { @@ -90,6 +93,10 @@ resource "aws_eks_node_group" "group" { max_size = local.asg_max_size min_size = local.asg_min_size } + + lifecycle { + create_before_destroy = true + } } resource "aws_autoscaling_schedule" "eks" { diff --git a/_sub/compute/eks-nodegroup-managed/user-data.sh.tftpl b/_sub/compute/eks-nodegroup-managed/user-data.sh.tftpl index 71e64af3c..92650d982 100644 --- a/_sub/compute/eks-nodegroup-managed/user-data.sh.tftpl +++ b/_sub/compute/eks-nodegroup-managed/user-data.sh.tftpl @@ -29,32 +29,18 @@ Content-Type: application/node.eks.aws apiVersion: node.eks.aws/v1alpha1 kind: NodeConfig spec: - cluster: - name: ${cluster_name} - apiServerEndpoint: ${eks_endpoint} - certificateAuthority: ${eks_certificate_authority} - cidr: ${cidr} - kubelet: - config: - maxPods: ${max_pods} - registryPullQPS: 0 -%{ if kube_cpu != null || kube_memory != null } - kubeReserved: -%{ endif ~} -%{ if kube_cpu != null ~} - cpu: ${kube_cpu} -%{ endif ~} -%{ if kube_memory != null ~} - memory: ${kube_memory} -%{ endif ~} -%{ if sys_cpu != null || sys_memory != null ~} - systemReserved: -%{ endif ~} -%{ if sys_cpu != null ~} - cpu: ${sys_cpu} -%{ endif ~} -%{ if sys_memory != null ~} - memory: ${sys_memory} -%{ endif ~} + cluster: + name: ${cluster_name} + apiServerEndpoint: ${eks_endpoint} + certificateAuthority: ${eks_certificate_authority} + cidr: ${cidr} + kubelet: + config: + maxPods: ${max_pods} + registryPullQPS: 0 + kubeReserved: + memory: ${kubelet_memory_reservation} + systemReserved: + memory: 200Mi --BOUNDARY-- diff --git a/_sub/compute/eks-nodegroup-managed/vars.tf b/_sub/compute/eks-nodegroup-managed/vars.tf index 85d422dc4..08a1fb772 100644 --- a/_sub/compute/eks-nodegroup-managed/vars.tf +++ b/_sub/compute/eks-nodegroup-managed/vars.tf @@ -113,34 +113,4 @@ variable "max_unavailable_percentage" { type = number description = "Desired max percentage of unavailable worker nodes during node group update." default = null -} - -variable "max_pods" { - type = number - description = "Maximum number of Pods that can run on the Kubelet" - default = 110 -} - -variable "kube_reserved_cpu" { - type = string - description = "CPU reserved for kubernetes system components" - default = null -} - -variable "kube_reserved_memory" { - type = string - description = "Memory reserved for kubernetes system components" - default = null -} - -variable "system_reserved_cpu" { - type = string - description = "CPU reserved for system components" - default = null -} - -variable "system_reserved_memory" { - type = string - description = "Memory reserved for system components" - default = null -} +} \ No newline at end of file diff --git a/compute/eks-ec2/main.tf b/compute/eks-ec2/main.tf index 8a438fe6c..7d22e60f8 100644 --- a/compute/eks-ec2/main.tf +++ b/compute/eks-ec2/main.tf @@ -270,11 +270,6 @@ module "eks_managed_workers_node_group" { subnet_ids = length(each.value.availability_zones) == 0 ? module.eks_managed_workers_subnet.subnet_ids : [ for sn in module.eks_managed_workers_subnet.subnets : sn.id if contains(each.value.availability_zones, sn.availability_zone) ] - max_pods = each.value.max_pods - kube_reserved_cpu = each.value.kube_cpu - kube_reserved_memory = each.value.kube_memory - system_reserved_cpu = each.value.sys_cpu - system_reserved_memory = each.value.sys_memory # Docker Hub credentials docker_hub_creds_ssm_path = aws_ssm_parameter.dockerhub.name diff --git a/compute/eks-ec2/vars.tf b/compute/eks-ec2/vars.tf index ce7c450de..9b42074e9 100644 --- a/compute/eks-ec2/vars.tf +++ b/compute/eks-ec2/vars.tf @@ -193,11 +193,6 @@ variable "eks_managed_nodegroups" { effect = string })), []) labels = optional(map(string), {}) - max_pods = optional(number, 110) - sys_cpu = optional(string, null) - sys_memory = optional(string, null) - kube_cpu = optional(string, null) - kube_memory = optional(string, null) })) default = {} validation { diff --git a/test/integration/eu-west-1/k8s-qa/cluster/terragrunt.hcl b/test/integration/eu-west-1/k8s-qa/cluster/terragrunt.hcl index 1489e5204..ba008c9b8 100644 --- a/test/integration/eu-west-1/k8s-qa/cluster/terragrunt.hcl +++ b/test/integration/eu-west-1/k8s-qa/cluster/terragrunt.hcl @@ -67,10 +67,6 @@ inputs = { ami_id = "ami-0275aedb4db249683" availability_zones = ["eu-west-1a", "eu-west-1b", "eu-west-1c"] max_unavailable_percentage = 50 - kube_memory = "1024Mi" - kube_cpu = "500m" - sys_memory = "768Mi" - sys_cpu = "300m" labels = { "karpenter.sh/controller" = "true" # required for Karpenter } @@ -93,10 +89,6 @@ inputs = { labels = { dedicated = "observability" } - kube_memory = "585Mi" - kube_cpu = "90m" - sys_memory = "585Mi" - sys_cpu = "90m" } }