traefik and nlb with support for issuing certificates

_sub/compute/eks-nlb/main.tf

@@ -4,6 +4,8 @@ resource "aws_lb" "nlb" {
   internal           = false #tfsec:ignore:aws-elbv2-alb-not-public tfsec:ignore:aws-elb-alb-not-public
   load_balancer_type = "network"
   subnets            = var.subnet_ids
+
+  security_groups = var.deploy ? [aws_security_group.traefik[0].id, aws_security_group.traefik_deployment[0].id] : []
 }
 
 resource "aws_autoscaling_attachment" "nlb" {
@@ -15,38 +17,120 @@ resource "aws_autoscaling_attachment" "nlb" {
 resource "aws_lb_target_group" "nlb" {
   count       = var.deploy ? 1 : 0
   name_prefix = substr(var.cluster_name, 0, min(6, length(var.cluster_name)))
-  port        = 30000
+  port        = var.target_http_port # 30000 #
   protocol    = "TCP"
   vpc_id      = var.vpc_id
 
+  health_check {
+    path     = var.health_check_path # "/ping" #
+    port     = var.target_admin_port # 30001 #
+    protocol = "HTTP"
+    matcher  = 200
+  }
+
   lifecycle {
     create_before_destroy = true
   }
 }
 
-#
 resource "aws_lb_listener" "nlb" {
   count             = var.deploy ? 1 : 0
   load_balancer_arn = aws_lb.nlb[0].arn
   port              = "443"
-  protocol          = "TLS"
-  ssl_policy        = "ELBSecurityPolicy-TLS-1-2-2017-01"
-  certificate_arn   = var.nlb_certificate_arn
+  protocol          = "TCP"
+
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.nlb[0].arn
+  }
+}
+
+resource "aws_lb_listener" "nlb_insecure" {
+  count             = var.deploy ? 1 : 0
+  load_balancer_arn = aws_lb.nlb[0].arn
+  port              = "80"
+  protocol          = "TCP"
 
   default_action {
     type             = "forward"
     target_group_arn = aws_lb_target_group.nlb[0].arn
   }
 }
 
-resource "aws_security_group_rule" "allow_argo" {
+
+resource "aws_security_group_rule" "traefik" { # on the nodes
   count       = var.deploy ? 1 : 0
   type        = "ingress"
   description = "Allow inbound HTTP traffic"
-  from_port   = 30000
-  to_port     = 30001
+  from_port   = var.target_http_port # 30000 #
+  to_port     = var.target_admin_port # 30001 #
   protocol    = "tcp"
 
-  cidr_blocks       = var.cidr_blocks
   security_group_id = var.nodes_sg_id
+
+  source_security_group_id = aws_security_group.traefik[0].id
+}
+
+resource "aws_security_group" "traefik" {
+  count       = var.deploy ? 1 : 0
+  name_prefix = "allow_traefik-${var.cluster_name}-nlb"
+  description = "Allow traefik connection for ${var.cluster_name}"
+  vpc_id      = var.vpc_id
+
+  ingress {
+    description = "Ingress on standard HTTP port"
+    from_port   = 80
+    to_port     = 80
+    protocol    = "TCP"
+    cidr_blocks = ["0.0.0.0/0"] #tfsec:ignore:aws-vpc-no-public-ingress-sg tfsec:ignore:aws-ec2-no-public-ingress-sgr
+  }
+
+  ingress {
+    description = "Ingress on standard HTTPS port"
+    from_port   = 443
+    to_port     = 443
+    protocol    = "TCP"
+    cidr_blocks = ["0.0.0.0/0"] #tfsec:ignore:aws-vpc-no-public-ingress-sg tfsec:ignore:aws-ec2-no-public-ingress-sgr
+  }
+
+  tags = {
+    Name = "${var.cluster_name}-traefik-sg"
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+}
+
+resource "aws_security_group" "traefik_deployment" { # on the load balancer
+  count       = var.deploy ? 1 : 0
+  name_prefix = "allow_traefik_deployment-${var.cluster_name}"
+  description = "Allow traefik connection related to the traefik deployment for ${var.cluster_name}"
+  vpc_id      = var.vpc_id
+
+  ingress {
+    description = "Ingress on var.target_admin_port"
+    from_port   = var.target_admin_port # 30001 #
+    to_port     = var.target_admin_port # 31001 #
+    protocol    = "TCP"
+    self        = true
+  }
+
+  egress {
+    description = "Egress from var.target_http_port to var.target_admin_port"
+    from_port   = var.target_http_port # 30000 #
+    to_port     = var.target_admin_port # 30001 #
+    protocol    = "TCP"
+    cidr_blocks = ["0.0.0.0/0"] #tfsec:ignore:aws-vpc-no-public-egress-sg tfsec:ignore:aws-ec2-no-public-egress-sgr
+  }
+
+  tags = {
+    Name = "${var.cluster_name}-traefik-blue-sg"
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
 }

_sub/compute/eks-nlb/vars.tf

@@ -17,18 +17,31 @@ variable "nodes_sg_id" {
   type = string
 }
 
-variable "cidr_blocks" {
-  type = list(string)
-}
+# variable "cidr_blocks" {
+#   type = list(string)
+# }
 
 variable "subnet_ids" {
   type = list(string)
 }
 
-variable "nlb_certificate_arn" {
-  type = string
-}
+# variable "nlb_certificate_arn" {
+#   type = string
+# }
 
 variable "autoscaling_group_ids" {
   type = list(string)
 }
+
+
+variable "target_http_port" {
+  type = number
+}
+
+variable "target_admin_port" {
+  type = number
+}
+
+variable "health_check_path" {
+  type = string
+}

_sub/compute/k8s-traefik-flux/dependencies.tf

@@ -8,4 +8,5 @@ locals {
   cluster_repo_path   = "clusters/${var.cluster_name}"
   helm_repo_path      = "platform-apps/${var.cluster_name}/${var.deploy_name}/helm"
   app_install_name    = "platform-apps-${var.deploy_name}"
+  ca_server_arg           =  var.certificate_resolver_is_staging ? ["--certificatesresolvers.clusterresolver.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory"] : []
 }

_sub/compute/k8s-traefik-flux/main.tf

@@ -37,9 +37,15 @@ resource "github_repository_file" "traefik_helm_patch" {
     helm_chart_version     = var.helm_chart_version
     http_nodeport          = var.http_nodeport
     admin_nodeport         = var.admin_nodeport
-    additional_args        = var.additional_args
+    additional_args        = concat(var.additional_args, var.certficate_resolver_args, local.ca_server_arg)
     replicas               = var.replicas
     dashboard_ingress_host = var.dashboard_ingress_host
+    certificateResolverEnabled = var.enable_certificate_resolver
+    certificateResolverEmail = var.certificate_resolver_email
+    certificateResolverStorageEnabled = var.certificate_resolver_storage_enabled
+    certificateResolverStorageClass = var.certificate_resolver_storage_class
+    certificateResolverStorageAccessMode = var.certificate_resolver_storage_access_mode
+    certificateResolverStorageSize = var.certificate_resolver_storage_size
   })
   overwrite_on_create = var.overwrite_on_create
 }

_sub/compute/k8s-traefik-flux/values/helm-patch.yaml

@@ -25,3 +25,21 @@ spec:
         enabled: true
         matchRule: Host(`${dashboard_ingress_host}`) && (PathPrefix(`/api`) || PathPrefix(`/dashboard`))
         entryPoints: [traefik, web]
+%{ if certificateResolverEnabled ~}
+    certificatesResolvers:
+      clusterresolver:
+        acme:
+          email: ${certificateResolverEmail}
+          storage: /data/acme.json
+    persistence:
+      enabled: ${certificateResolverStorageEnabled}
+      storageClass: ${certificateResolverStorageClass}
+      accessMode: ${certificateResolverStorageAccessMode}
+      size: ${certificateResolverStorageSize}
+      path: /data
+%{ endif ~}
+    # logs:
+    #   general:
+    #     level: DEBUG
+    #   # access:
+    #   #   enabled: true

_sub/compute/k8s-traefik-flux/vars.tf

@@ -85,3 +85,48 @@ variable "prune" {
   default     = true
   description = "Enable Garbage collection"
 }
+
+variable "enable_certificate_resolver" {
+  type = bool
+  default = false
+  description = "Enable the use of a certificate resolver (e.g. for Let's Encrypt)"
+}
+
+variable "certificate_resolver_email" {
+  type = string
+  default = ""
+  description = "Email address to use for the certificate resolver registration (e.g. Let's Encrypt)"
+}
+
+variable "certificate_resolver_storage_enabled" {
+  type = bool
+  default = false
+  description = "Enable persistent storage for the certificate resolver (e.g. for Let's Encrypt)"
+}
+variable "certificate_resolver_storage_class" {
+  type = string
+  default = "gp2"
+  description = "Storage class to use for the certificate resolver persistent storage (e.g. for Let's Encrypt)"
+}
+variable "certificate_resolver_storage_access_mode" {
+  type = string
+  default = "ReadWriteOnce"
+  description = "Access mode to use for the certificate resolver persistent storage (e.g. for Let's Encrypt)"
+}
+variable "certificate_resolver_storage_size" {
+  type = string
+  default = "128Mi"
+  description = "Size of the persistent storage to use for the certificate resolver (e.g. for Let's Encrypt)"
+}
+
+variable "certficate_resolver_args" {
+  type = list(any)
+  default = []
+  description = "Additional arguments to configure the certificate resolver (e.g. for Let's Encrypt)"
+}
+
+variable "certificate_resolver_is_staging" {
+  type = bool
+  default = false
+  description = "Use the staging environment for the certificate resolver (e.g. for Let's Encrypt)"
+}

compute/k8s-services/main.tf

@@ -189,6 +189,65 @@ module "traefik_alb_anon" {
   green_variant_weight            = var.traefik_green_variant_weight
 }
 
+module "nlb" {
+  source = "../../_sub/compute/eks-nlb"
+  # deploy  = var.traefik_nlb_deploy ? 1 : 0
+  cluster_name   = "${var.eks_cluster_name}-traefik-nlb"
+  nodes_sg_id = data.terraform_remote_state.cluster.outputs.eks_cluster_nodes_sg_id
+  vpc_id = data.aws_eks_cluster.eks.vpc_config[0].vpc_id
+  subnet_ids =data.terraform_remote_state.cluster.outputs.eks_control_subnet_ids #data.terraform_remote_state.cluster.outputs.eks_control_subnet_ids #
+  autoscaling_group_ids = data.terraform_remote_state.cluster.outputs.eks_worker_autoscaling_group_ids
+  target_http_port = var.traefik_nlb_http_nodeport
+  target_admin_port = var.traefik_nlb_admin_nodeport
+  health_check_path = "/ping"
+}
+
+module "traefik_nlb_flux_manifests" {
+  source                  = "../../_sub/compute/k8s-traefik-flux"
+  count                   = var.traefik_nlb_deploy ? 1 : 0
+  cluster_name            = var.eks_cluster_name
+  deploy_name             = "traefik-nlb"
+  namespace               = "traefik-nlb"
+  helm_chart_version      = var.traefik_nlb_helm_chart_version
+  replicas                = 1 # length(data.terraform_remote_state.cluster.outputs.eks_worker_subnet_ids) cannot use the built-in Traefik ACME (Let's Encrypt) provider running multiple replicas
+  http_nodeport           = var.traefik_nlb_http_nodeport
+  admin_nodeport          = var.traefik_nlb_admin_nodeport
+  github_owner            = var.fluxcd_bootstrap_repo_owner
+  repo_name               = var.fluxcd_bootstrap_repo_name
+  repo_branch             = var.fluxcd_bootstrap_repo_branch
+  additional_args         = var.traefik_nlb_additional_args
+  enable_certificate_resolver = var.traefik_nlb_enable_certificate_resolver
+  certificate_resolver_email = var.traefik_nlb_certificate_resolver_email
+  certificate_resolver_storage_enabled = var.traefik_nlb_certificate_resolver_storage_enabled
+  certificate_resolver_storage_class = var.traefik_nlb_certificate_resolver_storage_class
+  certificate_resolver_storage_access_mode = var.traefik_nlb_certificate_resolver_storage_access_mode
+  certificate_resolver_storage_size = var.traefik_nlb_certificate_resolver_storage_size
+  certficate_resolver_args = var.traefik_nlb_certficate_resolver_args
+  certificate_resolver_is_staging = var.traefik_nlb_certificate_resolver_is_staging
+  dashboard_ingress_host  = "traefik-nlb.${var.eks_cluster_name}.${var.workload_dns_zone_name}"
+  overwrite_on_create     = var.fluxcd_bootstrap_overwrite_on_create
+  gitops_apps_repo_url    = local.fluxcd_apps_repo_url
+  gitops_apps_repo_branch = var.fluxcd_apps_repo_branch
+  prune                   = var.fluxcd_prune
+
+  providers = {
+    github = github.fluxcd
+  }
+
+  depends_on = [module.platform_fluxcd]
+}
+
+module "traefik_nlb_dns_for_traefik_nlb_dashboard" {
+  source       = "../../_sub/network/route53-record"
+  count        = var.traefik_nlb_deploy ? 1 : 0
+  deploy       = true
+  zone_id      = local.workload_dns_zone_id
+  record_name  = ["traefik-nlb.${var.eks_cluster_name}.${var.workload_dns_zone_name}"]
+  record_type  = "CNAME"
+  record_ttl   = "900"
+  record_value = "${module.nlb.nlb_fqdn}."
+}
+
 module "traefik_alb_anon_dns" {
   source       = "../../_sub/network/route53-record"
   deploy       = var.traefik_alb_anon_deploy