diff --git a/_sub/compute/eks-nlb/main.tf b/_sub/compute/eks-nlb/main.tf index 8b458bfe2..b3bd095d0 100644 --- a/_sub/compute/eks-nlb/main.tf +++ b/_sub/compute/eks-nlb/main.tf @@ -4,6 +4,8 @@ resource "aws_lb" "nlb" { internal = false #tfsec:ignore:aws-elbv2-alb-not-public tfsec:ignore:aws-elb-alb-not-public load_balancer_type = "network" subnets = var.subnet_ids + + security_groups = var.deploy ? [aws_security_group.traefik[0].id, aws_security_group.traefik_deployment[0].id] : [] } resource "aws_autoscaling_attachment" "nlb" { @@ -15,23 +17,39 @@ resource "aws_autoscaling_attachment" "nlb" { resource "aws_lb_target_group" "nlb" { count = var.deploy ? 1 : 0 name_prefix = substr(var.cluster_name, 0, min(6, length(var.cluster_name))) - port = 30000 + port = var.target_http_port # 30000 # protocol = "TCP" vpc_id = var.vpc_id + health_check { + path = var.health_check_path # "/ping" # + port = var.target_admin_port # 30001 # + protocol = "HTTP" + matcher = 200 + } + lifecycle { create_before_destroy = true } } -# resource "aws_lb_listener" "nlb" { count = var.deploy ? 1 : 0 load_balancer_arn = aws_lb.nlb[0].arn port = "443" - protocol = "TLS" - ssl_policy = "ELBSecurityPolicy-TLS-1-2-2017-01" - certificate_arn = var.nlb_certificate_arn + protocol = "TCP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.nlb[0].arn + } +} + +resource "aws_lb_listener" "nlb_insecure" { + count = var.deploy ? 1 : 0 + load_balancer_arn = aws_lb.nlb[0].arn + port = "80" + protocol = "TCP" default_action { type = "forward" @@ -39,14 +57,80 @@ resource "aws_lb_listener" "nlb" { } } -resource "aws_security_group_rule" "allow_argo" { + +resource "aws_security_group_rule" "traefik" { # on the nodes count = var.deploy ? 1 : 0 type = "ingress" description = "Allow inbound HTTP traffic" - from_port = 30000 - to_port = 30001 + from_port = var.target_http_port # 30000 # + to_port = var.target_admin_port # 30001 # protocol = "tcp" - cidr_blocks = var.cidr_blocks security_group_id = var.nodes_sg_id + + source_security_group_id = aws_security_group.traefik[0].id +} + +resource "aws_security_group" "traefik" { + count = var.deploy ? 1 : 0 + name_prefix = "allow_traefik-${var.cluster_name}-nlb" + description = "Allow traefik connection for ${var.cluster_name}" + vpc_id = var.vpc_id + + ingress { + description = "Ingress on standard HTTP port" + from_port = 80 + to_port = 80 + protocol = "TCP" + cidr_blocks = ["0.0.0.0/0"] #tfsec:ignore:aws-vpc-no-public-ingress-sg tfsec:ignore:aws-ec2-no-public-ingress-sgr + } + + ingress { + description = "Ingress on standard HTTPS port" + from_port = 443 + to_port = 443 + protocol = "TCP" + cidr_blocks = ["0.0.0.0/0"] #tfsec:ignore:aws-vpc-no-public-ingress-sg tfsec:ignore:aws-ec2-no-public-ingress-sgr + } + + tags = { + Name = "${var.cluster_name}-traefik-sg" + } + + lifecycle { + create_before_destroy = true + } + +} + +resource "aws_security_group" "traefik_deployment" { # on the load balancer + count = var.deploy ? 1 : 0 + name_prefix = "allow_traefik_deployment-${var.cluster_name}" + description = "Allow traefik connection related to the traefik deployment for ${var.cluster_name}" + vpc_id = var.vpc_id + + ingress { + description = "Ingress on var.target_admin_port" + from_port = var.target_admin_port # 30001 # + to_port = var.target_admin_port # 31001 # + protocol = "TCP" + self = true + } + + egress { + description = "Egress from var.target_http_port to var.target_admin_port" + from_port = var.target_http_port # 30000 # + to_port = var.target_admin_port # 30001 # + protocol = "TCP" + cidr_blocks = ["0.0.0.0/0"] #tfsec:ignore:aws-vpc-no-public-egress-sg tfsec:ignore:aws-ec2-no-public-egress-sgr + } + + tags = { + Name = "${var.cluster_name}-traefik-blue-sg" + } + + lifecycle { + create_before_destroy = true + } + } diff --git a/_sub/compute/eks-nlb/vars.tf b/_sub/compute/eks-nlb/vars.tf index 85c9bd9e6..261271d31 100644 --- a/_sub/compute/eks-nlb/vars.tf +++ b/_sub/compute/eks-nlb/vars.tf @@ -17,18 +17,31 @@ variable "nodes_sg_id" { type = string } -variable "cidr_blocks" { - type = list(string) -} +# variable "cidr_blocks" { +# type = list(string) +# } variable "subnet_ids" { type = list(string) } -variable "nlb_certificate_arn" { - type = string -} +# variable "nlb_certificate_arn" { +# type = string +# } variable "autoscaling_group_ids" { type = list(string) } + + +variable "target_http_port" { + type = number +} + +variable "target_admin_port" { + type = number +} + +variable "health_check_path" { + type = string +} \ No newline at end of file diff --git a/_sub/compute/k8s-traefik-flux/dependencies.tf b/_sub/compute/k8s-traefik-flux/dependencies.tf index 0539a29c0..ebe6b4d3a 100644 --- a/_sub/compute/k8s-traefik-flux/dependencies.tf +++ b/_sub/compute/k8s-traefik-flux/dependencies.tf @@ -8,4 +8,5 @@ locals { cluster_repo_path = "clusters/${var.cluster_name}" helm_repo_path = "platform-apps/${var.cluster_name}/${var.deploy_name}/helm" app_install_name = "platform-apps-${var.deploy_name}" + ca_server_arg = var.certificate_resolver_is_staging ? ["--certificatesresolvers.clusterresolver.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory"] : [] } diff --git a/_sub/compute/k8s-traefik-flux/main.tf b/_sub/compute/k8s-traefik-flux/main.tf index cb20aa1d6..899b35ff6 100644 --- a/_sub/compute/k8s-traefik-flux/main.tf +++ b/_sub/compute/k8s-traefik-flux/main.tf @@ -37,9 +37,15 @@ resource "github_repository_file" "traefik_helm_patch" { helm_chart_version = var.helm_chart_version http_nodeport = var.http_nodeport admin_nodeport = var.admin_nodeport - additional_args = var.additional_args + additional_args = concat(var.additional_args, var.certficate_resolver_args, local.ca_server_arg) replicas = var.replicas dashboard_ingress_host = var.dashboard_ingress_host + certificateResolverEnabled = var.enable_certificate_resolver + certificateResolverEmail = var.certificate_resolver_email + certificateResolverStorageEnabled = var.certificate_resolver_storage_enabled + certificateResolverStorageClass = var.certificate_resolver_storage_class + certificateResolverStorageAccessMode = var.certificate_resolver_storage_access_mode + certificateResolverStorageSize = var.certificate_resolver_storage_size }) overwrite_on_create = var.overwrite_on_create } diff --git a/_sub/compute/k8s-traefik-flux/values/helm-patch.yaml b/_sub/compute/k8s-traefik-flux/values/helm-patch.yaml index 68758b0ec..8305f6731 100644 --- a/_sub/compute/k8s-traefik-flux/values/helm-patch.yaml +++ b/_sub/compute/k8s-traefik-flux/values/helm-patch.yaml @@ -25,3 +25,21 @@ spec: enabled: true matchRule: Host(`${dashboard_ingress_host}`) && (PathPrefix(`/api`) || PathPrefix(`/dashboard`)) entryPoints: [traefik, web] +%{ if certificateResolverEnabled ~} + certificatesResolvers: + clusterresolver: + acme: + email: ${certificateResolverEmail} + storage: /data/acme.json + persistence: + enabled: ${certificateResolverStorageEnabled} + storageClass: ${certificateResolverStorageClass} + accessMode: ${certificateResolverStorageAccessMode} + size: ${certificateResolverStorageSize} + path: /data +%{ endif ~} + # logs: + # general: + # level: DEBUG + # # access: + # # enabled: true diff --git a/_sub/compute/k8s-traefik-flux/vars.tf b/_sub/compute/k8s-traefik-flux/vars.tf index 7bf0a3003..b826f5942 100644 --- a/_sub/compute/k8s-traefik-flux/vars.tf +++ b/_sub/compute/k8s-traefik-flux/vars.tf @@ -85,3 +85,48 @@ variable "prune" { default = true description = "Enable Garbage collection" } + +variable "enable_certificate_resolver" { + type = bool + default = false + description = "Enable the use of a certificate resolver (e.g. for Let's Encrypt)" +} + +variable "certificate_resolver_email" { + type = string + default = "" + description = "Email address to use for the certificate resolver registration (e.g. Let's Encrypt)" +} + +variable "certificate_resolver_storage_enabled" { + type = bool + default = false + description = "Enable persistent storage for the certificate resolver (e.g. for Let's Encrypt)" +} +variable "certificate_resolver_storage_class" { + type = string + default = "gp2" + description = "Storage class to use for the certificate resolver persistent storage (e.g. for Let's Encrypt)" +} +variable "certificate_resolver_storage_access_mode" { + type = string + default = "ReadWriteOnce" + description = "Access mode to use for the certificate resolver persistent storage (e.g. for Let's Encrypt)" +} +variable "certificate_resolver_storage_size" { + type = string + default = "128Mi" + description = "Size of the persistent storage to use for the certificate resolver (e.g. for Let's Encrypt)" +} + +variable "certficate_resolver_args" { + type = list(any) + default = [] + description = "Additional arguments to configure the certificate resolver (e.g. for Let's Encrypt)" +} + +variable "certificate_resolver_is_staging" { + type = bool + default = false + description = "Use the staging environment for the certificate resolver (e.g. for Let's Encrypt)" +} \ No newline at end of file diff --git a/compute/k8s-services/main.tf b/compute/k8s-services/main.tf index 157543eb6..d24745c7c 100644 --- a/compute/k8s-services/main.tf +++ b/compute/k8s-services/main.tf @@ -189,6 +189,65 @@ module "traefik_alb_anon" { green_variant_weight = var.traefik_green_variant_weight } +module "nlb" { + source = "../../_sub/compute/eks-nlb" + # deploy = var.traefik_nlb_deploy ? 1 : 0 + cluster_name = "${var.eks_cluster_name}-traefik-nlb" + nodes_sg_id = data.terraform_remote_state.cluster.outputs.eks_cluster_nodes_sg_id + vpc_id = data.aws_eks_cluster.eks.vpc_config[0].vpc_id + subnet_ids =data.terraform_remote_state.cluster.outputs.eks_control_subnet_ids #data.terraform_remote_state.cluster.outputs.eks_control_subnet_ids # + autoscaling_group_ids = data.terraform_remote_state.cluster.outputs.eks_worker_autoscaling_group_ids + target_http_port = var.traefik_nlb_http_nodeport + target_admin_port = var.traefik_nlb_admin_nodeport + health_check_path = "/ping" +} + +module "traefik_nlb_flux_manifests" { + source = "../../_sub/compute/k8s-traefik-flux" + count = var.traefik_nlb_deploy ? 1 : 0 + cluster_name = var.eks_cluster_name + deploy_name = "traefik-nlb" + namespace = "traefik-nlb" + helm_chart_version = var.traefik_nlb_helm_chart_version + replicas = 1 # length(data.terraform_remote_state.cluster.outputs.eks_worker_subnet_ids) cannot use the built-in Traefik ACME (Let's Encrypt) provider running multiple replicas + http_nodeport = var.traefik_nlb_http_nodeport + admin_nodeport = var.traefik_nlb_admin_nodeport + github_owner = var.fluxcd_bootstrap_repo_owner + repo_name = var.fluxcd_bootstrap_repo_name + repo_branch = var.fluxcd_bootstrap_repo_branch + additional_args = var.traefik_nlb_additional_args + enable_certificate_resolver = var.traefik_nlb_enable_certificate_resolver + certificate_resolver_email = var.traefik_nlb_certificate_resolver_email + certificate_resolver_storage_enabled = var.traefik_nlb_certificate_resolver_storage_enabled + certificate_resolver_storage_class = var.traefik_nlb_certificate_resolver_storage_class + certificate_resolver_storage_access_mode = var.traefik_nlb_certificate_resolver_storage_access_mode + certificate_resolver_storage_size = var.traefik_nlb_certificate_resolver_storage_size + certficate_resolver_args = var.traefik_nlb_certficate_resolver_args + certificate_resolver_is_staging = var.traefik_nlb_certificate_resolver_is_staging + dashboard_ingress_host = "traefik-nlb.${var.eks_cluster_name}.${var.workload_dns_zone_name}" + overwrite_on_create = var.fluxcd_bootstrap_overwrite_on_create + gitops_apps_repo_url = local.fluxcd_apps_repo_url + gitops_apps_repo_branch = var.fluxcd_apps_repo_branch + prune = var.fluxcd_prune + + providers = { + github = github.fluxcd + } + + depends_on = [module.platform_fluxcd] +} + +module "traefik_nlb_dns_for_traefik_nlb_dashboard" { + source = "../../_sub/network/route53-record" + count = var.traefik_nlb_deploy ? 1 : 0 + deploy = true + zone_id = local.workload_dns_zone_id + record_name = ["traefik-nlb.${var.eks_cluster_name}.${var.workload_dns_zone_name}"] + record_type = "CNAME" + record_ttl = "900" + record_value = "${module.nlb.nlb_fqdn}." +} + module "traefik_alb_anon_dns" { source = "../../_sub/network/route53-record" deploy = var.traefik_alb_anon_deploy diff --git a/compute/k8s-services/vars.tf b/compute/k8s-services/vars.tf index 45b1b5653..9eeee941e 100644 --- a/compute/k8s-services/vars.tf +++ b/compute/k8s-services/vars.tf @@ -535,6 +535,83 @@ variable "traefik_green_variant_weight" { default = 0 } +# NLB +variable "traefik_nlb_helm_chart_version" { + type = string + description = "Helm Chart version to be used to deploy Traefik" + default = "" +} + +variable "traefik_nlb_http_nodeport" { + type = number + description = "Nodeport used by NLB's to connect to the Traefik instance" + default = 30000 +} + +variable "traefik_nlb_admin_nodeport" { + type = number + description = "Nodeport used by NLB's to connect to the Traefik instance admin page" + default = 30001 +} + +variable "traefik_nlb_additional_args" { + type = list(any) + description = "Pass arguments to the additionalArguments node in the Traefik Helm chart" + default = ["--metrics.prometheus"] +} + +variable "traefik_nlb_enable_certificate_resolver" { + type = bool + default = false + description = "Enable the use of a certificate resolver (e.g. for Let's Encrypt)" +} + +variable "traefik_nlb_certificate_resolver_email" { + type = string + default = "" + description = "Email address to use for the certificate resolver registration (e.g. Let's Encrypt)" +} + +variable "traefik_nlb_certificate_resolver_storage_enabled" { + type = bool + default = false + description = "Enable persistent storage for the certificate resolver (e.g. for Let's Encrypt)" +} + +variable "traefik_nlb_certificate_resolver_storage_class" { + type = string + default = "gp2" + description = "Storage class to use for the certificate resolver persistent storage (e.g. Let's Encrypt)" +} + +variable "traefik_nlb_certificate_resolver_storage_access_mode" { + type = string + default = "readWriteOnce" + description = "Access mode to use for the certificate resolver persistent storage (e.g. Let's Encrypt)" +} + +variable "traefik_nlb_certificate_resolver_storage_size" { + type = string + default = "128Mi" + description = "Size of the persistent volume to use for the certificate resolver persistent storage (e.g. Let's Encrypt)" +} + +variable "traefik_nlb_certficate_resolver_args" { + type = list(any) + description = "Pass arguments to the additionalArguments node in the Traefik Helm chart for the certificate resolver" + default = [ + "--entryPoints.web.http.redirections.entrypoint.scheme=https", + "--entrypoints.web.http.redirections.entryPoint.to=:443", + "--certificatesresolvers.clusterresolver.acme.tlschallenge", + "--api.insecure=true" + ] +} + +variable "traefik_nlb_certificate_resolver_is_staging" { + type = bool + default = false + description = "Use the staging environment for the certificate resolver (e.g. for Let's Encrypt). Should be false in production." +} # -------------------------------------------------- # Blackbox Exporter # --------------------------------------------------