From d5926fc4adf59f3d955c91da0c769f615c1c400e Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Mon, 3 Sep 2018 11:24:51 +0100 Subject: [PATCH 01/71] WIP: scrunching terraform --- terraform/aws/bosh.tf | 0 terraform/aws/cf.tf | 0 terraform/aws/dns.tf | 4 +- terraform/aws/locals.tf | 6 ++ terraform/aws/prometheus.tf | 0 terraform/aws/routes.tf | 13 ---- terraform/aws/routing.tf | 59 +++++++++++++++++++ terraform/aws/subnets.tf | 39 ++++++++++++ terraform/aws/{ => todo}/outputs.tf | 0 terraform/aws/vars.tf | 21 +++---- terraform/aws/vpc.tf | 11 +++- .../concourse/aws/internet-access-nat.tf | 56 ------------------ terraform/concourse/aws/subnets.tf | 35 ----------- 13 files changed, 123 insertions(+), 121 deletions(-) create mode 100644 terraform/aws/bosh.tf create mode 100644 terraform/aws/cf.tf create mode 100644 terraform/aws/locals.tf create mode 100644 terraform/aws/prometheus.tf delete mode 100644 terraform/aws/routes.tf create mode 100644 terraform/aws/routing.tf create mode 100644 terraform/aws/subnets.tf rename terraform/aws/{ => todo}/outputs.tf (100%) delete mode 100644 terraform/concourse/aws/internet-access-nat.tf delete mode 100644 terraform/concourse/aws/subnets.tf diff --git a/terraform/aws/bosh.tf b/terraform/aws/bosh.tf new file mode 100644 index 00000000..e69de29b diff --git a/terraform/aws/cf.tf b/terraform/aws/cf.tf new file mode 100644 index 00000000..e69de29b diff --git a/terraform/aws/dns.tf b/terraform/aws/dns.tf index 7f80a93e..d9856e12 100644 --- a/terraform/aws/dns.tf +++ b/terraform/aws/dns.tf @@ -26,7 +26,7 @@ resource "aws_route53_record" "child_ns" { } resource "aws_route53_zone" "private" { - name = "${var.environment}-paas-private" + name = "paas-private" comment = "Private DNS zone for internal CF add-ons" vpc_id = "${aws_vpc.default.id}" vpc_region = "${var.region}" @@ -35,4 +35,4 @@ resource "aws_route53_zone" "private" { Name = "${var.environment}-paas-private-zone" Environment = "${var.environment}" } -} +} \ No newline at end of file diff --git a/terraform/aws/locals.tf b/terraform/aws/locals.tf new file mode 100644 index 00000000..5c8915c3 --- /dev/null +++ b/terraform/aws/locals.tf @@ -0,0 +1,6 @@ +locals { + num_azs = "${length(var.availability_zones)}" + public_subnet_offset = 1 + internal_subnet_offset = "${local.public_subnet_offset + local.num_azs}" + services_subnet_offset = "${local.internal_subnet_offset + local.num_azs}" +} \ No newline at end of file diff --git a/terraform/aws/prometheus.tf b/terraform/aws/prometheus.tf new file mode 100644 index 00000000..e69de29b diff --git a/terraform/aws/routes.tf b/terraform/aws/routes.tf deleted file mode 100644 index 8f243340..00000000 --- a/terraform/aws/routes.tf +++ /dev/null @@ -1,13 +0,0 @@ -resource "aws_internet_gateway" "default" { - vpc_id = "${aws_vpc.default.id}" - - tags { - Name = "${var.environment}" - } -} - -resource "aws_route" "internet_access" { - route_table_id = "${aws_vpc.default.main_route_table_id}" - destination_cidr_block = "0.0.0.0/0" - gateway_id = "${aws_internet_gateway.default.id}" -} diff --git a/terraform/aws/routing.tf b/terraform/aws/routing.tf new file mode 100644 index 00000000..bc1c42b1 --- /dev/null +++ b/terraform/aws/routing.tf @@ -0,0 +1,59 @@ +resource "aws_eip" "nat" { + count = "${local.num_azs}" + vpc = true + + tags { + Name = "${var.environment}-nat-gateway-az${count.index+1}-eip" + Environment = "${var.environment}" + } +} + +resource "aws_nat_gateway" "nat" { + count = "${local.num_azs}" + allocation_id = "${element(aws_eip.nat.*.id, count.index)}" + subnet_id = "${element(aws_subnet.public.*.id, count.index)}" + + tags { + Name = "${var.environment}-az${count.index+1}-nat" + Environment = "${var.environment}" + } +} + +resource "aws_route_table" "az" { + count = "${local.num_azs}" + vpc_id = "${aws_vpc.default.id}" + + lifecycle { + create_before_destroy = true + } + + tags { + Name = "${var.environment}-az${count.index+1}-route-table" + Environment = "${var.environment}" + } +} + +resource "aws_route" "internet_access" { + route_table_id = "${aws_vpc.default.main_route_table_id}" + destination_cidr_block = "0.0.0.0/0" + gateway_id = "${aws_internet_gateway.default.id}" +} + +resource "aws_route" "nat" { + count = "${local.num_azs}" + route_table_id = "${element(aws_route_table.az.*.id, count.index)}" + destination_cidr_block = "0.0.0.0/0" + nat_gateway_id = "${element(aws_nat_gateway.nat.*.id, count.index)}" +} + +resource "aws_route_table_association" "internal" { + count = "${local.num_azs}" + subnet_id = "${element(aws_subnet.internal.*.id, count.index)}" + route_table_id = "${element(aws_route_table.az.*.id, count.index)}" +} + +resource "aws_route_table_association" "services" { + count = "${local.num_azs}" + subnet_id = "${element(aws_subnet.services.*.id, count.index)}" + route_table_id = "${element(aws_route_table.az.*.id, count.index)}" +} \ No newline at end of file diff --git a/terraform/aws/subnets.tf b/terraform/aws/subnets.tf new file mode 100644 index 00000000..cd6ffdc6 --- /dev/null +++ b/terraform/aws/subnets.tf @@ -0,0 +1,39 @@ + +resource "aws_subnet" "public" { + count = "${local.num_azs}" + vpc_id = "${aws_vpc.default.id}" + cidr_block = "${cidrsubnet(aws_vpc.default.cidr_block, 8, count.index+local.public_subnet_offset)}" + availability_zone = "${element(var.availability_zones, count.index)}" + + tags { + Name = "${var.environment}-public-az${count.index+1}-subnet" + Environment = "${var.environment}" + Visibility = "public" + } +} + +resource "aws_subnet" "internal" { + count = "${local.num_azs}" + vpc_id = "${aws_vpc.default.id}" + cidr_block = "${cidrsubnet(aws_vpc.default.cidr_block, 8, count.index+local.internal_subnet_offset)}" + availability_zone = "${element(var.availability_zones, count.index)}" + + tags { + Name = "${var.environment}-internal-az${count.index+1}-subnet" + Environment = "${var.environment}" + Visibility = "private" + } +} + +resource "aws_subnet" "services" { + count = "${local.num_azs}" + vpc_id = "${aws_vpc.default.id}" + cidr_block = "${cidrsubnet(aws_vpc.default.cidr_block, 8, count.index+local.services_subnet_offset)}" + availability_zone = "${element(var.availability_zones, count.index)}" + + tags { + Name = "${var.environment}-services-az${count.index+1}-subnet" + Environment = "${var.environment}" + Visibility = "private" + } +} \ No newline at end of file diff --git a/terraform/aws/outputs.tf b/terraform/aws/todo/outputs.tf similarity index 100% rename from terraform/aws/outputs.tf rename to terraform/aws/todo/outputs.tf diff --git a/terraform/aws/vars.tf b/terraform/aws/vars.tf index 8d53d8ac..2b6e40fe 100644 --- a/terraform/aws/vars.tf +++ b/terraform/aws/vars.tf @@ -1,26 +1,19 @@ -variable "region" { - default = "eu-west-1" -} +variable "region" {} variable "environment" {} variable "parent_dns_zone" {} +variable "cidr_block" {} + variable "ingress_whitelist" { type = "list" - default = ["0.0.0.0/0"] -} - -variable "az1" { - default = "eu-west-1a" -} - -variable "az2" { - default = "eu-west-1b" + default = [] } -variable "az3" { - default = "eu-west-1c" +variable "availability_zones" { + type = "list" + default = [] } variable "s3_prefix" { diff --git a/terraform/aws/vpc.tf b/terraform/aws/vpc.tf index 57913906..4eceb98d 100644 --- a/terraform/aws/vpc.tf +++ b/terraform/aws/vpc.tf @@ -1,5 +1,5 @@ resource "aws_vpc" "default" { - cidr_block = "10.0.0.0/16" + cidr_block = "${var.cidr_block}" enable_dns_support = true enable_dns_hostnames = true @@ -8,3 +8,12 @@ resource "aws_vpc" "default" { Environment = "${var.environment}" } } + +resource "aws_internet_gateway" "default" { + vpc_id = "${aws_vpc.default.id}" + + tags { + Name = "${var.environment}" + } +} + diff --git a/terraform/concourse/aws/internet-access-nat.tf b/terraform/concourse/aws/internet-access-nat.tf deleted file mode 100644 index 7063edf7..00000000 --- a/terraform/concourse/aws/internet-access-nat.tf +++ /dev/null @@ -1,56 +0,0 @@ -resource "aws_eip" "nat_az1" { - vpc = true - - tags { - Name = "${var.environment}-nat-gateway-az1-eip" - Environment = "${var.environment}" - } -} - -resource "aws_eip" "nat_az2" { - vpc = true - - tags { - Name = "${var.environment}-nat-gateway-az2-eip" - Environment = "${var.environment}" - } -} - -resource "aws_eip" "nat_az3" { - vpc = true - - tags { - Name = "${var.environment}-nat-gateway-az3-eip" - Environment = "${var.environment}" - } -} - -resource "aws_nat_gateway" "nat_az1" { - allocation_id = "${aws_eip.nat_az1.id}" - subnet_id = "${aws_subnet.az1.id}" - - tags { - Name = "${var.environment}-az1-nat" - Environment = "${var.environment}" - } -} - -resource "aws_nat_gateway" "nat_az2" { - allocation_id = "${aws_eip.nat_az2.id}" - subnet_id = "${aws_subnet.az2.id}" - - tags { - Name = "${var.environment}-az2-nat" - Environment = "${var.environment}" - } -} - -resource "aws_nat_gateway" "nat_az3" { - allocation_id = "${aws_eip.nat_az3.id}" - subnet_id = "${aws_subnet.az3.id}" - - tags { - Name = "${var.environment}-az3-nat" - Environment = "${var.environment}" - } -} diff --git a/terraform/concourse/aws/subnets.tf b/terraform/concourse/aws/subnets.tf deleted file mode 100644 index 3dab1e19..00000000 --- a/terraform/concourse/aws/subnets.tf +++ /dev/null @@ -1,35 +0,0 @@ -resource "aws_subnet" "az1" { - vpc_id = "${var.vpc_id}" - cidr_block = "10.0.1.0/24" - availability_zone = "${var.az1}" - - tags { - Name = "${var.environment}-az1-subnet" - Environment = "${var.environment}" - Visibility = "public" - } -} - -resource "aws_subnet" "az2" { - vpc_id = "${var.vpc_id}" - cidr_block = "10.0.2.0/24" - availability_zone = "${var.az2}" - - tags { - Name = "${var.environment}-az2-subnet" - Environment = "${var.environment}" - Visibility = "public" - } -} - -resource "aws_subnet" "az3" { - vpc_id = "${var.vpc_id}" - cidr_block = "10.0.12.0/24" - availability_zone = "${var.az3}" - - tags { - Name = "${var.environment}-az3-subnet" - Environment = "${var.environment}" - Visibility = "public" - } -} From 5052b7057f650d7408c8020656bab97851d92f80 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Tue, 4 Sep 2018 11:15:40 +0100 Subject: [PATCH 02/71] Base bosh infrastructure --- terraform/aws/bosh.tf | 192 ++++++++++++++++++ terraform/aws/bosh_rds.tf | 0 terraform/aws/concourse.tf | 11 + terraform/aws/data.tf | 6 + terraform/aws/dns.tf | 4 - terraform/aws/jumpbox.tf | 11 + terraform/aws/locals.tf | 11 +- terraform/aws/subnets.tf | 6 +- .../templates/bosh_iam_policy.json} | 0 terraform/aws/vars.tf | 6 +- terraform/aws/vpc.tf | 2 +- 11 files changed, 236 insertions(+), 13 deletions(-) create mode 100644 terraform/aws/bosh_rds.tf create mode 100644 terraform/aws/concourse.tf create mode 100644 terraform/aws/data.tf create mode 100644 terraform/aws/jumpbox.tf rename terraform/{bosh/aws/templates/iam_policy.json => aws/templates/bosh_iam_policy.json} (100%) diff --git a/terraform/aws/bosh.tf b/terraform/aws/bosh.tf index e69de29b..b8c7d6c3 100644 --- a/terraform/aws/bosh.tf +++ b/terraform/aws/bosh.tf @@ -0,0 +1,192 @@ +# DNS +resource "aws_route53_record" "bosh_director" { + zone_id = "${aws_route53_zone.private.zone_id}" + name = "bosh_director.${aws_route53_zone.private.name}" + type = "A" + ttl = "30" + + records = ["${local.bosh_private_ip}"] +} + +#IAM + +resource "aws_iam_instance_profile" "bosh" { + name = "${var.environment}_bosh_profile" + role = "${aws_iam_role.bosh.name}" +} + +resource "aws_iam_role" "bosh" { + name = "${var.environment}_bosh_role" + path = "/" + + assume_role_policy = < Date: Tue, 4 Sep 2018 13:47:15 +0100 Subject: [PATCH 03/71] Add bosh RDS terraform --- Makefile | 64 ++++--------------- bin/outputs.sh | 7 ++ bin/rds.sh | 17 +++++ bin/terraform.sh | 12 ++++ terraform/aws/bosh_rds.tf | 0 terraform/{aws => base}/aws.tf | 0 terraform/{aws => base}/bosh.tf | 39 +++++++++++ terraform/{aws => base}/cf.tf | 0 terraform/{aws => base}/concourse.tf | 0 terraform/{aws => base}/data.tf | 0 terraform/{aws => base}/dns.tf | 0 terraform/{aws => base}/jumpbox.tf | 0 terraform/{aws => base}/locals.tf | 0 terraform/base/outputs.tf | 7 ++ terraform/{aws => base}/prometheus.tf | 0 terraform/base/rds.tf | 5 ++ terraform/{aws => base}/routing.tf | 0 terraform/{aws => base}/s3.tf | 0 terraform/{aws => base}/subnets.tf | 13 ++++ .../templates/bosh_iam_policy.json | 0 terraform/{aws => base}/todo/outputs.tf | 0 terraform/{aws => base}/vars.tf | 4 ++ terraform/{aws => base}/vpc.tf | 0 .../bosh-databases/aws/bosh_databases.tf | 0 .../{ => old}/bosh-databases/aws/outputs.tf | 0 .../{ => old}/bosh-databases/aws/vars.tf | 0 terraform/{ => old}/bosh/aws/aws.tf | 0 terraform/{ => old}/bosh/aws/dns.tf | 0 terraform/{ => old}/bosh/aws/iam.tf | 0 .../{ => old}/bosh/aws/internet-access-nat.tf | 0 terraform/{ => old}/bosh/aws/outputs.tf | 0 terraform/{ => old}/bosh/aws/rds.tf | 0 terraform/{ => old}/bosh/aws/s3.tf | 0 .../{ => old}/bosh/aws/security-groups.tf | 0 terraform/{ => old}/bosh/aws/subnets.tf | 0 terraform/{ => old}/bosh/aws/vars.tf | 0 .../cf-databases/aws/cf_databases.tf | 0 .../{ => old}/cf-databases/aws/outputs.tf | 0 terraform/{ => old}/cf-databases/aws/vars.tf | 0 terraform/{ => old}/cf/aws/alb.tf | 0 terraform/{ => old}/cf/aws/aws.tf | 0 terraform/{ => old}/cf/aws/cert.tf | 0 terraform/{ => old}/cf/aws/data.tf | 0 terraform/{ => old}/cf/aws/dns.tf | 0 terraform/{ => old}/cf/aws/elb.tf | 0 terraform/{ => old}/cf/aws/iam.tf | 0 .../{ => old}/cf/aws/internet-access-nat.tf | 0 terraform/{ => old}/cf/aws/outputs.tf | 0 terraform/{ => old}/cf/aws/rds.tf | 0 terraform/{ => old}/cf/aws/s3-blobstore.tf | 0 terraform/{ => old}/cf/aws/s3.tf | 0 terraform/{ => old}/cf/aws/security-groups.tf | 0 terraform/{ => old}/cf/aws/subnets.tf | 0 terraform/{ => old}/cf/aws/template.tf | 0 .../templates/cloud_controller_policy.json | 0 terraform/{ => old}/cf/aws/vars.tf | 0 terraform/{ => old}/concourse/aws/alb.tf | 0 terraform/{ => old}/concourse/aws/aws.tf | 0 terraform/{ => old}/concourse/aws/cert.tf | 0 terraform/{ => old}/concourse/aws/dns.tf | 0 terraform/{ => old}/concourse/aws/eips.tf | 0 terraform/{ => old}/concourse/aws/iam.tf | 0 .../{ => old}/concourse/aws/key-pairs.tf | 0 terraform/{ => old}/concourse/aws/outputs.tf | 0 .../concourse/aws/security-groups.tf | 0 terraform/{ => old}/concourse/aws/template.tf | 0 .../concourse/aws/templates/iam_policy.json | 0 terraform/{ => old}/concourse/aws/vars.tf | 0 terraform/{ => old}/jumpbox/aws/aws.tf | 0 terraform/{ => old}/jumpbox/aws/dns.tf | 0 terraform/{ => old}/jumpbox/aws/eips.tf | 0 terraform/{ => old}/jumpbox/aws/outputs.tf | 0 terraform/{ => old}/jumpbox/aws/s3.tf | 0 .../{ => old}/jumpbox/aws/security-groups.tf | 0 terraform/{ => old}/jumpbox/aws/vars.tf | 0 terraform/{ => old}/prometheus/aws/alb.tf | 0 terraform/{ => old}/prometheus/aws/aws.tf | 0 terraform/{ => old}/prometheus/aws/cert.tf | 0 terraform/{ => old}/prometheus/aws/data.tf | 0 terraform/{ => old}/prometheus/aws/dns.tf | 0 .../prometheus/aws/internet-access-nat.tf | 0 terraform/{ => old}/prometheus/aws/outputs.tf | 0 terraform/{ => old}/prometheus/aws/s3.tf | 0 .../prometheus/aws/security-groups.tf | 0 terraform/{ => old}/prometheus/aws/subnets.tf | 0 terraform/{ => old}/prometheus/aws/vars.tf | 0 terraform/{ => old}/rds-broker/aws/iam.tf | 0 .../{ => old}/rds-broker/aws/template.tf | 0 .../rds-broker/aws/templates/iam_policy.json | 0 terraform/{ => old}/rds-broker/aws/vars.tf | 0 terraform/rds/aws.tf | 3 + terraform/rds/bosh.tf | 32 ++++++++++ terraform/rds/vars.tf | 5 ++ 93 files changed, 157 insertions(+), 51 deletions(-) create mode 100755 bin/outputs.sh create mode 100755 bin/rds.sh create mode 100755 bin/terraform.sh delete mode 100644 terraform/aws/bosh_rds.tf rename terraform/{aws => base}/aws.tf (100%) rename terraform/{aws => base}/bosh.tf (78%) rename terraform/{aws => base}/cf.tf (100%) rename terraform/{aws => base}/concourse.tf (100%) rename terraform/{aws => base}/data.tf (100%) rename terraform/{aws => base}/dns.tf (100%) rename terraform/{aws => base}/jumpbox.tf (100%) rename terraform/{aws => base}/locals.tf (100%) create mode 100644 terraform/base/outputs.tf rename terraform/{aws => base}/prometheus.tf (100%) create mode 100644 terraform/base/rds.tf rename terraform/{aws => base}/routing.tf (100%) rename terraform/{aws => base}/s3.tf (100%) rename terraform/{aws => base}/subnets.tf (75%) rename terraform/{aws => base}/templates/bosh_iam_policy.json (100%) rename terraform/{aws => base}/todo/outputs.tf (100%) rename terraform/{aws => base}/vars.tf (87%) rename terraform/{aws => base}/vpc.tf (100%) rename terraform/{ => old}/bosh-databases/aws/bosh_databases.tf (100%) rename terraform/{ => old}/bosh-databases/aws/outputs.tf (100%) rename terraform/{ => old}/bosh-databases/aws/vars.tf (100%) rename terraform/{ => old}/bosh/aws/aws.tf (100%) rename terraform/{ => old}/bosh/aws/dns.tf (100%) rename terraform/{ => old}/bosh/aws/iam.tf (100%) rename terraform/{ => old}/bosh/aws/internet-access-nat.tf (100%) rename terraform/{ => old}/bosh/aws/outputs.tf (100%) rename terraform/{ => old}/bosh/aws/rds.tf (100%) rename terraform/{ => old}/bosh/aws/s3.tf (100%) rename terraform/{ => old}/bosh/aws/security-groups.tf (100%) rename terraform/{ => old}/bosh/aws/subnets.tf (100%) rename terraform/{ => old}/bosh/aws/vars.tf (100%) rename terraform/{ => old}/cf-databases/aws/cf_databases.tf (100%) rename terraform/{ => old}/cf-databases/aws/outputs.tf (100%) rename terraform/{ => old}/cf-databases/aws/vars.tf (100%) rename terraform/{ => old}/cf/aws/alb.tf (100%) rename terraform/{ => old}/cf/aws/aws.tf (100%) rename terraform/{ => old}/cf/aws/cert.tf (100%) rename terraform/{ => old}/cf/aws/data.tf (100%) rename terraform/{ => old}/cf/aws/dns.tf (100%) rename terraform/{ => old}/cf/aws/elb.tf (100%) rename terraform/{ => old}/cf/aws/iam.tf (100%) rename terraform/{ => old}/cf/aws/internet-access-nat.tf (100%) rename terraform/{ => old}/cf/aws/outputs.tf (100%) rename terraform/{ => old}/cf/aws/rds.tf (100%) rename terraform/{ => old}/cf/aws/s3-blobstore.tf (100%) rename terraform/{ => old}/cf/aws/s3.tf (100%) rename terraform/{ => old}/cf/aws/security-groups.tf (100%) rename terraform/{ => old}/cf/aws/subnets.tf (100%) rename terraform/{ => old}/cf/aws/template.tf (100%) rename terraform/{ => old}/cf/aws/templates/cloud_controller_policy.json (100%) rename terraform/{ => old}/cf/aws/vars.tf (100%) rename terraform/{ => old}/concourse/aws/alb.tf (100%) rename terraform/{ => old}/concourse/aws/aws.tf (100%) rename terraform/{ => old}/concourse/aws/cert.tf (100%) rename terraform/{ => old}/concourse/aws/dns.tf (100%) rename terraform/{ => old}/concourse/aws/eips.tf (100%) rename terraform/{ => old}/concourse/aws/iam.tf (100%) rename terraform/{ => old}/concourse/aws/key-pairs.tf (100%) rename terraform/{ => old}/concourse/aws/outputs.tf (100%) rename terraform/{ => old}/concourse/aws/security-groups.tf (100%) rename terraform/{ => old}/concourse/aws/template.tf (100%) rename terraform/{ => old}/concourse/aws/templates/iam_policy.json (100%) rename terraform/{ => old}/concourse/aws/vars.tf (100%) rename terraform/{ => old}/jumpbox/aws/aws.tf (100%) rename terraform/{ => old}/jumpbox/aws/dns.tf (100%) rename terraform/{ => old}/jumpbox/aws/eips.tf (100%) rename terraform/{ => old}/jumpbox/aws/outputs.tf (100%) rename terraform/{ => old}/jumpbox/aws/s3.tf (100%) rename terraform/{ => old}/jumpbox/aws/security-groups.tf (100%) rename terraform/{ => old}/jumpbox/aws/vars.tf (100%) rename terraform/{ => old}/prometheus/aws/alb.tf (100%) rename terraform/{ => old}/prometheus/aws/aws.tf (100%) rename terraform/{ => old}/prometheus/aws/cert.tf (100%) rename terraform/{ => old}/prometheus/aws/data.tf (100%) rename terraform/{ => old}/prometheus/aws/dns.tf (100%) rename terraform/{ => old}/prometheus/aws/internet-access-nat.tf (100%) rename terraform/{ => old}/prometheus/aws/outputs.tf (100%) rename terraform/{ => old}/prometheus/aws/s3.tf (100%) rename terraform/{ => old}/prometheus/aws/security-groups.tf (100%) rename terraform/{ => old}/prometheus/aws/subnets.tf (100%) rename terraform/{ => old}/prometheus/aws/vars.tf (100%) rename terraform/{ => old}/rds-broker/aws/iam.tf (100%) rename terraform/{ => old}/rds-broker/aws/template.tf (100%) rename terraform/{ => old}/rds-broker/aws/templates/iam_policy.json (100%) rename terraform/{ => old}/rds-broker/aws/vars.tf (100%) create mode 100644 terraform/rds/aws.tf create mode 100644 terraform/rds/bosh.tf create mode 100644 terraform/rds/vars.tf diff --git a/Makefile b/Makefile index 1bb69067..f66d9e36 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ .EXPORT_ALL_VARIABLES: +.PHONY: terraform # Environment variables that you have to set yourself: # @@ -9,66 +10,27 @@ # You could override these, but you really shouldn't VAR_FILE = ${ENVIRONMENT}_vpc.tfvars -VPC_STATE_FILE = ${ENVIRONMENT}_vpc.tfstate.json -CONCOURSE_TERRAFORM_STATE_FILE = ${ENVIRONMENT}_concourse.tfstate.json -CONCOURSE_STATE_FILE = ${ENVIRONMENT}_concourse.state.json -CONCOURSE_CREDS_FILE = ${ENVIRONMENT}_concourse.creds.yml -PRIVATE_KEY_FILE = ${ENVIRONMENT}_concourse.pem -PUBLIC_KEY_FILE = ${PRIVATE_KEY_FILE}.pub - -## FIXME: delete -JUMPBOX_TERRAFORM_STATE_FILE = ${ENVIRONMENT}_jumpbox.tfstate.json help: @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' -require_vars: - @bin/require_vars.sh - -vpc: require_vars ## Setup a VPC to deploy concourse to - @bin/create_vpc.sh - -keypair: ## Create the SSH key pair used to log into the VMs - @bin/create_ssh_keypair.sh - -concourse_network: vpc keypair ## Setup networks for concourse to consume - @bin/create_concourse_network.sh - -concourse: concourse_network ## Deploy concourse with all prereqs - @bin/deploy_concourse.sh - -concourse_password: ## Retrieves the concourse password for a given environment - @bin/concourse_creds.sh - -concourse_login: require_vars ## Logs fly into concourse - @bin/login_fly.sh - -prometheus_credentials: ## Get credentials for prometheus stack - @bin/prometheus_credentials.sh - -test_pipeline: require_vars ## Deploy a test pipeline to concourse - @test/deploy_test_pipeline.sh - -test_s3_pipeline: require_vars ## Deploy a pipeline that tests S3 access to concourse - @test/deploy_s3_test_pipeline.sh - -deploy_pipeline: require_vars ## Deploy the CF deployment pipeline - @ci/deploy_pipeline.sh +terraform: ## create main terraform environment + @bin/terraform.sh apply -destruction_pipeline: require_vars ## Deploy the CF destruction pipeline - @ci/destruction_pipeline.sh +terraform_init: ## initialize terraform provider + @terraform init terraform/base -docker_image: ## Build the general-purpose tooled docker image for Concourse tasks - @bin/create_docker_image.sh +terraform_plan: ## plan what would be applied if we ran make terraform + @bin/terraform.sh plan -destroy: destroy_concourse_network ## Destroy an entire environment - @bin/delete_vpc.sh +destroy_terraform: ## destroy main terraform environment + @bin/terraform.sh destroy -destroy_concourse_network: destroy_concourse ## Destroy concourse and its network - @bin/delete_concourse_network.sh +outputs: ## base terraform outputs + @bin/outputs.sh -destroy_concourse: require_vars ## Destroy concourse only - @bin/delete_concourse.sh +rds: ## create rds instances + @bin/rds.sh apply decode_aws_error: ## Decode AWS message @aws sts decode-authorization-message --encoded-message ${DECODE_MESSAGE} | jq -r .DecodedMessage | jq . diff --git a/bin/outputs.sh b/bin/outputs.sh new file mode 100755 index 00000000..85af2460 --- /dev/null +++ b/bin/outputs.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -euo pipefail + +: $ENVIRONMENT + +terraform output -state="$ENVIRONMENT.tfstate" -json | jq '. | with_entries(.value = .value.value)' \ No newline at end of file diff --git a/bin/rds.sh b/bin/rds.sh new file mode 100755 index 00000000..b7d55965 --- /dev/null +++ b/bin/rds.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -euo pipefail + +: $ENVIRONMENT + +COMMAND=${1:-plan} +OPTS= +[ "$COMMAND" = plan ] || OPTS=-auto-approve + +terraform init terraform/rds + +terraform $COMMAND \ + $OPTS \ + -var-file="$ENVIRONMENT.tfvars" \ + -var-file=<(bin/outputs.sh) \ + -state="$ENVIRONMENT-rds.tfstate" terraform/rds/ \ No newline at end of file diff --git a/bin/terraform.sh b/bin/terraform.sh new file mode 100755 index 00000000..a0b1a272 --- /dev/null +++ b/bin/terraform.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -euo pipefail + +: $ENVIRONMENT + +COMMAND=${1:-plan} +OPTS= +[ "$COMMAND" = plan ] || OPTS=-auto-approve + +terraform init terraform/base +terraform $COMMAND $OPTS -var-file="$ENVIRONMENT.tfvars" -state="$ENVIRONMENT.tfstate" terraform/base/ diff --git a/terraform/aws/bosh_rds.tf b/terraform/aws/bosh_rds.tf deleted file mode 100644 index e69de29b..00000000 diff --git a/terraform/aws/aws.tf b/terraform/base/aws.tf similarity index 100% rename from terraform/aws/aws.tf rename to terraform/base/aws.tf diff --git a/terraform/aws/bosh.tf b/terraform/base/bosh.tf similarity index 78% rename from terraform/aws/bosh.tf rename to terraform/base/bosh.tf index b8c7d6c3..746e731e 100644 --- a/terraform/aws/bosh.tf +++ b/terraform/base/bosh.tf @@ -51,6 +51,26 @@ resource "aws_iam_role_policy" "bosh" { policy = "${data.template_file.iam_policy.rendered}" } + +# S3 +resource "aws_s3_bucket_object" "bosh-var-store" { + bucket = "${aws_s3_bucket.paas_states.id}" + acl = "private" + key = "bosh/bosh-variables.yml" + source = "/dev/null" + server_side_encryption = "aws:kms" + kms_key_id = "${aws_kms_key.paas_state_key.arn}" +} + +resource "aws_s3_bucket_object" "bosh-state" { + bucket = "${aws_s3_bucket.paas_states.id}" + acl = "private" + key = "bosh/bosh-state.json" + content = "{}" + server_side_encryption = "aws:kms" + kms_key_id = "${aws_kms_key.paas_state_key.arn}" +} + # SG resource "aws_security_group_rule" "allow-all" { security_group_id = "${aws_security_group.bosh.id}" @@ -190,3 +210,22 @@ resource "aws_security_group" "bosh_rds" { Environment = "${var.environment}" } } +resource "aws_security_group_rule" "allow_postgres_from_concourse" { + security_group_id = "${aws_security_group.bosh_rds.id}" + type = "ingress" + protocol = "tcp" + from_port = "${var.bosh_rds_port}" + to_port = "${var.bosh_rds_port}" + source_security_group_id = "${aws_security_group.concourse.id}" + description = "Provide ingress PostgreSQL traffic from Concourse" +} + +resource "aws_security_group_rule" "allow_postgres_from_bosh" { + security_group_id = "${aws_security_group.bosh_rds.id}" + type = "ingress" + protocol = "tcp" + from_port = "${var.bosh_rds_port}" + to_port = "${var.bosh_rds_port}" + source_security_group_id = "${aws_security_group.bosh.id}" + description = "Provide ingress PostgreSQL traffic from BOSH" +} \ No newline at end of file diff --git a/terraform/aws/cf.tf b/terraform/base/cf.tf similarity index 100% rename from terraform/aws/cf.tf rename to terraform/base/cf.tf diff --git a/terraform/aws/concourse.tf b/terraform/base/concourse.tf similarity index 100% rename from terraform/aws/concourse.tf rename to terraform/base/concourse.tf diff --git a/terraform/aws/data.tf b/terraform/base/data.tf similarity index 100% rename from terraform/aws/data.tf rename to terraform/base/data.tf diff --git a/terraform/aws/dns.tf b/terraform/base/dns.tf similarity index 100% rename from terraform/aws/dns.tf rename to terraform/base/dns.tf diff --git a/terraform/aws/jumpbox.tf b/terraform/base/jumpbox.tf similarity index 100% rename from terraform/aws/jumpbox.tf rename to terraform/base/jumpbox.tf diff --git a/terraform/aws/locals.tf b/terraform/base/locals.tf similarity index 100% rename from terraform/aws/locals.tf rename to terraform/base/locals.tf diff --git a/terraform/base/outputs.tf b/terraform/base/outputs.tf new file mode 100644 index 00000000..48ebaff5 --- /dev/null +++ b/terraform/base/outputs.tf @@ -0,0 +1,7 @@ +output "bosh_rds_security_group_id" { + value = "${aws_security_group.bosh_rds.id}" +} + +output "rds_db_subnet_group_id" { + value = "${aws_db_subnet_group.rds.id}" +} \ No newline at end of file diff --git a/terraform/aws/prometheus.tf b/terraform/base/prometheus.tf similarity index 100% rename from terraform/aws/prometheus.tf rename to terraform/base/prometheus.tf diff --git a/terraform/base/rds.tf b/terraform/base/rds.tf new file mode 100644 index 00000000..f7600578 --- /dev/null +++ b/terraform/base/rds.tf @@ -0,0 +1,5 @@ +resource "aws_db_subnet_group" "rds" { + name = "${var.environment}-rds-subnet-group" + description = "RDS subnet group" + subnet_ids = ["${aws_subnet.rds.*.id}"] +} \ No newline at end of file diff --git a/terraform/aws/routing.tf b/terraform/base/routing.tf similarity index 100% rename from terraform/aws/routing.tf rename to terraform/base/routing.tf diff --git a/terraform/aws/s3.tf b/terraform/base/s3.tf similarity index 100% rename from terraform/aws/s3.tf rename to terraform/base/s3.tf diff --git a/terraform/aws/subnets.tf b/terraform/base/subnets.tf similarity index 75% rename from terraform/aws/subnets.tf rename to terraform/base/subnets.tf index b7ce29c9..f1927326 100644 --- a/terraform/aws/subnets.tf +++ b/terraform/base/subnets.tf @@ -36,4 +36,17 @@ resource "aws_subnet" "services" { Environment = "${var.environment}" Visibility = "private" } +} + +resource "aws_subnet" "rds" { + count = "${local.num_azs}" + vpc_id = "${aws_vpc.default.id}" + cidr_block = "${element(local.rds_subnets, count.index)}" + availability_zone = "${element(var.availability_zones, count.index)}" + + tags { + Name = "${var.environment}-rds-az${count.index+1}-subnet" + Environment = "${var.environment}" + Visibility = "private" + } } \ No newline at end of file diff --git a/terraform/aws/templates/bosh_iam_policy.json b/terraform/base/templates/bosh_iam_policy.json similarity index 100% rename from terraform/aws/templates/bosh_iam_policy.json rename to terraform/base/templates/bosh_iam_policy.json diff --git a/terraform/aws/todo/outputs.tf b/terraform/base/todo/outputs.tf similarity index 100% rename from terraform/aws/todo/outputs.tf rename to terraform/base/todo/outputs.tf diff --git a/terraform/aws/vars.tf b/terraform/base/vars.tf similarity index 87% rename from terraform/aws/vars.tf rename to terraform/base/vars.tf index fc21dc36..ad2efc4b 100644 --- a/terraform/aws/vars.tf +++ b/terraform/base/vars.tf @@ -22,4 +22,8 @@ variable "s3_prefix" { variable "cidr_blocks" { type = "map" +} + +variable "bosh_rds_port" { + default = "5432" } \ No newline at end of file diff --git a/terraform/aws/vpc.tf b/terraform/base/vpc.tf similarity index 100% rename from terraform/aws/vpc.tf rename to terraform/base/vpc.tf diff --git a/terraform/bosh-databases/aws/bosh_databases.tf b/terraform/old/bosh-databases/aws/bosh_databases.tf similarity index 100% rename from terraform/bosh-databases/aws/bosh_databases.tf rename to terraform/old/bosh-databases/aws/bosh_databases.tf diff --git a/terraform/bosh-databases/aws/outputs.tf b/terraform/old/bosh-databases/aws/outputs.tf similarity index 100% rename from terraform/bosh-databases/aws/outputs.tf rename to terraform/old/bosh-databases/aws/outputs.tf diff --git a/terraform/bosh-databases/aws/vars.tf b/terraform/old/bosh-databases/aws/vars.tf similarity index 100% rename from terraform/bosh-databases/aws/vars.tf rename to terraform/old/bosh-databases/aws/vars.tf diff --git a/terraform/bosh/aws/aws.tf b/terraform/old/bosh/aws/aws.tf similarity index 100% rename from terraform/bosh/aws/aws.tf rename to terraform/old/bosh/aws/aws.tf diff --git a/terraform/bosh/aws/dns.tf b/terraform/old/bosh/aws/dns.tf similarity index 100% rename from terraform/bosh/aws/dns.tf rename to terraform/old/bosh/aws/dns.tf diff --git a/terraform/bosh/aws/iam.tf b/terraform/old/bosh/aws/iam.tf similarity index 100% rename from terraform/bosh/aws/iam.tf rename to terraform/old/bosh/aws/iam.tf diff --git a/terraform/bosh/aws/internet-access-nat.tf b/terraform/old/bosh/aws/internet-access-nat.tf similarity index 100% rename from terraform/bosh/aws/internet-access-nat.tf rename to terraform/old/bosh/aws/internet-access-nat.tf diff --git a/terraform/bosh/aws/outputs.tf b/terraform/old/bosh/aws/outputs.tf similarity index 100% rename from terraform/bosh/aws/outputs.tf rename to terraform/old/bosh/aws/outputs.tf diff --git a/terraform/bosh/aws/rds.tf b/terraform/old/bosh/aws/rds.tf similarity index 100% rename from terraform/bosh/aws/rds.tf rename to terraform/old/bosh/aws/rds.tf diff --git a/terraform/bosh/aws/s3.tf b/terraform/old/bosh/aws/s3.tf similarity index 100% rename from terraform/bosh/aws/s3.tf rename to terraform/old/bosh/aws/s3.tf diff --git a/terraform/bosh/aws/security-groups.tf b/terraform/old/bosh/aws/security-groups.tf similarity index 100% rename from terraform/bosh/aws/security-groups.tf rename to terraform/old/bosh/aws/security-groups.tf diff --git a/terraform/bosh/aws/subnets.tf b/terraform/old/bosh/aws/subnets.tf similarity index 100% rename from terraform/bosh/aws/subnets.tf rename to terraform/old/bosh/aws/subnets.tf diff --git a/terraform/bosh/aws/vars.tf b/terraform/old/bosh/aws/vars.tf similarity index 100% rename from terraform/bosh/aws/vars.tf rename to terraform/old/bosh/aws/vars.tf diff --git a/terraform/cf-databases/aws/cf_databases.tf b/terraform/old/cf-databases/aws/cf_databases.tf similarity index 100% rename from terraform/cf-databases/aws/cf_databases.tf rename to terraform/old/cf-databases/aws/cf_databases.tf diff --git a/terraform/cf-databases/aws/outputs.tf b/terraform/old/cf-databases/aws/outputs.tf similarity index 100% rename from terraform/cf-databases/aws/outputs.tf rename to terraform/old/cf-databases/aws/outputs.tf diff --git a/terraform/cf-databases/aws/vars.tf b/terraform/old/cf-databases/aws/vars.tf similarity index 100% rename from terraform/cf-databases/aws/vars.tf rename to terraform/old/cf-databases/aws/vars.tf diff --git a/terraform/cf/aws/alb.tf b/terraform/old/cf/aws/alb.tf similarity index 100% rename from terraform/cf/aws/alb.tf rename to terraform/old/cf/aws/alb.tf diff --git a/terraform/cf/aws/aws.tf b/terraform/old/cf/aws/aws.tf similarity index 100% rename from terraform/cf/aws/aws.tf rename to terraform/old/cf/aws/aws.tf diff --git a/terraform/cf/aws/cert.tf b/terraform/old/cf/aws/cert.tf similarity index 100% rename from terraform/cf/aws/cert.tf rename to terraform/old/cf/aws/cert.tf diff --git a/terraform/cf/aws/data.tf b/terraform/old/cf/aws/data.tf similarity index 100% rename from terraform/cf/aws/data.tf rename to terraform/old/cf/aws/data.tf diff --git a/terraform/cf/aws/dns.tf b/terraform/old/cf/aws/dns.tf similarity index 100% rename from terraform/cf/aws/dns.tf rename to terraform/old/cf/aws/dns.tf diff --git a/terraform/cf/aws/elb.tf b/terraform/old/cf/aws/elb.tf similarity index 100% rename from terraform/cf/aws/elb.tf rename to terraform/old/cf/aws/elb.tf diff --git a/terraform/cf/aws/iam.tf b/terraform/old/cf/aws/iam.tf similarity index 100% rename from terraform/cf/aws/iam.tf rename to terraform/old/cf/aws/iam.tf diff --git a/terraform/cf/aws/internet-access-nat.tf b/terraform/old/cf/aws/internet-access-nat.tf similarity index 100% rename from terraform/cf/aws/internet-access-nat.tf rename to terraform/old/cf/aws/internet-access-nat.tf diff --git a/terraform/cf/aws/outputs.tf b/terraform/old/cf/aws/outputs.tf similarity index 100% rename from terraform/cf/aws/outputs.tf rename to terraform/old/cf/aws/outputs.tf diff --git a/terraform/cf/aws/rds.tf b/terraform/old/cf/aws/rds.tf similarity index 100% rename from terraform/cf/aws/rds.tf rename to terraform/old/cf/aws/rds.tf diff --git a/terraform/cf/aws/s3-blobstore.tf b/terraform/old/cf/aws/s3-blobstore.tf similarity index 100% rename from terraform/cf/aws/s3-blobstore.tf rename to terraform/old/cf/aws/s3-blobstore.tf diff --git a/terraform/cf/aws/s3.tf b/terraform/old/cf/aws/s3.tf similarity index 100% rename from terraform/cf/aws/s3.tf rename to terraform/old/cf/aws/s3.tf diff --git a/terraform/cf/aws/security-groups.tf b/terraform/old/cf/aws/security-groups.tf similarity index 100% rename from terraform/cf/aws/security-groups.tf rename to terraform/old/cf/aws/security-groups.tf diff --git a/terraform/cf/aws/subnets.tf b/terraform/old/cf/aws/subnets.tf similarity index 100% rename from terraform/cf/aws/subnets.tf rename to terraform/old/cf/aws/subnets.tf diff --git a/terraform/cf/aws/template.tf b/terraform/old/cf/aws/template.tf similarity index 100% rename from terraform/cf/aws/template.tf rename to terraform/old/cf/aws/template.tf diff --git a/terraform/cf/aws/templates/cloud_controller_policy.json b/terraform/old/cf/aws/templates/cloud_controller_policy.json similarity index 100% rename from terraform/cf/aws/templates/cloud_controller_policy.json rename to terraform/old/cf/aws/templates/cloud_controller_policy.json diff --git a/terraform/cf/aws/vars.tf b/terraform/old/cf/aws/vars.tf similarity index 100% rename from terraform/cf/aws/vars.tf rename to terraform/old/cf/aws/vars.tf diff --git a/terraform/concourse/aws/alb.tf b/terraform/old/concourse/aws/alb.tf similarity index 100% rename from terraform/concourse/aws/alb.tf rename to terraform/old/concourse/aws/alb.tf diff --git a/terraform/concourse/aws/aws.tf b/terraform/old/concourse/aws/aws.tf similarity index 100% rename from terraform/concourse/aws/aws.tf rename to terraform/old/concourse/aws/aws.tf diff --git a/terraform/concourse/aws/cert.tf b/terraform/old/concourse/aws/cert.tf similarity index 100% rename from terraform/concourse/aws/cert.tf rename to terraform/old/concourse/aws/cert.tf diff --git a/terraform/concourse/aws/dns.tf b/terraform/old/concourse/aws/dns.tf similarity index 100% rename from terraform/concourse/aws/dns.tf rename to terraform/old/concourse/aws/dns.tf diff --git a/terraform/concourse/aws/eips.tf b/terraform/old/concourse/aws/eips.tf similarity index 100% rename from terraform/concourse/aws/eips.tf rename to terraform/old/concourse/aws/eips.tf diff --git a/terraform/concourse/aws/iam.tf b/terraform/old/concourse/aws/iam.tf similarity index 100% rename from terraform/concourse/aws/iam.tf rename to terraform/old/concourse/aws/iam.tf diff --git a/terraform/concourse/aws/key-pairs.tf b/terraform/old/concourse/aws/key-pairs.tf similarity index 100% rename from terraform/concourse/aws/key-pairs.tf rename to terraform/old/concourse/aws/key-pairs.tf diff --git a/terraform/concourse/aws/outputs.tf b/terraform/old/concourse/aws/outputs.tf similarity index 100% rename from terraform/concourse/aws/outputs.tf rename to terraform/old/concourse/aws/outputs.tf diff --git a/terraform/concourse/aws/security-groups.tf b/terraform/old/concourse/aws/security-groups.tf similarity index 100% rename from terraform/concourse/aws/security-groups.tf rename to terraform/old/concourse/aws/security-groups.tf diff --git a/terraform/concourse/aws/template.tf b/terraform/old/concourse/aws/template.tf similarity index 100% rename from terraform/concourse/aws/template.tf rename to terraform/old/concourse/aws/template.tf diff --git a/terraform/concourse/aws/templates/iam_policy.json b/terraform/old/concourse/aws/templates/iam_policy.json similarity index 100% rename from terraform/concourse/aws/templates/iam_policy.json rename to terraform/old/concourse/aws/templates/iam_policy.json diff --git a/terraform/concourse/aws/vars.tf b/terraform/old/concourse/aws/vars.tf similarity index 100% rename from terraform/concourse/aws/vars.tf rename to terraform/old/concourse/aws/vars.tf diff --git a/terraform/jumpbox/aws/aws.tf b/terraform/old/jumpbox/aws/aws.tf similarity index 100% rename from terraform/jumpbox/aws/aws.tf rename to terraform/old/jumpbox/aws/aws.tf diff --git a/terraform/jumpbox/aws/dns.tf b/terraform/old/jumpbox/aws/dns.tf similarity index 100% rename from terraform/jumpbox/aws/dns.tf rename to terraform/old/jumpbox/aws/dns.tf diff --git a/terraform/jumpbox/aws/eips.tf b/terraform/old/jumpbox/aws/eips.tf similarity index 100% rename from terraform/jumpbox/aws/eips.tf rename to terraform/old/jumpbox/aws/eips.tf diff --git a/terraform/jumpbox/aws/outputs.tf b/terraform/old/jumpbox/aws/outputs.tf similarity index 100% rename from terraform/jumpbox/aws/outputs.tf rename to terraform/old/jumpbox/aws/outputs.tf diff --git a/terraform/jumpbox/aws/s3.tf b/terraform/old/jumpbox/aws/s3.tf similarity index 100% rename from terraform/jumpbox/aws/s3.tf rename to terraform/old/jumpbox/aws/s3.tf diff --git a/terraform/jumpbox/aws/security-groups.tf b/terraform/old/jumpbox/aws/security-groups.tf similarity index 100% rename from terraform/jumpbox/aws/security-groups.tf rename to terraform/old/jumpbox/aws/security-groups.tf diff --git a/terraform/jumpbox/aws/vars.tf b/terraform/old/jumpbox/aws/vars.tf similarity index 100% rename from terraform/jumpbox/aws/vars.tf rename to terraform/old/jumpbox/aws/vars.tf diff --git a/terraform/prometheus/aws/alb.tf b/terraform/old/prometheus/aws/alb.tf similarity index 100% rename from terraform/prometheus/aws/alb.tf rename to terraform/old/prometheus/aws/alb.tf diff --git a/terraform/prometheus/aws/aws.tf b/terraform/old/prometheus/aws/aws.tf similarity index 100% rename from terraform/prometheus/aws/aws.tf rename to terraform/old/prometheus/aws/aws.tf diff --git a/terraform/prometheus/aws/cert.tf b/terraform/old/prometheus/aws/cert.tf similarity index 100% rename from terraform/prometheus/aws/cert.tf rename to terraform/old/prometheus/aws/cert.tf diff --git a/terraform/prometheus/aws/data.tf b/terraform/old/prometheus/aws/data.tf similarity index 100% rename from terraform/prometheus/aws/data.tf rename to terraform/old/prometheus/aws/data.tf diff --git a/terraform/prometheus/aws/dns.tf b/terraform/old/prometheus/aws/dns.tf similarity index 100% rename from terraform/prometheus/aws/dns.tf rename to terraform/old/prometheus/aws/dns.tf diff --git a/terraform/prometheus/aws/internet-access-nat.tf b/terraform/old/prometheus/aws/internet-access-nat.tf similarity index 100% rename from terraform/prometheus/aws/internet-access-nat.tf rename to terraform/old/prometheus/aws/internet-access-nat.tf diff --git a/terraform/prometheus/aws/outputs.tf b/terraform/old/prometheus/aws/outputs.tf similarity index 100% rename from terraform/prometheus/aws/outputs.tf rename to terraform/old/prometheus/aws/outputs.tf diff --git a/terraform/prometheus/aws/s3.tf b/terraform/old/prometheus/aws/s3.tf similarity index 100% rename from terraform/prometheus/aws/s3.tf rename to terraform/old/prometheus/aws/s3.tf diff --git a/terraform/prometheus/aws/security-groups.tf b/terraform/old/prometheus/aws/security-groups.tf similarity index 100% rename from terraform/prometheus/aws/security-groups.tf rename to terraform/old/prometheus/aws/security-groups.tf diff --git a/terraform/prometheus/aws/subnets.tf b/terraform/old/prometheus/aws/subnets.tf similarity index 100% rename from terraform/prometheus/aws/subnets.tf rename to terraform/old/prometheus/aws/subnets.tf diff --git a/terraform/prometheus/aws/vars.tf b/terraform/old/prometheus/aws/vars.tf similarity index 100% rename from terraform/prometheus/aws/vars.tf rename to terraform/old/prometheus/aws/vars.tf diff --git a/terraform/rds-broker/aws/iam.tf b/terraform/old/rds-broker/aws/iam.tf similarity index 100% rename from terraform/rds-broker/aws/iam.tf rename to terraform/old/rds-broker/aws/iam.tf diff --git a/terraform/rds-broker/aws/template.tf b/terraform/old/rds-broker/aws/template.tf similarity index 100% rename from terraform/rds-broker/aws/template.tf rename to terraform/old/rds-broker/aws/template.tf diff --git a/terraform/rds-broker/aws/templates/iam_policy.json b/terraform/old/rds-broker/aws/templates/iam_policy.json similarity index 100% rename from terraform/rds-broker/aws/templates/iam_policy.json rename to terraform/old/rds-broker/aws/templates/iam_policy.json diff --git a/terraform/rds-broker/aws/vars.tf b/terraform/old/rds-broker/aws/vars.tf similarity index 100% rename from terraform/rds-broker/aws/vars.tf rename to terraform/old/rds-broker/aws/vars.tf diff --git a/terraform/rds/aws.tf b/terraform/rds/aws.tf new file mode 100644 index 00000000..0977ba55 --- /dev/null +++ b/terraform/rds/aws.tf @@ -0,0 +1,3 @@ +provider "aws" { + region = "${var.region}" +} diff --git a/terraform/rds/bosh.tf b/terraform/rds/bosh.tf new file mode 100644 index 00000000..66d78d71 --- /dev/null +++ b/terraform/rds/bosh.tf @@ -0,0 +1,32 @@ +resource "random_string" "bosh_rds_password" { + length = 16 + special = false +} + +resource "aws_db_instance" "bosh_rds" { + identifier = "${var.environment}-bosh-rds" + allocated_storage = "100" + engine = "postgres" + engine_version = "9.6.8" + instance_class = "db.m4.xlarge" + username = "bosh_admin" + password = "${random_string.bosh_rds_password.result}" + vpc_security_group_ids = ["${var.bosh_rds_security_group_id}"] + db_subnet_group_name = "${var.rds_db_subnet_group_id}" + apply_immediately = true + backup_window = "01:00-02:00" + maintenance_window = "Tue:02:15-Tue:04:15" + copy_tags_to_snapshot = true + final_snapshot_identifier = "${var.environment}-bosh-rds-final" + multi_az = true + storage_encrypted = true + storage_type = "gp2" + skip_final_snapshot = true + name = "paastest" + port = "5432" + + tags { + Name = "${var.environment}-bosh-rds" + Environment = "${var.environment}" + } +} \ No newline at end of file diff --git a/terraform/rds/vars.tf b/terraform/rds/vars.tf new file mode 100644 index 00000000..3c883ced --- /dev/null +++ b/terraform/rds/vars.tf @@ -0,0 +1,5 @@ +variable "environment" {} +variable "region" {} + +variable "bosh_rds_security_group_id" {} +variable "rds_db_subnet_group_id" {} From 38db93d340ccb917db462fe6854653125654addb Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Tue, 4 Sep 2018 16:08:28 +0100 Subject: [PATCH 04/71] Manually create databases --- .gitignore | 1 + Makefile | 3 ++ bin/databases.sh | 38 ++++++++++++++++++++ bin/outputs.sh | 4 ++- bin/rds.sh | 4 +-- bin/terraform.sh | 2 +- terraform/base/bosh.tf | 10 ++++++ terraform/base/jumpbox.tf | 63 ++++++++++++++++++++++++++++++++++ terraform/base/outputs.tf | 10 +++++- terraform/databases/bosh.tf | 13 +++++++ terraform/databases/outputs.tf | 3 ++ terraform/databases/vars.tf | 10 ++++++ terraform/rds/outputs.tf | 23 +++++++++++++ 13 files changed, 179 insertions(+), 5 deletions(-) create mode 100755 bin/databases.sh create mode 100644 terraform/databases/bosh.tf create mode 100644 terraform/databases/outputs.tf create mode 100644 terraform/databases/vars.tf create mode 100644 terraform/rds/outputs.tf diff --git a/.gitignore b/.gitignore index 93221a17..23f53217 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ foo.yml *.lock.info **/.DS_Store +data/** \ No newline at end of file diff --git a/Makefile b/Makefile index f66d9e36..ad929227 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,9 @@ outputs: ## base terraform outputs rds: ## create rds instances @bin/rds.sh apply +databases: ## create databases + @bin/databases.sh + decode_aws_error: ## Decode AWS message @aws sts decode-authorization-message --encoded-message ${DECODE_MESSAGE} | jq -r .DecodedMessage | jq . diff --git a/bin/databases.sh b/bin/databases.sh new file mode 100755 index 00000000..fd14c109 --- /dev/null +++ b/bin/databases.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -euo pipefail + +: $ENVIRONMENT + +LOCAL_PORT=30201 +DB_USER=$(bin/outputs.sh rds | jq -r .bosh_db_username) +DB_HOST=$(bin/outputs.sh rds | jq -r .bosh_db_host) +DB_PORT=$(bin/outputs.sh rds | jq -r .bosh_db_port) +DB_PASS=$(bin/outputs.sh rds | jq -r .bosh_rds_password) +JUMPBOX_IP=$(bin/outputs.sh | jq -r .jumpbox_public_ip) +KEYFILE=~/.ssh/jumpbox.$$.pem +bin/outputs.sh | jq -r .jumpbox_private_key >$KEYFILE +chmod 600 $KEYFILE + +cleanup() { + rm -f $KEYFILE + kill $(ps -ef | awk "/ssh.*$LOCAL_PORT/ && ! /awk/ { print \$2 }") +} +ssh -fN -L $LOCAL_PORT:$DB_HOST:$DB_PORT ubuntu@$JUMPBOX_IP -i $KEYFILE +trap cleanup EXIT + +sql() { + PGPASSWORD=$DB_PASS psql -h localhost -p $LOCAL_PORT -U $DB_USER -d paastest -c "$*" +} + +exists() { + PGPASSWORD=$DB_PASS psql -h localhost -p $LOCAL_PORT -U $DB_USER -d paastest -l | grep -q $1 +} + +create() { + exists $1 || { echo "create database $1"; sql "create database $1"; } +} + +create bosh +create uaa +create credhub diff --git a/bin/outputs.sh b/bin/outputs.sh index 85af2460..ee76aeb0 100755 --- a/bin/outputs.sh +++ b/bin/outputs.sh @@ -4,4 +4,6 @@ set -euo pipefail : $ENVIRONMENT -terraform output -state="$ENVIRONMENT.tfstate" -json | jq '. | with_entries(.value = .value.value)' \ No newline at end of file +STEP=${1:+-$1} + +terraform output -state="data/$ENVIRONMENT$STEP.tfstate" -json | jq '. | with_entries(.value = .value.value)' \ No newline at end of file diff --git a/bin/rds.sh b/bin/rds.sh index b7d55965..699e9c9c 100755 --- a/bin/rds.sh +++ b/bin/rds.sh @@ -12,6 +12,6 @@ terraform init terraform/rds terraform $COMMAND \ $OPTS \ - -var-file="$ENVIRONMENT.tfvars" \ + -var-file="data/$ENVIRONMENT.tfvars" \ -var-file=<(bin/outputs.sh) \ - -state="$ENVIRONMENT-rds.tfstate" terraform/rds/ \ No newline at end of file + -state="data/$ENVIRONMENT-rds.tfstate" terraform/rds/ \ No newline at end of file diff --git a/bin/terraform.sh b/bin/terraform.sh index a0b1a272..1aa0e8c4 100755 --- a/bin/terraform.sh +++ b/bin/terraform.sh @@ -9,4 +9,4 @@ OPTS= [ "$COMMAND" = plan ] || OPTS=-auto-approve terraform init terraform/base -terraform $COMMAND $OPTS -var-file="$ENVIRONMENT.tfvars" -state="$ENVIRONMENT.tfstate" terraform/base/ +terraform $COMMAND $OPTS -var-file="data/$ENVIRONMENT.tfvars" -state="data/$ENVIRONMENT.tfstate" terraform/base/ diff --git a/terraform/base/bosh.tf b/terraform/base/bosh.tf index 746e731e..ef0eeb19 100644 --- a/terraform/base/bosh.tf +++ b/terraform/base/bosh.tf @@ -228,4 +228,14 @@ resource "aws_security_group_rule" "allow_postgres_from_bosh" { to_port = "${var.bosh_rds_port}" source_security_group_id = "${aws_security_group.bosh.id}" description = "Provide ingress PostgreSQL traffic from BOSH" +} + +resource "aws_security_group_rule" "allow_postgres_from_jumpbox" { + security_group_id = "${aws_security_group.bosh_rds.id}" + type = "ingress" + protocol = "tcp" + from_port = "${var.bosh_rds_port}" + to_port = "${var.bosh_rds_port}" + source_security_group_id = "${aws_security_group.jumpbox.id}" + description = "Provide ingress PostgreSQL traffic from jumpbox" } \ No newline at end of file diff --git a/terraform/base/jumpbox.tf b/terraform/base/jumpbox.tf index 4abdcdbf..c4140377 100644 --- a/terraform/base/jumpbox.tf +++ b/terraform/base/jumpbox.tf @@ -9,3 +9,66 @@ resource "aws_security_group" "jumpbox" { Environment = "${var.environment}" } } +resource "aws_security_group_rule" "jumpbox_ssh" { + security_group_id = "${aws_security_group.jumpbox.id}" + type = "ingress" + protocol = "tcp" + from_port = 22 + to_port = 22 + cidr_blocks = "${var.ingress_whitelist}" +} + +resource "aws_security_group_rule" "allow_jumpbox_to_postgres" { + security_group_id = "${aws_security_group.jumpbox.id}" + type = "egress" + protocol = "tcp" + from_port = "${var.bosh_rds_port}" + to_port = "${var.bosh_rds_port}" + source_security_group_id = "${aws_security_group.bosh_rds.id}" + description = "Provide egress PostgreSQL traffic from jumpbox" +} + +# SSH +resource "tls_private_key" "jumpbox" { + algorithm = "RSA" + rsa_bits = "2048" +} + +resource "aws_key_pair" "jumpbox" { + key_name = "${var.environment}-jumpbox" + public_key = "${tls_private_key.jumpbox.public_key_openssh}" +} +# instance +data "aws_ami" "ubuntu" { + most_recent = true + + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-trusty-14.04-amd64-server-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + owners = ["099720109477"] # Canonical +} + +resource "aws_instance" "jumpbox" { + ami = "${data.aws_ami.ubuntu.id}" + instance_type = "t2.micro" + key_name = "${aws_key_pair.jumpbox.key_name}" + subnet_id = "${aws_subnet.public.0.id}" + vpc_security_group_ids = ["${aws_security_group.jumpbox.id}"] + + tags { + Name = "${var.environment}-jumpbox" + Environment = "${var.environment}" + } +} + +resource "aws_eip" "jumpbox" { + instance = "${aws_instance.jumpbox.id}" + vpc = true +} \ No newline at end of file diff --git a/terraform/base/outputs.tf b/terraform/base/outputs.tf index 48ebaff5..cf5170bb 100644 --- a/terraform/base/outputs.tf +++ b/terraform/base/outputs.tf @@ -4,4 +4,12 @@ output "bosh_rds_security_group_id" { output "rds_db_subnet_group_id" { value = "${aws_db_subnet_group.rds.id}" -} \ No newline at end of file +} + +output "jumpbox_private_key" { + value = "${tls_private_key.jumpbox.private_key_pem}" +} + +output "jumpbox_public_ip" { + value = "${aws_instance.jumpbox.public_ip}" +} diff --git a/terraform/databases/bosh.tf b/terraform/databases/bosh.tf new file mode 100644 index 00000000..4956366a --- /dev/null +++ b/terraform/databases/bosh.tf @@ -0,0 +1,13 @@ +provider "postgresql" { + alias = "bosh" + host = "${var.bosh_db_host}" + port = "${var.bosh_db_port}" + username = "${var.bosh_db_username}" + password = "${var.bosh_rds_password}" + expected_version = "${var.bosh_db_engine_version}" +} + +resource "postgresql_database" "bosh" { + provider = "postgresql.bosh" + name = "bosh" +} diff --git a/terraform/databases/outputs.tf b/terraform/databases/outputs.tf new file mode 100644 index 00000000..165153b6 --- /dev/null +++ b/terraform/databases/outputs.tf @@ -0,0 +1,3 @@ +output "bosh_database_name" { + value = "${postgresql_database.bosh.name}" +} diff --git a/terraform/databases/vars.tf b/terraform/databases/vars.tf new file mode 100644 index 00000000..ff038ff4 --- /dev/null +++ b/terraform/databases/vars.tf @@ -0,0 +1,10 @@ +variable "bosh_db_host" {} +variable "bosh_db_username" {} + +variable "bosh_rds_password" {} + +variable "bosh_db_engine_version" {} +variable "bosh_db_port" {} + +variable "jumpbox_private_key" {} +variable "jumpbox_public_ip" {} \ No newline at end of file diff --git a/terraform/rds/outputs.tf b/terraform/rds/outputs.tf new file mode 100644 index 00000000..95e38b66 --- /dev/null +++ b/terraform/rds/outputs.tf @@ -0,0 +1,23 @@ +output "bosh_db_type" { + value = "${aws_db_instance.bosh_rds.engine}" +} + +output "bosh_db_host" { + value = "${aws_db_instance.bosh_rds.address}" +} + +output "bosh_db_port" { + value = "${aws_db_instance.bosh_rds.port}" +} + +output "bosh_db_engine_version" { + value = "${aws_db_instance.bosh_rds.engine_version}" +} + +output "bosh_db_username" { + value = "${aws_db_instance.bosh_rds.username}" +} + +output "bosh_rds_password" { + value = "${random_string.bosh_rds_password.result}" +} \ No newline at end of file From f2196f948f19e58e23b0bdd9b3e57f72395941c9 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Tue, 4 Sep 2018 16:17:15 +0100 Subject: [PATCH 05/71] Add environment to temporary pkey name --- bin/databases.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/databases.sh b/bin/databases.sh index fd14c109..77447e21 100755 --- a/bin/databases.sh +++ b/bin/databases.sh @@ -10,7 +10,7 @@ DB_HOST=$(bin/outputs.sh rds | jq -r .bosh_db_host) DB_PORT=$(bin/outputs.sh rds | jq -r .bosh_db_port) DB_PASS=$(bin/outputs.sh rds | jq -r .bosh_rds_password) JUMPBOX_IP=$(bin/outputs.sh | jq -r .jumpbox_public_ip) -KEYFILE=~/.ssh/jumpbox.$$.pem +KEYFILE=~/.ssh/$ENVIRONMENT.jumpbox.$$.pem bin/outputs.sh | jq -r .jumpbox_private_key >$KEYFILE chmod 600 $KEYFILE From 2cf2c589a2a72df27c3cff63f9dc20c3fde246c3 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Wed, 5 Sep 2018 10:17:59 +0100 Subject: [PATCH 06/71] Standup BosH and Jumpbox --- .gitmodules | 3 + Makefile | 3 + bin/bosh_credentials.sh | 73 +++++++---------------- bin/create_bosh.sh | 59 ++++++++++++++++++ bin/outputs.sh | 4 +- bin/terraform.sh | 2 +- bosh-deployment | 1 + terraform/base/bosh.tf | 20 +++++++ terraform/base/jumpbox.tf | 27 +++++++++ terraform/base/locals.tf | 7 ++- terraform/base/outputs.tf | 42 +++++++++++++ terraform/base/templates/jumpbox_init.cfg | 5 ++ terraform/base/vars.tf | 4 ++ terraform/databases/bosh.tf | 13 ---- terraform/databases/outputs.tf | 3 - terraform/databases/vars.tf | 10 ---- terraform/rds/outputs.tf | 3 + 17 files changed, 197 insertions(+), 82 deletions(-) create mode 100755 bin/create_bosh.sh create mode 160000 bosh-deployment create mode 100644 terraform/base/templates/jumpbox_init.cfg delete mode 100644 terraform/databases/bosh.tf delete mode 100644 terraform/databases/outputs.tf delete mode 100644 terraform/databases/vars.tf diff --git a/.gitmodules b/.gitmodules index 69342210..d6d36bc9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "concourse-bosh-deployment"] path = concourse-bosh-deployment url = https://github.com/concourse/concourse-bosh-deployment.git +[submodule "bosh-deployment"] + path = bosh-deployment + url = https://github.com/cloudfoundry/bosh-deployment.git diff --git a/Makefile b/Makefile index ad929227..4b33396b 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,9 @@ rds: ## create rds instances databases: ## create databases @bin/databases.sh +bosh: ## create bosh + @bin/create_bosh.sh + decode_aws_error: ## Decode AWS message @aws sts decode-authorization-message --encoded-message ${DECODE_MESSAGE} | jq -r .DecodedMessage | jq . diff --git a/bin/bosh_credentials.sh b/bin/bosh_credentials.sh index dc96ba16..1253bf4e 100755 --- a/bin/bosh_credentials.sh +++ b/bin/bosh_credentials.sh @@ -1,87 +1,56 @@ #!/bin/bash # # This script will set up a connection to a BOSH environment for you: -# -# - downloads from S3 the environment state files -# - sets up an SSH socks5 proxy (listening on BOSH_PROXY_PORT) -# - runs `bosh alias-env ` -# - exports the BOSH_CLIENT, BOSH_CLIENT_SECRET and BOSH_ALL_PROXY environment variables -# -# You can either run it in the foreground (-f), or source the script. In the first case, -# you will run in a subshell and everything will be cleaned up when you exit that shell. -# In the second, you are responsible for killing the SSH proxy. usage() { - echo "Usage: [source] $(basename $0) -e [-p ssh_port] [-f] [command]" + echo "Usage: [source] $(basename $0) -e [command]" echo echo "Options:" echo " -e Connect to this environment, e.g. eng2 (ENVIRONMENT)" - echo " -p Have SSH proxy listen on this port (BOSH_PROXY_PORT)" - echo " -f Run this session in a shell, when you exit, the SSH proxy will be killed" echo echo " If you specify a command, it will run, then detach the proxy" + echo " otherwise, you will be placed in a shell with the environment set" exit 1 } -: ${BOSH_PROXY_PORT:=30123} -FOREGROUND=false -while getopts 'e:p:f' option; do +while getopts 'e:f' option; do case $option in e) ENVIRONMENT="$OPTARG";; - p) BOSH_PROXY_PORT="$OPTARG";; - f) FOREGROUND=true;; *) usage;; esac done +export ENVIRONMENT shift $((OPTIND-1)) COMMAND=$* -VARS=/var/tmp/tmp$$ -mkdir -p "$VARS" -trap 'rm -rf "$VARS"' EXIT - -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/vpc/tfstate.json" "$VARS/" -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/jumpbox/jumpbox-variables.yml" "$VARS/" -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/bosh/bosh-variables.yml" "$VARS/" -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/bosh/bosh.yml" "$VARS/" -bosh int --path /jumpbox_ssh/private_key "$VARS/jumpbox-variables.yml" > "$VARS/jumpbox.key" -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "$VARS/tfstate.json" > "$VARS/vars.json" +JUMPBOX_IP=$(bin/outputs.sh base | jq -r .jumpbox_public_ip) +JUMPBOX_KEY=~/.ssh/$ENVIRONMENT.jumpbox.$$.pem +bin/outputs.sh base | jq -r .jumpbox_private_key >$JUMPBOX_KEY +chmod 600 $JUMPBOX_KEY +export BOSH_CA_CERT=/var/tmp/bosh_ca.$$.pem +bosh int --path /default_ca/ca "data/$ENVIRONMENT-bosh-variables.yml" >$BOSH_CA_CERT -ZONE=$(jq -r '.dns_zone' < "$VARS/vars.json" | sed 's/\.$//') -JUMPBOX_TARGET="jumpbox.${ZONE}" -chmod 600 "$VARS/jumpbox.key" -DIRECTOR_IP=$(bosh int --path '/cloud_provider/ssh_tunnel/host' "$VARS/bosh.yml") +cleanup() { + rm -f $JUMPBOX_KEY + rm -f $BOSH_CA_CERT +} +trap cleanup EXIT -if ! netstat -na | grep -q "127.0.0.1.${BOSH_PROXY_PORT}.*LISTEN"; then - ssh -4 -D $BOSH_PROXY_PORT -fNC jumpbox@${JUMPBOX_TARGET} -i "${VARS}/jumpbox.key" && STARTED_SSH=true -fi -bosh int --path /default_ca/ca "$VARS/bosh-variables.yml" > "${VARS}/bosh_ca.pem" -export BOSH_CA_CERT="${VARS}/bosh_ca.pem" export BOSH_CLIENT=admin -export BOSH_CLIENT_SECRET=$(bosh int --path /admin_password "$VARS/bosh-variables.yml") -export BOSH_ALL_PROXY="socks5://localhost:$BOSH_PROXY_PORT" +export BOSH_CLIENT_SECRET=$(bosh int --path /admin_password "data/$ENVIRONMENT-bosh-variables.yml") export BOSH_ENVIRONMENT="$ENVIRONMENT" +export BOSH_ALL_PROXY=ssh+socks5://ubuntu@$JUMPBOX_IP:22?private-key=$JUMPBOX_KEY +DIRECTOR_IP=$(bin/outputs.sh base | jq -r .bosh_private_ip) bosh alias-env "${ENVIRONMENT}" -e "${DIRECTOR_IP}" -kill_tunnel() { - if [ -n "$STARTED_SSH" ]; then - echo "Killing the SSH proxy on $BOSH_PROXY_PORT" - kill $(ps -ef | awk "/ssh -4 -D $BOSH_PROXY_PORT/ && ! /awk/ { print \$2 }") - fi -} - -if [ "$FOREGROUND" = true -o -n "$COMMAND" ]; then - trap 'kill_tunnel' EXIT -fi - -if [ "$FOREGROUND" = true ]; then +if [ -n "$COMMAND" ]; then + ""$COMMAND"" +else echo "OK, you are set up" export PS1="BOSH<$ENVIRONMENT>:\W \u\$ " bash -elif [ -n "$COMMAND" ]; then - ""$COMMAND"" fi \ No newline at end of file diff --git a/bin/create_bosh.sh b/bin/create_bosh.sh new file mode 100755 index 00000000..ba23774f --- /dev/null +++ b/bin/create_bosh.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +set -euo pipefail + +output() { + FILE=$1 + QUERY=$2 + + bin/outputs.sh $FILE | jq -r "$QUERY" +} + +bosh int \ + bosh-deployment/bosh.yml \ + --vars-store data/$ENVIRONMENT-bosh-variables.yml \ + -o bosh-deployment/aws/cpi.yml \ + -o bosh-deployment/misc/external-db.yml \ + -o operations/bosh/tags.yml \ + -o operations/bosh/dns-resolution.yml \ + -o operations/bosh/certificate.yml \ + -v director_name=bosh \ + -v internal_cidr="$(output base .bosh_subnet_cidr_block)" \ + -v internal_gw="$(output base .bosh_gateway_ip)" \ + -v internal_ip="$(output base .bosh_private_ip)" \ + -v bosh_director_fqdn="$(output base .bosh_director_fqdn)" \ + -v private_dns_nameserver="$(output base .vpc_dns_nameserver)" \ + -v region="$(jq -r .region < data/$ENVIRONMENT.tfvars)" \ + -v bosh_iam_instance_profile="$(output base .bosh_iam_instance_profile)" \ + -v az="$(jq -r .availability_zones[0] < data/$ENVIRONMENT.tfvars)" \ + -v default_key_name="$(output base .bosh_key_name)" \ + -v default_security_groups="$(output base .bosh_security_group_ids)" \ + --var-file private_key=<(output base .bosh_private_key) \ + -v subnet_id="$(output base .bosh_subnet_id)" \ + -v environment="${ENVIRONMENT}" \ + -v external_db_host="$(output rds .bosh_rds_fqdn)" \ + -v external_db_port="$(output rds .bosh_db_port)" \ + -v external_db_user="$(output rds .bosh_db_username)" \ + -v external_db_password="$(output rds .bosh_rds_password)" \ + -v external_db_adapter="$(output rds .bosh_db_type)" \ + -v external_db_name='bosh' \ + -v access_key_id="$AWS_ACCESS_KEY_ID" \ + -v secret_access_key="$AWS_SECRET_ACCESS_KEY" \ + > data/$ENVIRONMENT-bosh-manifest.yml + + +JUMPBOX_IP=$(output base .jumpbox_public_ip) +JUMPBOX_KEY=~/.ssh/$ENVIRONMENT.jumpbox.$$.pem +output base .jumpbox_private_key >$JUMPBOX_KEY +chmod 600 $JUMPBOX_KEY + +cleanup() { + rm -f $JUMPBOX_KEY +} +trap cleanup EXIT + +export BOSH_ALL_PROXY=ssh+socks5://ubuntu@$JUMPBOX_IP:22?private-key=$JUMPBOX_KEY + +bosh create-env \ + data/$ENVIRONMENT-bosh-manifest.yml \ + --state data/$ENVIRONMENT-bosh-state.json \ No newline at end of file diff --git a/bin/outputs.sh b/bin/outputs.sh index ee76aeb0..5567e2a9 100755 --- a/bin/outputs.sh +++ b/bin/outputs.sh @@ -4,6 +4,6 @@ set -euo pipefail : $ENVIRONMENT -STEP=${1:+-$1} +STEP=${1:-base} -terraform output -state="data/$ENVIRONMENT$STEP.tfstate" -json | jq '. | with_entries(.value = .value.value)' \ No newline at end of file +terraform output -state="data/$ENVIRONMENT-$STEP.tfstate" -json | jq '. | with_entries(.value = .value.value)' \ No newline at end of file diff --git a/bin/terraform.sh b/bin/terraform.sh index 1aa0e8c4..2ec00284 100755 --- a/bin/terraform.sh +++ b/bin/terraform.sh @@ -9,4 +9,4 @@ OPTS= [ "$COMMAND" = plan ] || OPTS=-auto-approve terraform init terraform/base -terraform $COMMAND $OPTS -var-file="data/$ENVIRONMENT.tfvars" -state="data/$ENVIRONMENT.tfstate" terraform/base/ +terraform $COMMAND $OPTS -var-file="data/$ENVIRONMENT.tfvars" -state="data/$ENVIRONMENT-base.tfstate" terraform/base/ diff --git a/bosh-deployment b/bosh-deployment new file mode 160000 index 00000000..ff0bc984 --- /dev/null +++ b/bosh-deployment @@ -0,0 +1 @@ +Subproject commit ff0bc984c72d5053656c666ae7d7788a78f6b7a8 diff --git a/terraform/base/bosh.tf b/terraform/base/bosh.tf index ef0eeb19..93fe87d0 100644 --- a/terraform/base/bosh.tf +++ b/terraform/base/bosh.tf @@ -110,6 +110,15 @@ resource "aws_security_group_rule" "bosh_uaa_concourse" { source_security_group_id = "${aws_security_group.concourse.id}" } +resource "aws_security_group_rule" "bosh_mbus_jumpbox" { + security_group_id = "${aws_security_group.bosh.id}" + type = "ingress" + protocol = "tcp" + from_port = 6868 + to_port = 6868 + source_security_group_id = "${aws_security_group.jumpbox.id}" +} + resource "aws_security_group_rule" "bosh_uaa_jumpbox" { security_group_id = "${aws_security_group.bosh.id}" type = "ingress" @@ -238,4 +247,15 @@ resource "aws_security_group_rule" "allow_postgres_from_jumpbox" { to_port = "${var.bosh_rds_port}" source_security_group_id = "${aws_security_group.jumpbox.id}" description = "Provide ingress PostgreSQL traffic from jumpbox" +} + +# BOSH default key pair +resource "tls_private_key" "bosh" { + algorithm = "RSA" + rsa_bits = "2048" +} + +resource "aws_key_pair" "bosh" { + key_name = "${var.environment}-bosh" + public_key = "${tls_private_key.bosh.public_key_openssh}" } \ No newline at end of file diff --git a/terraform/base/jumpbox.tf b/terraform/base/jumpbox.tf index c4140377..94a54f27 100644 --- a/terraform/base/jumpbox.tf +++ b/terraform/base/jumpbox.tf @@ -28,6 +28,16 @@ resource "aws_security_group_rule" "allow_jumpbox_to_postgres" { description = "Provide egress PostgreSQL traffic from jumpbox" } +resource "aws_security_group_rule" "allow_jumpbox_to_the_world" { + security_group_id = "${aws_security_group.jumpbox.id}" + type = "egress" + protocol = "tcp" + from_port = "0" + to_port = "65535" + cidr_blocks = ["0.0.0.0/0"] + description = "Sod it, let jumpbox see the world" +} + # SSH resource "tls_private_key" "jumpbox" { algorithm = "RSA" @@ -61,6 +71,7 @@ resource "aws_instance" "jumpbox" { key_name = "${aws_key_pair.jumpbox.key_name}" subnet_id = "${aws_subnet.public.0.id}" vpc_security_group_ids = ["${aws_security_group.jumpbox.id}"] + user_data = "${data.template_cloudinit_config.jumpbox.rendered}" tags { Name = "${var.environment}-jumpbox" @@ -71,4 +82,20 @@ resource "aws_instance" "jumpbox" { resource "aws_eip" "jumpbox" { instance = "${aws_instance.jumpbox.id}" vpc = true +} + +data "template_file" "jumpbox_cloudinit" { + template = "${file("${path.module}/templates/jumpbox_init.cfg")}" +} + +data "template_cloudinit_config" "jumpbox" { + + gzip = false + base64_encode = false + + part { + filename = "init.cfg" + content_type = "text/cloud-config" + content = "${data.template_file.jumpbox_cloudinit.rendered}" + } } \ No newline at end of file diff --git a/terraform/base/locals.tf b/terraform/base/locals.tf index ed0c18f9..60b73b54 100644 --- a/terraform/base/locals.tf +++ b/terraform/base/locals.tf @@ -5,5 +5,10 @@ locals { internal_subnets = "${var.cidr_blocks["internal"]}" services_subnets = "${var.cidr_blocks["services"]}" rds_subnets = "${var.cidr_blocks["rds"]}" - bosh_private_ip = "${cidrhost(local.public_subnets[0], 6)}" + + bosh_subnet_id = "${aws_subnet.public.*.id[var.bosh_availability_zone_index]}" + + bosh_subnet_cidr_block = "${aws_subnet.public.*.cidr_block[var.bosh_availability_zone_index]}" + bosh_private_ip = "${cidrhost(aws_subnet.public.*.cidr_block[var.bosh_availability_zone_index], 6)}" + bosh_gateway_ip = "${cidrhost(aws_subnet.public.*.cidr_block[var.bosh_availability_zone_index], 1)}" } diff --git a/terraform/base/outputs.tf b/terraform/base/outputs.tf index cf5170bb..d6a826a0 100644 --- a/terraform/base/outputs.tf +++ b/terraform/base/outputs.tf @@ -1,3 +1,7 @@ +output "vpc_dns_nameserver" { + value = "${cidrhost(aws_vpc.default.cidr_block, 2)}" +} + output "bosh_rds_security_group_id" { value = "${aws_security_group.bosh_rds.id}" } @@ -13,3 +17,41 @@ output "jumpbox_private_key" { output "jumpbox_public_ip" { value = "${aws_instance.jumpbox.public_ip}" } + +output "bosh_private_ip" { + value = "${local.bosh_private_ip}" +} + +output "bosh_subnet_cidr_block" { + value = "${local.bosh_subnet_cidr_block}" +} + +output "bosh_subnet_id" { + value = "${local.bosh_subnet_id}" +} + +output "bosh_gateway_ip" { + value = "${local.bosh_gateway_ip}" +} + +output "bosh_director_fqdn" { + # FIXME: DNS entry + value = "${local.bosh_private_ip}" +} + +output "bosh_iam_instance_profile" { + value = "${aws_iam_instance_profile.bosh.name}" +} + +output "bosh_key_name" { + value = "${aws_key_pair.bosh.key_name}" +} + +output "bosh_security_group_ids" { + value = [ "${aws_security_group.bosh.id}"] +} + +output "bosh_private_key" { + value = "${tls_private_key.bosh.private_key_pem}" +} + diff --git a/terraform/base/templates/jumpbox_init.cfg b/terraform/base/templates/jumpbox_init.cfg new file mode 100644 index 00000000..3a0863d3 --- /dev/null +++ b/terraform/base/templates/jumpbox_init.cfg @@ -0,0 +1,5 @@ +#cloud-config + +runcmd: + - sed -i -e '/^\(UseDNS\).*/{s//\1 no/;:a;n;:ba;q}' -e '$aUseDNS no' /etc/ssh/sshd_config + - restart ssh diff --git a/terraform/base/vars.tf b/terraform/base/vars.tf index ad2efc4b..bd44cab8 100644 --- a/terraform/base/vars.tf +++ b/terraform/base/vars.tf @@ -26,4 +26,8 @@ variable "cidr_blocks" { variable "bosh_rds_port" { default = "5432" +} + +variable "bosh_availability_zone_index" { + default = "0" } \ No newline at end of file diff --git a/terraform/databases/bosh.tf b/terraform/databases/bosh.tf deleted file mode 100644 index 4956366a..00000000 --- a/terraform/databases/bosh.tf +++ /dev/null @@ -1,13 +0,0 @@ -provider "postgresql" { - alias = "bosh" - host = "${var.bosh_db_host}" - port = "${var.bosh_db_port}" - username = "${var.bosh_db_username}" - password = "${var.bosh_rds_password}" - expected_version = "${var.bosh_db_engine_version}" -} - -resource "postgresql_database" "bosh" { - provider = "postgresql.bosh" - name = "bosh" -} diff --git a/terraform/databases/outputs.tf b/terraform/databases/outputs.tf deleted file mode 100644 index 165153b6..00000000 --- a/terraform/databases/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "bosh_database_name" { - value = "${postgresql_database.bosh.name}" -} diff --git a/terraform/databases/vars.tf b/terraform/databases/vars.tf deleted file mode 100644 index ff038ff4..00000000 --- a/terraform/databases/vars.tf +++ /dev/null @@ -1,10 +0,0 @@ -variable "bosh_db_host" {} -variable "bosh_db_username" {} - -variable "bosh_rds_password" {} - -variable "bosh_db_engine_version" {} -variable "bosh_db_port" {} - -variable "jumpbox_private_key" {} -variable "jumpbox_public_ip" {} \ No newline at end of file diff --git a/terraform/rds/outputs.tf b/terraform/rds/outputs.tf index 95e38b66..b38c5d63 100644 --- a/terraform/rds/outputs.tf +++ b/terraform/rds/outputs.tf @@ -2,6 +2,9 @@ output "bosh_db_type" { value = "${aws_db_instance.bosh_rds.engine}" } +output "bosh_rds_fqdn" { + value = "${aws_db_instance.bosh_rds.address}" +} output "bosh_db_host" { value = "${aws_db_instance.bosh_rds.address}" } From 5d36a07256efbf8f933cdcabcd470144023a383e Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Wed, 5 Sep 2018 12:00:05 +0100 Subject: [PATCH 07/71] Create cloud config (except for prometheus) --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 4b33396b..7614b683 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,9 @@ databases: ## create databases bosh: ## create bosh @bin/create_bosh.sh +cloud_config: ## Deploy cloud config + @bin/cloud_config.sh + decode_aws_error: ## Decode AWS message @aws sts decode-authorization-message --encoded-message ${DECODE_MESSAGE} | jq -r .DecodedMessage | jq . From 9e744688e1c0815e20e1fd781217f61051306f6f Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Wed, 5 Sep 2018 12:00:47 +0100 Subject: [PATCH 08/71] More create cloud config --- bin/bosh_credentials.sh | 5 +- bin/cloud_config.sh | 54 ++ terraform/base/cf.tf | 634 ++++++++++++++++++ terraform/base/concourse.tf | 5 + terraform/base/locals.tf | 1 + terraform/base/outputs.tf | 59 ++ .../templates/cloud_controller_policy.json | 30 + terraform/base/vars.tf | 3 +- terraform/rds/cf.tf | 48 ++ terraform/rds/outputs.tf | 23 + terraform/rds/vars.tf | 3 + 11 files changed, 860 insertions(+), 5 deletions(-) create mode 100755 bin/cloud_config.sh create mode 100644 terraform/base/templates/cloud_controller_policy.json create mode 100644 terraform/rds/cf.tf diff --git a/bin/bosh_credentials.sh b/bin/bosh_credentials.sh index 1253bf4e..8f8d2df2 100755 --- a/bin/bosh_credentials.sh +++ b/bin/bosh_credentials.sh @@ -41,12 +41,9 @@ trap cleanup EXIT export BOSH_CLIENT=admin export BOSH_CLIENT_SECRET=$(bosh int --path /admin_password "data/$ENVIRONMENT-bosh-variables.yml") -export BOSH_ENVIRONMENT="$ENVIRONMENT" +export BOSH_ENVIRONMENT=$(bin/outputs.sh base | jq -r .bosh_private_ip) export BOSH_ALL_PROXY=ssh+socks5://ubuntu@$JUMPBOX_IP:22?private-key=$JUMPBOX_KEY -DIRECTOR_IP=$(bin/outputs.sh base | jq -r .bosh_private_ip) -bosh alias-env "${ENVIRONMENT}" -e "${DIRECTOR_IP}" - if [ -n "$COMMAND" ]; then ""$COMMAND"" else diff --git a/bin/cloud_config.sh b/bin/cloud_config.sh new file mode 100755 index 00000000..d59fada5 --- /dev/null +++ b/bin/cloud_config.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set -euo pipefail + +: $ENVIRONMENT + +output() { + FILE=$1 + QUERY=$2 + + bin/outputs.sh $FILE | jq -r "$QUERY" +} + +bin/bosh_credentials.sh -e $ENVIRONMENT bosh update-cloud-config -n \ + ./cloud-config/cf/cloud-config.yml \ + -o ./operations/cloud-config/router-extensions.yml \ + -o ./operations/cloud-config/cf-scheduler-extensions.yml \ + -o ./operations/cloud-config/cf-s3-blobstore.yml \ + -o ./operations/cloud-config/cf-rds-sec-group.yml \ + -v az1="$(jq -r .availability_zones[0] < data/$ENVIRONMENT.tfvars)" \ + -v az2="$(jq -r .availability_zones[1] < data/$ENVIRONMENT.tfvars)" \ + -v az3="$(jq -r .availability_zones[2] < data/$ENVIRONMENT.tfvars)" \ + -v private_subnet_az1_gateway="$(output base .internal_subnet_gateway_ips[0])" \ + -v private_subnet_az2_gateway="$(output base .internal_subnet_gateway_ips[1])" \ + -v private_subnet_az3_gateway="$(output base .internal_subnet_gateway_ips[2])" \ + -v reserved_az1_cidr="$(output base .internal_subnet_reserved_cidr_blocks[0])" \ + -v reserved_az2_cidr="$(output base .internal_subnet_reserved_cidr_blocks[1])" \ + -v reserved_az3_cidr="$(output base .internal_subnet_reserved_cidr_blocks[2])" \ + -v private_dns_nameserver="$(output base .vpc_dns_nameserver)" \ + -v internal_security_group="$(output base .cf_internal_security_group_id)" \ + -v private_subnet_az1_id="$(output base .internal_subnet_ids[0])" \ + -v private_subnet_az2_id="$(output base .internal_subnet_ids[1])" \ + -v private_subnet_az3_id="$(output base .internal_subnet_ids[2])" \ + -v private_subnet_az1_cidr="$(output base .internal_subnet_cidr_blocks[0])" \ + -v private_subnet_az2_cidr="$(output base .internal_subnet_cidr_blocks[1])" \ + -v private_subnet_az3_cidr="$(output base .internal_subnet_cidr_blocks[2])" \ + -v cf-router-target-group-name="$(output base .cf_router_target_group_name)" \ + -v cf-router-lb-internal-security-group-id="$(output base .cf_router_lb_internal_security_group_id)" \ + -v cf-internal-security-group-id="$(output base .cf_internal_security_group_id)" \ + -v cf-ssh-internal="$(output base .cf_ssh_internal_security_group_id)" \ + -v cf-ssh-lb="$(output base .cf_ssh_lb_name)" \ + -v cf_s3_iam_instance_profile="$(output base .cf_s3_iam_instance_profile)" \ + -v cf-rds-client-security-group="$(output base .cf_rds_client_security_group_id)" + +# -o ./operations/cloud-config/prometheus.yml \ +# -v prometheus_subnet_az1_cidr="$(jq -r .prometheus_subnet_az1_cidr < prometheus-vars.json)" \ +# -v prometheus_subnet_az1_id="$(jq -r .prometheus_subnet_az1_id < prometheus-vars.json)" \ +# -v prometheus_security_group="$(jq -r .prometheus_security_group_id < prometheus-vars.json)" \ +# -v prometheus_subnet_az1_gateway="$(jq -r .prometheus_subnet_az1_cidr < prometheus-vars.json | sed 's#0/24#1#')" \ +# -v grafana_target_group_name="$(jq -r .grafana_target_group_name < prometheus-vars.json)" \ +# -v prometheus_target_group_name="$(jq -r .prometheus_target_group_name < prometheus-vars.json)" \ +# -v alertmanager_target_group_name="$(jq -r .alertmanager_target_group_name < prometheus-vars.json)" + +bin/bosh_credentials.sh -e $ENVIRONMENT bosh cloud-config > data/$ENVIRONMENT-cloud-config.yml \ No newline at end of file diff --git a/terraform/base/cf.tf b/terraform/base/cf.tf index e69de29b..9acbebb5 100644 --- a/terraform/base/cf.tf +++ b/terraform/base/cf.tf @@ -0,0 +1,634 @@ +# Load Balancers +resource "aws_lb" "cf" { + name = "${var.environment}-cf-alb" + subnets = ["${aws_subnet.public.*.id}"] + security_groups = ["${aws_security_group.cf_alb.id}"] + load_balancer_type = "application" + internal = false + enable_cross_zone_load_balancing = true + + tags { + Name = "${var.environment}-cf-alb" + Environment = "${var.environment}" + } + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_lb_listener" "cf_443" { + depends_on = ["aws_acm_certificate_validation.cf"] + load_balancer_arn = "${aws_lb.cf.arn}" + port = "443" + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-2015-05" + certificate_arn = "${aws_acm_certificate.cf.arn}" + + default_action { + target_group_arn = "${aws_lb_target_group.cf.arn}" + type = "forward" + } + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_lb_listener" "cf_4443" { + depends_on = ["aws_acm_certificate_validation.cf"] + load_balancer_arn = "${aws_lb.cf.arn}" + port = "4443" + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-2015-05" + certificate_arn = "${aws_acm_certificate.cf.arn}" + + default_action { + target_group_arn = "${aws_lb_target_group.cf.arn}" + type = "forward" + } + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_lb_target_group" "cf" { + name = "${var.environment}-cf-target-group" + port = 443 + protocol = "HTTPS" + vpc_id = "${aws_vpc.default.id}" + + health_check { + path = "/health" + port = 8080 + protocol = "HTTP" + healthy_threshold = 2 + unhealthy_threshold = 2 + timeout = 3 + interval = 5 + } + + tags { + Name = "${var.environment}-cf-target-group" + Environment = "${var.environment}" + } + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_elb" "cf-ssh-lb" { + name = "${var.environment}-cf-ssh-lb" + internal = false + subnets = ["${aws_subnet.public.*.id}"] + cross_zone_load_balancing = "true" + + security_groups = ["${aws_security_group.cf_ssh_lb.id}"] + + health_check { + target = "TCP:2222" + healthy_threshold = 2 + unhealthy_threshold = 2 + timeout = 3 + interval = 30 + } + + listener { + instance_port = 2222 + instance_protocol = "tcp" + lb_port = 2222 + lb_protocol = "tcp" + } + + tags { + Name = "${var.environment}-cf-ssh-lb" + Environment = "${var.environment}" + } + + lifecycle { + create_before_destroy = true + } +} +# Certificates +resource "aws_acm_certificate" "cf" { + domain_name = "${var.environment}.${var.parent_dns_zone}" + validation_method = "DNS" + subject_alternative_names = ["*.system.${var.environment}.${var.parent_dns_zone}", "*.apps.${var.environment}.${var.parent_dns_zone}"] + + tags { + Name = "${var.environment}-cf-system-cert" + Environment = "${var.environment}" + } +} + +resource "aws_route53_record" "cf_validation" { + count = 3 + + name = "${lookup(aws_acm_certificate.cf.domain_validation_options[count.index], "resource_record_name")}" + type = "${lookup(aws_acm_certificate.cf.domain_validation_options[count.index], "resource_record_type")}" + zone_id = "${aws_route53_zone.child_zone.zone_id}" + records = ["${lookup(aws_acm_certificate.cf.domain_validation_options[count.index], "resource_record_value")}"] + ttl = 30 +} + +resource "aws_acm_certificate_validation" "cf" { + certificate_arn = "${aws_acm_certificate.cf.arn}" + validation_record_fqdns = ["${aws_route53_record.cf_validation.*.fqdn}"] +} + +resource "aws_route53_record" "cf_system" { + zone_id = "${aws_route53_zone.child_zone.zone_id}" + name = "*.system.${var.environment}.${var.parent_dns_zone}" + type = "CNAME" + ttl = "30" + + records = ["${aws_lb.cf.dns_name}"] +} + +resource "aws_route53_record" "cf_apps" { + zone_id = "${aws_route53_zone.child_zone.zone_id}" + name = "*.apps.${var.environment}.${var.parent_dns_zone}" + type = "CNAME" + ttl = "30" + + records = ["${aws_lb.cf.dns_name}"] +} + +resource "aws_route53_record" "cf_ssh" { + zone_id = "${aws_route53_zone.child_zone.zone_id}" + name = "ssh.system.${var.environment}.${var.parent_dns_zone}" + type = "CNAME" + ttl = "30" + + records = ["${aws_elb.cf-ssh-lb.dns_name}"] +} + +# Security Group + +resource "aws_security_group" "cf_alb" { + name = "${var.environment}_cf_alb_security_group" + description = "CF public access" + vpc_id = "${aws_vpc.default.id}" + + tags { + Name = "${var.environment}-cf-alb-security-group" + Environment = "${var.environment}" + } +} + +resource "aws_security_group_rule" "cf_alb_https" { + security_group_id = "${aws_security_group.cf_alb.id}" + type = "ingress" + protocol = "tcp" + from_port = 443 + to_port = 443 + cidr_blocks = ["${local.loadbalancer_whitelist}"] + description = "Whitelist administrator access for HTTPS" +} + +resource "aws_security_group_rule" "cf_alb_4443" { + security_group_id = "${aws_security_group.cf_alb.id}" + type = "ingress" + protocol = "tcp" + from_port = 4443 + to_port = 4443 + cidr_blocks = ["${local.loadbalancer_whitelist}"] + description = "Whitelist administrator access for HTTPS/4443" +} + +resource "aws_security_group_rule" "cf_alb_egress_internal" { + security_group_id = "${aws_security_group.cf_alb.id}" + type = "egress" + protocol = "-1" + from_port = 0 + to_port = 0 + cidr_blocks = ["0.0.0.0/0"] + description = "Allow access to (eventually) internal services" +} + +resource "aws_security_group" "internal" { + name = "${var.environment}_cf_internal_security_group" + description = "Internal" + vpc_id = "${aws_vpc.default.id}" + + tags { + Name = "${var.environment}-cf-internal-security-group" + Environment = "${var.environment}" + } +} + +resource "aws_security_group_rule" "internal_rule_tcp" { + security_group_id = "${aws_security_group.internal.id}" + type = "ingress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + self = true +} + +resource "aws_security_group_rule" "internal_rule_udp" { + security_group_id = "${aws_security_group.internal.id}" + type = "ingress" + protocol = "udp" + from_port = 0 + to_port = 65535 + self = true +} + +resource "aws_security_group_rule" "internal_rule_icmp" { + security_group_id = "${aws_security_group.internal.id}" + type = "ingress" + protocol = "icmp" + from_port = -1 + to_port = -1 + self = true +} + +resource "aws_security_group_rule" "internal_rule_allow_internet" { + security_group_id = "${aws_security_group.internal.id}" + type = "egress" + protocol = "-1" + from_port = 0 + to_port = 0 + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "cf_from_bosh_rule_tcp_ssh" { + security_group_id = "${aws_security_group.internal.id}" + type = "ingress" + protocol = "tcp" + from_port = 22 + to_port = 22 + source_security_group_id = "${aws_security_group.bosh.id}" +} + +resource "aws_security_group_rule" "cf_from_bosh_rule_tcp_bosh_agent" { + security_group_id = "${aws_security_group.internal.id}" + type = "ingress" + protocol = "tcp" + from_port = 6868 + to_port = 6868 + source_security_group_id = "${aws_security_group.bosh.id}" +} + +resource "aws_security_group_rule" "cf_from_jumpbox_rule_tcp_ssh" { + security_group_id = "${aws_security_group.internal.id}" + type = "ingress" + protocol = "tcp" + from_port = 22 + to_port = 22 + source_security_group_id = "${aws_security_group.jumpbox.id}" +} + +resource "aws_security_group_rule" "bosh_ssh_cf" { + security_group_id = "${aws_security_group.bosh.id}" + type = "egress" + protocol = "tcp" + from_port = 22 + to_port = 22 + source_security_group_id = "${aws_security_group.internal.id}" +} + +resource "aws_security_group_rule" "bosh_mbus_cf" { + security_group_id = "${aws_security_group.bosh.id}" + type = "egress" + protocol = "tcp" + from_port = 6868 + to_port = 6868 + source_security_group_id = "${aws_security_group.internal.id}" +} + +resource "aws_security_group_rule" "bosh_tcp_from_cf" { + security_group_id = "${aws_security_group.bosh.id}" + type = "ingress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + source_security_group_id = "${aws_security_group.internal.id}" +} + +resource "aws_security_group_rule" "bosh_udp_from_cf" { + security_group_id = "${aws_security_group.bosh.id}" + type = "ingress" + protocol = "udp" + from_port = 0 + to_port = 65535 + source_security_group_id = "${aws_security_group.internal.id}" +} + +resource "aws_security_group_rule" "jumpbox_ssh_cf" { + security_group_id = "${aws_security_group.jumpbox.id}" + type = "egress" + protocol = "tcp" + from_port = 22 + to_port = 22 + source_security_group_id = "${aws_security_group.internal.id}" +} + +resource "aws_security_group" "cf_router_lb_internal_security_group" { + name = "${var.environment}_cf_router_lb_internal_security_group" + description = "CF Router Internal" + vpc_id = "${aws_vpc.default.id}" + + ingress { + security_groups = ["${aws_security_group.cf_alb.id}"] + protocol = "tcp" + from_port = 443 + to_port = 443 + } + + ingress { + security_groups = ["${aws_security_group.cf_alb.id}"] + protocol = "tcp" + from_port = 8080 + to_port = 8080 + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags { + Name = "${var.environment}-cf-router-lb-internal-security-group" + Environment = "${var.environment}" + } +} + +resource "aws_security_group" "cf_ssh_lb" { + name = "${var.environment}_cf_ssh_lb" + description = "CF SSH traffic from load balancer" + vpc_id = "${aws_vpc.default.id}" + + tags { + Name = "${var.environment}-cf-ssh-lb-security-group" + Environment = "${var.environment}" + } +} + +resource "aws_security_group_rule" "allow_tcp_2222_from_whitelist" { + security_group_id = "${aws_security_group.cf_ssh_lb.id}" + type = "ingress" + protocol = "tcp" + from_port = 2222 + to_port = 2222 + cidr_blocks = ["${var.ingress_whitelist}"] + description = "Allow SSH proxy traffic from whitelist" +} + +resource "aws_security_group_rule" "allow_tcp_2222_from_nat_gateways" { + security_group_id = "${aws_security_group.cf_ssh_lb.id}" + type = "ingress" + protocol = "tcp" + from_port = 2222 + to_port = 2222 + cidr_blocks = ["${formatlist("%s/32", aws_nat_gateway.nat.*.public_ip)}"] + description = "Allow SSH proxy traffic from internal components" +} + +resource "aws_security_group_rule" "allow_tcp_2222_from_concourse" { + security_group_id = "${aws_security_group.cf_ssh_lb.id}" + type = "ingress" + protocol = "tcp" + from_port = 2222 + to_port = 2222 + cidr_blocks = ["${aws_eip.concourse.public_ip}/32"] + description = "Allow SSH proxy traffic from Concourse" +} + +resource "aws_security_group_rule" "allow_tcp_2222_to_proxies" { + security_group_id = "${aws_security_group.cf_ssh_lb.id}" + type = "egress" + protocol = "tcp" + from_port = 2222 + to_port = 2222 + source_security_group_id = "${aws_security_group.cf_ssh_internal.id}" + description = "Provide egress SSH traffic" +} + +resource "aws_security_group" "cf_ssh_internal" { + name = "${var.environment}_cf_ssh_internal" + description = "CF SSH internal access" + vpc_id = "${aws_vpc.default.id}" + + tags { + Name = "${var.environment}-cf-ssh-internal-security-group" + Environment = "${var.environment}" + } +} + +resource "aws_security_group_rule" "allow_tcp_2222_from_ssh_lb" { + security_group_id = "${aws_security_group.cf_ssh_internal.id}" + type = "ingress" + protocol = "tcp" + from_port = 2222 + to_port = 2222 + source_security_group_id = "${aws_security_group.cf_ssh_lb.id}" + description = "Provide ingress SSH traffic" +} + +resource "aws_security_group" "cf_rds" { + name = "${var.environment}_cf_rds_security_group" + description = "CF rds access" + vpc_id = "${aws_vpc.default.id}" + + tags { + Name = "${var.environment}-cf-rds-security-group" + Environment = "${var.environment}" + } +} + +resource "aws_security_group_rule" "allow_mysql_from_concourse" { + security_group_id = "${aws_security_group.cf_rds.id}" + type = "ingress" + protocol = "tcp" + from_port = 3306 + to_port = 3306 + source_security_group_id = "${aws_security_group.concourse.id}" + description = "Provide ingress MySQL traffic from Concourse" +} + +resource "aws_security_group_rule" "allow_mysql_from_cf_internal_clients" { + security_group_id = "${aws_security_group.cf_rds.id}" + type = "ingress" + protocol = "tcp" + from_port = 3306 + to_port = 3306 + source_security_group_id = "${aws_security_group.cf_rds_client.id}" + description = "Provide ingress MySQL traffic from CF" +} + +resource "aws_security_group" "cf_rds_client" { + name = "${var.environment}_cf_rds_client_security_group" + description = "CF rds consumer" + vpc_id = "${aws_vpc.default.id}" + + tags { + Name = "${var.environment}-cf-rds-client-security-group" + Environment = "${var.environment}" + } +} + + +# IAM + +resource "aws_iam_instance_profile" "s3_blobstore" { + name = "${var.environment}_s3_blobstore" + role = "${aws_iam_role.s3_blobstore.name}" +} + +resource "aws_iam_role" "s3_blobstore" { + name = "${var.environment}_s3_blobstore_role" + path = "/" + + assume_role_policy = < Date: Wed, 5 Sep 2018 14:19:35 +0100 Subject: [PATCH 09/71] Create CF databases. Deploy configs. --- .gitmodules | 3 ++ Makefile | 6 +++ bin/cloud_config.sh | 6 ++- bin/databases.sh | 50 +++++++++++++++------ bin/deploy_cf.sh | 92 +++++++++++++++++++++++++++++++++++++++ bin/jumpbox_ssh.sh | 24 +++++----- bin/runtime_config.sh | 15 +++++++ bin/stemcells.sh | 19 ++++++++ cf-deployment | 1 + terraform/base/cf.tf | 10 ++++- terraform/base/jumpbox.tf | 10 +++++ terraform/base/locals.tf | 8 ++-- terraform/base/outputs.tf | 33 ++++++++++++-- terraform/base/vars.tf | 3 ++ terraform/rds/outputs.tf | 4 ++ 15 files changed, 247 insertions(+), 37 deletions(-) create mode 100755 bin/deploy_cf.sh create mode 100755 bin/runtime_config.sh create mode 100755 bin/stemcells.sh create mode 160000 cf-deployment diff --git a/.gitmodules b/.gitmodules index d6d36bc9..681a60e6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "bosh-deployment"] path = bosh-deployment url = https://github.com/cloudfoundry/bosh-deployment.git +[submodule "cf-deployment"] + path = cf-deployment + url = https://github.com/cloudfoundry/cf-deployment.git diff --git a/Makefile b/Makefile index 7614b683..ac80afc4 100644 --- a/Makefile +++ b/Makefile @@ -38,9 +38,15 @@ databases: ## create databases bosh: ## create bosh @bin/create_bosh.sh +runtime_config: ## Deploy runtime config + @bin/runtime_config.sh + cloud_config: ## Deploy cloud config @bin/cloud_config.sh +stemcells: ## Upload stemcells + @bin/stemcells.sh + decode_aws_error: ## Decode AWS message @aws sts decode-authorization-message --encoded-message ${DECODE_MESSAGE} | jq -r .DecodedMessage | jq . diff --git a/bin/cloud_config.sh b/bin/cloud_config.sh index d59fada5..3169005a 100755 --- a/bin/cloud_config.sh +++ b/bin/cloud_config.sh @@ -11,7 +11,9 @@ output() { bin/outputs.sh $FILE | jq -r "$QUERY" } -bin/bosh_credentials.sh -e $ENVIRONMENT bosh update-cloud-config -n \ +BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" + +$BOSH update-cloud-config -n \ ./cloud-config/cf/cloud-config.yml \ -o ./operations/cloud-config/router-extensions.yml \ -o ./operations/cloud-config/cf-scheduler-extensions.yml \ @@ -51,4 +53,4 @@ bin/bosh_credentials.sh -e $ENVIRONMENT bosh update-cloud-config -n \ # -v prometheus_target_group_name="$(jq -r .prometheus_target_group_name < prometheus-vars.json)" \ # -v alertmanager_target_group_name="$(jq -r .alertmanager_target_group_name < prometheus-vars.json)" -bin/bosh_credentials.sh -e $ENVIRONMENT bosh cloud-config > data/$ENVIRONMENT-cloud-config.yml \ No newline at end of file +$BOSH cloud-config > data/$ENVIRONMENT-cloud-config.yml \ No newline at end of file diff --git a/bin/databases.sh b/bin/databases.sh index 77447e21..7b83d47d 100755 --- a/bin/databases.sh +++ b/bin/databases.sh @@ -4,35 +4,59 @@ set -euo pipefail : $ENVIRONMENT -LOCAL_PORT=30201 -DB_USER=$(bin/outputs.sh rds | jq -r .bosh_db_username) -DB_HOST=$(bin/outputs.sh rds | jq -r .bosh_db_host) -DB_PORT=$(bin/outputs.sh rds | jq -r .bosh_db_port) -DB_PASS=$(bin/outputs.sh rds | jq -r .bosh_rds_password) + +BOSH_DB_USER=$(bin/outputs.sh rds | jq -r .bosh_db_username) +BOSH_DB_HOST=$(bin/outputs.sh rds | jq -r .bosh_db_host) +BOSH_DB_PORT=$(bin/outputs.sh rds | jq -r .bosh_db_port) +BOSH_DB_PASS=$(bin/outputs.sh rds | jq -r .bosh_rds_password) + +CF_DB_USER=$(bin/outputs.sh rds | jq -r .cf_db_username) +CF_DB_HOST=$(bin/outputs.sh rds | jq -r .cf_db_host) +CF_DB_PORT=$(bin/outputs.sh rds | jq -r .cf_db_port) +CF_DB_PASS=$(bin/outputs.sh rds | jq -r .cf_rds_password) + + JUMPBOX_IP=$(bin/outputs.sh | jq -r .jumpbox_public_ip) KEYFILE=~/.ssh/$ENVIRONMENT.jumpbox.$$.pem bin/outputs.sh | jq -r .jumpbox_private_key >$KEYFILE chmod 600 $KEYFILE +BOSH_LOCAL_PORT=30201 +CF_LOCAL_PORT=30202 + cleanup() { rm -f $KEYFILE - kill $(ps -ef | awk "/ssh.*$LOCAL_PORT/ && ! /awk/ { print \$2 }") + kill $(ps -ef | awk "/ssh.*$BOSH_LOCAL_PORT/ && ! /awk/ { print \$2 }") + kill $(ps -ef | awk "/ssh.*$CF_LOCAL_PORT/ && ! /awk/ { print \$2 }") } -ssh -fN -L $LOCAL_PORT:$DB_HOST:$DB_PORT ubuntu@$JUMPBOX_IP -i $KEYFILE +ssh -fN -L $BOSH_LOCAL_PORT:$BOSH_DB_HOST:$BOSH_DB_PORT ubuntu@$JUMPBOX_IP -i $KEYFILE +ssh -fN -L $CF_LOCAL_PORT:$CF_DB_HOST:$CF_DB_PORT ubuntu@$JUMPBOX_IP -i $KEYFILE trap cleanup EXIT sql() { - PGPASSWORD=$DB_PASS psql -h localhost -p $LOCAL_PORT -U $DB_USER -d paastest -c "$*" + PGPASSWORD=$BOSH_DB_PASS psql -h localhost -p $BOSH_LOCAL_PORT -U $BOSH_DB_USER -d paastest -c "$*" } exists() { - PGPASSWORD=$DB_PASS psql -h localhost -p $LOCAL_PORT -U $DB_USER -d paastest -l | grep -q $1 + PGPASSWORD=$BOSH_DB_PASS psql -h localhost -p $BOSH_LOCAL_PORT -U $BOSH_DB_USER -d paastest -l | grep -q $1 } -create() { +create_bosh() { exists $1 || { echo "create database $1"; sql "create database $1"; } } -create bosh -create uaa -create credhub +create_cf() { + echo "create database if not exists $1" | MYSQL_PWD=$CF_DB_PASS mysql --protocol=tcp -h localhost -P $CF_LOCAL_PORT -u $CF_DB_USER +} + +create_bosh bosh +create_bosh uaa +create_bosh credhub + +create_cf uaa +create_cf cc +create_cf bbs +create_cf routing +create_cf policy_server +create_cf silk_controller +create_cf locket \ No newline at end of file diff --git a/bin/deploy_cf.sh b/bin/deploy_cf.sh new file mode 100755 index 00000000..7708c48f --- /dev/null +++ b/bin/deploy_cf.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +set -euo pipefail + +: $ENVIRONMENT + +output() { + FILE=$1 + QUERY=$2 + + bin/outputs.sh $FILE | jq -r "$QUERY" +} + +BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" + +DOMAIN="$(output base .domain)" +SYSTEM_DOMAIN="system.${DOMAIN}" +APPS_DOMAIN="apps.${DOMAIN}" + +CF_DB_ENDPOINT="$(output rds .cf_rds_fqdn)" +CF_DB_USERNAME="$(output rds .cf_db_username)" +CF_DB_PASSWORD="$(output rds .cf_rds_password)" + +if [[ ${ENVIRONMENT} = "eng"* ]]; then + profile="engineering" +else + profile="${ENVIRONMENT}" +fi +instance_count_file=./profiles/${profile}/instance-count.yml + +bosh int \ + ./cf-deployment/cf-deployment.yml \ + --vars-store data/$ENVIRONMENT-cf-variables.yml \ + --vars-file="${instance_count_file}" \ + -o cf-deployment/operations/aws.yml \ + -o cf-deployment/operations/override-app-domains.yml \ + -o cf-deployment/operations/use-external-blobstore.yml \ + -o cf-deployment/operations/use-external-dbs.yml \ + -o ./operations/bosh/tags.yml \ + -o ./operations/cf/stemcells.yml \ + -o ./operations/cf/router-sec-group.yml \ + -o ./operations/cf/scheduler.yml \ + -o ./operations/cf/s3_blobstore_with_kms_and_iam.yml \ + -o ./operations/cf/azs.yml \ + -o ./operations/cf/instance-counts.yml \ + -o ./operations/cf/rds-access.yml \ + -o ./operations/cf/uaa-clients.yml \ + -o ./operations/cf/test-user.yml \ + -v environment="${ENVIRONMENT}" \ + -v region="$(jq -r .region < data/$ENVIRONMENT.tfvars)" \ + -v system_domain="${SYSTEM_DOMAIN}" \ + -v app_domains="[${APPS_DOMAIN}]" \ + -v smoke_test_app_domain="${APPS_DOMAIN}" \ + -v buildpack_directory_key="$(output base .cf_buildpacks_bucket_name)" \ + -v droplet_directory_key="$(output base .cf_droplets_bucket_name)" \ + -v app_package_directory_key="$(output base .cf_packages_bucket_name)" \ + -v resource_directory_key="$(output base .cf_resource_pool_bucket_name)" \ + -v cf_blobstore_s3_kms_key_id="$(output base .cf_blobstore_s3_kms_key_id)" \ + -v external_database_type="$(output rds .cf_db_type)" \ + -v external_database_port="$(output rds .cf_db_port)" \ + -v external_uaa_database_name="uaa" \ + -v external_uaa_database_address="${CF_DB_ENDPOINT}" \ + -v external_uaa_database_password="${CF_DB_PASSWORD}" \ + -v external_uaa_database_username="${CF_DB_USERNAME}" \ + -v external_cc_database_name="cc" \ + -v external_cc_database_address="${CF_DB_ENDPOINT}" \ + -v external_cc_database_password="${CF_DB_PASSWORD}" \ + -v external_cc_database_username="${CF_DB_USERNAME}" \ + -v external_bbs_database_name="bbs" \ + -v external_bbs_database_address="${CF_DB_ENDPOINT}" \ + -v external_bbs_database_password="${CF_DB_PASSWORD}" \ + -v external_bbs_database_username="${CF_DB_USERNAME}" \ + -v external_routing_api_database_name="routing_api" \ + -v external_routing_api_database_address="${CF_DB_ENDPOINT}" \ + -v external_routing_api_database_password="${CF_DB_PASSWORD}" \ + -v external_routing_api_database_username="${CF_DB_USERNAME}" \ + -v external_policy_server_database_name="policy_server" \ + -v external_policy_server_database_address="${CF_DB_ENDPOINT}" \ + -v external_policy_server_database_password="${CF_DB_PASSWORD}" \ + -v external_policy_server_database_username="${CF_DB_USERNAME}" \ + -v external_silk_controller_database_name="silk_controller" \ + -v external_silk_controller_database_address="${CF_DB_ENDPOINT}" \ + -v external_silk_controller_database_password="${CF_DB_PASSWORD}" \ + -v external_silk_controller_database_username="${CF_DB_USERNAME}" \ + -v external_locket_database_name="locket" \ + -v external_locket_database_address="${CF_DB_ENDPOINT}" \ + -v external_locket_database_password="${CF_DB_PASSWORD}" \ + -v external_locket_database_username="${CF_DB_USERNAME}" \ + > cf-manifests/cf.yml +\ + +# -o prometheus-deployment-git/manifests/operators/cf/add-prometheus-uaa-clients.yml \ \ No newline at end of file diff --git a/bin/jumpbox_ssh.sh b/bin/jumpbox_ssh.sh index ef3bdcd7..4b1a0796 100755 --- a/bin/jumpbox_ssh.sh +++ b/bin/jumpbox_ssh.sh @@ -19,22 +19,18 @@ if [ "$1" = -e ]; then ENVIRONMENT=$2 shift 2 fi +export ENVIRONMENT -VARS=/var/tmp/tmp$$ -mkdir -p "$VARS" -trap 'rm -rf "$VARS"' EXIT +JUMPBOX_IP=$(bin/outputs.sh base | jq -r .jumpbox_public_ip) +JUMPBOX_KEY=~/.ssh/$ENVIRONMENT.jumpbox.$$.pem +bin/outputs.sh base | jq -r .jumpbox_private_key >$JUMPBOX_KEY +chmod 600 $JUMPBOX_KEY -set -euo pipefail - -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/vpc/tfstate.json" "$VARS/" -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/jumpbox/jumpbox-variables.yml" "$VARS/" -bosh int --path /jumpbox_ssh/private_key "$VARS/jumpbox-variables.yml" > "$VARS/jumpbox.key" -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "$VARS/tfstate.json" > "$VARS/vars.json" - -ZONE=$(jq -r '.dns_zone' < "$VARS/vars.json" | sed 's/\.$//') -JUMPBOX_TARGET="jumpbox.${ZONE}" -chmod 600 "$VARS/jumpbox.key" +cleanup() { + rm -f $JUMPBOX_KEY +} +trap cleanup EXIT set +e -ssh jumpbox@${JUMPBOX_TARGET} -i "${VARS}/jumpbox.key" $* +ssh ubuntu@${JUMPBOX_IP} -i $JUMPBOX_KEY $* diff --git a/bin/runtime_config.sh b/bin/runtime_config.sh new file mode 100755 index 00000000..d960b514 --- /dev/null +++ b/bin/runtime_config.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -euo pipefail + +: $ENVIRONMENT + +output() { + FILE=$1 + QUERY=$2 + + bin/outputs.sh $FILE | jq -r "$QUERY" +} + +BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" +$BOSH update-runtime-config bosh-deployment/runtime-configs/dns.yml --name dns -n diff --git a/bin/stemcells.sh b/bin/stemcells.sh new file mode 100755 index 00000000..d411054f --- /dev/null +++ b/bin/stemcells.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -euo pipefail + +: $ENVIRONMENT + +output() { + FILE=$1 + QUERY=$2 + + bin/outputs.sh $FILE | jq -r "$QUERY" +} +export IAAS_INFO=aws-xen-hvm +export STEMCELL_VERSION=$(bosh interpolate cf-deployment/cf-deployment.yml --path=/stemcells/alias=default/version) + +BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" +if ! $BOSH -n stemcells | grep -q "$STEMCELL_VERSION"; then + $BOSH upload-stemcell https://bosh.io/d/stemcells/bosh-${IAAS_INFO}-ubuntu-trusty-go_agent?v=${STEMCELL_VERSION} +fi \ No newline at end of file diff --git a/cf-deployment b/cf-deployment new file mode 160000 index 00000000..16897e00 --- /dev/null +++ b/cf-deployment @@ -0,0 +1 @@ +Subproject commit 16897e006dcae39e50b94734dc060be9ed54c934 diff --git a/terraform/base/cf.tf b/terraform/base/cf.tf index 9acbebb5..7039ef8d 100644 --- a/terraform/base/cf.tf +++ b/terraform/base/cf.tf @@ -461,7 +461,15 @@ resource "aws_security_group_rule" "allow_mysql_from_cf_internal_clients" { source_security_group_id = "${aws_security_group.cf_rds_client.id}" description = "Provide ingress MySQL traffic from CF" } - +resource "aws_security_group_rule" "allow_mysql_from_jumpbox" { + security_group_id = "${aws_security_group.cf_rds.id}" + type = "ingress" + protocol = "tcp" + from_port = "${var.cf_rds_port}" + to_port = "${var.cf_rds_port}" + source_security_group_id = "${aws_security_group.jumpbox.id}" + description = "Provide ingress MySQL traffic from jumpbox" +} resource "aws_security_group" "cf_rds_client" { name = "${var.environment}_cf_rds_client_security_group" description = "CF rds consumer" diff --git a/terraform/base/jumpbox.tf b/terraform/base/jumpbox.tf index 94a54f27..07c17b5d 100644 --- a/terraform/base/jumpbox.tf +++ b/terraform/base/jumpbox.tf @@ -28,6 +28,16 @@ resource "aws_security_group_rule" "allow_jumpbox_to_postgres" { description = "Provide egress PostgreSQL traffic from jumpbox" } +resource "aws_security_group_rule" "allow_jumpbox_to_mysql" { + security_group_id = "${aws_security_group.jumpbox.id}" + type = "egress" + protocol = "tcp" + from_port = "${var.cf_rds_port}" + to_port = "${var.cf_rds_port}" + source_security_group_id = "${aws_security_group.cf_rds.id}" + description = "Provide egress MySQL traffic from jumpbox" +} + resource "aws_security_group_rule" "allow_jumpbox_to_the_world" { security_group_id = "${aws_security_group.jumpbox.id}" type = "egress" diff --git a/terraform/base/locals.tf b/terraform/base/locals.tf index b417ede3..ba1b95df 100644 --- a/terraform/base/locals.tf +++ b/terraform/base/locals.tf @@ -6,10 +6,10 @@ locals { services_subnets = "${var.cidr_blocks["services"]}" rds_subnets = "${var.cidr_blocks["rds"]}" - bosh_subnet_id = "${aws_subnet.public.*.id[var.bosh_availability_zone_index]}" + bosh_subnet_id = "${aws_subnet.internal.*.id[var.bosh_availability_zone_index]}" - bosh_subnet_cidr_block = "${aws_subnet.public.*.cidr_block[var.bosh_availability_zone_index]}" - bosh_private_ip = "${cidrhost(aws_subnet.public.*.cidr_block[var.bosh_availability_zone_index], 6)}" - bosh_gateway_ip = "${cidrhost(aws_subnet.public.*.cidr_block[var.bosh_availability_zone_index], 1)}" + bosh_subnet_cidr_block = "${aws_subnet.internal.*.cidr_block[var.bosh_availability_zone_index]}" + bosh_private_ip = "${cidrhost(aws_subnet.internal.*.cidr_block[var.bosh_availability_zone_index], 6)}" + bosh_gateway_ip = "${cidrhost(aws_subnet.internal.*.cidr_block[var.bosh_availability_zone_index], 1)}" loadbalancer_whitelist = ["${concat(var.ingress_whitelist,formatlist("%s/32", concat(list(aws_eip.concourse.public_ip), aws_nat_gateway.nat.*.public_ip)))}"] } diff --git a/terraform/base/outputs.tf b/terraform/base/outputs.tf index dffd9318..78a2be9f 100644 --- a/terraform/base/outputs.tf +++ b/terraform/base/outputs.tf @@ -2,6 +2,10 @@ output "vpc_dns_nameserver" { value = "${cidrhost(aws_vpc.default.cidr_block, 2)}" } +output "domain" { + value = "${replace(aws_route53_zone.child_zone.name, "/\\.$/", "")}" +} + output "bosh_rds_security_group_id" { value = "${aws_security_group.bosh_rds.id}" } @@ -73,9 +77,9 @@ output "internal_subnet_gateway_ips" { # so you will have to change this if you add more AZs output "internal_subnet_reserved_cidr_blocks" { value = [ - "${cidrsubnet(aws_subnet.internal.*.cidr_block[0],8,0)}", - "${cidrsubnet(aws_subnet.internal.*.cidr_block[1],8,0)}", - "${cidrsubnet(aws_subnet.internal.*.cidr_block[2],8,0)}" + "${cidrsubnet(aws_subnet.internal.*.cidr_block[0],7,0)}", + "${cidrsubnet(aws_subnet.internal.*.cidr_block[1],7,0)}", + "${cidrsubnet(aws_subnet.internal.*.cidr_block[2],7,0)}" ] } @@ -114,3 +118,26 @@ output "cf_s3_iam_instance_profile" { output "cf_rds_client_security_group_id" { value = "${aws_security_group.cf_rds_client.id}" } + +output "cf_buildpacks_bucket_name" { + value = "${aws_s3_bucket.cf_buildpacks.id}" +} + +output "cf_droplets_bucket_name" { + value = "${aws_s3_bucket.cf_droplets.id}" +} + +output "cf_resource_pool_bucket_name" { + value = "${aws_s3_bucket.cf_resource_pool.id}" +} + +output "cf_packages_bucket_name" { + value = "${aws_s3_bucket.cf_packages.id}" +} +output "cf_blobstore_s3_kms_key_id" { + value = "${aws_kms_key.cf_blobstore_key.id}" +} + +output "cf_blobstore_s3_kms_key_arn" { + value = "${aws_kms_key.cf_blobstore_key.arn}" +} \ No newline at end of file diff --git a/terraform/base/vars.tf b/terraform/base/vars.tf index 767087a8..5b98c3f7 100644 --- a/terraform/base/vars.tf +++ b/terraform/base/vars.tf @@ -27,6 +27,9 @@ variable "cidr_blocks" { variable "bosh_rds_port" { default = "5432" } +variable "cf_rds_port" { + default = "3306" +} variable "bosh_availability_zone_index" { default = "0" diff --git a/terraform/rds/outputs.tf b/terraform/rds/outputs.tf index 4bee0a32..89fd92c2 100644 --- a/terraform/rds/outputs.tf +++ b/terraform/rds/outputs.tf @@ -40,6 +40,10 @@ output "cf_rds_fqdn" { value = "${aws_db_instance.cf_rds.address}" } +output "cf_db_host" { + value = "${aws_db_instance.cf_rds.address}" +} + output "cf_db_endpoint" { value = "${aws_db_instance.cf_rds.endpoint}" } From 381821e787f85e3b1c9aa163953352ececbb9214 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Wed, 5 Sep 2018 15:38:14 +0100 Subject: [PATCH 10/71] First cut of cf deploy --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index ac80afc4..000aabf0 100644 --- a/Makefile +++ b/Makefile @@ -47,6 +47,9 @@ cloud_config: ## Deploy cloud config stemcells: ## Upload stemcells @bin/stemcells.sh +cf: ## Deploy CF + @bin/deploy_cf.sh + decode_aws_error: ## Decode AWS message @aws sts decode-authorization-message --encoded-message ${DECODE_MESSAGE} | jq -r .DecodedMessage | jq . From b726e56e4e53b3ee62dc61803a8b09ff1826387d Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Wed, 5 Sep 2018 15:38:32 +0100 Subject: [PATCH 11/71] First cut of cf_deploy (continued) --- bin/create_bosh.sh | 6 ++++-- bin/databases.sh | 3 ++- bin/deploy_cf.sh | 22 +++++++++++++--------- cf-deployment | 2 +- operations/bosh/external-credhub-db.yml | 11 +++++++++++ operations/bosh/external-uaa-db.yml | 14 ++++++++++++++ operations/cf/instance-counts.yml | 4 ++++ operations/cf/rds-access.yml | 4 ++++ operations/cf/stemcells.yml | 4 ---- profiles/engineering/instance-count.yml | 1 + profiles/staging/instance-count.yml | 1 + 11 files changed, 55 insertions(+), 17 deletions(-) create mode 100644 operations/bosh/external-credhub-db.yml create mode 100644 operations/bosh/external-uaa-db.yml delete mode 100644 operations/cf/stemcells.yml diff --git a/bin/create_bosh.sh b/bin/create_bosh.sh index ba23774f..365398c0 100755 --- a/bin/create_bosh.sh +++ b/bin/create_bosh.sh @@ -13,10 +13,13 @@ bosh int \ bosh-deployment/bosh.yml \ --vars-store data/$ENVIRONMENT-bosh-variables.yml \ -o bosh-deployment/aws/cpi.yml \ + -o bosh-deployment/uaa.yml \ + -o bosh-deployment/credhub.yml \ -o bosh-deployment/misc/external-db.yml \ -o operations/bosh/tags.yml \ - -o operations/bosh/dns-resolution.yml \ -o operations/bosh/certificate.yml \ + -o operations/bosh/external-uaa-db.yml \ + -o operations/bosh/external-credhub-db.yml \ -v director_name=bosh \ -v internal_cidr="$(output base .bosh_subnet_cidr_block)" \ -v internal_gw="$(output base .bosh_gateway_ip)" \ @@ -41,7 +44,6 @@ bosh int \ -v secret_access_key="$AWS_SECRET_ACCESS_KEY" \ > data/$ENVIRONMENT-bosh-manifest.yml - JUMPBOX_IP=$(output base .jumpbox_public_ip) JUMPBOX_KEY=~/.ssh/$ENVIRONMENT.jumpbox.$$.pem output base .jumpbox_private_key >$JUMPBOX_KEY diff --git a/bin/databases.sh b/bin/databases.sh index 7b83d47d..3cf8ec68 100755 --- a/bin/databases.sh +++ b/bin/databases.sh @@ -59,4 +59,5 @@ create_cf bbs create_cf routing create_cf policy_server create_cf silk_controller -create_cf locket \ No newline at end of file +create_cf locket +create_cf credhub \ No newline at end of file diff --git a/bin/deploy_cf.sh b/bin/deploy_cf.sh index 7708c48f..85ae3274 100755 --- a/bin/deploy_cf.sh +++ b/bin/deploy_cf.sh @@ -21,13 +21,12 @@ CF_DB_ENDPOINT="$(output rds .cf_rds_fqdn)" CF_DB_USERNAME="$(output rds .cf_db_username)" CF_DB_PASSWORD="$(output rds .cf_rds_password)" -if [[ ${ENVIRONMENT} = "eng"* ]]; then - profile="engineering" -else - profile="${ENVIRONMENT}" -fi +profile="${ENVIRONMENT}" +[ -d "profiles/${ENVIRONMENT}" ] || profile="engineering" + instance_count_file=./profiles/${profile}/instance-count.yml +echo "Interpolating" bosh int \ ./cf-deployment/cf-deployment.yml \ --vars-store data/$ENVIRONMENT-cf-variables.yml \ @@ -35,9 +34,8 @@ bosh int \ -o cf-deployment/operations/aws.yml \ -o cf-deployment/operations/override-app-domains.yml \ -o cf-deployment/operations/use-external-blobstore.yml \ - -o cf-deployment/operations/use-external-dbs.yml \ + -o cf-deployment/operations/use-external-dbs.yml \ -o ./operations/bosh/tags.yml \ - -o ./operations/cf/stemcells.yml \ -o ./operations/cf/router-sec-group.yml \ -o ./operations/cf/scheduler.yml \ -o ./operations/cf/s3_blobstore_with_kms_and_iam.yml \ @@ -86,7 +84,13 @@ bosh int \ -v external_locket_database_address="${CF_DB_ENDPOINT}" \ -v external_locket_database_password="${CF_DB_PASSWORD}" \ -v external_locket_database_username="${CF_DB_USERNAME}" \ - > cf-manifests/cf.yml -\ + -v external_credhub_database_name="credhub" \ + -v external_credhub_database_address="${CF_DB_ENDPOINT}" \ + -v external_credhub_database_password="${CF_DB_PASSWORD}" \ + -v external_credhub_database_username="${CF_DB_USERNAME}" \ + > data/$ENVIRONMENT-cf-manifest.yml + +echo "Deploying" +$BOSH deploy -n -d cf data/$ENVIRONMENT-cf-manifest.yml # -o prometheus-deployment-git/manifests/operators/cf/add-prometheus-uaa-clients.yml \ \ No newline at end of file diff --git a/cf-deployment b/cf-deployment index 16897e00..2f466e23 160000 --- a/cf-deployment +++ b/cf-deployment @@ -1 +1 @@ -Subproject commit 16897e006dcae39e50b94734dc060be9ed54c934 +Subproject commit 2f466e231a47809190f9793f7c7e0260727f7ed8 diff --git a/operations/bosh/external-credhub-db.yml b/operations/bosh/external-credhub-db.yml new file mode 100644 index 00000000..5478e130 --- /dev/null +++ b/operations/bosh/external-credhub-db.yml @@ -0,0 +1,11 @@ +--- +- type: replace + path: /instance_groups/name=bosh/jobs/name=credhub/properties/credhub/data_storage + value: + type: ((external_db_adapter)) + host: ((external_db_host)) + port: ((external_db_port)) + username: ((external_db_user)) + password: ((external_db_password)) + database: credhub + require_tls: false diff --git a/operations/bosh/external-uaa-db.yml b/operations/bosh/external-uaa-db.yml new file mode 100644 index 00000000..f5acc1a9 --- /dev/null +++ b/operations/bosh/external-uaa-db.yml @@ -0,0 +1,14 @@ +--- +- type: replace + path: /instance_groups/name=bosh/jobs/name=uaa/properties/uaadb + value: + address: ((external_db_host)) + port: ((external_db_port)) + db_scheme: ((external_db_adapter)) + databases: + - tag: uaa + name: uaa + roles: + - tag: admin + name: ((external_db_user)) + password: ((external_db_password)) \ No newline at end of file diff --git a/operations/cf/instance-counts.yml b/operations/cf/instance-counts.yml index 714d5e38..9f8d35ed 100644 --- a/operations/cf/instance-counts.yml +++ b/operations/cf/instance-counts.yml @@ -38,3 +38,7 @@ - type: replace path: /instance_groups/name=tcp-router/instances value: ((tcp-router-instance-count)) +- type: replace + path: /instance_groups/name=credhub/instances + value: ((credhub-instance-count)) + diff --git a/operations/cf/rds-access.yml b/operations/cf/rds-access.yml index ddc300cd..e5782a40 100644 --- a/operations/cf/rds-access.yml +++ b/operations/cf/rds-access.yml @@ -17,3 +17,7 @@ - type: replace path: /instance_groups/name=diego-api/vm_extensions?/- value: cf-rds-sec-group + +- type: replace + path: /instance_groups/name=credhub/vm_extensions?/- + value: cf-rds-sec-group \ No newline at end of file diff --git a/operations/cf/stemcells.yml b/operations/cf/stemcells.yml deleted file mode 100644 index ab97681c..00000000 --- a/operations/cf/stemcells.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- type: replace - path: /stemcells/alias=default/version - value: "3586.16" \ No newline at end of file diff --git a/profiles/engineering/instance-count.yml b/profiles/engineering/instance-count.yml index e567b3a8..530b7de3 100644 --- a/profiles/engineering/instance-count.yml +++ b/profiles/engineering/instance-count.yml @@ -12,3 +12,4 @@ adapter-instance-count: 1 doppler-instance-count: 1 log-api-instance-count: 1 tcp-router-instance-count: 1 +credhub-instance-count: 1 diff --git a/profiles/staging/instance-count.yml b/profiles/staging/instance-count.yml index d29d9ada..cd620282 100644 --- a/profiles/staging/instance-count.yml +++ b/profiles/staging/instance-count.yml @@ -12,3 +12,4 @@ adapter-instance-count: 3 doppler-instance-count: 3 log-api-instance-count: 3 tcp-router-instance-count: 3 +credhub-instance-count: 3 From 28d6df621e56fc7956b397ca5c576d338736a0b8 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Wed, 5 Sep 2018 17:01:43 +0100 Subject: [PATCH 12/71] CF deploys OK, though it's not currently using bosh credhub correctly --- bin/credhub_credentials.sh | 65 ++++++++++++++++++++++++++++++++++++++ bin/databases.sh | 2 +- terraform/base/bosh.tf | 18 +++++++++++ 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100755 bin/credhub_credentials.sh diff --git a/bin/credhub_credentials.sh b/bin/credhub_credentials.sh new file mode 100755 index 00000000..1bd4b159 --- /dev/null +++ b/bin/credhub_credentials.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# This script will set up a connection to a BOSH environment for you: + +usage() { + echo "Usage: [source] $(basename $0) -e [command]" + echo + echo "Options:" + echo " -e Connect to this environment, e.g. eng2 (ENVIRONMENT)" + echo + echo " If you specify a command, it will run, then detach the proxy" + echo " otherwise, you will be placed in a shell with the environment set" + exit 1 +} + + +while getopts 'e:f' option; do + case $option in + e) ENVIRONMENT="$OPTARG";; + *) usage;; + esac +done +export ENVIRONMENT + +shift $((OPTIND-1)) +COMMAND=$* + +PROXY_PORT=30125 +JUMPBOX_IP=$(bin/outputs.sh base | jq -r .jumpbox_public_ip) +JUMPBOX_KEY=~/.ssh/$ENVIRONMENT.jumpbox.$$.pem +bin/outputs.sh base | jq -r .jumpbox_private_key >$JUMPBOX_KEY +chmod 600 $JUMPBOX_KEY +export CREDHUB_CA_CERT=/var/tmp/credhub_ca.$$.pem +bosh int --path /credhub_ca/ca "data/$ENVIRONMENT-bosh-variables.yml" >$CREDHUB_CA_CERT +export UAA_CA_CERT=/var/tmp/uaa_ca.$$.pem +bosh int --path /uaa_ssl/ca "data/$ENVIRONMENT-bosh-variables.yml" >$UAA_CA_CERT + +cleanup() { + rm -f $JUMPBOX_KEY + rm -f $CREDHUB_CA_CERT + rm -f $UAA_CA_CERT + + if [ -n "$STARTED_SSH" ]; then + echo "Killing the SSH proxy on $PROXY_PORT" + kill $(ps -ef | awk "/ssh -4 -D $PROXY_PORT/ && ! /awk/ { print \$2 }") + fi +} +trap cleanup EXIT + +if ! netstat -na | grep -q "127.0.0.1.${PROXY_PORT}.*LISTEN"; then + ssh -4 -D $PROXY_PORT -fNC ubuntu@${JUMPBOX_IP} -i "$JUMPBOX_KEY" && STARTED_SSH=true +fi + +export CREDHUB_SERVER=https://$(bin/outputs.sh base | jq -r .bosh_private_ip):8844 +export CREDHUB_CLIENT=credhub-admin +export CREDHUB_SECRET=$(bosh int --path /credhub_admin_client_secret "data/$ENVIRONMENT-bosh-variables.yml") +export HTTPS_PROXY=socks5://localhost:$PROXY_PORT + +if [ -n "$COMMAND" ]; then + ""$COMMAND"" +else + echo "OK, you are set up" + export PS1="CREDHUB<$ENVIRONMENT>:\W \u\$ " + bash +fi \ No newline at end of file diff --git a/bin/databases.sh b/bin/databases.sh index 3cf8ec68..db1bd65a 100755 --- a/bin/databases.sh +++ b/bin/databases.sh @@ -56,7 +56,7 @@ create_bosh credhub create_cf uaa create_cf cc create_cf bbs -create_cf routing +create_cf routing_api create_cf policy_server create_cf silk_controller create_cf locket diff --git a/terraform/base/bosh.tf b/terraform/base/bosh.tf index 93fe87d0..77643372 100644 --- a/terraform/base/bosh.tf +++ b/terraform/base/bosh.tf @@ -146,6 +146,24 @@ resource "aws_security_group_rule" "bosh_ssh_jumpbox" { source_security_group_id = "${aws_security_group.jumpbox.id}" } +resource "aws_security_group_rule" "bosh_credhub_jumpbox" { + security_group_id = "${aws_security_group.bosh.id}" + type = "ingress" + protocol = "tcp" + from_port = 8844 + to_port = 8844 + source_security_group_id = "${aws_security_group.jumpbox.id}" +} + +resource "aws_security_group_rule" "bosh_credhub_concourse" { + security_group_id = "${aws_security_group.bosh.id}" + type = "ingress" + protocol = "tcp" + from_port = 8844 + to_port = 8844 + source_security_group_id = "${aws_security_group.concourse.id}" +} + resource "aws_security_group_rule" "bosh_director_concourse" { security_group_id = "${aws_security_group.bosh.id}" type = "ingress" From 05a05a078a3087b7fedd3c5de6fcb1cfe7c1b6dd Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Thu, 6 Sep 2018 12:34:33 +0100 Subject: [PATCH 13/71] Deployed prometheus --- .gitmodules | 3 + Makefile | 8 +- bin/cf_password.sh | 3 +- bin/cloud_config.sh | 24 +- bin/credhub_credentials.sh | 4 +- bin/deploy_cf.sh | 13 +- bin/deploy_prometheus.sh | 46 ++ bin/runtime_config.sh | 1 + bin/secret.sh | 15 + cloud-config/{cf/cloud-config.yml => cf.yml} | 0 operations/cloud-config/prometheus.yml | 2 +- prometheus-boshrelease | 1 + runtime-config/bosh/node-exporter-config.yml | 4 +- terraform/base/bosh.tf | 25 -- terraform/base/concourse.tf | 215 ++++++++- terraform/base/locals.tf | 3 +- terraform/base/outputs.tf | 61 ++- terraform/base/prometheus.tf | 450 +++++++++++++++++++ terraform/base/routing.tf | 6 + terraform/base/subnets.tf | 13 + 20 files changed, 840 insertions(+), 57 deletions(-) create mode 100755 bin/deploy_prometheus.sh create mode 100755 bin/secret.sh rename cloud-config/{cf/cloud-config.yml => cf.yml} (100%) create mode 160000 prometheus-boshrelease diff --git a/.gitmodules b/.gitmodules index 681a60e6..27acba40 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "cf-deployment"] path = cf-deployment url = https://github.com/cloudfoundry/cf-deployment.git +[submodule "prometheus-boshrelease"] + path = prometheus-boshrelease + url = https://github.com/bosh-prometheus/prometheus-boshrelease.git diff --git a/Makefile b/Makefile index 000aabf0..c9bfa8c2 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,13 @@ stemcells: ## Upload stemcells cf: ## Deploy CF @bin/deploy_cf.sh - + +prometheus: ## Deploy Prometheus + @bin/deploy_prometheus.sh + +concourse: ## Deploy concourse + @bin/deploy_concourse.sh + decode_aws_error: ## Decode AWS message @aws sts decode-authorization-message --encoded-message ${DECODE_MESSAGE} | jq -r .DecodedMessage | jq . diff --git a/bin/cf_password.sh b/bin/cf_password.sh index 9371296c..0cdc633e 100755 --- a/bin/cf_password.sh +++ b/bin/cf_password.sh @@ -10,5 +10,4 @@ done : $ENVIRONMENT -PASSPATH='/instance_groups/name=uaa/jobs/name=uaa/properties/uaa/scim/users/name=admin/password' -bosh int --path $PASSPATH <(aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/cf/cf.yml" -) \ No newline at end of file +bin/credhub_credentials.sh -e $ENVIRONMENT credhub get -n /bosh/cf/cf_admin_password -j | jq -r .value \ No newline at end of file diff --git a/bin/cloud_config.sh b/bin/cloud_config.sh index 3169005a..59b8c935 100755 --- a/bin/cloud_config.sh +++ b/bin/cloud_config.sh @@ -14,11 +14,12 @@ output() { BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" $BOSH update-cloud-config -n \ - ./cloud-config/cf/cloud-config.yml \ + ./cloud-config/cf.yml \ -o ./operations/cloud-config/router-extensions.yml \ -o ./operations/cloud-config/cf-scheduler-extensions.yml \ -o ./operations/cloud-config/cf-s3-blobstore.yml \ -o ./operations/cloud-config/cf-rds-sec-group.yml \ + -o ./operations/cloud-config/prometheus.yml \ -v az1="$(jq -r .availability_zones[0] < data/$ENVIRONMENT.tfvars)" \ -v az2="$(jq -r .availability_zones[1] < data/$ENVIRONMENT.tfvars)" \ -v az3="$(jq -r .availability_zones[2] < data/$ENVIRONMENT.tfvars)" \ @@ -42,15 +43,12 @@ $BOSH update-cloud-config -n \ -v cf-ssh-internal="$(output base .cf_ssh_internal_security_group_id)" \ -v cf-ssh-lb="$(output base .cf_ssh_lb_name)" \ -v cf_s3_iam_instance_profile="$(output base .cf_s3_iam_instance_profile)" \ - -v cf-rds-client-security-group="$(output base .cf_rds_client_security_group_id)" - -# -o ./operations/cloud-config/prometheus.yml \ -# -v prometheus_subnet_az1_cidr="$(jq -r .prometheus_subnet_az1_cidr < prometheus-vars.json)" \ -# -v prometheus_subnet_az1_id="$(jq -r .prometheus_subnet_az1_id < prometheus-vars.json)" \ -# -v prometheus_security_group="$(jq -r .prometheus_security_group_id < prometheus-vars.json)" \ -# -v prometheus_subnet_az1_gateway="$(jq -r .prometheus_subnet_az1_cidr < prometheus-vars.json | sed 's#0/24#1#')" \ -# -v grafana_target_group_name="$(jq -r .grafana_target_group_name < prometheus-vars.json)" \ -# -v prometheus_target_group_name="$(jq -r .prometheus_target_group_name < prometheus-vars.json)" \ -# -v alertmanager_target_group_name="$(jq -r .alertmanager_target_group_name < prometheus-vars.json)" - -$BOSH cloud-config > data/$ENVIRONMENT-cloud-config.yml \ No newline at end of file + -v cf-rds-client-security-group="$(output base .cf_rds_client_security_group_id)" \ + -v prometheus_subnet_az1_cidr="$(output base .prometheus_subnet_cidr_blocks[0])" \ + -v prometheus_subnet_az1_id="$(output base .prometheus_subnet_ids[0])" \ + -v prometheus_security_group="$(output base .prometheus_security_group_id)" \ + -v prometheus_subnet_az1_gateway="$(output base .prometheus_subnet_gateway_ips[0])" \ + -v reserved_prometheus_az1_cidr="$(output base .prometheus_subnet_reserved_cidr_blocks[0])" \ + -v grafana_target_group_name="$(output base .grafana_target_group_name)" \ + -v prometheus_target_group_name="$(output base .prometheus_target_group_name)" \ + -v alertmanager_target_group_name="$(output base .alertmanager_target_group_name)" diff --git a/bin/credhub_credentials.sh b/bin/credhub_credentials.sh index 1bd4b159..c98c18dd 100755 --- a/bin/credhub_credentials.sh +++ b/bin/credhub_credentials.sh @@ -1,7 +1,6 @@ #!/bin/bash # # This script will set up a connection to a BOSH environment for you: - usage() { echo "Usage: [source] $(basename $0) -e [command]" echo @@ -41,7 +40,7 @@ cleanup() { rm -f $UAA_CA_CERT if [ -n "$STARTED_SSH" ]; then - echo "Killing the SSH proxy on $PROXY_PORT" + [ -z "$COMMAND" ] && echo "Killing the SSH proxy on $PROXY_PORT" kill $(ps -ef | awk "/ssh -4 -D $PROXY_PORT/ && ! /awk/ { print \$2 }") fi } @@ -55,6 +54,7 @@ export CREDHUB_SERVER=https://$(bin/outputs.sh base | jq -r .bosh_private_ip):88 export CREDHUB_CLIENT=credhub-admin export CREDHUB_SECRET=$(bosh int --path /credhub_admin_client_secret "data/$ENVIRONMENT-bosh-variables.yml") export HTTPS_PROXY=socks5://localhost:$PROXY_PORT +credhub login --skip-tls-validation >/dev/null if [ -n "$COMMAND" ]; then ""$COMMAND"" diff --git a/bin/deploy_cf.sh b/bin/deploy_cf.sh index 85ae3274..28475e4c 100755 --- a/bin/deploy_cf.sh +++ b/bin/deploy_cf.sh @@ -26,15 +26,15 @@ profile="${ENVIRONMENT}" instance_count_file=./profiles/${profile}/instance-count.yml -echo "Interpolating" -bosh int \ +echo "Deploying" +$BOSH deploy -n \ ./cf-deployment/cf-deployment.yml \ - --vars-store data/$ENVIRONMENT-cf-variables.yml \ --vars-file="${instance_count_file}" \ -o cf-deployment/operations/aws.yml \ -o cf-deployment/operations/override-app-domains.yml \ -o cf-deployment/operations/use-external-blobstore.yml \ -o cf-deployment/operations/use-external-dbs.yml \ + -o prometheus-boshrelease/manifests/operators/cf/add-prometheus-uaa-clients.yml \ -o ./operations/bosh/tags.yml \ -o ./operations/cf/router-sec-group.yml \ -o ./operations/cf/scheduler.yml \ @@ -87,10 +87,5 @@ bosh int \ -v external_credhub_database_name="credhub" \ -v external_credhub_database_address="${CF_DB_ENDPOINT}" \ -v external_credhub_database_password="${CF_DB_PASSWORD}" \ - -v external_credhub_database_username="${CF_DB_USERNAME}" \ - > data/$ENVIRONMENT-cf-manifest.yml - -echo "Deploying" -$BOSH deploy -n -d cf data/$ENVIRONMENT-cf-manifest.yml + -v external_credhub_database_username="${CF_DB_USERNAME}" -# -o prometheus-deployment-git/manifests/operators/cf/add-prometheus-uaa-clients.yml \ \ No newline at end of file diff --git a/bin/deploy_prometheus.sh b/bin/deploy_prometheus.sh new file mode 100755 index 00000000..136723d2 --- /dev/null +++ b/bin/deploy_prometheus.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +set -euo pipefail + +: $ENVIRONMENT + +output() { + FILE=$1 + QUERY=$2 + + bin/outputs.sh $FILE | jq -r "$QUERY" +} + +secret() { + KEY=$1 + bin/secret.sh -e $ENVIRONMENT -d cf -k $KEY +} + +BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" + +SYSTEM_DOMAIN="system.$(output base .domain)" +METRON_DEPLOYMENT_NAME=cf + +export BOSH_CA_CERT=/var/tmp/bosh_ca.$$.pem +bosh int --path /default_ca/ca "data/$ENVIRONMENT-bosh-variables.yml" >$BOSH_CA_CERT + +cleanup() { + rm -f $BOSH_CA_CERT +} +trap cleanup EXIT + +$BOSH -d prometheus deploy -n prometheus-boshrelease/manifests/prometheus.yml \ + -o prometheus-boshrelease/manifests/operators/monitor-bosh.yml \ + -o prometheus-boshrelease/manifests/operators/monitor-cf.yml \ + -o ./operations/prometheus/networks.yml \ + -v bosh_url="$(output base .bosh_director_fqdn)" \ + -v bosh_username="admin" \ + -v bosh_password="$(bosh interpolate --path /admin_password data/$ENVIRONMENT-bosh-variables.yml)" \ + --var-file bosh_ca_cert=$BOSH_CA_CERT \ + -v metrics_environment="$ENVIRONMENT" \ + -v metron_deployment_name="$METRON_DEPLOYMENT_NAME" \ + -v system_domain="$SYSTEM_DOMAIN" \ + -v uaa_clients_cf_exporter_secret="$(secret uaa_clients_cf_exporter_secret)" \ + -v uaa_clients_firehose_exporter_secret="$(secret uaa_clients_firehose_exporter_secret)" \ + -v traffic_controller_external_port="$(output base .cf_traffic_controller_port)" \ + -v skip_ssl_verify=false diff --git a/bin/runtime_config.sh b/bin/runtime_config.sh index d960b514..865042b7 100755 --- a/bin/runtime_config.sh +++ b/bin/runtime_config.sh @@ -13,3 +13,4 @@ output() { BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" $BOSH update-runtime-config bosh-deployment/runtime-configs/dns.yml --name dns -n +$BOSH update-runtime-config ./runtime-config/bosh/node-exporter-config.yml --name node_exporter -n \ No newline at end of file diff --git a/bin/secret.sh b/bin/secret.sh new file mode 100755 index 00000000..d6d58a41 --- /dev/null +++ b/bin/secret.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -euo pipefail + +while getopts e:d:k: option; do + case $option in + e) ENVIRONMENT="$OPTARG";; + d) DEPLOYMENT="$OPTARG";; + k) KEY="$OPTARG";; + esac +done + +: $ENVIRONMENT + +bin/credhub_credentials.sh -e $ENVIRONMENT credhub get -n "/bosh/$DEPLOYMENT/$KEY" -j | jq -r .value \ No newline at end of file diff --git a/cloud-config/cf/cloud-config.yml b/cloud-config/cf.yml similarity index 100% rename from cloud-config/cf/cloud-config.yml rename to cloud-config/cf.yml diff --git a/operations/cloud-config/prometheus.yml b/operations/cloud-config/prometheus.yml index 3524cc68..19f0d066 100644 --- a/operations/cloud-config/prometheus.yml +++ b/operations/cloud-config/prometheus.yml @@ -18,7 +18,7 @@ dns: - ((private_dns_nameserver)) - 1.1.1.1 - reserved: [10.0.11.0-10.0.11.3] + reserved: [((reserved_prometheus_az1_cidr))] cloud_properties: subnet: ((prometheus_subnet_az1_id)) security_groups: [ ((prometheus_security_group))] diff --git a/prometheus-boshrelease b/prometheus-boshrelease new file mode 160000 index 00000000..0a68d32e --- /dev/null +++ b/prometheus-boshrelease @@ -0,0 +1 @@ +Subproject commit 0a68d32ef3a9bbae455f579b81d99484cb2241a9 diff --git a/runtime-config/bosh/node-exporter-config.yml b/runtime-config/bosh/node-exporter-config.yml index 0a3f5d26..80c82170 100644 --- a/runtime-config/bosh/node-exporter-config.yml +++ b/runtime-config/bosh/node-exporter-config.yml @@ -1,6 +1,8 @@ releases: - name: node-exporter - version: 4.0.0 + version: 4.0.1 + url: https://github.com/bosh-prometheus/node-exporter-boshrelease/releases/download/v4.0.1/node-exporter-4.0.1.tgz + sha1: 8f00d257838f33d5022d6c356c35d2922e75c9b4 addons: - name: node_exporter diff --git a/terraform/base/bosh.tf b/terraform/base/bosh.tf index 77643372..68dff2b5 100644 --- a/terraform/base/bosh.tf +++ b/terraform/base/bosh.tf @@ -92,14 +92,6 @@ resource "aws_security_group" "bosh" { } } -resource "aws_security_group_rule" "bosh_mbus_concourse" { - security_group_id = "${aws_security_group.bosh.id}" - type = "ingress" - protocol = "tcp" - from_port = 6868 - to_port = 6868 - source_security_group_id = "${aws_security_group.concourse.id}" -} resource "aws_security_group_rule" "bosh_uaa_concourse" { security_group_id = "${aws_security_group.bosh.id}" @@ -128,14 +120,6 @@ resource "aws_security_group_rule" "bosh_uaa_jumpbox" { source_security_group_id = "${aws_security_group.jumpbox.id}" } -resource "aws_security_group_rule" "bosh_ssh_concourse" { - security_group_id = "${aws_security_group.bosh.id}" - type = "ingress" - protocol = "tcp" - from_port = 22 - to_port = 22 - source_security_group_id = "${aws_security_group.concourse.id}" -} resource "aws_security_group_rule" "bosh_ssh_jumpbox" { security_group_id = "${aws_security_group.bosh.id}" @@ -164,15 +148,6 @@ resource "aws_security_group_rule" "bosh_credhub_concourse" { source_security_group_id = "${aws_security_group.concourse.id}" } -resource "aws_security_group_rule" "bosh_director_concourse" { - security_group_id = "${aws_security_group.bosh.id}" - type = "ingress" - protocol = "tcp" - from_port = 25555 - to_port = 25555 - source_security_group_id = "${aws_security_group.concourse.id}" -} - resource "aws_security_group_rule" "bosh_director_jumpbox" { security_group_id = "${aws_security_group.bosh.id}" type = "ingress" diff --git a/terraform/base/concourse.tf b/terraform/base/concourse.tf index 482ccab1..b7796af7 100644 --- a/terraform/base/concourse.tf +++ b/terraform/base/concourse.tf @@ -10,7 +10,220 @@ resource "aws_security_group" "concourse" { } } +resource "aws_security_group_rule" "concourse_web" { + security_group_id = "${aws_security_group.concourse.id}" + type = "ingress" + protocol = "tcp" + from_port = 8080 + to_port = 8080 + source_security_group_id = "${aws_security_group.concourse_alb.id}" +} + +resource "aws_security_group_rule" "concourse_bosh_director" { + security_group_id = "${aws_security_group.bosh.id}" + type = "ingress" + protocol = "tcp" + from_port = 25555 + to_port = 25555 + description = "Allow concourse to access bosh director" + source_security_group_id = "${aws_security_group.concourse.id}" +} + +resource "aws_security_group_rule" "concourse_from_bosh_rule_tcp_ssh" { + security_group_id = "${aws_security_group.concourse.id}" + type = "ingress" + protocol = "tcp" + from_port = 22 + to_port = 22 + source_security_group_id = "${aws_security_group.bosh.id}" +} + +resource "aws_security_group_rule" "concourse_from_bosh_rule_tcp_bosh_agent" { + security_group_id = "${aws_security_group.concourse.id}" + type = "ingress" + protocol = "tcp" + from_port = 6868 + to_port = 6868 + source_security_group_id = "${aws_security_group.bosh.id}" +} + +resource "aws_security_group_rule" "concourse_from_jumpbox_rule_tcp_ssh" { + security_group_id = "${aws_security_group.concourse.id}" + type = "ingress" + protocol = "tcp" + from_port = 22 + to_port = 22 + source_security_group_id = "${aws_security_group.jumpbox.id}" +} + +resource "aws_security_group_rule" "bosh_ssh_concourse" { + security_group_id = "${aws_security_group.bosh.id}" + type = "egress" + protocol = "tcp" + from_port = 22 + to_port = 22 + source_security_group_id = "${aws_security_group.concourse.id}" +} + +resource "aws_security_group_rule" "bosh_mbus_concourse" { + security_group_id = "${aws_security_group.bosh.id}" + type = "egress" + protocol = "tcp" + from_port = 6868 + to_port = 6868 + source_security_group_id = "${aws_security_group.concourse.id}" +} + +resource "aws_security_group_rule" "bosh_tcp_from_concourse" { + security_group_id = "${aws_security_group.bosh.id}" + type = "ingress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + source_security_group_id = "${aws_security_group.concourse.id}" +} + +resource "aws_security_group_rule" "bosh_udp_from_concourse" { + security_group_id = "${aws_security_group.bosh.id}" + type = "ingress" + protocol = "udp" + from_port = 0 + to_port = 65535 + source_security_group_id = "${aws_security_group.concourse.id}" +} + +resource "aws_security_group_rule" "jumpbox_ssh_concourse" { + security_group_id = "${aws_security_group.jumpbox.id}" + type = "egress" + protocol = "tcp" + from_port = 22 + to_port = 22 + source_security_group_id = "${aws_security_group.concourse.id}" +} + + + +resource "aws_security_group_rule" "concourse_outbound" { + security_group_id = "${aws_security_group.concourse.id}" + type = "egress" + protocol = "all" + from_port = 0 + to_port = 0 + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group" "concourse_alb" { + name = "${var.environment}_concourse_alb_security_group" + description = "Concourse public access" + vpc_id = "${aws_vpc.default.id}" + + tags { + Name = "${var.environment}-concourse-alb-security-group" + Environment = "${var.environment}" + } +} + +resource "aws_security_group_rule" "concourse_alb_https" { + security_group_id = "${aws_security_group.concourse_alb.id}" + type = "ingress" + protocol = "tcp" + from_port = 443 + to_port = 443 + cidr_blocks = "${var.ingress_whitelist}" +} + +resource "aws_security_group_rule" "concourse_alb_to_web" { + security_group_id = "${aws_security_group.concourse_alb.id}" + type = "egress" + protocol = "tcp" + from_port = 8080 + to_port = 8080 + source_security_group_id = "${aws_security_group.concourse.id}" +} + + # IP resource "aws_eip" "concourse" { vpc = true -} \ No newline at end of file +} + +# Load Balancer +resource "aws_lb" "concourse" { + name = "${var.environment}-concourse-alb" + subnets = ["${aws_subnet.public.*.id}"] + security_groups = ["${aws_security_group.concourse_alb.id}"] + load_balancer_type = "application" + internal = false + enable_cross_zone_load_balancing = true + + tags { + Name = "${var.environment}-concourse-alb" + Environment = "${var.environment}" + } +} + +resource "aws_lb_listener" "concourse" { + load_balancer_arn = "${aws_lb.concourse.arn}" + port = "443" + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-2015-05" + certificate_arn = "${aws_acm_certificate.concourse.arn}" + + default_action { + target_group_arn = "${aws_lb_target_group.concourse.arn}" + type = "forward" + } +} + +resource "aws_lb_target_group" "concourse" { + name = "${var.environment}-concourse-target-group" + port = 8080 + protocol = "HTTP" + vpc_id = "${aws_vpc.default.id}" + + health_check { + healthy_threshold = 2 + unhealthy_threshold = 2 + timeout = 3 + interval = 5 + } + + tags { + Name = "${var.environment}-concourse-target-group" + Environment = "${var.environment}" + } +} + +# Certificate +resource "aws_acm_certificate" "concourse" { + domain_name = "ci.${local.domain}" + validation_method = "DNS" + + tags { + Name = "${var.environment}-ci-cert" + Environment = "${var.environment}" + } +} + +resource "aws_route53_record" "concourse_validation" { + name = "${aws_acm_certificate.concourse.domain_validation_options.0.resource_record_name}" + type = "${aws_acm_certificate.concourse.domain_validation_options.0.resource_record_type}" + zone_id = "${aws_route53_zone.child_zone.zone_id}" + records = ["${aws_acm_certificate.concourse.domain_validation_options.0.resource_record_value}"] + ttl = 30 +} + +resource "aws_acm_certificate_validation" "concourse" { + certificate_arn = "${aws_acm_certificate.concourse.arn}" + validation_record_fqdns = ["${aws_route53_record.concourse_validation.fqdn}"] +} + +# DNS +resource "aws_route53_record" "concourse" { + zone_id = "${aws_route53_zone.child_zone.zone_id}" + name = "ci.${local.domain}" + type = "CNAME" + ttl = "30" + + records = ["${aws_lb.concourse.dns_name}"] +} diff --git a/terraform/base/locals.tf b/terraform/base/locals.tf index ba1b95df..dbb9e23f 100644 --- a/terraform/base/locals.tf +++ b/terraform/base/locals.tf @@ -1,11 +1,12 @@ locals { account_id = "${data.aws_caller_identity.current.account_id}" + domain = "${replace(aws_route53_zone.child_zone.name, "/\\.$/", "")}" num_azs = "${length(var.availability_zones)}" public_subnets = "${var.cidr_blocks["public"]}" internal_subnets = "${var.cidr_blocks["internal"]}" services_subnets = "${var.cidr_blocks["services"]}" rds_subnets = "${var.cidr_blocks["rds"]}" - + prometheus_subnets = "${var.cidr_blocks["prometheus"]}" bosh_subnet_id = "${aws_subnet.internal.*.id[var.bosh_availability_zone_index]}" bosh_subnet_cidr_block = "${aws_subnet.internal.*.cidr_block[var.bosh_availability_zone_index]}" diff --git a/terraform/base/outputs.tf b/terraform/base/outputs.tf index 78a2be9f..87d52551 100644 --- a/terraform/base/outputs.tf +++ b/terraform/base/outputs.tf @@ -3,7 +3,7 @@ output "vpc_dns_nameserver" { } output "domain" { - value = "${replace(aws_route53_zone.child_zone.name, "/\\.$/", "")}" + value = "${local.domain}" } output "bosh_rds_security_group_id" { @@ -140,4 +140,63 @@ output "cf_blobstore_s3_kms_key_id" { output "cf_blobstore_s3_kms_key_arn" { value = "${aws_kms_key.cf_blobstore_key.arn}" +} + +output "prometheus_subnet_gateway_ips" { + value = [ + "${cidrhost(aws_subnet.prometheus.*.cidr_block[0],1)}", + "${cidrhost(aws_subnet.prometheus.*.cidr_block[1],1)}", + "${cidrhost(aws_subnet.prometheus.*.cidr_block[2],1)}" + ] +} + +# NASTY HACK ALERT - we cannot find a way in terraform to perform an action on all elements of a list +# so you will have to change this if you add more AZs +output "prometheus_subnet_reserved_cidr_blocks" { + value = [ + "${cidrsubnet(aws_subnet.prometheus.*.cidr_block[0],6,0)}", + "${cidrsubnet(aws_subnet.prometheus.*.cidr_block[1],6,0)}", + "${cidrsubnet(aws_subnet.prometheus.*.cidr_block[2],6,0)}" + ] +} + +output "prometheus_subnet_ids" { + value = ["${aws_subnet.prometheus.*.id}"] +} + +output "prometheus_subnet_cidr_blocks" { + value = ["${aws_subnet.prometheus.*.cidr_block}"] +} + + +output "prometheus_security_group_id" { + value = "${aws_security_group.prometheus.id}" +} + +output "grafana_target_group_name" { + value = "${aws_lb_target_group.grafana.name}" +} + +output "prometheus_target_group_name" { + value = "${aws_lb_target_group.prometheus.name}" +} + +output "alertmanager_target_group_name" { + value = "${aws_lb_target_group.alertmanager.name}" +} + +output "grafana_fqdn" { + value = "${aws_route53_record.grafana.name}" +} + +output "prometheus_fqdn" { + value = "${aws_route53_record.prometheus.fqdn}" +} + +output "alertmanager_fqdn" { + value = "${aws_route53_record.alertmanager.fqdn}" +} + +output "cf_traffic_controller_port" { + value = "${aws_lb_listener.cf_4443.port}" } \ No newline at end of file diff --git a/terraform/base/prometheus.tf b/terraform/base/prometheus.tf index e69de29b..b7861717 100644 --- a/terraform/base/prometheus.tf +++ b/terraform/base/prometheus.tf @@ -0,0 +1,450 @@ +# Load balancing + +resource "aws_lb" "prometheus" { + name = "${var.environment}-prometheus-alb" + subnets = ["${aws_subnet.public.*.id}"] + security_groups = ["${aws_security_group.prometheus_alb.id}"] + load_balancer_type = "application" + internal = false + enable_cross_zone_load_balancing = true + + tags { + Name = "${var.environment}-prometheus-alb" + Environment = "${var.environment}" + } + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_lb_listener" "prometheus_443" { + depends_on = ["aws_acm_certificate_validation.prometheus"] + load_balancer_arn = "${aws_lb.prometheus.arn}" + port = "443" + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-2015-05" + certificate_arn = "${aws_acm_certificate.prometheus.arn}" + + default_action { + target_group_arn = "${aws_lb_target_group.grafana.arn}" + type = "forward" + } + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_lb_target_group" "grafana" { + name = "${var.environment}-grafana-target-group" + port = 3000 + protocol = "HTTP" + vpc_id = "${aws_vpc.default.id}" + deregistration_delay = "30" + + tags { + Name = "${var.environment}-grafana-target-group" + Environment = "${var.environment}" + } + + health_check { + path = "/metrics" + healthy_threshold = 2 + unhealthy_threshold = 2 + timeout = 3 + interval = 5 + } +} + +resource "aws_lb_target_group" "prometheus" { + name = "${var.environment}-prometheus-target-group" + port = 9090 + protocol = "HTTP" + vpc_id = "${aws_vpc.default.id}" + deregistration_delay = "30" + + tags { + Name = "${var.environment}-prometheus-target-group" + Environment = "${var.environment}" + } + + health_check { + path = "/login" + healthy_threshold = 2 + unhealthy_threshold = 2 + timeout = 3 + interval = 5 + matcher = "401" + } +} + +resource "aws_lb_listener_rule" "prometheus_host_routing" { + listener_arn = "${aws_lb_listener.prometheus_443.arn}" + priority = 4 + + action { + type = "forward" + target_group_arn = "${aws_lb_target_group.prometheus.arn}" + } + + condition { + field = "host-header" + values = ["prometheus.*"] + } +} + +resource "aws_lb_target_group" "alertmanager" { + name = "${var.environment}-alertmgr-target-group" + port = 9093 + protocol = "HTTP" + vpc_id = "${aws_vpc.default.id}" + deregistration_delay = "30" + + tags { + Name = "${var.environment}-alertmgr-target-group" + Environment = "${var.environment}" + } + + health_check { + path = "/login" + healthy_threshold = 2 + unhealthy_threshold = 2 + timeout = 3 + interval = 5 + matcher = "401" + } +} + +resource "aws_lb_listener_rule" "alertmanager_host_routing" { + listener_arn = "${aws_lb_listener.prometheus_443.arn}" + priority = 6 + + action { + type = "forward" + target_group_arn = "${aws_lb_target_group.alertmanager.arn}" + } + + condition { + field = "host-header" + values = ["alertmanager.*"] + } +} + +# Certificates + +resource "aws_acm_certificate" "prometheus" { + domain_name = "*.prometheus.${local.domain}" + + subject_alternative_names = [ + "prometheus.${local.domain}", + "grafana.${local.domain}", + "alertmanager.${local.domain}" + ] + + validation_method = "DNS" + + tags { + Name = "${var.environment}-prometheus-cert" + Environment = "${var.environment}" + } +} + +resource "aws_route53_record" "prometheus_validation" { + count = 4 + + name = "${lookup(aws_acm_certificate.prometheus.domain_validation_options[count.index], "resource_record_name")}" + type = "${lookup(aws_acm_certificate.prometheus.domain_validation_options[count.index], "resource_record_type")}" + zone_id = "${aws_route53_zone.child_zone.zone_id}" + records = ["${lookup(aws_acm_certificate.prometheus.domain_validation_options[count.index], "resource_record_value")}"] + ttl = 30 +} + +resource "aws_acm_certificate_validation" "prometheus" { + certificate_arn = "${aws_acm_certificate.prometheus.arn}" + validation_record_fqdns = ["${aws_route53_record.prometheus_validation.*.fqdn}"] +} + +# DNS + +resource "aws_route53_record" "prometheus" { + zone_id = "${aws_route53_zone.child_zone.zone_id}" + name = "prometheus.${local.domain}" + type = "CNAME" + ttl = "30" + + records = ["${aws_lb.prometheus.dns_name}"] +} + +resource "aws_route53_record" "grafana" { + zone_id = "${aws_route53_zone.child_zone.zone_id}" + name = "grafana.${local.domain}" + type = "CNAME" + ttl = "30" + + records = ["${aws_lb.prometheus.dns_name}"] +} + +resource "aws_route53_record" "alertmanager" { + zone_id = "${aws_route53_zone.child_zone.zone_id}" + name = "alertmanager.${local.domain}" + type = "CNAME" + ttl = "30" + + records = ["${aws_lb.prometheus.dns_name}"] +} + +# S3 + +# resource "aws_s3_bucket_object" "prometheus-var-store" { +# bucket = "${aws_s3_bucket.paas_states.id}" +# acl = "private" +# key = "prometheus/prometheus-variables.yml" +# source = "/dev/null" +# server_side_encryption = "aws:kms" +# kms_key_id = "${var.s3_kms_key_arn}" +# } + +# Security groups + +resource "aws_security_group" "prometheus" { + name = "${var.environment}_prometheus_security_group" + description = "Prometheus node exporter access" + vpc_id = "${aws_vpc.default.id}" + + tags { + Name = "${var.environment}-prometheus-security-group" + Environment = "${var.environment}" + } +} + +resource "aws_security_group_rule" "prometheus_bosh_node_exporters" { + security_group_id = "${aws_security_group.bosh.id}" + type = "ingress" + protocol = "tcp" + from_port = 9190 + to_port = 9190 + description = "Allow prometheus to access bosh node exporter" + source_security_group_id = "${aws_security_group.prometheus.id}" +} + +resource "aws_security_group_rule" "prometheus_cf_node_exporters" { + security_group_id = "${aws_security_group.internal.id}" + type = "ingress" + protocol = "tcp" + from_port = 9100 + to_port = 9100 + description = "Allow prometheus to access cf node exporter" + source_security_group_id = "${aws_security_group.prometheus.id}" +} + +resource "aws_security_group_rule" "prometheus_cf_nats" { + security_group_id = "${aws_security_group.internal.id}" + type = "ingress" + protocol = "tcp" + from_port = 4222 + to_port = 4222 + description = "Allow prometheus to access cf nats" + source_security_group_id = "${aws_security_group.prometheus.id}" +} + +resource "aws_security_group_rule" "prometheus_bosh_director" { + security_group_id = "${aws_security_group.bosh.id}" + type = "ingress" + protocol = "tcp" + from_port = 25555 + to_port = 25555 + description = "Allow prometheus to access bosh director" + source_security_group_id = "${aws_security_group.prometheus.id}" +} + +resource "aws_security_group_rule" "prometheus_outbound" { + security_group_id = "${aws_security_group.prometheus.id}" + type = "egress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + description = "Allow prometheus to access everything" + cidr_blocks = ["0.0.0.0/0"] +} + +resource "aws_security_group_rule" "prometheus_rule_tcp" { + security_group_id = "${aws_security_group.prometheus.id}" + type = "ingress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + self = true +} + +resource "aws_security_group_rule" "prometheus_rule_udp" { + security_group_id = "${aws_security_group.prometheus.id}" + type = "ingress" + protocol = "udp" + from_port = 0 + to_port = 65535 + self = true +} + +resource "aws_security_group_rule" "prometheus_rule_icmp" { + security_group_id = "${aws_security_group.prometheus.id}" + type = "ingress" + protocol = "icmp" + from_port = -1 + to_port = -1 + self = true +} + +resource "aws_security_group_rule" "prometheus_from_bosh_rule_tcp_ssh" { + security_group_id = "${aws_security_group.prometheus.id}" + type = "ingress" + protocol = "tcp" + from_port = 22 + to_port = 22 + source_security_group_id = "${aws_security_group.bosh.id}" +} + +resource "aws_security_group_rule" "prometheus_from_bosh_rule_tcp_bosh_agent" { + security_group_id = "${aws_security_group.prometheus.id}" + type = "ingress" + protocol = "tcp" + from_port = 6868 + to_port = 6868 + source_security_group_id = "${aws_security_group.bosh.id}" +} + +resource "aws_security_group_rule" "prometheus_from_jumpbox_rule_tcp_ssh" { + security_group_id = "${aws_security_group.prometheus.id}" + type = "ingress" + protocol = "tcp" + from_port = 22 + to_port = 22 + source_security_group_id = "${aws_security_group.jumpbox.id}" +} + +resource "aws_security_group_rule" "bosh_ssh_prometheus" { + security_group_id = "${aws_security_group.bosh.id}" + type = "egress" + protocol = "tcp" + from_port = 22 + to_port = 22 + source_security_group_id = "${aws_security_group.prometheus.id}" +} + +resource "aws_security_group_rule" "bosh_mbus_prometheus" { + security_group_id = "${aws_security_group.bosh.id}" + type = "egress" + protocol = "tcp" + from_port = 6868 + to_port = 6868 + source_security_group_id = "${aws_security_group.prometheus.id}" +} + +resource "aws_security_group_rule" "bosh_tcp_from_prometheus" { + security_group_id = "${aws_security_group.bosh.id}" + type = "ingress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + source_security_group_id = "${aws_security_group.prometheus.id}" +} + +resource "aws_security_group_rule" "bosh_udp_from_prometheus" { + security_group_id = "${aws_security_group.bosh.id}" + type = "ingress" + protocol = "udp" + from_port = 0 + to_port = 65535 + source_security_group_id = "${aws_security_group.prometheus.id}" +} + +resource "aws_security_group_rule" "jumpbox_ssh_prometheus" { + security_group_id = "${aws_security_group.jumpbox.id}" + type = "egress" + protocol = "tcp" + from_port = 22 + to_port = 22 + source_security_group_id = "${aws_security_group.prometheus.id}" +} + +resource "aws_security_group" "prometheus_alb" { + name = "${var.environment}_prometheus_alb_security_group" + description = "Prometheus web access" + vpc_id = "${aws_vpc.default.id}" + + tags { + Name = "${var.environment}-prometheus-alb-security-group" + Environment = "${var.environment}" + } +} + +resource "aws_security_group_rule" "prometheus_alb_web_access" { + security_group_id = "${aws_security_group.prometheus_alb.id}" + type = "ingress" + protocol = "tcp" + from_port = 443 + to_port = 443 + cidr_blocks = ["${local.loadbalancer_whitelist}"] + description = "External access to Prometheus service" +} + +resource "aws_security_group_rule" "prometheus_alb_to_grafana_access" { + security_group_id = "${aws_security_group.prometheus_alb.id}" + type = "egress" + protocol = "tcp" + from_port = 3000 + to_port = 3000 + source_security_group_id = "${aws_security_group.prometheus.id}" + description = "Route traffic to Grafana" +} + +resource "aws_security_group_rule" "grafana_from_prometheus_alb_access" { + security_group_id = "${aws_security_group.prometheus.id}" + type = "ingress" + protocol = "tcp" + from_port = 3000 + to_port = 3000 + source_security_group_id = "${aws_security_group.prometheus_alb.id}" + description = "Access to Grafana" +} + +resource "aws_security_group_rule" "prometheus_alb_to_prometheus_access" { + security_group_id = "${aws_security_group.prometheus_alb.id}" + type = "egress" + protocol = "tcp" + from_port = 9090 + to_port = 9090 + source_security_group_id = "${aws_security_group.prometheus.id}" + description = "Route traffic to Prometheus" +} + +resource "aws_security_group_rule" "prometheus_alb_access" { + security_group_id = "${aws_security_group.prometheus.id}" + type = "ingress" + protocol = "tcp" + from_port = 9090 + to_port = 9090 + source_security_group_id = "${aws_security_group.prometheus_alb.id}" + description = "Access to Prometheus" +} + +resource "aws_security_group_rule" "prometheus_alb_to_alertmanager_access" { + security_group_id = "${aws_security_group.prometheus_alb.id}" + type = "egress" + protocol = "tcp" + from_port = 9093 + to_port = 9093 + source_security_group_id = "${aws_security_group.prometheus.id}" + description = "Route traffic to Alertmanager" +} + +resource "aws_security_group_rule" "alertmanager_alb_access" { + security_group_id = "${aws_security_group.prometheus.id}" + type = "ingress" + protocol = "tcp" + from_port = 9093 + to_port = 9093 + source_security_group_id = "${aws_security_group.prometheus_alb.id}" + description = "Access to Alertmanager" +} + diff --git a/terraform/base/routing.tf b/terraform/base/routing.tf index bc1c42b1..718ad699 100644 --- a/terraform/base/routing.tf +++ b/terraform/base/routing.tf @@ -56,4 +56,10 @@ resource "aws_route_table_association" "services" { count = "${local.num_azs}" subnet_id = "${element(aws_subnet.services.*.id, count.index)}" route_table_id = "${element(aws_route_table.az.*.id, count.index)}" +} + +resource "aws_route_table_association" "prometheus" { + count = "${local.num_azs}" + subnet_id = "${element(aws_subnet.prometheus.*.id, count.index)}" + route_table_id = "${element(aws_route_table.az.*.id, count.index)}" } \ No newline at end of file diff --git a/terraform/base/subnets.tf b/terraform/base/subnets.tf index f1927326..e9d4e0a1 100644 --- a/terraform/base/subnets.tf +++ b/terraform/base/subnets.tf @@ -49,4 +49,17 @@ resource "aws_subnet" "rds" { Environment = "${var.environment}" Visibility = "private" } +} + +resource "aws_subnet" "prometheus" { + count = "${local.num_azs}" + vpc_id = "${aws_vpc.default.id}" + cidr_block = "${element(local.prometheus_subnets, count.index)}" + availability_zone = "${element(var.availability_zones, count.index)}" + + tags { + Name = "${var.environment}-prometheus-az${count.index+1}-subnet" + Environment = "${var.environment}" + Visibility = "private" + } } \ No newline at end of file From 449e0ee57711b8a73325dda49aa22eb3038069ec Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Thu, 6 Sep 2018 12:44:10 +0100 Subject: [PATCH 14/71] Set correct versions for submodules --- README.md | 7 +++++++ bosh-deployment | 2 +- concourse-bosh-deployment | 2 +- prometheus-boshrelease | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 41df814c..4b675422 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,13 @@ or # it also sets BOSH_CLIENT, BOSH_CLIENT_SECRET environment variables ``` +## Versions + +concourse v4.1.0 +cf v4.0.0 +bosh v1.1.0 +prometheus v23.2.0 + ## LICENCE Copyright (c) 2018 Crown Copyright (Office for National Statistics) diff --git a/bosh-deployment b/bosh-deployment index ff0bc984..04c85a5c 160000 --- a/bosh-deployment +++ b/bosh-deployment @@ -1 +1 @@ -Subproject commit ff0bc984c72d5053656c666ae7d7788a78f6b7a8 +Subproject commit 04c85a5c79a9fa6b92775386a334104b9a165013 diff --git a/concourse-bosh-deployment b/concourse-bosh-deployment index cae8cb40..70640ac7 160000 --- a/concourse-bosh-deployment +++ b/concourse-bosh-deployment @@ -1 +1 @@ -Subproject commit cae8cb402a470c7c318867e8070d57afcebc8b9c +Subproject commit 70640ac7707bc476f506fe6f6228fdb4422aa14f diff --git a/prometheus-boshrelease b/prometheus-boshrelease index 0a68d32e..746e3bb6 160000 --- a/prometheus-boshrelease +++ b/prometheus-boshrelease @@ -1 +1 @@ -Subproject commit 0a68d32ef3a9bbae455f579b81d99484cb2241a9 +Subproject commit 746e3bb6309d6a994d4b2dc5be40de96cfc60aab From 9cd1793ed1fbc58f5017bdf5ab22d674dc6a9bfa Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Thu, 6 Sep 2018 16:20:38 +0100 Subject: [PATCH 15/71] Deploy Concourse --- Makefile | 6 ++ bin/cloud_config.sh | 9 +- bin/deploy_concourse.sh | 91 +++++++++---------- bin/login_fly.sh | 20 ++-- bin/stemcells.sh | 14 ++- ci/cf.yml | 45 +++++++++ ci/cf_pipeline.sh | 21 +++++ .../bosh/db_connectivity_test/task.sh | 0 .../bosh/db_connectivity_test/task.yml | 0 ci/{tasks => tasks.old}/bosh/deploy/task.sh | 0 ci/{tasks => tasks.old}/bosh/deploy/task.yml | 0 .../bosh/deploy_node_exporter/task.sh | 0 .../bosh/deploy_node_exporter/task.yml | 0 ci/{tasks => tasks.old}/bosh/destroy/task.sh | 0 ci/{tasks => tasks.old}/bosh/destroy/task.yml | 0 .../bosh/get_database_vars/task.sh | 0 .../bosh/get_database_vars/task.yml | 0 .../bosh/get_terraform_vars/task.sh | 0 .../bosh/get_terraform_vars/task.yml | 0 .../bosh/interpolate/task.sh | 0 .../bosh/interpolate/task.yml | 0 ci/{tasks => tasks.old}/bosh/test/task.sh | 0 ci/{tasks => tasks.old}/bosh/test/task.yml | 0 .../cf/cloud_config/task.sh | 0 .../cf/cloud_config/task.yml | 0 .../cf/databases/db_connectivity_test/task.sh | 0 .../databases/db_connectivity_test/task.yml | 0 .../cf/databases/get_database_vars/task.sh | 0 .../cf/databases/get_database_vars/task.yml | 0 ci/{tasks => tasks.old}/cf/deploy_cf/task.sh | 0 ci/{tasks => tasks.old}/cf/deploy_cf/task.yml | 0 ci/{tasks => tasks.old}/cf/destroy/task.sh | 0 ci/{tasks => tasks.old}/cf/destroy/task.yml | 0 .../cf/get_management_vars/task.sh | 0 .../cf/get_management_vars/task.yml | 0 .../cf/get_terraform_vars/task.sh | 0 .../cf/get_terraform_vars/task.yml | 0 .../cf/interpolate/task.sh | 0 .../cf/interpolate/task.yml | 0 ci/tasks.old/cf/interpolate_cats/task.sh | 14 +++ ci/tasks.old/cf/interpolate_cats/task.yml | 21 +++++ ci/{tasks => tasks.old}/cf/management/task.sh | 0 .../cf/management/task.yml | 0 .../cf/smoke_tests/task.sh | 0 .../cf/smoke_tests/task.yml | 0 ci/{tasks => tasks.old}/cf/test/task.sh | 0 ci/{tasks => tasks.old}/cf/test/task.yml | 0 .../cf/upload_stemcell/task.sh | 0 .../cf/upload_stemcell/task.yml | 0 .../common/delete_s3/task.sh | 0 .../common/delete_s3/task.yml | 0 .../common/get_cf_tester_credentials/task.sh | 0 .../common/get_cf_tester_credentials/task.yml | 0 .../common/test_connectivity/task.sh | 0 .../common/test_connectivity/task.yml | 0 .../common/test_endpoint/task.sh | 0 .../common/test_endpoint/task.yml | 0 .../concourse/test_fqdn/task.sh | 0 .../concourse/test_fqdn/task.yml | 0 .../concourse/test_iam_policy/task.sh | 0 .../concourse/test_iam_policy/task.yml | 0 .../destroy/empty_buckets/task.sh | 0 .../destroy/empty_buckets/task.yml | 0 .../jumpbox/deploy/task.sh | 0 .../jumpbox/deploy/task.yml | 0 .../jumpbox/destroy/task.sh | 0 .../jumpbox/destroy/task.yml | 0 .../jumpbox/get_terraform_vars/task.sh | 0 .../jumpbox/get_terraform_vars/task.yml | 0 .../jumpbox/interpolate/task.sh | 0 .../jumpbox/interpolate/task.yml | 0 ci/{tasks => tasks.old}/jumpbox/test/task.sh | 0 ci/{tasks => tasks.old}/jumpbox/test/task.yml | 0 .../jumpbox/test_fqdn/task.sh | 0 .../jumpbox/test_fqdn/task.yml | 0 .../prometheus/dashboard/task.sh | 0 .../prometheus/dashboard/task.yml | 0 .../prometheus/deploy/task.sh | 0 .../prometheus/deploy/task.yml | 0 .../prometheus/destroy/task.sh | 0 .../prometheus/destroy/task.yml | 0 .../prometheus/get_terraform_vars/task.sh | 0 .../prometheus/get_terraform_vars/task.yml | 0 .../prometheus/interpolate/task.sh | 0 .../prometheus/interpolate/task.yml | 0 .../rds_broker/test_shared/task.sh | 0 .../rds_broker/test_shared/task.yml | 0 ci/tasks/cf/interpolate_cats/task.sh | 2 +- ci/tasks/cf/interpolate_cats/task.yml | 1 - operations/bosh/concourse-client.yml | 1 + operations/cloud-config/concourse.yml | 35 +++++++ operations/concourse/alb.yml | 4 +- operations/concourse/basic-auth.yml | 17 ---- operations/concourse/ephemeral_disk.yml | 5 - operations/concourse/fqdn.yml | 8 -- operations/concourse/iam_instance_profile.yml | 4 - operations/concourse/local-auth.yml | 13 +++ operations/concourse/public-network.yml | 21 ----- runtime-config/bosh/node-exporter-config.yml | 1 + terraform/base/cf.tf | 21 +---- terraform/base/concourse.tf | 31 ++++++- terraform/base/locals.tf | 3 +- terraform/base/outputs.tf | 39 ++++++++ terraform/base/prometheus.tf | 10 ++ terraform/base/routing.tf | 6 ++ terraform/base/subnets.tf | 13 +++ 106 files changed, 331 insertions(+), 145 deletions(-) create mode 100644 ci/cf.yml create mode 100755 ci/cf_pipeline.sh rename ci/{tasks => tasks.old}/bosh/db_connectivity_test/task.sh (100%) rename ci/{tasks => tasks.old}/bosh/db_connectivity_test/task.yml (100%) rename ci/{tasks => tasks.old}/bosh/deploy/task.sh (100%) rename ci/{tasks => tasks.old}/bosh/deploy/task.yml (100%) rename ci/{tasks => tasks.old}/bosh/deploy_node_exporter/task.sh (100%) rename ci/{tasks => tasks.old}/bosh/deploy_node_exporter/task.yml (100%) rename ci/{tasks => tasks.old}/bosh/destroy/task.sh (100%) rename ci/{tasks => tasks.old}/bosh/destroy/task.yml (100%) rename ci/{tasks => tasks.old}/bosh/get_database_vars/task.sh (100%) rename ci/{tasks => tasks.old}/bosh/get_database_vars/task.yml (100%) rename ci/{tasks => tasks.old}/bosh/get_terraform_vars/task.sh (100%) rename ci/{tasks => tasks.old}/bosh/get_terraform_vars/task.yml (100%) rename ci/{tasks => tasks.old}/bosh/interpolate/task.sh (100%) rename ci/{tasks => tasks.old}/bosh/interpolate/task.yml (100%) rename ci/{tasks => tasks.old}/bosh/test/task.sh (100%) rename ci/{tasks => tasks.old}/bosh/test/task.yml (100%) rename ci/{tasks => tasks.old}/cf/cloud_config/task.sh (100%) rename ci/{tasks => tasks.old}/cf/cloud_config/task.yml (100%) rename ci/{tasks => tasks.old}/cf/databases/db_connectivity_test/task.sh (100%) rename ci/{tasks => tasks.old}/cf/databases/db_connectivity_test/task.yml (100%) rename ci/{tasks => tasks.old}/cf/databases/get_database_vars/task.sh (100%) rename ci/{tasks => tasks.old}/cf/databases/get_database_vars/task.yml (100%) rename ci/{tasks => tasks.old}/cf/deploy_cf/task.sh (100%) rename ci/{tasks => tasks.old}/cf/deploy_cf/task.yml (100%) rename ci/{tasks => tasks.old}/cf/destroy/task.sh (100%) rename ci/{tasks => tasks.old}/cf/destroy/task.yml (100%) rename ci/{tasks => tasks.old}/cf/get_management_vars/task.sh (100%) rename ci/{tasks => tasks.old}/cf/get_management_vars/task.yml (100%) rename ci/{tasks => tasks.old}/cf/get_terraform_vars/task.sh (100%) rename ci/{tasks => tasks.old}/cf/get_terraform_vars/task.yml (100%) rename ci/{tasks => tasks.old}/cf/interpolate/task.sh (100%) rename ci/{tasks => tasks.old}/cf/interpolate/task.yml (100%) create mode 100755 ci/tasks.old/cf/interpolate_cats/task.sh create mode 100644 ci/tasks.old/cf/interpolate_cats/task.yml rename ci/{tasks => tasks.old}/cf/management/task.sh (100%) rename ci/{tasks => tasks.old}/cf/management/task.yml (100%) rename ci/{tasks => tasks.old}/cf/smoke_tests/task.sh (100%) rename ci/{tasks => tasks.old}/cf/smoke_tests/task.yml (100%) rename ci/{tasks => tasks.old}/cf/test/task.sh (100%) rename ci/{tasks => tasks.old}/cf/test/task.yml (100%) rename ci/{tasks => tasks.old}/cf/upload_stemcell/task.sh (100%) rename ci/{tasks => tasks.old}/cf/upload_stemcell/task.yml (100%) rename ci/{tasks => tasks.old}/common/delete_s3/task.sh (100%) rename ci/{tasks => tasks.old}/common/delete_s3/task.yml (100%) rename ci/{tasks => tasks.old}/common/get_cf_tester_credentials/task.sh (100%) rename ci/{tasks => tasks.old}/common/get_cf_tester_credentials/task.yml (100%) rename ci/{tasks => tasks.old}/common/test_connectivity/task.sh (100%) rename ci/{tasks => tasks.old}/common/test_connectivity/task.yml (100%) rename ci/{tasks => tasks.old}/common/test_endpoint/task.sh (100%) rename ci/{tasks => tasks.old}/common/test_endpoint/task.yml (100%) rename ci/{tasks => tasks.old}/concourse/test_fqdn/task.sh (100%) rename ci/{tasks => tasks.old}/concourse/test_fqdn/task.yml (100%) rename ci/{tasks => tasks.old}/concourse/test_iam_policy/task.sh (100%) rename ci/{tasks => tasks.old}/concourse/test_iam_policy/task.yml (100%) rename ci/{tasks => tasks.old}/destroy/empty_buckets/task.sh (100%) rename ci/{tasks => tasks.old}/destroy/empty_buckets/task.yml (100%) rename ci/{tasks => tasks.old}/jumpbox/deploy/task.sh (100%) rename ci/{tasks => tasks.old}/jumpbox/deploy/task.yml (100%) rename ci/{tasks => tasks.old}/jumpbox/destroy/task.sh (100%) rename ci/{tasks => tasks.old}/jumpbox/destroy/task.yml (100%) rename ci/{tasks => tasks.old}/jumpbox/get_terraform_vars/task.sh (100%) rename ci/{tasks => tasks.old}/jumpbox/get_terraform_vars/task.yml (100%) rename ci/{tasks => tasks.old}/jumpbox/interpolate/task.sh (100%) rename ci/{tasks => tasks.old}/jumpbox/interpolate/task.yml (100%) rename ci/{tasks => tasks.old}/jumpbox/test/task.sh (100%) rename ci/{tasks => tasks.old}/jumpbox/test/task.yml (100%) rename ci/{tasks => tasks.old}/jumpbox/test_fqdn/task.sh (100%) rename ci/{tasks => tasks.old}/jumpbox/test_fqdn/task.yml (100%) rename ci/{tasks => tasks.old}/prometheus/dashboard/task.sh (100%) rename ci/{tasks => tasks.old}/prometheus/dashboard/task.yml (100%) rename ci/{tasks => tasks.old}/prometheus/deploy/task.sh (100%) rename ci/{tasks => tasks.old}/prometheus/deploy/task.yml (100%) rename ci/{tasks => tasks.old}/prometheus/destroy/task.sh (100%) rename ci/{tasks => tasks.old}/prometheus/destroy/task.yml (100%) rename ci/{tasks => tasks.old}/prometheus/get_terraform_vars/task.sh (100%) rename ci/{tasks => tasks.old}/prometheus/get_terraform_vars/task.yml (100%) rename ci/{tasks => tasks.old}/prometheus/interpolate/task.sh (100%) rename ci/{tasks => tasks.old}/prometheus/interpolate/task.yml (100%) rename ci/{tasks => tasks.old}/rds_broker/test_shared/task.sh (100%) rename ci/{tasks => tasks.old}/rds_broker/test_shared/task.yml (100%) create mode 100644 operations/bosh/concourse-client.yml create mode 100644 operations/cloud-config/concourse.yml delete mode 100644 operations/concourse/basic-auth.yml delete mode 100644 operations/concourse/ephemeral_disk.yml delete mode 100644 operations/concourse/fqdn.yml delete mode 100644 operations/concourse/iam_instance_profile.yml create mode 100644 operations/concourse/local-auth.yml delete mode 100644 operations/concourse/public-network.yml diff --git a/Makefile b/Makefile index c9bfa8c2..b0217e69 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,12 @@ prometheus: ## Deploy Prometheus concourse: ## Deploy concourse @bin/deploy_concourse.sh +login_fly: ## Log in to fly + @bin/login_fly.sh + +pipelines: login_fly ## Deploy pipelines + @ci/cf_pipeline.sh + decode_aws_error: ## Decode AWS message @aws sts decode-authorization-message --encoded-message ${DECODE_MESSAGE} | jq -r .DecodedMessage | jq . diff --git a/bin/cloud_config.sh b/bin/cloud_config.sh index 59b8c935..7988d29c 100755 --- a/bin/cloud_config.sh +++ b/bin/cloud_config.sh @@ -20,6 +20,7 @@ $BOSH update-cloud-config -n \ -o ./operations/cloud-config/cf-s3-blobstore.yml \ -o ./operations/cloud-config/cf-rds-sec-group.yml \ -o ./operations/cloud-config/prometheus.yml \ + -o ./operations/cloud-config/concourse.yml \ -v az1="$(jq -r .availability_zones[0] < data/$ENVIRONMENT.tfvars)" \ -v az2="$(jq -r .availability_zones[1] < data/$ENVIRONMENT.tfvars)" \ -v az3="$(jq -r .availability_zones[2] < data/$ENVIRONMENT.tfvars)" \ @@ -51,4 +52,10 @@ $BOSH update-cloud-config -n \ -v reserved_prometheus_az1_cidr="$(output base .prometheus_subnet_reserved_cidr_blocks[0])" \ -v grafana_target_group_name="$(output base .grafana_target_group_name)" \ -v prometheus_target_group_name="$(output base .prometheus_target_group_name)" \ - -v alertmanager_target_group_name="$(output base .alertmanager_target_group_name)" + -v alertmanager_target_group_name="$(output base .alertmanager_target_group_name)" \ + -v concourse_subnet_az1_cidr="$(output base .concourse_subnet_cidr_blocks[0])" \ + -v concourse_subnet_az1_gateway="$(output base .concourse_subnet_gateway_ips[0])" \ + -v reserved_concourse_az1_cidr="$(output base .concourse_subnet_reserved_cidr_blocks[0])" \ + -v concourse_subnet_az1_id="$(output base .concourse_subnet_ids[0])" \ + -v concourse_security_group="$(output base .concourse_security_group_id)" \ + -v concourse_alb_target_group="$(output base .concourse_alb_target_group_name)" \ No newline at end of file diff --git a/bin/deploy_concourse.sh b/bin/deploy_concourse.sh index a40f16b9..87ead940 100755 --- a/bin/deploy_concourse.sh +++ b/bin/deploy_concourse.sh @@ -5,52 +5,47 @@ set -euo pipefail : $ENVIRONMENT -# BOSH create-env still requires the access credentials, rather than AWS_PROFILE -if [ -n "${AWS_PROFILE:-}" ]; then - export AWS_ACCESS_KEY_ID=$(aws configure get aws_access_key_id) - export AWS_SECRET_ACCESS_KEY=$(aws configure get aws_secret_access_key) -fi -: $AWS_ACCESS_KEY_ID -: $AWS_SECRET_ACCESS_KEY -: $CONCOURSE_TERRAFORM_STATE_FILE -: $CONCOURSE_STATE_FILE -: $CONCOURSE_CREDS_FILE -: $PRIVATE_KEY_FILE - -git submodule update --init - -# Convert the terraform outputs to YAML -_vars_file=tmp.$$.yml -trap 'rm -f $_vars_file' EXIT -terraform output -state="$CONCOURSE_TERRAFORM_STATE_FILE" -json | jq 'with_entries(.value = .value.value)' | yq r - >"$_vars_file" - -SUBMODULE=concourse-bosh-deployment - -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/creds.yml" "${CONCOURSE_CREDS_FILE}" || - echo "Remote concourse creds do not exist, assuming they need to be generated" - -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/state.json" "${CONCOURSE_STATE_FILE}" || - echo "Remote concourse state does not exist, assuming this is a new deployment" - -bosh create-env "$SUBMODULE"/lite/concourse.yml \ - -o "$SUBMODULE"/lite/infrastructures/aws.yml \ - -o operations/concourse/public-network.yml \ - -o operations/concourse/basic-auth.yml \ - -o operations/concourse/alb.yml \ - -o operations/concourse/fqdn.yml \ - -o operations/concourse/iam_instance_profile.yml \ - -o operations/concourse/ephemeral_disk.yml \ - -o operations/concourse/tags.yml \ - -l "$SUBMODULE"/versions.yml \ - -l "$_vars_file" \ - -v environment="$ENVIRONMENT" \ - -v access_key_id="$AWS_ACCESS_KEY_ID" \ - -v secret_access_key="$AWS_SECRET_ACCESS_KEY" \ - --var-file private_key="$PRIVATE_KEY_FILE" \ - --vars-store "$CONCOURSE_CREDS_FILE" \ - --state "$CONCOURSE_STATE_FILE" - -aws s3 cp "${CONCOURSE_CREDS_FILE}" "s3://ons-paas-${ENVIRONMENT}-states/concourse/creds.yml" --acl=private -aws s3 cp "${CONCOURSE_STATE_FILE}" "s3://ons-paas-${ENVIRONMENT}-states/concourse/state.json" --acl=private - +output() { + FILE=$1 + QUERY=$2 + + bin/outputs.sh $FILE | jq -r "$QUERY" +} + +secret() { + KEY=$1 + bin/secret.sh -e $ENVIRONMENT -d cf -k $KEY +} + +BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" + +CREDHUB_CA_CERT=/var/tmp/credhub_ca.$$.pem +bosh int --path /credhub_ca/ca "data/$ENVIRONMENT-bosh-variables.yml" >$CREDHUB_CA_CERT +CREDHUB_SERVER=https://$(output base .bosh_private_ip):8844 +CREDHUB_CLIENT=credhub-admin +CREDHUB_SECRET=$(bosh int --path /credhub_admin_client_secret "data/$ENVIRONMENT-bosh-variables.yml") + +cleanup() { + rm -f $CREDHUB_CA_CERT +} +trap cleanup EXIT + +$BOSH -d concourse deploy -n concourse-bosh-deployment/cluster/concourse.yml \ + -l concourse-bosh-deployment/versions.yml \ + -o concourse-bosh-deployment/cluster/operations/credhub.yml \ + -o ./operations/concourse/alb.yml \ + -o ./operations/concourse/tags.yml \ + -o ./operations/concourse/local-auth.yml \ + -v environment=$ENVIRONMENT \ + -v external_url=https://$(output base .concourse_fqdn) \ + -v network_name=concourse \ + -v web_vm_type=concourse \ + -v db_vm_type=concourse \ + -v db_persistent_disk_type=10GB \ + -v worker_vm_type=concourse \ + -v deployment_name=concourse \ + -v credhub_url="$CREDHUB_SERVER" \ + -v credhub_client_id="$CREDHUB_CLIENT" \ + -v credhub_client_secret="$CREDHUB_SECRET" \ + --var-file credhub_ca_cert="$CREDHUB_CA_CERT" \ No newline at end of file diff --git a/bin/login_fly.sh b/bin/login_fly.sh index 594ba26c..4777dbdc 100755 --- a/bin/login_fly.sh +++ b/bin/login_fly.sh @@ -2,16 +2,18 @@ set -euo pipefail -: $ENVIRONMENT -: ${USERNAME:=admin} -: $CONCOURSE_TERRAFORM_STATE_FILE -: $CONCOURSE_CREDS_FILE +USERNAME=admin + +while getopts 'e:u:' option; do + case $option in + e) export ENVIRONMENT="$OPTARG";; + u) USERNAME="$OPTARG";; + esac +done -# Grab pre-requisite files from S3 -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/tfstate.json" "${CONCOURSE_TERRAFORM_STATE_FILE}" -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/creds.yml" "${CONCOURSE_CREDS_FILE}" +: $ENVIRONMENT -FQDN=$(terraform output -state=${ENVIRONMENT}_concourse.tfstate.json concourse_fqdn) -PASSWORD=$(bosh int --path /admin_password ${ENVIRONMENT}_concourse.creds.yml) +PASSWORD=$(bin/secret.sh -e $ENVIRONMENT -d concourse -k admin_password) +FQDN=$(bin/outputs.sh base | jq -r .concourse_fqdn) fly login -t "$ENVIRONMENT" -u "$USERNAME" -p "$PASSWORD" -c "https://$FQDN" \ No newline at end of file diff --git a/bin/stemcells.sh b/bin/stemcells.sh index d411054f..efeda8d7 100755 --- a/bin/stemcells.sh +++ b/bin/stemcells.sh @@ -10,10 +10,16 @@ output() { bin/outputs.sh $FILE | jq -r "$QUERY" } -export IAAS_INFO=aws-xen-hvm -export STEMCELL_VERSION=$(bosh interpolate cf-deployment/cf-deployment.yml --path=/stemcells/alias=default/version) + +IAAS_INFO=aws-xen-hvm +CF_STEMCELL_VERSION=$(bosh interpolate cf-deployment/cf-deployment.yml --path=/stemcells/alias=default/version) +CONCOURSE_STEMCELL_VERSION=97.15 # FIXME: interpolate from somewhere BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" -if ! $BOSH -n stemcells | grep -q "$STEMCELL_VERSION"; then - $BOSH upload-stemcell https://bosh.io/d/stemcells/bosh-${IAAS_INFO}-ubuntu-trusty-go_agent?v=${STEMCELL_VERSION} +if ! $BOSH -n stemcells | grep -q "ubuntu-trusty.*$CF_STEMCELL_VERSION"; then + $BOSH upload-stemcell https://bosh.io/d/stemcells/bosh-${IAAS_INFO}-ubuntu-trusty-go_agent?v=${CF_STEMCELL_VERSION} +fi + +if ! $BOSH -n stemcells | grep -q "ubuntu-xenial.*$CONCOURSE_STEMCELL_VERSION"; then + $BOSH upload-stemcell https://bosh.io/d/stemcells/bosh-${IAAS_INFO}-ubuntu-xenial-go_agent?v=${CONCOURSE_STEMCELL_VERSION} fi \ No newline at end of file diff --git a/ci/cf.yml b/ci/cf.yml new file mode 100644 index 00000000..ce53498b --- /dev/null +++ b/ci/cf.yml @@ -0,0 +1,45 @@ +--- +resources: +- name: paas-bootstrap-git + type: git + source: + uri: https://github.com/ONSdigital/paas-bootstrap.git + branch: ((branch)) + +- name: cf-deployment-concourse-tasks-git + type: git + source: + uri: https://github.com/cloudfoundry/cf-acceptance-tests.git + +- name: cf-acceptance-tests-git + type: git + source: + uri: https://github.com/cloudfoundry/cf-acceptance-tests.git + + +jobs: +- name: cats + serial: true + serial_groups: [cats] + plan: + - aggregate: + - get: paas-bootstrap-git + - get: cf-deployment-concourse-tasks-git + - get: cf-acceptance-tests-git + - task: interpolate CATS config + file: paas-bootstrap-git/ci/tasks/cf/interpolate_cats/task.yml + params: + DOMAIN: ((domain)) + ENVIRONMENT: ((environment)) + CATS_CONFIG_FILE: paas-bootstrap-git/profiles/staging/cats_config.json + CF_ADMIN_PASSWORD: ((/bosh/cf/cf_admin_password)) + - task: run CATS tests + file: cf-deployment-concourse-tasks-git/run-cats/task.yml + input_mapping: + cf-deployment-concourse-tasks: cf-deployment-concourse-tasks-git + cf-acceptance-tests: cf-acceptance-tests-git + params: + CAPTURE_CONFIG: true + NODES: 6 + + diff --git a/ci/cf_pipeline.sh b/ci/cf_pipeline.sh new file mode 100755 index 00000000..f758d2d2 --- /dev/null +++ b/ci/cf_pipeline.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -euo pipefail + +: $ENVIRONMENT +: ${BRANCH:="$(git rev-parse --abbrev-ref HEAD)"} + +echo "Using branch '${BRANCH}'" + +DOMAIN="$(bin/outputs.sh base | jq -r .domain)" + +bin/login_fly.sh + +fly -t "$ENVIRONMENT" set-pipeline \ + -v environment="$ENVIRONMENT" \ + -v branch="$BRANCH" \ + -v domain="$DOMAIN" \ + -c ci/cf.yml -p cf -n + +fly -t "$ENVIRONMENT" unpause-pipeline -p cf +fly -t "$ENVIRONMENT" expose-pipeline -p cf diff --git a/ci/tasks/bosh/db_connectivity_test/task.sh b/ci/tasks.old/bosh/db_connectivity_test/task.sh similarity index 100% rename from ci/tasks/bosh/db_connectivity_test/task.sh rename to ci/tasks.old/bosh/db_connectivity_test/task.sh diff --git a/ci/tasks/bosh/db_connectivity_test/task.yml b/ci/tasks.old/bosh/db_connectivity_test/task.yml similarity index 100% rename from ci/tasks/bosh/db_connectivity_test/task.yml rename to ci/tasks.old/bosh/db_connectivity_test/task.yml diff --git a/ci/tasks/bosh/deploy/task.sh b/ci/tasks.old/bosh/deploy/task.sh similarity index 100% rename from ci/tasks/bosh/deploy/task.sh rename to ci/tasks.old/bosh/deploy/task.sh diff --git a/ci/tasks/bosh/deploy/task.yml b/ci/tasks.old/bosh/deploy/task.yml similarity index 100% rename from ci/tasks/bosh/deploy/task.yml rename to ci/tasks.old/bosh/deploy/task.yml diff --git a/ci/tasks/bosh/deploy_node_exporter/task.sh b/ci/tasks.old/bosh/deploy_node_exporter/task.sh similarity index 100% rename from ci/tasks/bosh/deploy_node_exporter/task.sh rename to ci/tasks.old/bosh/deploy_node_exporter/task.sh diff --git a/ci/tasks/bosh/deploy_node_exporter/task.yml b/ci/tasks.old/bosh/deploy_node_exporter/task.yml similarity index 100% rename from ci/tasks/bosh/deploy_node_exporter/task.yml rename to ci/tasks.old/bosh/deploy_node_exporter/task.yml diff --git a/ci/tasks/bosh/destroy/task.sh b/ci/tasks.old/bosh/destroy/task.sh similarity index 100% rename from ci/tasks/bosh/destroy/task.sh rename to ci/tasks.old/bosh/destroy/task.sh diff --git a/ci/tasks/bosh/destroy/task.yml b/ci/tasks.old/bosh/destroy/task.yml similarity index 100% rename from ci/tasks/bosh/destroy/task.yml rename to ci/tasks.old/bosh/destroy/task.yml diff --git a/ci/tasks/bosh/get_database_vars/task.sh b/ci/tasks.old/bosh/get_database_vars/task.sh similarity index 100% rename from ci/tasks/bosh/get_database_vars/task.sh rename to ci/tasks.old/bosh/get_database_vars/task.sh diff --git a/ci/tasks/bosh/get_database_vars/task.yml b/ci/tasks.old/bosh/get_database_vars/task.yml similarity index 100% rename from ci/tasks/bosh/get_database_vars/task.yml rename to ci/tasks.old/bosh/get_database_vars/task.yml diff --git a/ci/tasks/bosh/get_terraform_vars/task.sh b/ci/tasks.old/bosh/get_terraform_vars/task.sh similarity index 100% rename from ci/tasks/bosh/get_terraform_vars/task.sh rename to ci/tasks.old/bosh/get_terraform_vars/task.sh diff --git a/ci/tasks/bosh/get_terraform_vars/task.yml b/ci/tasks.old/bosh/get_terraform_vars/task.yml similarity index 100% rename from ci/tasks/bosh/get_terraform_vars/task.yml rename to ci/tasks.old/bosh/get_terraform_vars/task.yml diff --git a/ci/tasks/bosh/interpolate/task.sh b/ci/tasks.old/bosh/interpolate/task.sh similarity index 100% rename from ci/tasks/bosh/interpolate/task.sh rename to ci/tasks.old/bosh/interpolate/task.sh diff --git a/ci/tasks/bosh/interpolate/task.yml b/ci/tasks.old/bosh/interpolate/task.yml similarity index 100% rename from ci/tasks/bosh/interpolate/task.yml rename to ci/tasks.old/bosh/interpolate/task.yml diff --git a/ci/tasks/bosh/test/task.sh b/ci/tasks.old/bosh/test/task.sh similarity index 100% rename from ci/tasks/bosh/test/task.sh rename to ci/tasks.old/bosh/test/task.sh diff --git a/ci/tasks/bosh/test/task.yml b/ci/tasks.old/bosh/test/task.yml similarity index 100% rename from ci/tasks/bosh/test/task.yml rename to ci/tasks.old/bosh/test/task.yml diff --git a/ci/tasks/cf/cloud_config/task.sh b/ci/tasks.old/cf/cloud_config/task.sh similarity index 100% rename from ci/tasks/cf/cloud_config/task.sh rename to ci/tasks.old/cf/cloud_config/task.sh diff --git a/ci/tasks/cf/cloud_config/task.yml b/ci/tasks.old/cf/cloud_config/task.yml similarity index 100% rename from ci/tasks/cf/cloud_config/task.yml rename to ci/tasks.old/cf/cloud_config/task.yml diff --git a/ci/tasks/cf/databases/db_connectivity_test/task.sh b/ci/tasks.old/cf/databases/db_connectivity_test/task.sh similarity index 100% rename from ci/tasks/cf/databases/db_connectivity_test/task.sh rename to ci/tasks.old/cf/databases/db_connectivity_test/task.sh diff --git a/ci/tasks/cf/databases/db_connectivity_test/task.yml b/ci/tasks.old/cf/databases/db_connectivity_test/task.yml similarity index 100% rename from ci/tasks/cf/databases/db_connectivity_test/task.yml rename to ci/tasks.old/cf/databases/db_connectivity_test/task.yml diff --git a/ci/tasks/cf/databases/get_database_vars/task.sh b/ci/tasks.old/cf/databases/get_database_vars/task.sh similarity index 100% rename from ci/tasks/cf/databases/get_database_vars/task.sh rename to ci/tasks.old/cf/databases/get_database_vars/task.sh diff --git a/ci/tasks/cf/databases/get_database_vars/task.yml b/ci/tasks.old/cf/databases/get_database_vars/task.yml similarity index 100% rename from ci/tasks/cf/databases/get_database_vars/task.yml rename to ci/tasks.old/cf/databases/get_database_vars/task.yml diff --git a/ci/tasks/cf/deploy_cf/task.sh b/ci/tasks.old/cf/deploy_cf/task.sh similarity index 100% rename from ci/tasks/cf/deploy_cf/task.sh rename to ci/tasks.old/cf/deploy_cf/task.sh diff --git a/ci/tasks/cf/deploy_cf/task.yml b/ci/tasks.old/cf/deploy_cf/task.yml similarity index 100% rename from ci/tasks/cf/deploy_cf/task.yml rename to ci/tasks.old/cf/deploy_cf/task.yml diff --git a/ci/tasks/cf/destroy/task.sh b/ci/tasks.old/cf/destroy/task.sh similarity index 100% rename from ci/tasks/cf/destroy/task.sh rename to ci/tasks.old/cf/destroy/task.sh diff --git a/ci/tasks/cf/destroy/task.yml b/ci/tasks.old/cf/destroy/task.yml similarity index 100% rename from ci/tasks/cf/destroy/task.yml rename to ci/tasks.old/cf/destroy/task.yml diff --git a/ci/tasks/cf/get_management_vars/task.sh b/ci/tasks.old/cf/get_management_vars/task.sh similarity index 100% rename from ci/tasks/cf/get_management_vars/task.sh rename to ci/tasks.old/cf/get_management_vars/task.sh diff --git a/ci/tasks/cf/get_management_vars/task.yml b/ci/tasks.old/cf/get_management_vars/task.yml similarity index 100% rename from ci/tasks/cf/get_management_vars/task.yml rename to ci/tasks.old/cf/get_management_vars/task.yml diff --git a/ci/tasks/cf/get_terraform_vars/task.sh b/ci/tasks.old/cf/get_terraform_vars/task.sh similarity index 100% rename from ci/tasks/cf/get_terraform_vars/task.sh rename to ci/tasks.old/cf/get_terraform_vars/task.sh diff --git a/ci/tasks/cf/get_terraform_vars/task.yml b/ci/tasks.old/cf/get_terraform_vars/task.yml similarity index 100% rename from ci/tasks/cf/get_terraform_vars/task.yml rename to ci/tasks.old/cf/get_terraform_vars/task.yml diff --git a/ci/tasks/cf/interpolate/task.sh b/ci/tasks.old/cf/interpolate/task.sh similarity index 100% rename from ci/tasks/cf/interpolate/task.sh rename to ci/tasks.old/cf/interpolate/task.sh diff --git a/ci/tasks/cf/interpolate/task.yml b/ci/tasks.old/cf/interpolate/task.yml similarity index 100% rename from ci/tasks/cf/interpolate/task.yml rename to ci/tasks.old/cf/interpolate/task.yml diff --git a/ci/tasks.old/cf/interpolate_cats/task.sh b/ci/tasks.old/cf/interpolate_cats/task.sh new file mode 100755 index 00000000..acbf85f0 --- /dev/null +++ b/ci/tasks.old/cf/interpolate_cats/task.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -eu + +: $DOMAIN + +ADMIN_PASSWORD="$(bosh int cf-vars-s3/cf-variables.yml --path /cf_admin_password)" + +cat $CATS_CONFIG_FILE | jq " +.api = \"api.system.${DOMAIN}\" | +.apps_domain = \"apps.${DOMAIN}\" | +.admin_user = \"admin\" | +.admin_password = \"${ADMIN_PASSWORD}\" +" > integration-config/integration_config.json diff --git a/ci/tasks.old/cf/interpolate_cats/task.yml b/ci/tasks.old/cf/interpolate_cats/task.yml new file mode 100644 index 00000000..cb5d2d46 --- /dev/null +++ b/ci/tasks.old/cf/interpolate_cats/task.yml @@ -0,0 +1,21 @@ +--- +platform: linux +image_resource: + type: docker-image + source: { repository: onsdigital/paas-ci-gp, tag: latest } + +inputs: + - name: paas-bootstrap-git + - name: cf-acceptance-tests-git + - name: cf-vars-s3 + +outputs: + - name: integration-config + +run: + path: paas-bootstrap-git/ci/tasks/cf/interpolate_cats/task.sh + +params: + DOMAIN: + ENVIRONMENT: + CATS_CONFIG_FILE: diff --git a/ci/tasks/cf/management/task.sh b/ci/tasks.old/cf/management/task.sh similarity index 100% rename from ci/tasks/cf/management/task.sh rename to ci/tasks.old/cf/management/task.sh diff --git a/ci/tasks/cf/management/task.yml b/ci/tasks.old/cf/management/task.yml similarity index 100% rename from ci/tasks/cf/management/task.yml rename to ci/tasks.old/cf/management/task.yml diff --git a/ci/tasks/cf/smoke_tests/task.sh b/ci/tasks.old/cf/smoke_tests/task.sh similarity index 100% rename from ci/tasks/cf/smoke_tests/task.sh rename to ci/tasks.old/cf/smoke_tests/task.sh diff --git a/ci/tasks/cf/smoke_tests/task.yml b/ci/tasks.old/cf/smoke_tests/task.yml similarity index 100% rename from ci/tasks/cf/smoke_tests/task.yml rename to ci/tasks.old/cf/smoke_tests/task.yml diff --git a/ci/tasks/cf/test/task.sh b/ci/tasks.old/cf/test/task.sh similarity index 100% rename from ci/tasks/cf/test/task.sh rename to ci/tasks.old/cf/test/task.sh diff --git a/ci/tasks/cf/test/task.yml b/ci/tasks.old/cf/test/task.yml similarity index 100% rename from ci/tasks/cf/test/task.yml rename to ci/tasks.old/cf/test/task.yml diff --git a/ci/tasks/cf/upload_stemcell/task.sh b/ci/tasks.old/cf/upload_stemcell/task.sh similarity index 100% rename from ci/tasks/cf/upload_stemcell/task.sh rename to ci/tasks.old/cf/upload_stemcell/task.sh diff --git a/ci/tasks/cf/upload_stemcell/task.yml b/ci/tasks.old/cf/upload_stemcell/task.yml similarity index 100% rename from ci/tasks/cf/upload_stemcell/task.yml rename to ci/tasks.old/cf/upload_stemcell/task.yml diff --git a/ci/tasks/common/delete_s3/task.sh b/ci/tasks.old/common/delete_s3/task.sh similarity index 100% rename from ci/tasks/common/delete_s3/task.sh rename to ci/tasks.old/common/delete_s3/task.sh diff --git a/ci/tasks/common/delete_s3/task.yml b/ci/tasks.old/common/delete_s3/task.yml similarity index 100% rename from ci/tasks/common/delete_s3/task.yml rename to ci/tasks.old/common/delete_s3/task.yml diff --git a/ci/tasks/common/get_cf_tester_credentials/task.sh b/ci/tasks.old/common/get_cf_tester_credentials/task.sh similarity index 100% rename from ci/tasks/common/get_cf_tester_credentials/task.sh rename to ci/tasks.old/common/get_cf_tester_credentials/task.sh diff --git a/ci/tasks/common/get_cf_tester_credentials/task.yml b/ci/tasks.old/common/get_cf_tester_credentials/task.yml similarity index 100% rename from ci/tasks/common/get_cf_tester_credentials/task.yml rename to ci/tasks.old/common/get_cf_tester_credentials/task.yml diff --git a/ci/tasks/common/test_connectivity/task.sh b/ci/tasks.old/common/test_connectivity/task.sh similarity index 100% rename from ci/tasks/common/test_connectivity/task.sh rename to ci/tasks.old/common/test_connectivity/task.sh diff --git a/ci/tasks/common/test_connectivity/task.yml b/ci/tasks.old/common/test_connectivity/task.yml similarity index 100% rename from ci/tasks/common/test_connectivity/task.yml rename to ci/tasks.old/common/test_connectivity/task.yml diff --git a/ci/tasks/common/test_endpoint/task.sh b/ci/tasks.old/common/test_endpoint/task.sh similarity index 100% rename from ci/tasks/common/test_endpoint/task.sh rename to ci/tasks.old/common/test_endpoint/task.sh diff --git a/ci/tasks/common/test_endpoint/task.yml b/ci/tasks.old/common/test_endpoint/task.yml similarity index 100% rename from ci/tasks/common/test_endpoint/task.yml rename to ci/tasks.old/common/test_endpoint/task.yml diff --git a/ci/tasks/concourse/test_fqdn/task.sh b/ci/tasks.old/concourse/test_fqdn/task.sh similarity index 100% rename from ci/tasks/concourse/test_fqdn/task.sh rename to ci/tasks.old/concourse/test_fqdn/task.sh diff --git a/ci/tasks/concourse/test_fqdn/task.yml b/ci/tasks.old/concourse/test_fqdn/task.yml similarity index 100% rename from ci/tasks/concourse/test_fqdn/task.yml rename to ci/tasks.old/concourse/test_fqdn/task.yml diff --git a/ci/tasks/concourse/test_iam_policy/task.sh b/ci/tasks.old/concourse/test_iam_policy/task.sh similarity index 100% rename from ci/tasks/concourse/test_iam_policy/task.sh rename to ci/tasks.old/concourse/test_iam_policy/task.sh diff --git a/ci/tasks/concourse/test_iam_policy/task.yml b/ci/tasks.old/concourse/test_iam_policy/task.yml similarity index 100% rename from ci/tasks/concourse/test_iam_policy/task.yml rename to ci/tasks.old/concourse/test_iam_policy/task.yml diff --git a/ci/tasks/destroy/empty_buckets/task.sh b/ci/tasks.old/destroy/empty_buckets/task.sh similarity index 100% rename from ci/tasks/destroy/empty_buckets/task.sh rename to ci/tasks.old/destroy/empty_buckets/task.sh diff --git a/ci/tasks/destroy/empty_buckets/task.yml b/ci/tasks.old/destroy/empty_buckets/task.yml similarity index 100% rename from ci/tasks/destroy/empty_buckets/task.yml rename to ci/tasks.old/destroy/empty_buckets/task.yml diff --git a/ci/tasks/jumpbox/deploy/task.sh b/ci/tasks.old/jumpbox/deploy/task.sh similarity index 100% rename from ci/tasks/jumpbox/deploy/task.sh rename to ci/tasks.old/jumpbox/deploy/task.sh diff --git a/ci/tasks/jumpbox/deploy/task.yml b/ci/tasks.old/jumpbox/deploy/task.yml similarity index 100% rename from ci/tasks/jumpbox/deploy/task.yml rename to ci/tasks.old/jumpbox/deploy/task.yml diff --git a/ci/tasks/jumpbox/destroy/task.sh b/ci/tasks.old/jumpbox/destroy/task.sh similarity index 100% rename from ci/tasks/jumpbox/destroy/task.sh rename to ci/tasks.old/jumpbox/destroy/task.sh diff --git a/ci/tasks/jumpbox/destroy/task.yml b/ci/tasks.old/jumpbox/destroy/task.yml similarity index 100% rename from ci/tasks/jumpbox/destroy/task.yml rename to ci/tasks.old/jumpbox/destroy/task.yml diff --git a/ci/tasks/jumpbox/get_terraform_vars/task.sh b/ci/tasks.old/jumpbox/get_terraform_vars/task.sh similarity index 100% rename from ci/tasks/jumpbox/get_terraform_vars/task.sh rename to ci/tasks.old/jumpbox/get_terraform_vars/task.sh diff --git a/ci/tasks/jumpbox/get_terraform_vars/task.yml b/ci/tasks.old/jumpbox/get_terraform_vars/task.yml similarity index 100% rename from ci/tasks/jumpbox/get_terraform_vars/task.yml rename to ci/tasks.old/jumpbox/get_terraform_vars/task.yml diff --git a/ci/tasks/jumpbox/interpolate/task.sh b/ci/tasks.old/jumpbox/interpolate/task.sh similarity index 100% rename from ci/tasks/jumpbox/interpolate/task.sh rename to ci/tasks.old/jumpbox/interpolate/task.sh diff --git a/ci/tasks/jumpbox/interpolate/task.yml b/ci/tasks.old/jumpbox/interpolate/task.yml similarity index 100% rename from ci/tasks/jumpbox/interpolate/task.yml rename to ci/tasks.old/jumpbox/interpolate/task.yml diff --git a/ci/tasks/jumpbox/test/task.sh b/ci/tasks.old/jumpbox/test/task.sh similarity index 100% rename from ci/tasks/jumpbox/test/task.sh rename to ci/tasks.old/jumpbox/test/task.sh diff --git a/ci/tasks/jumpbox/test/task.yml b/ci/tasks.old/jumpbox/test/task.yml similarity index 100% rename from ci/tasks/jumpbox/test/task.yml rename to ci/tasks.old/jumpbox/test/task.yml diff --git a/ci/tasks/jumpbox/test_fqdn/task.sh b/ci/tasks.old/jumpbox/test_fqdn/task.sh similarity index 100% rename from ci/tasks/jumpbox/test_fqdn/task.sh rename to ci/tasks.old/jumpbox/test_fqdn/task.sh diff --git a/ci/tasks/jumpbox/test_fqdn/task.yml b/ci/tasks.old/jumpbox/test_fqdn/task.yml similarity index 100% rename from ci/tasks/jumpbox/test_fqdn/task.yml rename to ci/tasks.old/jumpbox/test_fqdn/task.yml diff --git a/ci/tasks/prometheus/dashboard/task.sh b/ci/tasks.old/prometheus/dashboard/task.sh similarity index 100% rename from ci/tasks/prometheus/dashboard/task.sh rename to ci/tasks.old/prometheus/dashboard/task.sh diff --git a/ci/tasks/prometheus/dashboard/task.yml b/ci/tasks.old/prometheus/dashboard/task.yml similarity index 100% rename from ci/tasks/prometheus/dashboard/task.yml rename to ci/tasks.old/prometheus/dashboard/task.yml diff --git a/ci/tasks/prometheus/deploy/task.sh b/ci/tasks.old/prometheus/deploy/task.sh similarity index 100% rename from ci/tasks/prometheus/deploy/task.sh rename to ci/tasks.old/prometheus/deploy/task.sh diff --git a/ci/tasks/prometheus/deploy/task.yml b/ci/tasks.old/prometheus/deploy/task.yml similarity index 100% rename from ci/tasks/prometheus/deploy/task.yml rename to ci/tasks.old/prometheus/deploy/task.yml diff --git a/ci/tasks/prometheus/destroy/task.sh b/ci/tasks.old/prometheus/destroy/task.sh similarity index 100% rename from ci/tasks/prometheus/destroy/task.sh rename to ci/tasks.old/prometheus/destroy/task.sh diff --git a/ci/tasks/prometheus/destroy/task.yml b/ci/tasks.old/prometheus/destroy/task.yml similarity index 100% rename from ci/tasks/prometheus/destroy/task.yml rename to ci/tasks.old/prometheus/destroy/task.yml diff --git a/ci/tasks/prometheus/get_terraform_vars/task.sh b/ci/tasks.old/prometheus/get_terraform_vars/task.sh similarity index 100% rename from ci/tasks/prometheus/get_terraform_vars/task.sh rename to ci/tasks.old/prometheus/get_terraform_vars/task.sh diff --git a/ci/tasks/prometheus/get_terraform_vars/task.yml b/ci/tasks.old/prometheus/get_terraform_vars/task.yml similarity index 100% rename from ci/tasks/prometheus/get_terraform_vars/task.yml rename to ci/tasks.old/prometheus/get_terraform_vars/task.yml diff --git a/ci/tasks/prometheus/interpolate/task.sh b/ci/tasks.old/prometheus/interpolate/task.sh similarity index 100% rename from ci/tasks/prometheus/interpolate/task.sh rename to ci/tasks.old/prometheus/interpolate/task.sh diff --git a/ci/tasks/prometheus/interpolate/task.yml b/ci/tasks.old/prometheus/interpolate/task.yml similarity index 100% rename from ci/tasks/prometheus/interpolate/task.yml rename to ci/tasks.old/prometheus/interpolate/task.yml diff --git a/ci/tasks/rds_broker/test_shared/task.sh b/ci/tasks.old/rds_broker/test_shared/task.sh similarity index 100% rename from ci/tasks/rds_broker/test_shared/task.sh rename to ci/tasks.old/rds_broker/test_shared/task.sh diff --git a/ci/tasks/rds_broker/test_shared/task.yml b/ci/tasks.old/rds_broker/test_shared/task.yml similarity index 100% rename from ci/tasks/rds_broker/test_shared/task.yml rename to ci/tasks.old/rds_broker/test_shared/task.yml diff --git a/ci/tasks/cf/interpolate_cats/task.sh b/ci/tasks/cf/interpolate_cats/task.sh index acbf85f0..3294f27f 100755 --- a/ci/tasks/cf/interpolate_cats/task.sh +++ b/ci/tasks/cf/interpolate_cats/task.sh @@ -10,5 +10,5 @@ cat $CATS_CONFIG_FILE | jq " .api = \"api.system.${DOMAIN}\" | .apps_domain = \"apps.${DOMAIN}\" | .admin_user = \"admin\" | -.admin_password = \"${ADMIN_PASSWORD}\" +.admin_password = \"${CF_ADMIN_PASSWORD}\" " > integration-config/integration_config.json diff --git a/ci/tasks/cf/interpolate_cats/task.yml b/ci/tasks/cf/interpolate_cats/task.yml index cb5d2d46..7ebfe7f7 100644 --- a/ci/tasks/cf/interpolate_cats/task.yml +++ b/ci/tasks/cf/interpolate_cats/task.yml @@ -7,7 +7,6 @@ image_resource: inputs: - name: paas-bootstrap-git - name: cf-acceptance-tests-git - - name: cf-vars-s3 outputs: - name: integration-config diff --git a/operations/bosh/concourse-client.yml b/operations/bosh/concourse-client.yml new file mode 100644 index 00000000..ed97d539 --- /dev/null +++ b/operations/bosh/concourse-client.yml @@ -0,0 +1 @@ +--- diff --git a/operations/cloud-config/concourse.yml b/operations/cloud-config/concourse.yml new file mode 100644 index 00000000..11b5478f --- /dev/null +++ b/operations/cloud-config/concourse.yml @@ -0,0 +1,35 @@ +--- +- type: replace + path: /networks/name=concourse? + value: + name: concourse + type: manual + subnets: + - range: ((concourse_subnet_az1_cidr)) + gateway: ((concourse_subnet_az1_gateway)) + az: z1 + dns: + - ((private_dns_nameserver)) + - 1.1.1.1 + reserved: [((reserved_concourse_az1_cidr))] + cloud_properties: + subnet: ((concourse_subnet_az1_id)) + security_groups: [ ((concourse_security_group))] + +- type: replace + path: /vm_types/- + value: + name: concourse + network: concourse + ephemeral_disk: + size: 100_000 + type: gp2 + cloud_properties: + instance_type: c5.large + +- type: replace + path: /vm_extensions/- + value: + name: concourse_web + cloud_properties: + lb_target_groups: [((concourse_alb_target_group))] \ No newline at end of file diff --git a/operations/concourse/alb.yml b/operations/concourse/alb.yml index 7b93aba0..a949f802 100644 --- a/operations/concourse/alb.yml +++ b/operations/concourse/alb.yml @@ -1,4 +1,4 @@ --- - type: replace - path: /resource_pools/name=vms/cloud_properties/lb_target_groups? - value: [((concourse_alb_target_group))] + path: /instance_groups/name=web/vm_extensions?/- + value: concourse_web diff --git a/operations/concourse/basic-auth.yml b/operations/concourse/basic-auth.yml deleted file mode 100644 index 9321cd7c..00000000 --- a/operations/concourse/basic-auth.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -- type: remove - path: /instance_groups/name=concourse/jobs/name=atc/properties/no_really_i_dont_want_any_auth - -- type: replace - path: /instance_groups/name=concourse/jobs/name=atc/properties/basic_auth_username? - value: admin - -- type: replace - path: /instance_groups/name=concourse/jobs/name=atc/properties/basic_auth_password? - value: ((admin_password)) - -- type: replace - path: /variables/- - value: - type: password - name: admin_password \ No newline at end of file diff --git a/operations/concourse/ephemeral_disk.yml b/operations/concourse/ephemeral_disk.yml deleted file mode 100644 index 1df5f5c1..00000000 --- a/operations/concourse/ephemeral_disk.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -# Configure sizes -- type: replace - path: /resource_pools/name=vms/cloud_properties/ephemeral_disk/size - value: 100_000 \ No newline at end of file diff --git a/operations/concourse/fqdn.yml b/operations/concourse/fqdn.yml deleted file mode 100644 index ff7ea7a5..00000000 --- a/operations/concourse/fqdn.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -- type: replace - path: /instance_groups/name=concourse/jobs/name=atc/properties/external_url - value: https://((concourse_fqdn)) - -- type: replace - path: /cloud_provider/mbus - value: https://mbus:((mbus_bootstrap_password))@((concourse_direct_fqdn)):6868 diff --git a/operations/concourse/iam_instance_profile.yml b/operations/concourse/iam_instance_profile.yml deleted file mode 100644 index 641cde89..00000000 --- a/operations/concourse/iam_instance_profile.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- type: replace - path: /resource_pools/name=vms/cloud_properties/iam_instance_profile? - value: ((concourse_iam_instance_profile)) diff --git a/operations/concourse/local-auth.yml b/operations/concourse/local-auth.yml new file mode 100644 index 00000000..84b7324e --- /dev/null +++ b/operations/concourse/local-auth.yml @@ -0,0 +1,13 @@ +- type: replace + path: /instance_groups/name=web/jobs/name=atc/properties/main_team?/auth/local/users + value: ['admin'] + +- type: replace + path: /instance_groups/name=web/jobs/name=atc/properties/add_local_users? + value: ['admin:((admin_password))'] + +- type: replace + path: /variables/- + value: + type: password + name: admin_password \ No newline at end of file diff --git a/operations/concourse/public-network.yml b/operations/concourse/public-network.yml deleted file mode 100644 index e4ccf92b..00000000 --- a/operations/concourse/public-network.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -- type: replace - path: /networks/- - value: - name: public - type: vip - -- type: replace - path: /networks/name=default/subnets/0/dns/0:before - value: ((vpc_dns_nameserver)) - -- type: replace - path: /instance_groups/name=concourse/networks/- - value: - name: public - static_ips: [((public_ip))] - -- type: replace - path: /cloud_provider/ssh_tunnel/host? - value: ((concourse_direct_fqdn)) - diff --git a/runtime-config/bosh/node-exporter-config.yml b/runtime-config/bosh/node-exporter-config.yml index 80c82170..d8bd5a89 100644 --- a/runtime-config/bosh/node-exporter-config.yml +++ b/runtime-config/bosh/node-exporter-config.yml @@ -12,4 +12,5 @@ addons: include: stemcell: - os: ubuntu-trusty + - os: ubuntu-xenial properties: {} diff --git a/terraform/base/cf.tf b/terraform/base/cf.tf index 7039ef8d..dc35eab2 100644 --- a/terraform/base/cf.tf +++ b/terraform/base/cf.tf @@ -370,35 +370,16 @@ resource "aws_security_group" "cf_ssh_lb" { } } -resource "aws_security_group_rule" "allow_tcp_2222_from_whitelist" { - security_group_id = "${aws_security_group.cf_ssh_lb.id}" - type = "ingress" - protocol = "tcp" - from_port = 2222 - to_port = 2222 - cidr_blocks = ["${var.ingress_whitelist}"] - description = "Allow SSH proxy traffic from whitelist" -} - resource "aws_security_group_rule" "allow_tcp_2222_from_nat_gateways" { security_group_id = "${aws_security_group.cf_ssh_lb.id}" type = "ingress" protocol = "tcp" from_port = 2222 to_port = 2222 - cidr_blocks = ["${formatlist("%s/32", aws_nat_gateway.nat.*.public_ip)}"] + cidr_blocks = ["${local.loadbalancer_whitelist}"] description = "Allow SSH proxy traffic from internal components" } -resource "aws_security_group_rule" "allow_tcp_2222_from_concourse" { - security_group_id = "${aws_security_group.cf_ssh_lb.id}" - type = "ingress" - protocol = "tcp" - from_port = 2222 - to_port = 2222 - cidr_blocks = ["${aws_eip.concourse.public_ip}/32"] - description = "Allow SSH proxy traffic from Concourse" -} resource "aws_security_group_rule" "allow_tcp_2222_to_proxies" { security_group_id = "${aws_security_group.cf_ssh_lb.id}" diff --git a/terraform/base/concourse.tf b/terraform/base/concourse.tf index b7796af7..f2bd35c4 100644 --- a/terraform/base/concourse.tf +++ b/terraform/base/concourse.tf @@ -101,8 +101,6 @@ resource "aws_security_group_rule" "jumpbox_ssh_concourse" { source_security_group_id = "${aws_security_group.concourse.id}" } - - resource "aws_security_group_rule" "concourse_outbound" { security_group_id = "${aws_security_group.concourse.id}" type = "egress" @@ -112,6 +110,33 @@ resource "aws_security_group_rule" "concourse_outbound" { cidr_blocks = ["0.0.0.0/0"] } +resource "aws_security_group_rule" "concourse_rule_tcp" { + security_group_id = "${aws_security_group.concourse.id}" + type = "ingress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + self = true +} + +resource "aws_security_group_rule" "concourse_rule_udp" { + security_group_id = "${aws_security_group.concourse.id}" + type = "ingress" + protocol = "udp" + from_port = 0 + to_port = 65535 + self = true +} + +resource "aws_security_group_rule" "concourse_rule_icmp" { + security_group_id = "${aws_security_group.concourse.id}" + type = "ingress" + protocol = "icmp" + from_port = -1 + to_port = -1 + self = true +} + resource "aws_security_group" "concourse_alb" { name = "${var.environment}_concourse_alb_security_group" description = "Concourse public access" @@ -129,7 +154,7 @@ resource "aws_security_group_rule" "concourse_alb_https" { protocol = "tcp" from_port = 443 to_port = 443 - cidr_blocks = "${var.ingress_whitelist}" + cidr_blocks = ["${local.loadbalancer_whitelist}"] } resource "aws_security_group_rule" "concourse_alb_to_web" { diff --git a/terraform/base/locals.tf b/terraform/base/locals.tf index dbb9e23f..31b4f48b 100644 --- a/terraform/base/locals.tf +++ b/terraform/base/locals.tf @@ -7,10 +7,11 @@ locals { services_subnets = "${var.cidr_blocks["services"]}" rds_subnets = "${var.cidr_blocks["rds"]}" prometheus_subnets = "${var.cidr_blocks["prometheus"]}" + concourse_subnets = "${var.cidr_blocks["concourse"]}" bosh_subnet_id = "${aws_subnet.internal.*.id[var.bosh_availability_zone_index]}" bosh_subnet_cidr_block = "${aws_subnet.internal.*.cidr_block[var.bosh_availability_zone_index]}" bosh_private_ip = "${cidrhost(aws_subnet.internal.*.cidr_block[var.bosh_availability_zone_index], 6)}" bosh_gateway_ip = "${cidrhost(aws_subnet.internal.*.cidr_block[var.bosh_availability_zone_index], 1)}" - loadbalancer_whitelist = ["${concat(var.ingress_whitelist,formatlist("%s/32", concat(list(aws_eip.concourse.public_ip), aws_nat_gateway.nat.*.public_ip)))}"] + loadbalancer_whitelist = ["${concat(var.ingress_whitelist,formatlist("%s/32", aws_nat_gateway.nat.*.public_ip))}"] } diff --git a/terraform/base/outputs.tf b/terraform/base/outputs.tf index 87d52551..5152b06a 100644 --- a/terraform/base/outputs.tf +++ b/terraform/base/outputs.tf @@ -199,4 +199,43 @@ output "alertmanager_fqdn" { output "cf_traffic_controller_port" { value = "${aws_lb_listener.cf_4443.port}" +} + +output "concourse_subnet_gateway_ips" { + value = [ + "${cidrhost(aws_subnet.concourse.*.cidr_block[0],1)}", + "${cidrhost(aws_subnet.concourse.*.cidr_block[1],1)}", + "${cidrhost(aws_subnet.concourse.*.cidr_block[2],1)}" + ] +} + +# NASTY HACK ALERT - we cannot find a way in terraform to perform an action on all elements of a list +# so you will have to change this if you add more AZs +output "concourse_subnet_reserved_cidr_blocks" { + value = [ + "${cidrsubnet(aws_subnet.concourse.*.cidr_block[0],6,0)}", + "${cidrsubnet(aws_subnet.concourse.*.cidr_block[1],6,0)}", + "${cidrsubnet(aws_subnet.concourse.*.cidr_block[2],6,0)}" + ] +} + +output "concourse_subnet_ids" { + value = ["${aws_subnet.concourse.*.id}"] +} + +output "concourse_subnet_cidr_blocks" { + value = ["${aws_subnet.concourse.*.cidr_block}"] +} + + +output "concourse_security_group_id" { + value = "${aws_security_group.concourse.id}" +} + +output "concourse_alb_target_group_name" { + value = "${aws_lb_target_group.concourse.name}" +} + +output "concourse_fqdn" { + value = "${aws_route53_record.concourse.name}" } \ No newline at end of file diff --git a/terraform/base/prometheus.tf b/terraform/base/prometheus.tf index b7861717..b9f89b43 100644 --- a/terraform/base/prometheus.tf +++ b/terraform/base/prometheus.tf @@ -238,6 +238,16 @@ resource "aws_security_group_rule" "prometheus_cf_node_exporters" { source_security_group_id = "${aws_security_group.prometheus.id}" } +resource "aws_security_group_rule" "prometheus_concourse_node_exporters" { + security_group_id = "${aws_security_group.concourse.id}" + type = "ingress" + protocol = "tcp" + from_port = 9100 + to_port = 9100 + description = "Allow prometheus to access concourse node exporter" + source_security_group_id = "${aws_security_group.prometheus.id}" +} + resource "aws_security_group_rule" "prometheus_cf_nats" { security_group_id = "${aws_security_group.internal.id}" type = "ingress" diff --git a/terraform/base/routing.tf b/terraform/base/routing.tf index 718ad699..16e8ad92 100644 --- a/terraform/base/routing.tf +++ b/terraform/base/routing.tf @@ -62,4 +62,10 @@ resource "aws_route_table_association" "prometheus" { count = "${local.num_azs}" subnet_id = "${element(aws_subnet.prometheus.*.id, count.index)}" route_table_id = "${element(aws_route_table.az.*.id, count.index)}" +} + +resource "aws_route_table_association" "concourse" { + count = "${local.num_azs}" + subnet_id = "${element(aws_subnet.concourse.*.id, count.index)}" + route_table_id = "${element(aws_route_table.az.*.id, count.index)}" } \ No newline at end of file diff --git a/terraform/base/subnets.tf b/terraform/base/subnets.tf index e9d4e0a1..cec21eb4 100644 --- a/terraform/base/subnets.tf +++ b/terraform/base/subnets.tf @@ -62,4 +62,17 @@ resource "aws_subnet" "prometheus" { Environment = "${var.environment}" Visibility = "private" } +} + +resource "aws_subnet" "concourse" { + count = "${local.num_azs}" + vpc_id = "${aws_vpc.default.id}" + cidr_block = "${element(local.concourse_subnets, count.index)}" + availability_zone = "${element(var.availability_zones, count.index)}" + + tags { + Name = "${var.environment}-concourse-az${count.index+1}-subnet" + Environment = "${var.environment}" + Visibility = "private" + } } \ No newline at end of file From 50cb0486a9acfd90fc34075190e85a941fd907bc Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Thu, 6 Sep 2018 16:23:02 +0100 Subject: [PATCH 16/71] Propagate CF_ADMIN_PASSWORD --- ci/tasks/cf/interpolate_cats/task.sh | 3 +-- ci/tasks/cf/interpolate_cats/task.yml | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/tasks/cf/interpolate_cats/task.sh b/ci/tasks/cf/interpolate_cats/task.sh index 3294f27f..acc08067 100755 --- a/ci/tasks/cf/interpolate_cats/task.sh +++ b/ci/tasks/cf/interpolate_cats/task.sh @@ -3,8 +3,7 @@ set -eu : $DOMAIN - -ADMIN_PASSWORD="$(bosh int cf-vars-s3/cf-variables.yml --path /cf_admin_password)" +: $CF_ADMIN_PASSWORD cat $CATS_CONFIG_FILE | jq " .api = \"api.system.${DOMAIN}\" | diff --git a/ci/tasks/cf/interpolate_cats/task.yml b/ci/tasks/cf/interpolate_cats/task.yml index 7ebfe7f7..ef51cd35 100644 --- a/ci/tasks/cf/interpolate_cats/task.yml +++ b/ci/tasks/cf/interpolate_cats/task.yml @@ -18,3 +18,4 @@ params: DOMAIN: ENVIRONMENT: CATS_CONFIG_FILE: + CF_ADMIN_PASSWORD: From 7fde2f71943a5c04b22abb3a7046568fbd8186c7 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Thu, 6 Sep 2018 17:22:56 +0100 Subject: [PATCH 17/71] CATS pipeline --- Makefile | 3 +++ bin/concourse_creds.sh | 17 ----------------- bin/concourse_password.sh | 3 +-- bin/deploy_concourse.sh | 1 + bin/set_concourse_secrets.sh | 14 ++++++++++++++ ci/cf.yml | 4 ++-- ci/cf_pipeline.sh | 2 -- operations/cloud-config/concourse.yml | 6 +++--- .../concourse/skip-credhub-tls-validation.yml | 4 ++++ 9 files changed, 28 insertions(+), 26 deletions(-) delete mode 100755 bin/concourse_creds.sh create mode 100755 bin/set_concourse_secrets.sh create mode 100644 operations/concourse/skip-credhub-tls-validation.yml diff --git a/Makefile b/Makefile index b0217e69..6735ccd0 100644 --- a/Makefile +++ b/Makefile @@ -59,6 +59,9 @@ concourse: ## Deploy concourse login_fly: ## Log in to fly @bin/login_fly.sh +set_concourse_secrets: ## Set the secrets that Concourse needs to run its pipelines + @bin/set_concourse_secrets.sh + pipelines: login_fly ## Deploy pipelines @ci/cf_pipeline.sh diff --git a/bin/concourse_creds.sh b/bin/concourse_creds.sh deleted file mode 100755 index c9f3f362..00000000 --- a/bin/concourse_creds.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# Retrieves the concourse admin credentails - -set -euo pipefail - -: $ENVIRONMENT -if [ -z "${AWS_PROFILE:-}" ]; then - echo "Profile not set" - exit 1 -fi - -_tmp_creds=tmp$$.tmp_creds.yml -trap 'rm -f $_tmp_creds' EXIT - -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/creds.yml" "${_tmp_creds}" -bosh int --path /admin_password $_tmp_creds diff --git a/bin/concourse_password.sh b/bin/concourse_password.sh index 3c295a8c..71dd1879 100755 --- a/bin/concourse_password.sh +++ b/bin/concourse_password.sh @@ -10,5 +10,4 @@ done : $ENVIRONMENT -PASSPATH='/instance_groups/name=uaa/jobs/name=uaa/properties/uaa/scim/users/name=admin/password' -bosh int --path /admin_password <(aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/creds.yml" -) \ No newline at end of file +bin/credhub_credentials.sh -e $ENVIRONMENT credhub get -n /bosh/concourse/admin_password -j | jq -r .value \ No newline at end of file diff --git a/bin/deploy_concourse.sh b/bin/deploy_concourse.sh index 87ead940..bcff34a3 100755 --- a/bin/deploy_concourse.sh +++ b/bin/deploy_concourse.sh @@ -37,6 +37,7 @@ $BOSH -d concourse deploy -n concourse-bosh-deployment/cluster/concourse.yml \ -o ./operations/concourse/alb.yml \ -o ./operations/concourse/tags.yml \ -o ./operations/concourse/local-auth.yml \ + -o ./operations/concourse/skip-credhub-tls-validation.yml \ -v environment=$ENVIRONMENT \ -v external_url=https://$(output base .concourse_fqdn) \ -v network_name=concourse \ diff --git a/bin/set_concourse_secrets.sh b/bin/set_concourse_secrets.sh new file mode 100755 index 00000000..f45e30e4 --- /dev/null +++ b/bin/set_concourse_secrets.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -euo pipefail + +while getopts e: option; do + case $option in + e) export ENVIRONMENT="$OPTARG";; + esac +done + +: $ENVIRONMENT + +bin/login_fly.sh +bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /concourse/main/cf_admin_password --type value --value $(bin/cf_password.sh -e $ENVIRONMENT) \ No newline at end of file diff --git a/ci/cf.yml b/ci/cf.yml index ce53498b..b1e45b31 100644 --- a/ci/cf.yml +++ b/ci/cf.yml @@ -9,7 +9,7 @@ resources: - name: cf-deployment-concourse-tasks-git type: git source: - uri: https://github.com/cloudfoundry/cf-acceptance-tests.git + uri: https://github.com/cloudfoundry/cf-deployment-concourse-tasks.git - name: cf-acceptance-tests-git type: git @@ -32,7 +32,7 @@ jobs: DOMAIN: ((domain)) ENVIRONMENT: ((environment)) CATS_CONFIG_FILE: paas-bootstrap-git/profiles/staging/cats_config.json - CF_ADMIN_PASSWORD: ((/bosh/cf/cf_admin_password)) + CF_ADMIN_PASSWORD: ((cf_admin_password)) - task: run CATS tests file: cf-deployment-concourse-tasks-git/run-cats/task.yml input_mapping: diff --git a/ci/cf_pipeline.sh b/ci/cf_pipeline.sh index f758d2d2..229e813a 100755 --- a/ci/cf_pipeline.sh +++ b/ci/cf_pipeline.sh @@ -9,8 +9,6 @@ echo "Using branch '${BRANCH}'" DOMAIN="$(bin/outputs.sh base | jq -r .domain)" -bin/login_fly.sh - fly -t "$ENVIRONMENT" set-pipeline \ -v environment="$ENVIRONMENT" \ -v branch="$BRANCH" \ diff --git a/operations/cloud-config/concourse.yml b/operations/cloud-config/concourse.yml index 11b5478f..f39451f0 100644 --- a/operations/cloud-config/concourse.yml +++ b/operations/cloud-config/concourse.yml @@ -21,11 +21,11 @@ value: name: concourse network: concourse - ephemeral_disk: - size: 100_000 - type: gp2 cloud_properties: instance_type: c5.large + ephemeral_disk: + size: 100_000 + type: gp2 - type: replace path: /vm_extensions/- diff --git a/operations/concourse/skip-credhub-tls-validation.yml b/operations/concourse/skip-credhub-tls-validation.yml new file mode 100644 index 00000000..c9cb599d --- /dev/null +++ b/operations/concourse/skip-credhub-tls-validation.yml @@ -0,0 +1,4 @@ +--- +- type: replace + path: /instance_groups/name=web/jobs/name=atc/properties/credhub?/tls?/insecure_skip_verify? + value: true From 7ace5d9d0bfd165ccbdca4035e38c856e4d1f183 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Fri, 7 Sep 2018 11:51:44 +0100 Subject: [PATCH 18/71] Persist states to S3. Delete unused scripts. Use more recent bosh version. Bosh S3 blobstore. --- bin/bosh_credentials.sh | 2 + bin/cloud_config.sh | 2 + bin/create_bosh.sh | 15 +++- bin/create_concourse_network.sh | 51 ------------ bin/create_ssh_keypair.sh | 29 ------- bin/create_vpc.sh | 32 ------- bin/credhub_credentials.sh | 2 + bin/delete_bosh.sh | 27 ------ bin/delete_concourse.sh | 71 ---------------- bin/delete_concourse_network.sh | 50 ----------- bin/delete_jumpbox.sh | 26 ------ bin/delete_vpc.sh | 26 ------ bin/deploy_cf.sh | 2 + bin/deploy_concourse.sh | 2 + bin/deploy_prometheus.sh | 2 + bin/get_states.sh | 29 +++++++ bin/outputs.sh | 9 ++ bin/persist_states.sh | 23 +++++ bin/prometheus_credentials.sh | 19 +---- bin/rds.sh | 8 +- bin/require_vars.sh | 16 ---- bin/set_concourse_secrets.sh | 4 +- bin/terraform.sh | 7 +- bin/tfstate_to_tfvars.sh | 9 -- bin/tfstate_to_yml.sh | 9 -- bosh-deployment | 2 +- ci/cf.yml | 38 ++++++++- ci/tasks/cf/interpolate_variables/task.sh | 97 ++++++++++++++++++++++ ci/tasks/cf/interpolate_variables/task.yml | 21 +++++ terraform/base/bosh.tf | 10 +++ terraform/base/outputs.tf | 4 + 31 files changed, 274 insertions(+), 370 deletions(-) delete mode 100755 bin/create_concourse_network.sh delete mode 100755 bin/create_ssh_keypair.sh delete mode 100755 bin/create_vpc.sh delete mode 100755 bin/delete_bosh.sh delete mode 100755 bin/delete_concourse.sh delete mode 100755 bin/delete_concourse_network.sh delete mode 100755 bin/delete_jumpbox.sh delete mode 100755 bin/delete_vpc.sh create mode 100755 bin/get_states.sh create mode 100755 bin/persist_states.sh delete mode 100755 bin/require_vars.sh delete mode 100755 bin/tfstate_to_tfvars.sh delete mode 100755 bin/tfstate_to_yml.sh create mode 100755 ci/tasks/cf/interpolate_variables/task.sh create mode 100644 ci/tasks/cf/interpolate_variables/task.yml diff --git a/bin/bosh_credentials.sh b/bin/bosh_credentials.sh index 8f8d2df2..e4b40bb0 100755 --- a/bin/bosh_credentials.sh +++ b/bin/bosh_credentials.sh @@ -25,6 +25,8 @@ export ENVIRONMENT shift $((OPTIND-1)) COMMAND=$* +bin/get_states.sh -e $ENVIRONMENT -x -f $ENVIRONMENT-bosh-variables.yml + JUMPBOX_IP=$(bin/outputs.sh base | jq -r .jumpbox_public_ip) JUMPBOX_KEY=~/.ssh/$ENVIRONMENT.jumpbox.$$.pem bin/outputs.sh base | jq -r .jumpbox_private_key >$JUMPBOX_KEY diff --git a/bin/cloud_config.sh b/bin/cloud_config.sh index 7988d29c..1ca6f0a5 100755 --- a/bin/cloud_config.sh +++ b/bin/cloud_config.sh @@ -11,6 +11,8 @@ output() { bin/outputs.sh $FILE | jq -r "$QUERY" } +bin/get_states.sh -e $ENVIRONMENT -x -f $ENVIRONMENT.tfvars + BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" $BOSH update-cloud-config -n \ diff --git a/bin/create_bosh.sh b/bin/create_bosh.sh index 365398c0..ef58c1fc 100755 --- a/bin/create_bosh.sh +++ b/bin/create_bosh.sh @@ -2,6 +2,10 @@ set -euo pipefail +: $ENVIRONMENT +: $AWS_ACCESS_KEY_ID +: $AWS_SECRET_ACCESS_KEY + output() { FILE=$1 QUERY=$2 @@ -9,6 +13,8 @@ output() { bin/outputs.sh $FILE | jq -r "$QUERY" } +bin/get_states.sh -e $ENVIRONMENT + bosh int \ bosh-deployment/bosh.yml \ --vars-store data/$ENVIRONMENT-bosh-variables.yml \ @@ -16,6 +22,7 @@ bosh int \ -o bosh-deployment/uaa.yml \ -o bosh-deployment/credhub.yml \ -o bosh-deployment/misc/external-db.yml \ + -o bosh-deployment/aws/s3-blobstore.yml \ -o operations/bosh/tags.yml \ -o operations/bosh/certificate.yml \ -o operations/bosh/external-uaa-db.yml \ @@ -40,6 +47,10 @@ bosh int \ -v external_db_password="$(output rds .bosh_rds_password)" \ -v external_db_adapter="$(output rds .bosh_db_type)" \ -v external_db_name='bosh' \ + -v s3-bucket-name="$(output base .bosh_blobstore_bucket_name)" \ + -v s3-access-key-id="$AWS_ACCESS_KEY_ID" \ + -v s3-secret-access-key="$AWS_SECRET_ACCESS_KEY" \ + -v s3-region="$(jq -r .region < data/$ENVIRONMENT.tfvars)" \ -v access_key_id="$AWS_ACCESS_KEY_ID" \ -v secret_access_key="$AWS_SECRET_ACCESS_KEY" \ > data/$ENVIRONMENT-bosh-manifest.yml @@ -58,4 +69,6 @@ export BOSH_ALL_PROXY=ssh+socks5://ubuntu@$JUMPBOX_IP:22?private-key=$JUMPBOX_KE bosh create-env \ data/$ENVIRONMENT-bosh-manifest.yml \ - --state data/$ENVIRONMENT-bosh-state.json \ No newline at end of file + --state data/$ENVIRONMENT-bosh-state.json + + bin/persist_states.sh -e $ENVIRONMENT -f $ENVIRONMENT-bosh-manifest.yml -f $ENVIRONMENT-bosh-variables.yml -f $ENVIRONMENT-bosh-state.json \ No newline at end of file diff --git a/bin/create_concourse_network.sh b/bin/create_concourse_network.sh deleted file mode 100755 index cfeef70b..00000000 --- a/bin/create_concourse_network.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -# Creates the Concourse AWS network environment, using the outputs from the create_vpc.sh step - -set -euo pipefail - -: $ENVIRONMENT -if [ -z "${AWS_PROFILE:-}" ]; then - : $AWS_ACCESS_KEY_ID - : $AWS_SECRET_ACCESS_KEY -fi -: $VPC_STATE_FILE -: $CONCOURSE_TERRAFORM_STATE_FILE -: $PUBLIC_KEY_FILE -: $VAR_FILE - -TERRAFORM_DIR=terraform/concourse/aws - -_tmp_vars=tmp$$.tfvars.json -trap 'rm -f $_tmp_vars' EXIT -STATE_FILE="$VPC_STATE_FILE" bin/tfstate_to_tfvars.sh >"$_tmp_vars" -_public_key=$(cat $PUBLIC_KEY_FILE) - -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/tfstate.json" "${CONCOURSE_TERRAFORM_STATE_FILE}" || - echo "Remote Concourse Terraform state does not exist. Assuming this is a new deployment" - -terraform init "$TERRAFORM_DIR" -terraform plan \ - -var "environment=$ENVIRONMENT" \ - -var "public_key=$_public_key" \ - -var-file="$VAR_FILE" \ - -var-file="$_tmp_vars" \ - -state="$CONCOURSE_TERRAFORM_STATE_FILE" \ - "$TERRAFORM_DIR" - -save_state() { - if [ -f "${CONCOURSE_TERRAFORM_STATE_FILE}" ]; then - aws s3 cp "${CONCOURSE_TERRAFORM_STATE_FILE}" "s3://ons-paas-${ENVIRONMENT}-states/concourse/tfstate.json" --acl=private - fi -} - -trap save_state EXIT - -terraform apply -auto-approve \ - -var "environment=$ENVIRONMENT" \ - -var "public_key=$_public_key" \ - -var-file="$VAR_FILE" \ - -var-file="$_tmp_vars" \ - -state="$CONCOURSE_TERRAFORM_STATE_FILE" \ - "$TERRAFORM_DIR" - diff --git a/bin/create_ssh_keypair.sh b/bin/create_ssh_keypair.sh deleted file mode 100755 index edd1d910..00000000 --- a/bin/create_ssh_keypair.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# Creates a new key pair if it doesn't already exist - -set -euo pipefail - -: $ENVIRONMENT -: $PRIVATE_KEY_FILE -: $PUBLIC_KEY_FILE - -uploadKeys() { - aws s3 cp "${PRIVATE_KEY_FILE}" "s3://ons-paas-${ENVIRONMENT}-states/concourse/ssh-key.pem" --acl=private - aws s3 cp "${PUBLIC_KEY_FILE}" "s3://ons-paas-${ENVIRONMENT}-states/concourse/ssh-key.pem.pub" --acl=private -} - -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/ssh-key.pem" "${PRIVATE_KEY_FILE}" || true -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/ssh-key.pem.pub" "${PUBLIC_KEY_FILE}" || true - -[ -f "$PRIVATE_KEY_FILE" ] && [ -f "$PUBLIC_KEY_FILE" ] && - { - echo "Key pair $PRIVATE_KEY_FILE exists, skipping generation"; - uploadKeys; - exit 0 - } - -ssh-keygen -t rsa -f "$PRIVATE_KEY_FILE" -N foobar -ssh-keygen -p -f "$PRIVATE_KEY_FILE" -P foobar -N '' - -uploadKeys diff --git a/bin/create_vpc.sh b/bin/create_vpc.sh deleted file mode 100755 index c4267e2c..00000000 --- a/bin/create_vpc.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -: $ENVIRONMENT -: $VAR_FILE -: $VPC_STATE_FILE -if [ -z "${AWS_PROFILE:-}" ]; then - : $AWS_ACCESS_KEY_ID - : $AWS_SECRET_ACCESS_KEY -fi - -TERRAFORM_DIR=terraform/aws - -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/vpc/tfstate.json" "${VPC_STATE_FILE}" || - echo "Remote VPC Terraform state does not exist. Assuming this is a new deployment" - -terraform init "$TERRAFORM_DIR" -terraform plan \ - -var "environment=$ENVIRONMENT" \ - -var-file="$VAR_FILE" \ - -state="$VPC_STATE_FILE" \ - "$TERRAFORM_DIR" - -terraform apply -auto-approve \ - -var "environment=$ENVIRONMENT" \ - -var-file="$VAR_FILE" \ - -state="$VPC_STATE_FILE" \ - "$TERRAFORM_DIR" - -aws s3 cp "${VPC_STATE_FILE}" "s3://ons-paas-${ENVIRONMENT}-states/vpc/tfstate.json" --acl=private -aws s3 cp "${VAR_FILE}" "s3://ons-paas-${ENVIRONMENT}-states/vpc/vars.tfvars" --acl=private diff --git a/bin/credhub_credentials.sh b/bin/credhub_credentials.sh index c98c18dd..4ede618f 100755 --- a/bin/credhub_credentials.sh +++ b/bin/credhub_credentials.sh @@ -24,6 +24,8 @@ export ENVIRONMENT shift $((OPTIND-1)) COMMAND=$* +bin/get_states.sh -e $ENVIRONMENT -x -f $ENVIRONMENT-bosh-variables.yml + PROXY_PORT=30125 JUMPBOX_IP=$(bin/outputs.sh base | jq -r .jumpbox_public_ip) JUMPBOX_KEY=~/.ssh/$ENVIRONMENT.jumpbox.$$.pem diff --git a/bin/delete_bosh.sh b/bin/delete_bosh.sh deleted file mode 100755 index 768cb734..00000000 --- a/bin/delete_bosh.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -# Puts BOSH out of its misery - -set -euo pipefail - -: $ENVIRONMENT -if [ -z "${AWS_PROFILE:-}" ]; then - : $AWS_ACCESS_KEY_ID - : $AWS_SECRET_ACCESS_KEY -fi - - -VARS=/var/tmp/tmp$$ -trap 'rm -rf $VARS' EXIT -mkdir -p "$VARS" - -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/bosh/bosh.yml" "${VARS}/bosh.yml" -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/bosh/bosh-state.json" "${VARS}/bosh-state.json" - -bin/bosh_credentials.sh -e "$ENVIRONMENT" bosh delete-env \ - --state "${VARS}/bosh-state.json" \ - "$VARS/bosh.yml" - -aws s3 rm "s3://ons-paas-${ENVIRONMENT}-states/bosh/bosh.yml" || true -aws s3 rm "s3://ons-paas-${ENVIRONMENT}-states/bosh/bosh-variables.yml" || true -aws s3 rm "s3://ons-paas-${ENVIRONMENT}-states/bosh/bosh-state.json" || true diff --git a/bin/delete_concourse.sh b/bin/delete_concourse.sh deleted file mode 100755 index 3cf1d107..00000000 --- a/bin/delete_concourse.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -# Creates a wee Concourse - -set -euo pipefail - -: $ENVIRONMENT -if [ -n "${AWS_PROFILE:-}" ]; then - export AWS_ACCESS_KEY_ID=$(aws configure get aws_access_key_id) - export AWS_SECRET_ACCESS_KEY=$(aws configure get aws_secret_access_key) -fi -: $AWS_ACCESS_KEY_ID -: $AWS_SECRET_ACCESS_KEY -: $CONCOURSE_TERRAFORM_STATE_FILE -: $CONCOURSE_STATE_FILE -: $CONCOURSE_CREDS_FILE -: $PRIVATE_KEY_FILE - -git submodule update --init - -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/tfstate.json" "${CONCOURSE_TERRAFORM_STATE_FILE}" || - { - echo "Remote terraform state for concourse does not exist, assuming the environment does not exist..."; - exit 0 - } - -# Convert the terraform outputs to YAML -_vars_file=tmp.$$.yml -trap 'rm -f $_vars_file' EXIT -terraform output -state="$CONCOURSE_TERRAFORM_STATE_FILE" -json | jq 'with_entries(.value = .value.value)' | yq r - >"$_vars_file" - -SUBMODULE=concourse-bosh-deployment - -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/creds.yml" "${CONCOURSE_CREDS_FILE}" || - { - echo "Remote concourse creds do not exist, assuming the environment does not exist..."; - exit 0 - } - -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/state.json" "${CONCOURSE_STATE_FILE}" || - { - echo "Remote concourse state does not exist, assuming the environment does not exist..."; - exit 0 - } - -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/ssh-key.pem" "${PRIVATE_KEY_FILE}" || - { - echo "Remote concourse pruvate key does not exist, assuming the environment does not exist..."; - exit 0 - } - -bosh delete-env "$SUBMODULE"/lite/concourse.yml \ - -o "$SUBMODULE"/lite/infrastructures/aws.yml \ - -o operations/concourse/public-network.yml \ - -o operations/concourse/basic-auth.yml \ - -o operations/concourse/alb.yml \ - -o operations/concourse/fqdn.yml \ - -l "$SUBMODULE"/versions.yml \ - -l "$_vars_file" \ - -v access_key_id="$AWS_ACCESS_KEY_ID" \ - -v secret_access_key="$AWS_SECRET_ACCESS_KEY" \ - --var-file private_key="$PRIVATE_KEY_FILE" \ - --vars-store "$CONCOURSE_CREDS_FILE" \ - --state "$CONCOURSE_STATE_FILE" - -aws s3 rm "s3://ons-paas-${ENVIRONMENT}-states/concourse/creds.yml" || true -rm "${CONCOURSE_CREDS_FILE}" || true -aws s3 rm "s3://ons-paas-${ENVIRONMENT}-states/concourse/state.json" || true -rm "${CONCOURSE_STATE_FILE}" || true -aws s3 rm "s3://ons-paas-${ENVIRONMENT}-states/concourse/ssh-key.pem" || true -rm "${PRIVATE_KEY_FILE}" || true \ No newline at end of file diff --git a/bin/delete_concourse_network.sh b/bin/delete_concourse_network.sh deleted file mode 100755 index 6b60526e..00000000 --- a/bin/delete_concourse_network.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -# Creates the Concourse AWS network environment, using the outputs from the create_vpc.sh step - -set -euo pipefail - -: $ENVIRONMENT -if [ -z "${AWS_PROFILE:-}" ]; then - : $AWS_ACCESS_KEY_ID - : $AWS_SECRET_ACCESS_KEY -fi -: $VPC_STATE_FILE -: $CONCOURSE_TERRAFORM_STATE_FILE -: $PUBLIC_KEY_FILE -: $VAR_FILE - -TERRAFORM_DIR=terraform/concourse/aws - -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/ssh-key.pem.pub" "${PUBLIC_KEY_FILE}" || - { - echo "Remote SSH key does not exist. Assuming the deployment does not exist"; - exit 0 - } -_public_key=$(cat $PUBLIC_KEY_FILE) - -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/tfstate.json" "${CONCOURSE_TERRAFORM_STATE_FILE}" || - { - echo "Remote Concourse Terraform state does not exist. Assuming the deployment does not exist"; - exit 0 - } - -_tmp_vars=tmp$$.tfvars.json -trap 'rm -f $_tmp_vars' EXIT -STATE_FILE="$VPC_STATE_FILE" bin/tfstate_to_tfvars.sh >"$_tmp_vars" - -terraform init "$TERRAFORM_DIR" -terraform destroy -auto-approve \ - -var "environment=$ENVIRONMENT" \ - -var "public_key=$_public_key" \ - -var-file="$VAR_FILE" \ - -var-file="$_tmp_vars" \ - -state="$CONCOURSE_TERRAFORM_STATE_FILE" \ - "$TERRAFORM_DIR" - -aws s3 rm "s3://ons-paas-${ENVIRONMENT}-states/concourse/tfstate.json" || true -rm "${CONCOURSE_TERRAFORM_STATE_FILE}" || true -aws s3 rm "s3://ons-paas-${ENVIRONMENT}-states/concourse/ssh-key.pem" || true -rm "${PRIVATE_KEY_FILE}" || true -aws s3 rm "s3://ons-paas-${ENVIRONMENT}-states/concourse/ssh-key.pem.pub" || true -rm "${PUBLIC_KEY_FILE}" || true diff --git a/bin/delete_jumpbox.sh b/bin/delete_jumpbox.sh deleted file mode 100755 index 6e6d452b..00000000 --- a/bin/delete_jumpbox.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# Euthanise Jumbox - -set -euo pipefail - -: $ENVIRONMENT -if [ -z "${AWS_PROFILE:-}" ]; then - : $AWS_ACCESS_KEY_ID - : $AWS_SECRET_ACCESS_KEY -fi - -VARS=/var/tmp/tmp$$ -trap 'rm -rf $VARS' EXIT -mkdir -p "$VARS" - -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/jumpbox/jumpbox.yml" "${VARS}/jumpbox.yml" -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/jumpbox/jumpbox-state.json" "${VARS}/jumpbox-state.json" - -bosh delete-env \ - --state "${VARS}/jumpbox-state.json" \ - "$VARS/jumpbox.yml" - -aws s3 rm "s3://ons-paas-${ENVIRONMENT}-states/jumpbox/jumpbox.yml" || true -aws s3 rm "s3://ons-paas-${ENVIRONMENT}-states/jumpbox/jumpbox-variables.yml" || true -aws s3 rm "s3://ons-paas-${ENVIRONMENT}-states/jumpbox/jumpbox-state.json" || true diff --git a/bin/delete_vpc.sh b/bin/delete_vpc.sh deleted file mode 100755 index 87913735..00000000 --- a/bin/delete_vpc.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -: $ENVIRONMENT -: $VAR_FILE -: $VPC_STATE_FILE - -TERRAFORM_DIR=terraform/aws - -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/vpc/tfstate.json" "${VPC_STATE_FILE}" || - { - echo "Remote VPC Terraform state does not exist. Assuming is has already been deleted"; - exit 0 - } - -bin/wipe_s3_bucket.sh - -terraform destroy -auto-approve \ - -var "environment=$ENVIRONMENT" \ - -var-file="$VAR_FILE" \ - -state="$VPC_STATE_FILE" \ - "$TERRAFORM_DIR" - -rm "${VPC_STATE_FILE}" || true -rm "${VAR_FILE}" || true diff --git a/bin/deploy_cf.sh b/bin/deploy_cf.sh index 28475e4c..c811f3a0 100755 --- a/bin/deploy_cf.sh +++ b/bin/deploy_cf.sh @@ -11,6 +11,8 @@ output() { bin/outputs.sh $FILE | jq -r "$QUERY" } +bin/get_states.sh -e $ENVIRONMENT + BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" DOMAIN="$(output base .domain)" diff --git a/bin/deploy_concourse.sh b/bin/deploy_concourse.sh index bcff34a3..b0db20b1 100755 --- a/bin/deploy_concourse.sh +++ b/bin/deploy_concourse.sh @@ -18,6 +18,8 @@ secret() { bin/secret.sh -e $ENVIRONMENT -d cf -k $KEY } +bin/get_states.sh -e $ENVIRONMENT + BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" CREDHUB_CA_CERT=/var/tmp/credhub_ca.$$.pem diff --git a/bin/deploy_prometheus.sh b/bin/deploy_prometheus.sh index 136723d2..372ccbd8 100755 --- a/bin/deploy_prometheus.sh +++ b/bin/deploy_prometheus.sh @@ -16,6 +16,8 @@ secret() { bin/secret.sh -e $ENVIRONMENT -d cf -k $KEY } +bin/get_states.sh -e $ENVIRONMENT + BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" SYSTEM_DOMAIN="system.$(output base .domain)" diff --git a/bin/get_states.sh b/bin/get_states.sh new file mode 100755 index 00000000..206e478f --- /dev/null +++ b/bin/get_states.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -euo pipefail + +FILES= +OPTIONAL= +EXISTS= + +while getopts e:f:ox option; do + case $option in + e) export ENVIRONMENT="$OPTARG";; + f) FILES="$OPTARG $FILES";; + o) OPTIONAL=true;; + x) EXISTS=true;; + esac +done + +: $ENVIRONMENT + +[ "$OPTIONAL" = true ] && set +e + +if [ -z "$FILES" ]; then + aws s3 cp --recursive "s3://ons-paas-${ENVIRONMENT}-states" "data/" --acl=private +else + for FILE in $FILES; do + [ "$EXISTS" = true -a -f data/$FILE ] && continue + aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/$FILE" "data/" --acl=private + done +fi \ No newline at end of file diff --git a/bin/outputs.sh b/bin/outputs.sh index 5567e2a9..b3c61cd1 100755 --- a/bin/outputs.sh +++ b/bin/outputs.sh @@ -2,8 +2,17 @@ set -euo pipefail +while getopts 'e:f' option; do + case $option in + e) export ENVIRONMENT="$OPTARG";; + esac +done +shift $((OPTIND-1)) + : $ENVIRONMENT STEP=${1:-base} +bin/get_states.sh -e $ENVIRONMENT -x -f $ENVIRONMENT-$STEP.tfstate + terraform output -state="data/$ENVIRONMENT-$STEP.tfstate" -json | jq '. | with_entries(.value = .value.value)' \ No newline at end of file diff --git a/bin/persist_states.sh b/bin/persist_states.sh new file mode 100755 index 00000000..25ec406d --- /dev/null +++ b/bin/persist_states.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -euo pipefail + +FILES= + +while getopts e:f: option; do + case $option in + e) export ENVIRONMENT="$OPTARG";; + f) FILES="$OPTARG $FILES";; + esac +done + +: $ENVIRONMENT + +cd data +trap 'cd -' EXIT + +[ -z "$FILES" ] && FILES=""$ENVIRONMENT-*"" +for FILE in $FILES; do + [[ $FILE = *backup ]] && continue + aws s3 cp "$FILE" "s3://ons-paas-${ENVIRONMENT}-states/$FILE" --acl=private +done \ No newline at end of file diff --git a/bin/prometheus_credentials.sh b/bin/prometheus_credentials.sh index 805791b9..952d53c5 100755 --- a/bin/prometheus_credentials.sh +++ b/bin/prometheus_credentials.sh @@ -2,33 +2,20 @@ set -euo pipefail -if [ -z "${AWS_PROFILE:-}" ]; then - echo "AWS_PROFILE not set." - exit 1 -fi - while getopts e: option; do case $option in - e) ENVIRONMENT="$OPTARG";; + e) export ENVIRONMENT="$OPTARG";; esac done : $ENVIRONMENT -VARS=/var/tmp/tmp$$ -mkdir -p "$VARS" -trap 'rm -rf "$VARS"' EXIT - -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/prometheus/prometheus-variables.yml" $VARS/prometheus-creds #--quiet -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/prometheus/${ENVIRONMENT}.tfstate" $VARS/state-tmp #--quiet -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "$VARS/state-tmp" > "$VARS/vars.json" - creds() { SVC=$1 KEY=$2 echo "$SVC" - echo "URL https://$(jq -r .${KEY}_fqdn < "$VARS/vars.json")" - egrep "^${KEY}_password:" $VARS/prometheus-creds + echo " URL: https://$(bin/outputs.sh base | jq -r .${KEY}_fqdn)" + echo " Pass: $(bin/secret.sh -e $ENVIRONMENT -d prometheus -k ${KEY}_password)" echo } diff --git a/bin/rds.sh b/bin/rds.sh index 699e9c9c..e260e15a 100755 --- a/bin/rds.sh +++ b/bin/rds.sh @@ -8,10 +8,14 @@ COMMAND=${1:-plan} OPTS= [ "$COMMAND" = plan ] || OPTS=-auto-approve +bin/get_states -e $ENVIRONMENT -f $ENVIRONMENT.tfvars -f $ENVIRONMENT-rds.tfstate + terraform init terraform/rds terraform $COMMAND \ $OPTS \ -var-file="data/$ENVIRONMENT.tfvars" \ - -var-file=<(bin/outputs.sh) \ - -state="data/$ENVIRONMENT-rds.tfstate" terraform/rds/ \ No newline at end of file + -var-file=<(bin/outputs.sh base) \ + -state="data/$ENVIRONMENT-rds.tfstate" terraform/rds/ + + bin/persist_states -e $ENVIRONMENT -f $ENVIRONMENT-rds.tfstate \ No newline at end of file diff --git a/bin/require_vars.sh b/bin/require_vars.sh deleted file mode 100755 index 8c440234..00000000 --- a/bin/require_vars.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -: $ENVIRONMENT -if [ -z "${AWS_PROFILE:-}" ]; then - : $AWS_ACCESS_KEY_ID - : $AWS_SECRET_ACCESS_KEY -fi - -[ -f "$VAR_FILE" ] || - aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/vpc/vars.tfvars" "$VAR_FILE" || - { - echo "$VAR_FILE does not exist locally or in s3, please create manually"; - exit 1 - } diff --git a/bin/set_concourse_secrets.sh b/bin/set_concourse_secrets.sh index f45e30e4..0daeea33 100755 --- a/bin/set_concourse_secrets.sh +++ b/bin/set_concourse_secrets.sh @@ -11,4 +11,6 @@ done : $ENVIRONMENT bin/login_fly.sh -bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /concourse/main/cf_admin_password --type value --value $(bin/cf_password.sh -e $ENVIRONMENT) \ No newline at end of file +bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /concourse/main/cf_admin_password --type value --value $(bin/cf_password.sh -e $ENVIRONMENT) +bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /concourse/main/cf_tag --type value --value v4.2.0 +bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /concourse/main/prometheus_tag --type value --value v23.2.0 \ No newline at end of file diff --git a/bin/terraform.sh b/bin/terraform.sh index 2ec00284..b0e9d0cf 100755 --- a/bin/terraform.sh +++ b/bin/terraform.sh @@ -8,5 +8,10 @@ COMMAND=${1:-plan} OPTS= [ "$COMMAND" = plan ] || OPTS=-auto-approve +STATE="$ENVIRONMENT-base.tfstate" +ENV_VARS="$ENVIRONMENT.tfvars" + +bin/get_states.sh -e $ENVIRONMENT -o -f $STATE -f $ENV_VARS terraform init terraform/base -terraform $COMMAND $OPTS -var-file="data/$ENVIRONMENT.tfvars" -state="data/$ENVIRONMENT-base.tfstate" terraform/base/ +terraform $COMMAND $OPTS -var-file="data/$ENV_VARS" -state="data/$STATE" terraform/base/ +bin/persist_states.sh -e $ENVIRONMENT -f $STATE -f $ENV_VARS diff --git a/bin/tfstate_to_tfvars.sh b/bin/tfstate_to_tfvars.sh deleted file mode 100755 index 90b93274..00000000 --- a/bin/tfstate_to_tfvars.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# This script will translate terraform outputs into .tfvars - -set -euo pipefail - -: $STATE_FILE - -terraform output -state="$STATE_FILE" -json | jq 'with_entries(.value = .value.value)' diff --git a/bin/tfstate_to_yml.sh b/bin/tfstate_to_yml.sh deleted file mode 100755 index bc75e72b..00000000 --- a/bin/tfstate_to_yml.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# This script will translate terraform outputs into .tfvars - -set -euo pipefail - -: $STATE_FILE - -terraform output -state="$STATE_FILE" -json | yq r \ No newline at end of file diff --git a/bosh-deployment b/bosh-deployment index 04c85a5c..ff0bc984 160000 --- a/bosh-deployment +++ b/bosh-deployment @@ -1 +1 @@ -Subproject commit 04c85a5c79a9fa6b92775386a334104b9a165013 +Subproject commit ff0bc984c72d5053656c666ae7d7788a78f6b7a8 diff --git a/ci/cf.yml b/ci/cf.yml index b1e45b31..74b780c7 100644 --- a/ci/cf.yml +++ b/ci/cf.yml @@ -16,16 +16,50 @@ resources: source: uri: https://github.com/cloudfoundry/cf-acceptance-tests.git +- name: cf-deployment-git + type: git + source: + uri: https://github.com/cloudfoundry/cf-deployment.git + tag_filter: v* + +- name: prometheus-boshrelease-git + type: git + source: + uri: https://github.com/bosh-prometheus/prometheus-boshrelease.git + tag_filter: v* jobs: +- name: deploy cloud foundry + serial: true + serial_groups: [cf, casts] + plan: + - aggregate: + - get: paas-bootstrap-git + - get: cf-deployment-concourse-tasks-git + - get: cf-deployment-git + version: { ref: ((cf_tag)) } + - get: prometheus-boshrelease-git + version: { ref: ((prometheus_tag)) } + - task: interpolate cf deployment variables and ops files + file: paas-bootstrap-git/ci/tasks/cf/interpolate-variables/task.yml + - task: deploy cf + file: cf-deployment-concourse-tasks-git/bosh-deploy/task.yml + input_mapping: + cf-deployment-concourse-tasks: cf-deployment-concourse-tasks-git + cf-deployment: cf-deployment-git + params: + OPS_FILES: instance-counts.yml ops.yml + SYSTEM_DOMAIN: system.((domain)) + + - name: cats serial: true - serial_groups: [cats] + serial_groups: [cf, cats] plan: - aggregate: - get: paas-bootstrap-git - get: cf-deployment-concourse-tasks-git - - get: cf-acceptance-tests-git + - get: cf-acceptance-tests-git - task: interpolate CATS config file: paas-bootstrap-git/ci/tasks/cf/interpolate_cats/task.yml params: diff --git a/ci/tasks/cf/interpolate_variables/task.sh b/ci/tasks/cf/interpolate_variables/task.sh new file mode 100755 index 00000000..42c4ea24 --- /dev/null +++ b/ci/tasks/cf/interpolate_variables/task.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +set -euo pipefail + +: $ENVIRONMENT + +output() { + FILE=$1 + QUERY=$2 + + cd paas-bootstrap-git + bin/outputs.sh $FILE | jq -r "$QUERY" + cd - +} + +BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" + +DOMAIN="$(output base .domain)" +SYSTEM_DOMAIN="system.${DOMAIN}" +APPS_DOMAIN="apps.${DOMAIN}" + +CF_DB_ENDPOINT="$(output rds .cf_rds_fqdn)" +CF_DB_USERNAME="$(output rds .cf_db_username)" +CF_DB_PASSWORD="$(output rds .cf_rds_password)" + +profile="${ENVIRONMENT}" +[ -d "profiles/${ENVIRONMENT}" ] || profile="engineering" + +instance_count_file=./profiles/${profile}/instance-count.yml + +echo "Constructing consolidated ops file" +cat \ + cf-deployment-git/operations/aws.yml \ + cf-deployment-git/operations/override-app-domains.yml \ + cf-deployment-git/operations/use-external-blobstore.yml \ + cf-deployment-git/operations/use-external-dbs.yml \ + prometheus-boshrelease-it/manifests/operators/cf/add-prometheus-uaa-clients.yml \ + paas-bootstrap-git/operations/bosh/tags.yml \ + paas-bootstrap-git/operations/cf/router-sec-group.yml \ + paas-bootstrap-git/operations/cf/scheduler.yml \ + paas-bootstrap-git/operations/cf/s3_blobstore_with_kms_and_iam.yml \ + paas-bootstrap-git/operations/cf/azs.yml \ + paas-bootstrap-git/operations/cf/instance-counts.yml \ + paas-bootstrap-git/operations/cf/rds-access.yml \ + paas-bootstrap-git/operations/cf/uaa-clients.yml \ + paas-bootstrap-git/operations/cf/test-user.yml \ + > ops-files/ops.yml + +echo "Constructing variables" +cp "${instance_count_file}" vars-files/instance-counts.yml + +cat >vars-files/variables.yml <<-EOS +environment: ${ENVIRONMENT} +region: $(jq -r .region < data/$ENVIRONMENT.tfvars) +system_domain:${SYSTEM_DOMAIN} +app_domains: [${APPS_DOMAIN}] +smoke_test_app_domain: ${APPS_DOMAIN} +buildpack_directory_key: $(output base .cf_buildpacks_bucket_name) +droplet_directory_key: $(output base .cf_droplets_bucket_name) +app_package_directory_key: $(output base .cf_packages_bucket_name) +resource_directory_key: $(output base .cf_resource_pool_bucket_name) +cf_blobstore_s3_kms_key_id: $(output base .cf_blobstore_s3_kms_key_id) +external_database_type: $(output rds .cf_db_type) +external_database_port: $(output rds .cf_db_port) +external_uaa_database_name: uaa +external_uaa_database_address: ${CF_DB_ENDPOINT} +external_uaa_database_password: ${CF_DB_PASSWORD} +external_uaa_database_username: ${CF_DB_USERNAME} +external_cc_database_name: cc +external_cc_database_address: ${CF_DB_ENDPOINT} +external_cc_database_password: ${CF_DB_PASSWORD} +external_cc_database_username: ${CF_DB_USERNAME} +external_bbs_database_name: bbs +external_bbs_database_address: ${CF_DB_ENDPOINT} +external_bbs_database_password: ${CF_DB_PASSWORD} +external_bbs_database_username: ${CF_DB_USERNAME} +external_routing_api_database_name: routing_api +external_routing_api_database_address: ${CF_DB_ENDPOINT} +external_routing_api_database_password: ${CF_DB_PASSWORD} +external_routing_api_database_username: ${CF_DB_USERNAME} +external_policy_server_database_name: policy_server +external_policy_server_database_address: ${CF_DB_ENDPOINT} +external_policy_server_database_password: ${CF_DB_PASSWORD} +external_policy_server_database_username: ${CF_DB_USERNAME} +external_silk_controller_database_name: silk_controller +external_silk_controller_database_address: ${CF_DB_ENDPOINT} +external_silk_controller_database_password: ${CF_DB_PASSWORD} +external_silk_controller_database_username: ${CF_DB_USERNAME} +external_locket_database_name: locket +external_locket_database_address: ${CF_DB_ENDPOINT} +external_locket_database_password: ${CF_DB_PASSWORD} +external_locket_database_username: ${CF_DB_USERNAME} +external_credhub_database_name: credhub +external_credhub_database_address: ${CF_DB_ENDPOINT} +external_credhub_database_password: ${CF_DB_PASSWORD} +external_credhub_database_username: ${CF_DB_USERNAME} +EOS \ No newline at end of file diff --git a/ci/tasks/cf/interpolate_variables/task.yml b/ci/tasks/cf/interpolate_variables/task.yml new file mode 100644 index 00000000..e4711736 --- /dev/null +++ b/ci/tasks/cf/interpolate_variables/task.yml @@ -0,0 +1,21 @@ +--- +platform: linux +image_resource: + type: docker-image + source: { repository: onsdigital/paas-ci-gp, tag: latest } + +inputs: + - name: paas-bootstrap-git + - name: cf-deployment-git + - name: prometheus-boshrelease-git + +outputs: + - name: vars-files + - name: ops-files + +run: + path: paas-bootstrap-git/ci/tasks/cf/interpolate_variables/task.sh + +params: + DOMAIN: + ENVIRONMENT: diff --git a/terraform/base/bosh.tf b/terraform/base/bosh.tf index 68dff2b5..8ad0cd84 100644 --- a/terraform/base/bosh.tf +++ b/terraform/base/bosh.tf @@ -71,6 +71,16 @@ resource "aws_s3_bucket_object" "bosh-state" { kms_key_id = "${aws_kms_key.paas_state_key.arn}" } +resource "aws_s3_bucket" "bosh_blobstore" { + bucket = "${var.s3_prefix}-${var.environment}-bosh-blobstore" + acl = "private" + + tags { + Name = "${var.s3_prefix}-${var.environment}-bosh-blobstore" + Environment = "${var.environment}" + } +} + # SG resource "aws_security_group_rule" "allow-all" { security_group_id = "${aws_security_group.bosh.id}" diff --git a/terraform/base/outputs.tf b/terraform/base/outputs.tf index 5152b06a..43578ed0 100644 --- a/terraform/base/outputs.tf +++ b/terraform/base/outputs.tf @@ -59,6 +59,10 @@ output "bosh_private_key" { value = "${tls_private_key.bosh.private_key_pem}" } +output "bosh_blobstore_bucket_name" { + value = "${aws_s3_bucket.bosh_blobstore.id}" +} + output "cf_rds_security_group_id" { value = "${aws_security_group.cf_rds.id}" } From 807a4868bdb0de5155830611c6e4d56c739c02d5 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Fri, 7 Sep 2018 13:04:33 +0100 Subject: [PATCH 19/71] Destroy bosh. --- Makefile | 3 +++ bin/database_client.sh | 56 ++++++++++++++++++++++++++++++++++++++++++ bin/databases.sh | 2 +- bin/destroy_bosh.sh | 47 +++++++++++++++++++++++++++++++++++ 4 files changed, 107 insertions(+), 1 deletion(-) create mode 100755 bin/database_client.sh create mode 100755 bin/destroy_bosh.sh diff --git a/Makefile b/Makefile index 6735ccd0..00f67519 100644 --- a/Makefile +++ b/Makefile @@ -65,6 +65,9 @@ set_concourse_secrets: ## Set the secrets that Concourse needs to run its pipeli pipelines: login_fly ## Deploy pipelines @ci/cf_pipeline.sh +destroy_bosh: ## Kill off bosh + @bin/destroy_bosh.sh + decode_aws_error: ## Decode AWS message @aws sts decode-authorization-message --encoded-message ${DECODE_MESSAGE} | jq -r .DecodedMessage | jq . diff --git a/bin/database_client.sh b/bin/database_client.sh new file mode 100755 index 00000000..d2deca16 --- /dev/null +++ b/bin/database_client.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +set -euo pipefail + + +while getopts e:i:d: option; do + case $option in + e) export ENVIRONMENT=$OPTARG;; + i) DB_INSTANCE=$OPTARG;; + d) DATABASE=$OPTARG;; + esac +done + +: $ENVIRONMENT +: $DB_INSTANCE + +BOSH_DB_USER=$(bin/outputs.sh rds | jq -r .bosh_db_username) +BOSH_DB_HOST=$(bin/outputs.sh rds | jq -r .bosh_db_host) +BOSH_DB_PORT=$(bin/outputs.sh rds | jq -r .bosh_db_port) +BOSH_DB_PASS=$(bin/outputs.sh rds | jq -r .bosh_rds_password) + +CF_DB_USER=$(bin/outputs.sh rds | jq -r .cf_db_username) +CF_DB_HOST=$(bin/outputs.sh rds | jq -r .cf_db_host) +CF_DB_PORT=$(bin/outputs.sh rds | jq -r .cf_db_port) +CF_DB_PASS=$(bin/outputs.sh rds | jq -r .cf_rds_password) + + +JUMPBOX_IP=$(bin/outputs.sh | jq -r .jumpbox_public_ip) +KEYFILE=~/.ssh/$ENVIRONMENT.jumpbox.$$.pem +bin/outputs.sh | jq -r .jumpbox_private_key >$KEYFILE +chmod 600 $KEYFILE + +BOSH_LOCAL_PORT=30201 +CF_LOCAL_PORT=30202 + +cleanup() { + rm -f $KEYFILE + kill $(ps -ef | awk "/ssh.*$BOSH_LOCAL_PORT/ && ! /awk/ { print \$2 }") + kill $(ps -ef | awk "/ssh.*$CF_LOCAL_PORT/ && ! /awk/ { print \$2 }") +} +ssh -fN -L $BOSH_LOCAL_PORT:$BOSH_DB_HOST:$BOSH_DB_PORT ubuntu@$JUMPBOX_IP -i $KEYFILE +ssh -fN -L $CF_LOCAL_PORT:$CF_DB_HOST:$CF_DB_PORT ubuntu@$JUMPBOX_IP -i $KEYFILE +trap cleanup EXIT + +bosh_db_client() { + PGPASSWORD=$BOSH_DB_PASS psql -h localhost -p $BOSH_LOCAL_PORT -U $BOSH_DB_USER -d $DATABASE +} + +cf_db_client() { + MYSQL_PWD=$CF_DB_PASS mysql --protocol=tcp -h localhost -P $CF_LOCAL_PORT -u $CF_DB_USER $DATABASE +} + +case $DB_INSTANCE in + bosh) bosh_db_client;; + cf) cf_db_client;; +esac \ No newline at end of file diff --git a/bin/databases.sh b/bin/databases.sh index db1bd65a..36729968 100755 --- a/bin/databases.sh +++ b/bin/databases.sh @@ -38,7 +38,7 @@ sql() { } exists() { - PGPASSWORD=$BOSH_DB_PASS psql -h localhost -p $BOSH_LOCAL_PORT -U $BOSH_DB_USER -d paastest -l | grep -q $1 + PGPASSWORD=$BOSH_DB_PASS psql -h localhost -p $BOSH_LOCAL_PORT -U $BOSH_DB_USER -d paastest -l | grep -q "^\s*$1\s" } create_bosh() { diff --git a/bin/destroy_bosh.sh b/bin/destroy_bosh.sh new file mode 100755 index 00000000..db56872f --- /dev/null +++ b/bin/destroy_bosh.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -euo pipefail + +: $ENVIRONMENT +: $AWS_ACCESS_KEY_ID +: $AWS_SECRET_ACCESS_KEY +: ${VERIFY:=true} + +output() { + FILE=$1 + QUERY=$2 + + bin/outputs.sh $FILE | jq -r "$QUERY" +} + +if [ "$VERIFY" = true ]; then + echo "This is probably a bad idea. Of course, all of the important stuff *should* be in the bosh database, but you ought to be well confident that you want to do this" + echo + echo "So do ya, punk? (only 'yes' is acceptable)" + read ans + case $ans in + [Yy][Ee][Ss]);; + *) echo "Bailing out"; exit 1;; + esac +fi + +bin/get_states.sh -e $ENVIRONMENT + +JUMPBOX_IP=$(output base .jumpbox_public_ip) +JUMPBOX_KEY=~/.ssh/$ENVIRONMENT.jumpbox.$$.pem +output base .jumpbox_private_key >$JUMPBOX_KEY +chmod 600 $JUMPBOX_KEY + +cleanup() { + rm -f $JUMPBOX_KEY +} +trap cleanup EXIT + +export BOSH_ALL_PROXY=ssh+socks5://ubuntu@$JUMPBOX_IP:22?private-key=$JUMPBOX_KEY + +bosh delete-env \ + data/$ENVIRONMENT-bosh-manifest.yml \ + --state data/$ENVIRONMENT-bosh-state.json + +echo "{}" >data/$ENVIRONMENT-bosh-state.json +bin/persist_states.sh -e $ENVIRONMENT -f $ENVIRONMENT-bosh-state.json \ No newline at end of file From 12299187bc4840bc508eff3e667e2737b9bbcdcc Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Fri, 7 Sep 2018 13:10:02 +0100 Subject: [PATCH 20/71] Remove old ci tasks --- .../bosh/db_connectivity_test/task.sh | 20 ----- .../bosh/db_connectivity_test/task.yml | 14 ---- ci/tasks.old/bosh/deploy/task.sh | 9 -- ci/tasks.old/bosh/deploy/task.yml | 16 ---- .../bosh/deploy_node_exporter/task.sh | 19 ----- .../bosh/deploy_node_exporter/task.yml | 16 ---- ci/tasks.old/bosh/destroy/task.sh | 11 --- ci/tasks.old/bosh/destroy/task.yml | 16 ---- ci/tasks.old/bosh/get_database_vars/task.sh | 13 --- ci/tasks.old/bosh/get_database_vars/task.yml | 18 ---- ci/tasks.old/bosh/get_terraform_vars/task.sh | 7 -- ci/tasks.old/bosh/get_terraform_vars/task.yml | 23 ----- ci/tasks.old/bosh/interpolate/task.sh | 43 ---------- ci/tasks.old/bosh/interpolate/task.yml | 24 ------ ci/tasks.old/bosh/test/task.sh | 8 -- ci/tasks.old/bosh/test/task.yml | 16 ---- ci/tasks.old/cf/cloud_config/task.sh | 58 ------------- ci/tasks.old/cf/cloud_config/task.yml | 25 ------ .../cf/databases/db_connectivity_test/task.sh | 20 ----- .../databases/db_connectivity_test/task.yml | 14 ---- .../cf/databases/get_database_vars/task.sh | 13 --- .../cf/databases/get_database_vars/task.yml | 18 ---- ci/tasks.old/cf/deploy_cf/task.sh | 15 ---- ci/tasks.old/cf/deploy_cf/task.yml | 17 ---- ci/tasks.old/cf/destroy/task.sh | 14 ---- ci/tasks.old/cf/destroy/task.yml | 13 --- ci/tasks.old/cf/get_management_vars/task.sh | 9 -- ci/tasks.old/cf/get_management_vars/task.yml | 20 ----- ci/tasks.old/cf/get_terraform_vars/task.sh | 8 -- ci/tasks.old/cf/get_terraform_vars/task.yml | 25 ------ ci/tasks.old/cf/interpolate/task.sh | 84 ------------------- ci/tasks.old/cf/interpolate/task.yml | 24 ------ ci/tasks.old/cf/interpolate_cats/task.sh | 14 ---- ci/tasks.old/cf/interpolate_cats/task.yml | 21 ----- ci/tasks.old/cf/management/task.sh | 21 ----- ci/tasks.old/cf/management/task.yml | 16 ---- ci/tasks.old/cf/smoke_tests/task.sh | 15 ---- ci/tasks.old/cf/smoke_tests/task.yml | 13 --- ci/tasks.old/cf/test/task.sh | 12 --- ci/tasks.old/cf/test/task.yml | 16 ---- ci/tasks.old/cf/upload_stemcell/task.sh | 18 ---- ci/tasks.old/cf/upload_stemcell/task.yml | 17 ---- ci/tasks.old/common/delete_s3/task.sh | 8 -- ci/tasks.old/common/delete_s3/task.yml | 18 ---- .../common/get_cf_tester_credentials/task.sh | 11 --- .../common/get_cf_tester_credentials/task.yml | 19 ----- ci/tasks.old/common/test_connectivity/task.sh | 21 ----- .../common/test_connectivity/task.yml | 18 ---- ci/tasks.old/common/test_endpoint/task.sh | 15 ---- ci/tasks.old/common/test_endpoint/task.yml | 18 ---- ci/tasks.old/concourse/test_fqdn/task.sh | 13 --- ci/tasks.old/concourse/test_fqdn/task.yml | 17 ---- .../concourse/test_iam_policy/task.sh | 10 --- .../concourse/test_iam_policy/task.yml | 13 --- ci/tasks.old/destroy/empty_buckets/task.sh | 41 --------- ci/tasks.old/destroy/empty_buckets/task.yml | 11 --- ci/tasks.old/jumpbox/deploy/task.sh | 9 -- ci/tasks.old/jumpbox/deploy/task.yml | 16 ---- ci/tasks.old/jumpbox/destroy/task.sh | 11 --- ci/tasks.old/jumpbox/destroy/task.yml | 17 ---- .../jumpbox/get_terraform_vars/task.sh | 6 -- .../jumpbox/get_terraform_vars/task.yml | 18 ---- ci/tasks.old/jumpbox/interpolate/task.sh | 26 ------ ci/tasks.old/jumpbox/interpolate/task.yml | 21 ----- ci/tasks.old/jumpbox/test/task.sh | 17 ---- ci/tasks.old/jumpbox/test/task.yml | 17 ---- ci/tasks.old/jumpbox/test_fqdn/task.sh | 15 ---- ci/tasks.old/jumpbox/test_fqdn/task.yml | 17 ---- ci/tasks.old/prometheus/dashboard/task.sh | 15 ---- ci/tasks.old/prometheus/dashboard/task.yml | 20 ----- ci/tasks.old/prometheus/deploy/task.sh | 15 ---- ci/tasks.old/prometheus/deploy/task.yml | 17 ---- ci/tasks.old/prometheus/destroy/task.sh | 14 ---- ci/tasks.old/prometheus/destroy/task.yml | 14 ---- .../prometheus/get_terraform_vars/task.sh | 9 -- .../prometheus/get_terraform_vars/task.yml | 27 ------ ci/tasks.old/prometheus/interpolate/task.sh | 34 -------- ci/tasks.old/prometheus/interpolate/task.yml | 27 ------ ci/tasks.old/rds_broker/test_shared/task.sh | 25 ------ ci/tasks.old/rds_broker/test_shared/task.yml | 14 ---- 80 files changed, 1467 deletions(-) delete mode 100755 ci/tasks.old/bosh/db_connectivity_test/task.sh delete mode 100644 ci/tasks.old/bosh/db_connectivity_test/task.yml delete mode 100755 ci/tasks.old/bosh/deploy/task.sh delete mode 100644 ci/tasks.old/bosh/deploy/task.yml delete mode 100755 ci/tasks.old/bosh/deploy_node_exporter/task.sh delete mode 100644 ci/tasks.old/bosh/deploy_node_exporter/task.yml delete mode 100755 ci/tasks.old/bosh/destroy/task.sh delete mode 100644 ci/tasks.old/bosh/destroy/task.yml delete mode 100755 ci/tasks.old/bosh/get_database_vars/task.sh delete mode 100644 ci/tasks.old/bosh/get_database_vars/task.yml delete mode 100755 ci/tasks.old/bosh/get_terraform_vars/task.sh delete mode 100644 ci/tasks.old/bosh/get_terraform_vars/task.yml delete mode 100755 ci/tasks.old/bosh/interpolate/task.sh delete mode 100644 ci/tasks.old/bosh/interpolate/task.yml delete mode 100755 ci/tasks.old/bosh/test/task.sh delete mode 100644 ci/tasks.old/bosh/test/task.yml delete mode 100755 ci/tasks.old/cf/cloud_config/task.sh delete mode 100644 ci/tasks.old/cf/cloud_config/task.yml delete mode 100755 ci/tasks.old/cf/databases/db_connectivity_test/task.sh delete mode 100644 ci/tasks.old/cf/databases/db_connectivity_test/task.yml delete mode 100755 ci/tasks.old/cf/databases/get_database_vars/task.sh delete mode 100644 ci/tasks.old/cf/databases/get_database_vars/task.yml delete mode 100755 ci/tasks.old/cf/deploy_cf/task.sh delete mode 100644 ci/tasks.old/cf/deploy_cf/task.yml delete mode 100755 ci/tasks.old/cf/destroy/task.sh delete mode 100644 ci/tasks.old/cf/destroy/task.yml delete mode 100755 ci/tasks.old/cf/get_management_vars/task.sh delete mode 100644 ci/tasks.old/cf/get_management_vars/task.yml delete mode 100755 ci/tasks.old/cf/get_terraform_vars/task.sh delete mode 100644 ci/tasks.old/cf/get_terraform_vars/task.yml delete mode 100755 ci/tasks.old/cf/interpolate/task.sh delete mode 100644 ci/tasks.old/cf/interpolate/task.yml delete mode 100755 ci/tasks.old/cf/interpolate_cats/task.sh delete mode 100644 ci/tasks.old/cf/interpolate_cats/task.yml delete mode 100755 ci/tasks.old/cf/management/task.sh delete mode 100644 ci/tasks.old/cf/management/task.yml delete mode 100755 ci/tasks.old/cf/smoke_tests/task.sh delete mode 100644 ci/tasks.old/cf/smoke_tests/task.yml delete mode 100755 ci/tasks.old/cf/test/task.sh delete mode 100644 ci/tasks.old/cf/test/task.yml delete mode 100755 ci/tasks.old/cf/upload_stemcell/task.sh delete mode 100644 ci/tasks.old/cf/upload_stemcell/task.yml delete mode 100755 ci/tasks.old/common/delete_s3/task.sh delete mode 100644 ci/tasks.old/common/delete_s3/task.yml delete mode 100755 ci/tasks.old/common/get_cf_tester_credentials/task.sh delete mode 100644 ci/tasks.old/common/get_cf_tester_credentials/task.yml delete mode 100755 ci/tasks.old/common/test_connectivity/task.sh delete mode 100644 ci/tasks.old/common/test_connectivity/task.yml delete mode 100755 ci/tasks.old/common/test_endpoint/task.sh delete mode 100644 ci/tasks.old/common/test_endpoint/task.yml delete mode 100755 ci/tasks.old/concourse/test_fqdn/task.sh delete mode 100644 ci/tasks.old/concourse/test_fqdn/task.yml delete mode 100755 ci/tasks.old/concourse/test_iam_policy/task.sh delete mode 100644 ci/tasks.old/concourse/test_iam_policy/task.yml delete mode 100755 ci/tasks.old/destroy/empty_buckets/task.sh delete mode 100644 ci/tasks.old/destroy/empty_buckets/task.yml delete mode 100755 ci/tasks.old/jumpbox/deploy/task.sh delete mode 100644 ci/tasks.old/jumpbox/deploy/task.yml delete mode 100755 ci/tasks.old/jumpbox/destroy/task.sh delete mode 100644 ci/tasks.old/jumpbox/destroy/task.yml delete mode 100755 ci/tasks.old/jumpbox/get_terraform_vars/task.sh delete mode 100644 ci/tasks.old/jumpbox/get_terraform_vars/task.yml delete mode 100755 ci/tasks.old/jumpbox/interpolate/task.sh delete mode 100644 ci/tasks.old/jumpbox/interpolate/task.yml delete mode 100755 ci/tasks.old/jumpbox/test/task.sh delete mode 100644 ci/tasks.old/jumpbox/test/task.yml delete mode 100755 ci/tasks.old/jumpbox/test_fqdn/task.sh delete mode 100644 ci/tasks.old/jumpbox/test_fqdn/task.yml delete mode 100755 ci/tasks.old/prometheus/dashboard/task.sh delete mode 100644 ci/tasks.old/prometheus/dashboard/task.yml delete mode 100755 ci/tasks.old/prometheus/deploy/task.sh delete mode 100644 ci/tasks.old/prometheus/deploy/task.yml delete mode 100755 ci/tasks.old/prometheus/destroy/task.sh delete mode 100644 ci/tasks.old/prometheus/destroy/task.yml delete mode 100755 ci/tasks.old/prometheus/get_terraform_vars/task.sh delete mode 100644 ci/tasks.old/prometheus/get_terraform_vars/task.yml delete mode 100755 ci/tasks.old/prometheus/interpolate/task.sh delete mode 100644 ci/tasks.old/prometheus/interpolate/task.yml delete mode 100755 ci/tasks.old/rds_broker/test_shared/task.sh delete mode 100644 ci/tasks.old/rds_broker/test_shared/task.yml diff --git a/ci/tasks.old/bosh/db_connectivity_test/task.sh b/ci/tasks.old/bosh/db_connectivity_test/task.sh deleted file mode 100755 index 6a62e234..00000000 --- a/ci/tasks.old/bosh/db_connectivity_test/task.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -set -a -source bosh-vars/env -set +a - -: $USER_DB -: $FQDN_DB -: $PASSWORD_DB -: $DUMMY_DB - -pg_isready -h ${FQDN_DB} -U ${USER_DB} - -set +e -while : -do - PGPASSWORD=$PASSWORD_DB psql -c "select 1" -h $FQDN_DB -U $USER_DB -d $DUMMY_DB && break -done \ No newline at end of file diff --git a/ci/tasks.old/bosh/db_connectivity_test/task.yml b/ci/tasks.old/bosh/db_connectivity_test/task.yml deleted file mode 100644 index 1661fce6..00000000 --- a/ci/tasks.old/bosh/db_connectivity_test/task.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: -- name: paas-bootstrap-git -- name: bosh-vars - -run: - path: paas-bootstrap-git/ci/tasks/bosh/db_connectivity_test/task.sh - diff --git a/ci/tasks.old/bosh/deploy/task.sh b/ci/tasks.old/bosh/deploy/task.sh deleted file mode 100755 index b1a06e8a..00000000 --- a/ci/tasks.old/bosh/deploy/task.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -cp bosh-state-s3/bosh-state.json bosh-state/bosh-state.json - -bosh create-env \ - bosh-manifests/bosh.yml \ - --state bosh-state/bosh-state.json \ No newline at end of file diff --git a/ci/tasks.old/bosh/deploy/task.yml b/ci/tasks.old/bosh/deploy/task.yml deleted file mode 100644 index 50a75716..00000000 --- a/ci/tasks.old/bosh/deploy/task.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: bosh-state-s3 - - name: bosh-manifests - -outputs: - - name: bosh-state - -run: - path: paas-bootstrap-git/ci/tasks/bosh/deploy/task.sh diff --git a/ci/tasks.old/bosh/deploy_node_exporter/task.sh b/ci/tasks.old/bosh/deploy_node_exporter/task.sh deleted file mode 100755 index a26fd542..00000000 --- a/ci/tasks.old/bosh/deploy_node_exporter/task.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -bosh_admin_password=$(bosh int bosh-vars-s3/bosh-variables.yml --path /admin_password) -bosh int bosh-vars-s3/bosh-variables.yml --path /default_ca/ca > bosh_ca.pem -bosh_ip=$(bosh int --path '/cloud_provider/ssh_tunnel/host' bosh-manifest-s3/bosh.yml) - -export BOSH_CLIENT=admin -export BOSH_CLIENT_SECRET="${bosh_admin_password}" -export BOSH_ENVIRONMENT="https://${bosh_ip}:25555" -export BOSH_CA_CERT=$(cat bosh_ca.pem) - -RUNTIME_CONFIG=paas-bootstrap-git/runtime-config/bosh/node-exporter-config.yml - -#TODO Needs to be parameterised -bosh upload-release https://github.com/bosh-prometheus/node-exporter-boshrelease/releases/download/v${NODE_EXPORTER_VERSION}/node-exporter-${NODE_EXPORTER_VERSION}.tgz - -bosh update-runtime-config --non-interactive ${RUNTIME_CONFIG} diff --git a/ci/tasks.old/bosh/deploy_node_exporter/task.yml b/ci/tasks.old/bosh/deploy_node_exporter/task.yml deleted file mode 100644 index 081ab86d..00000000 --- a/ci/tasks.old/bosh/deploy_node_exporter/task.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: bosh-manifest-s3 - - name: bosh-vars-s3 - -run: - path: paas-bootstrap-git/ci/tasks/bosh/deploy_node_exporter/task.sh - -params: - NODE_EXPORTER_VERSION: \ No newline at end of file diff --git a/ci/tasks.old/bosh/destroy/task.sh b/ci/tasks.old/bosh/destroy/task.sh deleted file mode 100755 index 4abc6b40..00000000 --- a/ci/tasks.old/bosh/destroy/task.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -cp bosh-state-s3/bosh-state.json bosh-state/bosh-state.json - -bosh delete-env \ - bosh-manifest-s3/bosh.yml \ - --state bosh-state/bosh-state.json - -echo '{}' > bosh-state/bosh-state.json \ No newline at end of file diff --git a/ci/tasks.old/bosh/destroy/task.yml b/ci/tasks.old/bosh/destroy/task.yml deleted file mode 100644 index 17473245..00000000 --- a/ci/tasks.old/bosh/destroy/task.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: bosh-state-s3 - - name: bosh-manifest-s3 - -outputs: - - name: bosh-state - -run: - path: paas-bootstrap-git/ci/tasks/bosh/destroy/task.sh diff --git a/ci/tasks.old/bosh/get_database_vars/task.sh b/ci/tasks.old/bosh/get_database_vars/task.sh deleted file mode 100755 index 1069ed4f..00000000 --- a/ci/tasks.old/bosh/get_database_vars/task.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -jq '.modules[0].outputs | with_entries(.value = .value.value)' < vpc-tfstate-s3/tfstate.json > vpc-vars/vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "bosh-terraform/terraform.tfstate" > bosh-vars/vars.json - -cat >bosh-vars/env <<-EOS -USER_DB=$(jq -r .bosh_db_username < bosh-vars/vars.json) -DUMMY_DB=$(jq -r .bosh_dummy_db < bosh-vars/vars.json) -PASSWORD_DB=$(jq -r .bosh_rds_password < bosh-vars/vars.json) -FQDN_DB=$(jq -r .bosh_rds_fqdn < bosh-vars/vars.json) -EOS diff --git a/ci/tasks.old/bosh/get_database_vars/task.yml b/ci/tasks.old/bosh/get_database_vars/task.yml deleted file mode 100644 index 85415ecd..00000000 --- a/ci/tasks.old/bosh/get_database_vars/task.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: -- name: paas-bootstrap-git -- name: vpc-tfstate-s3 -- name: bosh-terraform - -outputs: -- name: vpc-vars -- name: bosh-vars - -run: - path: paas-bootstrap-git/ci/tasks/bosh/get_database_vars/task.sh \ No newline at end of file diff --git a/ci/tasks.old/bosh/get_terraform_vars/task.sh b/ci/tasks.old/bosh/get_terraform_vars/task.sh deleted file mode 100755 index 8dc40ecb..00000000 --- a/ci/tasks.old/bosh/get_terraform_vars/task.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -jq '.modules[0].outputs | with_entries(.value = .value.value)' < vpc-tfstate-s3/tfstate.json > vpc-vars/vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < concourse-tfstate-s3/tfstate.json > concourse-vars/vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "jumpbox-tfstate-s3/${ENVIRONMENT}.tfstate" > jumpbox-vars/vars.json diff --git a/ci/tasks.old/bosh/get_terraform_vars/task.yml b/ci/tasks.old/bosh/get_terraform_vars/task.yml deleted file mode 100644 index c332c01f..00000000 --- a/ci/tasks.old/bosh/get_terraform_vars/task.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: -- name: paas-bootstrap-git -- name: vpc-tfstate-s3 -- name: concourse-tfstate-s3 -- name: jumpbox-tfstate-s3 - -outputs: -- name: vpc-vars -- name: concourse-vars -- name: jumpbox-vars - -run: - path: paas-bootstrap-git/ci/tasks/bosh/get_terraform_vars/task.sh - -params: - ENVIRONMENT: \ No newline at end of file diff --git a/ci/tasks.old/bosh/interpolate/task.sh b/ci/tasks.old/bosh/interpolate/task.sh deleted file mode 100755 index d928df94..00000000 --- a/ci/tasks.old/bosh/interpolate/task.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -cp bosh-vars-s3/bosh-variables.yml bosh-manifests/bosh-variables.yml - -jq '.modules[0].outputs | with_entries(.value = .value.value)' < vpc-tfstate-s3/tfstate.json > vpc-vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < concourse-tfstate-s3/tfstate.json > concourse-vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "bosh-tfstate-s3/${ENVIRONMENT}.tfstate" > bosh-vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "bosh-databases-tfstate-s3/${ENVIRONMENT}.tfstate" > databases-vars.json - - -bosh int \ - ./bosh-deployment-git/bosh.yml \ - --vars-store bosh-manifests/bosh-variables.yml \ - -o bosh-deployment-git/aws/cpi.yml \ - -o bosh-deployment-git/aws/cli-iam-instance-profile.yml \ - -o bosh-deployment-git/misc/external-db.yml \ - -o paas-bootstrap-git/operations/bosh/iam-instance-profile.yml \ - -o paas-bootstrap-git/operations/bosh/tags.yml \ - -o paas-bootstrap-git/operations/bosh/dns-resolution.yml \ - -o paas-bootstrap-git/operations/bosh/certificate.yml \ - -v director_name=bosh \ - -v internal_cidr="$(jq -r .internal_cidr < bosh-vars.json)" \ - -v internal_gw="$(jq -r .internal_cidr < bosh-vars.json | sed 's#0/24#1#')" \ - -v internal_ip="$(jq -r .internal_cidr < bosh-vars.json | sed 's#0/24#6#')" \ - -v bosh_director_fqdn="$(jq -r .bosh_director_fqdn < bosh-vars.json)" \ - -v private_dns_nameserver="$(jq -r '.vpc_dns_nameserver' < vpc-vars.json)" \ - -v region="$(jq -r .region < bosh-vars.json)" \ - -v bosh_iam_instance_profile="$(jq -r .bosh_iam_instance_profile < bosh-vars.json)" \ - -v az="$(jq -r .az < bosh-vars.json)" \ - -v default_key_name="$(jq -r .default_key_name < concourse-vars.json)" \ - -v default_security_groups="$(jq -r .default_security_groups < bosh-terraform/metadata)" \ - --var-file private_key=ssh-private-key-s3/ssh-key.pem \ - -v subnet_id="$(jq -r .subnet_id < bosh-vars.json)" \ - -v environment="${ENVIRONMENT}" \ - -v external_db_host="$(jq -r '.bosh_rds_fqdn' < bosh-vars.json)" \ - -v external_db_port="$(jq -r '.bosh_db_port' < bosh-vars.json)" \ - -v external_db_user="$(jq -r '.bosh_db_username' < bosh-vars.json)" \ - -v external_db_password="$(jq -r '.bosh_rds_password' < bosh-vars.json)" \ - -v external_db_adapter="$(jq -r '.bosh_db_type' < bosh-vars.json)" \ - -v external_db_name="$(jq -r '.bosh_database_name' < databases-vars.json)" \ - > bosh-manifests/bosh.yml diff --git a/ci/tasks.old/bosh/interpolate/task.yml b/ci/tasks.old/bosh/interpolate/task.yml deleted file mode 100644 index f0e040dc..00000000 --- a/ci/tasks.old/bosh/interpolate/task.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: vpc-tfstate-s3 - - name: bosh-terraform - - name: bosh-tfstate-s3 - - name: bosh-deployment-git - - name: bosh-vars-s3 - - name: ssh-private-key-s3 - - name: concourse-tfstate-s3 - - name: bosh-databases-tfstate-s3 -outputs: - - name: bosh-manifests - -run: - path: paas-bootstrap-git/ci/tasks/bosh/interpolate/task.sh - -params: - ENVIRONMENT: diff --git a/ci/tasks.old/bosh/test/task.sh b/ci/tasks.old/bosh/test/task.sh deleted file mode 100755 index 3caef80d..00000000 --- a/ci/tasks.old/bosh/test/task.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -INTERNAL_IP=$(bosh int --path /instance_groups/name=bosh/networks/name=default/static_ips/0 bosh-manifest-s3/bosh.yml) - -wget -qO - -t 1 -T 5 "https://${INTERNAL_IP}:25555/info" \ - --ca-certificate=<(bosh int --path /cloud_provider/cert/ca bosh-manifest-s3/bosh.yml) | jq . \ No newline at end of file diff --git a/ci/tasks.old/bosh/test/task.yml b/ci/tasks.old/bosh/test/task.yml deleted file mode 100644 index b6b78b89..00000000 --- a/ci/tasks.old/bosh/test/task.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: -- name: paas-bootstrap-git -- name: bosh-manifest-s3 - -run: - path: paas-bootstrap-git/ci/tasks/bosh/test/task.sh - -params: - ENVIRONMENT: diff --git a/ci/tasks.old/cf/cloud_config/task.sh b/ci/tasks.old/cf/cloud_config/task.sh deleted file mode 100755 index 62de02ac..00000000 --- a/ci/tasks.old/cf/cloud_config/task.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -jq '.modules[0].outputs | with_entries(.value = .value.value)' < vpc-tfstate-s3/tfstate.json > vpc-vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < concourse-tfstate-s3/tfstate.json > concourse-vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "cf-tfstate-s3/${ENVIRONMENT}.tfstate" > cf-vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "prometheus-tfstate-s3/${ENVIRONMENT}.tfstate" > prometheus-vars.json - -bosh_admin_password=$(bosh int bosh-vars-s3/bosh-variables.yml --path /admin_password) -bosh int bosh-vars-s3/bosh-variables.yml --path /default_ca/ca > bosh_ca.pem -bosh_ip=$(bosh int --path '/cloud_provider/ssh_tunnel/host' bosh-manifest-s3/bosh.yml) - -export BOSH_CLIENT=admin -export BOSH_CLIENT_SECRET="${bosh_admin_password}" -export BOSH_ENVIRONMENT="https://${bosh_ip}:25555" -export BOSH_CA_CERT=bosh_ca.pem - -bosh update-cloud-config -n \ - paas-bootstrap-git/cloud-config/cf/cloud-config.yml \ - -o paas-bootstrap-git/operations/cloud-config/router-extensions.yml \ - -o paas-bootstrap-git/operations/cloud-config/cf-scheduler-extensions.yml \ - -o paas-bootstrap-git/operations/cloud-config/cf-s3-blobstore.yml \ - -o paas-bootstrap-git/operations/cloud-config/cf-rds-sec-group.yml \ - -o paas-bootstrap-git/operations/cloud-config/prometheus.yml \ - -v az1="$(jq -r .az1 < vpc-vars.json)" \ - -v az2="$(jq -r .az2 < vpc-vars.json)" \ - -v az3="$(jq -r .az3 < vpc-vars.json)" \ - -v private_subnet_az1_gateway="$(jq -r '.cf_internal_subnet_az1_cidr' < cf-vars.json | sed 's#0/24#1#')" \ - -v private_subnet_az2_gateway="$(jq -r '.cf_internal_subnet_az2_cidr' < cf-vars.json | sed 's#0/24#1#')" \ - -v private_subnet_az3_gateway="$(jq -r '.cf_internal_subnet_az3_cidr' < cf-vars.json | sed 's#0/24#1#')" \ - -v reserved_az1_cidr="$(jq -r '.cf_internal_subnet_az1_cidr' < cf-vars.json | sed 's#0/24#1/30#')" \ - -v reserved_az2_cidr="$(jq -r '.cf_internal_subnet_az2_cidr' < cf-vars.json | sed 's#0/24#1/30#')" \ - -v reserved_az3_cidr="$(jq -r '.cf_internal_subnet_az3_cidr' < cf-vars.json | sed 's#0/24#1/30#')" \ - -v private_dns_nameserver="$(jq -r '.vpc_dns_nameserver' < vpc-vars.json)" \ - -v internal_security_group="$(jq -r '.cf_internal_security_group_id' < cf-vars.json)" \ - -v private_subnet_az1_id="$(jq -r '.cf_internal_subnet_az1_id' < cf-vars.json)" \ - -v private_subnet_az2_id="$(jq -r '.cf_internal_subnet_az2_id' < cf-vars.json)" \ - -v private_subnet_az3_id="$(jq -r '.cf_internal_subnet_az3_id' < cf-vars.json)" \ - -v private_subnet_az1_cidr="$(jq -r '.cf_internal_subnet_az1_cidr' < cf-vars.json)" \ - -v private_subnet_az2_cidr="$(jq -r '.cf_internal_subnet_az2_cidr' < cf-vars.json)" \ - -v private_subnet_az3_cidr="$(jq -r '.cf_internal_subnet_az3_cidr' < cf-vars.json)" \ - -v cf-router-target-group-name="$(jq -r '.cf_router_target_group_name' < cf-vars.json)" \ - -v cf-router-lb-internal-security-group-id="$(jq -r '.cf_router_lb_internal_security_group_id' < cf-vars.json)" \ - -v cf-internal-security-group-id="$(jq -r '.cf_internal_security_group_id' < cf-vars.json)" \ - -v cf-ssh-internal="$(jq -r '.cf_ssh_internal' < cf-vars.json)" \ - -v cf-ssh-lb="$(jq -r '.cf_ssh_lb' < cf-vars.json)" \ - -v cf_s3_iam_instance_profile="$(jq -r '.cf_s3_iam_instance_profile' < cf-vars.json)" \ - -v cf-rds-client-security-group="$(jq -r '.cf_rds_client_security_group_id' < cf-vars.json)" \ - -v prometheus_subnet_az1_cidr="$(jq -r .prometheus_subnet_az1_cidr < prometheus-vars.json)" \ - -v prometheus_subnet_az1_id="$(jq -r .prometheus_subnet_az1_id < prometheus-vars.json)" \ - -v prometheus_security_group="$(jq -r .prometheus_security_group_id < prometheus-vars.json)" \ - -v prometheus_subnet_az1_gateway="$(jq -r .prometheus_subnet_az1_cidr < prometheus-vars.json | sed 's#0/24#1#')" \ - -v grafana_target_group_name="$(jq -r .grafana_target_group_name < prometheus-vars.json)" \ - -v prometheus_target_group_name="$(jq -r .prometheus_target_group_name < prometheus-vars.json)" \ - -v alertmanager_target_group_name="$(jq -r .alertmanager_target_group_name < prometheus-vars.json)" - -bosh cloud-config > cf-manifests/cloud-config.yml diff --git a/ci/tasks.old/cf/cloud_config/task.yml b/ci/tasks.old/cf/cloud_config/task.yml deleted file mode 100644 index b8991827..00000000 --- a/ci/tasks.old/cf/cloud_config/task.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: cf-deployment-git - - name: vpc-tfstate-s3 - - name: cf-tfstate-s3 - - name: bosh-vars-s3 - - name: bosh-manifest-s3 - - name: concourse-tfstate-s3 - - name: prometheus-tfstate-s3 - -outputs: - - name: cf-manifests - -run: - path: paas-bootstrap-git/ci/tasks/cf/cloud_config/task.sh - -params: - ENVIRONMENT: - DOMAIN: diff --git a/ci/tasks.old/cf/databases/db_connectivity_test/task.sh b/ci/tasks.old/cf/databases/db_connectivity_test/task.sh deleted file mode 100755 index de98270f..00000000 --- a/ci/tasks.old/cf/databases/db_connectivity_test/task.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -set -a -source cf-vars/env -set +a - -: $USER_DB -: $FQDN_DB -: $PASSWORD_DB -: $DUMMY_DB - - -set +e -while : -do - mysql -u$USER_DB -p$PASSWORD_DB -h $FQDN_DB $DUMMY_DB <<<"select 1;" && break - sleep 1 -done \ No newline at end of file diff --git a/ci/tasks.old/cf/databases/db_connectivity_test/task.yml b/ci/tasks.old/cf/databases/db_connectivity_test/task.yml deleted file mode 100644 index 3c64ed39..00000000 --- a/ci/tasks.old/cf/databases/db_connectivity_test/task.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: -- name: paas-bootstrap-git -- name: cf-vars - -run: - path: paas-bootstrap-git/ci/tasks/cf/databases/db_connectivity_test/task.sh - diff --git a/ci/tasks.old/cf/databases/get_database_vars/task.sh b/ci/tasks.old/cf/databases/get_database_vars/task.sh deleted file mode 100755 index 56dff149..00000000 --- a/ci/tasks.old/cf/databases/get_database_vars/task.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -jq '.modules[0].outputs | with_entries(.value = .value.value)' < vpc-tfstate-s3/tfstate.json > vpc-vars/vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "cf-terraform/terraform.tfstate" > cf-vars/vars.json - -cat >cf-vars/env <<-EOS -USER_DB=$(jq -r .cf_db_username < cf-vars/vars.json) -DUMMY_DB=$(jq -r .cf_dummy_db < cf-vars/vars.json) -PASSWORD_DB=$(jq -r .cf_rds_password < cf-vars/vars.json) -FQDN_DB=$(jq -r .cf_rds_fqdn < cf-vars/vars.json) -EOS \ No newline at end of file diff --git a/ci/tasks.old/cf/databases/get_database_vars/task.yml b/ci/tasks.old/cf/databases/get_database_vars/task.yml deleted file mode 100644 index d53cd5cc..00000000 --- a/ci/tasks.old/cf/databases/get_database_vars/task.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: -- name: paas-bootstrap-git -- name: vpc-tfstate-s3 -- name: cf-terraform - -outputs: -- name: vpc-vars -- name: cf-vars - -run: - path: paas-bootstrap-git/ci/tasks/cf/databases/get_database_vars/task.sh diff --git a/ci/tasks.old/cf/deploy_cf/task.sh b/ci/tasks.old/cf/deploy_cf/task.sh deleted file mode 100755 index 61707775..00000000 --- a/ci/tasks.old/cf/deploy_cf/task.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -bosh_admin_password=$(bosh int bosh-vars-s3/bosh-variables.yml --path /admin_password) -bosh int bosh-vars-s3/bosh-variables.yml --path /default_ca/ca > bosh_ca.pem -bosh_ip=$(bosh int --path '/cloud_provider/ssh_tunnel/host' bosh-manifest-s3/bosh.yml) - -export BOSH_CLIENT=admin -export BOSH_CLIENT_SECRET="${bosh_admin_password}" -export BOSH_ENVIRONMENT="https://${bosh_ip}:25555" -export BOSH_CA_CERT=$(cat bosh_ca.pem) - -export BOSH_LOG_LEVEL=info -bosh -d cf --non-interactive deploy ./cf-manifest-s3/cf.yml diff --git a/ci/tasks.old/cf/deploy_cf/task.yml b/ci/tasks.old/cf/deploy_cf/task.yml deleted file mode 100644 index e4e8870f..00000000 --- a/ci/tasks.old/cf/deploy_cf/task.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: cf-manifest-s3 - - name: bosh-vars-s3 - - name: bosh-manifest-s3 - -outputs: - -run: - path: paas-bootstrap-git/ci/tasks/cf/deploy_cf/task.sh - diff --git a/ci/tasks.old/cf/destroy/task.sh b/ci/tasks.old/cf/destroy/task.sh deleted file mode 100755 index 50893ee1..00000000 --- a/ci/tasks.old/cf/destroy/task.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -bosh_admin_password=$(bosh int bosh-vars-s3/bosh-variables.yml --path /admin_password) -bosh int bosh-vars-s3/bosh-variables.yml --path /default_ca/ca > bosh_ca.pem -bosh_ip=$(bosh int --path '/cloud_provider/ssh_tunnel/host' bosh-manifest-s3/bosh.yml) - -export BOSH_CLIENT=admin -export BOSH_CLIENT_SECRET="${bosh_admin_password}" -export BOSH_ENVIRONMENT="https://${bosh_ip}:25555" -export BOSH_CA_CERT=$(cat bosh_ca.pem) - -bosh -d cf --non-interactive delete-deployment \ No newline at end of file diff --git a/ci/tasks.old/cf/destroy/task.yml b/ci/tasks.old/cf/destroy/task.yml deleted file mode 100644 index 131e649b..00000000 --- a/ci/tasks.old/cf/destroy/task.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: bosh-vars-s3 - - name: bosh-manifest-s3 - -run: - path: paas-bootstrap-git/ci/tasks/cf/destroy/task.sh diff --git a/ci/tasks.old/cf/get_management_vars/task.sh b/ci/tasks.old/cf/get_management_vars/task.sh deleted file mode 100755 index ecc5eebf..00000000 --- a/ci/tasks.old/cf/get_management_vars/task.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -cat >cf_mgmt_vars/env <<-EOS -USER_ID=cf_mgmt -CLIENT_SECRET=$(bosh interpolate --path /uaa_clients_cf_mgmt_secret cf-vars-s3/cf-variables.yml) -SYSTEM_DOMAIN=system.$DOMAIN -EOS \ No newline at end of file diff --git a/ci/tasks.old/cf/get_management_vars/task.yml b/ci/tasks.old/cf/get_management_vars/task.yml deleted file mode 100644 index 22748efb..00000000 --- a/ci/tasks.old/cf/get_management_vars/task.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: cf-vars-s3 - -outputs: - - name: cf_mgmt_vars - -run: - path: paas-bootstrap-git/ci/tasks/cf/get_management_vars/task.sh - -params: - ENVIRONMENT: - DOMAIN: \ No newline at end of file diff --git a/ci/tasks.old/cf/get_terraform_vars/task.sh b/ci/tasks.old/cf/get_terraform_vars/task.sh deleted file mode 100755 index 8403488d..00000000 --- a/ci/tasks.old/cf/get_terraform_vars/task.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -jq '.modules[0].outputs | with_entries(.value = .value.value)' < vpc-tfstate-s3/tfstate.json > vpc-vars/vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < concourse-tfstate-s3/tfstate.json > concourse-vars/vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "bosh-tfstate-s3/${ENVIRONMENT}.tfstate" > bosh-vars/vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "jumpbox-tfstate-s3/${ENVIRONMENT}.tfstate" > jumpbox-vars/vars.json diff --git a/ci/tasks.old/cf/get_terraform_vars/task.yml b/ci/tasks.old/cf/get_terraform_vars/task.yml deleted file mode 100644 index 3958cd4f..00000000 --- a/ci/tasks.old/cf/get_terraform_vars/task.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: -- name: paas-bootstrap-git -- name: vpc-tfstate-s3 -- name: jumpbox-tfstate-s3 -- name: bosh-tfstate-s3 -- name: concourse-tfstate-s3 - -outputs: -- name: vpc-vars -- name: jumpbox-vars -- name: bosh-vars -- name: concourse-vars - -run: - path: paas-bootstrap-git/ci/tasks/cf/get_terraform_vars/task.sh - -params: - ENVIRONMENT: diff --git a/ci/tasks.old/cf/interpolate/task.sh b/ci/tasks.old/cf/interpolate/task.sh deleted file mode 100755 index c7434d87..00000000 --- a/ci/tasks.old/cf/interpolate/task.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -cp cf-vars-s3/cf-variables.yml cf-manifests/cf-variables.yml -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "cf-tfstate-s3/${ENVIRONMENT}.tfstate" > cf-vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < vpc-tfstate-s3/tfstate.json > vpc-vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "databases-tfstate-s3/${ENVIRONMENT}.tfstate" > databases-vars.json - - -SYSTEM_DOMAIN="system.${DOMAIN}" -APPS_DOMAIN="apps.${DOMAIN}" - -CF_DB_ENDPOINT="$(jq -r '.cf_rds_fqdn' < cf-vars.json)" -CF_DB_USERNAME="$(jq -r '.cf_db_username' < cf-vars.json)" -CF_DB_PASSWORD="$(jq -r '.cf_rds_password' < cf-vars.json)" - -if [[ ${ENVIRONMENT} = "eng"* ]]; then - profile="engineering" -else - profile="${ENVIRONMENT}" -fi -instance_count_file=paas-bootstrap-git/profiles/${profile}/instance-count.yml - -bosh int \ - ./cf-deployment-git/cf-deployment.yml \ - --vars-store cf-manifests/cf-variables.yml \ - --vars-file="${instance_count_file}" \ - -o cf-deployment-git/operations/aws.yml \ - -o cf-deployment-git/operations/override-app-domains.yml \ - -o cf-deployment-git/operations/use-external-blobstore.yml \ - -o cf-deployment-git/operations/use-external-dbs.yml \ - -o prometheus-deployment-git/manifests/operators/cf/add-prometheus-uaa-clients.yml \ - -o paas-bootstrap-git/operations/bosh/tags.yml \ - -o paas-bootstrap-git/operations/cf/stemcells.yml \ - -o paas-bootstrap-git/operations/cf/router-sec-group.yml \ - -o paas-bootstrap-git/operations/cf/scheduler.yml \ - -o paas-bootstrap-git/operations/cf/s3_blobstore_with_kms_and_iam.yml \ - -o paas-bootstrap-git/operations/cf/azs.yml \ - -o paas-bootstrap-git/operations/cf/instance-counts.yml \ - -o paas-bootstrap-git/operations/cf/rds-access.yml \ - -o paas-bootstrap-git/operations/cf/uaa-clients.yml \ - -o paas-bootstrap-git/operations/cf/test-user.yml \ - -v environment="${ENVIRONMENT}" \ - -v region="$(jq -r .region < vpc-vars.json)" \ - -v system_domain="${SYSTEM_DOMAIN}" \ - -v app_domains="[${APPS_DOMAIN}]" \ - -v smoke_test_app_domain="${APPS_DOMAIN}" \ - -v buildpack_directory_key="$(jq -r '.cf_buildpacks_bucket_name' < cf-vars.json)" \ - -v droplet_directory_key="$(jq -r '.cf_droplets_bucket_name' < cf-vars.json)" \ - -v app_package_directory_key="$(jq -r '.cf_packages_bucket_name' < cf-vars.json)" \ - -v resource_directory_key="$(jq -r '.cf_resource_pool_bucket_name' < cf-vars.json)" \ - -v cf_blobstore_s3_kms_key_id="$(jq -r '.cf_blobstore_s3_kms_key_id' < cf-vars.json)" \ - -v external_database_type="$(jq -r '.cf_db_type' < cf-vars.json)" \ - -v external_database_port="$(jq -r '.cf_db_port' < cf-vars.json)" \ - -v external_uaa_database_name="$(jq -r '.uaa_database_name' < databases-vars.json)" \ - -v external_uaa_database_address="${CF_DB_ENDPOINT}" \ - -v external_uaa_database_password="${CF_DB_PASSWORD}" \ - -v external_uaa_database_username="${CF_DB_USERNAME}" \ - -v external_cc_database_name="$(jq -r '.cc_database_name' < databases-vars.json)" \ - -v external_cc_database_address="${CF_DB_ENDPOINT}" \ - -v external_cc_database_password="${CF_DB_PASSWORD}" \ - -v external_cc_database_username="${CF_DB_USERNAME}" \ - -v external_bbs_database_name="$(jq -r '.bbs_database_name' < databases-vars.json)" \ - -v external_bbs_database_address="${CF_DB_ENDPOINT}" \ - -v external_bbs_database_password="${CF_DB_PASSWORD}" \ - -v external_bbs_database_username="${CF_DB_USERNAME}" \ - -v external_routing_api_database_name="$(jq -r '.routing_api_database_name' < databases-vars.json)" \ - -v external_routing_api_database_address="${CF_DB_ENDPOINT}" \ - -v external_routing_api_database_password="${CF_DB_PASSWORD}" \ - -v external_routing_api_database_username="${CF_DB_USERNAME}" \ - -v external_policy_server_database_name="$(jq -r '.policy_server_database_name' < databases-vars.json)" \ - -v external_policy_server_database_address="${CF_DB_ENDPOINT}" \ - -v external_policy_server_database_password="${CF_DB_PASSWORD}" \ - -v external_policy_server_database_username="${CF_DB_USERNAME}" \ - -v external_silk_controller_database_name="$(jq -r '.silk_controller_database_name' < databases-vars.json)" \ - -v external_silk_controller_database_address="${CF_DB_ENDPOINT}" \ - -v external_silk_controller_database_password="${CF_DB_PASSWORD}" \ - -v external_silk_controller_database_username="${CF_DB_USERNAME}" \ - -v external_locket_database_name="$(jq -r '.locket_database_name' < databases-vars.json)" \ - -v external_locket_database_address="${CF_DB_ENDPOINT}" \ - -v external_locket_database_password="${CF_DB_PASSWORD}" \ - -v external_locket_database_username="${CF_DB_USERNAME}" \ - > cf-manifests/cf.yml diff --git a/ci/tasks.old/cf/interpolate/task.yml b/ci/tasks.old/cf/interpolate/task.yml deleted file mode 100644 index 656d52fd..00000000 --- a/ci/tasks.old/cf/interpolate/task.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: cf-deployment-git - - name: cf-vars-s3 - - name: cf-tfstate-s3 - - name: vpc-tfstate-s3 - - name: databases-tfstate-s3 - - name: prometheus-deployment-git - -outputs: - - name: cf-manifests - -run: - path: paas-bootstrap-git/ci/tasks/cf/interpolate/task.sh - -params: - ENVIRONMENT: - DOMAIN: diff --git a/ci/tasks.old/cf/interpolate_cats/task.sh b/ci/tasks.old/cf/interpolate_cats/task.sh deleted file mode 100755 index acbf85f0..00000000 --- a/ci/tasks.old/cf/interpolate_cats/task.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -eu - -: $DOMAIN - -ADMIN_PASSWORD="$(bosh int cf-vars-s3/cf-variables.yml --path /cf_admin_password)" - -cat $CATS_CONFIG_FILE | jq " -.api = \"api.system.${DOMAIN}\" | -.apps_domain = \"apps.${DOMAIN}\" | -.admin_user = \"admin\" | -.admin_password = \"${ADMIN_PASSWORD}\" -" > integration-config/integration_config.json diff --git a/ci/tasks.old/cf/interpolate_cats/task.yml b/ci/tasks.old/cf/interpolate_cats/task.yml deleted file mode 100644 index cb5d2d46..00000000 --- a/ci/tasks.old/cf/interpolate_cats/task.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: cf-acceptance-tests-git - - name: cf-vars-s3 - -outputs: - - name: integration-config - -run: - path: paas-bootstrap-git/ci/tasks/cf/interpolate_cats/task.sh - -params: - DOMAIN: - ENVIRONMENT: - CATS_CONFIG_FILE: diff --git a/ci/tasks.old/cf/management/task.sh b/ci/tasks.old/cf/management/task.sh deleted file mode 100755 index bc2c3477..00000000 --- a/ci/tasks.old/cf/management/task.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -set -e - -: ENVIRONMENT - -set -a -source cf_mgmt_vars/env -set +a - -if [[ ${ENVIRONMENT} = "eng"* ]]; then - profile="engineering" -else - profile="${ENVIRONMENT}" -fi - -cd paas-bootstrap-git/profiles/${profile} - -cf-mgmt create-orgs -cf-mgmt create-spaces -cf-mgmt update-org-users -cf-mgmt update-space-users diff --git a/ci/tasks.old/cf/management/task.yml b/ci/tasks.old/cf/management/task.yml deleted file mode 100644 index 60280d5c..00000000 --- a/ci/tasks.old/cf/management/task.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: {repository: pivotalservices/cf-mgmt, tag: "latest"} - -inputs: - - name: paas-bootstrap-git - - name: cf_mgmt_vars - -run: - path: paas-bootstrap-git/ci/tasks/cf/management/task.sh - -params: - ENVIRONMENT: diff --git a/ci/tasks.old/cf/smoke_tests/task.sh b/ci/tasks.old/cf/smoke_tests/task.sh deleted file mode 100755 index a8fa83a2..00000000 --- a/ci/tasks.old/cf/smoke_tests/task.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -bosh_admin_password=$(bosh int bosh-vars-s3/bosh-variables.yml --path /admin_password) -bosh int bosh-vars-s3/bosh-variables.yml --path /default_ca/ca > bosh_ca.pem -bosh_ip=$(bosh int --path '/cloud_provider/ssh_tunnel/host' bosh-manifest-s3/bosh.yml) - -export BOSH_CLIENT=admin -export BOSH_CLIENT_SECRET="${bosh_admin_password}" -export BOSH_ENVIRONMENT="https://${bosh_ip}:25555" -export BOSH_CA_CERT=$(cat bosh_ca.pem) - -export BOSH_LOG_LEVEL=info -bosh -d cf run-errand smoke-tests \ No newline at end of file diff --git a/ci/tasks.old/cf/smoke_tests/task.yml b/ci/tasks.old/cf/smoke_tests/task.yml deleted file mode 100644 index 63881efa..00000000 --- a/ci/tasks.old/cf/smoke_tests/task.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: bosh-vars-s3 - - name: bosh-manifest-s3 - -run: - path: paas-bootstrap-git/ci/tasks/cf/smoke_tests/task.sh diff --git a/ci/tasks.old/cf/test/task.sh b/ci/tasks.old/cf/test/task.sh deleted file mode 100755 index 8cbb13d4..00000000 --- a/ci/tasks.old/cf/test/task.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -: $HOST -: $DOMAIN - -FQDN="$HOST.$DOMAIN" -RESOLVE=$(host "$FQDN" | awk '/has address/ { print $4 }') - -echo "FQDN: $FQDN" -test -n "$RESOLVE" \ No newline at end of file diff --git a/ci/tasks.old/cf/test/task.yml b/ci/tasks.old/cf/test/task.yml deleted file mode 100644 index 67731628..00000000 --- a/ci/tasks.old/cf/test/task.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: -- name: paas-bootstrap-git - -run: - path: paas-bootstrap-git/ci/tasks/cf/test/task.sh - -params: - HOST: - DOMAIN: diff --git a/ci/tasks.old/cf/upload_stemcell/task.sh b/ci/tasks.old/cf/upload_stemcell/task.sh deleted file mode 100755 index aab6ad90..00000000 --- a/ci/tasks.old/cf/upload_stemcell/task.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -bosh_admin_password=$(bosh int bosh-vars-s3/bosh-variables.yml --path /admin_password) -bosh int bosh-vars-s3/bosh-variables.yml --path /default_ca/ca > bosh_ca.pem -bosh_ip=$(bosh int --path '/cloud_provider/ssh_tunnel/host' bosh-manifest-s3/bosh.yml) -stemcell_version=$(bosh int cf-manifest-s3/cf.yml --path=/stemcells/alias=default/version) - -export BOSH_CLIENT=admin -export BOSH_CLIENT_SECRET="${bosh_admin_password}" -export BOSH_ENVIRONMENT="https://${bosh_ip}:25555" -export BOSH_CA_CERT=$(cat bosh_ca.pem) - -# Upload the stemcell only if it doesn't already exist -if ! bosh -n stemcells | grep -q "$stemcell_version"; then - bosh upload-stemcell https://s3.amazonaws.com/bosh-aws-light-stemcells/light-bosh-stemcell-${stemcell_version}-aws-xen-hvm-ubuntu-trusty-go_agent.tgz -fi diff --git a/ci/tasks.old/cf/upload_stemcell/task.yml b/ci/tasks.old/cf/upload_stemcell/task.yml deleted file mode 100644 index 6fffcd3a..00000000 --- a/ci/tasks.old/cf/upload_stemcell/task.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: cf-manifest-s3 - - name: bosh-vars-s3 - - name: bosh-manifest-s3 - -outputs: - -run: - path: paas-bootstrap-git/ci/tasks/cf/upload_stemcell/task.sh - diff --git a/ci/tasks.old/common/delete_s3/task.sh b/ci/tasks.old/common/delete_s3/task.sh deleted file mode 100755 index aef7d686..00000000 --- a/ci/tasks.old/common/delete_s3/task.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -set -eu - -: $BUCKET -: $FILE - -aws s3 rm "s3://$BUCKET/$FILE" \ No newline at end of file diff --git a/ci/tasks.old/common/delete_s3/task.yml b/ci/tasks.old/common/delete_s3/task.yml deleted file mode 100644 index 18a9f4ca..00000000 --- a/ci/tasks.old/common/delete_s3/task.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: governmentpaas/awscli, tag: latest } - -inputs: -- name: paas-bootstrap-git - -run: - path: paas-bootstrap-git/ci/tasks/common/delete_s3/task.sh - -params: - HOST: - DOMAIN: - PORT: - EXPECTED_RESULT: success diff --git a/ci/tasks.old/common/get_cf_tester_credentials/task.sh b/ci/tasks.old/common/get_cf_tester_credentials/task.sh deleted file mode 100755 index 45283e11..00000000 --- a/ci/tasks.old/common/get_cf_tester_credentials/task.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -cat >cf_user_credentials/env <<-EOS -USER_ID=cf_test_user -PASSWORD=$(bosh interpolate --path /cf_test_user_password cf-vars-s3/cf-variables.yml) -SYSTEM_DOMAIN=system.${DOMAIN} -APPS_DOMAIN=apps.${DOMAIN} -API=api.system.${DOMAIN} -EOS \ No newline at end of file diff --git a/ci/tasks.old/common/get_cf_tester_credentials/task.yml b/ci/tasks.old/common/get_cf_tester_credentials/task.yml deleted file mode 100644 index a1fd534f..00000000 --- a/ci/tasks.old/common/get_cf_tester_credentials/task.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: cf-vars-s3 - -outputs: - - name: cf_user_credentials - -run: - path: paas-bootstrap-git/ci/tasks/common/get_cf_tester_credentials/task.sh - -params: - DOMAIN: \ No newline at end of file diff --git a/ci/tasks.old/common/test_connectivity/task.sh b/ci/tasks.old/common/test_connectivity/task.sh deleted file mode 100755 index ddf22ad5..00000000 --- a/ci/tasks.old/common/test_connectivity/task.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -: $PORT -: $HOST -: $DOMAIN -: ${EXPECTED_OUTCOME:=success} - -FQDN="$HOST.$DOMAIN" - -set +e -echo "foo" | nc -w 1 "$FQDN" "$PORT" -RESULT=$? -if [ "$EXPECTED_OUTCOME" = success ]; then - exit $RESULT -elif [ "$RESULT" = 0 ]; then - exit 1 -fi -exit 0 - diff --git a/ci/tasks.old/common/test_connectivity/task.yml b/ci/tasks.old/common/test_connectivity/task.yml deleted file mode 100644 index b3f8722a..00000000 --- a/ci/tasks.old/common/test_connectivity/task.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: -- name: paas-bootstrap-git - -run: - path: paas-bootstrap-git/ci/tasks/common/test_connectivity/task.sh - -params: - HOST: - DOMAIN: - PORT: - EXPECTED_RESULT: success diff --git a/ci/tasks.old/common/test_endpoint/task.sh b/ci/tasks.old/common/test_endpoint/task.sh deleted file mode 100755 index ede1aefa..00000000 --- a/ci/tasks.old/common/test_endpoint/task.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -: $DOMAIN -: $HOST -: ${QUERY_PATH=/} -: ${REQUIRE_JSON:=false} - -URL="https://${HOST}.${DOMAIN}${QUERY_PATH}" -if [ "$REQUIRE_JSON" = true ]; then - wget -O - -nv -t 1 -T 5 --header='Accept: application/json' "$URL" -else - wget -O - -nv -t 1 -T 5 "$URL" -fi \ No newline at end of file diff --git a/ci/tasks.old/common/test_endpoint/task.yml b/ci/tasks.old/common/test_endpoint/task.yml deleted file mode 100644 index abaf7020..00000000 --- a/ci/tasks.old/common/test_endpoint/task.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: -- name: paas-bootstrap-git - -run: - path: paas-bootstrap-git/ci/tasks/common/test_endpoint/task.sh - -params: - HOST: - DOMAIN: - QUERY_PATH: '/' - REQUIRE_JSON: 'false' \ No newline at end of file diff --git a/ci/tasks.old/concourse/test_fqdn/task.sh b/ci/tasks.old/concourse/test_fqdn/task.sh deleted file mode 100755 index 687a063c..00000000 --- a/ci/tasks.old/concourse/test_fqdn/task.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -: $HOST -: $DOMAIN - -FQDN="$HOST.$DOMAIN" -RESOLVE=$(host "$FQDN" | awk '/has address/ { print $4 }') -CONCOURSE_EIP=$(cat concourse-tfstate-s3/tfstate.json | jq -r ".modules[].outputs | with_entries(.value = .value.value) | .concourse_public_ip") - -echo "FQDN: $FQDN, expect: $CONCOURSE_EIP, actual: $RESOLVE" -test "$RESOLVE" = "$CONCOURSE_EIP" \ No newline at end of file diff --git a/ci/tasks.old/concourse/test_fqdn/task.yml b/ci/tasks.old/concourse/test_fqdn/task.yml deleted file mode 100644 index 6a47f47b..00000000 --- a/ci/tasks.old/concourse/test_fqdn/task.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: -- name: paas-bootstrap-git -- name: concourse-tfstate-s3 - -run: - path: paas-bootstrap-git/ci/tasks/concourse/test_fqdn/task.sh - -params: - HOST: - DOMAIN: diff --git a/ci/tasks.old/concourse/test_iam_policy/task.sh b/ci/tasks.old/concourse/test_iam_policy/task.sh deleted file mode 100755 index 0906fa37..00000000 --- a/ci/tasks.old/concourse/test_iam_policy/task.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -: ${BUCKET:=omg-this-bucket-no-existy} - -touch foo -aws s3 mb "s3://$BUCKET" || exit 0 -echo "Oh crap, managed to create an S3 bucket ($BUCKET) and put stuff in it" -echo -aws s3 rb "s3://$BUCKET" -exit 1 \ No newline at end of file diff --git a/ci/tasks.old/concourse/test_iam_policy/task.yml b/ci/tasks.old/concourse/test_iam_policy/task.yml deleted file mode 100644 index 60ce7232..00000000 --- a/ci/tasks.old/concourse/test_iam_policy/task.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: governmentpaas/awscli, tag: latest } - -inputs: -- name: paas-bootstrap-git - -run: - path: paas-bootstrap-git/ci/tasks/concourse/test_iam_policy/task.sh - diff --git a/ci/tasks.old/destroy/empty_buckets/task.sh b/ci/tasks.old/destroy/empty_buckets/task.sh deleted file mode 100755 index 6fce5e73..00000000 --- a/ci/tasks.old/destroy/empty_buckets/task.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -: $ONS_PREFIX -: $ENVIRONMENT -: $NAMESPACE - -prefix="${ONS_PREFIX}-${ENVIRONMENT}-${NAMESPACE}" - -NAMES=$(aws s3api list-buckets --query "Buckets[?starts_with(Name, '${prefix}')].Name" --output text) - -echo "Removing all files/versions from ${prefix}-* (${NAMES})" - -function emptyBucket { - local bucket="$1"; - echo "Emptying ${bucket}" - aws s3 rm "s3://${bucket}" --recursive -} - -function emptyVersionedBucket { - local bucket="$1"; - - echo "Deleting versions and delete markers from ${bucket}" - - aws s3api list-object-versions --bucket "${bucket}" | \ - jq -r '.Versions + .DeleteMarkers | .[] | @text "\(.Key) \(.VersionId)"' | \ - while read key versionId - do - echo "- $bucket $key $versionId" - aws s3api delete-object --bucket "${bucket}" --key "${key}" --version-id "${versionId}" - done - - return 0 -} - -for name in $NAMES -do - emptyVersionedBucket "${name}" - emptyBucket "${name}" -done diff --git a/ci/tasks.old/destroy/empty_buckets/task.yml b/ci/tasks.old/destroy/empty_buckets/task.yml deleted file mode 100644 index 2f1a1ed2..00000000 --- a/ci/tasks.old/destroy/empty_buckets/task.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: { repository: onsdigital/dp-concourse-tools-awscli, tag: latest } - -inputs: - - name: paas-bootstrap-git - -run: - path: paas-bootstrap-git/ci/tasks/destroy/empty_buckets/task.sh diff --git a/ci/tasks.old/jumpbox/deploy/task.sh b/ci/tasks.old/jumpbox/deploy/task.sh deleted file mode 100755 index 8434c23e..00000000 --- a/ci/tasks.old/jumpbox/deploy/task.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -cp jumpbox-state-s3/jumpbox-state.json jumpbox-state/jumpbox-state.json - -bosh create-env \ - jumpbox-manifests/jumpbox.yml \ - --state jumpbox-state/jumpbox-state.json \ No newline at end of file diff --git a/ci/tasks.old/jumpbox/deploy/task.yml b/ci/tasks.old/jumpbox/deploy/task.yml deleted file mode 100644 index b2b3459f..00000000 --- a/ci/tasks.old/jumpbox/deploy/task.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: jumpbox-state-s3 - - name: jumpbox-manifests - -outputs: - - name: jumpbox-state - -run: - path: paas-bootstrap-git/ci/tasks/jumpbox/deploy/task.sh diff --git a/ci/tasks.old/jumpbox/destroy/task.sh b/ci/tasks.old/jumpbox/destroy/task.sh deleted file mode 100755 index 8a46fcbd..00000000 --- a/ci/tasks.old/jumpbox/destroy/task.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -cp jumpbox-state-s3/jumpbox-state.json jumpbox-state/jumpbox-state.json - -bosh delete-env \ - jumpbox-manifest-s3/jumpbox.yml \ - --state jumpbox-state/jumpbox-state.json - -echo '{}' > jumpbox-state/jumpbox-state.json diff --git a/ci/tasks.old/jumpbox/destroy/task.yml b/ci/tasks.old/jumpbox/destroy/task.yml deleted file mode 100644 index 4f5a27f9..00000000 --- a/ci/tasks.old/jumpbox/destroy/task.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: jumpbox-state-s3 - - name: jumpbox-manifest-s3 - -outputs: - - name: jumpbox-state - - -run: - path: paas-bootstrap-git/ci/tasks/jumpbox/destroy/task.sh diff --git a/ci/tasks.old/jumpbox/get_terraform_vars/task.sh b/ci/tasks.old/jumpbox/get_terraform_vars/task.sh deleted file mode 100755 index 9b958507..00000000 --- a/ci/tasks.old/jumpbox/get_terraform_vars/task.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -jq '.modules[0].outputs | with_entries(.value = .value.value)' < vpc-tfstate-s3/tfstate.json > vpc-vars/vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < concourse-tfstate-s3/tfstate.json > concourse-vars/vars.json diff --git a/ci/tasks.old/jumpbox/get_terraform_vars/task.yml b/ci/tasks.old/jumpbox/get_terraform_vars/task.yml deleted file mode 100644 index 46c29276..00000000 --- a/ci/tasks.old/jumpbox/get_terraform_vars/task.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: -- name: paas-bootstrap-git -- name: vpc-tfstate-s3 -- name: concourse-tfstate-s3 - -outputs: -- name: vpc-vars -- name: concourse-vars - -run: - path: paas-bootstrap-git/ci/tasks/jumpbox/get_terraform_vars/task.sh diff --git a/ci/tasks.old/jumpbox/interpolate/task.sh b/ci/tasks.old/jumpbox/interpolate/task.sh deleted file mode 100755 index 1ed7b007..00000000 --- a/ci/tasks.old/jumpbox/interpolate/task.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -cp jumpbox-vars-s3/jumpbox-variables.yml jumpbox-manifests/jumpbox-variables.yml - -cat concourse-tfstate-s3/tfstate.json | jq '.modules[0].outputs | with_entries(.value = .value.value)' > concourse-vars.json - -bosh int \ - ./jumpbox-deployment-git/jumpbox.yml \ - --vars-store jumpbox-manifests/jumpbox-variables.yml \ - -v internal_cidr="$(jq -r .internal_cidr < concourse-vars.json)" \ - -v internal_ip="$(jq -r .internal_cidr < concourse-vars.json | sed 's#0/24#5#')" \ - -v internal_gw="$(jq -r .internal_cidr < concourse-vars.json | sed 's#0/24#1#')" \ - -v default_security_groups="[$(jq -r .jumpbox_security_group < jumpbox-terraform/metadata)]" \ - -v external_ip="$(jq -r .jumpbox_external_ip < jumpbox-terraform/metadata)" \ - -v region="$(jq -r .region < concourse-vars.json)" \ - -v az="$(jq -r .az < concourse-vars.json)" \ - -v subnet_id="$(jq -r .subnet_id < concourse-vars.json)" \ - -v default_key_name="$(jq -r .default_key_name < concourse-vars.json)" \ - -v environment="${ENVIRONMENT}" \ - --var-file private_key=ssh-private-key-s3/ssh-key.pem \ - -o ./jumpbox-deployment-git/aws/cpi.yml \ - -o ./paas-bootstrap-git/operations/jumpbox/aws_cpi.yml \ - -o ./paas-bootstrap-git/operations/jumpbox/private_mbus.yml \ - -o ./paas-bootstrap-git/operations/jumpbox/tags.yml > jumpbox-manifests/jumpbox.yml diff --git a/ci/tasks.old/jumpbox/interpolate/task.yml b/ci/tasks.old/jumpbox/interpolate/task.yml deleted file mode 100644 index 002b4aea..00000000 --- a/ci/tasks.old/jumpbox/interpolate/task.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: jumpbox-terraform - - name: jumpbox-deployment-git - - name: jumpbox-vars-s3 - - name: ssh-private-key-s3 - - name: concourse-tfstate-s3 -outputs: - - name: jumpbox-manifests - -run: - path: paas-bootstrap-git/ci/tasks/jumpbox/interpolate/task.sh - -params: - ENVIRONMENT: diff --git a/ci/tasks.old/jumpbox/test/task.sh b/ci/tasks.old/jumpbox/test/task.sh deleted file mode 100755 index 012460c6..00000000 --- a/ci/tasks.old/jumpbox/test/task.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -: $ENVIRONMENT - -USERNAME=jumpbox -SSH_KEY=$(bosh int --path /jumpbox_ssh/private_key jumpbox-vars-s3/jumpbox-variables.yml) -INTERNAL_IP=$(bosh int --path /instance_groups/name=jumpbox/networks/name=private/static_ips/0 jumpbox-manifest-s3/jumpbox.yml) - -_keyfile=/var/tmp/tmp$$ -echo "$SSH_KEY" > $_keyfile -chmod 600 $_keyfile -trap 'rm -f $_keyfile' EXIT - -ssh -o BatchMode=yes -o StrictHostKeyChecking=no -i $_keyfile "${USERNAME}@${INTERNAL_IP}" "date" - diff --git a/ci/tasks.old/jumpbox/test/task.yml b/ci/tasks.old/jumpbox/test/task.yml deleted file mode 100644 index 7e3b2247..00000000 --- a/ci/tasks.old/jumpbox/test/task.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: -- name: paas-bootstrap-git -- name: jumpbox-vars-s3 -- name: jumpbox-manifest-s3 - -run: - path: paas-bootstrap-git/ci/tasks/jumpbox/test/task.sh - -params: - ENVIRONMENT: diff --git a/ci/tasks.old/jumpbox/test_fqdn/task.sh b/ci/tasks.old/jumpbox/test_fqdn/task.sh deleted file mode 100755 index ff39014c..00000000 --- a/ci/tasks.old/jumpbox/test_fqdn/task.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -: $HOST -: $DOMAIN - -FQDN="$HOST.$DOMAIN" -RESOLVE=$(host "$FQDN" | awk '/has address/ { print $4 }') -JUMPBOX_EIP=$(bosh int --path /instance_groups/name=jumpbox/networks/name=public/static_ips/0 jumpbox-manifest-s3/jumpbox.yml) - -echo "FQDN: $FQDN, expect: $JUMPBOX_EIP, actual: $RESOLVE" -test "$RESOLVE" = "$JUMPBOX_EIP" - - diff --git a/ci/tasks.old/jumpbox/test_fqdn/task.yml b/ci/tasks.old/jumpbox/test_fqdn/task.yml deleted file mode 100644 index 01e3d446..00000000 --- a/ci/tasks.old/jumpbox/test_fqdn/task.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: -- name: paas-bootstrap-git -- name: jumpbox-manifest-s3 - -run: - path: paas-bootstrap-git/ci/tasks/jumpbox/test_fqdn/task.sh - -params: - HOST: - DOMAIN: diff --git a/ci/tasks.old/prometheus/dashboard/task.sh b/ci/tasks.old/prometheus/dashboard/task.sh deleted file mode 100755 index 7bf6be3e..00000000 --- a/ci/tasks.old/prometheus/dashboard/task.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -: $ENVIRONMENT -: $SNAPSHOT_NAME -: $TEMPLATE - -# vars -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "prometheus-tfstate-s3/${ENVIRONMENT}.tfstate" > prometheus-vars.json -HOST="$(jq -r .grafana_fqdn < prometheus-vars.json)" -USER=$(bosh int prometheus-manifest-s3/prometheus.yml --path /instance_groups/name=grafana/jobs/name=grafana/properties/grafana/security/admin_user) -PASS=$(bosh int prometheus-manifest-s3/prometheus.yml --path /instance_groups/name=grafana/jobs/name=grafana/properties/grafana/security/admin_password) - -paas-bootstrap-git/bin/create_grafana_snapshot.sh -h "$HOST" -u "$USER" -p "$PASS" -f "$TEMPLATE" -n "$SNAPSHOT_NAME" \ No newline at end of file diff --git a/ci/tasks.old/prometheus/dashboard/task.yml b/ci/tasks.old/prometheus/dashboard/task.yml deleted file mode 100644 index ea6e6348..00000000 --- a/ci/tasks.old/prometheus/dashboard/task.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: prometheus-tfstate-s3 - - name: prometheus-manifest-s3 - -outputs: - -run: - path: paas-bootstrap-git/ci/tasks/prometheus/dashboard/task.sh - -params: - ENVIRONMENT: - SNAPSHOT_NAME: - TEMPLATE: \ No newline at end of file diff --git a/ci/tasks.old/prometheus/deploy/task.sh b/ci/tasks.old/prometheus/deploy/task.sh deleted file mode 100755 index 8f217c6c..00000000 --- a/ci/tasks.old/prometheus/deploy/task.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -bosh_admin_password=$(bosh int bosh-vars-s3/bosh-variables.yml --path /admin_password) -bosh int bosh-vars-s3/bosh-variables.yml --path /default_ca/ca > bosh_ca.pem -bosh_ip=$(bosh int --path '/cloud_provider/ssh_tunnel/host' bosh-manifest-s3/bosh.yml) - -export BOSH_CLIENT=admin -export BOSH_CLIENT_SECRET="${bosh_admin_password}" -export BOSH_ENVIRONMENT="https://${bosh_ip}:25555" -export BOSH_CA_CERT=$(cat bosh_ca.pem) - -export BOSH_LOG_LEVEL=info -bosh -d prometheus --non-interactive deploy ./prometheus-manifest-s3/prometheus.yml diff --git a/ci/tasks.old/prometheus/deploy/task.yml b/ci/tasks.old/prometheus/deploy/task.yml deleted file mode 100644 index 9118195c..00000000 --- a/ci/tasks.old/prometheus/deploy/task.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: prometheus-manifest-s3 - - name: bosh-vars-s3 - - name: bosh-manifest-s3 - -outputs: - -run: - path: paas-bootstrap-git/ci/tasks/prometheus/deploy/task.sh - diff --git a/ci/tasks.old/prometheus/destroy/task.sh b/ci/tasks.old/prometheus/destroy/task.sh deleted file mode 100755 index 2159bf0a..00000000 --- a/ci/tasks.old/prometheus/destroy/task.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -bosh_admin_password=$(bosh int bosh-vars-s3/bosh-variables.yml --path /admin_password) -bosh int bosh-vars-s3/bosh-variables.yml --path /default_ca/ca > bosh_ca.pem -bosh_ip=$(bosh int --path '/cloud_provider/ssh_tunnel/host' bosh-manifest-s3/bosh.yml) - -export BOSH_CLIENT=admin -export BOSH_CLIENT_SECRET="${bosh_admin_password}" -export BOSH_ENVIRONMENT="https://${bosh_ip}:25555" -export BOSH_CA_CERT=$(cat bosh_ca.pem) - -bosh -d prometheus --non-interactive delete-deployment \ No newline at end of file diff --git a/ci/tasks.old/prometheus/destroy/task.yml b/ci/tasks.old/prometheus/destroy/task.yml deleted file mode 100644 index fe93ad36..00000000 --- a/ci/tasks.old/prometheus/destroy/task.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: prometheus-manifest-s3 - - name: bosh-vars-s3 - - name: bosh-manifest-s3 - -run: - path: paas-bootstrap-git/ci/tasks/prometheus/destroy/task.sh diff --git a/ci/tasks.old/prometheus/get_terraform_vars/task.sh b/ci/tasks.old/prometheus/get_terraform_vars/task.sh deleted file mode 100755 index 592d80cc..00000000 --- a/ci/tasks.old/prometheus/get_terraform_vars/task.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -jq '.modules[0].outputs | with_entries(.value = .value.value)' < vpc-tfstate-s3/tfstate.json > vpc-vars/vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < concourse-tfstate-s3/tfstate.json > concourse-vars/vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "bosh-tfstate-s3/${ENVIRONMENT}.tfstate" > bosh-vars/vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "cf-tfstate-s3/${ENVIRONMENT}.tfstate" > cf-vars/vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "jumpbox-tfstate-s3/${ENVIRONMENT}.tfstate" > jumpbox-vars/vars.json \ No newline at end of file diff --git a/ci/tasks.old/prometheus/get_terraform_vars/task.yml b/ci/tasks.old/prometheus/get_terraform_vars/task.yml deleted file mode 100644 index 72254096..00000000 --- a/ci/tasks.old/prometheus/get_terraform_vars/task.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: -- name: paas-bootstrap-git -- name: vpc-tfstate-s3 -- name: concourse-tfstate-s3 -- name: bosh-tfstate-s3 -- name: cf-tfstate-s3 -- name: jumpbox-tfstate-s3 - -outputs: -- name: vpc-vars -- name: concourse-vars -- name: bosh-vars -- name: cf-vars -- name: jumpbox-vars - -run: - path: paas-bootstrap-git/ci/tasks/prometheus/get_terraform_vars/task.sh - -params: - ENVIRONMENT: diff --git a/ci/tasks.old/prometheus/interpolate/task.sh b/ci/tasks.old/prometheus/interpolate/task.sh deleted file mode 100755 index 5de40f1b..00000000 --- a/ci/tasks.old/prometheus/interpolate/task.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -cp prometheus-vars-s3/prometheus-variables.yml prometheus-manifests/prometheus-variables.yml -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "cf-tfstate-s3/${ENVIRONMENT}.tfstate" > cf-vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "bosh-tfstate-s3/${ENVIRONMENT}.tfstate" > bosh-vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < "prometheus-tfstate-s3/${ENVIRONMENT}.tfstate" > prometheus-vars.json -jq '.modules[0].outputs | with_entries(.value = .value.value)' < vpc-tfstate-s3/tfstate.json > vpc-vars.json - -PROMETHEUS_MANIFESTS=prometheus-deployment-git/manifests -SYSTEM_DOMAIN="system.${DOMAIN}" -METRON_DEPLOYMENT_NAME=cf - -bosh interpolate --path /default_ca/ca bosh-vars-s3/bosh-variables.yml > bosh_ca_cert.pem - -bosh -d prometheus interpolate "$PROMETHEUS_MANIFESTS"/prometheus.yml \ - --vars-store prometheus-manifests/prometheus-variables.yml \ - -o "$PROMETHEUS_MANIFESTS"/operators/monitor-bosh.yml \ - -o "$PROMETHEUS_MANIFESTS"/operators/monitor-cf.yml \ - -o paas-bootstrap-git/operations/prometheus/networks.yml \ - -v bosh_url="$(jq -r .bosh_director_fqdn < bosh-vars.json)" \ - -v bosh_username="admin" \ - -v bosh_password="$(bosh interpolate --path /admin_password bosh-vars-s3/bosh-variables.yml)" \ - --var-file bosh_ca_cert=bosh_ca_cert.pem \ - -v metrics_environment="$ENVIRONMENT" \ - -v metron_deployment_name="$METRON_DEPLOYMENT_NAME" \ - -v system_domain="$SYSTEM_DOMAIN" \ - -v uaa_clients_cf_exporter_secret="$(bosh interpolate --path /uaa_clients_cf_exporter_secret cf-vars-s3/cf-variables.yml)" \ - -v uaa_clients_firehose_exporter_secret="$(bosh interpolate --path /uaa_clients_firehose_exporter_secret cf-vars-s3/cf-variables.yml)" \ - -v traffic_controller_external_port="$(jq -r .cf_traffic_controller_port < cf-vars.json)" \ - -v skip_ssl_verify=false \ - > prometheus-manifests/prometheus.yml - diff --git a/ci/tasks.old/prometheus/interpolate/task.yml b/ci/tasks.old/prometheus/interpolate/task.yml deleted file mode 100644 index 675858e3..00000000 --- a/ci/tasks.old/prometheus/interpolate/task.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: prometheus-deployment-git - - name: prometheus-vars-s3 - - name: cf-vars-s3 - - name: cf-tfstate-s3 - - name: vpc-tfstate-s3 - - name: bosh-vars-s3 - - name: bosh-tfstate-s3 - - name: prometheus-tfstate-s3 - -outputs: - - name: prometheus-manifests - -run: - path: paas-bootstrap-git/ci/tasks/prometheus/interpolate/task.sh - -params: - ENVIRONMENT: - DOMAIN: - CF_DEPLOYMENT_NAME: diff --git a/ci/tasks.old/rds_broker/test_shared/task.sh b/ci/tasks.old/rds_broker/test_shared/task.sh deleted file mode 100755 index 3af281bd..00000000 --- a/ci/tasks.old/rds_broker/test_shared/task.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -set -a -source cf_user_credentials/env -set +a - -cd cf-tests-git/rds - -cf login -a $API -u $USER_ID -p $PASSWORD -o paas -s test - -cf create-service rds shared-psql paas-test-shared-rds - -cf push paas-test-shared-rds-app - -cf bind-service paas-test-shared-rds-app paas-test-shared-rds - -curl paas-test-shared-rds-app.$APPS_DOMAIN | grep "RDS service is OK" - -cf delete paas-test-shared-rds-app -f -r - -cf delete-service paas-test-shared-rds -f - -exit 0 \ No newline at end of file diff --git a/ci/tasks.old/rds_broker/test_shared/task.yml b/ci/tasks.old/rds_broker/test_shared/task.yml deleted file mode 100644 index 542615be..00000000 --- a/ci/tasks.old/rds_broker/test_shared/task.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: { repository: onsdigital/paas-ci-gp, tag: latest } - -inputs: - - name: paas-bootstrap-git - - name: cf_user_credentials - - name: cf-tests-git - -run: - path: paas-bootstrap-git/ci/tasks/rds_broker/test_shared/task.sh From 383387288d9cdaa5c9cb7552daf227323b8bfc74 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Fri, 7 Sep 2018 14:06:56 +0100 Subject: [PATCH 21/71] Update README for the new way Stick with CF v4.0.0 ftm Remove unnecessary S3 objects --- README.md | 143 ++++++++++++++++++++++------------- bin/deploy_cf.sh | 2 +- bin/set_concourse_secrets.sh | 2 +- ci/cf.yml | 1 - terraform/base/bosh.tf | 17 ----- terraform/base/cf.tf | 10 --- 6 files changed, 92 insertions(+), 83 deletions(-) diff --git a/README.md b/README.md index 4b675422..ae0c3d22 100644 --- a/README.md +++ b/README.md @@ -2,61 +2,106 @@ We use the code in this repository to bootstrap our AWS PaaS environment. The normal flow is: -1. Create the VPC under which the PaaS systems will live -2. Create a Concourse using `bosh create-env` -3. Deploy the pipelines that will spin up BOSH and CloudFoundry -4. Sit back and wait +1. Create the Infrastructure under which the Concourse, BOSH and PaaS systems will live +2. Deploy a BOSH director +3. Deploy CF, Concourse, Prometheus, etc. +4. Deploy the automation pipelines for upgrades ## Pre-requisites - AWS CLI - Terraform CLI +- BOSH CLI +- CF CLI +- CF Management CLIs +- Credhub CLI +- UAA CLI - Fly [CLI](https://concourse-ci.org/download.html) - [yq](https://github.com/mikefarah/yq) (or, `brew install yq`) - [jq](https://stedolan.github.io/jq/) (or, `brew install jq`) ## Creating a new environment -You'll need to create a `_vpc.tfvars` file with `az1`, `az2`, `region` and `parent_dns_zone`: +1. Update submodules -> **Note**: Multiple AZs are required in order to deploy an AWS ALB. +```sh + git submodule update --init +``` + +2. Create your environment directory and vars file + +```sh +mkdir data +touch data/.tfvars +``` -> set ingress_whitelist to the CIDRs that may access Concourse +You'll need to create a `.tfvars` file: ```json { -"az1": "eu-west-1a", -"az2": "eu-west-1b", -"region": "eu-west-1", -"parent_dns_zone": "", -"ingress_whitelist": ["0.0.0.0/0"], -"slack_webhook_uri": "https://hooks.slack.com/services/" + "environment": "", + "region": "eu-west-1", + "availability_zones": ["eu-west-1a", "eu-west-1b", "eu-west-1c"], + "parent_dns_zone": "", + "ingress_whitelist": ["0.0.0.0/0"], + "vpc_cidr_block": "10.121.0.0/16", + "cidr_blocks": { + "public": ["10.121.0.0/24", "10.121.1.0/24", "10.121.2.0/24"], + "internal": ["10.121.8.0/22", "10.121.12.0/22", "10.121.16.0/22"], + "services": ["10.121.28.0/22", "10.121.32.0/22", "10.121.36.0/22"], + "rds": ["10.121.50.0/24", "10.121.51.0/24", "10.121.52.0/24"], + "prometheus": ["10.121.53.0/24", "10.121.54.0/24", "10.121.55.0/24"], + "concourse": ["10.121.56.0/24", "10.121.57.0/24", "10.121.58.0/24"] + } } ``` -Example command: +The `vpc_cidr_block` parameter is an array of IP ranges that you want to be able to access +the PaaS. It should *not* be `0.0.0.0/0`. + +The `cidr_blocks` parameter specifies the IP subnet CIDR block ranges for each subnet and availability zone. +This is specified to allow you to define exactly how big to make each subnet. (We could have automatically +generated these values, but feel it is more comprehensible to view it in the base variables) + +3. Create an AWS user and access credentials + +You will need to create (manually) an AWS user and generate access and secret keys via the AWS console. + +4. Create the PaaS ```sh -git submodule update --init -ENVIRONMENT= AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= -make concourse + export AWS_ACCESS_KEY= + export AWS_SECRET_ACCESS_KEY= + export ENVIRONMENT= + make terraform + make rds + make databases # you will need to wait a bit after the previous step to give RDS time to initialise + make bosh + make runtime_config cloud_config stemcells + make concourse + make cf + make prometheus ``` -Where: +You can specify AWS_PROFILE, rather than the two AWS secrets, for every step except `make bosh`. +The `bosh create-env` command currently does not handle AWS_PROFILE correctly. -- `ENVIRONMENT` - a name for your environment -- `AWS_ACCESS_KEY_ID` - your aws access key id -- `AWS_SECRET_ACCESS_KEY` - your aws secret access key +5. Deploy the Concourse pipelines -You can specify AWS_PROFILE, rather than the two AWS secrets, for every step except for `make concourse`. -The `bosh create-env` command currently does not handle AWS_PROFILE correctly. +```sh + export ENVIRONMENT= + make set_concourse_secrets + make pipelines +``` + +## Connecting to components -## Connecting to Concourse +### Connecting to Concourse The dns name of Concourse is found by: ```sh -terraform output -state=_concourse.tfstate.json concourse_fqdn + bin/outputs.sh -e | jq .concourse_fqdn ``` Go to `https://` to login. @@ -67,58 +112,50 @@ The username is `admin` and you can get the password through: bin/concourse_password.sh -e ``` -or using - -```sh -make concourse_password ENVIRONMENT= -``` -## Testing that Concourse works +### SSHing onto the jumpbox ```sh -ENVIRONMENT= AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= make test_pipeline -fly -t trigger-job -j test/pipeline-test -w +bin/jumpbox_ssh -e ``` -## Installing the deployment pipeline - -The `deploy_pipeline` pipeline will spin up the jump box and BOSH director. +### Logging in to BOSH ```sh -ENVIRONMENT= AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= make deploy_pipeline -fly -t trigger-job -j deploy_pipeline/terraform-jumpbox -w + bin/bosh_credentials.sh -e + # spins up a subshell with a Socks5 proxy connection via jump box to BOSH ``` -If you are deploying from a branch, you should also specify it with the `BRANCH` environment variable, so that the pipeline will trigger correctly. +or, you can run a bosh command: ```sh -BRANCH= ... make deploy_pipeline + bin/bosh_credentials.sh -e bosh vms ``` +### Logging into CF as admin -## Logging in to BOSH - -Once the deployment pipeline has run to completion, you can set up your connection to BOSH easily using: +The CF API URL is `https://api.system..`. ```sh - bin/bosh_credentials.sh -e -f - # spins up a subshell with a Socks5 proxy connection via jump box to BOSH +cf login -a https://api.system.. -u admin -p $(bin/cf_password.sh -e ) ``` -or +### Connecting to Prometheus components ```sh - source bin/bosh_credentials.sh -e - # sets up the Socks5 proxy connection as above, but it's now your job to kill it - # it also sets BOSH_CLIENT, BOSH_CLIENT_SECRET environment variables +bin/prometheus_credentials.sh -e ``` +And use the displayed output to point the browser at the desired Prometheus component. + ## Versions -concourse v4.1.0 -cf v4.0.0 -bosh v1.1.0 -prometheus v23.2.0 +| Component | Version | +| ----------- | ------- | +| concourse | v4.1.0 | +| cf | v4.0.0 | +| bosh | v1.1.0 | +| prometheus | v23.2.0 | ## LICENCE diff --git a/bin/deploy_cf.sh b/bin/deploy_cf.sh index c811f3a0..c1c603c3 100755 --- a/bin/deploy_cf.sh +++ b/bin/deploy_cf.sh @@ -29,7 +29,7 @@ profile="${ENVIRONMENT}" instance_count_file=./profiles/${profile}/instance-count.yml echo "Deploying" -$BOSH deploy -n \ +$BOSH deploy -n -d cf \ ./cf-deployment/cf-deployment.yml \ --vars-file="${instance_count_file}" \ -o cf-deployment/operations/aws.yml \ diff --git a/bin/set_concourse_secrets.sh b/bin/set_concourse_secrets.sh index 0daeea33..1bbbda5c 100755 --- a/bin/set_concourse_secrets.sh +++ b/bin/set_concourse_secrets.sh @@ -12,5 +12,5 @@ done bin/login_fly.sh bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /concourse/main/cf_admin_password --type value --value $(bin/cf_password.sh -e $ENVIRONMENT) -bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /concourse/main/cf_tag --type value --value v4.2.0 +bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /concourse/main/cf_tag --type value --value v4.0.0 bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /concourse/main/prometheus_tag --type value --value v23.2.0 \ No newline at end of file diff --git a/ci/cf.yml b/ci/cf.yml index 74b780c7..2fd4091d 100644 --- a/ci/cf.yml +++ b/ci/cf.yml @@ -51,7 +51,6 @@ jobs: OPS_FILES: instance-counts.yml ops.yml SYSTEM_DOMAIN: system.((domain)) - - name: cats serial: true serial_groups: [cf, cats] diff --git a/terraform/base/bosh.tf b/terraform/base/bosh.tf index 8ad0cd84..6a8a350f 100644 --- a/terraform/base/bosh.tf +++ b/terraform/base/bosh.tf @@ -53,23 +53,6 @@ resource "aws_iam_role_policy" "bosh" { } # S3 -resource "aws_s3_bucket_object" "bosh-var-store" { - bucket = "${aws_s3_bucket.paas_states.id}" - acl = "private" - key = "bosh/bosh-variables.yml" - source = "/dev/null" - server_side_encryption = "aws:kms" - kms_key_id = "${aws_kms_key.paas_state_key.arn}" -} - -resource "aws_s3_bucket_object" "bosh-state" { - bucket = "${aws_s3_bucket.paas_states.id}" - acl = "private" - key = "bosh/bosh-state.json" - content = "{}" - server_side_encryption = "aws:kms" - kms_key_id = "${aws_kms_key.paas_state_key.arn}" -} resource "aws_s3_bucket" "bosh_blobstore" { bucket = "${var.s3_prefix}-${var.environment}-bosh-blobstore" diff --git a/terraform/base/cf.tf b/terraform/base/cf.tf index dc35eab2..b47cba19 100644 --- a/terraform/base/cf.tf +++ b/terraform/base/cf.tf @@ -611,13 +611,3 @@ resource "aws_s3_bucket" "cf_resource_pool" { Environment = "${var.environment}" } } - -# S3 General -resource "aws_s3_bucket_object" "cf-var-store" { - bucket = "${aws_s3_bucket.paas_states.id}" - acl = "private" - key = "cf/cf-variables.yml" - source = "/dev/null" - server_side_encryption = "aws:kms" - kms_key_id = "${aws_kms_key.paas_state_key.arn}" -} From 2c825b6a287e978ffe06731eef9f7bb531a92bfc Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Mon, 10 Sep 2018 13:48:18 +0100 Subject: [PATCH 22/71] Use just AWS DNS server in Concourse containers Bump API workers to 5 Specify repo versions as pipeline parameters, not secrets Remove some cruft --- bin/deploy_cf.sh | 1 + bin/deploy_concourse.sh | 4 +++- bin/set_concourse_secrets.sh | 2 -- ci/cf.yml | 10 ++++---- ci/cf_pipeline.sh | 11 +++++++++ {test => ci}/test_pipeline.yml | 0 operations/cf/api-workers.yml | 8 +++++++ operations/concourse/specific-dns.yml | 8 +++++++ test/deploy_s3_test_pipeline.sh | 26 -------------------- test/deploy_test_pipeline.sh | 20 ---------------- test/s3_test_pipeline.yml | 34 --------------------------- 11 files changed, 37 insertions(+), 87 deletions(-) rename {test => ci}/test_pipeline.yml (100%) create mode 100644 operations/cf/api-workers.yml create mode 100644 operations/concourse/specific-dns.yml delete mode 100755 test/deploy_s3_test_pipeline.sh delete mode 100755 test/deploy_test_pipeline.sh delete mode 100644 test/s3_test_pipeline.yml diff --git a/bin/deploy_cf.sh b/bin/deploy_cf.sh index c1c603c3..43acaa55 100755 --- a/bin/deploy_cf.sh +++ b/bin/deploy_cf.sh @@ -46,6 +46,7 @@ $BOSH deploy -n -d cf \ -o ./operations/cf/rds-access.yml \ -o ./operations/cf/uaa-clients.yml \ -o ./operations/cf/test-user.yml \ + -o ./operations/cf/api-workers.yml \ -v environment="${ENVIRONMENT}" \ -v region="$(jq -r .region < data/$ENVIRONMENT.tfvars)" \ -v system_domain="${SYSTEM_DOMAIN}" \ diff --git a/bin/deploy_concourse.sh b/bin/deploy_concourse.sh index b0db20b1..e635f860 100755 --- a/bin/deploy_concourse.sh +++ b/bin/deploy_concourse.sh @@ -40,6 +40,7 @@ $BOSH -d concourse deploy -n concourse-bosh-deployment/cluster/concourse.yml \ -o ./operations/concourse/tags.yml \ -o ./operations/concourse/local-auth.yml \ -o ./operations/concourse/skip-credhub-tls-validation.yml \ + -o ./operations/concourse/specific-dns.yml \ -v environment=$ENVIRONMENT \ -v external_url=https://$(output base .concourse_fqdn) \ -v network_name=concourse \ @@ -51,4 +52,5 @@ $BOSH -d concourse deploy -n concourse-bosh-deployment/cluster/concourse.yml \ -v credhub_url="$CREDHUB_SERVER" \ -v credhub_client_id="$CREDHUB_CLIENT" \ -v credhub_client_secret="$CREDHUB_SECRET" \ - --var-file credhub_ca_cert="$CREDHUB_CA_CERT" \ No newline at end of file + --var-file credhub_ca_cert="$CREDHUB_CA_CERT" \ + -v private_dns_nameserver="$(output base .vpc_dns_nameserver)" \ No newline at end of file diff --git a/bin/set_concourse_secrets.sh b/bin/set_concourse_secrets.sh index 1bbbda5c..3b4ee135 100755 --- a/bin/set_concourse_secrets.sh +++ b/bin/set_concourse_secrets.sh @@ -12,5 +12,3 @@ done bin/login_fly.sh bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /concourse/main/cf_admin_password --type value --value $(bin/cf_password.sh -e $ENVIRONMENT) -bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /concourse/main/cf_tag --type value --value v4.0.0 -bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /concourse/main/prometheus_tag --type value --value v23.2.0 \ No newline at end of file diff --git a/ci/cf.yml b/ci/cf.yml index 2fd4091d..1bef55d4 100644 --- a/ci/cf.yml +++ b/ci/cf.yml @@ -15,6 +15,7 @@ resources: type: git source: uri: https://github.com/cloudfoundry/cf-acceptance-tests.git + tag_filter: v* - name: cf-deployment-git type: git @@ -31,7 +32,7 @@ resources: jobs: - name: deploy cloud foundry serial: true - serial_groups: [cf, casts] + serial_groups: [casts] plan: - aggregate: - get: paas-bootstrap-git @@ -53,12 +54,13 @@ jobs: - name: cats serial: true - serial_groups: [cf, cats] + serial_groups: [cats] plan: - aggregate: - get: paas-bootstrap-git - get: cf-deployment-concourse-tasks-git - - get: cf-acceptance-tests-git + - get: cf-acceptance-tests-git + version: { ref: ((cats_tag)) } - task: interpolate CATS config file: paas-bootstrap-git/ci/tasks/cf/interpolate_cats/task.yml params: @@ -73,6 +75,6 @@ jobs: cf-acceptance-tests: cf-acceptance-tests-git params: CAPTURE_CONFIG: true - NODES: 6 + NODES: 4 diff --git a/ci/cf_pipeline.sh b/ci/cf_pipeline.sh index 229e813a..306d7204 100755 --- a/ci/cf_pipeline.sh +++ b/ci/cf_pipeline.sh @@ -9,11 +9,22 @@ echo "Using branch '${BRANCH}'" DOMAIN="$(bin/outputs.sh base | jq -r .domain)" +CATS_TAG="v1.6.0" +CF_TAG="v4.0.0" +PROMETHEUS_TAG="v23.2.0" + fly -t "$ENVIRONMENT" set-pipeline \ -v environment="$ENVIRONMENT" \ -v branch="$BRANCH" \ -v domain="$DOMAIN" \ + -v cats_tag="$CATS_TAG" \ + -v cf_tag="$CF_TAG" \ + -v prometheus_tag="$PROMETHEUS_TAG" \ -c ci/cf.yml -p cf -n fly -t "$ENVIRONMENT" unpause-pipeline -p cf fly -t "$ENVIRONMENT" expose-pipeline -p cf + +fly -t "$ENVIRONMENT" check-resource -r cf/cf-acceptance-tests-git --from "ref:${CATS_TAG}" +fly -t "$ENVIRONMENT" check-resource -r cf/cf-deployment-git --from "ref:${CF_TAG}" +fly -t "$ENVIRONMENT" check-resource -r cf/prometheus-boshrelease-git --from "ref:${PROMETHEUS_TAG}" \ No newline at end of file diff --git a/test/test_pipeline.yml b/ci/test_pipeline.yml similarity index 100% rename from test/test_pipeline.yml rename to ci/test_pipeline.yml diff --git a/operations/cf/api-workers.yml b/operations/cf/api-workers.yml new file mode 100644 index 00000000..69921fa6 --- /dev/null +++ b/operations/cf/api-workers.yml @@ -0,0 +1,8 @@ +--- +# Tweak number of API workers. This shonky hack allows us to run CATS on a single API VM. If +# all workers are in play, the route registrar will time out and deregister the API from the +# gorouter routes. This will result in errors like: +# API endpoint not found at 'https://api.' +- type: replace + path: /instance_groups/name=api/jobs/name=cloud_controller_ng/properties/cc/jobs?/local?/number_of_workers? + value: 5 \ No newline at end of file diff --git a/operations/concourse/specific-dns.yml b/operations/concourse/specific-dns.yml new file mode 100644 index 00000000..2244e75e --- /dev/null +++ b/operations/concourse/specific-dns.yml @@ -0,0 +1,8 @@ +--- + # With the latest BOSH they introduced bosh-dns, which pops a bosh-own DNS nameserver + # in front of the system ones, in the VMs and in the container. In our opinion, we + # shouldn't be exposing the BOSH-internal names in Concourse. Actually, the real + # reason is that bosh-dns is *slow* and causes CATS to time out and fail. +- type: replace + path: /instance_groups/name=worker/jobs/name=garden/properties/garden/dns_servers? + value: [ ((private_dns_nameserver)) ] \ No newline at end of file diff --git a/test/deploy_s3_test_pipeline.sh b/test/deploy_s3_test_pipeline.sh deleted file mode 100755 index bd63f88f..00000000 --- a/test/deploy_s3_test_pipeline.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -: $ENVIRONMENT -: ${USERNAME:=admin} -: $CONCOURSE_TERRAFORM_STATE_FILE -: $CONCOURSE_CREDS_FILE - -# Grab pre-requisite files from S3 -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/tfstate.json" "${CONCOURSE_TERRAFORM_STATE_FILE}" -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/creds.yml" "${CONCOURSE_CREDS_FILE}" - -FQDN=$(terraform output -state=${ENVIRONMENT}_concourse.tfstate.json concourse_fqdn) -PASSWORD=$(bosh int --path /admin_password ${ENVIRONMENT}_concourse.creds.yml) -REGION=$(terraform output -state=${ENVIRONMENT}_concourse.tfstate.json region) -S3_KMS_KEY_ID=$(terraform output -state=${ENVIRONMENT}_concourse.tfstate.json s3_kms_key_id) - -fly login -t "$ENVIRONMENT" -u "$USERNAME" -p "$PASSWORD" -c "https://$FQDN" -fly -t "$ENVIRONMENT" set-pipeline \ - -v environment="$ENVIRONMENT" \ - -v region="$REGION" \ - -v s3_kms_key_id="$S3_KMS_KEY_ID" \ - -c test/s3_test_pipeline.yml -p s3test -n -fly -t "$ENVIRONMENT" unpause-pipeline -p s3test - diff --git a/test/deploy_test_pipeline.sh b/test/deploy_test_pipeline.sh deleted file mode 100755 index 61ee0d55..00000000 --- a/test/deploy_test_pipeline.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -: $ENVIRONMENT -: ${USERNAME:=admin} -: $CONCOURSE_TERRAFORM_STATE_FILE -: $CONCOURSE_CREDS_FILE - -# Grab pre-requisite files from S3 -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/tfstate.json" "${CONCOURSE_TERRAFORM_STATE_FILE}" -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/creds.yml" "${CONCOURSE_CREDS_FILE}" - -FQDN=$(terraform output -state=${ENVIRONMENT}_concourse.tfstate.json concourse_fqdn) -PASSWORD=$(bosh int --path /admin_password ${ENVIRONMENT}_concourse.creds.yml) - -fly login -t "$ENVIRONMENT" -u "$USERNAME" -p "$PASSWORD" -c "https://$FQDN" -fly -t "$ENVIRONMENT" set-pipeline -c test/test_pipeline.yml -p test -n -fly -t "$ENVIRONMENT" unpause-pipeline -p test - diff --git a/test/s3_test_pipeline.yml b/test/s3_test_pipeline.yml deleted file mode 100644 index d75fa514..00000000 --- a/test/s3_test_pipeline.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -resources: -# GAH! s3 resource does not handle SSE for downloads -# - name: s3 -# type: s3 -# source: -# bucket: ons-paas-((environment))-states -# # private: true -# region_name: ((region)) -# versioned_file: vpc/vars.tfvars -# server_side_encryption: 'aws:kms' -# sse_kms_key_id: ((s3_kms_key_id)) - -jobs: -- name: pipeline-test - plan: - # - get: s3 - - task: get-s3 - config: - platform: linux - image_resource: - type: docker-image - source: { repository: 'mesosphere/aws-cli', tag: latest } - run: - path: aws - args: - - s3 - - cp - - 's3://ons-paas-((environment))-states/vpc/vars.tfvars' - - foo - - '--sse' - - 'aws:kms' - - '--sse-kms-key-id' - - ((s3_kms_key_id))] \ No newline at end of file From 1e974906bc202d605fb034d318c77bb0442afc88 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Mon, 10 Sep 2018 14:36:17 +0100 Subject: [PATCH 23/71] Attach IAM policy to Concourse worker. Read vars from S3 --- bin/cloud_config.sh | 3 +- bin/deploy_concourse.sh | 1 + ci/cf.yml | 38 +++++++++++++++- ci/tasks/cf/interpolate_variables/task.sh | 13 +++--- ci/tasks/cf/interpolate_variables/task.yml | 3 ++ operations/cloud-config/concourse.yml | 9 +++- operations/concourse/worker.yml | 4 ++ terraform/base/concourse.tf | 43 ++++++++++++++++++- terraform/base/outputs.tf | 4 ++ .../base/templates/concourse_iam_policy.json | 32 ++++++++++++++ 10 files changed, 137 insertions(+), 13 deletions(-) create mode 100644 operations/concourse/worker.yml create mode 100644 terraform/base/templates/concourse_iam_policy.json diff --git a/bin/cloud_config.sh b/bin/cloud_config.sh index 1ca6f0a5..4128ccc0 100755 --- a/bin/cloud_config.sh +++ b/bin/cloud_config.sh @@ -60,4 +60,5 @@ $BOSH update-cloud-config -n \ -v reserved_concourse_az1_cidr="$(output base .concourse_subnet_reserved_cidr_blocks[0])" \ -v concourse_subnet_az1_id="$(output base .concourse_subnet_ids[0])" \ -v concourse_security_group="$(output base .concourse_security_group_id)" \ - -v concourse_alb_target_group="$(output base .concourse_alb_target_group_name)" \ No newline at end of file + -v concourse_alb_target_group="$(output base .concourse_alb_target_group_name)" \ + -v concourse_worker_iam_profile="$(output base .concourse_iam_instance_profile)" \ No newline at end of file diff --git a/bin/deploy_concourse.sh b/bin/deploy_concourse.sh index e635f860..71fb0fbb 100755 --- a/bin/deploy_concourse.sh +++ b/bin/deploy_concourse.sh @@ -41,6 +41,7 @@ $BOSH -d concourse deploy -n concourse-bosh-deployment/cluster/concourse.yml \ -o ./operations/concourse/local-auth.yml \ -o ./operations/concourse/skip-credhub-tls-validation.yml \ -o ./operations/concourse/specific-dns.yml \ + -o ./operations/concourse/worker.yml \ -v environment=$ENVIRONMENT \ -v external_url=https://$(output base .concourse_fqdn) \ -v network_name=concourse \ diff --git a/ci/cf.yml b/ci/cf.yml index 1bef55d4..03295854 100644 --- a/ci/cf.yml +++ b/ci/cf.yml @@ -29,10 +29,40 @@ resources: uri: https://github.com/bosh-prometheus/prometheus-boshrelease.git tag_filter: v* +- name: states-variables-s3 + type: s3-iam + source: + bucket: ons-paas-((environment))-states + private: true + region_name: ((region)) + versioned_file: ((environment)).tfvars + server_side_encryption: 'aws:kms' + sse_kms_key_id: ((s3_kms_key_id)) + +- name: base-tfstate-s3 + type: s3-iam + source: + bucket: ons-paas-((environment))-states + private: true + region_name: ((region)) + versioned_file: ((environment))-base.tfstate + server_side_encryption: 'aws:kms' + sse_kms_key_id: ((s3_kms_key_id)) + +- name: rds-tfstate-s3 + type: s3-iam + source: + bucket: ons-paas-((environment))-states + private: true + region_name: ((region)) + versioned_file: ((environment))-rds.tfstate + server_side_encryption: 'aws:kms' + sse_kms_key_id: ((s3_kms_key_id)) + jobs: - name: deploy cloud foundry serial: true - serial_groups: [casts] + serial_groups: [cats] plan: - aggregate: - get: paas-bootstrap-git @@ -41,8 +71,11 @@ jobs: version: { ref: ((cf_tag)) } - get: prometheus-boshrelease-git version: { ref: ((prometheus_tag)) } + - get: states-variables-s3 + - get: base-tfstate-s3 + - get: rds-tfstate-s3 - task: interpolate cf deployment variables and ops files - file: paas-bootstrap-git/ci/tasks/cf/interpolate-variables/task.yml + file: paas-bootstrap-git/ci/tasks/cf/interpolate_variables/task.yml - task: deploy cf file: cf-deployment-concourse-tasks-git/bosh-deploy/task.yml input_mapping: @@ -58,6 +91,7 @@ jobs: plan: - aggregate: - get: paas-bootstrap-git + passed: ["deploy cloud foundry"] - get: cf-deployment-concourse-tasks-git - get: cf-acceptance-tests-git version: { ref: ((cats_tag)) } diff --git a/ci/tasks/cf/interpolate_variables/task.sh b/ci/tasks/cf/interpolate_variables/task.sh index 42c4ea24..9c03646e 100755 --- a/ci/tasks/cf/interpolate_variables/task.sh +++ b/ci/tasks/cf/interpolate_variables/task.sh @@ -8,13 +8,10 @@ output() { FILE=$1 QUERY=$2 - cd paas-bootstrap-git - bin/outputs.sh $FILE | jq -r "$QUERY" - cd - + FILE="${FILE}-states-s3/${ENVIRONMENT}-${FILE}.tfstate" + jq -r ". | with_entries(.value = .value.value) | $QUERY" } -BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" - DOMAIN="$(output base .domain)" SYSTEM_DOMAIN="system.${DOMAIN}" APPS_DOMAIN="apps.${DOMAIN}" @@ -24,9 +21,9 @@ CF_DB_USERNAME="$(output rds .cf_db_username)" CF_DB_PASSWORD="$(output rds .cf_rds_password)" profile="${ENVIRONMENT}" -[ -d "profiles/${ENVIRONMENT}" ] || profile="engineering" +[ -d "paas-bootstrap-git/profiles/${ENVIRONMENT}" ] || profile="engineering" -instance_count_file=./profiles/${profile}/instance-count.yml +instance_count_file=paas-bootstrap-git/profiles/${profile}/instance-count.yml echo "Constructing consolidated ops file" cat \ @@ -51,7 +48,7 @@ cp "${instance_count_file}" vars-files/instance-counts.yml cat >vars-files/variables.yml <<-EOS environment: ${ENVIRONMENT} -region: $(jq -r .region < data/$ENVIRONMENT.tfvars) +region: $(jq -r .region < state-variables-s3/$ENVIRONMENT.tfvars) system_domain:${SYSTEM_DOMAIN} app_domains: [${APPS_DOMAIN}] smoke_test_app_domain: ${APPS_DOMAIN} diff --git a/ci/tasks/cf/interpolate_variables/task.yml b/ci/tasks/cf/interpolate_variables/task.yml index e4711736..35aa3852 100644 --- a/ci/tasks/cf/interpolate_variables/task.yml +++ b/ci/tasks/cf/interpolate_variables/task.yml @@ -8,6 +8,9 @@ inputs: - name: paas-bootstrap-git - name: cf-deployment-git - name: prometheus-boshrelease-git + - name: states-variables-s3 + - name: base-tfstate-s3 + - name: rds-tfstate-s3 outputs: - name: vars-files diff --git a/operations/cloud-config/concourse.yml b/operations/cloud-config/concourse.yml index f39451f0..f4fa6975 100644 --- a/operations/cloud-config/concourse.yml +++ b/operations/cloud-config/concourse.yml @@ -32,4 +32,11 @@ value: name: concourse_web cloud_properties: - lb_target_groups: [((concourse_alb_target_group))] \ No newline at end of file + lb_target_groups: [((concourse_alb_target_group))] + +- type: replace + path: /vm_extensions/- + value: + name: concourse-worker + cloud_properties: + iam_instance_profile: ((concourse_worker_iam_profile)) \ No newline at end of file diff --git a/operations/concourse/worker.yml b/operations/concourse/worker.yml new file mode 100644 index 00000000..171c6026 --- /dev/null +++ b/operations/concourse/worker.yml @@ -0,0 +1,4 @@ +--- +- type: replace + path: /instance_groups/name=worker/vm_extensions?/- + value: concourse-worker diff --git a/terraform/base/concourse.tf b/terraform/base/concourse.tf index f2bd35c4..00d446c0 100644 --- a/terraform/base/concourse.tf +++ b/terraform/base/concourse.tf @@ -166,7 +166,6 @@ resource "aws_security_group_rule" "concourse_alb_to_web" { source_security_group_id = "${aws_security_group.concourse.id}" } - # IP resource "aws_eip" "concourse" { vpc = true @@ -252,3 +251,45 @@ resource "aws_route53_record" "concourse" { records = ["${aws_lb.concourse.dns_name}"] } + +data "template_file" "concourse_iam_policy" { + template = "${file("${path.module}/templates/concourse_iam_policy.json")}" + + vars { + s3_kms_key_arn = "${aws_kms_key.paas_state_key.arn}" + environment = "${var.environment}" + } +} + +resource "aws_iam_instance_profile" "concourse" { + name = "${var.environment}_concourse_profile" + role = "${aws_iam_role.concourse.name}" +} + +resource "aws_iam_role" "concourse" { + name = "${var.environment}_concourse_role" + path = "/" + + assume_role_policy = < Date: Mon, 10 Sep 2018 15:13:57 +0100 Subject: [PATCH 24/71] Fix hanging jq Add S3 and KMS rules to concourse profile --- ci/cf.yml | 6 ++++++ ci/cf_pipeline.sh | 4 ++++ ci/tasks/cf/interpolate_variables/task.sh | 2 +- terraform/base/concourse.tf | 2 ++ terraform/base/outputs.tf | 9 +++++++++ .../base/templates/concourse_iam_policy.json | 15 ++++++++++++++- 6 files changed, 36 insertions(+), 2 deletions(-) diff --git a/ci/cf.yml b/ci/cf.yml index 03295854..8bc3bd19 100644 --- a/ci/cf.yml +++ b/ci/cf.yml @@ -1,4 +1,10 @@ --- +resource_types: +- name: s3-iam + type: docker-image + source: + repository: governmentpaas/s3-resource + resources: - name: paas-bootstrap-git type: git diff --git a/ci/cf_pipeline.sh b/ci/cf_pipeline.sh index 306d7204..768bc10e 100755 --- a/ci/cf_pipeline.sh +++ b/ci/cf_pipeline.sh @@ -8,6 +8,8 @@ set -euo pipefail echo "Using branch '${BRANCH}'" DOMAIN="$(bin/outputs.sh base | jq -r .domain)" +REGION="$(bin/outputs.sh base | jq -r .region)" +KMS_KEY_ID="$(bin/outputs.sh base | jq -r .states_s3_kms_key_id)" CATS_TAG="v1.6.0" CF_TAG="v4.0.0" @@ -15,11 +17,13 @@ PROMETHEUS_TAG="v23.2.0" fly -t "$ENVIRONMENT" set-pipeline \ -v environment="$ENVIRONMENT" \ + -v region="$REGION" \ -v branch="$BRANCH" \ -v domain="$DOMAIN" \ -v cats_tag="$CATS_TAG" \ -v cf_tag="$CF_TAG" \ -v prometheus_tag="$PROMETHEUS_TAG" \ + -v s3_kms_key_id="$KMS_KEY_ID" \ -c ci/cf.yml -p cf -n fly -t "$ENVIRONMENT" unpause-pipeline -p cf diff --git a/ci/tasks/cf/interpolate_variables/task.sh b/ci/tasks/cf/interpolate_variables/task.sh index 9c03646e..fcca3a6a 100755 --- a/ci/tasks/cf/interpolate_variables/task.sh +++ b/ci/tasks/cf/interpolate_variables/task.sh @@ -9,7 +9,7 @@ output() { QUERY=$2 FILE="${FILE}-states-s3/${ENVIRONMENT}-${FILE}.tfstate" - jq -r ". | with_entries(.value = .value.value) | $QUERY" + jq -r ". | with_entries(.value = .value.value) | $QUERY" <"$FILE" } DOMAIN="$(output base .domain)" diff --git a/terraform/base/concourse.tf b/terraform/base/concourse.tf index 00d446c0..b704d6b5 100644 --- a/terraform/base/concourse.tf +++ b/terraform/base/concourse.tf @@ -258,6 +258,8 @@ data "template_file" "concourse_iam_policy" { vars { s3_kms_key_arn = "${aws_kms_key.paas_state_key.arn}" environment = "${var.environment}" + region = "${var.region}" + account_id = "${local.account_id}" } } diff --git a/terraform/base/outputs.tf b/terraform/base/outputs.tf index 182d6afd..d9865618 100644 --- a/terraform/base/outputs.tf +++ b/terraform/base/outputs.tf @@ -2,6 +2,15 @@ output "vpc_dns_nameserver" { value = "${cidrhost(aws_vpc.default.cidr_block, 2)}" } +output "region" { + value = "${var.region}" +} + + +output "states_s3_kms_key_id" { + value = "${aws_kms_key.paas_state_key.id}" +} + output "domain" { value = "${local.domain}" } diff --git a/terraform/base/templates/concourse_iam_policy.json b/terraform/base/templates/concourse_iam_policy.json index 69c2fe3c..a0efcd6e 100644 --- a/terraform/base/templates/concourse_iam_policy.json +++ b/terraform/base/templates/concourse_iam_policy.json @@ -18,6 +18,7 @@ { "Effect": "Allow", "Action": [ + "kms:Encrypt", "kms:Decrypt", "kms:DescribeKey", "kms:GetKeyPolicy", @@ -25,7 +26,19 @@ "kms:ListResourceTags" ], "Resource": [ - "${s3_kms_key_arn}" + "${s3_kms_key_arn}", + "arn:aws:kms:${region}:${account_id}:*", + "arn:aws:kms:${region}:${account_id}:key/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "kms:CreateKey", + "kms:GenerateDataKey" + ], + "Resource": [ + "*" ] } ] From 1785669e18249a1bb01cc46c924a40eaa6800b65 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Mon, 10 Sep 2018 15:18:09 +0100 Subject: [PATCH 25/71] Fix state path names --- ci/cf.yml | 2 ++ ci/tasks/cf/interpolate_variables/task.sh | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/cf.yml b/ci/cf.yml index 8bc3bd19..380788f2 100644 --- a/ci/cf.yml +++ b/ci/cf.yml @@ -82,6 +82,8 @@ jobs: - get: rds-tfstate-s3 - task: interpolate cf deployment variables and ops files file: paas-bootstrap-git/ci/tasks/cf/interpolate_variables/task.yml + params: + ENVIRONMENT: ((environment)) - task: deploy cf file: cf-deployment-concourse-tasks-git/bosh-deploy/task.yml input_mapping: diff --git a/ci/tasks/cf/interpolate_variables/task.sh b/ci/tasks/cf/interpolate_variables/task.sh index fcca3a6a..d29da22d 100755 --- a/ci/tasks/cf/interpolate_variables/task.sh +++ b/ci/tasks/cf/interpolate_variables/task.sh @@ -8,7 +8,7 @@ output() { FILE=$1 QUERY=$2 - FILE="${FILE}-states-s3/${ENVIRONMENT}-${FILE}.tfstate" + FILE="${FILE}-tfstate-s3/${ENVIRONMENT}-${FILE}.tfstate" jq -r ". | with_entries(.value = .value.value) | $QUERY" <"$FILE" } @@ -48,7 +48,7 @@ cp "${instance_count_file}" vars-files/instance-counts.yml cat >vars-files/variables.yml <<-EOS environment: ${ENVIRONMENT} -region: $(jq -r .region < state-variables-s3/$ENVIRONMENT.tfvars) +region: $(jq -r .region < states-variables-s3/$ENVIRONMENT.tfvars) system_domain:${SYSTEM_DOMAIN} app_domains: [${APPS_DOMAIN}] smoke_test_app_domain: ${APPS_DOMAIN} From f7a5f6061d3693f7eb07de14254967b79bc608c4 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Mon, 10 Sep 2018 15:21:15 +0100 Subject: [PATCH 26/71] Jq outputs --- ci/tasks/cf/interpolate_variables/task.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/tasks/cf/interpolate_variables/task.sh b/ci/tasks/cf/interpolate_variables/task.sh index d29da22d..f7ec3d2e 100755 --- a/ci/tasks/cf/interpolate_variables/task.sh +++ b/ci/tasks/cf/interpolate_variables/task.sh @@ -9,7 +9,7 @@ output() { QUERY=$2 FILE="${FILE}-tfstate-s3/${ENVIRONMENT}-${FILE}.tfstate" - jq -r ". | with_entries(.value = .value.value) | $QUERY" <"$FILE" + jq -r ".mnodules[0].outputs | with_entries(.value = .value.value) | $QUERY" <"$FILE" } DOMAIN="$(output base .domain)" From 5c7fde305073c69280c9c8700c93b97d45847e75 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Mon, 10 Sep 2018 15:23:21 +0100 Subject: [PATCH 27/71] Typos typos typos --- ci/tasks/cf/interpolate_variables/task.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/tasks/cf/interpolate_variables/task.sh b/ci/tasks/cf/interpolate_variables/task.sh index f7ec3d2e..cf3415a4 100755 --- a/ci/tasks/cf/interpolate_variables/task.sh +++ b/ci/tasks/cf/interpolate_variables/task.sh @@ -9,7 +9,7 @@ output() { QUERY=$2 FILE="${FILE}-tfstate-s3/${ENVIRONMENT}-${FILE}.tfstate" - jq -r ".mnodules[0].outputs | with_entries(.value = .value.value) | $QUERY" <"$FILE" + jq -r ".modules[0].outputs | with_entries(.value = .value.value) | $QUERY" <"$FILE" } DOMAIN="$(output base .domain)" From 9ab2f9c084b3a8acd62a5475f61517e0a51fb29e Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Mon, 10 Sep 2018 15:24:33 +0100 Subject: [PATCH 28/71] More typos --- ci/tasks/cf/interpolate_variables/task.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/tasks/cf/interpolate_variables/task.sh b/ci/tasks/cf/interpolate_variables/task.sh index cf3415a4..bd389f0c 100755 --- a/ci/tasks/cf/interpolate_variables/task.sh +++ b/ci/tasks/cf/interpolate_variables/task.sh @@ -31,7 +31,7 @@ cat \ cf-deployment-git/operations/override-app-domains.yml \ cf-deployment-git/operations/use-external-blobstore.yml \ cf-deployment-git/operations/use-external-dbs.yml \ - prometheus-boshrelease-it/manifests/operators/cf/add-prometheus-uaa-clients.yml \ + prometheus-boshrelease-git/manifests/operators/cf/add-prometheus-uaa-clients.yml \ paas-bootstrap-git/operations/bosh/tags.yml \ paas-bootstrap-git/operations/cf/router-sec-group.yml \ paas-bootstrap-git/operations/cf/scheduler.yml \ From 1bc3b7a1831a87aa6076f44420e03c5d4493e439 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Mon, 10 Sep 2018 15:26:38 +0100 Subject: [PATCH 29/71] Add bbl-state --- ci/tasks/cf/interpolate_variables/task.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/tasks/cf/interpolate_variables/task.yml b/ci/tasks/cf/interpolate_variables/task.yml index 35aa3852..4b365c95 100644 --- a/ci/tasks/cf/interpolate_variables/task.yml +++ b/ci/tasks/cf/interpolate_variables/task.yml @@ -15,6 +15,7 @@ inputs: outputs: - name: vars-files - name: ops-files + - name: bbl-state run: path: paas-bootstrap-git/ci/tasks/cf/interpolate_variables/task.sh From 2635f75f1576c572ee281ce627990214e8652633 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Mon, 10 Sep 2018 15:31:14 +0100 Subject: [PATCH 30/71] Create dummy bbl-state.json --- ci/tasks/cf/interpolate_variables/task.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/tasks/cf/interpolate_variables/task.sh b/ci/tasks/cf/interpolate_variables/task.sh index bd389f0c..ec75bd85 100755 --- a/ci/tasks/cf/interpolate_variables/task.sh +++ b/ci/tasks/cf/interpolate_variables/task.sh @@ -91,4 +91,8 @@ external_credhub_database_name: credhub external_credhub_database_address: ${CF_DB_ENDPOINT} external_credhub_database_password: ${CF_DB_PASSWORD} external_credhub_database_username: ${CF_DB_USERNAME} -EOS \ No newline at end of file +EOS + +echo "Creating dummy bbl-state.json" +mkdir bbl-state/bbl-state +echo "{}" >bbl-state/bbl-state/bbl-state.json \ No newline at end of file From 129721bdd1603c0bbee8dcb0944d2fa290f7356c Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Mon, 10 Sep 2018 15:43:26 +0100 Subject: [PATCH 31/71] Revert to deploying in our own way --- bin/bosh_password.sh | 14 ++++++++++++++ bin/set_concourse_secrets.sh | 1 + ci/cf.yml | 13 +++---------- .../cf/{interpolate_variables => deploy}/task.sh | 11 ++++++++--- .../cf/{interpolate_variables => deploy}/task.yml | 4 +++- 5 files changed, 29 insertions(+), 14 deletions(-) create mode 100755 bin/bosh_password.sh rename ci/tasks/cf/{interpolate_variables => deploy}/task.sh (95%) rename ci/tasks/cf/{interpolate_variables => deploy}/task.yml (81%) diff --git a/bin/bosh_password.sh b/bin/bosh_password.sh new file mode 100755 index 00000000..85f3dd32 --- /dev/null +++ b/bin/bosh_password.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# +# Returns the current BOSH password + +while getopts 'e:' option; do + case $option in + e) export ENVIRONMENT="$OPTARG";; + esac +done + +: $ENVIRONMENT + +bin/get_states.sh -e $ENVIRONMENT -x -f $ENVIRONMENT-bosh-variables.yml +bosh int --path /admin_password "data/$ENVIRONMENT-bosh-variables.yml" | head -1 diff --git a/bin/set_concourse_secrets.sh b/bin/set_concourse_secrets.sh index 3b4ee135..59577bc5 100755 --- a/bin/set_concourse_secrets.sh +++ b/bin/set_concourse_secrets.sh @@ -12,3 +12,4 @@ done bin/login_fly.sh bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /concourse/main/cf_admin_password --type value --value $(bin/cf_password.sh -e $ENVIRONMENT) +bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /concourse/main/bosh_admin_password --type value --value $(bin/bosh_password.sh -e $ENVIRONMENT) diff --git a/ci/cf.yml b/ci/cf.yml index 380788f2..6f019961 100644 --- a/ci/cf.yml +++ b/ci/cf.yml @@ -80,18 +80,11 @@ jobs: - get: states-variables-s3 - get: base-tfstate-s3 - get: rds-tfstate-s3 - - task: interpolate cf deployment variables and ops files - file: paas-bootstrap-git/ci/tasks/cf/interpolate_variables/task.yml + - task: interpolate and deploy + file: paas-bootstrap-git/ci/tasks/cf/deploy/task.yml params: ENVIRONMENT: ((environment)) - - task: deploy cf - file: cf-deployment-concourse-tasks-git/bosh-deploy/task.yml - input_mapping: - cf-deployment-concourse-tasks: cf-deployment-concourse-tasks-git - cf-deployment: cf-deployment-git - params: - OPS_FILES: instance-counts.yml ops.yml - SYSTEM_DOMAIN: system.((domain)) + BOSH_CLIENT_SECRET: ((bosh_admin_password)) - name: cats serial: true diff --git a/ci/tasks/cf/interpolate_variables/task.sh b/ci/tasks/cf/deploy/task.sh similarity index 95% rename from ci/tasks/cf/interpolate_variables/task.sh rename to ci/tasks/cf/deploy/task.sh index ec75bd85..c8980c0f 100755 --- a/ci/tasks/cf/interpolate_variables/task.sh +++ b/ci/tasks/cf/deploy/task.sh @@ -93,6 +93,11 @@ external_credhub_database_password: ${CF_DB_PASSWORD} external_credhub_database_username: ${CF_DB_USERNAME} EOS -echo "Creating dummy bbl-state.json" -mkdir bbl-state/bbl-state -echo "{}" >bbl-state/bbl-state/bbl-state.json \ No newline at end of file +echo "Deploying" + +export BOSH_ENVIRONMENT=$(output base .bosh_private_ip) + +bosh deploy -d cf \ + -o ops-files/ops.yml \ + -l vars-files/instance-counts.yml \ + -l vars-files/variables.yml \ No newline at end of file diff --git a/ci/tasks/cf/interpolate_variables/task.yml b/ci/tasks/cf/deploy/task.yml similarity index 81% rename from ci/tasks/cf/interpolate_variables/task.yml rename to ci/tasks/cf/deploy/task.yml index 4b365c95..174fcfc7 100644 --- a/ci/tasks/cf/interpolate_variables/task.yml +++ b/ci/tasks/cf/deploy/task.yml @@ -18,8 +18,10 @@ outputs: - name: bbl-state run: - path: paas-bootstrap-git/ci/tasks/cf/interpolate_variables/task.sh + path: paas-bootstrap-git/ci/tasks/cf/deploy/task.sh params: DOMAIN: ENVIRONMENT: + BOSH_CLIENT: admin + BOSH_CLIENT_SECRET: From a722971995a3908c29a4e587d2700026ca8b04b6 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Mon, 10 Sep 2018 15:46:44 +0100 Subject: [PATCH 32/71] Space needed when yml used --- ci/tasks/cf/deploy/task.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/tasks/cf/deploy/task.sh b/ci/tasks/cf/deploy/task.sh index c8980c0f..e46b6691 100755 --- a/ci/tasks/cf/deploy/task.sh +++ b/ci/tasks/cf/deploy/task.sh @@ -49,7 +49,7 @@ cp "${instance_count_file}" vars-files/instance-counts.yml cat >vars-files/variables.yml <<-EOS environment: ${ENVIRONMENT} region: $(jq -r .region < states-variables-s3/$ENVIRONMENT.tfvars) -system_domain:${SYSTEM_DOMAIN} +system_domain: ${SYSTEM_DOMAIN} app_domains: [${APPS_DOMAIN}] smoke_test_app_domain: ${APPS_DOMAIN} buildpack_directory_key: $(output base .cf_buildpacks_bucket_name) From 745878d6434702d4c7dde6552ecfd02ccbbe2e03 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Mon, 10 Sep 2018 15:51:55 +0100 Subject: [PATCH 33/71] Specify ops files in bosh deploy --- ci/tasks/cf/deploy/task.sh | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/ci/tasks/cf/deploy/task.sh b/ci/tasks/cf/deploy/task.sh index e46b6691..c36321b9 100755 --- a/ci/tasks/cf/deploy/task.sh +++ b/ci/tasks/cf/deploy/task.sh @@ -25,24 +25,6 @@ profile="${ENVIRONMENT}" instance_count_file=paas-bootstrap-git/profiles/${profile}/instance-count.yml -echo "Constructing consolidated ops file" -cat \ - cf-deployment-git/operations/aws.yml \ - cf-deployment-git/operations/override-app-domains.yml \ - cf-deployment-git/operations/use-external-blobstore.yml \ - cf-deployment-git/operations/use-external-dbs.yml \ - prometheus-boshrelease-git/manifests/operators/cf/add-prometheus-uaa-clients.yml \ - paas-bootstrap-git/operations/bosh/tags.yml \ - paas-bootstrap-git/operations/cf/router-sec-group.yml \ - paas-bootstrap-git/operations/cf/scheduler.yml \ - paas-bootstrap-git/operations/cf/s3_blobstore_with_kms_and_iam.yml \ - paas-bootstrap-git/operations/cf/azs.yml \ - paas-bootstrap-git/operations/cf/instance-counts.yml \ - paas-bootstrap-git/operations/cf/rds-access.yml \ - paas-bootstrap-git/operations/cf/uaa-clients.yml \ - paas-bootstrap-git/operations/cf/test-user.yml \ - > ops-files/ops.yml - echo "Constructing variables" cp "${instance_count_file}" vars-files/instance-counts.yml @@ -98,6 +80,20 @@ echo "Deploying" export BOSH_ENVIRONMENT=$(output base .bosh_private_ip) bosh deploy -d cf \ - -o ops-files/ops.yml \ + -o cf-deployment-git/operations/aws.yml \ + -o cf-deployment-git/operations/override-app-domains.yml \ + -o cf-deployment-git/operations/use-external-blobstore.yml \ + -o cf-deployment-git/operations/use-external-dbs.yml \ + -o prometheus-boshrelease-git/manifests/operators/cf/add-prometheus-uaa-clients.yml \ + -o paas-bootstrap-git/operations/bosh/tags.yml \ + -o paas-bootstrap-git/operations/cf/router-sec-group.yml \ + -o paas-bootstrap-git/operations/cf/scheduler.yml \ + -o paas-bootstrap-git/operations/cf/s3_blobstore_with_kms_and_iam.yml \ + -o paas-bootstrap-git/operations/cf/azs.yml \ + -o paas-bootstrap-git/operations/cf/instance-counts.yml \ + -o paas-bootstrap-git/operations/cf/rds-access.yml \ + -o paas-bootstrap-git/operations/cf/uaa-clients.yml \ + -o paas-bootstrap-git/operations/cf/test-user.yml \ + -o paas-bootstrap-git/operations/cf/api-workers.yml \ -l vars-files/instance-counts.yml \ -l vars-files/variables.yml \ No newline at end of file From 9ab3b043c015fbb144d08474649862e97e0d390a Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Mon, 10 Sep 2018 15:54:07 +0100 Subject: [PATCH 34/71] Add cf-deployment yml --- ci/tasks/cf/deploy/task.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/tasks/cf/deploy/task.sh b/ci/tasks/cf/deploy/task.sh index c36321b9..bfadc8b1 100755 --- a/ci/tasks/cf/deploy/task.sh +++ b/ci/tasks/cf/deploy/task.sh @@ -79,7 +79,7 @@ echo "Deploying" export BOSH_ENVIRONMENT=$(output base .bosh_private_ip) -bosh deploy -d cf \ +bosh deploy -d cf cf-deployment-git/cf-deployment.yml \ -o cf-deployment-git/operations/aws.yml \ -o cf-deployment-git/operations/override-app-domains.yml \ -o cf-deployment-git/operations/use-external-blobstore.yml \ From 85a734a8ef1e44ea1fa555eb21b963db1499fb0e Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Mon, 10 Sep 2018 16:00:51 +0100 Subject: [PATCH 35/71] Define BOSH_CA_CERT --- bin/bosh_ca_cert.sh | 14 ++++++++++++++ ci/cf.yml | 1 + ci/cf_pipeline.sh | 1 + ci/tasks/cf/deploy/task.yml | 1 + 4 files changed, 17 insertions(+) create mode 100755 bin/bosh_ca_cert.sh diff --git a/bin/bosh_ca_cert.sh b/bin/bosh_ca_cert.sh new file mode 100755 index 00000000..2a25c151 --- /dev/null +++ b/bin/bosh_ca_cert.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# +# Returns the current BOSH password + +while getopts 'e:' option; do + case $option in + e) export ENVIRONMENT="$OPTARG";; + esac +done + +: $ENVIRONMENT + +bin/get_states.sh -e $ENVIRONMENT -x -f $ENVIRONMENT-bosh-variables.yml +bosh int --path /default_ca/ca "data/$ENVIRONMENT-bosh-variables.yml" diff --git a/ci/cf.yml b/ci/cf.yml index 6f019961..daa77288 100644 --- a/ci/cf.yml +++ b/ci/cf.yml @@ -85,6 +85,7 @@ jobs: params: ENVIRONMENT: ((environment)) BOSH_CLIENT_SECRET: ((bosh_admin_password)) + BOSH_CA_CERT: ((bosh_ca_cert)) - name: cats serial: true diff --git a/ci/cf_pipeline.sh b/ci/cf_pipeline.sh index 768bc10e..1db4936c 100755 --- a/ci/cf_pipeline.sh +++ b/ci/cf_pipeline.sh @@ -24,6 +24,7 @@ fly -t "$ENVIRONMENT" set-pipeline \ -v cf_tag="$CF_TAG" \ -v prometheus_tag="$PROMETHEUS_TAG" \ -v s3_kms_key_id="$KMS_KEY_ID" \ + -v bosh_ca_cert="$(bin/bosh_ca_cert.sh -e $ENVIRONMENT)" \ -c ci/cf.yml -p cf -n fly -t "$ENVIRONMENT" unpause-pipeline -p cf diff --git a/ci/tasks/cf/deploy/task.yml b/ci/tasks/cf/deploy/task.yml index 174fcfc7..20f2a8b1 100644 --- a/ci/tasks/cf/deploy/task.yml +++ b/ci/tasks/cf/deploy/task.yml @@ -25,3 +25,4 @@ params: ENVIRONMENT: BOSH_CLIENT: admin BOSH_CLIENT_SECRET: + BOSH_CA_CERT: From 61f8361d0f7c6ac513bc68012818fbc7f9f70daf Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Mon, 10 Sep 2018 16:02:09 +0100 Subject: [PATCH 36/71] Non-interactive deploy --- ci/tasks/cf/deploy/task.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/tasks/cf/deploy/task.sh b/ci/tasks/cf/deploy/task.sh index bfadc8b1..79f42851 100755 --- a/ci/tasks/cf/deploy/task.sh +++ b/ci/tasks/cf/deploy/task.sh @@ -79,7 +79,7 @@ echo "Deploying" export BOSH_ENVIRONMENT=$(output base .bosh_private_ip) -bosh deploy -d cf cf-deployment-git/cf-deployment.yml \ +bosh deploy -n -d cf cf-deployment-git/cf-deployment.yml \ -o cf-deployment-git/operations/aws.yml \ -o cf-deployment-git/operations/override-app-domains.yml \ -o cf-deployment-git/operations/use-external-blobstore.yml \ From 5e917c27bff5eb8cdb7c31fa0cbee8e58762be7e Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Mon, 10 Sep 2018 16:13:34 +0100 Subject: [PATCH 37/71] Add bucket policy --- terraform/base/templates/concourse_iam_policy.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/terraform/base/templates/concourse_iam_policy.json b/terraform/base/templates/concourse_iam_policy.json index a0efcd6e..8baa2aba 100644 --- a/terraform/base/templates/concourse_iam_policy.json +++ b/terraform/base/templates/concourse_iam_policy.json @@ -12,7 +12,8 @@ "s3:GetObjectTagging" ], "Resource": [ - "arn:aws:s3:::ons-paas-${environment}-states/*" + "arn:aws:s3:::ons-paas-${environment}-states/*", + "arn:aws:s3:::ons-paas-${environment}-states" ] }, { From 808797a14190bc790b697dbcce837fe7dc9a95d2 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Tue, 11 Sep 2018 08:52:03 +0100 Subject: [PATCH 38/71] Add initialise step and stemcells upload to CI --- ci/cf.yml | 27 +++++++++++++++++++++++++++ ci/cf_pipeline.sh | 2 +- ci/tasks/cf/stemcells/task.sh | 21 +++++++++++++++++++++ ci/tasks/cf/stemcells/task.yml | 20 ++++++++++++++++++++ 4 files changed, 69 insertions(+), 1 deletion(-) create mode 100755 ci/tasks/cf/stemcells/task.sh create mode 100644 ci/tasks/cf/stemcells/task.yml diff --git a/ci/cf.yml b/ci/cf.yml index daa77288..73de35d1 100644 --- a/ci/cf.yml +++ b/ci/cf.yml @@ -66,12 +66,38 @@ resources: sse_kms_key_id: ((s3_kms_key_id)) jobs: +- name: initialise + serial: true + serial_groups: [cats] + plan: + - get: paas-bootstrap-git + trigger: true + +- name: upload stemcells + serial: true + serial_groups: [cats] + plan: + - aggregate: + - get: paas-bootstrap-git + passed: [initialise] + - get: cf-deployment-git + version: { ref: ((cf_tag)) } + - get: base-tfstate-s3 + - task: upload stemcells + file: paas-bootstrap-git/ci/tasks/cf/stemcells/task.yml + params: + ENVIRONMENT: ((environment)) + BOSH_CLIENT_SECRET: ((bosh_admin_password)) + BOSH_CA_CERT: ((bosh_ca_cert)) + - name: deploy cloud foundry serial: true serial_groups: [cats] plan: - aggregate: - get: paas-bootstrap-git + passed: ["upload stemcells"] + trigger: true - get: cf-deployment-concourse-tasks-git - get: cf-deployment-git version: { ref: ((cf_tag)) } @@ -94,6 +120,7 @@ jobs: - aggregate: - get: paas-bootstrap-git passed: ["deploy cloud foundry"] + trigger: true - get: cf-deployment-concourse-tasks-git - get: cf-acceptance-tests-git version: { ref: ((cats_tag)) } diff --git a/ci/cf_pipeline.sh b/ci/cf_pipeline.sh index 1db4936c..9fc0d5e0 100755 --- a/ci/cf_pipeline.sh +++ b/ci/cf_pipeline.sh @@ -12,7 +12,7 @@ REGION="$(bin/outputs.sh base | jq -r .region)" KMS_KEY_ID="$(bin/outputs.sh base | jq -r .states_s3_kms_key_id)" CATS_TAG="v1.6.0" -CF_TAG="v4.0.0" +CF_TAG="v4.2.0" PROMETHEUS_TAG="v23.2.0" fly -t "$ENVIRONMENT" set-pipeline \ diff --git a/ci/tasks/cf/stemcells/task.sh b/ci/tasks/cf/stemcells/task.sh new file mode 100755 index 00000000..d5b161ae --- /dev/null +++ b/ci/tasks/cf/stemcells/task.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -eu + +: $ENVIRONMENT + +output() { + FILE=$1 + QUERY=$2 + + FILE="${FILE}-tfstate-s3/${ENVIRONMENT}-${FILE}.tfstate" + jq -r ".modules[0].outputs | with_entries(.value = .value.value) | $QUERY" <"$FILE" +} + + +export BOSH_ENVIRONMENT=$(output base .bosh_private_ip) + +IAAS_INFO=aws-xen-hvm +CF_STEMCELL_VERSION=$(bosh interpolate cf-deployment/cf-deployment.yml --path=/stemcells/alias=default/version) + +bosh upload-stemcell https://bosh.io/d/stemcells/bosh-${IAAS_INFO}-ubuntu-trusty-go_agent?v=${CF_STEMCELL_VERSION} \ No newline at end of file diff --git a/ci/tasks/cf/stemcells/task.yml b/ci/tasks/cf/stemcells/task.yml new file mode 100644 index 00000000..92f107ca --- /dev/null +++ b/ci/tasks/cf/stemcells/task.yml @@ -0,0 +1,20 @@ +--- +platform: linux +image_resource: + type: docker-image + source: { repository: onsdigital/paas-ci-gp, tag: latest } + +inputs: + - name: paas-bootstrap-git + - name: cf-deployment-git + - name: base-tfstate-s3 + +run: + path: paas-bootstrap-git/ci/tasks/cf/stemcells/task.sh + +params: + DOMAIN: + ENVIRONMENT: + BOSH_CLIENT: admin + BOSH_CLIENT_SECRET: + BOSH_CA_CERT: \ No newline at end of file From 09cf444181b45d86dfb8e10c4dc22797249c7edd Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Tue, 11 Sep 2018 08:54:15 +0100 Subject: [PATCH 39/71] Fix cf-deployment path --- ci/tasks/cf/stemcells/task.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/tasks/cf/stemcells/task.sh b/ci/tasks/cf/stemcells/task.sh index d5b161ae..49625acc 100755 --- a/ci/tasks/cf/stemcells/task.sh +++ b/ci/tasks/cf/stemcells/task.sh @@ -16,6 +16,6 @@ output() { export BOSH_ENVIRONMENT=$(output base .bosh_private_ip) IAAS_INFO=aws-xen-hvm -CF_STEMCELL_VERSION=$(bosh interpolate cf-deployment/cf-deployment.yml --path=/stemcells/alias=default/version) +CF_STEMCELL_VERSION=$(bosh interpolate cf-deployment-git/cf-deployment.yml --path=/stemcells/alias=default/version) bosh upload-stemcell https://bosh.io/d/stemcells/bosh-${IAAS_INFO}-ubuntu-trusty-go_agent?v=${CF_STEMCELL_VERSION} \ No newline at end of file From 8bd49490aa99e78a289eafc59e3365c92791733a Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Tue, 11 Sep 2018 08:56:55 +0100 Subject: [PATCH 40/71] Increase CATS concurrency --- ci/cf.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/cf.yml b/ci/cf.yml index 73de35d1..61427b1f 100644 --- a/ci/cf.yml +++ b/ci/cf.yml @@ -138,6 +138,4 @@ jobs: cf-acceptance-tests: cf-acceptance-tests-git params: CAPTURE_CONFIG: true - NODES: 4 - - + NODES: 8 From a66cbcff4d519943399f1ea0bad95efcf41b4401 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Wed, 12 Sep 2018 11:58:29 +0100 Subject: [PATCH 41/71] Initial cut of RabbitMQ --- .gitmodules | 3 ++ Makefile | 4 ++ README.md | 4 +- bin/cloud_config.sh | 20 +++++++- bin/deploy_rabbitmq.sh | 54 +++++++++++++++++++++ cf-rabbitmq-multitenant-broker-release | 1 + cloud-config/cf.yml | 31 +++++++++++- operations/cloud-config/rabbitmq.yml | 16 ++++++ operations/rabbitmq/release.yml | 33 +++++++++++++ terraform/base/outputs.tf | 36 +++++++++++++- terraform/base/rabbitmq.tf | 22 +++++++++ terraform/base/todo/outputs.tf | 67 -------------------------- 12 files changed, 219 insertions(+), 72 deletions(-) create mode 100755 bin/deploy_rabbitmq.sh create mode 160000 cf-rabbitmq-multitenant-broker-release create mode 100644 operations/cloud-config/rabbitmq.yml create mode 100644 operations/rabbitmq/release.yml create mode 100644 terraform/base/rabbitmq.tf delete mode 100644 terraform/base/todo/outputs.tf diff --git a/.gitmodules b/.gitmodules index 27acba40..b4a94457 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "prometheus-boshrelease"] path = prometheus-boshrelease url = https://github.com/bosh-prometheus/prometheus-boshrelease.git +[submodule "cf-rabbitmq-multitenant-broker-release"] + path = cf-rabbitmq-multitenant-broker-release + url = https://github.com/pivotal-cf/cf-rabbitmq-multitenant-broker-release diff --git a/Makefile b/Makefile index 00f67519..0198010b 100644 --- a/Makefile +++ b/Makefile @@ -53,6 +53,9 @@ cf: ## Deploy CF prometheus: ## Deploy Prometheus @bin/deploy_prometheus.sh +rabbitmq: ## Deploy RabbitMQ service broker + @bin/deploy_rabbitmq.sh + concourse: ## Deploy concourse @bin/deploy_concourse.sh @@ -65,6 +68,7 @@ set_concourse_secrets: ## Set the secrets that Concourse needs to run its pipeli pipelines: login_fly ## Deploy pipelines @ci/cf_pipeline.sh + destroy_bosh: ## Kill off bosh @bin/destroy_bosh.sh diff --git a/README.md b/README.md index ae0c3d22..2b6f347a 100644 --- a/README.md +++ b/README.md @@ -153,8 +153,8 @@ And use the displayed output to point the browser at the desired Prometheus comp | Component | Version | | ----------- | ------- | | concourse | v4.1.0 | -| cf | v4.0.0 | -| bosh | v1.1.0 | +| cf | v4.2.0 | +| bosh | v1.2.0 | | prometheus | v23.2.0 | ## LICENCE diff --git a/bin/cloud_config.sh b/bin/cloud_config.sh index 4128ccc0..7e6eb8f1 100755 --- a/bin/cloud_config.sh +++ b/bin/cloud_config.sh @@ -23,6 +23,7 @@ $BOSH update-cloud-config -n \ -o ./operations/cloud-config/cf-rds-sec-group.yml \ -o ./operations/cloud-config/prometheus.yml \ -o ./operations/cloud-config/concourse.yml \ + -o ./operations/cloud-config/rabbitmq.yml \ -v az1="$(jq -r .availability_zones[0] < data/$ENVIRONMENT.tfvars)" \ -v az2="$(jq -r .availability_zones[1] < data/$ENVIRONMENT.tfvars)" \ -v az3="$(jq -r .availability_zones[2] < data/$ENVIRONMENT.tfvars)" \ @@ -61,4 +62,21 @@ $BOSH update-cloud-config -n \ -v concourse_subnet_az1_id="$(output base .concourse_subnet_ids[0])" \ -v concourse_security_group="$(output base .concourse_security_group_id)" \ -v concourse_alb_target_group="$(output base .concourse_alb_target_group_name)" \ - -v concourse_worker_iam_profile="$(output base .concourse_iam_instance_profile)" \ No newline at end of file + -v concourse_worker_iam_profile="$(output base .concourse_iam_instance_profile)" \ + -v services_subnet_az1_cidr="$(output base .services_subnet_cidr_blocks[0])" \ + -v services_subnet_az1_gateway="$(output base .services_subnet_gateway_ips[0])" \ + -v reserved_services_az1_cidr="$(output base .services_subnet_reserved_cidr_blocks[0])" \ + -v services_subnet_az1_id="$(output base .services_subnet_ids[0])" \ + -v services_subnet_az2_cidr="$(output base .services_subnet_cidr_blocks[1])" \ + -v services_subnet_az2_gateway="$(output base .services_subnet_gateway_ips[1])" \ + -v reserved_services_az2_cidr="$(output base .services_subnet_reserved_cidr_blocks[1])" \ + -v services_subnet_az2_id="$(output base .services_subnet_ids[1])" \ + -v services_subnet_az3_cidr="$(output base .services_subnet_cidr_blocks[2])" \ + -v services_subnet_az3_gateway="$(output base .services_subnet_gateway_ips[2])" \ + -v reserved_services_az3_cidr="$(output base .services_subnet_reserved_cidr_blocks[2])" \ + -v services_subnet_az3_id="$(output base .services_subnet_ids[2])" \ + -v rabbitmq-broker-security-group-id="$(output base .rabbitmq_broker_security_group_id)" \ + -v rabbitmq-server-security-group-id="$(output base .rabbitmq_server_security_group_id)" + + + diff --git a/bin/deploy_rabbitmq.sh b/bin/deploy_rabbitmq.sh new file mode 100755 index 00000000..35ec9f41 --- /dev/null +++ b/bin/deploy_rabbitmq.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set -euo pipefail + +: $ENVIRONMENT + +output() { + FILE=$1 + QUERY=$2 + + bin/outputs.sh $FILE | jq -r "$QUERY" +} + +secret() { + KEY=$1 + bin/secret.sh -e $ENVIRONMENT -d cf -k $KEY +} + +bin/get_states.sh -e $ENVIRONMENT + +BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" + +SYSTEM_DOMAIN="system.$(output base .domain)" + +export BOSH_CA_CERT=$(bosh int --path /default_ca/ca "data/$ENVIRONMENT-bosh-variables.yml") + +RABBITMQ_STEMCELL_VERSION=3586.36 + +$BOSH -d rabbitmq interpolate -n cf-rabbitmq-multitenant-broker-release/manifests/cf-rabbitmq-broker-template.yml \ + -o cf-rabbitmq-multitenant-broker-release/manifests/add-cf-rabbitmq.yml \ + -o ./operations/rabbitmq/release.yml \ + -v deployment-name=rabbitmq \ + -v stemcell-version=3586.16 \ + -v system-domain="$SYSTEM_DOMAIN" \ + -v multitenant-rabbitmq-broker-password=broker \ + -v multitenant-rabbitmq-broker-username=broker \ + -v product-name=p-rabbitmq \ + -v rabbitmq-management-password=admin \ + -v rabbitmq-management-username=admin \ + -v rabbitmq-broker-hostname=rabbitmq-multitenant-broker \ + -v rabbitmq-broker-password=broker \ + -v rabbitmq-broker-plan-uuid=22F0B28C-B886-4123-B01B-95E54D3DE6DA \ + -v rabbitmq-broker-protocol=http \ + -v rabbitmq-broker-username=broker \ + -v rabbitmq-broker-uuid=568725FD-AD46-44CA-9853-621416E983A4 \ + -v rabbitmq-management-hostname=pivotal-rabbitmq \ + -v cf-admin-password=admin \ + -v cf-admin-username=admin \ + -v cluster-partition-handling-strategy=autoheal \ + -v disk_alarm_threshold="{mem_relative,0.4}" \ + -v haproxy-instances=1 \ + -v haproxy-stats-password=admin \ + -v haproxy-stats-username=admin \ + -v rabbitmq-hosts=[] \ No newline at end of file diff --git a/cf-rabbitmq-multitenant-broker-release b/cf-rabbitmq-multitenant-broker-release new file mode 160000 index 00000000..a4dad11d --- /dev/null +++ b/cf-rabbitmq-multitenant-broker-release @@ -0,0 +1 @@ +Subproject commit a4dad11dfb34039f55404617e114d77d2d707a58 diff --git a/cloud-config/cf.yml b/cloud-config/cf.yml index 0fe2cad5..a6282b56 100644 --- a/cloud-config/cf.yml +++ b/cloud-config/cf.yml @@ -93,7 +93,36 @@ networks: cloud_properties: subnet: ((private_subnet_az3_id)) security_groups: [ ((internal_security_group))] - +- name: services + type: manual + subnets: + - range: ((services_subnet_az1_cidr)) + gateway: ((services_subnet_az1_gateway)) + az: z1 + dns: + - ((private_dns_nameserver)) + - 1.1.1.1 + reserved: [((reserved_services_az1_cidr))] + cloud_properties: + subnet: ((services_subnet_az1_id)) + - range: ((services_subnet_az2_cidr)) + gateway: ((services_subnet_az2_gateway)) + az: z2 + dns: + - ((private_dns_nameserver)) + - 1.1.1.1 + reserved: [((services_subnet_az2_cidr))] + cloud_properties: + subnet: ((services_subnet_az2_id)) + - range: ((services_subnet_az3_cidr)) + gateway: ((services_subnet_az3_gateway)) + az: z3 + dns: + - ((private_dns_nameserver)) + - 1.1.1.1 + reserved: [((reserved_services_az3_cidr))] + cloud_properties: + subnet: ((services_subnet_az3_id)) compilation: workers: 8 diff --git a/operations/cloud-config/rabbitmq.yml b/operations/cloud-config/rabbitmq.yml new file mode 100644 index 00000000..487f67e8 --- /dev/null +++ b/operations/cloud-config/rabbitmq.yml @@ -0,0 +1,16 @@ +--- +- type: replace + path: /vm_types/- + value: + name: n1-standard-1 + cloud_properties: + instance_type: t2.micro + security_groups: [ ((rabbitmq-broker-security-group-id))] + +- type: replace + path: /vm_types/- + value: + name: n1-himem-2 + cloud_properties: + instance_type: r5.large + security_groups: [ ((rabbitmq-server-security-group-id))] \ No newline at end of file diff --git a/operations/rabbitmq/release.yml b/operations/rabbitmq/release.yml new file mode 100644 index 00000000..ba9e8d1a --- /dev/null +++ b/operations/rabbitmq/release.yml @@ -0,0 +1,33 @@ +--- + +# https://bosh.io/releases/github.com/pivotal-cf/cf-rabbitmq-multitenant-broker-release?all=1 +- type: replace + path: /releases/name=cf-rabbitmq-multitenant-broker? + value: + name: "cf-rabbitmq-multitenant-broker" + version: "37.0.0" + url: "https://bosh.io/d/github.com/pivotal-cf/cf-rabbitmq-multitenant-broker-release?v=37.0.0" + sha1: "146049116e60593af959d1c738a139b59e8480a7" + +- type: remove + path: /releases/name=cf-rabbitmq-smoke-tests + +- type: replace + path: /releases/name=rabbitmq-smoke-tests? + value: + name: "rabbitmq-smoke-tests" + version: "4" + url: "https://bosh.io/d/github.com/cloudfoundry-community/rabbitmq-smoke-tests-boshrelease?v=4" + sha1: "06004fa77387f4443236117604870d9eeb93acb1" + +- type: replace + path: /releases/name=cf-rabbitmq? + value: + name: "cf-rabbitmq" + version: "251.0.0" + url: "https://bosh.io/d/github.com/pivotal-cf/cf-rabbitmq-release?v=251.0.0" + sha1: "6b26a898eb7d908b4325f6010917a228986c7ec0" + +- type: replace + path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests/release + value: rabbitmq-smoke-tests \ No newline at end of file diff --git a/terraform/base/outputs.tf b/terraform/base/outputs.tf index d9865618..452d692d 100644 --- a/terraform/base/outputs.tf +++ b/terraform/base/outputs.tf @@ -255,4 +255,38 @@ output "concourse_fqdn" { output "concourse_iam_instance_profile" { value = "${aws_iam_instance_profile.concourse.name}" -} \ No newline at end of file +} + +output "services_subnet_gateway_ips" { + value = [ + "${cidrhost(aws_subnet.services.*.cidr_block[0],1)}", + "${cidrhost(aws_subnet.services.*.cidr_block[1],1)}", + "${cidrhost(aws_subnet.services.*.cidr_block[2],1)}" + ] +} + +# NASTY HACK ALERT - we cannot find a way in terraform to perform an action on all elements of a list +# so you will have to change this if you add more AZs +output "services_subnet_reserved_cidr_blocks" { + value = [ + "${cidrsubnet(aws_subnet.services.*.cidr_block[0],8,0)}", + "${cidrsubnet(aws_subnet.services.*.cidr_block[1],8,0)}", + "${cidrsubnet(aws_subnet.services.*.cidr_block[2],8,0)}" + ] +} + +output "services_subnet_ids" { + value = ["${aws_subnet.services.*.id}"] +} + +output "services_subnet_cidr_blocks" { + value = ["${aws_subnet.services.*.cidr_block}"] +} + +output "rabbitmq_broker_security_group_id" { + value = "${aws_security_group.rabbitmq_broker.id}" +} + +output "rabbitmq_server_security_group_id" { + value = "${aws_security_group.rabbitmq_server.id}" +} diff --git a/terraform/base/rabbitmq.tf b/terraform/base/rabbitmq.tf new file mode 100644 index 00000000..0e25f71d --- /dev/null +++ b/terraform/base/rabbitmq.tf @@ -0,0 +1,22 @@ +resource "aws_security_group" "rabbitmq_broker" { + name = "${var.environment}_rabbitmq_broker_security_group" + description = "RabbitMQ service broker access" + vpc_id = "${aws_vpc.default.id}" + + tags { + Name = "${var.environment}-rabbitmq-broker-security-group" + Environment = "${var.environment}" + } +} + + +resource "aws_security_group" "rabbitmq_server" { + name = "${var.environment}_rabbitmq_server_security_group" + description = "RabbitMQ server access" + vpc_id = "${aws_vpc.default.id}" + + tags { + Name = "${var.environment}-rabbitmq-server-security-group" + Environment = "${var.environment}" + } +} \ No newline at end of file diff --git a/terraform/base/todo/outputs.tf b/terraform/base/todo/outputs.tf deleted file mode 100644 index 1efdab9a..00000000 --- a/terraform/base/todo/outputs.tf +++ /dev/null @@ -1,67 +0,0 @@ -output "vpc_id" { - value = "${aws_vpc.default.id}" -} - -output "vpc_dns_nameserver" { - value = "${cidrhost(aws_vpc.default.cidr_block, 2)}" -} - -output "region" { - value = "${var.region}" -} - -output "az1" { - value = "${var.az1}" -} - -output "az2" { - value = "${var.az2}" -} - -output "az3" { - value = "${var.az3}" -} - -output "default_route_table_id" { - value = "${aws_vpc.default.main_route_table_id}" -} - -output "environment" { - value = "${var.environment}" -} - -output "ingress_whitelist" { - value = "${var.ingress_whitelist}" -} - -output "s3_kms_key_id" { - value = "${aws_kms_key.paas_state_key.id}" -} - -output "s3_kms_key_arn" { - value = "${aws_kms_key.paas_state_key.arn}" -} - -output "dns_zone" { - value = "${aws_route53_zone.child_zone.name}" -} - -output "parent_dns_zone" { - value = "${var.parent_dns_zone}" -} - -output "state_bucket_id" { - value = "${aws_s3_bucket.paas_states.id}" -} - -output "aws_internet_gateway_id" { - value = "${aws_internet_gateway.default.id}" -} - -output "s3_prefix" { - value = "${var.s3_prefix}" -} - -output "private_dns_zone" { - value = "${aws_route53_zone.private.name}" -} From defdc2c1cdb42a9462dc7e539fb144c8d64c711f Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Wed, 12 Sep 2018 13:03:13 +0100 Subject: [PATCH 42/71] Get stuff in --- bin/cloud_config.sh | 1 + bin/deploy_rabbitmq.sh | 2 +- operations/cloud-config/rabbitmq.yml | 8 ++- terraform/base/bosh.tf | 56 ++++++++++++++++- terraform/base/outputs.tf | 4 ++ terraform/base/rabbitmq.tf | 89 ++++++++++++++++++++++++++++ 6 files changed, 154 insertions(+), 6 deletions(-) diff --git a/bin/cloud_config.sh b/bin/cloud_config.sh index 7e6eb8f1..14d34f4b 100755 --- a/bin/cloud_config.sh +++ b/bin/cloud_config.sh @@ -34,6 +34,7 @@ $BOSH update-cloud-config -n \ -v reserved_az2_cidr="$(output base .internal_subnet_reserved_cidr_blocks[1])" \ -v reserved_az3_cidr="$(output base .internal_subnet_reserved_cidr_blocks[2])" \ -v private_dns_nameserver="$(output base .vpc_dns_nameserver)" \ + -v bosh-managed-security-group-id="$(output base .bosh_managed_security_group_id)" \ -v internal_security_group="$(output base .cf_internal_security_group_id)" \ -v private_subnet_az1_id="$(output base .internal_subnet_ids[0])" \ -v private_subnet_az2_id="$(output base .internal_subnet_ids[1])" \ diff --git a/bin/deploy_rabbitmq.sh b/bin/deploy_rabbitmq.sh index 35ec9f41..77317745 100755 --- a/bin/deploy_rabbitmq.sh +++ b/bin/deploy_rabbitmq.sh @@ -26,7 +26,7 @@ export BOSH_CA_CERT=$(bosh int --path /default_ca/ca "data/$ENVIRONMENT-bosh-var RABBITMQ_STEMCELL_VERSION=3586.36 -$BOSH -d rabbitmq interpolate -n cf-rabbitmq-multitenant-broker-release/manifests/cf-rabbitmq-broker-template.yml \ +$BOSH -d rabbitmq deploy -n cf-rabbitmq-multitenant-broker-release/manifests/cf-rabbitmq-broker-template.yml \ -o cf-rabbitmq-multitenant-broker-release/manifests/add-cf-rabbitmq.yml \ -o ./operations/rabbitmq/release.yml \ -v deployment-name=rabbitmq \ diff --git a/operations/cloud-config/rabbitmq.yml b/operations/cloud-config/rabbitmq.yml index 487f67e8..e7778e6f 100644 --- a/operations/cloud-config/rabbitmq.yml +++ b/operations/cloud-config/rabbitmq.yml @@ -5,7 +5,9 @@ name: n1-standard-1 cloud_properties: instance_type: t2.micro - security_groups: [ ((rabbitmq-broker-security-group-id))] + security_groups: + - ((rabbitmq-broker-security-group-id)) + - ((bosh-managed-security-group-id)) - type: replace path: /vm_types/- @@ -13,4 +15,6 @@ name: n1-himem-2 cloud_properties: instance_type: r5.large - security_groups: [ ((rabbitmq-server-security-group-id))] \ No newline at end of file + security_groups: + - ((rabbitmq-server-security-group-id)) + - ((bosh-managed-security-group-id)) diff --git a/terraform/base/bosh.tf b/terraform/base/bosh.tf index 6a8a350f..ead326b4 100644 --- a/terraform/base/bosh.tf +++ b/terraform/base/bosh.tf @@ -85,7 +85,6 @@ resource "aws_security_group" "bosh" { } } - resource "aws_security_group_rule" "bosh_uaa_concourse" { security_group_id = "${aws_security_group.bosh.id}" type = "ingress" @@ -156,7 +155,7 @@ resource "aws_security_group_rule" "bosh_management_tcp" { protocol = "tcp" from_port = 0 to_port = 65535 - source_security_group_id = "${aws_security_group.bosh.id}" + self = true } resource "aws_security_group_rule" "bosh_management_udp" { @@ -165,7 +164,7 @@ resource "aws_security_group_rule" "bosh_management_udp" { protocol = "udp" from_port = 0 to_port = 65535 - source_security_group_id = "${aws_security_group.bosh.id}" + self = true } resource "aws_security_group_rule" "jumpbox_uaa_bosh" { @@ -244,4 +243,55 @@ resource "tls_private_key" "bosh" { resource "aws_key_pair" "bosh" { key_name = "${var.environment}-bosh" public_key = "${tls_private_key.bosh.public_key_openssh}" +} + +resource "aws_security_group" "bosh_managed" { + name = "${var.environment}_bosh_managed_security_group" + description = "Allow BOSH to manage this EC2 instance" + vpc_id = "${aws_vpc.default.id}" + + tags { + Name = "${var.environment}-bosh-managed-security-group" + Environment = "${var.environment}" + } +} + +resource "aws_security_group_rule" "bosh_to_managed_ssh" { + security_group_id = "${aws_security_group.bosh_managed.id}" + type = "ingress" + protocol = "tcp" + from_port = 22 + to_port = 22 + source_security_group_id = "${aws_security_group.bosh.id}" + description = "Allow BOSH to SSH to instance" +} + +resource "aws_security_group_rule" "bosh_to_managed_mbus" { + security_group_id = "${aws_security_group.bosh_managed.id}" + type = "ingress" + protocol = "tcp" + from_port = 6868 + to_port = 6868 + source_security_group_id = "${aws_security_group.bosh.id}" + description = "Allow BOSH to access the BOSH agent on instance" +} + +resource "aws_security_group_rule" "managed_to_bosh_tcp" { + security_group_id = "${aws_security_group.bosh_managed.id}" + type = "egress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + source_security_group_id = "${aws_security_group.bosh.id}" + description = "Allow managed instance to respond to BOSH" +} + +resource "aws_security_group_rule" "managed_to_bosh_udp" { + security_group_id = "${aws_security_group.bosh_managed.id}" + type = "egress" + protocol = "udp" + from_port = 0 + to_port = 65535 + source_security_group_id = "${aws_security_group.bosh.id}" + description = "Allow managed instance to respond to BOSH" } \ No newline at end of file diff --git a/terraform/base/outputs.tf b/terraform/base/outputs.tf index 452d692d..67675c02 100644 --- a/terraform/base/outputs.tf +++ b/terraform/base/outputs.tf @@ -64,6 +64,10 @@ output "bosh_security_group_ids" { value = [ "${aws_security_group.bosh.id}"] } +output "bosh_managed_security_group_id" { + value = "${aws_security_group.bosh_managed.id}" +} + output "bosh_private_key" { value = "${tls_private_key.bosh.private_key_pem}" } diff --git a/terraform/base/rabbitmq.tf b/terraform/base/rabbitmq.tf index 0e25f71d..9d0e5697 100644 --- a/terraform/base/rabbitmq.tf +++ b/terraform/base/rabbitmq.tf @@ -9,6 +9,25 @@ resource "aws_security_group" "rabbitmq_broker" { } } +resource "aws_security_group_rule" "cf_to_rmq_broker" { + security_group_id = "${aws_security_group.rabbitmq_broker.id}" + type = "ingress" + protocol = "tcp" + from_port = 4567 + to_port = 4567 + source_security_group_id = "${aws_security_group.internal.id}" + description = "CF may talk to RabbitMQ broker API" +} + +resource "aws_security_group_rule" "rmq_broker_outbound" { + security_group_id = "${aws_security_group.rabbitmq_broker.id}" + type = "egress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + cidr_blocks = ["0.0.0.0/0"] # FIXME: restrict to the CF ALB security group - 443, 8443 + description = "RabbitMQ broker outbound access" +} resource "aws_security_group" "rabbitmq_server" { name = "${var.environment}_rabbitmq_server_security_group" @@ -19,4 +38,74 @@ resource "aws_security_group" "rabbitmq_server" { Name = "${var.environment}-rabbitmq-server-security-group" Environment = "${var.environment}" } +} + +resource "aws_security_group_rule" "rmq_broker_to_rmq_server" { + security_group_id = "${aws_security_group.rabbitmq_server.id}" + type = "ingress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + source_security_group_id = "${aws_security_group.rabbitmq_broker.id}" + description = "RabbitMQ broker can do everything to the servers" +} + +resource "aws_security_group_rule" "cf_to_rmq_5671_2" { + security_group_id = "${aws_security_group.rabbitmq_server.id}" + type = "ingress" + protocol = "tcp" + from_port = 5671 + to_port = 5672 + source_security_group_id = "${aws_security_group.internal.id}" + description = "CF may talk to RabbitMQ on ports 5671,2" +} + +resource "aws_security_group_rule" "cf_to_rmq_1883" { + security_group_id = "${aws_security_group.rabbitmq_server.id}" + type = "ingress" + protocol = "tcp" + from_port = 1883 + to_port = 1883 + source_security_group_id = "${aws_security_group.internal.id}" + description = "CF may talk to RabbitMQ on ports 5671,2" +} + +resource "aws_security_group_rule" "cf_to_rmq_8883" { + security_group_id = "${aws_security_group.rabbitmq_server.id}" + type = "ingress" + protocol = "tcp" + from_port = 8883 + to_port = 8883 + source_security_group_id = "${aws_security_group.internal.id}" + description = "CF may talk to RabbitMQ on ports 8883" +} + +resource "aws_security_group_rule" "cf_to_rmq_61613_4" { + security_group_id = "${aws_security_group.rabbitmq_server.id}" + type = "ingress" + protocol = "tcp" + from_port = 61613 + to_port = 61614 + source_security_group_id = "${aws_security_group.internal.id}" + description = "CF may talk to RabbitMQ on ports 61613,4" +} + +resource "aws_security_group_rule" "cf_to_rmq_15672" { + security_group_id = "${aws_security_group.rabbitmq_server.id}" + type = "ingress" + protocol = "tcp" + from_port = 15672 + to_port = 15672 + source_security_group_id = "${aws_security_group.internal.id}" + description = "CF may talk to RabbitMQ on ports 15672" +} + +resource "aws_security_group_rule" "cf_to_rmq_15674" { + security_group_id = "${aws_security_group.rabbitmq_server.id}" + type = "ingress" + protocol = "tcp" + from_port = 15674 + to_port = 15674 + source_security_group_id = "${aws_security_group.internal.id}" + description = "CF may talk to RabbitMQ on ports 15674" } \ No newline at end of file From d44a77d1fc586f23513cb7dd2610e5b2b8f5eb28 Mon Sep 17 00:00:00 2001 From: Robert Gruber Date: Wed, 12 Sep 2018 13:07:04 +0100 Subject: [PATCH 43/71] Add Redis security group and subnet [#157117678] --- .gitmodules | 3 ++ Makefile | 2 ++ bin/deploy_redis.sh | 57 ++++++++++++++++++++++++++++++++ bin/terraform.sh | 7 +++- concourse-bosh-deployment | 2 +- elasticache-broker | 1 + operations/redis/credentials.yml | 7 ++++ terraform/base/outputs.tf | 8 +++++ terraform/base/services-redis.tf | 33 ++++++++++++++++++ 9 files changed, 118 insertions(+), 2 deletions(-) create mode 100755 bin/deploy_redis.sh create mode 160000 elasticache-broker create mode 100644 operations/redis/credentials.yml create mode 100644 terraform/base/services-redis.tf diff --git a/.gitmodules b/.gitmodules index b4a94457..4bbdc23b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "cf-rabbitmq-multitenant-broker-release"] path = cf-rabbitmq-multitenant-broker-release url = https://github.com/pivotal-cf/cf-rabbitmq-multitenant-broker-release +[submodule "elasticache-broker"] + path = elasticache-broker + url = https://github.com/cloudfoundry-community/elasticache-broker.git diff --git a/Makefile b/Makefile index 0198010b..742d261f 100644 --- a/Makefile +++ b/Makefile @@ -75,3 +75,5 @@ destroy_bosh: ## Kill off bosh decode_aws_error: ## Decode AWS message @aws sts decode-authorization-message --encoded-message ${DECODE_MESSAGE} | jq -r .DecodedMessage | jq . +deploy_redis: ## Deploy Redis (Elasticache) + @bin/deploy_redis.sh diff --git a/bin/deploy_redis.sh b/bin/deploy_redis.sh new file mode 100755 index 00000000..f22eeec2 --- /dev/null +++ b/bin/deploy_redis.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +set -euo pipefail + +: $ENVIRONMENT + +output() { + FILE=$1 + QUERY=$2 + + bin/outputs.sh $FILE | jq -r "$QUERY" +} + +bin/get_states.sh -e $ENVIRONMENT + +BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" + +REDIS_SUBMODULE="./elasticache-broker" + +# TODO provide credentials from credhub +bosh int $REDIS_SUBMODULE/manifest.yml \ + -o ./operations/redis/credentials.yml \ + -v aws_access_key_id="***REMOVED***" \ + -v aws_secret_access_key="***REMOVED***" \ + > foo.yml + +cat foo.yml + +REDIS_CONFIG="elasticache-broker/config.json" +touch $REDIS_CONFIG + +cleanup() { + rm -f $REDIS_CONFIG +} + +trap cleanup EXIT + +REDIS_BROKER_USER='test_user' +REDIS_BROKER_PASSWORD='test_password' +REDIS_SERVICE_UUID="test_uuid" +REDIS_SERVICE_NAME="test_redis_service_name" +REDIS_SERVICE_TAG="test_tag" +# Create redis subnet group +REDIS_CACHE_SUBNET_GROUP_NAME="test_cache_subnet_in_rds_subnets" +REDIS_CACHE_SECURITY_GROUP="test_redis_security_groups" +REGION='test_region' +jq '.username = "'${REDIS_BROKER_USER}'" | + .password = "'${REDIS_BROKER_PASSWORD}'" | + .elasticache_config.region = "'${REGION}'" | + .elasticache_config.catalog.services[0].id = "'${REDIS_SERVICE_UUID}'" | + .elasticache_config.catalog.services[0].tags[0] = "'${REDIS_SERVICE_TAG}'" | + .elasticache_config.catalog.services[0].name = "'${REDIS_SERVICE_NAME}'" | + .elasticache_config.catalog.services[0].plans[].elasticache_properties.cache_subnet_group_name = "'${REDIS_CACHE_SUBNET_GROUP_NAME}'" | + .elasticache_config.catalog.services[0].plans[].elasticache_properties.cache_security_groups[0]= "'${REDIS_CACHE_SECURITY_GROUP}'" ' \ + $REDIS_SUBMODULE/config-sample.json > "$REDIS_CONFIG" + +cat $REDIS_CONFIG \ No newline at end of file diff --git a/bin/terraform.sh b/bin/terraform.sh index b0e9d0cf..f149bdf7 100755 --- a/bin/terraform.sh +++ b/bin/terraform.sh @@ -11,7 +11,12 @@ OPTS= STATE="$ENVIRONMENT-base.tfstate" ENV_VARS="$ENVIRONMENT.tfvars" +persist() { + bin/persist_states.sh -e $ENVIRONMENT -f $STATE -f $ENV_VARS +} + +trap persist EXIT + bin/get_states.sh -e $ENVIRONMENT -o -f $STATE -f $ENV_VARS terraform init terraform/base terraform $COMMAND $OPTS -var-file="data/$ENV_VARS" -state="data/$STATE" terraform/base/ -bin/persist_states.sh -e $ENVIRONMENT -f $STATE -f $ENV_VARS diff --git a/concourse-bosh-deployment b/concourse-bosh-deployment index 70640ac7..cae8cb40 160000 --- a/concourse-bosh-deployment +++ b/concourse-bosh-deployment @@ -1 +1 @@ -Subproject commit 70640ac7707bc476f506fe6f6228fdb4422aa14f +Subproject commit cae8cb402a470c7c318867e8070d57afcebc8b9c diff --git a/elasticache-broker b/elasticache-broker new file mode 160000 index 00000000..d0a66eab --- /dev/null +++ b/elasticache-broker @@ -0,0 +1 @@ +Subproject commit d0a66eab280793e3d2f342996e6c16a5cc6f9c8f diff --git a/operations/redis/credentials.yml b/operations/redis/credentials.yml new file mode 100644 index 00000000..823d642b --- /dev/null +++ b/operations/redis/credentials.yml @@ -0,0 +1,7 @@ +--- +- type: replace + path: /applications/name=elasticache-broker/env? + value: + - AWS_ACCESS_KEY_ID: ((aws_access_key_id)) + - AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) + - GO15VENDOREXPERIMENT: 0 \ No newline at end of file diff --git a/terraform/base/outputs.tf b/terraform/base/outputs.tf index 67675c02..90da09ff 100644 --- a/terraform/base/outputs.tf +++ b/terraform/base/outputs.tf @@ -294,3 +294,11 @@ output "rabbitmq_broker_security_group_id" { output "rabbitmq_server_security_group_id" { value = "${aws_security_group.rabbitmq_server.id}" } +output "redis_cache_subnet_group_name" { + value = "${aws_elasticache_subnet_group.redis.name}" +} + +output "redis_security_group_name" { + value = "${aws_elasticache_security_group.redis.name}" +} + diff --git a/terraform/base/services-redis.tf b/terraform/base/services-redis.tf new file mode 100644 index 00000000..e49bd998 --- /dev/null +++ b/terraform/base/services-redis.tf @@ -0,0 +1,33 @@ +# Redis (Elasticache) +resource "aws_security_group" "redis" { + name = "${var.environment}_redis_security_group" + description = "Redis access" + vpc_id = "${aws_vpc.default.id}" + + tags { + Name = "${var.environment}-redis-security-group" + Environment = "${var.environment}" + } +} + +# resource "aws_security_group_rule" "allow-all" { +# security_group_id = "${aws_security_group.bosh.id}" +# type = "egress" +# protocol = "-1" +# from_port = 0 +# to_port = 0 +# cidr_blocks = ["0.0.0.0/0"] +# } + +# CREATE USER + + +resource "aws_elasticache_subnet_group" "redis" { + name = "${var.environment}-redis-subnet-group" + subnet_ids = ["${aws_subnet.rds.*.id}"] +} + +resource "aws_elasticache_security_group" "redis" { + name = "${var.environment}-redis-security-group" + security_group_names = ["${aws_security_group.redis.name}"] +} \ No newline at end of file From ccf938e3d2815014bf9b85e8286770115f07feec Mon Sep 17 00:00:00 2001 From: Robert Gruber Date: Wed, 12 Sep 2018 13:22:43 +0100 Subject: [PATCH 44/71] Remove secrets [#157117678] --- bin/deploy_redis.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/deploy_redis.sh b/bin/deploy_redis.sh index f22eeec2..3d5ce348 100755 --- a/bin/deploy_redis.sh +++ b/bin/deploy_redis.sh @@ -20,8 +20,8 @@ REDIS_SUBMODULE="./elasticache-broker" # TODO provide credentials from credhub bosh int $REDIS_SUBMODULE/manifest.yml \ -o ./operations/redis/credentials.yml \ - -v aws_access_key_id="***REMOVED***" \ - -v aws_secret_access_key="***REMOVED***" \ + -v aws_access_key_id="$REDIS_AWS_ACCESS_KEY_ID" \ + -v aws_secret_access_key="$REDIS_AWS_SECRET_ACCESS_KEY" \ > foo.yml cat foo.yml From 5aa25d3c5a1c01829db07d92a942961f7361dec1 Mon Sep 17 00:00:00 2001 From: Robert Gruber Date: Wed, 12 Sep 2018 14:07:33 +0100 Subject: [PATCH 45/71] Create Redis user [#157117678] --- Makefile | 2 +- terraform/base/outputs.tf | 5 ++- terraform/base/services-redis.tf | 60 +++++++++++++++++++++++++++++--- 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 742d261f..26cba7e5 100644 --- a/Makefile +++ b/Makefile @@ -75,5 +75,5 @@ destroy_bosh: ## Kill off bosh decode_aws_error: ## Decode AWS message @aws sts decode-authorization-message --encoded-message ${DECODE_MESSAGE} | jq -r .DecodedMessage | jq . -deploy_redis: ## Deploy Redis (Elasticache) +redis: ## Deploy Redis (Elasticache) @bin/deploy_redis.sh diff --git a/terraform/base/outputs.tf b/terraform/base/outputs.tf index 90da09ff..93c217fb 100644 --- a/terraform/base/outputs.tf +++ b/terraform/base/outputs.tf @@ -299,6 +299,9 @@ output "redis_cache_subnet_group_name" { } output "redis_security_group_name" { - value = "${aws_elasticache_security_group.redis.name}" + value = "${aws_security_group.redis.name}" } +output "redis_user_name" { + value = "${aws_iam_user.redis.name}" +} \ No newline at end of file diff --git a/terraform/base/services-redis.tf b/terraform/base/services-redis.tf index e49bd998..97787103 100644 --- a/terraform/base/services-redis.tf +++ b/terraform/base/services-redis.tf @@ -21,13 +21,63 @@ resource "aws_security_group" "redis" { # CREATE USER - resource "aws_elasticache_subnet_group" "redis" { name = "${var.environment}-redis-subnet-group" subnet_ids = ["${aws_subnet.rds.*.id}"] } -resource "aws_elasticache_security_group" "redis" { - name = "${var.environment}-redis-security-group" - security_group_names = ["${aws_security_group.redis.name}"] -} \ No newline at end of file +# User and permissions +resource "aws_iam_user" "redis" { + name = "${var.environment}-redis-user" + path = "/services/" +} + +resource "aws_iam_user_policy" "redis" { + name = "${var.environment}-redis-policy" + user = "${aws_iam_user.redis.name}" + + policy = < Date: Wed, 12 Sep 2018 14:28:12 +0100 Subject: [PATCH 46/71] More rabbitmq work --- bin/deploy_rabbitmq.sh | 4 ++- bin/stemcells.sh | 5 ++++ cloud-config/cf.yml | 5 +++- operations/cloud-config/rabbitmq.yml | 2 +- operations/rabbitmq/release.yml | 42 +++++++++++++++++++++++++++- terraform/base/outputs.tf | 6 ++-- terraform/base/prometheus.tf | 11 ++++++++ 7 files changed, 68 insertions(+), 7 deletions(-) diff --git a/bin/deploy_rabbitmq.sh b/bin/deploy_rabbitmq.sh index 77317745..ced20a23 100755 --- a/bin/deploy_rabbitmq.sh +++ b/bin/deploy_rabbitmq.sh @@ -21,6 +21,7 @@ bin/get_states.sh -e $ENVIRONMENT BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" SYSTEM_DOMAIN="system.$(output base .domain)" +APPS_DOMAIN="apps.$(output base .domain)" export BOSH_CA_CERT=$(bosh int --path /default_ca/ca "data/$ENVIRONMENT-bosh-variables.yml") @@ -32,6 +33,7 @@ $BOSH -d rabbitmq deploy -n cf-rabbitmq-multitenant-broker-release/manifests/cf- -v deployment-name=rabbitmq \ -v stemcell-version=3586.16 \ -v system-domain="$SYSTEM_DOMAIN" \ + -v apps_domain="$APPS_DOMAIN" \ -v multitenant-rabbitmq-broker-password=broker \ -v multitenant-rabbitmq-broker-username=broker \ -v product-name=p-rabbitmq \ @@ -47,7 +49,7 @@ $BOSH -d rabbitmq deploy -n cf-rabbitmq-multitenant-broker-release/manifests/cf- -v cf-admin-password=admin \ -v cf-admin-username=admin \ -v cluster-partition-handling-strategy=autoheal \ - -v disk_alarm_threshold="{mem_relative,0.4}" \ + -v disk_alarm_threshold="{mem_relative,'0.4'}" \ -v haproxy-instances=1 \ -v haproxy-stats-password=admin \ -v haproxy-stats-username=admin \ diff --git a/bin/stemcells.sh b/bin/stemcells.sh index efeda8d7..3258dc4c 100755 --- a/bin/stemcells.sh +++ b/bin/stemcells.sh @@ -14,6 +14,7 @@ output() { IAAS_INFO=aws-xen-hvm CF_STEMCELL_VERSION=$(bosh interpolate cf-deployment/cf-deployment.yml --path=/stemcells/alias=default/version) CONCOURSE_STEMCELL_VERSION=97.15 # FIXME: interpolate from somewhere +RABBITMQ_STEMCELL_VERSION=3586.16 BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" if ! $BOSH -n stemcells | grep -q "ubuntu-trusty.*$CF_STEMCELL_VERSION"; then @@ -22,4 +23,8 @@ fi if ! $BOSH -n stemcells | grep -q "ubuntu-xenial.*$CONCOURSE_STEMCELL_VERSION"; then $BOSH upload-stemcell https://bosh.io/d/stemcells/bosh-${IAAS_INFO}-ubuntu-xenial-go_agent?v=${CONCOURSE_STEMCELL_VERSION} +fi + +if ! $BOSH -n stemcells | grep -q "ubuntu-trusty.*$RABBITMQ_STEMCELL_VERSION"; then + $BOSH upload-stemcell https://bosh.io/d/stemcells/bosh-${IAAS_INFO}-ubuntu-trusty-go_agent?v=${RABBITMQ_STEMCELL_VERSION} fi \ No newline at end of file diff --git a/cloud-config/cf.yml b/cloud-config/cf.yml index a6282b56..95f67343 100644 --- a/cloud-config/cf.yml +++ b/cloud-config/cf.yml @@ -43,6 +43,9 @@ disk_types: - name: 10GB disk_size: 10_000 cloud_properties: {encrypted: true, type: gp2} +- name: 50GB + disk_size: 50_000 + cloud_properties: {encrypted: true, type: gp2} - name: 100GB disk_size: 100_000 cloud_properties: {encrypted: true, type: gp2} @@ -111,7 +114,7 @@ networks: dns: - ((private_dns_nameserver)) - 1.1.1.1 - reserved: [((services_subnet_az2_cidr))] + reserved: [((reserved_services_az2_cidr))] cloud_properties: subnet: ((services_subnet_az2_id)) - range: ((services_subnet_az3_cidr)) diff --git a/operations/cloud-config/rabbitmq.yml b/operations/cloud-config/rabbitmq.yml index e7778e6f..a314444f 100644 --- a/operations/cloud-config/rabbitmq.yml +++ b/operations/cloud-config/rabbitmq.yml @@ -12,7 +12,7 @@ - type: replace path: /vm_types/- value: - name: n1-himem-2 + name: n1-highmem-2 cloud_properties: instance_type: r5.large security_groups: diff --git a/operations/rabbitmq/release.yml b/operations/rabbitmq/release.yml index ba9e8d1a..0fc9b692 100644 --- a/operations/rabbitmq/release.yml +++ b/operations/rabbitmq/release.yml @@ -28,6 +28,46 @@ url: "https://bosh.io/d/github.com/pivotal-cf/cf-rabbitmq-release?v=251.0.0" sha1: "6b26a898eb7d908b4325f6010917a228986c7ec0" +- type: replace + path: /instance_groups/name=rmq-broker/networks + value: + - name: services + +- type: replace + path: /instance_groups/name=rmq/networks + value: + - name: services + +- type: replace + path: /instance_groups/name=haproxy/networks + value: + - name: services + +- type: replace + path: /instance_groups/name=rmq-broker/azs + value: [z1, z2, z3] + +- type: replace + path: /instance_groups/name=rmq/azs + value: [z1, z2, z3] + + +- type: replace + path: /instance_groups/name=haproxy/azs + value: [z1, z2, z3] + + +# The BOSH release appears not to be cf-rabbitmq-smoke-tests, so change to suit - type: replace path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests/release - value: rabbitmq-smoke-tests \ No newline at end of file + value: rabbitmq-smoke-tests + +- type: remove + path: /instance_groups/name=rmq-broker/jobs/name=rabbitmq-broker/properties/rabbitmq-broker/rabbitmq/hosts + +# - type: replace +# path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests/properties/cf/apps_domain? +# value: ((apps_domain)) + +- type: remove + path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests \ No newline at end of file diff --git a/terraform/base/outputs.tf b/terraform/base/outputs.tf index 93c217fb..e37b2832 100644 --- a/terraform/base/outputs.tf +++ b/terraform/base/outputs.tf @@ -273,9 +273,9 @@ output "services_subnet_gateway_ips" { # so you will have to change this if you add more AZs output "services_subnet_reserved_cidr_blocks" { value = [ - "${cidrsubnet(aws_subnet.services.*.cidr_block[0],8,0)}", - "${cidrsubnet(aws_subnet.services.*.cidr_block[1],8,0)}", - "${cidrsubnet(aws_subnet.services.*.cidr_block[2],8,0)}" + "${cidrsubnet(aws_subnet.services.*.cidr_block[0],7,0)}", + "${cidrsubnet(aws_subnet.services.*.cidr_block[1],7,0)}", + "${cidrsubnet(aws_subnet.services.*.cidr_block[2],7,0)}" ] } diff --git a/terraform/base/prometheus.tf b/terraform/base/prometheus.tf index b9f89b43..4e508845 100644 --- a/terraform/base/prometheus.tf +++ b/terraform/base/prometheus.tf @@ -228,6 +228,17 @@ resource "aws_security_group_rule" "prometheus_bosh_node_exporters" { source_security_group_id = "${aws_security_group.prometheus.id}" } + +resource "aws_security_group_rule" "managed_node_exporters" { + security_group_id = "${aws_security_group.bosh_managed.id}" + type = "ingress" + protocol = "tcp" + from_port = 9100 + to_port = 9100 + source_security_group_id = "${aws_security_group.prometheus.id}" + description = "Allow BOSH to access the BOSH agent on instance" +} + resource "aws_security_group_rule" "prometheus_cf_node_exporters" { security_group_id = "${aws_security_group.internal.id}" type = "ingress" From 895e78da4623d45f6d152b74a19247a1df4765a0 Mon Sep 17 00:00:00 2001 From: Robert Gruber Date: Wed, 12 Sep 2018 15:45:30 +0100 Subject: [PATCH 47/71] Store secrets in Credhub [#157117678] --- bin/deploy_redis.sh | 10 ++++----- bin/set_redis_secrets.sh | 38 ++++++++++++++++++++++++++++++++ terraform/base/outputs.tf | 6 ++++- terraform/base/services-redis.tf | 6 +++++ 4 files changed, 54 insertions(+), 6 deletions(-) create mode 100755 bin/set_redis_secrets.sh diff --git a/bin/deploy_redis.sh b/bin/deploy_redis.sh index 3d5ce348..e98c14d0 100755 --- a/bin/deploy_redis.sh +++ b/bin/deploy_redis.sh @@ -20,8 +20,8 @@ REDIS_SUBMODULE="./elasticache-broker" # TODO provide credentials from credhub bosh int $REDIS_SUBMODULE/manifest.yml \ -o ./operations/redis/credentials.yml \ - -v aws_access_key_id="$REDIS_AWS_ACCESS_KEY_ID" \ - -v aws_secret_access_key="$REDIS_AWS_SECRET_ACCESS_KEY" \ + -v aws_access_key_id="$(bin/credhub_credentials.sh -e $ENVIRONMENT credhub get -n /cf/redis/aws_access_key_id -j | jq -r .value)" \ + -v aws_secret_access_key="$(bin/credhub_credentials.sh -e $ENVIRONMENT credhub get -n /cf/redis/aws_secret_access_key_id -j | jq -r .value)" \ > foo.yml cat foo.yml @@ -41,9 +41,9 @@ REDIS_SERVICE_UUID="test_uuid" REDIS_SERVICE_NAME="test_redis_service_name" REDIS_SERVICE_TAG="test_tag" # Create redis subnet group -REDIS_CACHE_SUBNET_GROUP_NAME="test_cache_subnet_in_rds_subnets" -REDIS_CACHE_SECURITY_GROUP="test_redis_security_groups" -REGION='test_region' +REDIS_CACHE_SUBNET_GROUP_NAME="$(output base .redis_cache_subnet_group_name)" +REDIS_CACHE_SECURITY_GROUP="$(output base .redis_security_group_name)" +REGION="$(jq -r .region < data/$ENVIRONMENT.tfvars)" jq '.username = "'${REDIS_BROKER_USER}'" | .password = "'${REDIS_BROKER_PASSWORD}'" | .elasticache_config.region = "'${REGION}'" | diff --git a/bin/set_redis_secrets.sh b/bin/set_redis_secrets.sh new file mode 100755 index 00000000..da32927e --- /dev/null +++ b/bin/set_redis_secrets.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -euo pipefail + +while getopts e: option; do + case $option in + e) export ENVIRONMENT="$OPTARG";; + esac +done + +: $ENVIRONMENT + +output() { + FILE=$1 + QUERY=$2 + + bin/outputs.sh $FILE | jq -r "$QUERY" +} + +REDIS_USER=$(output base .redis_user_name) +KEY_METADATA=$(aws iam list-access-keys --user-name $REDIS_USER --max-items 1) + +if [[ $(echo $KEY_METADATA | jq '.AccessKeyMetadata[] | length > 0') ]]; then + CREDHUB_KEY=$(bin/credhub_credentials.sh -e $ENVIRONMENT credhub get -n /cf/redis/aws_access_key_id -j | jq -r .value) + AWS_KEY=$(echo $KEY_METADATA | jq -r .AccessKeyMetadata[0].AccessKeyId) + if [ -z $CREDHUB_KEY ] || [ "${CREDHUB_KEY}" != "${AWS_KEY}" ]; then + echo "The secret not in credhub, but it exists in AWS. Please remove this account manually from AWS and rerun." + exit 1 + else + echo "The secret is already in credhub." + fi +else + secrets=$(aws iam create-access-key --user-name $REDIS_USER) + bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /cf/redis/aws_access_key_id --type value --value $(echo $secrets | jq -r .AccessKey.AccessKeyId) + bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /cf/redis/aws_secret_access_key_id --type value --value $(echo $secrets | jq -r .AccessKey.SecretAccessKey) + bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /cf/redis/broker_user --type value --value "redis_admin" + bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /cf/redis/broker_password --type value --value "$(output base .redis_broker_password)" +fi diff --git a/terraform/base/outputs.tf b/terraform/base/outputs.tf index e37b2832..c4a22f78 100644 --- a/terraform/base/outputs.tf +++ b/terraform/base/outputs.tf @@ -304,4 +304,8 @@ output "redis_security_group_name" { output "redis_user_name" { value = "${aws_iam_user.redis.name}" -} \ No newline at end of file +} + +output "redis_broker_password" { + value = "${random_string.redis_broker_password.result}}" +} diff --git a/terraform/base/services-redis.tf b/terraform/base/services-redis.tf index 97787103..d7ab1823 100644 --- a/terraform/base/services-redis.tf +++ b/terraform/base/services-redis.tf @@ -81,3 +81,9 @@ resource "aws_iam_user_policy" "redis" { } EOF } + +resource "random_string" "redis_broker_password" { + length = 26 + special = false +} + From d2a43cd3b512e55adc4b261fd8ffca2e9e534b39 Mon Sep 17 00:00:00 2001 From: Robert Gruber Date: Wed, 12 Sep 2018 16:04:24 +0100 Subject: [PATCH 48/71] Store additional secrets in Credhub [#157117678] --- Makefile | 3 +++ bin/deploy_redis.sh | 20 ++++++++++---------- bin/set_redis_secrets.sh | 5 +++-- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 26cba7e5..0281c817 100644 --- a/Makefile +++ b/Makefile @@ -75,5 +75,8 @@ destroy_bosh: ## Kill off bosh decode_aws_error: ## Decode AWS message @aws sts decode-authorization-message --encoded-message ${DECODE_MESSAGE} | jq -r .DecodedMessage | jq . +set_redis_secrets: ## Set the secrets for Redis broker + @bin/set_redis_secrets.sh + redis: ## Deploy Redis (Elasticache) @bin/deploy_redis.sh diff --git a/bin/deploy_redis.sh b/bin/deploy_redis.sh index e98c14d0..07cc6ef7 100755 --- a/bin/deploy_redis.sh +++ b/bin/deploy_redis.sh @@ -17,29 +17,29 @@ BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" REDIS_SUBMODULE="./elasticache-broker" +MANIFEST=manifest.$$.yml +touch $MANIFEST # TODO provide credentials from credhub bosh int $REDIS_SUBMODULE/manifest.yml \ -o ./operations/redis/credentials.yml \ -v aws_access_key_id="$(bin/credhub_credentials.sh -e $ENVIRONMENT credhub get -n /cf/redis/aws_access_key_id -j | jq -r .value)" \ -v aws_secret_access_key="$(bin/credhub_credentials.sh -e $ENVIRONMENT credhub get -n /cf/redis/aws_secret_access_key_id -j | jq -r .value)" \ - > foo.yml - -cat foo.yml + > $MANIFEST REDIS_CONFIG="elasticache-broker/config.json" touch $REDIS_CONFIG cleanup() { rm -f $REDIS_CONFIG + rm -f $MANIFEST } - trap cleanup EXIT -REDIS_BROKER_USER='test_user' -REDIS_BROKER_PASSWORD='test_password' -REDIS_SERVICE_UUID="test_uuid" -REDIS_SERVICE_NAME="test_redis_service_name" -REDIS_SERVICE_TAG="test_tag" +REDIS_BROKER_USER="$(bin/credhub_credentials.sh -e $ENVIRONMENT credhub get -n /cf/redis/broker_user -j | jq -r .value)" +REDIS_BROKER_PASSWORD="$(bin/credhub_credentials.sh -e $ENVIRONMENT credhub get -n /cf/redis/broker_password -j | jq -r .value)" +REDIS_SERVICE_UUID="$(bin/credhub_credentials.sh -e $ENVIRONMENT credhub get -n /cf/redis/service_uuid -j | jq -r .value)" +REDIS_SERVICE_NAME="$ENVIRONMENT-redis-elasticcashe" +REDIS_SERVICE_TAG="$ENVIRONMENT" # Create redis subnet group REDIS_CACHE_SUBNET_GROUP_NAME="$(output base .redis_cache_subnet_group_name)" REDIS_CACHE_SECURITY_GROUP="$(output base .redis_security_group_name)" @@ -54,4 +54,4 @@ jq '.username = "'${REDIS_BROKER_USER}'" | .elasticache_config.catalog.services[0].plans[].elasticache_properties.cache_security_groups[0]= "'${REDIS_CACHE_SECURITY_GROUP}'" ' \ $REDIS_SUBMODULE/config-sample.json > "$REDIS_CONFIG" -cat $REDIS_CONFIG \ No newline at end of file +# cf push -f $MANIFEST \ No newline at end of file diff --git a/bin/set_redis_secrets.sh b/bin/set_redis_secrets.sh index da32927e..01efed06 100755 --- a/bin/set_redis_secrets.sh +++ b/bin/set_redis_secrets.sh @@ -24,10 +24,10 @@ if [[ $(echo $KEY_METADATA | jq '.AccessKeyMetadata[] | length > 0') ]]; then CREDHUB_KEY=$(bin/credhub_credentials.sh -e $ENVIRONMENT credhub get -n /cf/redis/aws_access_key_id -j | jq -r .value) AWS_KEY=$(echo $KEY_METADATA | jq -r .AccessKeyMetadata[0].AccessKeyId) if [ -z $CREDHUB_KEY ] || [ "${CREDHUB_KEY}" != "${AWS_KEY}" ]; then - echo "The secret not in credhub, but it exists in AWS. Please remove this account manually from AWS and rerun." + echo "AWS credential is not in credhub, but it exists in AWS. Please remove this account manually from AWS and rerun." exit 1 else - echo "The secret is already in credhub." + echo "AWS credential is already in credhub." fi else secrets=$(aws iam create-access-key --user-name $REDIS_USER) @@ -35,4 +35,5 @@ else bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /cf/redis/aws_secret_access_key_id --type value --value $(echo $secrets | jq -r .AccessKey.SecretAccessKey) bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /cf/redis/broker_user --type value --value "redis_admin" bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /cf/redis/broker_password --type value --value "$(output base .redis_broker_password)" + bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /cf/redis/service_uuid --type value --value "$(uuidgen)" fi From 808454b4c5735e5d3652dd2ff58d0880495a83a1 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Wed, 12 Sep 2018 16:10:23 +0100 Subject: [PATCH 49/71] WIP: fixing managed sec groups --- bin/deploy_rabbitmq.sh | 4 +-- bin/stemcells.sh | 2 +- concourse-bosh-deployment | 2 +- terraform/base/bosh.tf | 22 +++++++++++++- terraform/base/jumpbox.tf | 10 +++++++ terraform/base/rabbitmq.tf | 60 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 95 insertions(+), 5 deletions(-) diff --git a/bin/deploy_rabbitmq.sh b/bin/deploy_rabbitmq.sh index ced20a23..c216bf13 100755 --- a/bin/deploy_rabbitmq.sh +++ b/bin/deploy_rabbitmq.sh @@ -25,13 +25,13 @@ APPS_DOMAIN="apps.$(output base .domain)" export BOSH_CA_CERT=$(bosh int --path /default_ca/ca "data/$ENVIRONMENT-bosh-variables.yml") -RABBITMQ_STEMCELL_VERSION=3586.36 +RABBITMQ_STEMCELL_VERSION=3586.40 $BOSH -d rabbitmq deploy -n cf-rabbitmq-multitenant-broker-release/manifests/cf-rabbitmq-broker-template.yml \ -o cf-rabbitmq-multitenant-broker-release/manifests/add-cf-rabbitmq.yml \ -o ./operations/rabbitmq/release.yml \ -v deployment-name=rabbitmq \ - -v stemcell-version=3586.16 \ + -v stemcell-version="'$RABBITMQ_STEMCELL_VERSION'" \ -v system-domain="$SYSTEM_DOMAIN" \ -v apps_domain="$APPS_DOMAIN" \ -v multitenant-rabbitmq-broker-password=broker \ diff --git a/bin/stemcells.sh b/bin/stemcells.sh index 3258dc4c..bcde5b06 100755 --- a/bin/stemcells.sh +++ b/bin/stemcells.sh @@ -14,7 +14,7 @@ output() { IAAS_INFO=aws-xen-hvm CF_STEMCELL_VERSION=$(bosh interpolate cf-deployment/cf-deployment.yml --path=/stemcells/alias=default/version) CONCOURSE_STEMCELL_VERSION=97.15 # FIXME: interpolate from somewhere -RABBITMQ_STEMCELL_VERSION=3586.16 +RABBITMQ_STEMCELL_VERSION=3586.40 BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" if ! $BOSH -n stemcells | grep -q "ubuntu-trusty.*$CF_STEMCELL_VERSION"; then diff --git a/concourse-bosh-deployment b/concourse-bosh-deployment index cae8cb40..70640ac7 160000 --- a/concourse-bosh-deployment +++ b/concourse-bosh-deployment @@ -1 +1 @@ -Subproject commit cae8cb402a470c7c318867e8070d57afcebc8b9c +Subproject commit 70640ac7707bc476f506fe6f6228fdb4422aa14f diff --git a/terraform/base/bosh.tf b/terraform/base/bosh.tf index ead326b4..c6bcd745 100644 --- a/terraform/base/bosh.tf +++ b/terraform/base/bosh.tf @@ -294,4 +294,24 @@ resource "aws_security_group_rule" "managed_to_bosh_udp" { to_port = 65535 source_security_group_id = "${aws_security_group.bosh.id}" description = "Allow managed instance to respond to BOSH" -} \ No newline at end of file +} + +resource "aws_security_group_rule" "bosh_from_managed_tcp" { + security_group_id = "${aws_security_group.bosh.id}" + type = "ingress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + source_security_group_id = "${aws_security_group.bosh_managed.id}" + description = "Allow managed instance to respond to BOSH" +} + +resource "aws_security_group_rule" "bosh_from_managed" { + security_group_id = "${aws_security_group.bosh.id}" + type = "ingress" + protocol = "udp" + from_port = 0 + to_port = 65535 + source_security_group_id = "${aws_security_group.bosh_managed.id}" + description = "Allow managed instance to respond to BOSH" +} diff --git a/terraform/base/jumpbox.tf b/terraform/base/jumpbox.tf index 07c17b5d..a54e84ce 100644 --- a/terraform/base/jumpbox.tf +++ b/terraform/base/jumpbox.tf @@ -48,6 +48,16 @@ resource "aws_security_group_rule" "allow_jumpbox_to_the_world" { description = "Sod it, let jumpbox see the world" } +resource "aws_security_group_rule" "jumpbox_to_managed_ssh" { + security_group_id = "${aws_security_group.bosh_managed.id}" + type = "ingress" + protocol = "tcp" + from_port = 22 + to_port = 22 + source_security_group_id = "${aws_security_group.jumpbox.id}" + description = "Allow Jumpbox to SSH to instance" +} + # SSH resource "tls_private_key" "jumpbox" { algorithm = "RSA" diff --git a/terraform/base/rabbitmq.tf b/terraform/base/rabbitmq.tf index 9d0e5697..16691524 100644 --- a/terraform/base/rabbitmq.tf +++ b/terraform/base/rabbitmq.tf @@ -29,6 +29,36 @@ resource "aws_security_group_rule" "rmq_broker_outbound" { description = "RabbitMQ broker outbound access" } +resource "aws_security_group_rule" "rmq_broker_self_tcp" { + security_group_id = "${aws_security_group.rabbitmq_broker.id}" + type = "ingress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + self = true + description = "RabbitMQ broker self TCP" +} + +resource "aws_security_group_rule" "rmq_broker_self_udp" { + security_group_id = "${aws_security_group.rabbitmq_broker.id}" + type = "ingress" + protocol = "udp" + from_port = 0 + to_port = 65535 + self = true + description = "RabbitMQ broker self UDP" +} + +resource "aws_security_group_rule" "rmq_broker_self_icmp" { + security_group_id = "${aws_security_group.rabbitmq_broker.id}" + type = "ingress" + protocol = "icmp" + from_port = -1 + to_port = -1 + self = true + description = "RabbitMQ broker self ICMP" +} + resource "aws_security_group" "rabbitmq_server" { name = "${var.environment}_rabbitmq_server_security_group" description = "RabbitMQ server access" @@ -40,6 +70,36 @@ resource "aws_security_group" "rabbitmq_server" { } } +resource "aws_security_group_rule" "rmq_server_self_tcp" { + security_group_id = "${aws_security_group.rabbitmq_server.id}" + type = "ingress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + self = true + description = "RabbitMQ broker self TCP" +} + +resource "aws_security_group_rule" "rmq_server_self_udo" { + security_group_id = "${aws_security_group.rabbitmq_server.id}" + type = "ingress" + protocol = "udp" + from_port = 0 + to_port = 65535 + self = true + description = "RabbitMQ broker self UDP" +} + +resource "aws_security_group_rule" "rmq_server_self_icmp" { + security_group_id = "${aws_security_group.rabbitmq_server.id}" + type = "ingress" + protocol = "icmp" + from_port = -1 + to_port = -1 + self = true + description = "RabbitMQ server self ICMP" +} + resource "aws_security_group_rule" "rmq_broker_to_rmq_server" { security_group_id = "${aws_security_group.rabbitmq_server.id}" type = "ingress" From d0dcccc186b9b19dcd78d9c9333f274f7f8ce00e Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Wed, 12 Sep 2018 18:26:08 +0100 Subject: [PATCH 50/71] Install and test RabbitMQ --- bin/deploy_rabbitmq.sh | 28 +++++-- operations/cloud-config/rabbitmq.yml | 2 +- operations/rabbitmq/release.yml | 46 ++++++++-- terraform/base/bosh.tf | 11 +++ terraform/base/rabbitmq.tf | 120 +++++++++++++++------------ 5 files changed, 139 insertions(+), 68 deletions(-) diff --git a/bin/deploy_rabbitmq.sh b/bin/deploy_rabbitmq.sh index c216bf13..829933e9 100755 --- a/bin/deploy_rabbitmq.sh +++ b/bin/deploy_rabbitmq.sh @@ -26,6 +26,8 @@ APPS_DOMAIN="apps.$(output base .domain)" export BOSH_CA_CERT=$(bosh int --path /default_ca/ca "data/$ENVIRONMENT-bosh-variables.yml") RABBITMQ_STEMCELL_VERSION=3586.40 +BROKER_PLAN_UUID=22F0B28C-B886-4123-B01B-95E54D3DE6DA +BROKER_UUID=568725FD-AD46-44CA-9853-621416E983A4 $BOSH -d rabbitmq deploy -n cf-rabbitmq-multitenant-broker-release/manifests/cf-rabbitmq-broker-template.yml \ -o cf-rabbitmq-multitenant-broker-release/manifests/add-cf-rabbitmq.yml \ @@ -34,23 +36,31 @@ $BOSH -d rabbitmq deploy -n cf-rabbitmq-multitenant-broker-release/manifests/cf- -v stemcell-version="'$RABBITMQ_STEMCELL_VERSION'" \ -v system-domain="$SYSTEM_DOMAIN" \ -v apps_domain="$APPS_DOMAIN" \ - -v multitenant-rabbitmq-broker-password=broker \ -v multitenant-rabbitmq-broker-username=broker \ -v product-name=p-rabbitmq \ - -v rabbitmq-management-password=admin \ -v rabbitmq-management-username=admin \ -v rabbitmq-broker-hostname=rabbitmq-multitenant-broker \ -v rabbitmq-broker-password=broker \ - -v rabbitmq-broker-plan-uuid=22F0B28C-B886-4123-B01B-95E54D3DE6DA \ - -v rabbitmq-broker-protocol=http \ + -v rabbitmq-broker-plan-uuid="$BROKER_PLAN_UUID" \ + -v rabbitmq-broker-protocol=https \ -v rabbitmq-broker-username=broker \ - -v rabbitmq-broker-uuid=568725FD-AD46-44CA-9853-621416E983A4 \ + -v rabbitmq-broker-uuid="$BROKER_UUID" \ -v rabbitmq-management-hostname=pivotal-rabbitmq \ - -v cf-admin-password=admin \ -v cf-admin-username=admin \ + -v cf-admin-password=$(bin/cf_password.sh -e "$ENVIRONMENT") \ -v cluster-partition-handling-strategy=autoheal \ - -v disk_alarm_threshold="{mem_relative,'0.4'}" \ -v haproxy-instances=1 \ - -v haproxy-stats-password=admin \ -v haproxy-stats-username=admin \ - -v rabbitmq-hosts=[] \ No newline at end of file + -v rabbitmq-hosts=[] + +# NB: passwords are auto-generated in the deployment manifest - see operations/rabbitmq/release.yml + +$BOSH -d rabbitmq run-errand broker-registrar + +$BOSH -d rabbitmq run-errand smoke-tests + +echo +echo +echo ***************************************************** +echo The RabbitMQ service broker is installed and tested +echo ***************************************************** diff --git a/operations/cloud-config/rabbitmq.yml b/operations/cloud-config/rabbitmq.yml index a314444f..0e8ee4ef 100644 --- a/operations/cloud-config/rabbitmq.yml +++ b/operations/cloud-config/rabbitmq.yml @@ -14,7 +14,7 @@ value: name: n1-highmem-2 cloud_properties: - instance_type: r5.large + instance_type: m4.large security_groups: - ((rabbitmq-server-security-group-id)) - ((bosh-managed-security-group-id)) diff --git a/operations/rabbitmq/release.yml b/operations/rabbitmq/release.yml index 0fc9b692..721d6aab 100644 --- a/operations/rabbitmq/release.yml +++ b/operations/rabbitmq/release.yml @@ -57,6 +57,15 @@ value: [z1, z2, z3] +# FIXME: the pivotal RMQ release gives bogus examples +# - type: replace +# path: /instance_groups/name=rmq/jobs/name=rabbitmq-server/properties/rabbitmq-server/disk_alarm_threshold? +# value: +# mem_relative: 0.4 +- type: remove + path: /instance_groups/name=rmq/jobs/name=rabbitmq-server/properties/rabbitmq-server/disk_alarm_threshold + + # The BOSH release appears not to be cf-rabbitmq-smoke-tests, so change to suit - type: replace path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests/release @@ -65,9 +74,36 @@ - type: remove path: /instance_groups/name=rmq-broker/jobs/name=rabbitmq-broker/properties/rabbitmq-broker/rabbitmq/hosts -# - type: replace -# path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests/properties/cf/apps_domain? -# value: ((apps_domain)) +- type: replace + path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests/properties/cf/apps_domain? + value: ((apps_domain)) + +- type: replace + path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests/properties/rabbitmq?/plan_names? + value: [standard] + +- type: replace + path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests/properties/rabbitmq?/service_name? + value: ((product-name)) + +# - type: remove +# path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests + +- type: replace + path: /variables?/- + value: + name: rabbitmq-management-password + type: password + +- type: replace + path: /variables?/- + value: + name: multitenant-rabbitmq-broker-password + type: password + +- type: replace + path: /variables?/- + value: + name: haproxy-stats-password + type: password -- type: remove - path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests \ No newline at end of file diff --git a/terraform/base/bosh.tf b/terraform/base/bosh.tf index c6bcd745..568b56d5 100644 --- a/terraform/base/bosh.tf +++ b/terraform/base/bosh.tf @@ -306,6 +306,16 @@ resource "aws_security_group_rule" "bosh_from_managed_tcp" { description = "Allow managed instance to respond to BOSH" } +resource "aws_security_group_rule" "s3_and_stuff_from_managed" { + security_group_id = "${aws_security_group.bosh_managed.id}" + type = "egress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + cidr_blocks = ["0.0.0.0/0"] + description = "Allow managed instance to hit S3 for blobs and stuff (FIXME: narrow scope)" +} + resource "aws_security_group_rule" "bosh_from_managed" { security_group_id = "${aws_security_group.bosh.id}" type = "ingress" @@ -315,3 +325,4 @@ resource "aws_security_group_rule" "bosh_from_managed" { source_security_group_id = "${aws_security_group.bosh_managed.id}" description = "Allow managed instance to respond to BOSH" } + diff --git a/terraform/base/rabbitmq.tf b/terraform/base/rabbitmq.tf index 16691524..6c1bfd15 100644 --- a/terraform/base/rabbitmq.tf +++ b/terraform/base/rabbitmq.tf @@ -59,59 +59,18 @@ resource "aws_security_group_rule" "rmq_broker_self_icmp" { description = "RabbitMQ broker self ICMP" } -resource "aws_security_group" "rabbitmq_server" { - name = "${var.environment}_rabbitmq_server_security_group" - description = "RabbitMQ server access" - vpc_id = "${aws_vpc.default.id}" - - tags { - Name = "${var.environment}-rabbitmq-server-security-group" - Environment = "${var.environment}" - } -} - -resource "aws_security_group_rule" "rmq_server_self_tcp" { - security_group_id = "${aws_security_group.rabbitmq_server.id}" +resource "aws_security_group_rule" "rabbitmq_to_cf_nats" { + security_group_id = "${aws_security_group.internal.id}" type = "ingress" protocol = "tcp" - from_port = 0 - to_port = 65535 - self = true - description = "RabbitMQ broker self TCP" -} - -resource "aws_security_group_rule" "rmq_server_self_udo" { - security_group_id = "${aws_security_group.rabbitmq_server.id}" - type = "ingress" - protocol = "udp" - from_port = 0 - to_port = 65535 - self = true - description = "RabbitMQ broker self UDP" -} - -resource "aws_security_group_rule" "rmq_server_self_icmp" { - security_group_id = "${aws_security_group.rabbitmq_server.id}" - type = "ingress" - protocol = "icmp" - from_port = -1 - to_port = -1 - self = true - description = "RabbitMQ server self ICMP" -} - -resource "aws_security_group_rule" "rmq_broker_to_rmq_server" { - security_group_id = "${aws_security_group.rabbitmq_server.id}" - type = "ingress" - protocol = "tcp" - from_port = 0 - to_port = 65535 + from_port = 4222 + to_port = 4222 + description = "Allow rabbitmq broker to access cf nats" source_security_group_id = "${aws_security_group.rabbitmq_broker.id}" - description = "RabbitMQ broker can do everything to the servers" } resource "aws_security_group_rule" "cf_to_rmq_5671_2" { - security_group_id = "${aws_security_group.rabbitmq_server.id}" + security_group_id = "${aws_security_group.rabbitmq_broker.id}" type = "ingress" protocol = "tcp" from_port = 5671 @@ -121,7 +80,7 @@ resource "aws_security_group_rule" "cf_to_rmq_5671_2" { } resource "aws_security_group_rule" "cf_to_rmq_1883" { - security_group_id = "${aws_security_group.rabbitmq_server.id}" + security_group_id = "${aws_security_group.rabbitmq_broker.id}" type = "ingress" protocol = "tcp" from_port = 1883 @@ -131,7 +90,7 @@ resource "aws_security_group_rule" "cf_to_rmq_1883" { } resource "aws_security_group_rule" "cf_to_rmq_8883" { - security_group_id = "${aws_security_group.rabbitmq_server.id}" + security_group_id = "${aws_security_group.rabbitmq_broker.id}" type = "ingress" protocol = "tcp" from_port = 8883 @@ -141,7 +100,7 @@ resource "aws_security_group_rule" "cf_to_rmq_8883" { } resource "aws_security_group_rule" "cf_to_rmq_61613_4" { - security_group_id = "${aws_security_group.rabbitmq_server.id}" + security_group_id = "${aws_security_group.rabbitmq_broker.id}" type = "ingress" protocol = "tcp" from_port = 61613 @@ -151,7 +110,7 @@ resource "aws_security_group_rule" "cf_to_rmq_61613_4" { } resource "aws_security_group_rule" "cf_to_rmq_15672" { - security_group_id = "${aws_security_group.rabbitmq_server.id}" + security_group_id = "${aws_security_group.rabbitmq_broker.id}" type = "ingress" protocol = "tcp" from_port = 15672 @@ -161,11 +120,66 @@ resource "aws_security_group_rule" "cf_to_rmq_15672" { } resource "aws_security_group_rule" "cf_to_rmq_15674" { - security_group_id = "${aws_security_group.rabbitmq_server.id}" + security_group_id = "${aws_security_group.rabbitmq_broker.id}" type = "ingress" protocol = "tcp" from_port = 15674 to_port = 15674 source_security_group_id = "${aws_security_group.internal.id}" description = "CF may talk to RabbitMQ on ports 15674" -} \ No newline at end of file +} + + + +resource "aws_security_group" "rabbitmq_server" { + name = "${var.environment}_rabbitmq_server_security_group" + description = "RabbitMQ server access" + vpc_id = "${aws_vpc.default.id}" + + tags { + Name = "${var.environment}-rabbitmq-server-security-group" + Environment = "${var.environment}" + } +} + +resource "aws_security_group_rule" "rmq_server_self_tcp" { + security_group_id = "${aws_security_group.rabbitmq_server.id}" + type = "ingress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + self = true + description = "RabbitMQ broker self TCP" +} + +resource "aws_security_group_rule" "rmq_server_self_udo" { + security_group_id = "${aws_security_group.rabbitmq_server.id}" + type = "ingress" + protocol = "udp" + from_port = 0 + to_port = 65535 + self = true + description = "RabbitMQ broker self UDP" +} + +resource "aws_security_group_rule" "rmq_server_self_icmp" { + security_group_id = "${aws_security_group.rabbitmq_server.id}" + type = "ingress" + protocol = "icmp" + from_port = -1 + to_port = -1 + self = true + description = "RabbitMQ server self ICMP" +} + +resource "aws_security_group_rule" "rmq_broker_to_rmq_server" { + security_group_id = "${aws_security_group.rabbitmq_server.id}" + type = "ingress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + source_security_group_id = "${aws_security_group.rabbitmq_broker.id}" + description = "RabbitMQ broker can do everything to the servers" +} + + From 739974fef098ffd728f6df6456a08189a6d81822 Mon Sep 17 00:00:00 2001 From: Robert Gruber Date: Thu, 13 Sep 2018 09:22:16 +0100 Subject: [PATCH 51/71] Simplify and WIP [#157117678] --- bin/deploy_redis.sh | 4 ++-- bin/set_redis_secrets.sh | 4 +--- terraform/base/outputs.tf | 4 ---- terraform/base/services-redis.tf | 6 ------ 4 files changed, 3 insertions(+), 15 deletions(-) diff --git a/bin/deploy_redis.sh b/bin/deploy_redis.sh index 07cc6ef7..d0426469 100755 --- a/bin/deploy_redis.sh +++ b/bin/deploy_redis.sh @@ -35,9 +35,9 @@ cleanup() { } trap cleanup EXIT -REDIS_BROKER_USER="$(bin/credhub_credentials.sh -e $ENVIRONMENT credhub get -n /cf/redis/broker_user -j | jq -r .value)" +REDIS_BROKER_USER="admin" REDIS_BROKER_PASSWORD="$(bin/credhub_credentials.sh -e $ENVIRONMENT credhub get -n /cf/redis/broker_password -j | jq -r .value)" -REDIS_SERVICE_UUID="$(bin/credhub_credentials.sh -e $ENVIRONMENT credhub get -n /cf/redis/service_uuid -j | jq -r .value)" +REDIS_SERVICE_UUID="546C4866-99AB-4403-9121-C8D9E95EF10B" REDIS_SERVICE_NAME="$ENVIRONMENT-redis-elasticcashe" REDIS_SERVICE_TAG="$ENVIRONMENT" # Create redis subnet group diff --git a/bin/set_redis_secrets.sh b/bin/set_redis_secrets.sh index 01efed06..b23af128 100755 --- a/bin/set_redis_secrets.sh +++ b/bin/set_redis_secrets.sh @@ -33,7 +33,5 @@ else secrets=$(aws iam create-access-key --user-name $REDIS_USER) bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /cf/redis/aws_access_key_id --type value --value $(echo $secrets | jq -r .AccessKey.AccessKeyId) bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /cf/redis/aws_secret_access_key_id --type value --value $(echo $secrets | jq -r .AccessKey.SecretAccessKey) - bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /cf/redis/broker_user --type value --value "redis_admin" - bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /cf/redis/broker_password --type value --value "$(output base .redis_broker_password)" - bin/credhub_credentials.sh -e $ENVIRONMENT credhub set -n /cf/redis/service_uuid --type value --value "$(uuidgen)" + bin/credhub_credentials.sh -e $ENVIRONMENT credhub generate -n /cf/redis/broker_password --type password fi diff --git a/terraform/base/outputs.tf b/terraform/base/outputs.tf index c4a22f78..3a5ce727 100644 --- a/terraform/base/outputs.tf +++ b/terraform/base/outputs.tf @@ -305,7 +305,3 @@ output "redis_security_group_name" { output "redis_user_name" { value = "${aws_iam_user.redis.name}" } - -output "redis_broker_password" { - value = "${random_string.redis_broker_password.result}}" -} diff --git a/terraform/base/services-redis.tf b/terraform/base/services-redis.tf index d7ab1823..97787103 100644 --- a/terraform/base/services-redis.tf +++ b/terraform/base/services-redis.tf @@ -81,9 +81,3 @@ resource "aws_iam_user_policy" "redis" { } EOF } - -resource "random_string" "redis_broker_password" { - length = 26 - special = false -} - From 2ea267d0ab55c19b97acf0e9f0925439f5c9b418 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Thu, 13 Sep 2018 10:34:41 +0100 Subject: [PATCH 52/71] Create a service user for use by BOSH [#160478328] --- bin/create_bosh.sh | 10 ++-- terraform/base/bosh.tf | 52 +++++++++---------- terraform/base/iam_user.tf | 14 +++++ terraform/base/outputs.tf | 8 +++ terraform/base/templates/bosh_iam_policy.json | 10 ++++ 5 files changed, 62 insertions(+), 32 deletions(-) create mode 100644 terraform/base/iam_user.tf diff --git a/bin/create_bosh.sh b/bin/create_bosh.sh index ef58c1fc..7605b05d 100755 --- a/bin/create_bosh.sh +++ b/bin/create_bosh.sh @@ -3,8 +3,6 @@ set -euo pipefail : $ENVIRONMENT -: $AWS_ACCESS_KEY_ID -: $AWS_SECRET_ACCESS_KEY output() { FILE=$1 @@ -48,11 +46,11 @@ bosh int \ -v external_db_adapter="$(output rds .bosh_db_type)" \ -v external_db_name='bosh' \ -v s3-bucket-name="$(output base .bosh_blobstore_bucket_name)" \ - -v s3-access-key-id="$AWS_ACCESS_KEY_ID" \ - -v s3-secret-access-key="$AWS_SECRET_ACCESS_KEY" \ + -v s3-access-key-id="$(output base .service_user_access_key_id)" \ + -v s3-secret-access-key="$(output base .service_user_secret_access_key)" \ -v s3-region="$(jq -r .region < data/$ENVIRONMENT.tfvars)" \ - -v access_key_id="$AWS_ACCESS_KEY_ID" \ - -v secret_access_key="$AWS_SECRET_ACCESS_KEY" \ + -v access_key_id="$(output base .service_user_access_key_id)" \ + -v secret_access_key="$(output base .service_user_secret_access_key)" \ > data/$ENVIRONMENT-bosh-manifest.yml JUMPBOX_IP=$(output base .jumpbox_public_ip) diff --git a/terraform/base/bosh.tf b/terraform/base/bosh.tf index 568b56d5..0ba67b5e 100644 --- a/terraform/base/bosh.tf +++ b/terraform/base/bosh.tf @@ -40,8 +40,9 @@ data "template_file" "iam_policy" { template = "${file("${path.module}/templates/bosh_iam_policy.json")}" vars { - region = "${var.region}" - account_id = "${local.account_id}" + environment = "${var.environment}" + region = "${var.region}" + account_id = "${local.account_id}" } } @@ -112,7 +113,6 @@ resource "aws_security_group_rule" "bosh_uaa_jumpbox" { source_security_group_id = "${aws_security_group.jumpbox.id}" } - resource "aws_security_group_rule" "bosh_ssh_jumpbox" { security_group_id = "${aws_security_group.bosh.id}" type = "ingress" @@ -150,21 +150,21 @@ resource "aws_security_group_rule" "bosh_director_jumpbox" { } resource "aws_security_group_rule" "bosh_management_tcp" { - security_group_id = "${aws_security_group.bosh.id}" - type = "ingress" - protocol = "tcp" - from_port = 0 - to_port = 65535 - self = true + security_group_id = "${aws_security_group.bosh.id}" + type = "ingress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + self = true } resource "aws_security_group_rule" "bosh_management_udp" { - security_group_id = "${aws_security_group.bosh.id}" - type = "ingress" - protocol = "udp" - from_port = 0 - to_port = 65535 - self = true + security_group_id = "${aws_security_group.bosh.id}" + type = "ingress" + protocol = "udp" + from_port = 0 + to_port = 65535 + self = true } resource "aws_security_group_rule" "jumpbox_uaa_bosh" { @@ -204,6 +204,7 @@ resource "aws_security_group" "bosh_rds" { Environment = "${var.environment}" } } + resource "aws_security_group_rule" "allow_postgres_from_concourse" { security_group_id = "${aws_security_group.bosh_rds.id}" type = "ingress" @@ -236,12 +237,12 @@ resource "aws_security_group_rule" "allow_postgres_from_jumpbox" { # BOSH default key pair resource "tls_private_key" "bosh" { - algorithm = "RSA" - rsa_bits = "2048" + algorithm = "RSA" + rsa_bits = "2048" } resource "aws_key_pair" "bosh" { - key_name = "${var.environment}-bosh" + key_name = "${var.environment}-bosh" public_key = "${tls_private_key.bosh.public_key_openssh}" } @@ -307,13 +308,13 @@ resource "aws_security_group_rule" "bosh_from_managed_tcp" { } resource "aws_security_group_rule" "s3_and_stuff_from_managed" { - security_group_id = "${aws_security_group.bosh_managed.id}" - type = "egress" - protocol = "tcp" - from_port = 0 - to_port = 65535 - cidr_blocks = ["0.0.0.0/0"] - description = "Allow managed instance to hit S3 for blobs and stuff (FIXME: narrow scope)" + security_group_id = "${aws_security_group.bosh_managed.id}" + type = "egress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + cidr_blocks = ["0.0.0.0/0"] + description = "Allow managed instance to hit S3 for blobs and stuff (FIXME: narrow scope)" } resource "aws_security_group_rule" "bosh_from_managed" { @@ -325,4 +326,3 @@ resource "aws_security_group_rule" "bosh_from_managed" { source_security_group_id = "${aws_security_group.bosh_managed.id}" description = "Allow managed instance to respond to BOSH" } - diff --git a/terraform/base/iam_user.tf b/terraform/base/iam_user.tf new file mode 100644 index 00000000..69c4d54f --- /dev/null +++ b/terraform/base/iam_user.tf @@ -0,0 +1,14 @@ +resource "aws_iam_user" "service_user" { + name = "${var.environment}-service-user" +} + +resource "aws_iam_user_policy" "service_user_policy" { + name = "${var.environment}-service-user-policy" + user = "${aws_iam_user.service_user.name}" + + policy = "${data.template_file.iam_policy.rendered}" +} + +resource "aws_iam_access_key" "service_user" { + user = "${aws_iam_user.service_user.name}" +} \ No newline at end of file diff --git a/terraform/base/outputs.tf b/terraform/base/outputs.tf index 3a5ce727..0267235c 100644 --- a/terraform/base/outputs.tf +++ b/terraform/base/outputs.tf @@ -15,6 +15,14 @@ output "domain" { value = "${local.domain}" } +output "service_user_access_key_id" { + value = "${aws_iam_access_key.service_user.id}" +} + +output "service_user_secret_access_key" { + value = "${aws_iam_access_key.service_user.secret}" +} + output "bosh_rds_security_group_id" { value = "${aws_security_group.bosh_rds.id}" } diff --git a/terraform/base/templates/bosh_iam_policy.json b/terraform/base/templates/bosh_iam_policy.json index c924b78b..06ffa37f 100644 --- a/terraform/base/templates/bosh_iam_policy.json +++ b/terraform/base/templates/bosh_iam_policy.json @@ -61,6 +61,16 @@ "Resource": [ "*" ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:*" + ], + "Resource": [ + "arn:aws:s3:::ons-paas-${environment}-*", + "arn:aws:s3:::ons-paas-${environment}-*/*" + ] } ] } \ No newline at end of file From 1d023525bcad4db2e7a334520f1f70e36bda84ef Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Thu, 13 Sep 2018 10:59:14 +0100 Subject: [PATCH 53/71] Restrict bosh_managed SG access to BOSH --- bin/deploy_rabbitmq.sh | 6 +++--- terraform/base/bosh.tf | 32 ++++++-------------------------- 2 files changed, 9 insertions(+), 29 deletions(-) diff --git a/bin/deploy_rabbitmq.sh b/bin/deploy_rabbitmq.sh index 829933e9..f51f5884 100755 --- a/bin/deploy_rabbitmq.sh +++ b/bin/deploy_rabbitmq.sh @@ -61,6 +61,6 @@ $BOSH -d rabbitmq run-errand smoke-tests echo echo -echo ***************************************************** -echo The RabbitMQ service broker is installed and tested -echo ***************************************************** +echo '*****************************************************' +echo 'The RabbitMQ service broker is installed and tested' +echo '*****************************************************' diff --git a/terraform/base/bosh.tf b/terraform/base/bosh.tf index 0ba67b5e..bb84f1ed 100644 --- a/terraform/base/bosh.tf +++ b/terraform/base/bosh.tf @@ -267,42 +267,22 @@ resource "aws_security_group_rule" "bosh_to_managed_ssh" { description = "Allow BOSH to SSH to instance" } -resource "aws_security_group_rule" "bosh_to_managed_mbus" { - security_group_id = "${aws_security_group.bosh_managed.id}" - type = "ingress" - protocol = "tcp" - from_port = 6868 - to_port = 6868 - source_security_group_id = "${aws_security_group.bosh.id}" - description = "Allow BOSH to access the BOSH agent on instance" -} - -resource "aws_security_group_rule" "managed_to_bosh_tcp" { +resource "aws_security_group_rule" "managed_to_bosh_4222" { security_group_id = "${aws_security_group.bosh_managed.id}" type = "egress" protocol = "tcp" - from_port = 0 - to_port = 65535 + from_port = 4222 + to_port = 4222 source_security_group_id = "${aws_security_group.bosh.id}" description = "Allow managed instance to respond to BOSH" } -resource "aws_security_group_rule" "managed_to_bosh_udp" { - security_group_id = "${aws_security_group.bosh_managed.id}" - type = "egress" - protocol = "udp" - from_port = 0 - to_port = 65535 - source_security_group_id = "${aws_security_group.bosh.id}" - description = "Allow managed instance to respond to BOSH" -} - -resource "aws_security_group_rule" "bosh_from_managed_tcp" { +resource "aws_security_group_rule" "bosh_from_managed_4222" { security_group_id = "${aws_security_group.bosh.id}" type = "ingress" protocol = "tcp" - from_port = 0 - to_port = 65535 + from_port = 4222 + to_port = 4222 source_security_group_id = "${aws_security_group.bosh_managed.id}" description = "Allow managed instance to respond to BOSH" } From d75f549b0949a1401b8bfc61ec66bc51dbd200a9 Mon Sep 17 00:00:00 2001 From: Robert Gruber Date: Thu, 13 Sep 2018 11:17:03 +0100 Subject: [PATCH 54/71] Fix script names Update Readme [#157117678] --- README.md | 2 ++ bin/get_states.sh | 4 +++- bin/rds.sh | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2b6f347a..a2c3fd2e 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ We use the code in this repository to bootstrap our AWS PaaS environment. The no - CF Management CLIs - Credhub CLI - UAA CLI +- PSQL client +- MySQL client - Fly [CLI](https://concourse-ci.org/download.html) - [yq](https://github.com/mikefarah/yq) (or, `brew install yq`) - [jq](https://stedolan.github.io/jq/) (or, `brew install jq`) diff --git a/bin/get_states.sh b/bin/get_states.sh index 206e478f..759d70e5 100755 --- a/bin/get_states.sh +++ b/bin/get_states.sh @@ -26,4 +26,6 @@ else [ "$EXISTS" = true -a -f data/$FILE ] && continue aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/$FILE" "data/" --acl=private done -fi \ No newline at end of file +fi + +exit 0 \ No newline at end of file diff --git a/bin/rds.sh b/bin/rds.sh index e260e15a..d426b14e 100755 --- a/bin/rds.sh +++ b/bin/rds.sh @@ -8,7 +8,7 @@ COMMAND=${1:-plan} OPTS= [ "$COMMAND" = plan ] || OPTS=-auto-approve -bin/get_states -e $ENVIRONMENT -f $ENVIRONMENT.tfvars -f $ENVIRONMENT-rds.tfstate +bin/get_states.sh -e $ENVIRONMENT -o -f $ENVIRONMENT.tfvars -f $ENVIRONMENT-rds.tfstate terraform init terraform/rds @@ -18,4 +18,4 @@ terraform $COMMAND \ -var-file=<(bin/outputs.sh base) \ -state="data/$ENVIRONMENT-rds.tfstate" terraform/rds/ - bin/persist_states -e $ENVIRONMENT -f $ENVIRONMENT-rds.tfstate \ No newline at end of file +bin/persist_states.sh -e $ENVIRONMENT -f $ENVIRONMENT-rds.tfstate \ No newline at end of file From 594ce76282f15ff6922b96fe2df90eb3f03d1a9b Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Thu, 13 Sep 2018 11:21:49 +0100 Subject: [PATCH 55/71] Shift concourse to managed_bosh SG [#158185683] --- operations/cloud-config/concourse.yml | 4 +- terraform/base/aws.tf | 2 + terraform/base/bosh.tf | 29 +++++++---- terraform/base/concourse.tf | 72 --------------------------- 4 files changed, 24 insertions(+), 83 deletions(-) diff --git a/operations/cloud-config/concourse.yml b/operations/cloud-config/concourse.yml index f4fa6975..cd2fa16d 100644 --- a/operations/cloud-config/concourse.yml +++ b/operations/cloud-config/concourse.yml @@ -14,7 +14,9 @@ reserved: [((reserved_concourse_az1_cidr))] cloud_properties: subnet: ((concourse_subnet_az1_id)) - security_groups: [ ((concourse_security_group))] + security_groups: + - ((concourse_security_group)) + - ((bosh-managed-security-group-id)) - type: replace path: /vm_types/- diff --git a/terraform/base/aws.tf b/terraform/base/aws.tf index 0977ba55..30a23ddf 100644 --- a/terraform/base/aws.tf +++ b/terraform/base/aws.tf @@ -1,3 +1,5 @@ provider "aws" { region = "${var.region}" } + +provider "random" {} diff --git a/terraform/base/bosh.tf b/terraform/base/bosh.tf index bb84f1ed..67efce4d 100644 --- a/terraform/base/bosh.tf +++ b/terraform/base/bosh.tf @@ -287,6 +287,25 @@ resource "aws_security_group_rule" "bosh_from_managed_4222" { description = "Allow managed instance to respond to BOSH" } +resource "aws_security_group_rule" "managed_to_bosh_25777" { + security_group_id = "${aws_security_group.bosh_managed.id}" + type = "egress" + protocol = "tcp" + from_port = 25777 + to_port = 25777 + source_security_group_id = "${aws_security_group.bosh.id}" + description = "Allow managed instance to respond to BOSH" +} +resource "aws_security_group_rule" "bosh_from_managed_25777" { + security_group_id = "${aws_security_group.bosh.id}" + type = "ingress" + protocol = "tcp" + from_port = 25777 + to_port = 25777 + source_security_group_id = "${aws_security_group.bosh_managed.id}" + description = "Allow managed instance to respond to BOSH" +} + resource "aws_security_group_rule" "s3_and_stuff_from_managed" { security_group_id = "${aws_security_group.bosh_managed.id}" type = "egress" @@ -296,13 +315,3 @@ resource "aws_security_group_rule" "s3_and_stuff_from_managed" { cidr_blocks = ["0.0.0.0/0"] description = "Allow managed instance to hit S3 for blobs and stuff (FIXME: narrow scope)" } - -resource "aws_security_group_rule" "bosh_from_managed" { - security_group_id = "${aws_security_group.bosh.id}" - type = "ingress" - protocol = "udp" - from_port = 0 - to_port = 65535 - source_security_group_id = "${aws_security_group.bosh_managed.id}" - description = "Allow managed instance to respond to BOSH" -} diff --git a/terraform/base/concourse.tf b/terraform/base/concourse.tf index b704d6b5..2eebeab4 100644 --- a/terraform/base/concourse.tf +++ b/terraform/base/concourse.tf @@ -29,78 +29,6 @@ resource "aws_security_group_rule" "concourse_bosh_director" { source_security_group_id = "${aws_security_group.concourse.id}" } -resource "aws_security_group_rule" "concourse_from_bosh_rule_tcp_ssh" { - security_group_id = "${aws_security_group.concourse.id}" - type = "ingress" - protocol = "tcp" - from_port = 22 - to_port = 22 - source_security_group_id = "${aws_security_group.bosh.id}" -} - -resource "aws_security_group_rule" "concourse_from_bosh_rule_tcp_bosh_agent" { - security_group_id = "${aws_security_group.concourse.id}" - type = "ingress" - protocol = "tcp" - from_port = 6868 - to_port = 6868 - source_security_group_id = "${aws_security_group.bosh.id}" -} - -resource "aws_security_group_rule" "concourse_from_jumpbox_rule_tcp_ssh" { - security_group_id = "${aws_security_group.concourse.id}" - type = "ingress" - protocol = "tcp" - from_port = 22 - to_port = 22 - source_security_group_id = "${aws_security_group.jumpbox.id}" -} - -resource "aws_security_group_rule" "bosh_ssh_concourse" { - security_group_id = "${aws_security_group.bosh.id}" - type = "egress" - protocol = "tcp" - from_port = 22 - to_port = 22 - source_security_group_id = "${aws_security_group.concourse.id}" -} - -resource "aws_security_group_rule" "bosh_mbus_concourse" { - security_group_id = "${aws_security_group.bosh.id}" - type = "egress" - protocol = "tcp" - from_port = 6868 - to_port = 6868 - source_security_group_id = "${aws_security_group.concourse.id}" -} - -resource "aws_security_group_rule" "bosh_tcp_from_concourse" { - security_group_id = "${aws_security_group.bosh.id}" - type = "ingress" - protocol = "tcp" - from_port = 0 - to_port = 65535 - source_security_group_id = "${aws_security_group.concourse.id}" -} - -resource "aws_security_group_rule" "bosh_udp_from_concourse" { - security_group_id = "${aws_security_group.bosh.id}" - type = "ingress" - protocol = "udp" - from_port = 0 - to_port = 65535 - source_security_group_id = "${aws_security_group.concourse.id}" -} - -resource "aws_security_group_rule" "jumpbox_ssh_concourse" { - security_group_id = "${aws_security_group.jumpbox.id}" - type = "egress" - protocol = "tcp" - from_port = 22 - to_port = 22 - source_security_group_id = "${aws_security_group.concourse.id}" -} - resource "aws_security_group_rule" "concourse_outbound" { security_group_id = "${aws_security_group.concourse.id}" type = "egress" From 16b0351b1b6985cc35ea0b870aed8bf846b58210 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Thu, 13 Sep 2018 11:33:03 +0100 Subject: [PATCH 56/71] Tighten prometheus -> bosh SG rules [#158185683] --- operations/cloud-config/prometheus.yml | 4 +- terraform/base/prometheus.tf | 72 -------------------------- 2 files changed, 3 insertions(+), 73 deletions(-) diff --git a/operations/cloud-config/prometheus.yml b/operations/cloud-config/prometheus.yml index 19f0d066..bd55edd7 100644 --- a/operations/cloud-config/prometheus.yml +++ b/operations/cloud-config/prometheus.yml @@ -21,7 +21,9 @@ reserved: [((reserved_prometheus_az1_cidr))] cloud_properties: subnet: ((prometheus_subnet_az1_id)) - security_groups: [ ((prometheus_security_group))] + security_groups: + - ((prometheus_security_group)) + - ((bosh-managed-security-group-id)) - type: replace path: /vm_extensions/- diff --git a/terraform/base/prometheus.tf b/terraform/base/prometheus.tf index 4e508845..0a499d15 100644 --- a/terraform/base/prometheus.tf +++ b/terraform/base/prometheus.tf @@ -316,78 +316,6 @@ resource "aws_security_group_rule" "prometheus_rule_icmp" { self = true } -resource "aws_security_group_rule" "prometheus_from_bosh_rule_tcp_ssh" { - security_group_id = "${aws_security_group.prometheus.id}" - type = "ingress" - protocol = "tcp" - from_port = 22 - to_port = 22 - source_security_group_id = "${aws_security_group.bosh.id}" -} - -resource "aws_security_group_rule" "prometheus_from_bosh_rule_tcp_bosh_agent" { - security_group_id = "${aws_security_group.prometheus.id}" - type = "ingress" - protocol = "tcp" - from_port = 6868 - to_port = 6868 - source_security_group_id = "${aws_security_group.bosh.id}" -} - -resource "aws_security_group_rule" "prometheus_from_jumpbox_rule_tcp_ssh" { - security_group_id = "${aws_security_group.prometheus.id}" - type = "ingress" - protocol = "tcp" - from_port = 22 - to_port = 22 - source_security_group_id = "${aws_security_group.jumpbox.id}" -} - -resource "aws_security_group_rule" "bosh_ssh_prometheus" { - security_group_id = "${aws_security_group.bosh.id}" - type = "egress" - protocol = "tcp" - from_port = 22 - to_port = 22 - source_security_group_id = "${aws_security_group.prometheus.id}" -} - -resource "aws_security_group_rule" "bosh_mbus_prometheus" { - security_group_id = "${aws_security_group.bosh.id}" - type = "egress" - protocol = "tcp" - from_port = 6868 - to_port = 6868 - source_security_group_id = "${aws_security_group.prometheus.id}" -} - -resource "aws_security_group_rule" "bosh_tcp_from_prometheus" { - security_group_id = "${aws_security_group.bosh.id}" - type = "ingress" - protocol = "tcp" - from_port = 0 - to_port = 65535 - source_security_group_id = "${aws_security_group.prometheus.id}" -} - -resource "aws_security_group_rule" "bosh_udp_from_prometheus" { - security_group_id = "${aws_security_group.bosh.id}" - type = "ingress" - protocol = "udp" - from_port = 0 - to_port = 65535 - source_security_group_id = "${aws_security_group.prometheus.id}" -} - -resource "aws_security_group_rule" "jumpbox_ssh_prometheus" { - security_group_id = "${aws_security_group.jumpbox.id}" - type = "egress" - protocol = "tcp" - from_port = 22 - to_port = 22 - source_security_group_id = "${aws_security_group.prometheus.id}" -} - resource "aws_security_group" "prometheus_alb" { name = "${var.environment}_prometheus_alb_security_group" description = "Prometheus web access" From 10a69051a0ae73e3f000e01f7a77650f7c341e99 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Thu, 13 Sep 2018 12:33:50 +0100 Subject: [PATCH 57/71] Tighten CF -> BOSH rules [#158185683] --- cloud-config/cf.yml | 17 +++-- operations/cloud-config/cf-rds-sec-group.yml | 6 +- .../cloud-config/cf-scheduler-extensions.yml | 4 +- operations/cloud-config/router-extensions.yml | 5 +- terraform/base/cf.tf | 72 ------------------- 5 files changed, 22 insertions(+), 82 deletions(-) diff --git a/cloud-config/cf.yml b/cloud-config/cf.yml index 95f67343..9c1a1fb1 100644 --- a/cloud-config/cf.yml +++ b/cloud-config/cf.yml @@ -31,8 +31,9 @@ vm_types: cloud_properties: instance_type: c4.xlarge ephemeral_disk: {size: 50_000, type: gp2} - security_groups: [ ((internal_security_group))] - + security_groups: + - ((internal_security_group)) + - ((bosh-managed-security-group-id)) disk_types: - name: default disk_size: 3000 @@ -75,7 +76,9 @@ networks: reserved: [((reserved_az1_cidr))] cloud_properties: subnet: ((private_subnet_az1_id)) - security_groups: [ ((internal_security_group))] + security_groups: + - ((internal_security_group)) + - ((bosh-managed-security-group-id)) - range: ((private_subnet_az2_cidr)) gateway: ((private_subnet_az2_gateway)) az: z2 @@ -85,7 +88,9 @@ networks: reserved: [((reserved_az2_cidr))] cloud_properties: subnet: ((private_subnet_az2_id)) - security_groups: [ ((internal_security_group))] + security_groups: + - ((internal_security_group)) + - ((bosh-managed-security-group-id)) - range: ((private_subnet_az3_cidr)) gateway: ((private_subnet_az3_gateway)) az: z3 @@ -95,7 +100,9 @@ networks: reserved: [((reserved_az3_cidr))] cloud_properties: subnet: ((private_subnet_az3_id)) - security_groups: [ ((internal_security_group))] + security_groups: + - ((internal_security_group)) + - ((bosh-managed-security-group-id)) - name: services type: manual subnets: diff --git a/operations/cloud-config/cf-rds-sec-group.yml b/operations/cloud-config/cf-rds-sec-group.yml index ed266da2..cc3ed112 100644 --- a/operations/cloud-config/cf-rds-sec-group.yml +++ b/operations/cloud-config/cf-rds-sec-group.yml @@ -4,8 +4,10 @@ name: cf-rds-sec-group cloud_properties: security_groups: - - ((cf-rds-client-security-group)) - - ((internal_security_group)) + - ((cf-rds-client-security-group)) + - ((internal_security_group)) + - ((bosh-managed-security-group-id)) + # Note: requires cf-scheduler-sec-groiup to exist, which implies ordering - type: replace diff --git a/operations/cloud-config/cf-scheduler-extensions.yml b/operations/cloud-config/cf-scheduler-extensions.yml index 0fc57380..98f8ea43 100644 --- a/operations/cloud-config/cf-scheduler-extensions.yml +++ b/operations/cloud-config/cf-scheduler-extensions.yml @@ -5,7 +5,7 @@ name: diego-ssh-proxy-network-properties cloud_properties: elbs: - - ((cf-ssh-lb)) + - ((cf-ssh-lb)) - type: replace path: /vm_extensions/- @@ -15,3 +15,5 @@ security_groups: - ((internal_security_group)) - ((cf-ssh-internal)) + - ((bosh-managed-security-group-id)) + diff --git a/operations/cloud-config/router-extensions.yml b/operations/cloud-config/router-extensions.yml index 52932b17..62487111 100644 --- a/operations/cloud-config/router-extensions.yml +++ b/operations/cloud-config/router-extensions.yml @@ -12,5 +12,6 @@ name: cf-router-sec-group cloud_properties: security_groups: - - ((cf-router-lb-internal-security-group-id)) - - ((cf-internal-security-group-id)) \ No newline at end of file + - ((cf-router-lb-internal-security-group-id)) + - ((cf-internal-security-group-id)) + - ((bosh-managed-security-group-id)) diff --git a/terraform/base/cf.tf b/terraform/base/cf.tf index b47cba19..b934d4b3 100644 --- a/terraform/base/cf.tf +++ b/terraform/base/cf.tf @@ -255,78 +255,6 @@ resource "aws_security_group_rule" "internal_rule_allow_internet" { cidr_blocks = ["0.0.0.0/0"] } -resource "aws_security_group_rule" "cf_from_bosh_rule_tcp_ssh" { - security_group_id = "${aws_security_group.internal.id}" - type = "ingress" - protocol = "tcp" - from_port = 22 - to_port = 22 - source_security_group_id = "${aws_security_group.bosh.id}" -} - -resource "aws_security_group_rule" "cf_from_bosh_rule_tcp_bosh_agent" { - security_group_id = "${aws_security_group.internal.id}" - type = "ingress" - protocol = "tcp" - from_port = 6868 - to_port = 6868 - source_security_group_id = "${aws_security_group.bosh.id}" -} - -resource "aws_security_group_rule" "cf_from_jumpbox_rule_tcp_ssh" { - security_group_id = "${aws_security_group.internal.id}" - type = "ingress" - protocol = "tcp" - from_port = 22 - to_port = 22 - source_security_group_id = "${aws_security_group.jumpbox.id}" -} - -resource "aws_security_group_rule" "bosh_ssh_cf" { - security_group_id = "${aws_security_group.bosh.id}" - type = "egress" - protocol = "tcp" - from_port = 22 - to_port = 22 - source_security_group_id = "${aws_security_group.internal.id}" -} - -resource "aws_security_group_rule" "bosh_mbus_cf" { - security_group_id = "${aws_security_group.bosh.id}" - type = "egress" - protocol = "tcp" - from_port = 6868 - to_port = 6868 - source_security_group_id = "${aws_security_group.internal.id}" -} - -resource "aws_security_group_rule" "bosh_tcp_from_cf" { - security_group_id = "${aws_security_group.bosh.id}" - type = "ingress" - protocol = "tcp" - from_port = 0 - to_port = 65535 - source_security_group_id = "${aws_security_group.internal.id}" -} - -resource "aws_security_group_rule" "bosh_udp_from_cf" { - security_group_id = "${aws_security_group.bosh.id}" - type = "ingress" - protocol = "udp" - from_port = 0 - to_port = 65535 - source_security_group_id = "${aws_security_group.internal.id}" -} - -resource "aws_security_group_rule" "jumpbox_ssh_cf" { - security_group_id = "${aws_security_group.jumpbox.id}" - type = "egress" - protocol = "tcp" - from_port = 22 - to_port = 22 - source_security_group_id = "${aws_security_group.internal.id}" -} - resource "aws_security_group" "cf_router_lb_internal_security_group" { name = "${var.environment}_cf_router_lb_internal_security_group" description = "CF Router Internal" From 45afd9c092c519b25eb39f9d3e807b80bcc0a889 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Thu, 13 Sep 2018 12:40:19 +0100 Subject: [PATCH 58/71] Add destroy make targets --- Makefile | 20 +++++++++++++++++--- bin/destroy_deployment.sh | 14 ++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) create mode 100755 bin/destroy_deployment.sh diff --git a/Makefile b/Makefile index 0281c817..16f2113a 100644 --- a/Makefile +++ b/Makefile @@ -23,9 +23,6 @@ terraform_init: ## initialize terraform provider terraform_plan: ## plan what would be applied if we ran make terraform @bin/terraform.sh plan -destroy_terraform: ## destroy main terraform environment - @bin/terraform.sh destroy - outputs: ## base terraform outputs @bin/outputs.sh @@ -68,10 +65,27 @@ set_concourse_secrets: ## Set the secrets that Concourse needs to run its pipeli pipelines: login_fly ## Deploy pipelines @ci/cf_pipeline.sh +destroy_concourse: ## Delete the concourse deployment + @bin/destroy_deployment.sh concourse + +destroy_rabbitmq: ## Delete the rabbitmq deployment + @bin/destroy_deployment.sh rabbitmq + +destroy_prometheus: ## Delete the prometheus deployment + @bin/destroy_deployment.sh prometheus + +destroy_cf: ## Delete the CF deployment + @bin/destroy_deployment.sh cf destroy_bosh: ## Kill off bosh @bin/destroy_bosh.sh +destroy_rds: ## destroy the RDS instances + @bin/rds.sh destroy + +destroy_terraform: ## destroy main terraform environment + @bin/terraform.sh destroy + decode_aws_error: ## Decode AWS message @aws sts decode-authorization-message --encoded-message ${DECODE_MESSAGE} | jq -r .DecodedMessage | jq . diff --git a/bin/destroy_deployment.sh b/bin/destroy_deployment.sh new file mode 100755 index 00000000..fd2f3506 --- /dev/null +++ b/bin/destroy_deployment.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -euo pipefail + +: $ENVIRONMENT + +DEPLOYMENT=$1 + +bin/get_states.sh -e $ENVIRONMENT + +BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" + +echo "Deploying" +$BOSH delete-deployment -n -d $DEPLOYMENT \ No newline at end of file From 99c0bd917b8c250ce9bd225029e04f5e550de008 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Thu, 13 Sep 2018 12:51:04 +0100 Subject: [PATCH 59/71] Remove unneeded node_exporter ingress rules [#158185683] --- terraform/base/prometheus.tf | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/terraform/base/prometheus.tf b/terraform/base/prometheus.tf index 0a499d15..11528573 100644 --- a/terraform/base/prometheus.tf +++ b/terraform/base/prometheus.tf @@ -218,46 +218,37 @@ resource "aws_security_group" "prometheus" { } } -resource "aws_security_group_rule" "prometheus_bosh_node_exporters" { +resource "aws_security_group_rule" "prometheus_bosh_data_exporters" { security_group_id = "${aws_security_group.bosh.id}" type = "ingress" protocol = "tcp" from_port = 9190 to_port = 9190 - description = "Allow prometheus to access bosh node exporter" + description = "Allow prometheus to access bosh data exporter" source_security_group_id = "${aws_security_group.prometheus.id}" } - -resource "aws_security_group_rule" "managed_node_exporters" { - security_group_id = "${aws_security_group.bosh_managed.id}" +resource "aws_security_group_rule" "prometheus_bosh_node_exporters" { + security_group_id = "${aws_security_group.bosh.id}" type = "ingress" protocol = "tcp" from_port = 9100 to_port = 9100 + description = "Allow prometheus to access bosh node exporter" source_security_group_id = "${aws_security_group.prometheus.id}" - description = "Allow BOSH to access the BOSH agent on instance" } -resource "aws_security_group_rule" "prometheus_cf_node_exporters" { - security_group_id = "${aws_security_group.internal.id}" +resource "aws_security_group_rule" "managed_node_exporters" { + security_group_id = "${aws_security_group.bosh_managed.id}" type = "ingress" protocol = "tcp" from_port = 9100 to_port = 9100 - description = "Allow prometheus to access cf node exporter" source_security_group_id = "${aws_security_group.prometheus.id}" + description = "Allow BOSH to access the BOSH agent on instance" } -resource "aws_security_group_rule" "prometheus_concourse_node_exporters" { - security_group_id = "${aws_security_group.concourse.id}" - type = "ingress" - protocol = "tcp" - from_port = 9100 - to_port = 9100 - description = "Allow prometheus to access concourse node exporter" - source_security_group_id = "${aws_security_group.prometheus.id}" -} + resource "aws_security_group_rule" "prometheus_cf_nats" { security_group_id = "${aws_security_group.internal.id}" From 47e552249e8fc8376cd17746f6a98e212df2e4de Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Thu, 13 Sep 2018 13:31:29 +0100 Subject: [PATCH 60/71] Add UAA ops files to prometheus and bosh to allow prometheus to get BOSH data [#160480416] --- bin/create_bosh.sh | 3 ++- bin/deploy_prometheus.sh | 7 +++++++ operations/bosh/concourse-client.yml | 1 - terraform/base/prometheus.tf | 10 ++++++++++ 4 files changed, 19 insertions(+), 2 deletions(-) delete mode 100644 operations/bosh/concourse-client.yml diff --git a/bin/create_bosh.sh b/bin/create_bosh.sh index 7605b05d..a9ee2f6c 100755 --- a/bin/create_bosh.sh +++ b/bin/create_bosh.sh @@ -25,6 +25,7 @@ bosh int \ -o operations/bosh/certificate.yml \ -o operations/bosh/external-uaa-db.yml \ -o operations/bosh/external-credhub-db.yml \ + -o prometheus-boshrelease/manifests/operators/bosh/add-bosh-exporter-uaa-clients.yml \ -v director_name=bosh \ -v internal_cidr="$(output base .bosh_subnet_cidr_block)" \ -v internal_gw="$(output base .bosh_gateway_ip)" \ @@ -69,4 +70,4 @@ bosh create-env \ data/$ENVIRONMENT-bosh-manifest.yml \ --state data/$ENVIRONMENT-bosh-state.json - bin/persist_states.sh -e $ENVIRONMENT -f $ENVIRONMENT-bosh-manifest.yml -f $ENVIRONMENT-bosh-variables.yml -f $ENVIRONMENT-bosh-state.json \ No newline at end of file + bin/persist_states.sh -e $ENVIRONMENT -f $ENVIRONMENT-bosh-manifest.yml -f $ENVIRONMENT-bosh-variables.yml -f $ENVIRONMENT-bosh-state.json diff --git a/bin/deploy_prometheus.sh b/bin/deploy_prometheus.sh index 372ccbd8..737863be 100755 --- a/bin/deploy_prometheus.sh +++ b/bin/deploy_prometheus.sh @@ -31,9 +31,16 @@ cleanup() { } trap cleanup EXIT + +bin/credhub_credentials.sh -e $ENVIRONMENT \ + credhub set -n /bosh/prometheus/uaa_bosh_exporter_client_secret \ + --type value \ + --value "$(bosh int --path=/uaa_bosh_exporter_client_secret data/$ENVIRONMENT-bosh-variables.yml)" + $BOSH -d prometheus deploy -n prometheus-boshrelease/manifests/prometheus.yml \ -o prometheus-boshrelease/manifests/operators/monitor-bosh.yml \ -o prometheus-boshrelease/manifests/operators/monitor-cf.yml \ + -o prometheus-boshrelease/manifests/operators/enable-bosh-uaa.yml \ -o ./operations/prometheus/networks.yml \ -v bosh_url="$(output base .bosh_director_fqdn)" \ -v bosh_username="admin" \ diff --git a/operations/bosh/concourse-client.yml b/operations/bosh/concourse-client.yml deleted file mode 100644 index ed97d539..00000000 --- a/operations/bosh/concourse-client.yml +++ /dev/null @@ -1 +0,0 @@ ---- diff --git a/terraform/base/prometheus.tf b/terraform/base/prometheus.tf index 11528573..bbbb77e1 100644 --- a/terraform/base/prometheus.tf +++ b/terraform/base/prometheus.tf @@ -270,6 +270,16 @@ resource "aws_security_group_rule" "prometheus_bosh_director" { source_security_group_id = "${aws_security_group.prometheus.id}" } +resource "aws_security_group_rule" "prometheus_bosh_uaa" { + security_group_id = "${aws_security_group.bosh.id}" + type = "ingress" + protocol = "tcp" + from_port = 8443 + to_port = 8443 + description = "Allow prometheus to access bosh UAA" + source_security_group_id = "${aws_security_group.prometheus.id}" +} + resource "aws_security_group_rule" "prometheus_outbound" { security_group_id = "${aws_security_group.prometheus.id}" type = "egress" From acbdc09e942ece903c8347e56b7aab0e933ca359 Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Thu, 13 Sep 2018 13:59:49 +0100 Subject: [PATCH 61/71] Add rabbitmq version to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a2c3fd2e..ad0498a0 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,7 @@ And use the displayed output to point the browser at the desired Prometheus comp | cf | v4.2.0 | | bosh | v1.2.0 | | prometheus | v23.2.0 | +| rabbitmq | v37.0.0 | ## LICENCE From e6a624ecf346ef075282ece56f2edcfdd585abf5 Mon Sep 17 00:00:00 2001 From: Robert Gruber Date: Thu, 13 Sep 2018 14:07:37 +0100 Subject: [PATCH 62/71] Add target to update submodules --- Makefile | 3 +++ README.md | 4 ++-- bin/update_submodules.sh | 30 ++++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) create mode 100755 bin/update_submodules.sh diff --git a/Makefile b/Makefile index 0281c817..ae6a8454 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,9 @@ VAR_FILE = ${ENVIRONMENT}_vpc.tfvars help: @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' +submodules: ## checkout the right version of git submodules + @bin/update_submodules.sh + terraform: ## create main terraform environment @bin/terraform.sh apply diff --git a/README.md b/README.md index a2c3fd2e..a7138eb3 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ We use the code in this repository to bootstrap our AWS PaaS environment. The no 1. Update submodules ```sh - git submodule update --init + make submodules ``` 2. Create your environment directory and vars file @@ -156,7 +156,7 @@ And use the displayed output to point the browser at the desired Prometheus comp | ----------- | ------- | | concourse | v4.1.0 | | cf | v4.2.0 | -| bosh | v1.2.0 | +| bosh | 7375c31d59018203911da3881334f09d8e70deb5 | | prometheus | v23.2.0 | ## LICENCE diff --git a/bin/update_submodules.sh b/bin/update_submodules.sh new file mode 100755 index 00000000..eb75edb8 --- /dev/null +++ b/bin/update_submodules.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -euo pipefail + + +git submodule update --init + +cd concourse-bosh-deployment +git checkout v4.1.0 +cd .. + +cd bosh-deployment +git checkout 7375c31d59018203911da3881334f09d8e70deb5 +cd .. + +cd cf-deployment +git checkout v4.2.0 +cd .. + +cd prometheus-boshrelease +git checkout v23.2.0 +cd .. + +cd cf-rabbitmq-multitenant-broker-release +git checkout v37.0.0 +cd .. + +cd elasticache-broker +git checkout d0a66eab280793e3d2f342996e6c16a5cc6f9c8f +cd .. \ No newline at end of file From f74d753f9f3241c640b37804dbab8d83d545ec8a Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Thu, 13 Sep 2018 14:22:00 +0100 Subject: [PATCH 63/71] Downgrade CATS to 6 nodes, because API workers cannot keep up --- ci/cf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cf.yml b/ci/cf.yml index 61427b1f..221ece5a 100644 --- a/ci/cf.yml +++ b/ci/cf.yml @@ -138,4 +138,4 @@ jobs: cf-acceptance-tests: cf-acceptance-tests-git params: CAPTURE_CONFIG: true - NODES: 8 + NODES: 6 From 576a53356adddefeb5391e6241003d887a31d72b Mon Sep 17 00:00:00 2001 From: Joe Blackman Date: Thu, 13 Sep 2018 14:36:07 +0100 Subject: [PATCH 64/71] Add make rabbitmq to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ad0498a0..3d5cea5a 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,7 @@ You will need to create (manually) an AWS user and generate access and secret ke make concourse make cf make prometheus + make rabbitmq ``` You can specify AWS_PROFILE, rather than the two AWS secrets, for every step except `make bosh`. From bb53fdcc9db9884137d379b9cdc7883d59b41385 Mon Sep 17 00:00:00 2001 From: Robert Gruber Date: Thu, 13 Sep 2018 16:01:39 +0100 Subject: [PATCH 65/71] Specify release versions --- bin/update_submodules.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/update_submodules.sh b/bin/update_submodules.sh index eb75edb8..a6aa2045 100755 --- a/bin/update_submodules.sh +++ b/bin/update_submodules.sh @@ -10,7 +10,7 @@ git checkout v4.1.0 cd .. cd bosh-deployment -git checkout 7375c31d59018203911da3881334f09d8e70deb5 +git checkout ff0bc984c72d5053656c666ae7d7788a78f6b7a8 cd .. cd cf-deployment From 6936de9d7d8dbe242db02e3bbbadacdd456d9b21 Mon Sep 17 00:00:00 2001 From: Robert Gruber Date: Fri, 14 Sep 2018 12:00:36 +0100 Subject: [PATCH 66/71] Add destroy_database script Reduce cats instance count --- bin/destroy_database.sh | 66 +++++++++++++++++++++++++++++++++++++++++ ci/deploy_pipeline.yml | 2 +- 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100755 bin/destroy_database.sh diff --git a/bin/destroy_database.sh b/bin/destroy_database.sh new file mode 100755 index 00000000..9b1eaf52 --- /dev/null +++ b/bin/destroy_database.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +set -euo pipefail + +while getopts 'e:' option; do + case $option in + e) export ENVIRONMENT="$OPTARG";; + esac +done + +shift $((OPTIND-1)) + +: $ENVIRONMENT + +BOSH_DB_USER=$(bin/outputs.sh rds | jq -r .bosh_db_username) +BOSH_DB_HOST=$(bin/outputs.sh rds | jq -r .bosh_db_host) +BOSH_DB_PORT=$(bin/outputs.sh rds | jq -r .bosh_db_port) +BOSH_DB_PASS=$(bin/outputs.sh rds | jq -r .bosh_rds_password) + +CF_DB_USER=$(bin/outputs.sh rds | jq -r .cf_db_username) +CF_DB_HOST=$(bin/outputs.sh rds | jq -r .cf_db_host) +CF_DB_PORT=$(bin/outputs.sh rds | jq -r .cf_db_port) +CF_DB_PASS=$(bin/outputs.sh rds | jq -r .cf_rds_password) + + +JUMPBOX_IP=$(bin/outputs.sh | jq -r .jumpbox_public_ip) +KEYFILE=~/.ssh/$ENVIRONMENT.jumpbox.$$.pem +bin/outputs.sh | jq -r .jumpbox_private_key >$KEYFILE +chmod 600 $KEYFILE + +BOSH_LOCAL_PORT=30201 +CF_LOCAL_PORT=30202 + +cleanup() { + rm -f $KEYFILE + kill $(ps -ef | awk "/ssh.*$BOSH_LOCAL_PORT/ && ! /awk/ { print \$2 }") + kill $(ps -ef | awk "/ssh.*$CF_LOCAL_PORT/ && ! /awk/ { print \$2 }") +} + +ssh -fN -L $BOSH_LOCAL_PORT:$BOSH_DB_HOST:$BOSH_DB_PORT ubuntu@$JUMPBOX_IP -i $KEYFILE +ssh -fN -L $CF_LOCAL_PORT:$CF_DB_HOST:$CF_DB_PORT ubuntu@$JUMPBOX_IP -i $KEYFILE + +trap cleanup EXIT + +sql() { + PGPASSWORD=$BOSH_DB_PASS psql -h localhost -p $BOSH_LOCAL_PORT -U $BOSH_DB_USER -d paastest -c "$*" +} + +destroy_bosh() { + sql "drop database $1" +} + +destroy_cf() { + echo "drop database if exists $1" | MYSQL_PWD=$CF_DB_PASS mysql --protocol=tcp -h localhost -P $CF_LOCAL_PORT -u $CF_DB_USER +} + +case "$1" in +cf) + echo "Destroy CF db" + destroy_cf $2 + ;; +bosh) + echo "Destroy Bosh db" + destroy_bosh $2 + ;; +esac \ No newline at end of file diff --git a/ci/deploy_pipeline.yml b/ci/deploy_pipeline.yml index 197f1d7d..8fe235a5 100644 --- a/ci/deploy_pipeline.yml +++ b/ci/deploy_pipeline.yml @@ -799,7 +799,7 @@ jobs: cf-acceptance-tests: cf-acceptance-tests-git params: CAPTURE_CONFIG: true - NODES: 6 + NODES: 4 on_failure: put: slack-alert params: From 8143cdb84e8b06a286068fd48f1850a8a61133d2 Mon Sep 17 00:00:00 2001 From: Robert Gruber Date: Fri, 14 Sep 2018 12:05:02 +0100 Subject: [PATCH 67/71] Remove obsolete files Reduce cats nodes to 4 --- ci/cf.yml | 2 +- ci/deploy_pipeline.sh | 50 -- ci/deploy_pipeline.yml | 914 ------------------------------------ ci/destruction_pipeline.sh | 35 -- ci/destruction_pipeline.yml | 384 --------------- 5 files changed, 1 insertion(+), 1384 deletions(-) delete mode 100755 ci/deploy_pipeline.sh delete mode 100644 ci/deploy_pipeline.yml delete mode 100755 ci/destruction_pipeline.sh delete mode 100644 ci/destruction_pipeline.yml diff --git a/ci/cf.yml b/ci/cf.yml index 221ece5a..f0b047cd 100644 --- a/ci/cf.yml +++ b/ci/cf.yml @@ -138,4 +138,4 @@ jobs: cf-acceptance-tests: cf-acceptance-tests-git params: CAPTURE_CONFIG: true - NODES: 6 + NODES: 4 diff --git a/ci/deploy_pipeline.sh b/ci/deploy_pipeline.sh deleted file mode 100755 index 705883d1..00000000 --- a/ci/deploy_pipeline.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -: $ENVIRONMENT -: ${BRANCH:="$(git rev-parse --abbrev-ref HEAD)"} - -echo "Using branch '${BRANCH}'" - -bin/login_fly.sh - -jumpbox_commit_ref="32c162b16f2a5a2639c78d905ba852487b93d507" -bosh_commit_ref="04c85a5c79a9fa6b92775386a334104b9a165013" -prometheus_commit_ref="a381c0af550550fc1d740ef409c2e2f22a589202" -cf_tag="v2.7.0" - -node_exporter_version="4.0.0" - -# Grab pre-requisite files from S3 -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/tfstate.json" "${CONCOURSE_TERRAFORM_STATE_FILE}" -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/creds.yml" "${CONCOURSE_CREDS_FILE}" -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/vpc/tfstate.json" "${VPC_STATE_FILE}" - -REGION=$(terraform output -state="${ENVIRONMENT}_concourse.tfstate.json" region) -S3_KMS_KEY_ID=$(terraform output -state="${ENVIRONMENT}_concourse.tfstate.json" s3_kms_key_id) -DOMAIN=$(jq '.modules[0].outputs | with_entries(.value = .value.value)' "${VPC_STATE_FILE}" | jq -r '.dns_zone' | sed 's/\.$//') -slack_webhook_uri=$(jq -r .slack_webhook_uri < "$VAR_FILE") - -fly -t "$ENVIRONMENT" set-pipeline \ - -v environment="$ENVIRONMENT" \ - -v branch="$BRANCH" \ - -v region="$REGION" \ - -v domain="$DOMAIN" \ - -v s3_kms_key_id="$S3_KMS_KEY_ID" \ - -v jumpbox_commit_ref="$jumpbox_commit_ref" \ - -v bosh_commit_ref="$bosh_commit_ref" \ - -v cf_tag="$cf_tag" \ - -v slack_webhook_uri="$slack_webhook_uri" \ - -v prometheus_commit_ref="$prometheus_commit_ref" \ - -v node_exporter_version="$node_exporter_version" \ - -v cf_deployment_name="cf" \ - -c ci/deploy_pipeline.yml -p deploy_pipeline -n - -fly -t "$ENVIRONMENT" unpause-pipeline -p deploy_pipeline -fly -t "$ENVIRONMENT" expose-pipeline -p deploy_pipeline - -fly -t "$ENVIRONMENT" check-resource -r deploy_pipeline/jumpbox-deployment-git --from "ref:${jumpbox_commit_ref}" -fly -t "$ENVIRONMENT" check-resource -r deploy_pipeline/bosh-deployment-git --from "ref:${bosh_commit_ref}" -fly -t "$ENVIRONMENT" check-resource -r deploy_pipeline/cf-deployment-git --from "ref:${cf_tag}" -fly -t "$ENVIRONMENT" check-resource -r deploy_pipeline/prometheus-deployment-git --from "ref:${prometheus_commit_ref}" \ No newline at end of file diff --git a/ci/deploy_pipeline.yml b/ci/deploy_pipeline.yml deleted file mode 100644 index 8fe235a5..00000000 --- a/ci/deploy_pipeline.yml +++ /dev/null @@ -1,914 +0,0 @@ ---- -resource_types: -- name: terraform - type: docker-image - source: - repository: robertgruber/terraform-resource - -- name: s3-iam - type: docker-image - source: - repository: governmentpaas/s3-resource - -- name: slack-notification - type: docker-image - source: - repository: cfcommunity/slack-notification-resource - tag: latest - -resources: -- name: paas-bootstrap-git - type: git - source: - uri: https://github.com/ONSdigital/paas-bootstrap.git - branch: ((branch)) - -- name: jumpbox-deployment-git - type: git - source: - uri: https://github.com/cppforlife/jumpbox-deployment.git - -- name: bosh-deployment-git - type: git - source: - uri: https://github.com/cloudfoundry/bosh-deployment.git - -- name: cf-deployment-git - type: git - source: - uri: https://github.com/cloudfoundry/cf-deployment.git - tag_filter: v* - -- name: cf-deployment-concourse-tasks-git - type: git - source: - uri: https://github.com/cloudfoundry/cf-deployment-concourse-tasks.git - -- name: cf-acceptance-tests-git - type: git - source: - uri: https://github.com/cloudfoundry/cf-acceptance-tests.git - -- name: prometheus-deployment-git - type: git - source: - uri: https://github.com/bosh-prometheus/prometheus-boshrelease.git - -- name: cf-tests-git - type: git - source: - uri: https://github.com/ONSdigital/cf-tests - -- name: jumpbox-terraform - type: terraform - source: - storage: - bucket: ons-paas-((environment))-states - bucket_path: jumpbox/ - use_ec2_role: true - region_name: ((region)) - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: vpc-tfstate-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: vpc/tfstate.json - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: concourse-tfstate-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: concourse/tfstate.json - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: jumpbox-tfstate-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: jumpbox/((environment)).tfstate - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: cf-tfstate-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: cf/((environment)).tfstate - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: jumpbox-vars-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: jumpbox/jumpbox-variables.yml - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: jumpbox-state-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: jumpbox/jumpbox-state.json - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: jumpbox-manifest-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: jumpbox/jumpbox.yml - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: ssh-private-key-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: concourse/ssh-key.pem - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: bosh-terraform - type: terraform - source: - storage: - bucket: ons-paas-((environment))-states - bucket_path: bosh/ - use_ec2_role: true - region_name: ((region)) - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: databases-bosh-terraform - type: terraform - source: - storage: - bucket: ons-paas-((environment))-states - bucket_path: bosh-databases/ - use_ec2_role: true - region_name: ((region)) - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: cf-terraform - type: terraform - source: - storage: - bucket: ons-paas-((environment))-states - bucket_path: cf/ - use_ec2_role: true - region_name: ((region)) - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: prometheus-terraform - type: terraform - source: - storage: - bucket: ons-paas-((environment))-states - bucket_path: prometheus/ - use_ec2_role: true - region_name: ((region)) - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: cf-databases-terraform - type: terraform - source: - storage: - bucket: ons-paas-((environment))-states - bucket_path: cf-databases/ - use_ec2_role: true - region_name: ((region)) - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: cf-vars-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: cf/cf-variables.yml - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - initial_version: "-" - initial_content_text: | - --- - -- name: cf-manifest-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: cf/cf.yml - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - initial_version: "-" - initial_content_text: | - --- - -- name: prometheus-vars-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: prometheus/prometheus-variables.yml - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: prometheus-manifest-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: prometheus/prometheus.yml - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - initial_version: "-" - initial_content_text: | - --- - -- name: prometheus-tfstate-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: prometheus/((environment)).tfstate - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: cf-cloud-config-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: cf/cloud-config.yml - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: bosh-tfstate-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: bosh/((environment)).tfstate - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: bosh-vars-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: bosh/bosh-variables.yml - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: bosh-state-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: bosh/bosh-state.json - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: bosh-manifest-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: bosh/bosh.yml - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: databases-tfstate-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: cf-databases/((environment)).tfstate - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - - -- name: bosh-databases-tfstate-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: bosh-databases/((environment)).tfstate - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: slack-alert - type: slack-notification - source: - url: ((slack_webhook_uri)) - -jobs: -- name: verify-concourse - serial: true - plan: - - get: paas-bootstrap-git - trigger: true - - get: concourse-tfstate-s3 - - aggregate: - - task: ensure world cannot connect to concourse on port 22 - file: paas-bootstrap-git/ci/tasks/common/test_connectivity/task.yml - params: - HOST: concourse - DOMAIN: ((domain)) - PORT: 22 - EXPECTED_OUTCOME: failure - - task: ensure world cannot connect to concourse on port 6868 - file: paas-bootstrap-git/ci/tasks/common/test_connectivity/task.yml - params: - HOST: concourse - DOMAIN: ((domain)) - PORT: 6868 - EXPECTED_OUTCOME: failure - - task: ensure world cannot connect to concourse (ci) on port 443 - file: paas-bootstrap-git/ci/tasks/common/test_connectivity/task.yml - params: - HOST: ci - DOMAIN: ((domain)) - PORT: 443 - EXPECTED_OUTCOME: failure - - task: ensure that direct FQDN of concourse matches its endpoint - file: paas-bootstrap-git/ci/tasks/concourse/test_fqdn/task.yml - params: - HOST: concourse - DOMAIN: ((domain)) - - task: ensure concourse cannot perform disallowed action - file: paas-bootstrap-git/ci/tasks/concourse/test_iam_policy/task.yml - -- name: terraform-jumpbox - serial: true - serial_groups: [rds,prometheus,smoke-tests,cats] - plan: - - aggregate: - - get: concourse-tfstate-s3 - - get: vpc-tfstate-s3 - - get: paas-bootstrap-git - passed: [ verify-concourse ] - trigger: true - - task: get-terraform-vars - file: paas-bootstrap-git/ci/tasks/jumpbox/get_terraform_vars/task.yml - - put: jumpbox-terraform - params: - terraform_source: paas-bootstrap-git/terraform/jumpbox/aws - env_name: ((environment)) - var_files: - - concourse-vars/vars.json - - vpc-vars/vars.json - -- name: terraform-bosh - serial: true - serial_groups: [rds,prometheus,smoke-tests,cats] - plan: - - aggregate: - - get: concourse-tfstate-s3 - - get: jumpbox-terraform - - get: vpc-tfstate-s3 - - get: jumpbox-tfstate-s3 - - get: paas-bootstrap-git - passed: [terraform-jumpbox] - trigger: true - - task: get-terraform-vars - file: paas-bootstrap-git/ci/tasks/bosh/get_terraform_vars/task.yml - params: - ENVIRONMENT: ((environment)) - - put: bosh-terraform - params: - terraform_source: paas-bootstrap-git/terraform/bosh/aws - env_name: ((environment)) - var_files: - - concourse-vars/vars.json - - jumpbox-terraform/metadata - - vpc-vars/vars.json - -- name: terraform-bosh-databases - serial: true - serial_groups: [rds,prometheus,smoke-tests,cats] - plan: - - aggregate: - - get: vpc-tfstate-s3 - - get: bosh-terraform - passed: [terraform-bosh] - params: - output_statefile: true - - get: paas-bootstrap-git - trigger: true - passed: [terraform-bosh] - - task: get database vars - file: paas-bootstrap-git/ci/tasks/bosh/get_database_vars/task.yml - - task: bosh db connectivity test - file: paas-bootstrap-git/ci/tasks/bosh/db_connectivity_test/task.yml - timeout: 5m - - put: databases-bosh-terraform - params: - terraform_source: paas-bootstrap-git/terraform/bosh-databases/aws - env_name: ((environment)) - var_files: - - vpc-vars/vars.json - - bosh-vars/vars.json - -- name: terraform-cf - serial: true - serial_groups: [rds,prometheus,smoke-tests,cats] - plan: - - aggregate: - - get: vpc-tfstate-s3 - - get: jumpbox-tfstate-s3 - - get: bosh-tfstate-s3 - - get: concourse-tfstate-s3 - - get: paas-bootstrap-git - trigger: true - passed: [terraform-bosh-databases] - - task: get-terraform-vars - file: paas-bootstrap-git/ci/tasks/cf/get_terraform_vars/task.yml - params: - ENVIRONMENT: ((environment)) - - put: cf-terraform - params: - terraform_source: paas-bootstrap-git/terraform/cf/aws - env_name: ((environment)) - var_files: - - vpc-vars/vars.json - - jumpbox-vars/vars.json - - bosh-vars/vars.json - - concourse-vars/vars.json - -- name: terraform-cf-databases - serial: true - serial_groups: [rds,prometheus,smoke-tests,cats] - plan: - - aggregate: - - get: vpc-tfstate-s3 - - get: cf-terraform - passed: [terraform-cf] - params: - output_statefile: true - - get: paas-bootstrap-git - trigger: true - passed: [terraform-cf] - - task: get-database-vars - file: paas-bootstrap-git/ci/tasks/cf/databases/get_database_vars/task.yml - - task: cf db connectivity test - file: paas-bootstrap-git/ci/tasks/cf/databases/db_connectivity_test/task.yml - timeout: 5m - - put: cf-databases-terraform - params: - terraform_source: paas-bootstrap-git/terraform/cf-databases/aws - env_name: ((environment)) - var_files: - - vpc-vars/vars.json - - cf-vars/vars.json - -- name: verify-terraform-cf - serial: true - serial_groups: [rds,prometheus,smoke-tests,cats] - plan: - - get: paas-bootstrap-git - trigger: true - passed: [terraform-cf-databases] - - aggregate: - - task: ensure that system FQDN is resolvable - file: paas-bootstrap-git/ci/tasks/cf/test/task.yml - params: - HOST: test.system - DOMAIN: ((domain)) - - task: ensure that apps FQDN is resolvable - file: paas-bootstrap-git/ci/tasks/cf/test/task.yml - params: - HOST: test.apps - DOMAIN: ((domain)) - -- name: terraform-prometheus - serial: true - serial_groups: [rds,prometheus,smoke-tests,cats] - plan: - - aggregate: - - get: paas-bootstrap-git - passed: [verify-terraform-cf] - trigger: true - - get: vpc-tfstate-s3 - - get: concourse-tfstate-s3 - - get: bosh-tfstate-s3 - - get: cf-tfstate-s3 - - get: jumpbox-tfstate-s3 - - task: get-terraform-vars - file: paas-bootstrap-git/ci/tasks/prometheus/get_terraform_vars/task.yml - params: - ENVIRONMENT: ((environment)) - - put: prometheus-terraform - params: - terraform_source: paas-bootstrap-git/terraform/prometheus/aws - env_name: ((environment)) - var_files: - - vpc-vars/vars.json - - concourse-vars/vars.json - - bosh-vars/vars.json - - cf-vars/vars.json - - jumpbox-vars/vars.json - -- name: create-jumpbox - serial: true - serial_groups: [rds,prometheus,smoke-tests,cats] - plan: - - aggregate: - - get: paas-bootstrap-git - passed: [terraform-prometheus] - trigger: true - - get: jumpbox-deployment-git - version: { ref: ((jumpbox_commit_ref)) } - - get: jumpbox-terraform - passed: [terraform-jumpbox] - - get: jumpbox-vars-s3 - - get: jumpbox-state-s3 - - get: ssh-private-key-s3 - - get: concourse-tfstate-s3 - - task: interpolate-jumpbox - file: paas-bootstrap-git/ci/tasks/jumpbox/interpolate/task.yml - params: - ENVIRONMENT: ((environment)) - on_success: - aggregate: - - put: jumpbox-vars-s3 - params: - file: jumpbox-manifests/jumpbox-variables.yml - - put: jumpbox-manifest-s3 - params: - file: jumpbox-manifests/jumpbox.yml - - task: deploy-jumpbox - file: paas-bootstrap-git/ci/tasks/jumpbox/deploy/task.yml - on_success: - put: jumpbox-state-s3 - params: - file: jumpbox-state/jumpbox-state.json - on_failure: - put: jumpbox-state-s3 - params: - file: jumpbox-state/jumpbox-state.json - -- name: verify-jumpbox - serial: true - serial_groups: [rds,prometheus,smoke-tests,cats] - plan: - - get: paas-bootstrap-git - passed: [create-jumpbox] - trigger: true - - aggregate: - - get: jumpbox-vars-s3 - - get: jumpbox-manifest-s3 - - aggregate: - - task: ensure concourse can SSH to jumpbox - file: paas-bootstrap-git/ci/tasks/jumpbox/test/task.yml - params: - ENVIRONMENT: ((environment)) - - task: ensure world cannot connect to jumpbox on port 22 - file: paas-bootstrap-git/ci/tasks/common/test_connectivity/task.yml - params: - HOST: jumpbox - DOMAIN: ((domain)) - PORT: 22 - EXPECTED_OUTCOME: failure - - task: ensure world cannot connect to jumpbox on port 6868 - file: paas-bootstrap-git/ci/tasks/common/test_connectivity/task.yml - params: - HOST: jumpbox - DOMAIN: ((domain)) - PORT: 6868 - EXPECTED_OUTCOME: failure - - task: ensure that FQDN of jumpbox matches its endpoint - file: paas-bootstrap-git/ci/tasks/jumpbox/test_fqdn/task.yml - params: - HOST: jumpbox - DOMAIN: ((domain)) - -- name: create-bosh - serial: true - serial_groups: [rds,prometheus,smoke-tests,cats] - plan: - - aggregate: - - get: paas-bootstrap-git - passed: [verify-jumpbox] - trigger: true - - get: bosh-deployment-git - version: { ref: ((bosh_commit_ref)) } - - get: bosh-terraform - passed: [terraform-bosh] - - get: bosh-tfstate-s3 - - get: bosh-vars-s3 - - get: bosh-state-s3 - - get: ssh-private-key-s3 - - get: concourse-tfstate-s3 - - get: bosh-databases-tfstate-s3 - - get: vpc-tfstate-s3 - - task: interpolate-bosh - file: paas-bootstrap-git/ci/tasks/bosh/interpolate/task.yml - params: - ENVIRONMENT: ((environment)) - on_success: - aggregate: - - put: bosh-vars-s3 - params: - file: bosh-manifests/bosh-variables.yml - - put: bosh-manifest-s3 - params: - file: bosh-manifests/bosh.yml - - task: deploy-bosh - file: paas-bootstrap-git/ci/tasks/bosh/deploy/task.yml - on_success: - put: bosh-state-s3 - params: - file: bosh-state/bosh-state.json - - task: deploy-bosh-node-exporter - file: paas-bootstrap-git/ci/tasks/bosh/deploy_node_exporter/task.yml - params: - NODE_EXPORTER_VERSION: ((node_exporter_version)) - -- name: verify-bosh - serial: true - serial_groups: [rds,prometheus,smoke-tests,cats] - plan: - - get: paas-bootstrap-git - passed: [create-bosh] - trigger: true - - aggregate: - - get: bosh-vars-s3 - - get: bosh-manifest-s3 - - task: verify - file: paas-bootstrap-git/ci/tasks/bosh/test/task.yml - -- name: deploy-cf - serial: true - serial_groups: [rds,prometheus,smoke-tests,cats] - plan: - - aggregate: - - get: paas-bootstrap-git - trigger: true - passed: [verify-bosh] - - get: cf-deployment-git - version: { ref: ((cf_tag)) } - - get: prometheus-deployment-git - version: { ref: ((prometheus_commit_ref)) } - - get: vpc-tfstate-s3 - - get: bosh-tfstate-s3 - - get: cf-tfstate-s3 - - get: cf-vars-s3 - - get: bosh-vars-s3 - - get: bosh-manifest-s3 - - get: cf-manifest-s3 - - get: concourse-tfstate-s3 - - get: jumpbox-tfstate-s3 - - get: databases-tfstate-s3 - - get: prometheus-tfstate-s3 - - task: get-terraform-vars - file: paas-bootstrap-git/ci/tasks/cf/get_terraform_vars/task.yml - params: - ENVIRONMENT: ((environment)) - - task: interpolate-cf - file: paas-bootstrap-git/ci/tasks/cf/interpolate/task.yml - params: - ENVIRONMENT: ((environment)) - DOMAIN: ((domain)) - on_success: - aggregate: - - put: cf-vars-s3 - params: - file: cf-manifests/cf-variables.yml - - put: cf-manifest-s3 - params: - file: cf-manifests/cf.yml - - task: cloud_config - file: paas-bootstrap-git/ci/tasks/cf/cloud_config/task.yml - params: - ENVIRONMENT: ((environment)) - DOMAIN: ((domain)) - on_success: - aggregate: - - put: cf-cloud-config-s3 - params: - file: cf-manifests/cloud-config.yml - - task: deploy-stemcell - file: paas-bootstrap-git/ci/tasks/cf/upload_stemcell/task.yml - - task: deploy_cf - file: paas-bootstrap-git/ci/tasks/cf/deploy_cf/task.yml - on_failure: - put: slack-alert - params: - text: | - Environment: ((environment)) - Job/Task: deploy_cf/deploy_cf - Build Name: $BUILD_NAME - Build ID: $BUILD_ID - Result: FAIL - -- name: verify-cf-endpoints - serial: true - serial_groups: [rds,prometheus,smoke-tests,cats] - plan: - - get: paas-bootstrap-git - passed: [deploy-cf] - trigger: true - - aggregate: - - task: ensure CF api responds - file: paas-bootstrap-git/ci/tasks/common/test_endpoint/task.yml - params: - HOST: api.system - DOMAIN: ((domain)) - QUERY_PATH: /v2/info - - task: ensure CF UAA responds - file: paas-bootstrap-git/ci/tasks/common/test_endpoint/task.yml - params: - HOST: uaa.system - DOMAIN: ((domain)) - QUERY_PATH: /info - REQUIRE_JSON: true - -- name: cf-smoke-tests - serial: true - serial_groups: [smoke-tests] - plan: - - aggregate: - - get: paas-bootstrap-git - passed: [verify-cf-endpoints] - trigger: true - - get: bosh-vars-s3 - - get: bosh-manifest-s3 - - task: ensure smoke tests pass - file: paas-bootstrap-git/ci/tasks/cf/smoke_tests/task.yml - -- name: cats - serial: true - serial_groups: [cats] - plan: - - aggregate: - - get: paas-bootstrap-git - trigger: true - passed: [verify-cf-endpoints] - - get: cf-deployment-concourse-tasks-git - - get: cf-acceptance-tests-git - - get: cf-vars-s3 - - task: interpolate CATS config - file: paas-bootstrap-git/ci/tasks/cf/interpolate_cats/task.yml - params: - DOMAIN: ((domain)) - ENVIRONMENT: ((environment)) - CATS_CONFIG_FILE: paas-bootstrap-git/profiles/staging/cats_config.json - - task: run CATS tests - file: cf-deployment-concourse-tasks-git/run-cats/task.yml - input_mapping: - cf-deployment-concourse-tasks: cf-deployment-concourse-tasks-git - cf-acceptance-tests: cf-acceptance-tests-git - params: - CAPTURE_CONFIG: true - NODES: 4 - on_failure: - put: slack-alert - params: - text: | - Environment: ((environment)) - Job/Task: cats/run CATS tests - Build Name: $BUILD_NAME - Build ID: $BUILD_ID - Result: FAIL - -- name: deploy-prometheus - serial: true - serial_groups: [prometheus] - plan: - - aggregate: - - get: paas-bootstrap-git - trigger: true - passed: [verify-cf-endpoints] - - get: prometheus-deployment-git - version: { ref: ((prometheus_commit_ref)) } - - get: prometheus-vars-s3 - - get: vpc-tfstate-s3 - - get: bosh-tfstate-s3 - - get: cf-tfstate-s3 - - get: cf-vars-s3 - - get: bosh-vars-s3 - - get: bosh-manifest-s3 - - get: prometheus-tfstate-s3 - - get: jumpbox-tfstate-s3 - - task: interpolate-prometheus - file: paas-bootstrap-git/ci/tasks/prometheus/interpolate/task.yml - params: - ENVIRONMENT: ((environment)) - DOMAIN: ((domain)) - CF_DEPLOYMENT_NAME: ((cf_deployment_name)) - on_success: - aggregate: - - put: prometheus-vars-s3 - params: - file: prometheus-manifests/prometheus-variables.yml - - put: prometheus-manifest-s3 - params: - file: prometheus-manifests/prometheus.yml - - task: deploy_prometheus - file: paas-bootstrap-git/ci/tasks/prometheus/deploy/task.yml - -- name: verify-prometheus - serial: true - serial_groups: [prometheus] - plan: - - get: paas-bootstrap-git - passed: [deploy-prometheus] - trigger: true - - aggregate: - - task: ensure concourse can connect to grafana on port 443 - file: paas-bootstrap-git/ci/tasks/common/test_connectivity/task.yml - params: - HOST: grafana - DOMAIN: ((domain)) - PORT: 443 - EXPECTED_OUTCOME: success - -- name: deploy-dashboard-snapshots - serial: true - serial_groups: [prometheus] - plan: - - aggregate: - - get: paas-bootstrap-git - trigger: true - passed: [verify-prometheus] - - get: prometheus-manifest-s3 - - get: prometheus-tfstate-s3 - - task: create radiator snapshot(s) - file: paas-bootstrap-git/ci/tasks/prometheus/dashboard/task.yml - params: - ENVIRONMENT: ((environment)) - SNAPSHOT_NAME: radiator - TEMPLATE: paas-bootstrap-git/grafana/bosh_cpu.json - -- name: cf-management - serial: true - serial_groups: [rds] - plan: - - get: paas-bootstrap-git - trigger: true - passed: [verify-cf-endpoints] - - get: cf-vars-s3 - - task: get management variables - file: paas-bootstrap-git/ci/tasks/cf/get_management_vars/task.yml - params: - ENVIRONMENT: ((environment)) - DOMAIN: ((domain)) - - task: create orgs spaces etc - file: paas-bootstrap-git/ci/tasks/cf/management/task.yml - params: - ENVIRONMENT: ((environment)) - -- name: verify-shared-rds - serial: true - serial_groups: [rds] - plan: - - get: cf-tests-git - - get: paas-bootstrap-git - trigger: true - passed: [cf-management] - - get: cf-vars-s3 - - task: get cf credentials - file: paas-bootstrap-git/ci/tasks/common/get_cf_tester_credentials/task.yml - params: - DOMAIN: ((domain)) - - task: create service and test useability - file: paas-bootstrap-git/ci/tasks/rds_broker/test_shared/task.yml diff --git a/ci/destruction_pipeline.sh b/ci/destruction_pipeline.sh deleted file mode 100755 index ca480a03..00000000 --- a/ci/destruction_pipeline.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -: $ENVIRONMENT -: ${BRANCH:="$(git rev-parse --abbrev-ref HEAD)"} - -echo "Using branch '${BRANCH}'" - -bin/login_fly.sh - -jumpbox_commit_ref="32c162b16f2a5a2639c78d905ba852487b93d507" -bosh_commit_ref="010bd498bb97dee707c167e60469b0f5d2cc90fb" - -# Grab pre-requisite files from S3 -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/tfstate.json" "${CONCOURSE_TERRAFORM_STATE_FILE}" -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/concourse/creds.yml" "${CONCOURSE_CREDS_FILE}" -aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/vpc/tfstate.json" "${VPC_STATE_FILE}" - -REGION=$(terraform output -state="${ENVIRONMENT}_concourse.tfstate.json" region) -S3_KMS_KEY_ID=$(terraform output -state="${ENVIRONMENT}_concourse.tfstate.json" s3_kms_key_id) -DOMAIN=$(jq '.modules[0].outputs | with_entries(.value = .value.value)' "${VPC_STATE_FILE}" | jq -r '.dns_zone' | sed 's/\.$//') - -fly -t "$ENVIRONMENT" set-pipeline \ - -v environment="$ENVIRONMENT" \ - -v branch="$BRANCH" \ - -v region="$REGION" \ - -v domain="$DOMAIN" \ - -v s3_kms_key_id="$S3_KMS_KEY_ID" \ - -v jumpbox_commit_ref="$jumpbox_commit_ref" \ - -v bosh_commit_ref="$bosh_commit_ref" \ - -c ci/destruction_pipeline.yml -p destruction_pipeline -n -fly -t "$ENVIRONMENT" unpause-pipeline -p destruction_pipeline - -fly -t "$ENVIRONMENT" check-resource -r destruction_pipeline/bosh-deployment-git --from "ref:${bosh_commit_ref}" \ No newline at end of file diff --git a/ci/destruction_pipeline.yml b/ci/destruction_pipeline.yml deleted file mode 100644 index c962ef74..00000000 --- a/ci/destruction_pipeline.yml +++ /dev/null @@ -1,384 +0,0 @@ ---- -resource_types: -- name: terraform - type: docker-image - source: - repository: robertgruber/terraform-resource - -- name: s3-iam - type: docker-image - source: - repository: governmentpaas/s3-resource - -resources: -- name: paas-bootstrap-git - type: git - source: - uri: https://github.com/ONSdigital/paas-bootstrap.git - branch: ((branch)) - -- name: bosh-deployment-git - type: git - source: - uri: https://github.com/cloudfoundry/bosh-deployment.git - -- name: manual-force - type: time - source: - interval: 24000h - -- name: jumpbox-terraform - type: terraform - source: - storage: - bucket: ons-paas-((environment))-states - bucket_path: jumpbox/ - use_ec2_role: true - region_name: ((region)) - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: jumpbox-tfstate-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: jumpbox/((environment)).tfstate - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: jumpbox-state-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: jumpbox/jumpbox-state.json - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: jumpbox-manifest-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: jumpbox/jumpbox.yml - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: bosh-terraform - type: terraform - source: - storage: - bucket: ons-paas-((environment))-states - bucket_path: bosh/ - use_ec2_role: true - region_name: ((region)) - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: bosh-tfstate-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: bosh/((environment)).tfstate - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: bosh-vars-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: bosh/bosh-variables.yml - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: bosh-state-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: bosh/bosh-state.json - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: bosh-manifest-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: bosh/bosh.yml - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: vpc-tfstate-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: vpc/tfstate.json - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - - -- name: concourse-tfstate-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: concourse/tfstate.json - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: cf-terraform - type: terraform - source: - storage: - bucket: ons-paas-((environment))-states - bucket_path: cf/ - use_ec2_role: true - region_name: ((region)) - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: prometheus-terraform - type: terraform - source: - storage: - bucket: ons-paas-((environment))-states - bucket_path: prometheus/ - use_ec2_role: true - region_name: ((region)) - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id))c - -- name: cf-vars-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: cf/cf-variables.yml - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - initial_version: "-" - initial_content_text: | - --- - -- name: cf-tfstate-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: cf/((environment)).tfstate - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - -- name: prometheus-manifest-s3 - type: s3-iam - source: - bucket: ons-paas-((environment))-states - private: true - region_name: ((region)) - versioned_file: prometheus/prometheus.yml - server_side_encryption: 'aws:kms' - sse_kms_key_id: ((s3_kms_key_id)) - initial_version: "-" - initial_content_text: | - --- - -jobs: -- name: trigger-destroy - serial_groups: [destruction] - serial: true - plan: - - get: manual-force - -- name: destroy-prometheus - serial_groups: [destruction] - serial: true - plan: - - aggregate: - - get: manual-force - trigger: true - passed: [trigger-destroy] - - get: paas-bootstrap-git - - get: vpc-tfstate-s3 - - get: bosh-tfstate-s3 - - get: cf-tfstate-s3 - - get: cf-vars-s3 - - get: bosh-vars-s3 - - get: bosh-manifest-s3 - - get: jumpbox-tfstate-s3 - - get: concourse-tfstate-s3 - - get: prometheus-manifest-s3 - # - task: destroy-prometheus - # file: paas-bootstrap-git/ci/tasks/prometheus/destroy/task.yml - # params: - # ENVIRONMENT: ((environment)) - - task: empty-buckets - file: paas-bootstrap-git/ci/tasks/destroy/empty_buckets/task.yml - params: - ONS_PREFIX: ons-paas - ENVIRONMENT: ((environment)) - NAMESPACE: prometheus - - task: get_terraform_vars - file: paas-bootstrap-git/ci/tasks/prometheus/get_terraform_vars/task.yml - params: - ENVIRONMENT: ((environment)) - - put: prometheus-terraform - params: - terraform_source: paas-bootstrap-git/terraform/prometheus/aws - env_name: ((environment)) - var_files: - - vpc-vars/vars.json - - concourse-vars/vars.json - - bosh-vars/vars.json - - cf-vars/vars.json - - jumpbox-vars/vars.json - action: destroy - get_params: - action: destroy - -- name: destroy-cf - serial_groups: [destruction] - serial: true - plan: - - aggregate: - - get: manual-force - trigger: true - passed: [destroy-prometheus] - - get: paas-bootstrap-git - - get: bosh-vars-s3 - - get: bosh-manifest-s3 - - get: vpc-tfstate-s3 - - get: concourse-tfstate-s3 - - get: bosh-tfstate-s3 - - get: jumpbox-tfstate-s3 - # - task: destroy-cf - # file: paas-bootstrap-git/ci/tasks/cf/destroy/task.yml - # params: - # ENVIRONMENT: ((environment)) - - task: empty-buckets - file: paas-bootstrap-git/ci/tasks/destroy/empty_buckets/task.yml - params: - ONS_PREFIX: ons-paas - ENVIRONMENT: ((environment)) - NAMESPACE: cf - - task: get_terraform_vars - file: paas-bootstrap-git/ci/tasks/cf/get_terraform_vars/task.yml - params: - ENVIRONMENT: ((environment)) - - put: cf-terraform - params: - terraform_source: paas-bootstrap-git/terraform/cf/aws - env_name: ((environment)) - var_files: - - vpc-vars/vars.json - - bosh-vars/vars.json - - concourse-vars/vars.json - - jumpbox-vars/vars.json - action: destroy - get_params: - action: destroy - -- name: destroy-bosh - serial_groups: [destruction] - serial: true - plan: - - aggregate: - - get: manual-force - trigger: true - passed: [destroy-cf] - - get: paas-bootstrap-git - - get: bosh-deployment-git - version: { ref: ((bosh_commit_ref)) } - - get: bosh-state-s3 - - get: bosh-manifest-s3 - - get: vpc-tfstate-s3 - - get: concourse-tfstate-s3 - - get: jumpbox-tfstate-s3 - - task: get-terraform-vars - file: paas-bootstrap-git/ci/tasks/bosh/get_terraform_vars/task.yml - params: - ENVIRONMENT: ((environment)) - # - task: destroy-bosh - # file: paas-bootstrap-git/ci/tasks/bosh/destroy/task.yml - # on_success: - # put: bosh-state-s3 - # params: - # file: bosh-state/bosh-state.json - - task: empty-buckets - file: paas-bootstrap-git/ci/tasks/destroy/empty_buckets/task.yml - params: - ONS_PREFIX: ons-paas - ENVIRONMENT: ((environment)) - NAMESPACE: bosh - - task: get_terraform_vars - file: paas-bootstrap-git/ci/tasks/bosh/get_terraform_vars/task.yml - params: - ENVIRONMENT: ((environment)) - - put: bosh-terraform - params: - terraform_source: paas-bootstrap-git/terraform/bosh/aws - env_name: ((environment)) - var_files: - - vpc-vars/vars.json - - concourse-vars/vars.json - - jumpbox-vars/vars.json - action: destroy - get_params: - action: destroy - -- name: destroy-jumpbox - serial_groups: [destruction] - serial: true - plan: - - aggregate: - - get: manual-force - trigger: true - passed: [destroy-bosh] - - get: paas-bootstrap-git - - get: jumpbox-state-s3 - - get: jumpbox-manifest-s3 - - get: vpc-tfstate-s3 - - get: concourse-tfstate-s3 - - task: destroy-jumpbox - file: paas-bootstrap-git/ci/tasks/jumpbox/destroy/task.yml - params: - ENVIRONMENT: ((environment)) - on_success: - put: jumpbox-state-s3 - params: - file: jumpbox-state/jumpbox-state.json - - task: empty-buckets - file: paas-bootstrap-git/ci/tasks/destroy/empty_buckets/task.yml - params: - ONS_PREFIX: ons-paas - ENVIRONMENT: ((environment)) - NAMESPACE: jumpbox - - task: get_terraform_vars - file: paas-bootstrap-git/ci/tasks/jumpbox/get_terraform_vars/task.yml - params: - ENVIRONMENT: ((environment)) - - put: jumpbox-terraform - params: - terraform_source: paas-bootstrap-git/terraform/jumpbox/aws - env_name: ((environment)) - var_files: - - vpc-vars/vars.json - - concourse-vars/vars.json - action: destroy - get_params: - action: destroy From 9d9fcb7a739ed120c8dffbc9bff3ffa7c77f9993 Mon Sep 17 00:00:00 2001 From: Robert Gruber Date: Fri, 14 Sep 2018 12:07:55 +0100 Subject: [PATCH 68/71] Update git submodule for CF to v4.2.0 --- cf-deployment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf-deployment b/cf-deployment index 2f466e23..33abec0e 160000 --- a/cf-deployment +++ b/cf-deployment @@ -1 +1 @@ -Subproject commit 2f466e231a47809190f9793f7c7e0260727f7ed8 +Subproject commit 33abec0e839f8beb572c922fd2e8c870c3617bda From 4456bb91196d975fe02418364e2b24037df1a6f6 Mon Sep 17 00:00:00 2001 From: Robert Gruber Date: Thu, 20 Sep 2018 15:47:52 +0100 Subject: [PATCH 69/71] Force delete of S3 buckets if not empty Force delete of IAM user if keys exist --- terraform/base/bosh.tf | 1 + terraform/base/cf.tf | 4 ++++ terraform/base/s3.tf | 1 + terraform/base/services-redis.tf | 1 + 4 files changed, 7 insertions(+) diff --git a/terraform/base/bosh.tf b/terraform/base/bosh.tf index 67efce4d..77431898 100644 --- a/terraform/base/bosh.tf +++ b/terraform/base/bosh.tf @@ -58,6 +58,7 @@ resource "aws_iam_role_policy" "bosh" { resource "aws_s3_bucket" "bosh_blobstore" { bucket = "${var.s3_prefix}-${var.environment}-bosh-blobstore" acl = "private" + force_destroy = true tags { Name = "${var.s3_prefix}-${var.environment}-bosh-blobstore" diff --git a/terraform/base/cf.tf b/terraform/base/cf.tf index b934d4b3..3ab5ce83 100644 --- a/terraform/base/cf.tf +++ b/terraform/base/cf.tf @@ -451,6 +451,7 @@ resource "aws_kms_key" "cf_blobstore_key" { resource "aws_s3_bucket" "cf_buildpacks" { bucket = "${var.s3_prefix}-${var.environment}-cf-buildpacks" acl = "private" + force_destroy = true versioning { enabled = false @@ -474,6 +475,7 @@ resource "aws_s3_bucket" "cf_buildpacks" { resource "aws_s3_bucket" "cf_droplets" { bucket = "${var.s3_prefix}-${var.environment}-cf-droplets" acl = "private" + force_destroy = true versioning { enabled = false @@ -497,6 +499,7 @@ resource "aws_s3_bucket" "cf_droplets" { resource "aws_s3_bucket" "cf_packages" { bucket = "${var.s3_prefix}-${var.environment}-cf-packages" acl = "private" + force_destroy = true versioning { enabled = false @@ -520,6 +523,7 @@ resource "aws_s3_bucket" "cf_packages" { resource "aws_s3_bucket" "cf_resource_pool" { bucket = "${var.s3_prefix}-${var.environment}-cf-resource-pool" acl = "private" + force_destroy = true versioning { enabled = false diff --git a/terraform/base/s3.tf b/terraform/base/s3.tf index 9e311ec1..b7803d32 100644 --- a/terraform/base/s3.tf +++ b/terraform/base/s3.tf @@ -6,6 +6,7 @@ resource "aws_kms_key" "paas_state_key" { resource "aws_s3_bucket" "paas_states" { bucket = "ons-paas-${var.environment}-states" acl = "private" + force_destroy = true versioning { enabled = true diff --git a/terraform/base/services-redis.tf b/terraform/base/services-redis.tf index 97787103..68b3c3f0 100644 --- a/terraform/base/services-redis.tf +++ b/terraform/base/services-redis.tf @@ -30,6 +30,7 @@ resource "aws_elasticache_subnet_group" "redis" { resource "aws_iam_user" "redis" { name = "${var.environment}-redis-user" path = "/services/" + force_destroy = true } resource "aws_iam_user_policy" "redis" { From ba8993c7d1930877143ffd5d03b1918bd80324f8 Mon Sep 17 00:00:00 2001 From: J Gregory Date: Thu, 27 Sep 2018 16:37:13 +0100 Subject: [PATCH 70/71] Add vagrant build tools --- .gitignore | 5 +- tools/README.md | 23 ++++++++ tools/Vagrantfile | 14 +++++ tools/build/provision.sh | 98 +++++++++++++++++++++++++++++++++ tools/build/provision_nosudo.sh | 19 +++++++ 5 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 tools/README.md create mode 100644 tools/Vagrantfile create mode 100644 tools/build/provision.sh create mode 100644 tools/build/provision_nosudo.sh diff --git a/.gitignore b/.gitignore index 23f53217..ebb2c3fb 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,7 @@ foo.yml *.lock.info **/.DS_Store -data/** \ No newline at end of file +data/** + +.vagrant +*-cloudimg-console.log \ No newline at end of file diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 00000000..7d9542bf --- /dev/null +++ b/tools/README.md @@ -0,0 +1,23 @@ +# Vagrant Tools + +> Provides a vagrant image loaded with the tools required to deploy and run a Cloud Foundry instance from this repo + +## Prerequistes + +- `Virtualbox` +- `Vagrant` + +## To use + +``` +cd /tools/vagrant +vagrant up +``` + +Once you have a vm running, you can connect to it with + +``` +vagrant ssh +``` + +The vm mounts the main repo folder as a shared folder under `/src` \ No newline at end of file diff --git a/tools/Vagrantfile b/tools/Vagrantfile new file mode 100644 index 00000000..8363e292 --- /dev/null +++ b/tools/Vagrantfile @@ -0,0 +1,14 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + + config.vm.box = "ubuntu/xenial64" + config.vm.provision :shell, path: "build/provision.sh" + config.vm.provision :shell, privileged: false, path: "build/provision_nosudo.sh" + config.vm.provision :shell, inline: "printf '\nDone provisioning\nConnect to vm with vagrant ssh\n\n'" + config.vm.synced_folder "../..", "/src" + +end \ No newline at end of file diff --git a/tools/build/provision.sh b/tools/build/provision.sh new file mode 100644 index 00000000..61f69813 --- /dev/null +++ b/tools/build/provision.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# Assumed that this script is run with sudo permissions by vagrant provisioner + +locale-gen en_GB.UTF-8 + +BOSH_RELEASE=https://s3.amazonaws.com/bosh-cli-artifacts/bosh-cli-5.1.2-linux-amd64 + +TERRAFORM_VERSION=0.11.8 +TERRAFORM_RELEASE=terraform_${TERRAFORM_VERSION}_linux_amd64.zip +TERRAFORM_URL=https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${TERRAFORM_RELEASE} + +CREDHUB_VERSION=2.0.0 +CREDHUB_RELEASE=credhub-linux-${CREDHUB_VERSION}.tgz +CREDHUB_URL=https://github.com/cloudfoundry-incubator/credhub-cli/releases/download/${CREDHUB_VERSION}/${CREDHUB_RELEASE} + +FLY_VERSION=4.1.0 +FLY_RELEASE=fly_linux_amd64 +FLY_URL=https://github.com/concourse/concourse/releases/download/v${FLY_VERSION}/${FLY_RELEASE} + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +NC='\033[0m' + +echo "" +echo "== Provisioning tools ==" +echo "" + +# ainstall(apt-name,cmd) - install with apt +ainstall() { + printf "${NC}> %-20s -> " "$1" + if apt -y install $1 &>/dev/null; then + printf "${GREEN}%s\\n" "$($2 --version)" + else + printf "${RED}failed to install !!\n" + fi +} + +# cinstall(binary,url) - install with curl +cinstall() { + printf "${NC}> %-20s -> " "$1" + if curl -s -Lo $1 $2 && chmod +x $1 && mv -f $1 /usr/local/bin/$1 &>/dev/null; then + printf "${GREEN}$($1 --version)\n" + else + printf "${RED}failed to install !!\n" + fi +} + +# winstall(binary,url,archive,type) - install with wget (where type=tar|zip|none) +winstall() { + CMD=":" # default noop + if [ ${4} = "zip" ]; then CMD="unzip -qo ${3}"; fi + if [ ${4} = "tar" ]; then CMD="tar zxf ${3}"; fi + + printf "${NC}> %-20s -> " "${1}" + if wget -q ${2} && ${CMD} && chmod +x ${1} && mv -f ${1} /usr/local/bin/${1} ; then + printf "${GREEN}$(${1} --version)\n" + else + printf "${RED}failed to install !!\n" + fi +} + +echo "Installing tools (non-sudo)..." + + +echo "Installing prereqs ..." +{ + # yq + add-apt-repository -y ppa:rmescandon/yq + + # cf cli + wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add - + echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list + apt-get update + apt-get install -y unzip python-pip + + # prereqs for bosh to install ruby + apt-get -y --no-install-recommends install build-essential openssl libreadline6 libreadline6-dev curl git-core zlib1g zlib1g-dev libssl-dev libyaml-dev libsqlite3-dev sqlite3 libxml2-dev libxslt-dev autoconf libc6-dev libgdbm-dev ncurses-dev automake libtool bison subversion pkg-config libffi-dev + +} &>/dev/null # throw away output + +echo "Installing tools ..." +ainstall cf-cli cf # cf cli +ainstall mysql-client mysql # mysql +ainstall postgresql-client psql # postgres +ainstall jq jq # jq +ainstall yq yq # yq +cinstall bosh ${BOSH_RELEASE} # bosh +winstall terraform ${TERRAFORM_URL} ${TERRAFORM_RELEASE} zip # terraform +winstall credhub ${CREDHUB_URL} ${CREDHUB_RELEASE} tar # credhub +cinstall fly ${FLY_URL} # fly + +# Oddly the ubuntu user doesn't own it's own home folder (which causes issues when +# things don't have permission to create folders etc) - there may be a more official +# vagrant-y solution as this seems hacky +# This specifically is to be able to install the awscli (in provision_nosudo) +chown ubuntu /home/ubuntu +chgrp ubuntu /home/ubuntu diff --git a/tools/build/provision_nosudo.sh b/tools/build/provision_nosudo.sh new file mode 100644 index 00000000..5a45ad9f --- /dev/null +++ b/tools/build/provision_nosudo.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Assumes "provision.sh" has already been run for prereqs to have been installed +# +# This script deals with things that cannot be installed with vagrant's default "sudo" + +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' + +echo "Installing tools (no sudo) ..." + +printf "${NC}> %-20s -> " "awscli" +if pip install awscli -q --upgrade --user; then + echo "PATH=~/.local/bin:${PATH}" >> .bashrc + printf "${GREEN}$(aws --version 2>&1)\n" +else + printf "${RED}failed to install !!\n" +fi + From f12e5cf97af049c753449dcd14d44afc61355fc5 Mon Sep 17 00:00:00 2001 From: J Gregory Date: Thu, 27 Sep 2018 16:38:29 +0100 Subject: [PATCH 71/71] Refactor tool location --- tools/{ => vagrant}/README.md | 0 tools/{ => vagrant}/Vagrantfile | 0 tools/{ => vagrant}/build/provision.sh | 0 tools/{ => vagrant}/build/provision_nosudo.sh | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename tools/{ => vagrant}/README.md (100%) rename tools/{ => vagrant}/Vagrantfile (100%) rename tools/{ => vagrant}/build/provision.sh (100%) rename tools/{ => vagrant}/build/provision_nosudo.sh (100%) diff --git a/tools/README.md b/tools/vagrant/README.md similarity index 100% rename from tools/README.md rename to tools/vagrant/README.md diff --git a/tools/Vagrantfile b/tools/vagrant/Vagrantfile similarity index 100% rename from tools/Vagrantfile rename to tools/vagrant/Vagrantfile diff --git a/tools/build/provision.sh b/tools/vagrant/build/provision.sh similarity index 100% rename from tools/build/provision.sh rename to tools/vagrant/build/provision.sh diff --git a/tools/build/provision_nosudo.sh b/tools/vagrant/build/provision_nosudo.sh similarity index 100% rename from tools/build/provision_nosudo.sh rename to tools/vagrant/build/provision_nosudo.sh