diff --git a/.gitignore b/.gitignore index 93221a17..be043623 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,5 @@ foo.yml *.lock.info **/.DS_Store + +data/ \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index 69342210..ee12b242 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,7 @@ [submodule "concourse-bosh-deployment"] path = concourse-bosh-deployment url = https://github.com/concourse/concourse-bosh-deployment.git + +[submodule "cf-rabbitmq-multitenant-broker-release"] + path = cf-rabbitmq-multitenant-broker-release + url = https://github.com/pivotal-cf/cf-rabbitmq-multitenant-broker-release.git diff --git a/README.md b/README.md index 41df814c..48dd4354 100644 --- a/README.md +++ b/README.md @@ -25,12 +25,17 @@ You'll need to create a `_vpc.tfvars` file with `az1`, `az2`, `region` and ```json { -"az1": "eu-west-1a", -"az2": "eu-west-1b", -"region": "eu-west-1", -"parent_dns_zone": "", -"ingress_whitelist": ["0.0.0.0/0"], -"slack_webhook_uri": "https://hooks.slack.com/services/" + "environment": "engineering", + "az1": "eu-west-1a", + "az2": "eu-west-1b", + "region": "eu-west-1", + "parent_dns_zone": "", + "ingress_whitelist": ["0.0.0.0/0"], + "availability_zones": ["eu-west-1a", "eu-west-1b", "eu-west-1c"], + "slack_webhook_uri": "https://hooks.slack.com/services/", + "cidr_blocks": { + "services": ["10.121.28.0/22", "10.121.32.0/22", "10.121.36.0/22"], + } } ``` diff --git a/bin/deploy_rabbitmq_service_and_broker.sh b/bin/deploy_rabbitmq_service_and_broker.sh new file mode 100755 index 00000000..242dc832 --- /dev/null +++ b/bin/deploy_rabbitmq_service_and_broker.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +set -euo pipefail + +: $ENVIRONMENT + +output() { + FILE=$1 + QUERY=$2 + + bin/outputs.sh $FILE | jq -r "$QUERY" +} + +secret() { + KEY=$1 + bin/secret.sh -e $ENVIRONMENT -d cf -k $KEY +} + +bin/get_states.sh -e $ENVIRONMENT + +BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh" + +SYSTEM_DOMAIN="system.$(output base .domain)" +APPS_DOMAIN="apps.$(output base .domain)" + +export BOSH_CA_CERT=$(bosh int --path /default_ca/ca "data/$ENVIRONMENT-bosh-variables.yml") + +RABBITMQ_STEMCELL_VERSION=3586.16 +BROKER_PLAN_UUID=22F0B28C-B886-4123-B01B-95E54D3DE6DA +BROKER_UUID=568725FD-AD46-44CA-9853-621416E983A4 + +$BOSH -d rabbitmq deploy -n cf-rabbitmq-multitenant-broker-release/manifests/cf-rabbitmq-broker-template.yml \ + -o cf-rabbitmq-multitenant-broker-release/manifests/add-cf-rabbitmq.yml \ + -o ./operations/rabbitmq/release.yml \ + -v deployment-name=rabbitmq \ + -v stemcell-version="'$RABBITMQ_STEMCELL_VERSION'" \ + -v system-domain="$SYSTEM_DOMAIN" \ + -v apps_domain="$APPS_DOMAIN" \ + -v multitenant-rabbitmq-broker-username=broker \ + -v product-name=p-rabbitmq \ + -v rabbitmq-management-username=admin \ + -v rabbitmq-broker-hostname=rabbitmq-multitenant-broker \ + -v rabbitmq-broker-password=broker \ + -v rabbitmq-broker-plan-uuid="$BROKER_PLAN_UUID" \ + -v rabbitmq-broker-protocol=https \ + -v rabbitmq-broker-username=broker \ + -v rabbitmq-broker-uuid="$BROKER_UUID" \ + -v rabbitmq-management-hostname=pivotal-rabbitmq \ + -v cf-admin-username=admin \ + -v cf-admin-password=$(bin/cf_password.sh -e "$ENVIRONMENT") \ + -v cluster-partition-handling-strategy=autoheal \ + -v haproxy-instances=1 \ + -v haproxy-stats-username=admin \ + -v rabbitmq-hosts=[] \ + --vars-store "${ENVIRONMENT}_rabbitmq_broker.creds.yml" + +# NB: passwords are auto-generated in the deployment manifest - see operations/rabbitmq/release.yml + +$BOSH -d rabbitmq run-errand broker-registrar + +$BOSH -d rabbitmq run-errand smoke-tests + +echo +echo +echo '*****************************************************' +echo 'The RabbitMQ service broker is installed and tested' +echo '*****************************************************' \ No newline at end of file diff --git a/bin/get_states.sh b/bin/get_states.sh new file mode 100755 index 00000000..759d70e5 --- /dev/null +++ b/bin/get_states.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -euo pipefail + +FILES= +OPTIONAL= +EXISTS= + +while getopts e:f:ox option; do + case $option in + e) export ENVIRONMENT="$OPTARG";; + f) FILES="$OPTARG $FILES";; + o) OPTIONAL=true;; + x) EXISTS=true;; + esac +done + +: $ENVIRONMENT + +[ "$OPTIONAL" = true ] && set +e + +if [ -z "$FILES" ]; then + aws s3 cp --recursive "s3://ons-paas-${ENVIRONMENT}-states" "data/" --acl=private +else + for FILE in $FILES; do + [ "$EXISTS" = true -a -f data/$FILE ] && continue + aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/$FILE" "data/" --acl=private + done +fi + +exit 0 \ No newline at end of file diff --git a/bin/outputs.sh b/bin/outputs.sh new file mode 100755 index 00000000..0dde0d36 --- /dev/null +++ b/bin/outputs.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -euo pipefail + +while getopts 'e:f' option; do + case $option in + e) export ENVIRONMENT="$OPTARG";; + esac +done +shift $((OPTIND-1)) + +: $ENVIRONMENT + +# STEP=${1:-base} + +bin/get_states.sh -e $ENVIRONMENT -x -f bosh/$ENVIRONMENT.tfstate +# bin/get_states.sh -e $ENVIRONMENT -x -f $ENVIRONMENT-$STEP.tfstate + +terraform output -state="data/bosh/$ENVIRONMENT.tfstate" -json | jq '. | with_entries(.value = .value.value)' \ No newline at end of file diff --git a/ci/deploy_pipeline.sh b/ci/deploy_pipeline.sh index 705883d1..1b8b7526 100755 --- a/ci/deploy_pipeline.sh +++ b/ci/deploy_pipeline.sh @@ -13,6 +13,7 @@ jumpbox_commit_ref="32c162b16f2a5a2639c78d905ba852487b93d507" bosh_commit_ref="04c85a5c79a9fa6b92775386a334104b9a165013" prometheus_commit_ref="a381c0af550550fc1d740ef409c2e2f22a589202" cf_tag="v2.7.0" +rabbitmq_broker_tag="v37.0.0" node_exporter_version="4.0.0" @@ -38,6 +39,7 @@ fly -t "$ENVIRONMENT" set-pipeline \ -v slack_webhook_uri="$slack_webhook_uri" \ -v prometheus_commit_ref="$prometheus_commit_ref" \ -v node_exporter_version="$node_exporter_version" \ + -v rabbitmq_broker_tag="$rabbitmq_broker_tag" \ -v cf_deployment_name="cf" \ -c ci/deploy_pipeline.yml -p deploy_pipeline -n diff --git a/ci/deploy_pipeline.yml b/ci/deploy_pipeline.yml index 3ced5480..7e0474c2 100644 --- a/ci/deploy_pipeline.yml +++ b/ci/deploy_pipeline.yml @@ -56,17 +56,12 @@ resources: type: git source: uri: https://github.com/cloudfoundry/cf-acceptance-tests.git - + - name: prometheus-deployment-git type: git source: uri: https://github.com/bosh-prometheus/prometheus-boshrelease.git -- name: cf-tests-git - type: git - source: - uri: https://github.com/ONSdigital/cf-tests - - name: jumpbox-terraform type: terraform source: @@ -953,32 +948,14 @@ jobs: - get: paas-bootstrap-git trigger: true passed: [verify-cf-endpoints] - - get: cf-vars-s3 + - get: cf-vars-s3 - task: get management variables file: paas-bootstrap-git/ci/tasks/cf/get_management_vars/task.yml params: ENVIRONMENT: ((environment)) - DOMAIN: ((domain)) + DOMAIN: ((domain)) - task: create orgs spaces etc file: paas-bootstrap-git/ci/tasks/cf/management/task.yml params: ENVIRONMENT: ((environment)) -- name: verify-shared-rds - serial: true - serial_groups: [rds] - plan: - - get: cf-tests-git - - get: trigger-weekday-morning - trigger: true - passed: [cf-management] - - get: paas-bootstrap-git - trigger: true - passed: [cf-management] - - get: cf-vars-s3 - - task: get cf credentials - file: paas-bootstrap-git/ci/tasks/common/get_cf_tester_credentials/task.yml - params: - DOMAIN: ((domain)) - - task: create service and test useability - file: paas-bootstrap-git/ci/tasks/rds_broker/test_shared/task.yml diff --git a/ci/tasks/cf/cloud_config/task.sh b/ci/tasks/cf/cloud_config/task.sh index 62de02ac..9f1924f5 100755 --- a/ci/tasks/cf/cloud_config/task.sh +++ b/ci/tasks/cf/cloud_config/task.sh @@ -2,6 +2,7 @@ set -euo pipefail +jq '.modules[0].outputs | with_entries(.value = .value.value)' < "bosh-tfstate-s3/${ENVIRONMENT}.tfstate" > bosh-vars.json jq '.modules[0].outputs | with_entries(.value = .value.value)' < vpc-tfstate-s3/tfstate.json > vpc-vars.json jq '.modules[0].outputs | with_entries(.value = .value.value)' < concourse-tfstate-s3/tfstate.json > concourse-vars.json jq '.modules[0].outputs | with_entries(.value = .value.value)' < "cf-tfstate-s3/${ENVIRONMENT}.tfstate" > cf-vars.json @@ -23,6 +24,7 @@ bosh update-cloud-config -n \ -o paas-bootstrap-git/operations/cloud-config/cf-s3-blobstore.yml \ -o paas-bootstrap-git/operations/cloud-config/cf-rds-sec-group.yml \ -o paas-bootstrap-git/operations/cloud-config/prometheus.yml \ + -o paas-bootstrap-git/operations/cloud-config/rabbitmq.yml \ -v az1="$(jq -r .az1 < vpc-vars.json)" \ -v az2="$(jq -r .az2 < vpc-vars.json)" \ -v az3="$(jq -r .az3 < vpc-vars.json)" \ @@ -40,6 +42,18 @@ bosh update-cloud-config -n \ -v private_subnet_az1_cidr="$(jq -r '.cf_internal_subnet_az1_cidr' < cf-vars.json)" \ -v private_subnet_az2_cidr="$(jq -r '.cf_internal_subnet_az2_cidr' < cf-vars.json)" \ -v private_subnet_az3_cidr="$(jq -r '.cf_internal_subnet_az3_cidr' < cf-vars.json)" \ + -v services_subnet_az1_cidr="$(jq -r .services_subnet_cidr_blocks[0] < cf-vars.json)" \ + -v services_subnet_az1_gateway="$(jq -r .services_subnet_gateway_ips[0] < cf-vars.json)" \ + -v reserved_services_az1_cidr="$(jq -r .services_subnet_reserved_cidr_blocks[0] < cf-vars.json)" \ + -v services_subnet_az1_id="$(jq -r .services_subnet_ids[0] < cf-vars.json)" \ + -v services_subnet_az2_cidr="$(jq -r .services_subnet_cidr_blocks[1] < cf-vars.json)" \ + -v services_subnet_az2_gateway="$(jq -r .services_subnet_gateway_ips[1] < cf-vars.json)" \ + -v reserved_services_az2_cidr="$(jq -r .services_subnet_reserved_cidr_blocks[1] < cf-vars.json)" \ + -v services_subnet_az2_id="$(jq -r .services_subnet_ids[1] < cf-vars.json)" \ + -v services_subnet_az3_cidr="$(jq -r .services_subnet_cidr_blocks[2] < cf-vars.json)" \ + -v services_subnet_az3_gateway="$(jq -r .services_subnet_gateway_ips[2] < cf-vars.json)" \ + -v reserved_services_az3_cidr="$(jq -r .services_subnet_reserved_cidr_blocks[2] < cf-vars.json)" \ + -v services_subnet_az3_id="$(jq -r .services_subnet_ids[2] < cf-vars.json)" \ -v cf-router-target-group-name="$(jq -r '.cf_router_target_group_name' < cf-vars.json)" \ -v cf-router-lb-internal-security-group-id="$(jq -r '.cf_router_lb_internal_security_group_id' < cf-vars.json)" \ -v cf-internal-security-group-id="$(jq -r '.cf_internal_security_group_id' < cf-vars.json)" \ @@ -53,6 +67,9 @@ bosh update-cloud-config -n \ -v prometheus_subnet_az1_gateway="$(jq -r .prometheus_subnet_az1_cidr < prometheus-vars.json | sed 's#0/24#1#')" \ -v grafana_target_group_name="$(jq -r .grafana_target_group_name < prometheus-vars.json)" \ -v prometheus_target_group_name="$(jq -r .prometheus_target_group_name < prometheus-vars.json)" \ - -v alertmanager_target_group_name="$(jq -r .alertmanager_target_group_name < prometheus-vars.json)" + -v alertmanager_target_group_name="$(jq -r .alertmanager_target_group_name < prometheus-vars.json)" \ + -v rabbitmq-broker-security-group-id="$(jq -r .rabbitmq_broker_security_group_id < cf-vars.json)" \ + -v rabbitmq-server-security-group-id="$(jq -r .rabbitmq_server_security_group_id < cf-vars.json)" \ + -v bosh-managed-security-group-id="$(jq -r .bosh_security_group_id < bosh-vars.json)" bosh cloud-config > cf-manifests/cloud-config.yml diff --git a/ci/tasks/cf/cloud_config/task.yml b/ci/tasks/cf/cloud_config/task.yml index b8991827..bef6f2fa 100644 --- a/ci/tasks/cf/cloud_config/task.yml +++ b/ci/tasks/cf/cloud_config/task.yml @@ -11,6 +11,7 @@ inputs: - name: cf-tfstate-s3 - name: bosh-vars-s3 - name: bosh-manifest-s3 + - name: bosh-tfstate-s3 - name: concourse-tfstate-s3 - name: prometheus-tfstate-s3 diff --git a/ci/tasks/rabbitmq_broker/deploy/task.sh b/ci/tasks/rabbitmq_broker/deploy/task.sh new file mode 100755 index 00000000..a53d7adc --- /dev/null +++ b/ci/tasks/rabbitmq_broker/deploy/task.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -euo pipefail + +RABBITMQ_MANIFESTS=rabbitmq-broker-deployment-git/manifests +SYSTEM_DOMAIN="system.${DOMAIN}" +APPS_DOMAIN="apps.${DOMAIN}" + +# TODO better configuration +RABBITMQ_STEMCELL_VERSION=3586.40 +BROKER_PLAN_UUID=22F0B28C-B886-4123-B01B-95E54D3DE6DA +BROKER_UUID=568725FD-AD46-44CA-9853-621416E983A4 + +bosh -d rabbitmq deploy -n "$RABBITMQ_MANIFESTS"/cf-rabbitmq-broker-template.yml \ + -o "$RABBITMQ_MANIFESTS"/add-cf-rabbitmq.yml \ + -o paas-bootstrap-git/operations/rabbitmq/release.yml \ + -v deployment-name=rabbitmq \ + -v stemcell-version="'$RABBITMQ_STEMCELL_VERSION'" \ + -v system-domain="$SYSTEM_DOMAIN" \ + -v apps_domain="$APPS_DOMAIN" \ + -v multitenant-rabbitmq-broker-username=broker \ + -v product-name=p-rabbitmq \ + -v rabbitmq-management-username=admin \ + -v rabbitmq-broker-hostname=rabbitmq-multitenant-broker \ + -v rabbitmq-broker-password=broker \ + -v rabbitmq-broker-plan-uuid="$BROKER_PLAN_UUID" \ + -v rabbitmq-broker-protocol=https \ + -v rabbitmq-broker-username=broker \ + -v rabbitmq-broker-uuid="$BROKER_UUID" \ + -v rabbitmq-management-hostname=pivotal-rabbitmq \ + -v cf-admin-username=admin \ + -v cf-admin-password=$(bin/cf_password.sh -e "$ENVIRONMENT") \ + -v cluster-partition-handling-strategy=autoheal \ + -v haproxy-instances=1 \ + -v haproxy-stats-username=admin \ + -v rabbitmq-hosts=[] + diff --git a/ci/tasks/rabbitmq_broker/deploy/task.yml b/ci/tasks/rabbitmq_broker/deploy/task.yml new file mode 100644 index 00000000..c0b86ea9 --- /dev/null +++ b/ci/tasks/rabbitmq_broker/deploy/task.yml @@ -0,0 +1,19 @@ +--- +platform: linux +image_resource: + type: docker-image + source: { repository: onsdigital/paas-ci-gp, tag: latest } + +inputs: + - name: paas-bootstrap-git + - name: rabbitmq-broker-deployment-git + +outputs: + - name: rabbitmq-broker-manifests + +run: + path: paas-bootstrap-git/ci/tasks/rabbitmq_broker/deploy/task.sh + +params: + ENVIRONMENT: + DOMAIN: diff --git a/cloud-config/cf/cloud-config.yml b/cloud-config/cf/cloud-config.yml index 0fe2cad5..fc95b0ea 100644 --- a/cloud-config/cf/cloud-config.yml +++ b/cloud-config/cf/cloud-config.yml @@ -43,6 +43,9 @@ disk_types: - name: 10GB disk_size: 10_000 cloud_properties: {encrypted: true, type: gp2} +- name: 50GB + disk_size: 50_000 + cloud_properties: {encrypted: true, type: gp2} - name: 100GB disk_size: 100_000 cloud_properties: {encrypted: true, type: gp2} @@ -93,7 +96,37 @@ networks: cloud_properties: subnet: ((private_subnet_az3_id)) security_groups: [ ((internal_security_group))] - +- name: services + type: manual + subnets: + - range: ((services_subnet_az1_cidr)) + gateway: ((services_subnet_az1_gateway)) + az: z1 + dns: + - ((private_dns_nameserver)) + - 1.1.1.1 + reserved: [((reserved_services_az1_cidr))] + cloud_properties: + subnet: ((services_subnet_az1_id)) + - range: ((services_subnet_az2_cidr)) + gateway: ((services_subnet_az2_gateway)) + az: z2 + dns: + - ((private_dns_nameserver)) + - 1.1.1.1 + reserved: [((reserved_services_az2_cidr))] + cloud_properties: + subnet: ((services_subnet_az2_id)) + - range: ((services_subnet_az3_cidr)) + gateway: ((services_subnet_az3_gateway)) + az: z3 + dns: + - ((private_dns_nameserver)) + - 1.1.1.1 + reserved: [((reserved_services_az3_cidr))] + cloud_properties: + subnet: ((services_subnet_az3_id)) + compilation: workers: 8 diff --git a/operations/cloud-config/rabbitmq.yml b/operations/cloud-config/rabbitmq.yml new file mode 100644 index 00000000..0e8ee4ef --- /dev/null +++ b/operations/cloud-config/rabbitmq.yml @@ -0,0 +1,20 @@ +--- +- type: replace + path: /vm_types/- + value: + name: n1-standard-1 + cloud_properties: + instance_type: t2.micro + security_groups: + - ((rabbitmq-broker-security-group-id)) + - ((bosh-managed-security-group-id)) + +- type: replace + path: /vm_types/- + value: + name: n1-highmem-2 + cloud_properties: + instance_type: m4.large + security_groups: + - ((rabbitmq-server-security-group-id)) + - ((bosh-managed-security-group-id)) diff --git a/operations/rabbitmq/release.yml b/operations/rabbitmq/release.yml new file mode 100644 index 00000000..5ea3070d --- /dev/null +++ b/operations/rabbitmq/release.yml @@ -0,0 +1,108 @@ +--- + +# https://bosh.io/releases/github.com/pivotal-cf/cf-rabbitmq-multitenant-broker-release?all=1 +- type: replace + path: /releases/name=cf-rabbitmq-multitenant-broker? + value: + name: "cf-rabbitmq-multitenant-broker" + version: "37.0.0" + url: "https://bosh.io/d/github.com/pivotal-cf/cf-rabbitmq-multitenant-broker-release?v=37.0.0" + sha1: "146049116e60593af959d1c738a139b59e8480a7" + +- type: remove + path: /releases/name=cf-rabbitmq-smoke-tests + +- type: replace + path: /releases/name=rabbitmq-smoke-tests? + value: + name: "rabbitmq-smoke-tests" + version: "4" + url: "https://bosh.io/d/github.com/cloudfoundry-community/rabbitmq-smoke-tests-boshrelease?v=4" + sha1: "06004fa77387f4443236117604870d9eeb93acb1" + +- type: replace + path: /releases/name=cf-rabbitmq? + value: + name: "cf-rabbitmq" + version: "251.0.0" + url: "https://bosh.io/d/github.com/pivotal-cf/cf-rabbitmq-release?v=251.0.0" + sha1: "6b26a898eb7d908b4325f6010917a228986c7ec0" + +- type: replace + path: /instance_groups/name=rmq-broker/networks + value: + - name: services + +- type: replace + path: /instance_groups/name=rmq/networks + value: + - name: services + +- type: replace + path: /instance_groups/name=haproxy/networks + value: + - name: services + +- type: replace + path: /instance_groups/name=rmq-broker/azs + value: [z1, z2, z3] + +- type: replace + path: /instance_groups/name=rmq/azs + value: [z1, z2, z3] + + +- type: replace + path: /instance_groups/name=haproxy/azs + value: [z1, z2, z3] + + +# FIXME: the pivotal RMQ release gives bogus examples +# - type: replace +# path: /instance_groups/name=rmq/jobs/name=rabbitmq-server/properties/rabbitmq-server/disk_alarm_threshold? +# value: +# mem_relative: 0.4 +- type: remove + path: /instance_groups/name=rmq/jobs/name=rabbitmq-server/properties/rabbitmq-server/disk_alarm_threshold + + +# The BOSH release appears not to be cf-rabbitmq-smoke-tests, so change to suit +- type: replace + path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests/release + value: rabbitmq-smoke-tests + +- type: remove + path: /instance_groups/name=rmq-broker/jobs/name=rabbitmq-broker/properties/rabbitmq-broker/rabbitmq/hosts + +- type: replace + path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests/properties/cf/apps_domain? + value: ((apps_domain)) + +- type: replace + path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests/properties/rabbitmq?/plan_names? + value: [standard] + +- type: replace + path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests/properties/rabbitmq?/service_name? + value: ((product-name)) + +# - type: remove +# path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests + +- type: replace + path: /variables?/- + value: + name: rabbitmq-management-password + type: password + +- type: replace + path: /variables?/- + value: + name: multitenant-rabbitmq-broker-password + type: password + +- type: replace + path: /variables?/- + value: + name: haproxy-stats-password + type: password \ No newline at end of file diff --git a/terraform/aws/locals.tf b/terraform/aws/locals.tf new file mode 100644 index 00000000..fe6a5cd7 --- /dev/null +++ b/terraform/aws/locals.tf @@ -0,0 +1,5 @@ +locals { + domain = "${replace(aws_route53_zone.child_zone.name, "/\\.$/", "")}" + num_azs = "${length(var.availability_zones)}" + services_subnets = "${var.cidr_blocks["services"]}" +} \ No newline at end of file diff --git a/terraform/aws/outputs.tf b/terraform/aws/outputs.tf index 1efdab9a..8c6d2af7 100644 --- a/terraform/aws/outputs.tf +++ b/terraform/aws/outputs.tf @@ -65,3 +65,15 @@ output "s3_prefix" { output "private_dns_zone" { value = "${aws_route53_zone.private.name}" } + +output "cidr_blocks" { + value = "${var.cidr_blocks}" +} + +output "services_cidr_block" { + value = "${var.cidr_blocks["services"]}" +} + +output "availability_zones" { + value = ["${var.availability_zones}"] +} \ No newline at end of file diff --git a/terraform/aws/vars.tf b/terraform/aws/vars.tf index 8d53d8ac..90f4670b 100644 --- a/terraform/aws/vars.tf +++ b/terraform/aws/vars.tf @@ -26,3 +26,15 @@ variable "az3" { variable "s3_prefix" { default = "ons-paas" } + +variable "availability_zones" { + type = "list" + default = [] +} + +variable "vpc_cidr_block" {} + +variable "cidr_blocks" { + type = "map" +} + diff --git a/terraform/aws/vpc.tf b/terraform/aws/vpc.tf index 57913906..08a72e2a 100644 --- a/terraform/aws/vpc.tf +++ b/terraform/aws/vpc.tf @@ -1,5 +1,5 @@ resource "aws_vpc" "default" { - cidr_block = "10.0.0.0/16" + cidr_block = "${var.vpc_cidr_block}" enable_dns_support = true enable_dns_hostnames = true diff --git a/terraform/bosh/aws/outputs.tf b/terraform/bosh/aws/outputs.tf index 20b3d7fd..1a2aeac2 100644 --- a/terraform/bosh/aws/outputs.tf +++ b/terraform/bosh/aws/outputs.tf @@ -77,3 +77,7 @@ output "bosh_rds_password" { output "bosh_director_fqdn" { value = "${aws_route53_record.bosh_director.fqdn}" } + +output "cidr_blocks" { + value = "${var.cidr_blocks}" +} diff --git a/terraform/bosh/aws/vars.tf b/terraform/bosh/aws/vars.tf index e8149107..dc576055 100644 --- a/terraform/bosh/aws/vars.tf +++ b/terraform/bosh/aws/vars.tf @@ -40,3 +40,7 @@ variable jumpbox_security_group_id {} variable "nat_az1_id" {} variable "private_dns_zone" {} + +variable "cidr_blocks" { + type = "map" +} \ No newline at end of file diff --git a/terraform/cf/aws/locals.tf b/terraform/cf/aws/locals.tf new file mode 100644 index 00000000..e415e6ed --- /dev/null +++ b/terraform/cf/aws/locals.tf @@ -0,0 +1,4 @@ +locals { + num_azs = "${length(var.availability_zones)}" + services_subnets = "${var.services_cidr_block}" +} \ No newline at end of file diff --git a/terraform/cf/aws/outputs.tf b/terraform/cf/aws/outputs.tf index f0eb94d2..76477cbd 100644 --- a/terraform/cf/aws/outputs.tf +++ b/terraform/cf/aws/outputs.tf @@ -104,4 +104,38 @@ output "cf_traffic_controller_port" { output "cf_dummy_db" { value = "${aws_db_instance.cf_rds.name}" -} \ No newline at end of file +} + +output "services_subnet_gateway_ips" { + value = [ + "${cidrhost(aws_subnet.services.*.cidr_block[0],1)}", + "${cidrhost(aws_subnet.services.*.cidr_block[1],1)}", + "${cidrhost(aws_subnet.services.*.cidr_block[2],1)}" + ] +} + +# NASTY HACK ALERT - we cannot find a way in terraform to perform an action on all elements of a list +# so you will have to change this if you add more AZs +output "services_subnet_reserved_cidr_blocks" { + value = [ + "${cidrsubnet(aws_subnet.services.*.cidr_block[0],7,0)}", + "${cidrsubnet(aws_subnet.services.*.cidr_block[1],7,0)}", + "${cidrsubnet(aws_subnet.services.*.cidr_block[2],7,0)}" + ] +} + +output "services_subnet_ids" { + value = ["${aws_subnet.services.*.id}"] +} + +output "services_subnet_cidr_blocks" { + value = ["${aws_subnet.services.*.cidr_block}"] +} + +output "rabbitmq_broker_security_group_id" { + value = "${aws_security_group.rabbitmq_broker.id}" +} + +output "rabbitmq_server_security_group_id" { + value = "${aws_security_group.rabbitmq_server.id}" +} diff --git a/terraform/cf/aws/rabbitmq-broker.tf b/terraform/cf/aws/rabbitmq-broker.tf new file mode 100644 index 00000000..c4211e31 --- /dev/null +++ b/terraform/cf/aws/rabbitmq-broker.tf @@ -0,0 +1,185 @@ +resource "aws_security_group" "rabbitmq_broker" { + name = "${var.environment}_rabbitmq_broker_security_group" + description = "RabbitMQ service broker access" + vpc_id = "${var.vpc_id}" + + tags { + Name = "${var.environment}-rabbitmq-broker-security-group" + Environment = "${var.environment}" + } +} + +resource "aws_security_group_rule" "cf_to_rmq_broker" { + security_group_id = "${aws_security_group.rabbitmq_broker.id}" + type = "ingress" + protocol = "tcp" + from_port = 4567 + to_port = 4567 + source_security_group_id = "${aws_security_group.internal.id}" + description = "CF may talk to RabbitMQ broker API" +} + +resource "aws_security_group_rule" "rmq_broker_outbound" { + security_group_id = "${aws_security_group.rabbitmq_broker.id}" + type = "egress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + cidr_blocks = ["0.0.0.0/0"] # FIXME: restrict to the CF ALB security group - 443, 8443 + description = "RabbitMQ broker outbound access" +} + +resource "aws_security_group_rule" "rmq_broker_self_tcp" { + security_group_id = "${aws_security_group.rabbitmq_broker.id}" + type = "ingress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + self = true + description = "RabbitMQ broker self TCP" +} + +resource "aws_security_group_rule" "rmq_broker_self_udp" { + security_group_id = "${aws_security_group.rabbitmq_broker.id}" + type = "ingress" + protocol = "udp" + from_port = 0 + to_port = 65535 + self = true + description = "RabbitMQ broker self UDP" +} + +resource "aws_security_group_rule" "rmq_broker_self_icmp" { + security_group_id = "${aws_security_group.rabbitmq_broker.id}" + type = "ingress" + protocol = "icmp" + from_port = -1 + to_port = -1 + self = true + description = "RabbitMQ broker self ICMP" +} + +resource "aws_security_group_rule" "rabbitmq_to_cf_nats" { + security_group_id = "${aws_security_group.internal.id}" + type = "ingress" + protocol = "tcp" + from_port = 4222 + to_port = 4222 + description = "Allow rabbitmq broker to access cf nats" + source_security_group_id = "${aws_security_group.rabbitmq_broker.id}" +} + +resource "aws_security_group_rule" "cf_to_rmq_5671_2" { + security_group_id = "${aws_security_group.rabbitmq_broker.id}" + type = "ingress" + protocol = "tcp" + from_port = 5671 + to_port = 5672 + source_security_group_id = "${aws_security_group.internal.id}" + description = "CF may talk to RabbitMQ on ports 5671,2" +} + +resource "aws_security_group_rule" "cf_to_rmq_1883" { + security_group_id = "${aws_security_group.rabbitmq_broker.id}" + type = "ingress" + protocol = "tcp" + from_port = 1883 + to_port = 1883 + source_security_group_id = "${aws_security_group.internal.id}" + description = "CF may talk to RabbitMQ on ports 5671,2" +} + +resource "aws_security_group_rule" "cf_to_rmq_8883" { + security_group_id = "${aws_security_group.rabbitmq_broker.id}" + type = "ingress" + protocol = "tcp" + from_port = 8883 + to_port = 8883 + source_security_group_id = "${aws_security_group.internal.id}" + description = "CF may talk to RabbitMQ on ports 8883" +} + +resource "aws_security_group_rule" "cf_to_rmq_61613_4" { + security_group_id = "${aws_security_group.rabbitmq_broker.id}" + type = "ingress" + protocol = "tcp" + from_port = 61613 + to_port = 61614 + source_security_group_id = "${aws_security_group.internal.id}" + description = "CF may talk to RabbitMQ on ports 61613,4" +} + +resource "aws_security_group_rule" "cf_to_rmq_15672" { + security_group_id = "${aws_security_group.rabbitmq_broker.id}" + type = "ingress" + protocol = "tcp" + from_port = 15672 + to_port = 15672 + source_security_group_id = "${aws_security_group.internal.id}" + description = "CF may talk to RabbitMQ on ports 15672" +} + +resource "aws_security_group_rule" "cf_to_rmq_15674" { + security_group_id = "${aws_security_group.rabbitmq_broker.id}" + type = "ingress" + protocol = "tcp" + from_port = 15674 + to_port = 15674 + source_security_group_id = "${aws_security_group.internal.id}" + description = "CF may talk to RabbitMQ on ports 15674" +} + + + +resource "aws_security_group" "rabbitmq_server" { + name = "${var.environment}_rabbitmq_server_security_group" + description = "RabbitMQ server access" + vpc_id = "${var.vpc_id}" + + tags { + Name = "${var.environment}-rabbitmq-server-security-group" + Environment = "${var.environment}" + } +} + +resource "aws_security_group_rule" "rmq_server_self_tcp" { + security_group_id = "${aws_security_group.rabbitmq_server.id}" + type = "ingress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + self = true + description = "RabbitMQ broker self TCP" +} + +resource "aws_security_group_rule" "rmq_server_self_udo" { + security_group_id = "${aws_security_group.rabbitmq_server.id}" + type = "ingress" + protocol = "udp" + from_port = 0 + to_port = 65535 + self = true + description = "RabbitMQ broker self UDP" +} + +resource "aws_security_group_rule" "rmq_server_self_icmp" { + security_group_id = "${aws_security_group.rabbitmq_server.id}" + type = "ingress" + protocol = "icmp" + from_port = -1 + to_port = -1 + self = true + description = "RabbitMQ server self ICMP" +} + +resource "aws_security_group_rule" "rmq_broker_to_rmq_server" { + security_group_id = "${aws_security_group.rabbitmq_server.id}" + type = "ingress" + protocol = "tcp" + from_port = 0 + to_port = 65535 + source_security_group_id = "${aws_security_group.rabbitmq_broker.id}" + description = "RabbitMQ broker can do everything to the servers" +} + + diff --git a/terraform/cf/aws/subnets.tf b/terraform/cf/aws/subnets.tf index 9e831f64..2694bf6c 100644 --- a/terraform/cf/aws/subnets.tf +++ b/terraform/cf/aws/subnets.tf @@ -69,3 +69,17 @@ resource "aws_subnet" "rds_az3" { Visibility = "private" } } + + +resource "aws_subnet" "services" { + count = "${local.num_azs}" + vpc_id = "${var.vpc_id}" + cidr_block = "${element(local.services_subnets, count.index)}" + availability_zone = "${element(var.availability_zones, count.index)}" + + tags { + Name = "${var.environment}-services-az${count.index+1}-subnet" + Environment = "${var.environment}" + Visibility = "private" + } +} \ No newline at end of file diff --git a/terraform/cf/aws/vars.tf b/terraform/cf/aws/vars.tf index 11eb67c1..1827dc5f 100644 --- a/terraform/cf/aws/vars.tf +++ b/terraform/cf/aws/vars.tf @@ -43,3 +43,13 @@ variable "concourse_public_ip" { } variable "s3_prefix" {} + +variable "availability_zones" { + type = "list" + default = [] +} + +variable "services_cidr_block" { + type = "list" + default = [] +} \ No newline at end of file