Cf 2.7.0 rabbitmq broker

.gitignore

@@ -19,3 +19,5 @@ foo.yml
 
 *.lock.info
 **/.DS_Store
+
+data/

.gitmodules

@@ -1,3 +1,7 @@
 [submodule "concourse-bosh-deployment"]
 	path = concourse-bosh-deployment
 	url = https://github.com/concourse/concourse-bosh-deployment.git
+
+[submodule "cf-rabbitmq-multitenant-broker-release"]
+	path = cf-rabbitmq-multitenant-broker-release
+	url = https://github.com/pivotal-cf/cf-rabbitmq-multitenant-broker-release.git

README.md

@@ -25,12 +25,17 @@ You'll need to create a `<env>_vpc.tfvars` file with `az1`, `az2`, `region` and
 
 ```json
 {
-"az1": "eu-west-1a",
-"az2": "eu-west-1b",
-"region": "eu-west-1",
-"parent_dns_zone": "<domain>",
-"ingress_whitelist": ["0.0.0.0/0"],
-"slack_webhook_uri": "https://hooks.slack.com/services/<generated uri>"
+  "environment": "engineering",
+  "az1": "eu-west-1a",
+  "az2": "eu-west-1b",
+  "region": "eu-west-1",
+  "parent_dns_zone": "<domain>",
+  "ingress_whitelist": ["0.0.0.0/0"],
+  "availability_zones": ["eu-west-1a", "eu-west-1b", "eu-west-1c"],
+  "slack_webhook_uri": "https://hooks.slack.com/services/<generated uri>",
+  "cidr_blocks": {
+    "services":   ["10.121.28.0/22", "10.121.32.0/22", "10.121.36.0/22"],
+  }
 }
 ```
 

bin/deploy_rabbitmq_service_and_broker.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+set -euo pipefail
+
+: $ENVIRONMENT
+
+output() {
+    FILE=$1
+    QUERY=$2
+
+    bin/outputs.sh $FILE | jq -r "$QUERY"
+}
+
+secret() {
+  KEY=$1
+  bin/secret.sh -e $ENVIRONMENT -d cf -k $KEY
+}
+
+bin/get_states.sh -e $ENVIRONMENT
+
+BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh"
+
+SYSTEM_DOMAIN="system.$(output base .domain)"
+APPS_DOMAIN="apps.$(output base .domain)"
+
+export BOSH_CA_CERT=$(bosh int --path /default_ca/ca "data/$ENVIRONMENT-bosh-variables.yml")
+
+RABBITMQ_STEMCELL_VERSION=3586.16
+BROKER_PLAN_UUID=22F0B28C-B886-4123-B01B-95E54D3DE6DA
+BROKER_UUID=568725FD-AD46-44CA-9853-621416E983A4
+
+$BOSH -d rabbitmq deploy -n cf-rabbitmq-multitenant-broker-release/manifests/cf-rabbitmq-broker-template.yml \
+  -o cf-rabbitmq-multitenant-broker-release/manifests/add-cf-rabbitmq.yml \
+  -o ./operations/rabbitmq/release.yml \
+  -v deployment-name=rabbitmq \
+  -v stemcell-version="'$RABBITMQ_STEMCELL_VERSION'" \
+  -v system-domain="$SYSTEM_DOMAIN" \
+  -v apps_domain="$APPS_DOMAIN" \
+  -v multitenant-rabbitmq-broker-username=broker \
+  -v product-name=p-rabbitmq \
+  -v rabbitmq-management-username=admin \
+  -v rabbitmq-broker-hostname=rabbitmq-multitenant-broker \
+  -v rabbitmq-broker-password=broker \
+  -v rabbitmq-broker-plan-uuid="$BROKER_PLAN_UUID" \
+  -v rabbitmq-broker-protocol=https \
+  -v rabbitmq-broker-username=broker \
+  -v rabbitmq-broker-uuid="$BROKER_UUID" \
+  -v rabbitmq-management-hostname=pivotal-rabbitmq \
+  -v cf-admin-username=admin \
+  -v cf-admin-password=$(bin/cf_password.sh -e "$ENVIRONMENT") \
+  -v cluster-partition-handling-strategy=autoheal \
+  -v haproxy-instances=1 \
+  -v haproxy-stats-username=admin \
+  -v rabbitmq-hosts=[] \
+  --vars-store "${ENVIRONMENT}_rabbitmq_broker.creds.yml"
+
+# NB: passwords are auto-generated in the deployment manifest - see operations/rabbitmq/release.yml
+
+$BOSH -d rabbitmq run-errand broker-registrar
+
+$BOSH -d rabbitmq run-errand smoke-tests
+
+echo
+echo
+echo '*****************************************************'
+echo 'The RabbitMQ service broker is installed and tested'
+echo '*****************************************************'

bin/get_states.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -euo pipefail
+
+FILES=
+OPTIONAL=
+EXISTS=
+
+while getopts e:f:ox option; do
+  case $option in
+    e) export ENVIRONMENT="$OPTARG";;
+    f) FILES="$OPTARG $FILES";;
+    o) OPTIONAL=true;;
+    x) EXISTS=true;;
+  esac
+done
+
+: $ENVIRONMENT
+
+[ "$OPTIONAL" = true ] && set +e
+
+if [ -z "$FILES" ]; then
+  aws s3 cp --recursive "s3://ons-paas-${ENVIRONMENT}-states" "data/" --acl=private
+else
+  for FILE in $FILES; do
+    [ "$EXISTS" = true -a -f data/$FILE ] && continue
+    aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/$FILE" "data/" --acl=private
+  done
+fi
+
+exit 0

bin/outputs.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -euo pipefail
+
+while getopts 'e:f' option; do
+  case $option in
+    e) export ENVIRONMENT="$OPTARG";;
+  esac
+done
+shift $((OPTIND-1))
+
+: $ENVIRONMENT
+
+# STEP=${1:-base}
+
+bin/get_states.sh -e $ENVIRONMENT -x -f bosh/$ENVIRONMENT.tfstate
+# bin/get_states.sh -e $ENVIRONMENT -x -f $ENVIRONMENT-$STEP.tfstate
+
+terraform output -state="data/bosh/$ENVIRONMENT.tfstate" -json | jq '. | with_entries(.value = .value.value)'

ci/deploy_pipeline.sh

@@ -13,6 +13,7 @@ jumpbox_commit_ref="32c162b16f2a5a2639c78d905ba852487b93d507"
 bosh_commit_ref="04c85a5c79a9fa6b92775386a334104b9a165013"
 prometheus_commit_ref="a381c0af550550fc1d740ef409c2e2f22a589202"
 cf_tag="v2.7.0"
+rabbitmq_broker_tag="v37.0.0"
 
 node_exporter_version="4.0.0"
 
@@ -38,6 +39,7 @@ fly -t "$ENVIRONMENT" set-pipeline \
     -v slack_webhook_uri="$slack_webhook_uri" \
     -v prometheus_commit_ref="$prometheus_commit_ref" \
     -v node_exporter_version="$node_exporter_version" \
+    -v rabbitmq_broker_tag="$rabbitmq_broker_tag" \
     -v cf_deployment_name="cf" \
     -c ci/deploy_pipeline.yml -p deploy_pipeline -n
 

ci/deploy_pipeline.yml

@@ -56,17 +56,12 @@ resources:
   type: git
   source:
     uri: https://github.com/cloudfoundry/cf-acceptance-tests.git
-    
+
 - name: prometheus-deployment-git
   type: git
   source:
     uri: https://github.com/bosh-prometheus/prometheus-boshrelease.git
 
-- name: cf-tests-git
-  type: git
-  source:
-    uri: https://github.com/ONSdigital/cf-tests
-
 - name: jumpbox-terraform
   type: terraform
   source:
@@ -953,32 +948,14 @@ jobs:
   - get: paas-bootstrap-git
     trigger: true
     passed: [verify-cf-endpoints]
-  - get: cf-vars-s3  
+  - get: cf-vars-s3
   - task: get management variables
     file: paas-bootstrap-git/ci/tasks/cf/get_management_vars/task.yml
     params:
       ENVIRONMENT: ((environment))
-      DOMAIN: ((domain))  
+      DOMAIN: ((domain))
   - task: create orgs spaces etc
     file: paas-bootstrap-git/ci/tasks/cf/management/task.yml
     params:
       ENVIRONMENT: ((environment))
 
-- name: verify-shared-rds
-  serial: true
-  serial_groups: [rds]
-  plan:
-  - get: cf-tests-git
-  - get: trigger-weekday-morning
-    trigger: true
-    passed: [cf-management]
-  - get: paas-bootstrap-git
-    trigger: true
-    passed: [cf-management]
-  - get: cf-vars-s3
-  - task: get cf credentials
-    file: paas-bootstrap-git/ci/tasks/common/get_cf_tester_credentials/task.yml
-    params:
-      DOMAIN: ((domain)) 
-  - task: create service and test useability
-    file: paas-bootstrap-git/ci/tasks/rds_broker/test_shared/task.yml

ci/tasks/cf/cloud_config/task.sh

@@ -2,6 +2,7 @@
 
 set -euo pipefail
 
+jq '.modules[0].outputs | with_entries(.value = .value.value)' < "bosh-tfstate-s3/${ENVIRONMENT}.tfstate" > bosh-vars.json
 jq '.modules[0].outputs | with_entries(.value = .value.value)' < vpc-tfstate-s3/tfstate.json > vpc-vars.json
 jq '.modules[0].outputs | with_entries(.value = .value.value)' < concourse-tfstate-s3/tfstate.json > concourse-vars.json
 jq '.modules[0].outputs | with_entries(.value = .value.value)' < "cf-tfstate-s3/${ENVIRONMENT}.tfstate" > cf-vars.json
@@ -23,6 +24,7 @@ bosh update-cloud-config -n \
   -o paas-bootstrap-git/operations/cloud-config/cf-s3-blobstore.yml \
   -o paas-bootstrap-git/operations/cloud-config/cf-rds-sec-group.yml \
   -o paas-bootstrap-git/operations/cloud-config/prometheus.yml \
+  -o paas-bootstrap-git/operations/cloud-config/rabbitmq.yml \
   -v az1="$(jq -r .az1 < vpc-vars.json)" \
   -v az2="$(jq -r .az2 < vpc-vars.json)" \
   -v az3="$(jq -r .az3 < vpc-vars.json)" \
@@ -40,6 +42,18 @@ bosh update-cloud-config -n \
   -v private_subnet_az1_cidr="$(jq -r '.cf_internal_subnet_az1_cidr' < cf-vars.json)" \
   -v private_subnet_az2_cidr="$(jq -r '.cf_internal_subnet_az2_cidr' < cf-vars.json)" \
   -v private_subnet_az3_cidr="$(jq -r '.cf_internal_subnet_az3_cidr' < cf-vars.json)" \
+  -v services_subnet_az1_cidr="$(jq -r .services_subnet_cidr_blocks[0] < cf-vars.json)" \
+  -v services_subnet_az1_gateway="$(jq -r .services_subnet_gateway_ips[0] < cf-vars.json)" \
+  -v reserved_services_az1_cidr="$(jq -r .services_subnet_reserved_cidr_blocks[0] < cf-vars.json)" \
+  -v services_subnet_az1_id="$(jq -r .services_subnet_ids[0] < cf-vars.json)" \
+  -v services_subnet_az2_cidr="$(jq -r .services_subnet_cidr_blocks[1] < cf-vars.json)" \
+  -v services_subnet_az2_gateway="$(jq -r .services_subnet_gateway_ips[1] < cf-vars.json)" \
+  -v reserved_services_az2_cidr="$(jq -r .services_subnet_reserved_cidr_blocks[1] < cf-vars.json)" \
+  -v services_subnet_az2_id="$(jq -r .services_subnet_ids[1] < cf-vars.json)" \
+  -v services_subnet_az3_cidr="$(jq -r .services_subnet_cidr_blocks[2] < cf-vars.json)" \
+  -v services_subnet_az3_gateway="$(jq -r .services_subnet_gateway_ips[2] < cf-vars.json)" \
+  -v reserved_services_az3_cidr="$(jq -r .services_subnet_reserved_cidr_blocks[2] < cf-vars.json)" \
+  -v services_subnet_az3_id="$(jq -r .services_subnet_ids[2] < cf-vars.json)" \
   -v cf-router-target-group-name="$(jq -r '.cf_router_target_group_name' < cf-vars.json)" \
   -v cf-router-lb-internal-security-group-id="$(jq -r '.cf_router_lb_internal_security_group_id' < cf-vars.json)" \
   -v cf-internal-security-group-id="$(jq -r '.cf_internal_security_group_id' < cf-vars.json)" \
@@ -53,6 +67,9 @@ bosh update-cloud-config -n \
   -v prometheus_subnet_az1_gateway="$(jq -r .prometheus_subnet_az1_cidr < prometheus-vars.json | sed 's#0/24#1#')" \
   -v grafana_target_group_name="$(jq -r .grafana_target_group_name < prometheus-vars.json)" \
   -v prometheus_target_group_name="$(jq -r .prometheus_target_group_name < prometheus-vars.json)" \
-  -v alertmanager_target_group_name="$(jq -r .alertmanager_target_group_name < prometheus-vars.json)" 
+  -v alertmanager_target_group_name="$(jq -r .alertmanager_target_group_name < prometheus-vars.json)" \
+  -v rabbitmq-broker-security-group-id="$(jq -r .rabbitmq_broker_security_group_id < cf-vars.json)" \
+  -v rabbitmq-server-security-group-id="$(jq -r .rabbitmq_server_security_group_id < cf-vars.json)" \
+  -v bosh-managed-security-group-id="$(jq -r .bosh_security_group_id < bosh-vars.json)"
 
 bosh cloud-config > cf-manifests/cloud-config.yml

ci/tasks/cf/cloud_config/task.yml

@@ -11,6 +11,7 @@ inputs:
   - name: cf-tfstate-s3
   - name: bosh-vars-s3
   - name: bosh-manifest-s3
+  - name: bosh-tfstate-s3
   - name: concourse-tfstate-s3
   - name: prometheus-tfstate-s3
 

ci/tasks/rabbitmq_broker/deploy/task.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+set -euo pipefail
+
+RABBITMQ_MANIFESTS=rabbitmq-broker-deployment-git/manifests
+SYSTEM_DOMAIN="system.${DOMAIN}"
+APPS_DOMAIN="apps.${DOMAIN}"
+
+# TODO better configuration
+RABBITMQ_STEMCELL_VERSION=3586.40
+BROKER_PLAN_UUID=22F0B28C-B886-4123-B01B-95E54D3DE6DA
+BROKER_UUID=568725FD-AD46-44CA-9853-621416E983A4
+
+bosh -d rabbitmq deploy -n "$RABBITMQ_MANIFESTS"/cf-rabbitmq-broker-template.yml \
+  -o "$RABBITMQ_MANIFESTS"/add-cf-rabbitmq.yml \
+  -o paas-bootstrap-git/operations/rabbitmq/release.yml \
+  -v deployment-name=rabbitmq \
+  -v stemcell-version="'$RABBITMQ_STEMCELL_VERSION'" \
+  -v system-domain="$SYSTEM_DOMAIN" \
+  -v apps_domain="$APPS_DOMAIN" \
+  -v multitenant-rabbitmq-broker-username=broker \
+  -v product-name=p-rabbitmq \
+  -v rabbitmq-management-username=admin \
+  -v rabbitmq-broker-hostname=rabbitmq-multitenant-broker \
+  -v rabbitmq-broker-password=broker \
+  -v rabbitmq-broker-plan-uuid="$BROKER_PLAN_UUID" \
+  -v rabbitmq-broker-protocol=https \
+  -v rabbitmq-broker-username=broker \
+  -v rabbitmq-broker-uuid="$BROKER_UUID" \
+  -v rabbitmq-management-hostname=pivotal-rabbitmq \
+  -v cf-admin-username=admin \
+  -v cf-admin-password=$(bin/cf_password.sh -e "$ENVIRONMENT") \
+  -v cluster-partition-handling-strategy=autoheal \
+  -v haproxy-instances=1 \
+  -v haproxy-stats-username=admin \
+  -v rabbitmq-hosts=[]
+

ci/tasks/rabbitmq_broker/deploy/task.yml

@@ -0,0 +1,19 @@
+---
+platform: linux
+image_resource:
+  type: docker-image
+  source: { repository: onsdigital/paas-ci-gp, tag: latest }
+
+inputs:
+  - name: paas-bootstrap-git
+  - name: rabbitmq-broker-deployment-git
+
+outputs:
+  - name: rabbitmq-broker-manifests
+
+run:
+  path: paas-bootstrap-git/ci/tasks/rabbitmq_broker/deploy/task.sh
+
+params:
+  ENVIRONMENT:
+  DOMAIN:

cloud-config/cf/cloud-config.yml

@@ -43,6 +43,9 @@ disk_types:
 - name: 10GB
   disk_size: 10_000
   cloud_properties: {encrypted: true, type: gp2}
+- name: 50GB
+  disk_size: 50_000
+  cloud_properties: {encrypted: true, type: gp2}
 - name: 100GB
   disk_size: 100_000
   cloud_properties: {encrypted: true, type: gp2}
@@ -93,7 +96,37 @@ networks:
     cloud_properties:
       subnet: ((private_subnet_az3_id))
       security_groups: [ ((internal_security_group))]
-
+- name: services
+  type: manual
+  subnets:
+  - range: ((services_subnet_az1_cidr))
+    gateway: ((services_subnet_az1_gateway))
+    az: z1
+    dns:
+    - ((private_dns_nameserver))
+    - 1.1.1.1
+    reserved: [((reserved_services_az1_cidr))]
+    cloud_properties:
+      subnet: ((services_subnet_az1_id))
+  - range: ((services_subnet_az2_cidr))
+    gateway: ((services_subnet_az2_gateway))
+    az: z2
+    dns:
+    - ((private_dns_nameserver))
+    - 1.1.1.1
+    reserved: [((reserved_services_az2_cidr))]
+    cloud_properties:
+      subnet: ((services_subnet_az2_id))
+  - range: ((services_subnet_az3_cidr))
+    gateway: ((services_subnet_az3_gateway))
+    az: z3
+    dns:
+    - ((private_dns_nameserver))
+    - 1.1.1.1
+    reserved: [((reserved_services_az3_cidr))]
+    cloud_properties:
+      subnet: ((services_subnet_az3_id))
+
 
 compilation:
   workers: 8