From 0f6e2679125c25a8bd6b26e0330c8c625c0eaca6 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Thu, 4 Oct 2018 15:03:45 +0100
Subject: [PATCH 01/37] Add 'domain' to terraform output

---
 terraform/bosh/aws/outputs.tf | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/terraform/bosh/aws/outputs.tf b/terraform/bosh/aws/outputs.tf
index 20b3d7fd..16723fa0 100644
--- a/terraform/bosh/aws/outputs.tf
+++ b/terraform/bosh/aws/outputs.tf
@@ -77,3 +77,7 @@ output "bosh_rds_password" {
 output "bosh_director_fqdn" {
   value = "${aws_route53_record.bosh_director.fqdn}"
 }
+
+output "domain" {
+  value = "${local.domain}"
+}
\ No newline at end of file

From 99e45c3c4cb24d4893279af308b10b6ac61c227b Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Thu, 4 Oct 2018 15:40:31 +0100
Subject: [PATCH 02/37] Reset output

---
 terraform/bosh/aws/outputs.tf | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/terraform/bosh/aws/outputs.tf b/terraform/bosh/aws/outputs.tf
index 16723fa0..20b3d7fd 100644
--- a/terraform/bosh/aws/outputs.tf
+++ b/terraform/bosh/aws/outputs.tf
@@ -77,7 +77,3 @@ output "bosh_rds_password" {
 output "bosh_director_fqdn" {
   value = "${aws_route53_record.bosh_director.fqdn}"
 }
-
-output "domain" {
-  value = "${local.domain}"
-}
\ No newline at end of file

From 9b95fdd90f2c63b6a9d850463eaae4bc2b5e6b72 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Thu, 4 Oct 2018 15:59:57 +0100
Subject: [PATCH 03/37] Add basic deploy task for rabbitmq broker

---
 ci/deploy_pipeline.yml                   | 34 ++++++++++++++++++----
 ci/tasks/rabbitmq_broker/deploy/task.sh  | 37 ++++++++++++++++++++++++
 ci/tasks/rabbitmq_broker/deploy/task.yml | 19 ++++++++++++
 3 files changed, 84 insertions(+), 6 deletions(-)
 create mode 100755 ci/tasks/rabbitmq_broker/deploy/task.sh
 create mode 100644 ci/tasks/rabbitmq_broker/deploy/task.yml

diff --git a/ci/deploy_pipeline.yml b/ci/deploy_pipeline.yml
index 3ced5480..eabd6c45 100644
--- a/ci/deploy_pipeline.yml
+++ b/ci/deploy_pipeline.yml
@@ -56,12 +56,17 @@ resources:
   type: git
   source:
     uri: https://github.com/cloudfoundry/cf-acceptance-tests.git
-    
+
 - name: prometheus-deployment-git
   type: git
   source:
     uri: https://github.com/bosh-prometheus/prometheus-boshrelease.git
 
+- name: rabbitmq-broker-deployment-git
+  type: git
+  source:
+    uri: https://github.com/pivotal-cf/cf-rabbitmq-multitenant-broker-release.git
+
 - name: cf-tests-git
   type: git
   source:
@@ -794,8 +799,8 @@ jobs:
     trigger: true
     passed: [deploy-cf]
   - get: paas-bootstrap-git
-    passed: [deploy-cf]
-    trigger: true
+    # passed: [deploy-cf]
+    # trigger: true
   - aggregate:
     - task: ensure CF api responds
       file: paas-bootstrap-git/ci/tasks/common/test_endpoint/task.yml
@@ -827,6 +832,23 @@ jobs:
   - task: ensure smoke tests pass
     file: paas-bootstrap-git/ci/tasks/cf/smoke_tests/task.yml
 
+- name: rabbitmq-broker
+  plan:
+  - aggregate:
+    - get: trigger-weekday-morning
+      trigger: true
+      passed: [verify-cf-endpoints]
+    - get: paas-bootstrap-git
+      passed: [verify-cf-endpoints]
+      trigger: true
+    - get: rabbitmq-broker-deployment-git
+      version: { ref: ((prometheus_commit_ref)) }
+  - task: interpolate-rabbitmq-broker
+    file: paas-bootstrap-git/ci/tasks/rabbitmq_broker/deploy/task.yml
+    params:
+      ENVIRONMENT: ((environment))
+      DOMAIN: ((domain))
+
 - name: cats
   serial: true
   serial_groups: [cats]
@@ -953,12 +975,12 @@ jobs:
   - get: paas-bootstrap-git
     trigger: true
     passed: [verify-cf-endpoints]
-  - get: cf-vars-s3  
+  - get: cf-vars-s3
   - task: get management variables
     file: paas-bootstrap-git/ci/tasks/cf/get_management_vars/task.yml
     params:
       ENVIRONMENT: ((environment))
-      DOMAIN: ((domain))  
+      DOMAIN: ((domain))
   - task: create orgs spaces etc
     file: paas-bootstrap-git/ci/tasks/cf/management/task.yml
     params:
@@ -979,6 +1001,6 @@ jobs:
   - task: get cf credentials
     file: paas-bootstrap-git/ci/tasks/common/get_cf_tester_credentials/task.yml
     params:
-      DOMAIN: ((domain)) 
+      DOMAIN: ((domain))
   - task: create service and test useability
     file: paas-bootstrap-git/ci/tasks/rds_broker/test_shared/task.yml
diff --git a/ci/tasks/rabbitmq_broker/deploy/task.sh b/ci/tasks/rabbitmq_broker/deploy/task.sh
new file mode 100755
index 00000000..7f999fd1
--- /dev/null
+++ b/ci/tasks/rabbitmq_broker/deploy/task.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+set -euo pipefail
+
+RABBITMQ_MANIFESTS=rabbitmq-broker-deployment-git/manifests
+SYSTEM_DOMAIN="system.${DOMAIN}"
+APPS_DOMAIN="apps.${DOMAIN}"
+
+# TODO better configuration
+RABBITMQ_STEMCELL_VERSION=3586.40
+BROKER_PLAN_UUID=22F0B28C-B886-4123-B01B-95E54D3DE6DA
+BROKER_UUID=568725FD-AD46-44CA-9853-621416E983A4
+
+bosh -d rabbitmq deploy -n "$RABBITMQ_MANIFESTS"/cf-rabbitmq-broker-template.yml \
+  -o "$RABBITMQ_MANIFESTS"/add-cf-rabbitmq.yml \
+  -o ./operations/rabbitmq/release.yml \
+  -v deployment-name=rabbitmq \
+  -v stemcell-version="'$RABBITMQ_STEMCELL_VERSION'" \
+  -v system-domain="$SYSTEM_DOMAIN" \
+  -v apps_domain="$APPS_DOMAIN" \
+  -v multitenant-rabbitmq-broker-username=broker \
+  -v product-name=p-rabbitmq \
+  -v rabbitmq-management-username=admin \
+  -v rabbitmq-broker-hostname=rabbitmq-multitenant-broker \
+  -v rabbitmq-broker-password=broker \
+  -v rabbitmq-broker-plan-uuid="$BROKER_PLAN_UUID" \
+  -v rabbitmq-broker-protocol=https \
+  -v rabbitmq-broker-username=broker \
+  -v rabbitmq-broker-uuid="$BROKER_UUID" \
+  -v rabbitmq-management-hostname=pivotal-rabbitmq \
+  -v cf-admin-username=admin \
+  -v cf-admin-password=$(bin/cf_password.sh -e "$ENVIRONMENT") \
+  -v cluster-partition-handling-strategy=autoheal \
+  -v haproxy-instances=1 \
+  -v haproxy-stats-username=admin \
+  -v rabbitmq-hosts=[]
+
diff --git a/ci/tasks/rabbitmq_broker/deploy/task.yml b/ci/tasks/rabbitmq_broker/deploy/task.yml
new file mode 100644
index 00000000..c0b86ea9
--- /dev/null
+++ b/ci/tasks/rabbitmq_broker/deploy/task.yml
@@ -0,0 +1,19 @@
+---
+platform: linux
+image_resource:
+  type: docker-image
+  source: { repository: onsdigital/paas-ci-gp, tag: latest }
+
+inputs:
+  - name: paas-bootstrap-git
+  - name: rabbitmq-broker-deployment-git
+
+outputs:
+  - name: rabbitmq-broker-manifests
+
+run:
+  path: paas-bootstrap-git/ci/tasks/rabbitmq_broker/deploy/task.sh
+
+params:
+  ENVIRONMENT:
+  DOMAIN:

From 94deca121c0270e033ece9d3059c127d1f7d6275 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Thu, 4 Oct 2018 17:19:03 +0100
Subject: [PATCH 04/37] Fix ops file location in task

---
 ci/deploy_pipeline.sh                   | 3 +++
 ci/deploy_pipeline.yml                  | 7 ++++---
 ci/tasks/rabbitmq_broker/deploy/task.sh | 2 +-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/ci/deploy_pipeline.sh b/ci/deploy_pipeline.sh
index 705883d1..251bcf69 100755
--- a/ci/deploy_pipeline.sh
+++ b/ci/deploy_pipeline.sh
@@ -13,6 +13,7 @@ jumpbox_commit_ref="32c162b16f2a5a2639c78d905ba852487b93d507"
 bosh_commit_ref="04c85a5c79a9fa6b92775386a334104b9a165013"
 prometheus_commit_ref="a381c0af550550fc1d740ef409c2e2f22a589202"
 cf_tag="v2.7.0"
+rabbitmq_broker_tag="v37.0.0"
 
 node_exporter_version="4.0.0"
 
@@ -38,6 +39,7 @@ fly -t "$ENVIRONMENT" set-pipeline \
     -v slack_webhook_uri="$slack_webhook_uri" \
     -v prometheus_commit_ref="$prometheus_commit_ref" \
     -v node_exporter_version="$node_exporter_version" \
+    -v rabbitmq_broker_tag="$rabbitmq_broker_tag" \
     -v cf_deployment_name="cf" \
     -c ci/deploy_pipeline.yml -p deploy_pipeline -n
 
@@ -47,4 +49,5 @@ fly -t "$ENVIRONMENT" expose-pipeline -p deploy_pipeline
 fly -t "$ENVIRONMENT" check-resource -r deploy_pipeline/jumpbox-deployment-git --from "ref:${jumpbox_commit_ref}"
 fly -t "$ENVIRONMENT" check-resource -r deploy_pipeline/bosh-deployment-git --from "ref:${bosh_commit_ref}"
 fly -t "$ENVIRONMENT" check-resource -r deploy_pipeline/cf-deployment-git --from "ref:${cf_tag}"
+fly -t "$ENVIRONMENT" check-resource -r deploy_pipeline/rabbitmq-broker-deployment-git --from "ref:${rabbitmq_broker_tag}"
 fly -t "$ENVIRONMENT" check-resource -r deploy_pipeline/prometheus-deployment-git --from "ref:${prometheus_commit_ref}"
\ No newline at end of file
diff --git a/ci/deploy_pipeline.yml b/ci/deploy_pipeline.yml
index eabd6c45..d2d6cda0 100644
--- a/ci/deploy_pipeline.yml
+++ b/ci/deploy_pipeline.yml
@@ -66,6 +66,7 @@ resources:
   type: git
   source:
     uri: https://github.com/pivotal-cf/cf-rabbitmq-multitenant-broker-release.git
+    tag_filter: v*
 
 - name: cf-tests-git
   type: git
@@ -796,8 +797,8 @@ jobs:
   serial_groups: [rds,prometheus,smoke-tests,cats]
   plan:
   - get: trigger-weekday-morning
-    trigger: true
-    passed: [deploy-cf]
+    # trigger: true
+    # passed: [deploy-cf]
   - get: paas-bootstrap-git
     # passed: [deploy-cf]
     # trigger: true
@@ -842,7 +843,7 @@ jobs:
       passed: [verify-cf-endpoints]
       trigger: true
     - get: rabbitmq-broker-deployment-git
-      version: { ref: ((prometheus_commit_ref)) }
+      version: { ref: ((rabbitmq_broker_tag)) }
   - task: interpolate-rabbitmq-broker
     file: paas-bootstrap-git/ci/tasks/rabbitmq_broker/deploy/task.yml
     params:
diff --git a/ci/tasks/rabbitmq_broker/deploy/task.sh b/ci/tasks/rabbitmq_broker/deploy/task.sh
index 7f999fd1..a53d7adc 100755
--- a/ci/tasks/rabbitmq_broker/deploy/task.sh
+++ b/ci/tasks/rabbitmq_broker/deploy/task.sh
@@ -13,7 +13,7 @@ BROKER_UUID=568725FD-AD46-44CA-9853-621416E983A4
 
 bosh -d rabbitmq deploy -n "$RABBITMQ_MANIFESTS"/cf-rabbitmq-broker-template.yml \
   -o "$RABBITMQ_MANIFESTS"/add-cf-rabbitmq.yml \
-  -o ./operations/rabbitmq/release.yml \
+  -o paas-bootstrap-git/operations/rabbitmq/release.yml \
   -v deployment-name=rabbitmq \
   -v stemcell-version="'$RABBITMQ_STEMCELL_VERSION'" \
   -v system-domain="$SYSTEM_DOMAIN" \

From 453e2e2a65e8c3284842ddc67e8b2f81daef2b6c Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Thu, 4 Oct 2018 17:25:21 +0100
Subject: [PATCH 05/37] Add rabbitmq ops file from cf-4.2.0 branch

---
 operations/rabbitmq/release.yml | 108 ++++++++++++++++++++++++++++++++
 1 file changed, 108 insertions(+)
 create mode 100644 operations/rabbitmq/release.yml

diff --git a/operations/rabbitmq/release.yml b/operations/rabbitmq/release.yml
new file mode 100644
index 00000000..5ea3070d
--- /dev/null
+++ b/operations/rabbitmq/release.yml
@@ -0,0 +1,108 @@
+---
+
+# https://bosh.io/releases/github.com/pivotal-cf/cf-rabbitmq-multitenant-broker-release?all=1
+- type: replace
+  path: /releases/name=cf-rabbitmq-multitenant-broker?
+  value:
+    name: "cf-rabbitmq-multitenant-broker"
+    version: "37.0.0"
+    url: "https://bosh.io/d/github.com/pivotal-cf/cf-rabbitmq-multitenant-broker-release?v=37.0.0"
+    sha1: "146049116e60593af959d1c738a139b59e8480a7"
+
+- type: remove
+  path: /releases/name=cf-rabbitmq-smoke-tests
+
+- type: replace
+  path: /releases/name=rabbitmq-smoke-tests?
+  value:
+    name: "rabbitmq-smoke-tests"
+    version: "4"
+    url: "https://bosh.io/d/github.com/cloudfoundry-community/rabbitmq-smoke-tests-boshrelease?v=4"
+    sha1: "06004fa77387f4443236117604870d9eeb93acb1"
+
+- type: replace
+  path: /releases/name=cf-rabbitmq?
+  value:
+    name: "cf-rabbitmq"
+    version: "251.0.0"
+    url: "https://bosh.io/d/github.com/pivotal-cf/cf-rabbitmq-release?v=251.0.0"
+    sha1: "6b26a898eb7d908b4325f6010917a228986c7ec0"
+
+- type: replace
+  path: /instance_groups/name=rmq-broker/networks
+  value:
+  - name: services
+
+- type: replace
+  path: /instance_groups/name=rmq/networks
+  value:
+  - name: services
+
+- type: replace
+  path: /instance_groups/name=haproxy/networks
+  value:
+  - name: services
+
+- type: replace
+  path: /instance_groups/name=rmq-broker/azs
+  value: [z1, z2, z3]
+
+- type: replace
+  path: /instance_groups/name=rmq/azs
+  value: [z1, z2, z3]
+
+
+- type: replace
+  path: /instance_groups/name=haproxy/azs
+  value: [z1, z2, z3]
+
+
+# FIXME: the pivotal RMQ release gives bogus examples
+# - type: replace
+#   path: /instance_groups/name=rmq/jobs/name=rabbitmq-server/properties/rabbitmq-server/disk_alarm_threshold?
+#   value:
+#     mem_relative: 0.4
+- type: remove
+  path: /instance_groups/name=rmq/jobs/name=rabbitmq-server/properties/rabbitmq-server/disk_alarm_threshold
+
+
+# The BOSH release appears not to be cf-rabbitmq-smoke-tests, so change to suit
+- type: replace
+  path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests/release
+  value: rabbitmq-smoke-tests
+
+- type: remove
+  path: /instance_groups/name=rmq-broker/jobs/name=rabbitmq-broker/properties/rabbitmq-broker/rabbitmq/hosts
+
+- type: replace
+  path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests/properties/cf/apps_domain?
+  value: ((apps_domain))
+
+- type: replace
+  path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests/properties/rabbitmq?/plan_names?
+  value: [standard]
+
+- type: replace
+  path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests/properties/rabbitmq?/service_name?
+  value: ((product-name))
+
+# - type: remove
+#   path: /instance_groups/name=rmq-broker/jobs/name=smoke-tests
+
+- type: replace
+  path: /variables?/-
+  value:
+    name: rabbitmq-management-password
+    type: password
+
+- type: replace
+  path: /variables?/-
+  value:
+    name: multitenant-rabbitmq-broker-password
+    type: password
+
+- type: replace
+  path: /variables?/-
+  value:
+    name: haproxy-stats-password
+    type: password
\ No newline at end of file

From 869690285dc553b9b19e660da4ac18910f4392c6 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Fri, 5 Oct 2018 13:07:22 +0100
Subject: [PATCH 06/37] Add scripts and config for rabbitmq

---
 bin/deploy_rabbitmq_service_and_broker.sh | 66 +++++++++++++++++++++++
 bin/get_states.sh                         | 31 +++++++++++
 bin/outputs.sh                            | 19 +++++++
 ci/tasks/cf/cloud_config/task.sh          |  3 +-
 operations/cloud-config/rabbitmq.yml      | 20 +++++++
 5 files changed, 138 insertions(+), 1 deletion(-)
 create mode 100755 bin/deploy_rabbitmq_service_and_broker.sh
 create mode 100755 bin/get_states.sh
 create mode 100755 bin/outputs.sh
 create mode 100644 operations/cloud-config/rabbitmq.yml

diff --git a/bin/deploy_rabbitmq_service_and_broker.sh b/bin/deploy_rabbitmq_service_and_broker.sh
new file mode 100755
index 00000000..e0846337
--- /dev/null
+++ b/bin/deploy_rabbitmq_service_and_broker.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+set -euo pipefail
+
+: $ENVIRONMENT
+
+output() {
+    FILE=$1
+    QUERY=$2
+
+    bin/outputs.sh $FILE | jq -r "$QUERY"
+}
+
+secret() {
+  KEY=$1
+  bin/secret.sh -e $ENVIRONMENT -d cf -k $KEY
+}
+
+bin/get_states.sh -e $ENVIRONMENT
+
+BOSH="bin/bosh_credentials.sh -e $ENVIRONMENT bosh"
+
+SYSTEM_DOMAIN="system.$(output base .domain)"
+APPS_DOMAIN="apps.$(output base .domain)"
+
+export BOSH_CA_CERT=$(bosh int --path /default_ca/ca "data/$ENVIRONMENT-bosh-variables.yml")
+
+RABBITMQ_STEMCELL_VERSION=3586.40
+BROKER_PLAN_UUID=22F0B28C-B886-4123-B01B-95E54D3DE6DA
+BROKER_UUID=568725FD-AD46-44CA-9853-621416E983A4
+
+$BOSH -d rabbitmq deploy -n cf-rabbitmq-multitenant-broker-release/manifests/cf-rabbitmq-broker-template.yml \
+  -o cf-rabbitmq-multitenant-broker-release/manifests/add-cf-rabbitmq.yml \
+  -o ./operations/rabbitmq/release.yml \
+  -v deployment-name=rabbitmq \
+  -v stemcell-version="'$RABBITMQ_STEMCELL_VERSION'" \
+  -v system-domain="$SYSTEM_DOMAIN" \
+  -v apps_domain="$APPS_DOMAIN" \
+  -v multitenant-rabbitmq-broker-username=broker \
+  -v product-name=p-rabbitmq \
+  -v rabbitmq-management-username=admin \
+  -v rabbitmq-broker-hostname=rabbitmq-multitenant-broker \
+  -v rabbitmq-broker-password=broker \
+  -v rabbitmq-broker-plan-uuid="$BROKER_PLAN_UUID" \
+  -v rabbitmq-broker-protocol=https \
+  -v rabbitmq-broker-username=broker \
+  -v rabbitmq-broker-uuid="$BROKER_UUID" \
+  -v rabbitmq-management-hostname=pivotal-rabbitmq \
+  -v cf-admin-username=admin \
+  -v cf-admin-password=$(bin/cf_password.sh -e "$ENVIRONMENT") \
+  -v cluster-partition-handling-strategy=autoheal \
+  -v haproxy-instances=1 \
+  -v haproxy-stats-username=admin \
+  -v rabbitmq-hosts=[]
+
+# NB: passwords are auto-generated in the deployment manifest - see operations/rabbitmq/release.yml
+
+$BOSH -d rabbitmq run-errand broker-registrar
+
+$BOSH -d rabbitmq run-errand smoke-tests
+
+echo
+echo
+echo '*****************************************************'
+echo 'The RabbitMQ service broker is installed and tested'
+echo '*****************************************************'
\ No newline at end of file
diff --git a/bin/get_states.sh b/bin/get_states.sh
new file mode 100755
index 00000000..759d70e5
--- /dev/null
+++ b/bin/get_states.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -euo pipefail
+
+FILES=
+OPTIONAL=
+EXISTS=
+
+while getopts e:f:ox option; do
+  case $option in
+    e) export ENVIRONMENT="$OPTARG";;
+    f) FILES="$OPTARG $FILES";;
+    o) OPTIONAL=true;;
+    x) EXISTS=true;;
+  esac
+done
+
+: $ENVIRONMENT
+
+[ "$OPTIONAL" = true ] && set +e
+
+if [ -z "$FILES" ]; then
+  aws s3 cp --recursive "s3://ons-paas-${ENVIRONMENT}-states" "data/" --acl=private
+else
+  for FILE in $FILES; do
+    [ "$EXISTS" = true -a -f data/$FILE ] && continue
+    aws s3 cp "s3://ons-paas-${ENVIRONMENT}-states/$FILE" "data/" --acl=private
+  done
+fi
+
+exit 0
\ No newline at end of file
diff --git a/bin/outputs.sh b/bin/outputs.sh
new file mode 100755
index 00000000..0dde0d36
--- /dev/null
+++ b/bin/outputs.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -euo pipefail
+
+while getopts 'e:f' option; do
+  case $option in
+    e) export ENVIRONMENT="$OPTARG";;
+  esac
+done
+shift $((OPTIND-1))
+
+: $ENVIRONMENT
+
+# STEP=${1:-base}
+
+bin/get_states.sh -e $ENVIRONMENT -x -f bosh/$ENVIRONMENT.tfstate
+# bin/get_states.sh -e $ENVIRONMENT -x -f $ENVIRONMENT-$STEP.tfstate
+
+terraform output -state="data/bosh/$ENVIRONMENT.tfstate" -json | jq '. | with_entries(.value = .value.value)'
\ No newline at end of file
diff --git a/ci/tasks/cf/cloud_config/task.sh b/ci/tasks/cf/cloud_config/task.sh
index 62de02ac..2b56a098 100755
--- a/ci/tasks/cf/cloud_config/task.sh
+++ b/ci/tasks/cf/cloud_config/task.sh
@@ -23,6 +23,7 @@ bosh update-cloud-config -n \
   -o paas-bootstrap-git/operations/cloud-config/cf-s3-blobstore.yml \
   -o paas-bootstrap-git/operations/cloud-config/cf-rds-sec-group.yml \
   -o paas-bootstrap-git/operations/cloud-config/prometheus.yml \
+  -o paas-bootstrap-git/operations/cloud-config/rabbitmq.yml \
   -v az1="$(jq -r .az1 < vpc-vars.json)" \
   -v az2="$(jq -r .az2 < vpc-vars.json)" \
   -v az3="$(jq -r .az3 < vpc-vars.json)" \
@@ -53,6 +54,6 @@ bosh update-cloud-config -n \
   -v prometheus_subnet_az1_gateway="$(jq -r .prometheus_subnet_az1_cidr < prometheus-vars.json | sed 's#0/24#1#')" \
   -v grafana_target_group_name="$(jq -r .grafana_target_group_name < prometheus-vars.json)" \
   -v prometheus_target_group_name="$(jq -r .prometheus_target_group_name < prometheus-vars.json)" \
-  -v alertmanager_target_group_name="$(jq -r .alertmanager_target_group_name < prometheus-vars.json)" 
+  -v alertmanager_target_group_name="$(jq -r .alertmanager_target_group_name < prometheus-vars.json)"
 
 bosh cloud-config > cf-manifests/cloud-config.yml
diff --git a/operations/cloud-config/rabbitmq.yml b/operations/cloud-config/rabbitmq.yml
new file mode 100644
index 00000000..0e8ee4ef
--- /dev/null
+++ b/operations/cloud-config/rabbitmq.yml
@@ -0,0 +1,20 @@
+---
+- type: replace
+  path: /vm_types/-
+  value:
+    name: n1-standard-1
+    cloud_properties:
+      instance_type: t2.micro
+      security_groups:
+      - ((rabbitmq-broker-security-group-id))
+      - ((bosh-managed-security-group-id))
+
+- type: replace
+  path: /vm_types/-
+  value:
+    name: n1-highmem-2
+    cloud_properties:
+      instance_type: m4.large
+      security_groups:
+      - ((rabbitmq-server-security-group-id))
+      - ((bosh-managed-security-group-id))

From 9f7cd1b5b46cd0d9d8f9e4158814095196884c1e Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Fri, 5 Oct 2018 13:37:39 +0100
Subject: [PATCH 07/37] Adding 'services' subnet

---
 .gitignore                       |  2 ++
 .gitmodules                      |  4 ++++
 README.md                        | 15 +++++++++------
 ci/tasks/cf/cloud_config/task.sh | 10 ++++++++++
 cloud-config/cf/cloud-config.yml | 32 +++++++++++++++++++++++++++++++-
 terraform/aws/vars.tf            |  6 ++++++
 terraform/cf/aws/outputs.tf      | 26 ++++++++++++++++++++++++++
 terraform/cf/aws/subnets.tf      | 14 ++++++++++++++
 8 files changed, 102 insertions(+), 7 deletions(-)

diff --git a/.gitignore b/.gitignore
index 93221a17..be043623 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,5 @@ foo.yml
 
 *.lock.info
 **/.DS_Store
+
+data/
\ No newline at end of file
diff --git a/.gitmodules b/.gitmodules
index 69342210..ee12b242 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +1,7 @@
 [submodule "concourse-bosh-deployment"]
 	path = concourse-bosh-deployment
 	url = https://github.com/concourse/concourse-bosh-deployment.git
+
+[submodule "cf-rabbitmq-multitenant-broker-release"]
+	path = cf-rabbitmq-multitenant-broker-release
+	url = https://github.com/pivotal-cf/cf-rabbitmq-multitenant-broker-release.git
diff --git a/README.md b/README.md
index 41df814c..ef2105bc 100644
--- a/README.md
+++ b/README.md
@@ -25,12 +25,15 @@ You'll need to create a `<env>_vpc.tfvars` file with `az1`, `az2`, `region` and
 
 ```json
 {
-"az1": "eu-west-1a",
-"az2": "eu-west-1b",
-"region": "eu-west-1",
-"parent_dns_zone": "<domain>",
-"ingress_whitelist": ["0.0.0.0/0"],
-"slack_webhook_uri": "https://hooks.slack.com/services/<generated uri>"
+  "az1": "eu-west-1a",
+  "az2": "eu-west-1b",
+  "region": "eu-west-1",
+  "parent_dns_zone": "<domain>",
+  "ingress_whitelist": ["0.0.0.0/0"],
+  "slack_webhook_uri": "https://hooks.slack.com/services/<generated uri>",
+  "cidr_blocks": {
+    "services":   ["10.121.28.0/22", "10.121.32.0/22", "10.121.36.0/22"],
+  }
 }
 ```
 
diff --git a/ci/tasks/cf/cloud_config/task.sh b/ci/tasks/cf/cloud_config/task.sh
index 2b56a098..612dcccd 100755
--- a/ci/tasks/cf/cloud_config/task.sh
+++ b/ci/tasks/cf/cloud_config/task.sh
@@ -41,6 +41,16 @@ bosh update-cloud-config -n \
   -v private_subnet_az1_cidr="$(jq -r '.cf_internal_subnet_az1_cidr' < cf-vars.json)" \
   -v private_subnet_az2_cidr="$(jq -r '.cf_internal_subnet_az2_cidr' < cf-vars.json)" \
   -v private_subnet_az3_cidr="$(jq -r '.cf_internal_subnet_az3_cidr' < cf-vars.json)" \
+  -v services_subnet_az1_cidr="$(output base .services_subnet_cidr_blocks[0])" \
+  -v services_subnet_az1_gateway="$(output base .services_subnet_gateway_ips[0])" \
+  -v reserved_services_az1_cidr="$(output base .services_subnet_reserved_cidr_blocks[0])" \
+  -v services_subnet_az1_id="$(output base .services_subnet_ids[0])" \
+  -v services_subnet_az2_cidr="$(output base .services_subnet_cidr_blocks[1])" \
+  -v services_subnet_az2_gateway="$(output base .services_subnet_gateway_ips[1])" \
+  -v reserved_services_az2_cidr="$(output base .services_subnet_reserved_cidr_blocks[1])" \
+  -v services_subnet_az2_id="$(output base .services_subnet_ids[1])" \
+  -v services_subnet_az3_cidr="$(output base .services_subnet_cidr_blocks[2])" \
+  -v services_subnet_az3_gateway="$(output base .services_subnet_gateway_ips[2])" \
   -v cf-router-target-group-name="$(jq -r '.cf_router_target_group_name' < cf-vars.json)" \
   -v cf-router-lb-internal-security-group-id="$(jq -r '.cf_router_lb_internal_security_group_id' < cf-vars.json)" \
   -v cf-internal-security-group-id="$(jq -r '.cf_internal_security_group_id' < cf-vars.json)" \
diff --git a/cloud-config/cf/cloud-config.yml b/cloud-config/cf/cloud-config.yml
index 0fe2cad5..d124ef0d 100644
--- a/cloud-config/cf/cloud-config.yml
+++ b/cloud-config/cf/cloud-config.yml
@@ -93,7 +93,37 @@ networks:
     cloud_properties:
       subnet: ((private_subnet_az3_id))
       security_groups: [ ((internal_security_group))]
-  
+- name: services
+  type: manual
+  subnets:
+  - range: ((services_subnet_az1_cidr))
+    gateway: ((services_subnet_az1_gateway))
+    az: z1
+    dns:
+    - ((private_dns_nameserver))
+    - 1.1.1.1
+    reserved: [((reserved_services_az1_cidr))]
+    cloud_properties:
+      subnet: ((services_subnet_az1_id))
+  - range: ((services_subnet_az2_cidr))
+    gateway: ((services_subnet_az2_gateway))
+    az: z2
+    dns:
+    - ((private_dns_nameserver))
+    - 1.1.1.1
+    reserved: [((reserved_services_az2_cidr))]
+    cloud_properties:
+      subnet: ((services_subnet_az2_id))
+  - range: ((services_subnet_az3_cidr))
+    gateway: ((services_subnet_az3_gateway))
+    az: z3
+    dns:
+    - ((private_dns_nameserver))
+    - 1.1.1.1
+    reserved: [((reserved_services_az3_cidr))]
+    cloud_properties:
+      subnet: ((services_subnet_az3_id))
+
 
 compilation:
   workers: 8
diff --git a/terraform/aws/vars.tf b/terraform/aws/vars.tf
index 8d53d8ac..11b48bcc 100644
--- a/terraform/aws/vars.tf
+++ b/terraform/aws/vars.tf
@@ -26,3 +26,9 @@ variable "az3" {
 variable "s3_prefix" {
   default = "ons-paas"
 }
+
+variable "vpc_cidr_block" {}
+
+variable "cidr_blocks" {
+  type = "map"
+}
\ No newline at end of file
diff --git a/terraform/cf/aws/outputs.tf b/terraform/cf/aws/outputs.tf
index f0eb94d2..eeef4635 100644
--- a/terraform/cf/aws/outputs.tf
+++ b/terraform/cf/aws/outputs.tf
@@ -104,4 +104,30 @@ output "cf_traffic_controller_port" {
 
 output "cf_dummy_db" {
   value = "${aws_db_instance.cf_rds.name}"
+}
+
+output "services_subnet_gateway_ips" {
+  value = [
+      "${cidrhost(aws_subnet.services.*.cidr_block[0],1)}",
+      "${cidrhost(aws_subnet.services.*.cidr_block[1],1)}",
+      "${cidrhost(aws_subnet.services.*.cidr_block[2],1)}"
+  ]
+}
+
+# NASTY HACK ALERT - we cannot find a way in terraform to perform an action on all elements of a list
+#                    so you will have to change this if you add more AZs
+output "services_subnet_reserved_cidr_blocks" {
+  value = [
+      "${cidrsubnet(aws_subnet.services.*.cidr_block[0],7,0)}",
+      "${cidrsubnet(aws_subnet.services.*.cidr_block[1],7,0)}",
+      "${cidrsubnet(aws_subnet.services.*.cidr_block[2],7,0)}"
+  ]
+}
+
+output "services_subnet_ids" {
+    value = ["${aws_subnet.services.*.id}"]
+}
+
+output "services_subnet_cidr_blocks" {
+    value = ["${aws_subnet.services.*.cidr_block}"]
 }
\ No newline at end of file
diff --git a/terraform/cf/aws/subnets.tf b/terraform/cf/aws/subnets.tf
index 9e831f64..72e9424f 100644
--- a/terraform/cf/aws/subnets.tf
+++ b/terraform/cf/aws/subnets.tf
@@ -69,3 +69,17 @@ resource "aws_subnet" "rds_az3" {
     Visibility  = "private"
   }
 }
+
+
+resource "aws_subnet" "services" {
+  count = "${local.num_azs}"
+  vpc_id            = "${aws_vpc.default.id}"
+  cidr_block        = "${element(local.services_subnets, count.index)}"
+  availability_zone = "${element(var.availability_zones, count.index)}"
+
+  tags {
+    Name        = "${var.environment}-services-az${count.index+1}-subnet"
+    Environment = "${var.environment}"
+    Visibility  = "private"
+  }
+}
\ No newline at end of file

From 149e6a45d87c2d395a98c6930e422dab0cadb63a Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 8 Oct 2018 08:23:31 +0100
Subject: [PATCH 08/37] Update cf terraform

---
 terraform/aws/vpc.tf        | 2 +-
 terraform/cf/aws/subnets.tf | 2 +-
 terraform/cf/aws/vars.tf    | 5 +++++
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/terraform/aws/vpc.tf b/terraform/aws/vpc.tf
index 57913906..08a72e2a 100644
--- a/terraform/aws/vpc.tf
+++ b/terraform/aws/vpc.tf
@@ -1,5 +1,5 @@
 resource "aws_vpc" "default" {
-  cidr_block           = "10.0.0.0/16"
+  cidr_block           = "${var.vpc_cidr_block}"
   enable_dns_support   = true
   enable_dns_hostnames = true
 
diff --git a/terraform/cf/aws/subnets.tf b/terraform/cf/aws/subnets.tf
index 72e9424f..2694bf6c 100644
--- a/terraform/cf/aws/subnets.tf
+++ b/terraform/cf/aws/subnets.tf
@@ -73,7 +73,7 @@ resource "aws_subnet" "rds_az3" {
 
 resource "aws_subnet" "services" {
   count = "${local.num_azs}"
-  vpc_id            = "${aws_vpc.default.id}"
+  vpc_id            = "${var.vpc_id}"
   cidr_block        = "${element(local.services_subnets, count.index)}"
   availability_zone = "${element(var.availability_zones, count.index)}"
 
diff --git a/terraform/cf/aws/vars.tf b/terraform/cf/aws/vars.tf
index 11eb67c1..03230fce 100644
--- a/terraform/cf/aws/vars.tf
+++ b/terraform/cf/aws/vars.tf
@@ -43,3 +43,8 @@ variable "concourse_public_ip" {
 }
 
 variable "s3_prefix" {}
+
+variable "availability_zones" {
+  type = "list"
+  default = []
+}
\ No newline at end of file

From 2f2f07896a07b9ed0bde3431d428d3ad7403853b Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 8 Oct 2018 08:31:23 +0100
Subject: [PATCH 09/37] Add locals.tf

---
 terraform/aws/locals.tf | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 terraform/aws/locals.tf

diff --git a/terraform/aws/locals.tf b/terraform/aws/locals.tf
new file mode 100644
index 00000000..0606df9e
--- /dev/null
+++ b/terraform/aws/locals.tf
@@ -0,0 +1,6 @@
+locals {
+    account_id = "${data.aws_caller_identity.current.account_id}"
+    domain = "${replace(aws_route53_zone.child_zone.name, "/\\.$/", "")}"
+    num_azs = "${length(var.availability_zones)}"
+    services_subnets = "${var.cidr_blocks["services"]}"
+}
\ No newline at end of file

From 5a559ba98ffb3a2ec9d0bc4824314c4f7d564871 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 8 Oct 2018 08:35:52 +0100
Subject: [PATCH 10/37] Add locals.tf to cf terraform

---
 terraform/cf/aws/locals.tf | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 terraform/cf/aws/locals.tf

diff --git a/terraform/cf/aws/locals.tf b/terraform/cf/aws/locals.tf
new file mode 100644
index 00000000..0606df9e
--- /dev/null
+++ b/terraform/cf/aws/locals.tf
@@ -0,0 +1,6 @@
+locals {
+    account_id = "${data.aws_caller_identity.current.account_id}"
+    domain = "${replace(aws_route53_zone.child_zone.name, "/\\.$/", "")}"
+    num_azs = "${length(var.availability_zones)}"
+    services_subnets = "${var.cidr_blocks["services"]}"
+}
\ No newline at end of file

From 4b1ca5e62fcd6aaf79de5c11288119f921786cf3 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 8 Oct 2018 09:25:35 +0100
Subject: [PATCH 11/37] Add cidr_blocks var to cf tf

---
 terraform/cf/aws/vars.tf | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/terraform/cf/aws/vars.tf b/terraform/cf/aws/vars.tf
index 03230fce..50b10340 100644
--- a/terraform/cf/aws/vars.tf
+++ b/terraform/cf/aws/vars.tf
@@ -47,4 +47,8 @@ variable "s3_prefix" {}
 variable "availability_zones" {
   type = "list"
   default = []
+}
+
+variable "cidr_blocks" {
+  type = "map"
 }
\ No newline at end of file

From a1c309a33ed19f045515af990fe46bad092fc685 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 8 Oct 2018 09:51:28 +0100
Subject: [PATCH 12/37] Add cidr_blocks to base outputs

---
 terraform/aws/outputs.tf | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/terraform/aws/outputs.tf b/terraform/aws/outputs.tf
index 1efdab9a..2073254e 100644
--- a/terraform/aws/outputs.tf
+++ b/terraform/aws/outputs.tf
@@ -65,3 +65,7 @@ output "s3_prefix" {
 output "private_dns_zone" {
   value = "${aws_route53_zone.private.name}"
 }
+
+output "cidr_blocks" {
+  value = "${var.cidr_blocks}"
+}
\ No newline at end of file

From 0577647221abaf50250ebd1cffd1bb9141160886 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 8 Oct 2018 09:56:13 +0100
Subject: [PATCH 13/37] Add availability_zones var

---
 terraform/aws/vars.tf | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/terraform/aws/vars.tf b/terraform/aws/vars.tf
index 11b48bcc..90f4670b 100644
--- a/terraform/aws/vars.tf
+++ b/terraform/aws/vars.tf
@@ -27,8 +27,14 @@ variable "s3_prefix" {
   default = "ons-paas"
 }
 
+variable "availability_zones" {
+  type = "list"
+  default = []
+}
+
 variable "vpc_cidr_block" {}
 
 variable "cidr_blocks" {
   type = "map"
-}
\ No newline at end of file
+}
+

From 03c3fad5268f8ef89c42a282dd944c935758b225 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 8 Oct 2018 09:57:37 +0100
Subject: [PATCH 14/37] Remove unused local

---
 terraform/aws/locals.tf    | 1 -
 terraform/cf/aws/locals.tf | 1 -
 2 files changed, 2 deletions(-)

diff --git a/terraform/aws/locals.tf b/terraform/aws/locals.tf
index 0606df9e..fe6a5cd7 100644
--- a/terraform/aws/locals.tf
+++ b/terraform/aws/locals.tf
@@ -1,5 +1,4 @@
 locals {
-    account_id = "${data.aws_caller_identity.current.account_id}"
     domain = "${replace(aws_route53_zone.child_zone.name, "/\\.$/", "")}"
     num_azs = "${length(var.availability_zones)}"
     services_subnets = "${var.cidr_blocks["services"]}"
diff --git a/terraform/cf/aws/locals.tf b/terraform/cf/aws/locals.tf
index 0606df9e..fe6a5cd7 100644
--- a/terraform/cf/aws/locals.tf
+++ b/terraform/cf/aws/locals.tf
@@ -1,5 +1,4 @@
 locals {
-    account_id = "${data.aws_caller_identity.current.account_id}"
     domain = "${replace(aws_route53_zone.child_zone.name, "/\\.$/", "")}"
     num_azs = "${length(var.availability_zones)}"
     services_subnets = "${var.cidr_blocks["services"]}"

From 9fe3974ad1a0fc7460c35f9a162137a144432ae5 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 8 Oct 2018 10:30:36 +0100
Subject: [PATCH 15/37] Remove var

---
 terraform/aws/outputs.tf | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/terraform/aws/outputs.tf b/terraform/aws/outputs.tf
index 2073254e..1efdab9a 100644
--- a/terraform/aws/outputs.tf
+++ b/terraform/aws/outputs.tf
@@ -65,7 +65,3 @@ output "s3_prefix" {
 output "private_dns_zone" {
   value = "${aws_route53_zone.private.name}"
 }
-
-output "cidr_blocks" {
-  value = "${var.cidr_blocks}"
-}
\ No newline at end of file

From 422087d252e66fec7f433da81ae96d73fc58d6f8 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 8 Oct 2018 10:48:39 +0100
Subject: [PATCH 16/37] Reinstate outputs

---
 terraform/bosh/aws/outputs.tf | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/terraform/bosh/aws/outputs.tf b/terraform/bosh/aws/outputs.tf
index 20b3d7fd..86341b63 100644
--- a/terraform/bosh/aws/outputs.tf
+++ b/terraform/bosh/aws/outputs.tf
@@ -77,3 +77,7 @@ output "bosh_rds_password" {
 output "bosh_director_fqdn" {
   value = "${aws_route53_record.bosh_director.fqdn}"
 }
+
+output "cidr_blocks" {
+  value = "${va.cidr_blocks}"
+}
\ No newline at end of file

From 0b856a723f3f9257913847793d86632786aefd30 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 8 Oct 2018 10:53:11 +0100
Subject: [PATCH 17/37] Fix typo

---
 terraform/bosh/aws/outputs.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/bosh/aws/outputs.tf b/terraform/bosh/aws/outputs.tf
index 86341b63..3ce7dfc2 100644
--- a/terraform/bosh/aws/outputs.tf
+++ b/terraform/bosh/aws/outputs.tf
@@ -79,5 +79,5 @@ output "bosh_director_fqdn" {
 }
 
 output "cidr_blocks" {
-  value = "${va.cidr_blocks}"
+  value = "${var.cidr_blocks}"
 }
\ No newline at end of file

From 341915d684c08e44274b6026c8f2ecbd2a8364a6 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 8 Oct 2018 10:56:53 +0100
Subject: [PATCH 18/37] Add cidr_blocks var to bosh

---
 terraform/bosh/aws/vars.tf | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/terraform/bosh/aws/vars.tf b/terraform/bosh/aws/vars.tf
index e8149107..dc576055 100644
--- a/terraform/bosh/aws/vars.tf
+++ b/terraform/bosh/aws/vars.tf
@@ -40,3 +40,7 @@ variable jumpbox_security_group_id {}
 variable "nat_az1_id" {}
 
 variable "private_dns_zone" {}
+
+variable "cidr_blocks" {
+  type = "map"
+}
\ No newline at end of file

From fd544e1d1c16c7d3b9940f2aabf639dec86e9ec1 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 8 Oct 2018 10:59:58 +0100
Subject: [PATCH 19/37] Reinstate correct output

---
 terraform/aws/outputs.tf | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/terraform/aws/outputs.tf b/terraform/aws/outputs.tf
index 1efdab9a..2073254e 100644
--- a/terraform/aws/outputs.tf
+++ b/terraform/aws/outputs.tf
@@ -65,3 +65,7 @@ output "s3_prefix" {
 output "private_dns_zone" {
   value = "${aws_route53_zone.private.name}"
 }
+
+output "cidr_blocks" {
+  value = "${var.cidr_blocks}"
+}
\ No newline at end of file

From a4a9c97952ecfda11b4992204df7143354cd59be Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Wed, 10 Oct 2018 15:47:26 +0100
Subject: [PATCH 20/37] Test passing list instead of map

---
 terraform/aws/outputs.tf   | 6 +++++-
 terraform/cf/aws/locals.tf | 2 +-
 terraform/cf/aws/vars.tf   | 4 +---
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/terraform/aws/outputs.tf b/terraform/aws/outputs.tf
index 2073254e..b92d3e50 100644
--- a/terraform/aws/outputs.tf
+++ b/terraform/aws/outputs.tf
@@ -68,4 +68,8 @@ output "private_dns_zone" {
 
 output "cidr_blocks" {
   value = "${var.cidr_blocks}"
-}
\ No newline at end of file
+}
+
+output "services_cidr_block" {
+  value = "${var.cidr_blocks["services"]}"
+}
diff --git a/terraform/cf/aws/locals.tf b/terraform/cf/aws/locals.tf
index fe6a5cd7..bf133b27 100644
--- a/terraform/cf/aws/locals.tf
+++ b/terraform/cf/aws/locals.tf
@@ -1,5 +1,5 @@
 locals {
     domain = "${replace(aws_route53_zone.child_zone.name, "/\\.$/", "")}"
     num_azs = "${length(var.availability_zones)}"
-    services_subnets = "${var.cidr_blocks["services"]}"
+    services_subnets = "${var.services_cidr_block}"
 }
\ No newline at end of file
diff --git a/terraform/cf/aws/vars.tf b/terraform/cf/aws/vars.tf
index 50b10340..ae71548f 100644
--- a/terraform/cf/aws/vars.tf
+++ b/terraform/cf/aws/vars.tf
@@ -49,6 +49,4 @@ variable "availability_zones" {
   default = []
 }
 
-variable "cidr_blocks" {
-  type = "map"
-}
\ No newline at end of file
+variable "services_cidr_block" {}
\ No newline at end of file

From 101a29a28004a8c5188b246a4f784952be1156b1 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Wed, 10 Oct 2018 15:54:36 +0100
Subject: [PATCH 21/37] Fix var type

---
 terraform/cf/aws/vars.tf | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/terraform/cf/aws/vars.tf b/terraform/cf/aws/vars.tf
index ae71548f..1827dc5f 100644
--- a/terraform/cf/aws/vars.tf
+++ b/terraform/cf/aws/vars.tf
@@ -49,4 +49,7 @@ variable "availability_zones" {
   default = []
 }
 
-variable "services_cidr_block" {}
\ No newline at end of file
+variable "services_cidr_block" {
+  type = "list"
+  default = []
+}
\ No newline at end of file

From 012543a27d5d3be6f3933ead560fcbc7fac68f62 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Wed, 10 Oct 2018 15:58:24 +0100
Subject: [PATCH 22/37] Remove unused local

---
 terraform/cf/aws/locals.tf | 1 -
 1 file changed, 1 deletion(-)

diff --git a/terraform/cf/aws/locals.tf b/terraform/cf/aws/locals.tf
index bf133b27..e415e6ed 100644
--- a/terraform/cf/aws/locals.tf
+++ b/terraform/cf/aws/locals.tf
@@ -1,5 +1,4 @@
 locals {
-    domain = "${replace(aws_route53_zone.child_zone.name, "/\\.$/", "")}"
     num_azs = "${length(var.availability_zones)}"
     services_subnets = "${var.services_cidr_block}"
 }
\ No newline at end of file

From 2989d242d68984d489b55284a64af3f0cf4b9154 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Thu, 11 Oct 2018 11:43:41 +0100
Subject: [PATCH 23/37] Tweaking cf deploy task

---
 ci/tasks/cf/cloud_config/task.sh | 30 ++++++++++++++++++++----------
 terraform/aws/outputs.tf         |  4 ++++
 terraform/bosh/aws/outputs.tf    |  2 +-
 terraform/cf/aws/outputs.tf      |  2 +-
 4 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/ci/tasks/cf/cloud_config/task.sh b/ci/tasks/cf/cloud_config/task.sh
index 612dcccd..37e1d085 100755
--- a/ci/tasks/cf/cloud_config/task.sh
+++ b/ci/tasks/cf/cloud_config/task.sh
@@ -41,16 +41,26 @@ bosh update-cloud-config -n \
   -v private_subnet_az1_cidr="$(jq -r '.cf_internal_subnet_az1_cidr' < cf-vars.json)" \
   -v private_subnet_az2_cidr="$(jq -r '.cf_internal_subnet_az2_cidr' < cf-vars.json)" \
   -v private_subnet_az3_cidr="$(jq -r '.cf_internal_subnet_az3_cidr' < cf-vars.json)" \
-  -v services_subnet_az1_cidr="$(output base .services_subnet_cidr_blocks[0])" \
-  -v services_subnet_az1_gateway="$(output base .services_subnet_gateway_ips[0])" \
-  -v reserved_services_az1_cidr="$(output base .services_subnet_reserved_cidr_blocks[0])" \
-  -v services_subnet_az1_id="$(output base .services_subnet_ids[0])" \
-  -v services_subnet_az2_cidr="$(output base .services_subnet_cidr_blocks[1])" \
-  -v services_subnet_az2_gateway="$(output base .services_subnet_gateway_ips[1])" \
-  -v reserved_services_az2_cidr="$(output base .services_subnet_reserved_cidr_blocks[1])" \
-  -v services_subnet_az2_id="$(output base .services_subnet_ids[1])" \
-  -v services_subnet_az3_cidr="$(output base .services_subnet_cidr_blocks[2])" \
-  -v services_subnet_az3_gateway="$(output base .services_subnet_gateway_ips[2])" \
+  -v services_subnet_az1_cidr="$(jq -r .services_subnet_cidr_blocks[0] < cf-vars.json)" \
+  -v services_subnet_az1_gateway="$(jq -r .services_subnet_gateway_ips[0] < cf-vars.json)" \
+  -v reserved_services_az1_cidr="$(jq -r .services_subnet_reserved_cidr_blocks[0] < cf-vars.json)" \
+  -v services_subnet_az1_id="$(jq -r .services_subnet_ids[0] < cf-vars.json)" \
+  -v services_subnet_az2_cidr="$(jq -r .services_subnet_cidr_blocks[1] < cf-vars.json)" \
+  -v services_subnet_az2_gateway="$(jq -r .services_subnet_gateway_ips[1] < cf-vars.json)" \
+  -v reserved_services_az2_cidr="$(jq -r .services_subnet_reserved_cidr_blocks[1] < cf-vars.json)" \
+  -v services_subnet_az2_id="$(jq -r .services_subnet_ids[1] < cf-vars.json)" \
+  -v services_subnet_az3_cidr="$(jq -r .services_subnet_cidr_blocks[2] < cf-vars.json)" \
+  -v services_subnet_az3_gateway="$(jq -r .services_subnet_gateway_ips[2] < cf-vars.json)" \
+  # -v services_subnet_az1_cidr="$(output base .services_subnet_cidr_blocks[0])" \
+  # -v services_subnet_az1_gateway="$(output base .services_subnet_gateway_ips[0])" \
+  # -v reserved_services_az1_cidr="$(output base .services_subnet_reserved_cidr_blocks[0])" \
+  # -v services_subnet_az1_id="$(output base .services_subnet_ids[0])" \
+  # -v services_subnet_az2_cidr="$(output base .services_subnet_cidr_blocks[1])" \
+  # -v services_subnet_az2_gateway="$(output base .services_subnet_gateway_ips[1])" \
+  # -v reserved_services_az2_cidr="$(output base .services_subnet_reserved_cidr_blocks[1])" \
+  # -v services_subnet_az2_id="$(output base .services_subnet_ids[1])" \
+  # -v services_subnet_az3_cidr="$(output base .services_subnet_cidr_blocks[2])" \
+  # -v services_subnet_az3_gateway="$(output base .services_subnet_gateway_ips[2])" \
   -v cf-router-target-group-name="$(jq -r '.cf_router_target_group_name' < cf-vars.json)" \
   -v cf-router-lb-internal-security-group-id="$(jq -r '.cf_router_lb_internal_security_group_id' < cf-vars.json)" \
   -v cf-internal-security-group-id="$(jq -r '.cf_internal_security_group_id' < cf-vars.json)" \
diff --git a/terraform/aws/outputs.tf b/terraform/aws/outputs.tf
index b92d3e50..8c6d2af7 100644
--- a/terraform/aws/outputs.tf
+++ b/terraform/aws/outputs.tf
@@ -73,3 +73,7 @@ output "cidr_blocks" {
 output "services_cidr_block" {
   value = "${var.cidr_blocks["services"]}"
 }
+
+output "availability_zones" {
+  value = ["${var.availability_zones}"]
+}
\ No newline at end of file
diff --git a/terraform/bosh/aws/outputs.tf b/terraform/bosh/aws/outputs.tf
index 3ce7dfc2..1a2aeac2 100644
--- a/terraform/bosh/aws/outputs.tf
+++ b/terraform/bosh/aws/outputs.tf
@@ -80,4 +80,4 @@ output "bosh_director_fqdn" {
 
 output "cidr_blocks" {
   value = "${var.cidr_blocks}"
-}
\ No newline at end of file
+}
diff --git a/terraform/cf/aws/outputs.tf b/terraform/cf/aws/outputs.tf
index eeef4635..8fe60fcc 100644
--- a/terraform/cf/aws/outputs.tf
+++ b/terraform/cf/aws/outputs.tf
@@ -130,4 +130,4 @@ output "services_subnet_ids" {
 
 output "services_subnet_cidr_blocks" {
     value = ["${aws_subnet.services.*.cidr_block}"]
-}
\ No newline at end of file
+}

From b7f9f1fc4faa35d2fd89dd83a0cbc72d8f87e717 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Thu, 11 Oct 2018 11:52:38 +0100
Subject: [PATCH 24/37] Remove unneeded code

---
 ci/tasks/cf/cloud_config/task.sh | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/ci/tasks/cf/cloud_config/task.sh b/ci/tasks/cf/cloud_config/task.sh
index 37e1d085..8735619e 100755
--- a/ci/tasks/cf/cloud_config/task.sh
+++ b/ci/tasks/cf/cloud_config/task.sh
@@ -51,16 +51,6 @@ bosh update-cloud-config -n \
   -v services_subnet_az2_id="$(jq -r .services_subnet_ids[1] < cf-vars.json)" \
   -v services_subnet_az3_cidr="$(jq -r .services_subnet_cidr_blocks[2] < cf-vars.json)" \
   -v services_subnet_az3_gateway="$(jq -r .services_subnet_gateway_ips[2] < cf-vars.json)" \
-  # -v services_subnet_az1_cidr="$(output base .services_subnet_cidr_blocks[0])" \
-  # -v services_subnet_az1_gateway="$(output base .services_subnet_gateway_ips[0])" \
-  # -v reserved_services_az1_cidr="$(output base .services_subnet_reserved_cidr_blocks[0])" \
-  # -v services_subnet_az1_id="$(output base .services_subnet_ids[0])" \
-  # -v services_subnet_az2_cidr="$(output base .services_subnet_cidr_blocks[1])" \
-  # -v services_subnet_az2_gateway="$(output base .services_subnet_gateway_ips[1])" \
-  # -v reserved_services_az2_cidr="$(output base .services_subnet_reserved_cidr_blocks[1])" \
-  # -v services_subnet_az2_id="$(output base .services_subnet_ids[1])" \
-  # -v services_subnet_az3_cidr="$(output base .services_subnet_cidr_blocks[2])" \
-  # -v services_subnet_az3_gateway="$(output base .services_subnet_gateway_ips[2])" \
   -v cf-router-target-group-name="$(jq -r '.cf_router_target_group_name' < cf-vars.json)" \
   -v cf-router-lb-internal-security-group-id="$(jq -r '.cf_router_lb_internal_security_group_id' < cf-vars.json)" \
   -v cf-internal-security-group-id="$(jq -r '.cf_internal_security_group_id' < cf-vars.json)" \

From 02ba837c3831caacdf1bf23d583019099b49e8c4 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Thu, 11 Oct 2018 12:13:53 +0100
Subject: [PATCH 25/37] Add missing var to bosh update-cloud-config

---
 ci/tasks/cf/cloud_config/task.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/tasks/cf/cloud_config/task.sh b/ci/tasks/cf/cloud_config/task.sh
index 8735619e..c2f94cfe 100755
--- a/ci/tasks/cf/cloud_config/task.sh
+++ b/ci/tasks/cf/cloud_config/task.sh
@@ -51,6 +51,7 @@ bosh update-cloud-config -n \
   -v services_subnet_az2_id="$(jq -r .services_subnet_ids[1] < cf-vars.json)" \
   -v services_subnet_az3_cidr="$(jq -r .services_subnet_cidr_blocks[2] < cf-vars.json)" \
   -v services_subnet_az3_gateway="$(jq -r .services_subnet_gateway_ips[2] < cf-vars.json)" \
+  -v reserved_services_az3_cidr="$(jq -r .services_subnet_reserved_cidr_blocks[2] < cf-vars.json)" \
   -v cf-router-target-group-name="$(jq -r '.cf_router_target_group_name' < cf-vars.json)" \
   -v cf-router-lb-internal-security-group-id="$(jq -r '.cf_router_lb_internal_security_group_id' < cf-vars.json)" \
   -v cf-internal-security-group-id="$(jq -r '.cf_internal_security_group_id' < cf-vars.json)" \

From d170fe66acdd57e9e80fc486aa4902efb938251c Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Thu, 11 Oct 2018 13:36:01 +0100
Subject: [PATCH 26/37] Add 50G disk type to cloud config

---
 cloud-config/cf/cloud-config.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cloud-config/cf/cloud-config.yml b/cloud-config/cf/cloud-config.yml
index d124ef0d..fc95b0ea 100644
--- a/cloud-config/cf/cloud-config.yml
+++ b/cloud-config/cf/cloud-config.yml
@@ -43,6 +43,9 @@ disk_types:
 - name: 10GB
   disk_size: 10_000
   cloud_properties: {encrypted: true, type: gp2}
+- name: 50GB
+  disk_size: 50_000
+  cloud_properties: {encrypted: true, type: gp2}
 - name: 100GB
   disk_size: 100_000
   cloud_properties: {encrypted: true, type: gp2}

From 564bbf1c596f824340a0368a9ecc431b3a6f55fe Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Thu, 11 Oct 2018 16:20:38 +0100
Subject: [PATCH 27/37] Updating rabbitmq deploy - currently fails on reading
 vars

---
 bin/deploy_rabbitmq_service_and_broker.sh |  5 ++-
 ci/deploy_pipeline.sh                     |  1 -
 ci/deploy_pipeline.yml                    | 54 ++---------------------
 ci/tasks/cf/cloud_config/task.sh          |  2 +
 4 files changed, 9 insertions(+), 53 deletions(-)

diff --git a/bin/deploy_rabbitmq_service_and_broker.sh b/bin/deploy_rabbitmq_service_and_broker.sh
index e0846337..242dc832 100755
--- a/bin/deploy_rabbitmq_service_and_broker.sh
+++ b/bin/deploy_rabbitmq_service_and_broker.sh
@@ -25,7 +25,7 @@ APPS_DOMAIN="apps.$(output base .domain)"
 
 export BOSH_CA_CERT=$(bosh int --path /default_ca/ca "data/$ENVIRONMENT-bosh-variables.yml")
 
-RABBITMQ_STEMCELL_VERSION=3586.40
+RABBITMQ_STEMCELL_VERSION=3586.16
 BROKER_PLAN_UUID=22F0B28C-B886-4123-B01B-95E54D3DE6DA
 BROKER_UUID=568725FD-AD46-44CA-9853-621416E983A4
 
@@ -51,7 +51,8 @@ $BOSH -d rabbitmq deploy -n cf-rabbitmq-multitenant-broker-release/manifests/cf-
   -v cluster-partition-handling-strategy=autoheal \
   -v haproxy-instances=1 \
   -v haproxy-stats-username=admin \
-  -v rabbitmq-hosts=[]
+  -v rabbitmq-hosts=[] \
+  --vars-store "${ENVIRONMENT}_rabbitmq_broker.creds.yml"
 
 # NB: passwords are auto-generated in the deployment manifest - see operations/rabbitmq/release.yml
 
diff --git a/ci/deploy_pipeline.sh b/ci/deploy_pipeline.sh
index 251bcf69..1b8b7526 100755
--- a/ci/deploy_pipeline.sh
+++ b/ci/deploy_pipeline.sh
@@ -49,5 +49,4 @@ fly -t "$ENVIRONMENT" expose-pipeline -p deploy_pipeline
 fly -t "$ENVIRONMENT" check-resource -r deploy_pipeline/jumpbox-deployment-git --from "ref:${jumpbox_commit_ref}"
 fly -t "$ENVIRONMENT" check-resource -r deploy_pipeline/bosh-deployment-git --from "ref:${bosh_commit_ref}"
 fly -t "$ENVIRONMENT" check-resource -r deploy_pipeline/cf-deployment-git --from "ref:${cf_tag}"
-fly -t "$ENVIRONMENT" check-resource -r deploy_pipeline/rabbitmq-broker-deployment-git --from "ref:${rabbitmq_broker_tag}"
 fly -t "$ENVIRONMENT" check-resource -r deploy_pipeline/prometheus-deployment-git --from "ref:${prometheus_commit_ref}"
\ No newline at end of file
diff --git a/ci/deploy_pipeline.yml b/ci/deploy_pipeline.yml
index d2d6cda0..7e0474c2 100644
--- a/ci/deploy_pipeline.yml
+++ b/ci/deploy_pipeline.yml
@@ -62,17 +62,6 @@ resources:
   source:
     uri: https://github.com/bosh-prometheus/prometheus-boshrelease.git
 
-- name: rabbitmq-broker-deployment-git
-  type: git
-  source:
-    uri: https://github.com/pivotal-cf/cf-rabbitmq-multitenant-broker-release.git
-    tag_filter: v*
-
-- name: cf-tests-git
-  type: git
-  source:
-    uri: https://github.com/ONSdigital/cf-tests
-
 - name: jumpbox-terraform
   type: terraform
   source:
@@ -797,11 +786,11 @@ jobs:
   serial_groups: [rds,prometheus,smoke-tests,cats]
   plan:
   - get: trigger-weekday-morning
-    # trigger: true
-    # passed: [deploy-cf]
+    trigger: true
+    passed: [deploy-cf]
   - get: paas-bootstrap-git
-    # passed: [deploy-cf]
-    # trigger: true
+    passed: [deploy-cf]
+    trigger: true
   - aggregate:
     - task: ensure CF api responds
       file: paas-bootstrap-git/ci/tasks/common/test_endpoint/task.yml
@@ -833,23 +822,6 @@ jobs:
   - task: ensure smoke tests pass
     file: paas-bootstrap-git/ci/tasks/cf/smoke_tests/task.yml
 
-- name: rabbitmq-broker
-  plan:
-  - aggregate:
-    - get: trigger-weekday-morning
-      trigger: true
-      passed: [verify-cf-endpoints]
-    - get: paas-bootstrap-git
-      passed: [verify-cf-endpoints]
-      trigger: true
-    - get: rabbitmq-broker-deployment-git
-      version: { ref: ((rabbitmq_broker_tag)) }
-  - task: interpolate-rabbitmq-broker
-    file: paas-bootstrap-git/ci/tasks/rabbitmq_broker/deploy/task.yml
-    params:
-      ENVIRONMENT: ((environment))
-      DOMAIN: ((domain))
-
 - name: cats
   serial: true
   serial_groups: [cats]
@@ -987,21 +959,3 @@ jobs:
     params:
       ENVIRONMENT: ((environment))
 
-- name: verify-shared-rds
-  serial: true
-  serial_groups: [rds]
-  plan:
-  - get: cf-tests-git
-  - get: trigger-weekday-morning
-    trigger: true
-    passed: [cf-management]
-  - get: paas-bootstrap-git
-    trigger: true
-    passed: [cf-management]
-  - get: cf-vars-s3
-  - task: get cf credentials
-    file: paas-bootstrap-git/ci/tasks/common/get_cf_tester_credentials/task.yml
-    params:
-      DOMAIN: ((domain))
-  - task: create service and test useability
-    file: paas-bootstrap-git/ci/tasks/rds_broker/test_shared/task.yml
diff --git a/ci/tasks/cf/cloud_config/task.sh b/ci/tasks/cf/cloud_config/task.sh
index c2f94cfe..fd734223 100755
--- a/ci/tasks/cf/cloud_config/task.sh
+++ b/ci/tasks/cf/cloud_config/task.sh
@@ -66,5 +66,7 @@ bosh update-cloud-config -n \
   -v grafana_target_group_name="$(jq -r .grafana_target_group_name < prometheus-vars.json)" \
   -v prometheus_target_group_name="$(jq -r .prometheus_target_group_name < prometheus-vars.json)" \
   -v alertmanager_target_group_name="$(jq -r .alertmanager_target_group_name < prometheus-vars.json)"
+  -v rabbitmq-broker-security-group-id="($jq -r .rabbitmq_broker_security_group_id < )" \
+  -v rabbitmq-server-security-group-id="($jq -r .rabbitmq_server_security_group_id < )"
 
 bosh cloud-config > cf-manifests/cloud-config.yml

From 8897b6d2615e414d45e738ec9218d0ef416cd6ad Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Thu, 11 Oct 2018 17:21:08 +0100
Subject: [PATCH 28/37] Update vars example in readme

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index ef2105bc..48dd4354 100644
--- a/README.md
+++ b/README.md
@@ -25,11 +25,13 @@ You'll need to create a `<env>_vpc.tfvars` file with `az1`, `az2`, `region` and
 
 ```json
 {
+  "environment": "engineering",
   "az1": "eu-west-1a",
   "az2": "eu-west-1b",
   "region": "eu-west-1",
   "parent_dns_zone": "<domain>",
   "ingress_whitelist": ["0.0.0.0/0"],
+  "availability_zones": ["eu-west-1a", "eu-west-1b", "eu-west-1c"],
   "slack_webhook_uri": "https://hooks.slack.com/services/<generated uri>",
   "cidr_blocks": {
     "services":   ["10.121.28.0/22", "10.121.32.0/22", "10.121.36.0/22"],

From 350b3e17493cc7ac5ef9333dc509ef7b5d5a3f45 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 15 Oct 2018 11:57:40 +0100
Subject: [PATCH 29/37] Add rabbitmq terraform to cf

---
 terraform/cf/aws/rabbitmq-broker.tf | 185 ++++++++++++++++++++++++++++
 1 file changed, 185 insertions(+)
 create mode 100644 terraform/cf/aws/rabbitmq-broker.tf

diff --git a/terraform/cf/aws/rabbitmq-broker.tf b/terraform/cf/aws/rabbitmq-broker.tf
new file mode 100644
index 00000000..6c1bfd15
--- /dev/null
+++ b/terraform/cf/aws/rabbitmq-broker.tf
@@ -0,0 +1,185 @@
+resource "aws_security_group" "rabbitmq_broker" {
+  name        = "${var.environment}_rabbitmq_broker_security_group"
+  description = "RabbitMQ service broker access"
+  vpc_id      = "${aws_vpc.default.id}"
+
+  tags {
+    Name        = "${var.environment}-rabbitmq-broker-security-group"
+    Environment = "${var.environment}"
+  }
+}
+
+resource "aws_security_group_rule" "cf_to_rmq_broker" {
+  security_group_id        = "${aws_security_group.rabbitmq_broker.id}"
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 4567
+  to_port                  = 4567
+  source_security_group_id = "${aws_security_group.internal.id}"
+  description              = "CF may talk to RabbitMQ broker API"
+}
+
+resource "aws_security_group_rule" "rmq_broker_outbound" {
+  security_group_id        = "${aws_security_group.rabbitmq_broker.id}"
+  type                     = "egress"
+  protocol                 = "tcp"
+  from_port                = 0
+  to_port                  = 65535
+  cidr_blocks              = ["0.0.0.0/0"] # FIXME: restrict to the CF ALB security group - 443, 8443
+  description              = "RabbitMQ broker outbound access"
+}
+
+resource "aws_security_group_rule" "rmq_broker_self_tcp" {
+  security_group_id        = "${aws_security_group.rabbitmq_broker.id}"
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 0
+  to_port                  = 65535
+  self                     = true
+  description              = "RabbitMQ broker self TCP"
+}
+
+resource "aws_security_group_rule" "rmq_broker_self_udp" {
+  security_group_id        = "${aws_security_group.rabbitmq_broker.id}"
+  type                     = "ingress"
+  protocol                 = "udp"
+  from_port                = 0
+  to_port                  = 65535
+  self                     = true
+  description              = "RabbitMQ broker self UDP"
+}
+
+resource "aws_security_group_rule" "rmq_broker_self_icmp" {
+  security_group_id        = "${aws_security_group.rabbitmq_broker.id}"
+  type                     = "ingress"
+  protocol                 = "icmp"
+  from_port                = -1
+  to_port                  = -1
+  self                     = true
+  description              = "RabbitMQ broker self ICMP"
+}
+
+resource "aws_security_group_rule" "rabbitmq_to_cf_nats" {
+  security_group_id        = "${aws_security_group.internal.id}"
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 4222
+  to_port                  = 4222
+  description              = "Allow rabbitmq broker to access cf nats"
+  source_security_group_id = "${aws_security_group.rabbitmq_broker.id}"
+}
+
+resource "aws_security_group_rule" "cf_to_rmq_5671_2" {
+  security_group_id        = "${aws_security_group.rabbitmq_broker.id}"
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 5671
+  to_port                  = 5672
+  source_security_group_id = "${aws_security_group.internal.id}"
+  description              = "CF may talk to RabbitMQ on ports 5671,2"
+}
+
+resource "aws_security_group_rule" "cf_to_rmq_1883" {
+  security_group_id        = "${aws_security_group.rabbitmq_broker.id}"
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 1883
+  to_port                  = 1883
+  source_security_group_id = "${aws_security_group.internal.id}"
+  description              = "CF may talk to RabbitMQ on ports 5671,2"
+}
+
+resource "aws_security_group_rule" "cf_to_rmq_8883" {
+  security_group_id        = "${aws_security_group.rabbitmq_broker.id}"
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 8883
+  to_port                  = 8883
+  source_security_group_id = "${aws_security_group.internal.id}"
+  description              = "CF may talk to RabbitMQ on ports 8883"
+}
+
+resource "aws_security_group_rule" "cf_to_rmq_61613_4" {
+  security_group_id        = "${aws_security_group.rabbitmq_broker.id}"
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 61613
+  to_port                  = 61614
+  source_security_group_id = "${aws_security_group.internal.id}"
+  description              = "CF may talk to RabbitMQ on ports 61613,4"
+}
+
+resource "aws_security_group_rule" "cf_to_rmq_15672" {
+  security_group_id        = "${aws_security_group.rabbitmq_broker.id}"
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 15672
+  to_port                  = 15672
+  source_security_group_id = "${aws_security_group.internal.id}"
+  description              = "CF may talk to RabbitMQ on ports 15672"
+}
+
+resource "aws_security_group_rule" "cf_to_rmq_15674" {
+  security_group_id        = "${aws_security_group.rabbitmq_broker.id}"
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 15674
+  to_port                  = 15674
+  source_security_group_id = "${aws_security_group.internal.id}"
+  description              = "CF may talk to RabbitMQ on ports 15674"
+}
+
+
+
+resource "aws_security_group" "rabbitmq_server" {
+  name        = "${var.environment}_rabbitmq_server_security_group"
+  description = "RabbitMQ server access"
+  vpc_id      = "${aws_vpc.default.id}"
+
+  tags {
+    Name        = "${var.environment}-rabbitmq-server-security-group"
+    Environment = "${var.environment}"
+  }
+}
+
+resource "aws_security_group_rule" "rmq_server_self_tcp" {
+  security_group_id        = "${aws_security_group.rabbitmq_server.id}"
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 0
+  to_port                  = 65535
+  self                     = true
+  description              = "RabbitMQ broker self TCP"
+}
+
+resource "aws_security_group_rule" "rmq_server_self_udo" {
+  security_group_id        = "${aws_security_group.rabbitmq_server.id}"
+  type                     = "ingress"
+  protocol                 = "udp"
+  from_port                = 0
+  to_port                  = 65535
+  self                     = true
+  description              = "RabbitMQ broker self UDP"
+}
+
+resource "aws_security_group_rule" "rmq_server_self_icmp" {
+  security_group_id        = "${aws_security_group.rabbitmq_server.id}"
+  type                     = "ingress"
+  protocol                 = "icmp"
+  from_port                = -1
+  to_port                  = -1
+  self                     = true
+  description              = "RabbitMQ server self ICMP"
+}
+
+resource "aws_security_group_rule" "rmq_broker_to_rmq_server" {
+  security_group_id        = "${aws_security_group.rabbitmq_server.id}"
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 0
+  to_port                  = 65535
+  source_security_group_id = "${aws_security_group.rabbitmq_broker.id}"
+  description              = "RabbitMQ broker can do everything to the servers"
+}
+
+

From baf2f9131549b79acc129b3cdf1106d986827169 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 15 Oct 2018 12:54:08 +0100
Subject: [PATCH 30/37] Correct source for vpc id in rabbitmq terraform

---
 terraform/cf/aws/rabbitmq-broker.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/terraform/cf/aws/rabbitmq-broker.tf b/terraform/cf/aws/rabbitmq-broker.tf
index 6c1bfd15..c4211e31 100644
--- a/terraform/cf/aws/rabbitmq-broker.tf
+++ b/terraform/cf/aws/rabbitmq-broker.tf
@@ -1,7 +1,7 @@
 resource "aws_security_group" "rabbitmq_broker" {
   name        = "${var.environment}_rabbitmq_broker_security_group"
   description = "RabbitMQ service broker access"
-  vpc_id      = "${aws_vpc.default.id}"
+  vpc_id      = "${var.vpc_id}"
 
   tags {
     Name        = "${var.environment}-rabbitmq-broker-security-group"
@@ -134,7 +134,7 @@ resource "aws_security_group_rule" "cf_to_rmq_15674" {
 resource "aws_security_group" "rabbitmq_server" {
   name        = "${var.environment}_rabbitmq_server_security_group"
   description = "RabbitMQ server access"
-  vpc_id      = "${aws_vpc.default.id}"
+  vpc_id      = "${var.vpc_id}"
 
   tags {
     Name        = "${var.environment}-rabbitmq-server-security-group"

From 4efa2c6d4241caeaceb861c7329ac1f8c6669f27 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 15 Oct 2018 13:06:09 +0100
Subject: [PATCH 31/37] Add rabbitmq outputs to cf terraform

---
 ci/tasks/cf/cloud_config/task.sh | 4 ++--
 terraform/cf/aws/outputs.tf      | 8 ++++++++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/ci/tasks/cf/cloud_config/task.sh b/ci/tasks/cf/cloud_config/task.sh
index fd734223..4df012f2 100755
--- a/ci/tasks/cf/cloud_config/task.sh
+++ b/ci/tasks/cf/cloud_config/task.sh
@@ -66,7 +66,7 @@ bosh update-cloud-config -n \
   -v grafana_target_group_name="$(jq -r .grafana_target_group_name < prometheus-vars.json)" \
   -v prometheus_target_group_name="$(jq -r .prometheus_target_group_name < prometheus-vars.json)" \
   -v alertmanager_target_group_name="$(jq -r .alertmanager_target_group_name < prometheus-vars.json)"
-  -v rabbitmq-broker-security-group-id="($jq -r .rabbitmq_broker_security_group_id < )" \
-  -v rabbitmq-server-security-group-id="($jq -r .rabbitmq_server_security_group_id < )"
+  -v rabbitmq-broker-security-group-id="($jq -r .rabbitmq_broker_security_group_id < cf-vars.json)" \
+  -v rabbitmq-server-security-group-id="($jq -r .rabbitmq_server_security_group_id < cf-vars.json)"
 
 bosh cloud-config > cf-manifests/cloud-config.yml
diff --git a/terraform/cf/aws/outputs.tf b/terraform/cf/aws/outputs.tf
index 8fe60fcc..e01f382b 100644
--- a/terraform/cf/aws/outputs.tf
+++ b/terraform/cf/aws/outputs.tf
@@ -131,3 +131,11 @@ output "services_subnet_ids" {
 output "services_subnet_cidr_blocks" {
     value = ["${aws_subnet.services.*.cidr_block}"]
 }
+
+output "rabbitmq_broker_security_group_id" {
+  value = "${aws_security_group.rabbitmq_broker.id}"
+}
+
+output "rabbitmq_server_security_group_id" {
+  value = "${aws_security_group.rabbitmq_server.id}"
+}
\ No newline at end of file

From 229482fd596a32d8fd32728a0cb9c72bd46d799e Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 15 Oct 2018 13:31:05 +0100
Subject: [PATCH 32/37] Correct jq call

---
 ci/tasks/cf/cloud_config/task.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/tasks/cf/cloud_config/task.sh b/ci/tasks/cf/cloud_config/task.sh
index 4df012f2..0b3d6d70 100755
--- a/ci/tasks/cf/cloud_config/task.sh
+++ b/ci/tasks/cf/cloud_config/task.sh
@@ -66,7 +66,7 @@ bosh update-cloud-config -n \
   -v grafana_target_group_name="$(jq -r .grafana_target_group_name < prometheus-vars.json)" \
   -v prometheus_target_group_name="$(jq -r .prometheus_target_group_name < prometheus-vars.json)" \
   -v alertmanager_target_group_name="$(jq -r .alertmanager_target_group_name < prometheus-vars.json)"
-  -v rabbitmq-broker-security-group-id="($jq -r .rabbitmq_broker_security_group_id < cf-vars.json)" \
-  -v rabbitmq-server-security-group-id="($jq -r .rabbitmq_server_security_group_id < cf-vars.json)"
+  -v rabbitmq-broker-security-group-id="$(jq -r .rabbitmq_broker_security_group_id < cf-vars.json)" \
+  -v rabbitmq-server-security-group-id="$(jq -r .rabbitmq_server_security_group_id < cf-vars.json)"
 
 bosh cloud-config > cf-manifests/cloud-config.yml

From 4b8d71cd5bde691d896efa4fc10dce21e1bcb2ef Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 15 Oct 2018 13:40:16 +0100
Subject: [PATCH 33/37] Add missing slash

---
 ci/tasks/cf/cloud_config/task.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/tasks/cf/cloud_config/task.sh b/ci/tasks/cf/cloud_config/task.sh
index 0b3d6d70..083b8184 100755
--- a/ci/tasks/cf/cloud_config/task.sh
+++ b/ci/tasks/cf/cloud_config/task.sh
@@ -65,7 +65,7 @@ bosh update-cloud-config -n \
   -v prometheus_subnet_az1_gateway="$(jq -r .prometheus_subnet_az1_cidr < prometheus-vars.json | sed 's#0/24#1#')" \
   -v grafana_target_group_name="$(jq -r .grafana_target_group_name < prometheus-vars.json)" \
   -v prometheus_target_group_name="$(jq -r .prometheus_target_group_name < prometheus-vars.json)" \
-  -v alertmanager_target_group_name="$(jq -r .alertmanager_target_group_name < prometheus-vars.json)"
+  -v alertmanager_target_group_name="$(jq -r .alertmanager_target_group_name < prometheus-vars.json)" \
   -v rabbitmq-broker-security-group-id="$(jq -r .rabbitmq_broker_security_group_id < cf-vars.json)" \
   -v rabbitmq-server-security-group-id="$(jq -r .rabbitmq_server_security_group_id < cf-vars.json)"
 

From 59835417cbb0d7f491017d1f3ff393ac0d3594e9 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 15 Oct 2018 15:03:49 +0100
Subject: [PATCH 34/37] Add bosh security group id to rabbitmq deploy

---
 ci/tasks/cf/cloud_config/task.sh | 4 +++-
 terraform/cf/aws/outputs.tf      | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/ci/tasks/cf/cloud_config/task.sh b/ci/tasks/cf/cloud_config/task.sh
index 083b8184..ab675515 100755
--- a/ci/tasks/cf/cloud_config/task.sh
+++ b/ci/tasks/cf/cloud_config/task.sh
@@ -2,6 +2,7 @@
 
 set -euo pipefail
 
+jq '.modules[0].outputs | with_entries(.value = .value.value)' < bosh-tfstate-s3/tfstate.json > bosh-vars.json
 jq '.modules[0].outputs | with_entries(.value = .value.value)' < vpc-tfstate-s3/tfstate.json > vpc-vars.json
 jq '.modules[0].outputs | with_entries(.value = .value.value)' < concourse-tfstate-s3/tfstate.json > concourse-vars.json
 jq '.modules[0].outputs | with_entries(.value = .value.value)' < "cf-tfstate-s3/${ENVIRONMENT}.tfstate" > cf-vars.json
@@ -67,6 +68,7 @@ bosh update-cloud-config -n \
   -v prometheus_target_group_name="$(jq -r .prometheus_target_group_name < prometheus-vars.json)" \
   -v alertmanager_target_group_name="$(jq -r .alertmanager_target_group_name < prometheus-vars.json)" \
   -v rabbitmq-broker-security-group-id="$(jq -r .rabbitmq_broker_security_group_id < cf-vars.json)" \
-  -v rabbitmq-server-security-group-id="$(jq -r .rabbitmq_server_security_group_id < cf-vars.json)"
+  -v rabbitmq-server-security-group-id="$(jq -r .rabbitmq_server_security_group_id < cf-vars.json)" \
+  -v bosh-managed-security-group-id="$(jq -r .bosh_security_group_id < bosh-vars.json)"
 
 bosh cloud-config > cf-manifests/cloud-config.yml
diff --git a/terraform/cf/aws/outputs.tf b/terraform/cf/aws/outputs.tf
index e01f382b..76477cbd 100644
--- a/terraform/cf/aws/outputs.tf
+++ b/terraform/cf/aws/outputs.tf
@@ -138,4 +138,4 @@ output "rabbitmq_broker_security_group_id" {
 
 output "rabbitmq_server_security_group_id" {
   value = "${aws_security_group.rabbitmq_server.id}"
-}
\ No newline at end of file
+}

From 3b1ba86ff27d007b5a3410addc7494dffdd3b892 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 15 Oct 2018 15:17:28 +0100
Subject: [PATCH 35/37] Correct bosh tfstate file name

---
 ci/tasks/cf/cloud_config/task.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/tasks/cf/cloud_config/task.sh b/ci/tasks/cf/cloud_config/task.sh
index ab675515..52b146f7 100755
--- a/ci/tasks/cf/cloud_config/task.sh
+++ b/ci/tasks/cf/cloud_config/task.sh
@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-jq '.modules[0].outputs | with_entries(.value = .value.value)' < bosh-tfstate-s3/tfstate.json > bosh-vars.json
+jq '.modules[0].outputs | with_entries(.value = .value.value)' < "bosh-tfstate-s3/${ENVIRONMENT}.tfstate" > bosh-vars.json
 jq '.modules[0].outputs | with_entries(.value = .value.value)' < vpc-tfstate-s3/tfstate.json > vpc-vars.json
 jq '.modules[0].outputs | with_entries(.value = .value.value)' < concourse-tfstate-s3/tfstate.json > concourse-vars.json
 jq '.modules[0].outputs | with_entries(.value = .value.value)' < "cf-tfstate-s3/${ENVIRONMENT}.tfstate" > cf-vars.json

From 2d0ee82ef6353c33d253fed51d42d917b56e1b97 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 15 Oct 2018 15:35:46 +0100
Subject: [PATCH 36/37] Add bosh-tfstate-s3 resource to cloud config ci task

---
 ci/tasks/cf/cloud_config/task.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/tasks/cf/cloud_config/task.yml b/ci/tasks/cf/cloud_config/task.yml
index b8991827..bef6f2fa 100644
--- a/ci/tasks/cf/cloud_config/task.yml
+++ b/ci/tasks/cf/cloud_config/task.yml
@@ -11,6 +11,7 @@ inputs:
   - name: cf-tfstate-s3
   - name: bosh-vars-s3
   - name: bosh-manifest-s3
+  - name: bosh-tfstate-s3
   - name: concourse-tfstate-s3
   - name: prometheus-tfstate-s3
 

From e90b880b0f9d0a52553c1fd8196c48cb82563f19 Mon Sep 17 00:00:00 2001
From: J Gregory <john@johngregory.me.uk>
Date: Mon, 15 Oct 2018 15:41:54 +0100
Subject: [PATCH 37/37] Add missing services subnet az3 id

---
 ci/tasks/cf/cloud_config/task.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/tasks/cf/cloud_config/task.sh b/ci/tasks/cf/cloud_config/task.sh
index 52b146f7..9f1924f5 100755
--- a/ci/tasks/cf/cloud_config/task.sh
+++ b/ci/tasks/cf/cloud_config/task.sh
@@ -53,6 +53,7 @@ bosh update-cloud-config -n \
   -v services_subnet_az3_cidr="$(jq -r .services_subnet_cidr_blocks[2] < cf-vars.json)" \
   -v services_subnet_az3_gateway="$(jq -r .services_subnet_gateway_ips[2] < cf-vars.json)" \
   -v reserved_services_az3_cidr="$(jq -r .services_subnet_reserved_cidr_blocks[2] < cf-vars.json)" \
+  -v services_subnet_az3_id="$(jq -r .services_subnet_ids[2] < cf-vars.json)" \
   -v cf-router-target-group-name="$(jq -r '.cf_router_target_group_name' < cf-vars.json)" \
   -v cf-router-lb-internal-security-group-id="$(jq -r '.cf_router_lb_internal_security_group_id' < cf-vars.json)" \
   -v cf-internal-security-group-id="$(jq -r '.cf_internal_security_group_id' < cf-vars.json)" \