CF 4.xx

.gitignore

@@ -19,3 +19,7 @@ foo.yml
 
 *.lock.info
 **/.DS_Store
+data/**
+
+.vagrant
+*-cloudimg-console.log

.gitmodules

@@ -1,3 +1,18 @@
 [submodule "concourse-bosh-deployment"]
 	path = concourse-bosh-deployment
 	url = https://github.com/concourse/concourse-bosh-deployment.git
+[submodule "bosh-deployment"]
+	path = bosh-deployment
+	url = https://github.com/cloudfoundry/bosh-deployment.git
+[submodule "cf-deployment"]
+	path = cf-deployment
+	url = https://github.com/cloudfoundry/cf-deployment.git
+[submodule "prometheus-boshrelease"]
+	path = prometheus-boshrelease
+	url = https://github.com/bosh-prometheus/prometheus-boshrelease.git
+[submodule "cf-rabbitmq-multitenant-broker-release"]
+	path = cf-rabbitmq-multitenant-broker-release
+	url = https://github.com/pivotal-cf/cf-rabbitmq-multitenant-broker-release
+[submodule "elasticache-broker"]
+	path = elasticache-broker
+	url = https://github.com/cloudfoundry-community/elasticache-broker.git

Makefile

@@ -1,5 +1,6 @@
 
 .EXPORT_ALL_VARIABLES:
+.PHONY: terraform
 
 # Environment variables that you have to set yourself:
 #
@@ -9,67 +10,90 @@
 
 # You could override these, but you really shouldn't
 VAR_FILE = ${ENVIRONMENT}_vpc.tfvars
-VPC_STATE_FILE = ${ENVIRONMENT}_vpc.tfstate.json
-CONCOURSE_TERRAFORM_STATE_FILE = ${ENVIRONMENT}_concourse.tfstate.json
-CONCOURSE_STATE_FILE = ${ENVIRONMENT}_concourse.state.json
-CONCOURSE_CREDS_FILE = ${ENVIRONMENT}_concourse.creds.yml
-PRIVATE_KEY_FILE = ${ENVIRONMENT}_concourse.pem
-PUBLIC_KEY_FILE = ${PRIVATE_KEY_FILE}.pub
-
-## FIXME: delete
-JUMPBOX_TERRAFORM_STATE_FILE = ${ENVIRONMENT}_jumpbox.tfstate.json
 
 help:
 	@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-require_vars:
-	@bin/require_vars.sh
+submodules: ## checkout the right version of git submodules
+	@bin/update_submodules.sh 
+
+terraform: ## create main terraform environment
+	@bin/terraform.sh apply
+
+terraform_init: ## initialize terraform provider
+	@terraform init terraform/base	
+
+terraform_plan: ## plan what would be applied if we ran make terraform
+	@bin/terraform.sh plan
+
+outputs: ## base terraform outputs
+	@bin/outputs.sh
+
+rds: ## create rds instances
+	@bin/rds.sh apply
 
-vpc: require_vars ## Setup a VPC to deploy concourse to
-	@bin/create_vpc.sh
+databases: ## create databases
+	@bin/databases.sh
 
-keypair: ## Create the SSH key pair used to log into the VMs
-	@bin/create_ssh_keypair.sh
+bosh: ## create bosh
+	@bin/create_bosh.sh
 
-concourse_network: vpc keypair ## Setup networks for concourse to consume
-	@bin/create_concourse_network.sh
+runtime_config: ## Deploy runtime config
+	@bin/runtime_config.sh
 
-concourse: concourse_network ## Deploy concourse with all prereqs
-	@bin/deploy_concourse.sh
+cloud_config: ## Deploy cloud config	
+	@bin/cloud_config.sh
 
-concourse_password: ## Retrieves the concourse password for a given environment
-	@bin/concourse_creds.sh
+stemcells: ## Upload stemcells
+	@bin/stemcells.sh
 
-concourse_login: require_vars ## Logs fly into concourse
+cf: ## Deploy CF
+	@bin/deploy_cf.sh
+
+prometheus: ## Deploy Prometheus
+	@bin/deploy_prometheus.sh
+
+rabbitmq: ## Deploy RabbitMQ service broker
+	@bin/deploy_rabbitmq.sh
+
+concourse: ## Deploy concourse
+	@bin/deploy_concourse.sh	
+
+login_fly: ## Log in to fly
 	@bin/login_fly.sh
 
-prometheus_credentials: ## Get credentials for prometheus stack
-	@bin/prometheus_credentials.sh
+set_concourse_secrets: ## Set the secrets that Concourse needs to run its pipelines
+	@bin/set_concourse_secrets.sh
 
-test_pipeline: require_vars ## Deploy a test pipeline to concourse
-	@test/deploy_test_pipeline.sh
+pipelines: login_fly ## Deploy pipelines
+	@ci/cf_pipeline.sh
 
-test_s3_pipeline: require_vars ## Deploy a pipeline that tests S3 access to concourse
-	@test/deploy_s3_test_pipeline.sh
+destroy_concourse: ## Delete the concourse deployment
+	@bin/destroy_deployment.sh concourse
 
-deploy_pipeline: require_vars ## Deploy the CF deployment pipeline
-	@ci/deploy_pipeline.sh
+destroy_rabbitmq: ## Delete the rabbitmq deployment
+	@bin/destroy_deployment.sh rabbitmq
 
-destruction_pipeline: require_vars ## Deploy the CF destruction pipeline
-	@ci/destruction_pipeline.sh
+destroy_prometheus: ## Delete the prometheus deployment
+	@bin/destroy_deployment.sh prometheus
 
-docker_image:  ## Build the general-purpose tooled docker image for Concourse tasks
-	@bin/create_docker_image.sh
+destroy_cf: ## Delete the CF deployment
+	@bin/destroy_deployment.sh cf
 
-destroy: destroy_concourse_network ## Destroy an entire environment
-	@bin/delete_vpc.sh
+destroy_bosh: ## Kill off bosh
+	@bin/destroy_bosh.sh
 
-destroy_concourse_network: destroy_concourse ## Destroy concourse and its network
-	@bin/delete_concourse_network.sh
+destroy_rds: ## destroy the RDS instances
+	@bin/rds.sh destroy
 
-destroy_concourse: require_vars ## Destroy concourse only
-	@bin/delete_concourse.sh
+destroy_terraform: ## destroy main terraform environment
+	@bin/terraform.sh destroy
 
 decode_aws_error: ## Decode AWS message
 	@aws sts decode-authorization-message --encoded-message ${DECODE_MESSAGE} | jq -r .DecodedMessage | jq .
 
+set_redis_secrets: ## Set the secrets for Redis broker
+	@bin/set_redis_secrets.sh
+
+redis: ## Deploy Redis (Elasticache)
+	@bin/deploy_redis.sh

README.md

@@ -2,61 +2,109 @@
 
 We use the code in this repository to bootstrap our AWS PaaS environment. The normal flow is:
 
-1. Create the VPC under which the PaaS systems will live
-2. Create a Concourse using `bosh create-env`
-3. Deploy the pipelines that will spin up BOSH and CloudFoundry
-4. Sit back and wait
+1. Create the Infrastructure under which the Concourse, BOSH and PaaS systems will live
+2. Deploy a BOSH director
+3. Deploy CF, Concourse, Prometheus, etc.
+4. Deploy the automation pipelines for upgrades
 
 ## Pre-requisites
 
 - AWS CLI
 - Terraform CLI
+- BOSH CLI
+- CF CLI
+- CF Management CLIs
+- Credhub CLI
+- UAA CLI
+- PSQL client
+- MySQL client
 - Fly [CLI](https://concourse-ci.org/download.html)
 - [yq](https://github.com/mikefarah/yq) (or, `brew install yq`)
 - [jq](https://stedolan.github.io/jq/) (or, `brew install jq`)
 
 ## Creating a new environment
 
-You'll need to create a `<env>_vpc.tfvars` file with `az1`, `az2`, `region` and `parent_dns_zone`:
+1. Update submodules
 
-> **Note**: Multiple AZs are required in order to deploy an AWS ALB.
+```sh
+  make submodules
+```
 
-> set ingress_whitelist to the CIDRs that may access Concourse
+2. Create your environment directory and vars file
+
+```sh
+mkdir data
+touch data/<env>.tfvars
+```
+
+You'll need to create a `<env>.tfvars` file:
 
 ```json
 {
-"az1": "eu-west-1a",
-"az2": "eu-west-1b",
-"region": "eu-west-1",
-"parent_dns_zone": "<domain>",
-"ingress_whitelist": ["0.0.0.0/0"],
-"slack_webhook_uri": "https://hooks.slack.com/services/<generated uri>"
+    "environment": "<env>",
+    "region": "eu-west-1",
+    "availability_zones": ["eu-west-1a", "eu-west-1b", "eu-west-1c"],
+    "parent_dns_zone": "<parent domain>",
+    "ingress_whitelist": ["0.0.0.0/0"],
+    "vpc_cidr_block": "10.121.0.0/16",
+    "cidr_blocks": {
+        "public":     ["10.121.0.0/24", "10.121.1.0/24", "10.121.2.0/24"],
+        "internal":   ["10.121.8.0/22", "10.121.12.0/22", "10.121.16.0/22"],
+        "services":   ["10.121.28.0/22", "10.121.32.0/22", "10.121.36.0/22"],
+        "rds":        ["10.121.50.0/24", "10.121.51.0/24", "10.121.52.0/24"],
+        "prometheus": ["10.121.53.0/24", "10.121.54.0/24", "10.121.55.0/24"],
+        "concourse":  ["10.121.56.0/24", "10.121.57.0/24", "10.121.58.0/24"]
+    }
 }
 ```
 
-Example command:
+The `vpc_cidr_block` parameter is an array of IP ranges that you want to be able to access
+the PaaS. It should *not* be `0.0.0.0/0`.
+
+The `cidr_blocks` parameter specifies the IP subnet CIDR block ranges for each subnet and availability zone.
+This is specified to allow you to define exactly how big to make each subnet. (We could have automatically
+generated these values, but feel it is more comprehensible to view it in the base variables)
+
+3. Create an AWS user and access credentials
+
+You will need to create (manually) an AWS user and generate access and secret keys via the AWS console.
+
+4. Create the PaaS
 
 ```sh
-git submodule update --init
-ENVIRONMENT=<choose_a_name> AWS_ACCESS_KEY_ID=<your_key_id> AWS_SECRET_ACCESS_KEY=<your_secret_key>
-make concourse
+  export AWS_ACCESS_KEY=<your AWS key id>
+  export AWS_SECRET_ACCESS_KEY=<your AWS secret key>
+  export ENVIRONMENT=<env>
+  make terraform
+  make rds
+  make databases # you will need to wait a bit after the previous step to give RDS time to initialise
+  make bosh
+  make runtime_config cloud_config stemcells
+  make concourse
+  make cf
+  make prometheus
+  make rabbitmq
 ```
 
-Where:
+You can specify AWS_PROFILE, rather than the two AWS secrets, for every step except `make bosh`.
+The `bosh create-env` command currently does not handle AWS_PROFILE correctly.
 
-- `ENVIRONMENT` - a name for your environment
-- `AWS_ACCESS_KEY_ID` - your aws access key id
-- `AWS_SECRET_ACCESS_KEY` - your aws secret access key
+5. Deploy the Concourse pipelines
 
-You can specify AWS_PROFILE, rather than the two AWS secrets, for every step except for `make concourse`.
-The `bosh create-env` command currently does not handle AWS_PROFILE correctly.
+```sh
+  export ENVIRONMENT=<env>
+  make set_concourse_secrets
+  make pipelines
+```
+
+## Connecting to components
 
-## Connecting to Concourse
+### Connecting to Concourse
 
 The dns name of Concourse is found by:
 
 ```sh
-terraform output -state=<env>_concourse.tfstate.json concourse_fqdn
+  bin/outputs.sh -e <env> | jq .concourse_fqdn
 ```
 
 Go to `https://<concourse_fqdn>` to login.
@@ -67,52 +115,52 @@ The username is `admin` and you can get the password through:
 bin/concourse_password.sh -e <env>
 ```
 
-or using
-
-```sh
-make concourse_password ENVIRONMENT=<env>
-```
 
-## Testing that Concourse works
+### SSHing onto the jumpbox
 
 ```sh
-ENVIRONMENT=<env> AWS_ACCESS_KEY_ID=<your_key_id> AWS_SECRET_ACCESS_KEY=<your_secret_key> make test_pipeline
-fly -t <env> trigger-job -j test/pipeline-test -w
+bin/jumpbox_ssh -e <env>
 ```
 
-## Installing the deployment pipeline
-
-The `deploy_pipeline` pipeline will spin up the jump box and BOSH director.
+### Logging in to BOSH
 
 ```sh
-ENVIRONMENT=<env> AWS_ACCESS_KEY_ID=<your_key_id> AWS_SECRET_ACCESS_KEY=<your_secret_key> make deploy_pipeline
-fly -t <env> trigger-job -j deploy_pipeline/terraform-jumpbox -w
+  bin/bosh_credentials.sh -e <env>
+  # spins up a subshell with a Socks5 proxy connection via jump box to BOSH
 ```
 
-If you are deploying from a branch, you should also specify it with the `BRANCH` environment variable, so that the pipeline will trigger correctly.
+or, you can run a bosh command:
 
 ```sh
-BRANCH=<your git branch> ... make deploy_pipeline
+  bin/bosh_credentials.sh -e <env> bosh vms
 ```
 
+### Logging into CF as admin
 
-## Logging in to BOSH
-
-Once the deployment pipeline has run to completion, you can set up your connection to BOSH easily using:
+The CF API URL is `https://api.system.<env>.<domain>`.
 
 ```sh
-  bin/bosh_credentials.sh -e <env> -f
-  # spins up a subshell with a Socks5 proxy connection via jump box to BOSH
+cf login -a https://api.system.<env>.<parent domain> -u admin -p $(bin/cf_password.sh -e <env>)
 ```
 
-or
+### Connecting to Prometheus components
 
 ```sh
-  source bin/bosh_credentials.sh -e <env>
-  # sets up the Socks5 proxy connection as above, but it's now your job to kill it
-  # it also sets BOSH_CLIENT, BOSH_CLIENT_SECRET environment variables
+bin/prometheus_credentials.sh -e <env>
 ```
 
+And use the displayed output to point the browser at the desired Prometheus component.
+
+## Versions
+
+| Component   | Version |
+| ----------- | ------- |
+| concourse   | v4.1.0  |
+| cf          | v4.2.0  |
+| bosh        | 7375c31d59018203911da3881334f09d8e70deb5 |
+| prometheus  | v23.2.0 |
+| rabbitmq    | v37.0.0 |
+
 ## LICENCE
 
 Copyright (c) 2018 Crown Copyright (Office for National Statistics)

bin/bosh_ca_cert.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+#
+# Returns the current BOSH password
+
+while getopts 'e:' option; do
+  case $option in
+    e) export ENVIRONMENT="$OPTARG";;
+  esac
+done
+
+: $ENVIRONMENT
+
+bin/get_states.sh -e $ENVIRONMENT -x -f $ENVIRONMENT-bosh-variables.yml
+bosh int --path /default_ca/ca "data/$ENVIRONMENT-bosh-variables.yml"