From 137581ec0a08fe3408e2920d4659e5ef934d35c9 Mon Sep 17 00:00:00 2001
From: Jen Spinney <jspinney@pivotal.io>
Date: Fri, 28 Feb 2020 12:36:26 -0800
Subject: [PATCH 01/10] Pin CAPI version in a way compatible with `vendir sync`

[finishes #171549565]
---
 .../github.com/cloudfoundry/capi-k8s-release/values.yml       | 2 +-
 config/values.yml                                             | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/config/_ytt_lib/github.com/cloudfoundry/capi-k8s-release/values.yml b/config/_ytt_lib/github.com/cloudfoundry/capi-k8s-release/values.yml
index a4d834140..3b2859e93 100644
--- a/config/_ytt_lib/github.com/cloudfoundry/capi-k8s-release/values.yml
+++ b/config/_ytt_lib/github.com/cloudfoundry/capi-k8s-release/values.yml
@@ -3,7 +3,7 @@
 replicaCount: 1
 
 images:
-  ccng: cloudfoundry/cloud-controller-ng@sha256:402932286e330049ea8e6562db615080b26388d849c8bb4aa2eed544b33e2c2a
+  ccng: cloudfoundry/cloud-controller-ng
   nginx: nginx:latest
 namespace: cf-system
 
diff --git a/config/values.yml b/config/values.yml
index 8682b4f0b..5d0775fe0 100644
--- a/config/values.yml
+++ b/config/values.yml
@@ -22,7 +22,9 @@ cf_db:
   enabled: true
 
 images:
-  capi: ""
+#! NOTE: the CAPI image reference below is needed temporarily to address a build break.
+#! See https://www.pivotaltracker.com/story/show/171522062 and https://www.pivotaltracker.com/story/show/171549565 for details
+  capi: "cloudfoundry/cloud-controller-ng@sha256:402932286e330049ea8e6562db615080b26388d849c8bb4aa2eed544b33e2c2a"
   nginx: ""
   cfroutesync: ""
   log_cache: "logcache/log-cache"

From 06ecf6bbc34a97368b59b677395b99d17737ea25 Mon Sep 17 00:00:00 2001
From: Jen Spinney <jspinney@pivotal.io>
Date: Fri, 28 Feb 2020 12:44:49 -0800
Subject: [PATCH 02/10] Revert "Pin CAPI version in a way compatible with
 `vendir sync`"

This reverts commit 137581ec0a08fe3408e2920d4659e5ef934d35c9.
---
 .../github.com/cloudfoundry/capi-k8s-release/values.yml       | 2 +-
 config/values.yml                                             | 4 +---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/config/_ytt_lib/github.com/cloudfoundry/capi-k8s-release/values.yml b/config/_ytt_lib/github.com/cloudfoundry/capi-k8s-release/values.yml
index 3b2859e93..a4d834140 100644
--- a/config/_ytt_lib/github.com/cloudfoundry/capi-k8s-release/values.yml
+++ b/config/_ytt_lib/github.com/cloudfoundry/capi-k8s-release/values.yml
@@ -3,7 +3,7 @@
 replicaCount: 1
 
 images:
-  ccng: cloudfoundry/cloud-controller-ng
+  ccng: cloudfoundry/cloud-controller-ng@sha256:402932286e330049ea8e6562db615080b26388d849c8bb4aa2eed544b33e2c2a
   nginx: nginx:latest
 namespace: cf-system
 
diff --git a/config/values.yml b/config/values.yml
index 5d0775fe0..8682b4f0b 100644
--- a/config/values.yml
+++ b/config/values.yml
@@ -22,9 +22,7 @@ cf_db:
   enabled: true
 
 images:
-#! NOTE: the CAPI image reference below is needed temporarily to address a build break.
-#! See https://www.pivotaltracker.com/story/show/171522062 and https://www.pivotaltracker.com/story/show/171549565 for details
-  capi: "cloudfoundry/cloud-controller-ng@sha256:402932286e330049ea8e6562db615080b26388d849c8bb4aa2eed544b33e2c2a"
+  capi: ""
   nginx: ""
   cfroutesync: ""
   log_cache: "logcache/log-cache"

From 7bd5389d39e27157d2689bd2dade7d59e889cb8b Mon Sep 17 00:00:00 2001
From: Jen Spinney <jspinney@pivotal.io>
Date: Fri, 28 Feb 2020 13:47:40 -0800
Subject: [PATCH 03/10] Correctly pin CAPI version in a way compatible with
 `vendir sync`

This is the more functional version of a broken commit I had to revert.
---
 .../github.com/cloudfoundry/capi-k8s-release/values.yml       | 2 +-
 config/capi.yml                                               | 3 ++-
 config/values.yml                                             | 4 +++-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/config/_ytt_lib/github.com/cloudfoundry/capi-k8s-release/values.yml b/config/_ytt_lib/github.com/cloudfoundry/capi-k8s-release/values.yml
index a4d834140..3b2859e93 100644
--- a/config/_ytt_lib/github.com/cloudfoundry/capi-k8s-release/values.yml
+++ b/config/_ytt_lib/github.com/cloudfoundry/capi-k8s-release/values.yml
@@ -3,7 +3,7 @@
 replicaCount: 1
 
 images:
-  ccng: cloudfoundry/cloud-controller-ng@sha256:402932286e330049ea8e6562db615080b26388d849c8bb4aa2eed544b33e2c2a
+  ccng: cloudfoundry/cloud-controller-ng
   nginx: nginx:latest
 namespace: cf-system
 
diff --git a/config/capi.yml b/config/capi.yml
index 6c1171ecf..1d0b0dc9b 100644
--- a/config/capi.yml
+++ b/config/capi.yml
@@ -16,9 +16,10 @@
 #@ end
 
 #@ def capi_values():
-#@ if/end data.values.images.capi:
 images:
+#@ if/end data.values.images.capi:
   ccng: #@ data.values.images.capi
+#@ if/end data.values.images.nginx:
   nginx: #@ data.values.images.nginx
 
 namespace: #@ data.values.system_namespace
diff --git a/config/values.yml b/config/values.yml
index 8682b4f0b..5d0775fe0 100644
--- a/config/values.yml
+++ b/config/values.yml
@@ -22,7 +22,9 @@ cf_db:
   enabled: true
 
 images:
-  capi: ""
+#! NOTE: the CAPI image reference below is needed temporarily to address a build break.
+#! See https://www.pivotaltracker.com/story/show/171522062 and https://www.pivotaltracker.com/story/show/171549565 for details
+  capi: "cloudfoundry/cloud-controller-ng@sha256:402932286e330049ea8e6562db615080b26388d849c8bb4aa2eed544b33e2c2a"
   nginx: ""
   cfroutesync: ""
   log_cache: "logcache/log-cache"

From 6002631c54a7d6dd9cff3eec032a5f8605135c86 Mon Sep 17 00:00:00 2001
From: Andrew Costa <acosta@pivotal.io>
Date: Fri, 28 Feb 2020 15:06:23 -0800
Subject: [PATCH 04/10] Update regular cf-for-k8s interval to 20 min

---
 ci/pipelines/cf-for-k8s.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ci/pipelines/cf-for-k8s.yml b/ci/pipelines/cf-for-k8s.yml
index 722a56d42..2f1537830 100644
--- a/ci/pipelines/cf-for-k8s.yml
+++ b/ci/pipelines/cf-for-k8s.yml
@@ -10,11 +10,11 @@ resources:
     ignore_paths:
       - ci/**
 
-- name: every-10-minutes
+- name: every-20-minutes
   type: time
   icon: clock-outline
   source:
-    interval: 10m
+    interval: 20m
 
 # semaphore for the Kubernetes cluster (ci-test-cluster in GKE); it is shared between this pipeline and the CF for K8s PR test pipeline
 - name: pool
@@ -58,7 +58,7 @@ jobs:
     params: {acquire: true}
   - get: cf-for-k8s-master
     trigger: true
-  - get: every-10-minutes
+  - get: every-20-minutes
     trigger: true
   - task: delete-cf
     config:

From d8af66d619e659f358b949b1191e79bce867a2f3 Mon Sep 17 00:00:00 2001
From: Dave Walter <dwalter@pivotal.io>
Date: Mon, 2 Mar 2020 13:41:46 -0800
Subject: [PATCH 05/10] Switch back to a single DNS entry for our CI cluster

We previously tried to create a subdomain with a timestamp in it to
address some of the DNS flakiness we were seeing.

However, it seems that that change didn't actually make things better,
so we're reverting it to reduce the complexity of our DNS script.

[#171452798]

Signed-off-by: Jen Spinney <jspinney@pivotal.io>
---
 ci/pipelines/cf-for-k8s.yml |  1 -
 hack/update-gcp-dns.sh      | 11 +++++++----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/ci/pipelines/cf-for-k8s.yml b/ci/pipelines/cf-for-k8s.yml
index 2f1537830..244f41d78 100644
--- a/ci/pipelines/cf-for-k8s.yml
+++ b/ci/pipelines/cf-for-k8s.yml
@@ -90,7 +90,6 @@ jobs:
         - -ec
         - |
           export KUBECONFIG=kube-config.yml
-          export DNS_DOMAIN="b$(date +%s).${DNS_DOMAIN}"
           gcloud auth activate-service-account --key-file=<(echo '((ci_k8s_gcp_service_account_json))') --project='((ci_k8s_gcp_project_name))' 1>/dev/null 2>&1
           gcloud container clusters get-credentials '((ci_k8s_gke_cluster_name))' --zone '((ci_k8s_gcp_project_zone))' 1>/dev/null 2>&1
 
diff --git a/hack/update-gcp-dns.sh b/hack/update-gcp-dns.sh
index 87021edfa..5c8712f69 100755
--- a/hack/update-gcp-dns.sh
+++ b/hack/update-gcp-dns.sh
@@ -20,10 +20,13 @@ external_static_ip=$(kubectl get services/istio-ingressgateway -n istio-system -
 echo "Starting transaction..."
 gcloud dns record-sets transaction start --zone="${DNS_ZONE_NAME}"
 
-echo "Deleting existing DNS A records..."
-gcloud dns record-sets list --zone="${DNS_ZONE_NAME}" --format=json | \
-  jq -r '.[] | select(.type == "A") | ("\"" + .name + "\" \"" + (.rrdatas | join(" ")) + "\"")' | \
-  xargs -n2 -I{} -t sh -c "gcloud dns record-sets transaction remove --ttl=5 --type=A --zone=\"${DNS_ZONE_NAME}\" --name={} --verbosity=debug"
+gcp_records_json="$( gcloud dns record-sets list --zone="${DNS_ZONE_NAME}" --name "*.${DNS_DOMAIN}" --format=json )"
+record_count="$( echo "${gcp_records_json}" | jq 'length' )"
+if [ "${record_count}" != "0" ]; then
+	echo "Deleting existing DNS A record..."
+  existing_record_ip="$( echo "${gcp_records_json}" | jq -r '.[0].rrdatas | join(" ")' )"
+  gcloud dns record-sets transaction remove --name "*.${DNS_DOMAIN}" --type=A --zone="${DNS_ZONE_NAME}" --ttl=5 "${existing_record_ip}" --verbosity=debug
+fi
 
 echo "Configuring DNS for external IP \"${external_static_ip}\"..."
 gcloud dns record-sets transaction add --name "*.${DNS_DOMAIN}" --type=A --zone="${DNS_ZONE_NAME}" --ttl=5 "${external_static_ip}" --verbosity=debug

From 072ce22491cc083f2ea4ba818c43887611c5b03e Mon Sep 17 00:00:00 2001
From: Dave Walter <dwalter@pivotal.io>
Date: Mon, 2 Mar 2020 13:47:11 -0800
Subject: [PATCH 06/10] Remove timestamp'd subdomain from DNS name in PR
 pipeline

[#171452798]

Signed-off-by: Jen Spinney <jspinney@pivotal.io>
---
 ci/pipelines/cf-for-k8s-prs.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ci/pipelines/cf-for-k8s-prs.yml b/ci/pipelines/cf-for-k8s-prs.yml
index 9379ffc22..ead700361 100644
--- a/ci/pipelines/cf-for-k8s-prs.yml
+++ b/ci/pipelines/cf-for-k8s-prs.yml
@@ -96,7 +96,6 @@ jobs:
             - -ec
             - |
               export KUBECONFIG=kube-config.yml
-              export DNS_DOMAIN="b$(date +%s).${DNS_DOMAIN}"
               gcloud auth activate-service-account --key-file=<(echo '((ci_k8s_gcp_service_account_json))') --project='((ci_k8s_gcp_project_name))' 1>/dev/null 2>&1
               gcloud container clusters get-credentials '((ci_k8s_gke_cluster_name))' --zone '((ci_k8s_gcp_project_zone))' 1>/dev/null 2>&1
 

From e0d72c0b9200857af341d9d5f3a9f659a0dd2bbf Mon Sep 17 00:00:00 2001
From: Dave Walter <dwalter@pivotal.io>
Date: Mon, 2 Mar 2020 15:23:21 -0800
Subject: [PATCH 07/10] Split stability tests out into a separate job

Instead of running stability tests and commit validation in the same
job, we now have a separate job just for stability tests. This new job
lives in a separate group in the same pipeline as the commit validation
pipeline.

[#171451722]
[skip ci]

Signed-off-by: Jen Spinney <jspinney@pivotal.io>
---
 ci/pipelines/cf-for-k8s.yml | 113 ++++++++++++++++++++++++++++++++++--
 1 file changed, 109 insertions(+), 4 deletions(-)

diff --git a/ci/pipelines/cf-for-k8s.yml b/ci/pipelines/cf-for-k8s.yml
index 244f41d78..62a4e917c 100644
--- a/ci/pipelines/cf-for-k8s.yml
+++ b/ci/pipelines/cf-for-k8s.yml
@@ -1,4 +1,14 @@
 ---
+groups:
+- name: cf-for-k8s
+  jobs:
+  - validate-cf-for-k8s
+  - release-validation-pool-manual
+- name: Stability Validation
+  jobs:
+  - stability-test-cf-for-k8s
+  - release-stability-pool-manual
+
 resources:
 - name: cf-for-k8s-master
   type: git
@@ -50,7 +60,7 @@ test-runner-config: &common-test-config
       repository: relintdockerhubpushbot/cf-test-runner
 
 jobs:
-- name: deploy-cf
+- name: validate-cf-for-k8s
   serial: true
   public: true
   plan:
@@ -58,8 +68,6 @@ jobs:
     params: {acquire: true}
   - get: cf-for-k8s-master
     trigger: true
-  - get: every-20-minutes
-    trigger: true
   - task: delete-cf
     config:
       <<: *common-task-config
@@ -143,7 +151,95 @@ jobs:
       repos:
         - cf-for-k8s-master
 
-- name: release-pool-manual
+- name: stability-test-cf-for-k8s
+  serial: true
+  public: true
+  plan:
+  - put: pool
+    params: {acquire: true}
+  - get: cf-for-k8s-master
+  - get: every-20-minutes
+    trigger: true
+
+  - task: delete-cf
+    config:
+      <<: *common-task-config
+      run:
+        path: /bin/bash
+        args:
+        - -ec
+        - |
+          export KUBECONFIG=kube-config.yml
+          gcloud auth activate-service-account --key-file=<(echo '((ci_k8s_gcp_service_account_json))') --project='((ci_k8s_gcp_project_name))' 1>/dev/null 2>&1
+          gcloud container clusters get-credentials '((ci_k8s_gke_cluster_name))' --zone '((ci_k8s_gcp_project_zone))' 1>/dev/null 2>&1
+
+          kapp delete -a cf --yes
+
+  - task: install-cf
+    config:
+      <<: *common-task-config
+      inputs:
+      - name: cf-for-k8s-master
+      outputs:
+      - name: env-metadata
+      params:
+        DNS_DOMAIN: ((ci_k8s_domain))
+        DNS_ZONE_NAME: ((ci_k8s_dns_zone_name))
+      run:
+        path: /bin/bash
+        args:
+        - -ec
+        - |
+          export KUBECONFIG=kube-config.yml
+          gcloud auth activate-service-account --key-file=<(echo '((ci_k8s_gcp_service_account_json))') --project='((ci_k8s_gcp_project_name))' 1>/dev/null 2>&1
+          gcloud container clusters get-credentials '((ci_k8s_gke_cluster_name))' --zone '((ci_k8s_gcp_project_zone))' 1>/dev/null 2>&1
+
+          cf-for-k8s-master/hack/confirm-network-policy.sh '((ci_k8s_gke_cluster_name))' '((ci_k8s_gcp_project_zone))'
+          echo "Generating install values..."
+          cf-for-k8s-master/hack/generate-values.sh "${DNS_DOMAIN}" > cf-install-values.yml
+          echo "Installing CF..."
+          cf-for-k8s-master/bin/install-cf.sh cf-install-values.yml
+          bosh interpolate --path /cf_admin_password cf-install-values.yml > env-metadata/cf-admin-password.txt
+          echo "${DNS_DOMAIN}" > env-metadata/dns-domain.txt
+          echo "Configuring DNS..."
+          cf-for-k8s-master/hack/update-gcp-dns.sh "${DNS_DOMAIN}" "${DNS_ZONE_NAME}"
+
+  - task: run-smoke-test
+    config:
+      <<: *common-test-config
+      inputs:
+      - name: cf-for-k8s-master
+      - name: env-metadata
+      run:
+        path: /bin/sh
+        args:
+        - -ec
+        - |
+          DNS_DOMAIN=$(cat env-metadata/dns-domain.txt)
+          export SMOKE_TEST_API_ENDPOINT="https://api.${DNS_DOMAIN}"
+          export SMOKE_TEST_APPS_DOMAIN="${DNS_DOMAIN}"
+          export SMOKE_TEST_USERNAME=admin
+          export SMOKE_TEST_PASSWORD=$(cat env-metadata/cf-admin-password.txt)
+          cf-for-k8s-master/hack/run-smoke-tests.sh
+
+  - task: delete-cf
+    config:
+      <<: *common-task-config
+      run:
+        path: /bin/bash
+        args:
+        - -ec
+        - |
+          export KUBECONFIG=kube-config.yml
+          gcloud auth activate-service-account --key-file=<(echo '((ci_k8s_gcp_service_account_json))') --project='((ci_k8s_gcp_project_name))' 1>/dev/null 2>&1
+          gcloud container clusters get-credentials '((ci_k8s_gke_cluster_name))' --zone '((ci_k8s_gcp_project_zone))' 1>/dev/null 2>&1
+
+          kapp delete -a cf --yes
+
+  - put: pool
+    params: {release: pool}
+
+- name: release-validation-pool-manual
   public: true
   plan:
   - get: pool
@@ -151,3 +247,12 @@ jobs:
     try:
       put: pool
       params: {release: pool}
+
+- name: release-stability-pool-manual
+  public: true
+  plan:
+  - get: pool # TODO should be different pool
+  ensure:
+    try:
+      put: pool
+      params: {release: pool}

From 0863b5a58a645d3c6f13a3513b73f906af1002cc Mon Sep 17 00:00:00 2001
From: Jen Spinney <jspinney@pivotal.io>
Date: Mon, 2 Mar 2020 17:29:01 -0800
Subject: [PATCH 08/10] Split CI pool into two separate pools

Previously, the job that validated new commits and the job that tested
the stabilty of cf-for-k8s by running tests every 20 minutes shared the
same pool of environments.

Going forward, we'd like to grow the environment pool for testing
commits and PRs, but if the stability tests consume from the same
environment, we run into the risk of flakes consuming all available
environments. Because of this, we decided to isolate the stability
environment pool.

[#171451722]
---
 ci/pipelines/cf-for-k8s.yml | 57 +++++++++++++++++++++++++------------
 1 file changed, 39 insertions(+), 18 deletions(-)

diff --git a/ci/pipelines/cf-for-k8s.yml b/ci/pipelines/cf-for-k8s.yml
index 62a4e917c..4ec9a6c60 100644
--- a/ci/pipelines/cf-for-k8s.yml
+++ b/ci/pipelines/cf-for-k8s.yml
@@ -27,7 +27,7 @@ resources:
     interval: 20m
 
 # semaphore for the Kubernetes cluster (ci-test-cluster in GKE); it is shared between this pipeline and the CF for K8s PR test pipeline
-- name: pool
+- name: validation-pool
   type: pool
   icon: pool
   source:
@@ -36,6 +36,15 @@ resources:
     pool: cf-for-k8s
     private_key: ((relint_ci_pools_readwrite_deploy_key.private_key))
 
+- name: stability-pool
+  type: pool
+  icon: pool
+  source:
+    uri: git@github.com:cloudfoundry/relint-ci-pools
+    branch: master
+    pool: cf-for-k8s-stability
+    private_key: ((relint_ci_pools_readwrite_deploy_key.private_key))
+
 - name: deliver-tracker-stories
   type: tracker
   source:
@@ -64,7 +73,7 @@ jobs:
   serial: true
   public: true
   plan:
-  - put: pool
+  - put: validation-pool
     params: {acquire: true}
   - get: cf-for-k8s-master
     trigger: true
@@ -143,8 +152,8 @@ jobs:
 
           kapp delete -a cf --yes
 
-  - put: pool
-    params: {release: pool}
+  - put: validation-pool
+    params: {release: validation-pool}
 
   - put: deliver-tracker-stories
     params:
@@ -155,46 +164,54 @@ jobs:
   serial: true
   public: true
   plan:
-  - put: pool
+  - put: stability-pool
     params: {acquire: true}
   - get: cf-for-k8s-master
   - get: every-20-minutes
     trigger: true
 
   - task: delete-cf
+    input_mapping: {pool-lock: stability-pool}
     config:
       <<: *common-task-config
+      inputs:
+      - name: pool-lock
       run:
         path: /bin/bash
         args:
         - -ec
         - |
+          cluster_name=$(cat pool-lock/name)
           export KUBECONFIG=kube-config.yml
           gcloud auth activate-service-account --key-file=<(echo '((ci_k8s_gcp_service_account_json))') --project='((ci_k8s_gcp_project_name))' 1>/dev/null 2>&1
-          gcloud container clusters get-credentials '((ci_k8s_gke_cluster_name))' --zone '((ci_k8s_gcp_project_zone))' 1>/dev/null 2>&1
+          gcloud container clusters get-credentials "${cluster_name}" --zone '((ci_k8s_gcp_project_zone))' 1>/dev/null 2>&1
 
           kapp delete -a cf --yes
 
   - task: install-cf
+    input_mapping: {pool-lock: stability-pool}
     config:
       <<: *common-task-config
       inputs:
       - name: cf-for-k8s-master
+      - name: pool-lock
       outputs:
       - name: env-metadata
       params:
-        DNS_DOMAIN: ((ci_k8s_domain))
         DNS_ZONE_NAME: ((ci_k8s_dns_zone_name))
+        DNS_ROOT_DOMAIN: ((ci_k8s_root_domain))
       run:
         path: /bin/bash
         args:
         - -ec
         - |
           export KUBECONFIG=kube-config.yml
+          cluster_name=$(cat pool-lock/name)
           gcloud auth activate-service-account --key-file=<(echo '((ci_k8s_gcp_service_account_json))') --project='((ci_k8s_gcp_project_name))' 1>/dev/null 2>&1
-          gcloud container clusters get-credentials '((ci_k8s_gke_cluster_name))' --zone '((ci_k8s_gcp_project_zone))' 1>/dev/null 2>&1
+          gcloud container clusters get-credentials "${cluster_name}" --zone '((ci_k8s_gcp_project_zone))' 1>/dev/null 2>&1
 
-          cf-for-k8s-master/hack/confirm-network-policy.sh '((ci_k8s_gke_cluster_name))' '((ci_k8s_gcp_project_zone))'
+          export DNS_DOMAIN="${cluster_name}.${DNS_ROOT_DOMAIN}"
+          cf-for-k8s-master/hack/confirm-network-policy.sh "${cluster_name}" '((ci_k8s_gcp_project_zone))'
           echo "Generating install values..."
           cf-for-k8s-master/hack/generate-values.sh "${DNS_DOMAIN}" > cf-install-values.yml
           echo "Installing CF..."
@@ -223,36 +240,40 @@ jobs:
           cf-for-k8s-master/hack/run-smoke-tests.sh
 
   - task: delete-cf
+    input_mapping: {pool-lock: stability-pool}
     config:
       <<: *common-task-config
+      inputs:
+      - name: pool-lock
       run:
         path: /bin/bash
         args:
         - -ec
         - |
+          cluster_name=$(cat pool-lock/name)
           export KUBECONFIG=kube-config.yml
           gcloud auth activate-service-account --key-file=<(echo '((ci_k8s_gcp_service_account_json))') --project='((ci_k8s_gcp_project_name))' 1>/dev/null 2>&1
-          gcloud container clusters get-credentials '((ci_k8s_gke_cluster_name))' --zone '((ci_k8s_gcp_project_zone))' 1>/dev/null 2>&1
+          gcloud container clusters get-credentials "${cluster_name}" --zone '((ci_k8s_gcp_project_zone))' 1>/dev/null 2>&1
 
           kapp delete -a cf --yes
 
-  - put: pool
-    params: {release: pool}
+  - put: stability-pool
+    params: {release: stability-pool}
 
 - name: release-validation-pool-manual
   public: true
   plan:
-  - get: pool
+  - get: validation-pool
   ensure:
     try:
-      put: pool
-      params: {release: pool}
+      put: validation-pool
+      params: {release: validation-pool}
 
 - name: release-stability-pool-manual
   public: true
   plan:
-  - get: pool # TODO should be different pool
+  - get: stability-pool
   ensure:
     try:
-      put: pool
-      params: {release: pool}
+      put: stability-pool
+      params: {release: stability-pool}

From ac54e303db6737d6ec07d5fc040497146f0b3174 Mon Sep 17 00:00:00 2001
From: Andrew Costa <acosta@pivotal.io>
Date: Mon, 2 Mar 2020 15:54:53 -0800
Subject: [PATCH 09/10] Use image digest for unstable or absent image tags

- also update development doc to include image reference guidance.

[finishes #171454907](https://www.pivotaltracker.com/story/show/171454907)

Co-authored-by: Sebastian Vidrio <svidrio@pivotal.io>
Co-authored-by: Dave Walter <dwalter@pivotal.io>
---
 config/logging.yml  |  8 +++++++-
 config/values.yml   | 12 ++++++------
 docs/development.md |  4 ++++
 3 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/config/logging.yml b/config/logging.yml
index 167394956..791060c23 100644
--- a/config/logging.yml
+++ b/config/logging.yml
@@ -8,7 +8,13 @@
 system_domain: #@ data.values.system_domain
 system_namespace: #@ data.values.system_namespace
 
-log_cache_ca: 
+images:
+  log_cache: #@ data.values.images.log_cache
+  syslog_server: #@ data.values.images.syslog_server
+  log_cache_gateway: #@ data.values.images.log_cache_gateway
+  fluent: #@ data.values.images.fluent
+
+log_cache_ca:
   crt: #@ data.values.log_cache_ca.crt
   key: #@ data.values.log_cache_ca.key
 
diff --git a/config/values.yml b/config/values.yml
index 5d0775fe0..1282ad016 100644
--- a/config/values.yml
+++ b/config/values.yml
@@ -25,12 +25,12 @@ images:
 #! NOTE: the CAPI image reference below is needed temporarily to address a build break.
 #! See https://www.pivotaltracker.com/story/show/171522062 and https://www.pivotaltracker.com/story/show/171549565 for details
   capi: "cloudfoundry/cloud-controller-ng@sha256:402932286e330049ea8e6562db615080b26388d849c8bb4aa2eed544b33e2c2a"
-  nginx: ""
-  cfroutesync: ""
-  log_cache: "logcache/log-cache"
-  syslog_server: "logcache/syslog-server"
-  log_cache_gateway: "logcache/log-cache-gateway"
-  fluent: "logcache/cf-k8s-logging"
+  nginx: "nginx@sha256:4a50ed86d8c86e35f530d4a168173677a192177eed14146fbb5728b1b3a2d4de"
+  cfroutesync: "gcr.io/cf-networking-images/cf-k8s-networking/cfroutesync@sha256:758abd1b6144596cef74ad4805bc7e1a6055142cab5719f0d42d0321a14a190d"
+  log_cache: "logcache/log-cache@sha256:0bc1ee1934ecd2145a3372dc838747c7977d50ca58bd5c28db7f70b28b1021b4"
+  syslog_server: "logcache/syslog-server@sha256:f1f6def4af1261e4cafe4745d5bab8258a2e1ff9fd835c5021e00957ea42d7d4"
+  log_cache_gateway: "logcache/log-cache-gateway@sha256:dd639d3c4cbb120cbbd00850b319bc0c60fcc003979f8f8d6f5198ef985e6a8c"
+  fluent: "logcache/cf-k8s-logging@sha256:8107d5923e630f792631c21e6c862bc027cc1ae275202ac1be3f9b19f4991698"
 
 system_certificate:
   #! Base64-encoded certificate for the wildcard
diff --git a/docs/development.md b/docs/development.md
index cf9109f46..3e8ca0c77 100644
--- a/docs/development.md
+++ b/docs/development.md
@@ -32,6 +32,10 @@ SMOKE_TEST_API_ENDPOINT=https://api.system.cf.example.com SMOKE_TEST_USERNAME=ad
   - this directory is only used by cf-for-k8s maintainers
   - `build.sh` in each sub-directory has specific build instructions
 
+## Image References
+
+Image references are expected to use an image SHA digest. If using [kbld](https://get-kbld.io/) to build images as suggested in the component development flow, the image reference should include the digest by default.
+
 ## Tips
 
 - `alias k=kubectl`

From c92f2bfc25d3bc39425044f00e96c8387f453c2a Mon Sep 17 00:00:00 2001
From: Andrew Costa <acosta@pivotal.io>
Date: Fri, 28 Feb 2020 14:46:54 -0800
Subject: [PATCH 10/10] Implement dev to master CI

also refactor cf-for-k8s-pr and reliability testing into groups under
single pipeline

[#171524414](https://www.pivotaltracker.com/story/show/171524414)
[#171451722](https://www.pivotaltracker.com/story/show/171451722)

Co-authored-by: Sebastian Vidrio <svidrio@pivotal.io>
---
 ci/pipelines/cf-for-k8s-prs.yml | 164 -------------------------
 ci/pipelines/cf-for-k8s.yml     | 211 ++++++++++++++++++++++++++++----
 2 files changed, 186 insertions(+), 189 deletions(-)
 delete mode 100644 ci/pipelines/cf-for-k8s-prs.yml

diff --git a/ci/pipelines/cf-for-k8s-prs.yml b/ci/pipelines/cf-for-k8s-prs.yml
deleted file mode 100644
index ead700361..000000000
--- a/ci/pipelines/cf-for-k8s-prs.yml
+++ /dev/null
@@ -1,164 +0,0 @@
----
-resource_types:
-- name: pull-request
-  type: docker-image
-  source:
-    repository: teliaoss/github-pr-resource
-
-resources:
-- name: runtime-ci
-  type: git
-  icon: github-box
-  source:
-    branch: master
-    uri: https://github.com/cloudfoundry/runtime-ci.git
-
-- name: cf-for-k8s-pr
-  type: pull-request
-  source:
-    access_token: ((github_status_bot_repo_access_token))
-    repository: cloudfoundry/cf-for-k8s
-    disable_forks: true
-
-# semaphore for the Kubernetes cluster (ci-test-cluster in GKE); it is shared between this pipeline and the main CF for K8s test pipeline
-- name: pool
-  type: pool
-  icon: pool
-  source:
-    uri: git@github.com:cloudfoundry/relint-ci-pools
-    branch: master
-    pool: cf-for-k8s
-    private_key: ((relint_ci_pools_readwrite_deploy_key.private_key))
-
-# Extract common platform and image_resource details so task configs can be more concise
-x-config: &common-task-config
-  platform: linux
-  image_resource:
-    type: docker-image
-    source:
-      repository: relintdockerhubpushbot/cf-for-k8s-ci
-
-# Extract common platform and image_resource details so task configs can be more concise
-test-runner-config: &common-test-config
-  platform: linux
-  image_resource:
-    type: docker-image
-    source:
-      repository: relintdockerhubpushbot/cf-test-runner
-
-jobs:
-- name: run-tests-on-cf-for-k8s-pr
-  public: true
-  plan:
-  - in_parallel:
-    - get: runtime-ci
-    - get: cf-for-k8s-pr
-      params:
-        integration_tool: rebase
-      trigger: true
-      version: every
-  - put: cf-for-k8s-pr
-    params:
-      path: cf-for-k8s-pr
-      status: pending
-      context: tests
-  - do:
-    - put: pool
-      params: {acquire: true}
-    - do:
-      - task: delete-cf
-        config:
-          <<: *common-task-config
-          run:
-            path: /bin/bash
-            args:
-            - -ec
-            - |
-              export KUBECONFIG=kube-config.yml
-              gcloud auth activate-service-account --key-file=<(echo '((ci_k8s_gcp_service_account_json))') --project='((ci_k8s_gcp_project_name))' 1>/dev/null 2>&1
-              gcloud container clusters get-credentials '((ci_k8s_gke_cluster_name))' --zone '((ci_k8s_gcp_project_zone))' 1>/dev/null 2>&1
-
-              kapp delete -a cf --yes
-
-      - task: install-cf
-        config:
-          <<: *common-task-config
-          inputs:
-          - name: cf-for-k8s-pr
-          outputs:
-          - name: env-metadata
-          params:
-            DNS_DOMAIN: ((ci_k8s_domain))
-            DNS_ZONE_NAME: ((ci_k8s_dns_zone_name))
-          run:
-            path: /bin/bash
-            args:
-            - -ec
-            - |
-              export KUBECONFIG=kube-config.yml
-              gcloud auth activate-service-account --key-file=<(echo '((ci_k8s_gcp_service_account_json))') --project='((ci_k8s_gcp_project_name))' 1>/dev/null 2>&1
-              gcloud container clusters get-credentials '((ci_k8s_gke_cluster_name))' --zone '((ci_k8s_gcp_project_zone))' 1>/dev/null 2>&1
-
-              cf-for-k8s-pr/hack/confirm-network-policy.sh '((ci_k8s_gke_cluster_name))' '((ci_k8s_gcp_project_zone))'
-              echo "Generating install values..."
-              cf-for-k8s-pr/hack/generate-values.sh "${DNS_DOMAIN}" > cf-install-values.yml
-              echo "Installing CF..."
-              cf-for-k8s-pr/bin/install-cf.sh cf-install-values.yml
-              bosh interpolate --path /cf_admin_password cf-install-values.yml > env-metadata/cf-admin-password.txt
-              echo "${DNS_DOMAIN}" > env-metadata/dns-domain.txt
-              echo "Configuring DNS..."
-              cf-for-k8s-pr/hack/update-gcp-dns.sh "${DNS_DOMAIN}" "${DNS_ZONE_NAME}"
-
-      - task: run-smoke-test
-        config:
-          <<: *common-test-config
-          inputs:
-          - name: cf-for-k8s-pr
-          - name: env-metadata
-          run:
-            path: /bin/sh
-            args:
-            - -ec
-            - |
-              DNS_DOMAIN=$(cat env-metadata/dns-domain.txt)
-              export SMOKE_TEST_API_ENDPOINT="https://api.${DNS_DOMAIN}"
-              export SMOKE_TEST_APPS_DOMAIN="${DNS_DOMAIN}"
-              export SMOKE_TEST_USERNAME=admin
-              export SMOKE_TEST_PASSWORD=$(cat env-metadata/cf-admin-password.txt)
-              cf-for-k8s-pr/hack/run-smoke-tests.sh
-
-      - task: delete-cf
-        config:
-          <<: *common-task-config
-          run:
-            path: /bin/bash
-            args:
-            - -ec
-            - |
-              export KUBECONFIG=kube-config.yml
-              gcloud auth activate-service-account --key-file=<(echo '((ci_k8s_gcp_service_account_json))') --project='((ci_k8s_gcp_project_name))' 1>/dev/null 2>&1
-              gcloud container clusters get-credentials '((ci_k8s_gke_cluster_name))' --zone '((ci_k8s_gcp_project_zone))' 1>/dev/null 2>&1
-
-              kapp delete -a cf --yes
-
-      on_failure:
-        do:
-        - task: write-pr-check-failure-comment
-          file: runtime-ci/tasks/write-pr-check-failure-comment/task.yml
-          input_mapping:
-            pull-request: cf-for-k8s-pr
-        - put: cf-for-k8s-pr
-          params:
-            path: cf-for-k8s-pr
-            status: failure
-            context: tests
-            comment_file: pull-request-comment/comment
-      on_success:
-        put: cf-for-k8s-pr
-        params:
-          path: cf-for-k8s-pr
-          status: success
-          context: tests
-    ensure:
-      put: pool
-      params: {release: pool}
diff --git a/ci/pipelines/cf-for-k8s.yml b/ci/pipelines/cf-for-k8s.yml
index 4ec9a6c60..2585f3013 100644
--- a/ci/pipelines/cf-for-k8s.yml
+++ b/ci/pipelines/cf-for-k8s.yml
@@ -8,24 +8,63 @@ groups:
   jobs:
   - stability-test-cf-for-k8s
   - release-stability-pool-manual
+- name: PRs
+  jobs:
+  - run-tests-on-cf-for-k8s-pr
+
+resource_types:
+- name: pull-request
+  type: docker-image
+  source:
+    repository: teliaoss/github-pr-resource
 
 resources:
-- name: cf-for-k8s-master
+- name: cf-for-k8s-develop
   type: git
   icon: github-box
   source:
-    branch: master
+    branch: test-develop
     uri: git@github.com:cloudfoundry/cf-for-k8s
     private_key: ((cf_for_k8s_readonly_deploy_key.private_key))
     ignore_paths:
       - ci/**
 
+- name: cf-for-k8s-master
+  type: git
+  icon: github-box
+  source:
+    branch: test-master
+    uri: git@github.com:cloudfoundry/cf-for-k8s
+    private_key: ((cf_for_k8s_readonly_deploy_key.private_key))
+
+- name: cf-for-k8s-master-push
+  type: git
+  icon: github-box
+  source:
+    branch: test-master
+    uri: git@github.com:cloudfoundry/cf-for-k8s
+    private_key: ((cf_for_k8s_readwrite_deploy_key.private_key))
+
 - name: every-20-minutes
   type: time
   icon: clock-outline
   source:
     interval: 20m
 
+- name: runtime-ci
+  type: git
+  icon: github-box
+  source:
+    branch: master
+    uri: https://github.com/cloudfoundry/runtime-ci.git
+
+- name: cf-for-k8s-pr
+  type: pull-request
+  source:
+    access_token: ((github_status_bot_repo_access_token))
+    repository: cloudfoundry/cf-for-k8s
+    disable_forks: true
+
 # semaphore for the Kubernetes cluster (ci-test-cluster in GKE); it is shared between this pipeline and the CF for K8s PR test pipeline
 - name: validation-pool
   type: pool
@@ -75,7 +114,7 @@ jobs:
   plan:
   - put: validation-pool
     params: {acquire: true}
-  - get: cf-for-k8s-master
+  - get: cf-for-k8s-develop
     trigger: true
   - task: delete-cf
     config:
@@ -95,7 +134,7 @@ jobs:
     config:
       <<: *common-task-config
       inputs:
-      - name: cf-for-k8s-master
+      - name: cf-for-k8s-develop
       outputs:
       - name: env-metadata
       params:
@@ -110,21 +149,21 @@ jobs:
           gcloud auth activate-service-account --key-file=<(echo '((ci_k8s_gcp_service_account_json))') --project='((ci_k8s_gcp_project_name))' 1>/dev/null 2>&1
           gcloud container clusters get-credentials '((ci_k8s_gke_cluster_name))' --zone '((ci_k8s_gcp_project_zone))' 1>/dev/null 2>&1
 
-          cf-for-k8s-master/hack/confirm-network-policy.sh '((ci_k8s_gke_cluster_name))' '((ci_k8s_gcp_project_zone))'
+          cf-for-k8s-develop/hack/confirm-network-policy.sh '((ci_k8s_gke_cluster_name))' '((ci_k8s_gcp_project_zone))'
           echo "Generating install values..."
-          cf-for-k8s-master/hack/generate-values.sh "${DNS_DOMAIN}" > cf-install-values.yml
+          cf-for-k8s-develop/hack/generate-values.sh "${DNS_DOMAIN}" > cf-install-values.yml
           echo "Installing CF..."
-          cf-for-k8s-master/bin/install-cf.sh cf-install-values.yml
+          cf-for-k8s-develop/bin/install-cf.sh cf-install-values.yml
           bosh interpolate --path /cf_admin_password cf-install-values.yml > env-metadata/cf-admin-password.txt
           echo "${DNS_DOMAIN}" > env-metadata/dns-domain.txt
           echo "Configuring DNS..."
-          cf-for-k8s-master/hack/update-gcp-dns.sh "${DNS_DOMAIN}" "${DNS_ZONE_NAME}"
+          cf-for-k8s-develop/hack/update-gcp-dns.sh "${DNS_DOMAIN}" "${DNS_ZONE_NAME}"
 
   - task: run-smoke-test
     config:
       <<: *common-test-config
       inputs:
-      - name: cf-for-k8s-master
+      - name: cf-for-k8s-develop
       - name: env-metadata
       run:
         path: /bin/sh
@@ -136,7 +175,7 @@ jobs:
           export SMOKE_TEST_APPS_DOMAIN="${DNS_DOMAIN}"
           export SMOKE_TEST_USERNAME=admin
           export SMOKE_TEST_PASSWORD=$(cat env-metadata/cf-admin-password.txt)
-          cf-for-k8s-master/hack/run-smoke-tests.sh
+          cf-for-k8s-develop/hack/run-smoke-tests.sh
 
   - task: delete-cf
     config:
@@ -152,23 +191,37 @@ jobs:
 
           kapp delete -a cf --yes
 
-  - put: validation-pool
-    params: {release: validation-pool}
+  - put: cf-for-k8s-master-push
+    params:
+      repository: cf-for-k8s-develop
 
   - put: deliver-tracker-stories
     params:
       repos:
-        - cf-for-k8s-master
+      - cf-for-k8s-develop
+
+  - put: validation-pool
+    params: {release: validation-pool}
+
+- name: release-validation-pool-manual
+  public: true
+  plan:
+  - get: validation-pool
+  ensure:
+    try:
+      put: validation-pool
+      params: {release: validation-pool}
 
 - name: stability-test-cf-for-k8s
   serial: true
   public: true
   plan:
+  - in_parallel:
+    - get: cf-for-k8s-master
+    - get: every-20-minutes
+      trigger: true
   - put: stability-pool
     params: {acquire: true}
-  - get: cf-for-k8s-master
-  - get: every-20-minutes
-    trigger: true
 
   - task: delete-cf
     input_mapping: {pool-lock: stability-pool}
@@ -260,15 +313,6 @@ jobs:
   - put: stability-pool
     params: {release: stability-pool}
 
-- name: release-validation-pool-manual
-  public: true
-  plan:
-  - get: validation-pool
-  ensure:
-    try:
-      put: validation-pool
-      params: {release: validation-pool}
-
 - name: release-stability-pool-manual
   public: true
   plan:
@@ -277,3 +321,120 @@ jobs:
     try:
       put: stability-pool
       params: {release: stability-pool}
+
+- name: run-tests-on-cf-for-k8s-pr
+  public: true
+  plan:
+  - in_parallel:
+    - get: runtime-ci
+    - get: cf-for-k8s-pr
+      params:
+        integration_tool: rebase
+      trigger: true
+      version: every
+  - put: cf-for-k8s-pr
+    params:
+      path: cf-for-k8s-pr
+      status: pending
+      context: tests
+  - do:
+    - put: validation-pool
+      params: {acquire: true}
+    - do:
+      - task: delete-cf
+        config:
+          <<: *common-task-config
+          run:
+            path: /bin/bash
+            args:
+            - -ec
+            - |
+              export KUBECONFIG=kube-config.yml
+              gcloud auth activate-service-account --key-file=<(echo '((ci_k8s_gcp_service_account_json))') --project='((ci_k8s_gcp_project_name))' 1>/dev/null 2>&1
+              gcloud container clusters get-credentials '((ci_k8s_gke_cluster_name))' --zone '((ci_k8s_gcp_project_zone))' 1>/dev/null 2>&1
+
+              kapp delete -a cf --yes
+
+      - task: install-cf
+        config:
+          <<: *common-task-config
+          inputs:
+          - name: cf-for-k8s-pr
+          outputs:
+          - name: env-metadata
+          params:
+            DNS_DOMAIN: ((ci_k8s_domain))
+            DNS_ZONE_NAME: ((ci_k8s_dns_zone_name))
+          run:
+            path: /bin/bash
+            args:
+            - -ec
+            - |
+              export KUBECONFIG=kube-config.yml
+              export DNS_DOMAIN="b$(date +%s).${DNS_DOMAIN}"
+              gcloud auth activate-service-account --key-file=<(echo '((ci_k8s_gcp_service_account_json))') --project='((ci_k8s_gcp_project_name))' 1>/dev/null 2>&1
+              gcloud container clusters get-credentials '((ci_k8s_gke_cluster_name))' --zone '((ci_k8s_gcp_project_zone))' 1>/dev/null 2>&1
+
+              cf-for-k8s-pr/hack/confirm-network-policy.sh '((ci_k8s_gke_cluster_name))' '((ci_k8s_gcp_project_zone))'
+              echo "Generating install values..."
+              cf-for-k8s-pr/hack/generate-values.sh "${DNS_DOMAIN}" > cf-install-values.yml
+              echo "Installing CF..."
+              cf-for-k8s-pr/bin/install-cf.sh cf-install-values.yml
+              bosh interpolate --path /cf_admin_password cf-install-values.yml > env-metadata/cf-admin-password.txt
+              echo "${DNS_DOMAIN}" > env-metadata/dns-domain.txt
+              echo "Configuring DNS..."
+              cf-for-k8s-pr/hack/update-gcp-dns.sh "${DNS_DOMAIN}" "${DNS_ZONE_NAME}"
+
+      - task: run-smoke-test
+        config:
+          <<: *common-test-config
+          inputs:
+          - name: cf-for-k8s-pr
+          - name: env-metadata
+          run:
+            path: /bin/sh
+            args:
+            - -ec
+            - |
+              DNS_DOMAIN=$(cat env-metadata/dns-domain.txt)
+              export SMOKE_TEST_API_ENDPOINT="https://api.${DNS_DOMAIN}"
+              export SMOKE_TEST_APPS_DOMAIN="${DNS_DOMAIN}"
+              export SMOKE_TEST_USERNAME=admin
+              export SMOKE_TEST_PASSWORD=$(cat env-metadata/cf-admin-password.txt)
+              cf-for-k8s-pr/hack/run-smoke-tests.sh
+
+      - task: delete-cf
+        config:
+          <<: *common-task-config
+          run:
+            path: /bin/bash
+            args:
+            - -ec
+            - |
+              export KUBECONFIG=kube-config.yml
+              gcloud auth activate-service-account --key-file=<(echo '((ci_k8s_gcp_service_account_json))') --project='((ci_k8s_gcp_project_name))' 1>/dev/null 2>&1
+              gcloud container clusters get-credentials '((ci_k8s_gke_cluster_name))' --zone '((ci_k8s_gcp_project_zone))' 1>/dev/null 2>&1
+
+              kapp delete -a cf --yes
+
+      on_failure:
+        do:
+        - task: write-pr-check-failure-comment
+          file: runtime-ci/tasks/write-pr-check-failure-comment/task.yml
+          input_mapping:
+            pull-request: cf-for-k8s-pr
+        - put: cf-for-k8s-pr
+          params:
+            path: cf-for-k8s-pr
+            status: failure
+            context: tests
+            comment_file: pull-request-comment/comment
+      on_success:
+        put: cf-for-k8s-pr
+        params:
+          path: cf-for-k8s-pr
+          status: success
+          context: tests
+    ensure:
+      put: validation-pool
+      params: {release: validation-pool}