From a8ebbcc691b8a3912f1a88f4ae2f1cff7f34d3da Mon Sep 17 00:00:00 2001 From: Shwetha Gururaj Date: Tue, 5 Mar 2024 14:16:46 -0500 Subject: [PATCH 01/12] Update CLI to use ginkgo v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add user agent info for plugin api calls - Newer lager was needed with newer diego-ssh * Replace depracted ginkgo calls with non deprecated calls * Remove ACK_GINKGO_RC * Desperate attempt at resolving output issue by using serial decorator and using non deprecated pty * Turn off Gingkno v2 parallel mode * Adjust integration tests to use ginkgo v2 * Add Byte to lograte, disk and memory regexes used by int tests --------- Co-authored-by: M. Oleske Co-authored-by: Al Berez Co-authored-by: Cristhian Peña Co-authored-by: Marc Paquette --- .github/workflows/tests-integration-reusable.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests-integration-reusable.yml b/.github/workflows/tests-integration-reusable.yml index 856e88ee585..ba7758ebacc 100644 --- a/.github/workflows/tests-integration-reusable.yml +++ b/.github/workflows/tests-integration-reusable.yml @@ -32,7 +32,7 @@ jobs: runs-on: ${{ inputs.os }} steps: - - uses: LouisBrunner/checks-action@v2.0.0 + - uses: LouisBrunner/checks-action@v1.6.2 if: always() id: check with: @@ -301,7 +301,7 @@ jobs: grep -q -E "HTTP/[[:digit:]\.]{1,3} 404" headers.txt && echo $(echo "$output" | jq '.messages | join(", ")') && exit 2 grep -q -E "HTTP/[[:digit:]\.]{1,3} 202" headers.txt && break done - - uses: LouisBrunner/checks-action@v2.0.0 + - uses: LouisBrunner/checks-action@v1.6.2 if: always() with: token: ${{ secrets.GITHUB_TOKEN }} From 68d7e419dde9893cba0af4880b5ad7cb3c39d7dd Mon Sep 17 00:00:00 2001 From: Shwetha Gururaj Date: Tue, 5 Mar 2024 14:29:40 -0500 Subject: [PATCH 02/12] run v8 --- .github/workflows/tests-integration.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests-integration.yml b/.github/workflows/tests-integration.yml index 270e87cd38a..1ddb1901128 100644 --- a/.github/workflows/tests-integration.yml +++ b/.github/workflows/tests-integration.yml @@ -1,19 +1,22 @@ name: "Tests: Integration" -run-name: "Integration [${{ github.event.workflow_run.head_branch }}]: ${{ github.event.workflow_run.head_commit.message }}" +# run-name: "Integration [${{ github.event.workflow_run.head_branch }}]: ${{ github.event.workflow_run.head_commit.message }}" on: workflow_dispatch: + push: + branches: + - v8 workflow_run: - workflows: - - "Tests" - types: - - completed + # workflows: + # - "Tests" + # types: + # - completed jobs: run-integration-tests-cf-env: name: Integration tests - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + # if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} uses: ./.github/workflows/tests-integration-reusable.yml with: capi-version: edge @@ -24,7 +27,7 @@ jobs: run-integration-tests-cf-env-with-client-creds: name: client creds - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + # if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} uses: ./.github/workflows/tests-integration-reusable.yml with: capi-version: edge @@ -35,7 +38,7 @@ jobs: run-integration-tests-cf-env-with-min-capi: name: MIN CAPI - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + # if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} uses: ./.github/workflows/tests-integration-reusable.yml with: capi-version: min From 486350d06f575cbfe8cc11acef06a88f5cbbfbac Mon Sep 17 00:00:00 2001 From: Shwetha Gururaj Date: Tue, 5 Mar 2024 14:32:43 -0500 Subject: [PATCH 03/12] Fix syntax --- .github/workflows/tests-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-integration.yml b/.github/workflows/tests-integration.yml index 1ddb1901128..ea2cb5df1e0 100644 --- a/.github/workflows/tests-integration.yml +++ b/.github/workflows/tests-integration.yml @@ -4,10 +4,10 @@ name: "Tests: Integration" on: workflow_dispatch: + workflow_run: push: branches: - v8 - workflow_run: # workflows: # - "Tests" # types: From a565410e094563060df3a7759ebb64266bfe2c2e Mon Sep 17 00:00:00 2001 From: Shwetha Gururaj Date: Tue, 5 Mar 2024 14:33:46 -0500 Subject: [PATCH 04/12] Remove on workflow run --- .github/workflows/tests-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-integration.yml b/.github/workflows/tests-integration.yml index ea2cb5df1e0..ba850128a00 100644 --- a/.github/workflows/tests-integration.yml +++ b/.github/workflows/tests-integration.yml @@ -4,10 +4,10 @@ name: "Tests: Integration" on: workflow_dispatch: - workflow_run: push: branches: - v8 + # workflow_run: # workflows: # - "Tests" # types: From 38f730a09234f518b783e15d858d1929d03d2ac5 Mon Sep 17 00:00:00 2001 From: Shwetha Gururaj Date: Tue, 5 Mar 2024 14:34:51 -0500 Subject: [PATCH 05/12] Remove PR Check --- .../workflows/tests-integration-reusable.yml | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/tests-integration-reusable.yml b/.github/workflows/tests-integration-reusable.yml index ba7758ebacc..7e195ca399f 100644 --- a/.github/workflows/tests-integration-reusable.yml +++ b/.github/workflows/tests-integration-reusable.yml @@ -32,16 +32,16 @@ jobs: runs-on: ${{ inputs.os }} steps: - - uses: LouisBrunner/checks-action@v1.6.2 - if: always() - id: check - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: "${{ inputs.name }}" - status: in_progress - sha: ${{github.event.workflow_run.head_sha}} - output: | - {"title": "${{ inputs.name }}", "summary":"started ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"} + # - uses: LouisBrunner/checks-action@v1.6.2 + # if: always() + # id: check + # with: + # token: ${{ secrets.GITHUB_TOKEN }} + # name: "${{ inputs.name }}" + # status: in_progress + # sha: ${{github.event.workflow_run.head_sha}} + # output: | + # {"title": "${{ inputs.name }}", "summary":"started ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"} - name: Checkout cli uses: actions/checkout@v4 with: @@ -301,12 +301,12 @@ jobs: grep -q -E "HTTP/[[:digit:]\.]{1,3} 404" headers.txt && echo $(echo "$output" | jq '.messages | join(", ")') && exit 2 grep -q -E "HTTP/[[:digit:]\.]{1,3} 202" headers.txt && break done - - uses: LouisBrunner/checks-action@v1.6.2 - if: always() - with: - token: ${{ secrets.GITHUB_TOKEN }} - check_id: ${{ steps.check.outputs.check_id }} - conclusion: ${{ job.status }} - sha: ${{github.event.workflow_run.head_sha}} - output: | - {"title": "${{ inputs.name }}", "summary":"finished ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"} + # - uses: LouisBrunner/checks-action@v1.6.2 + # if: always() + # with: + # token: ${{ secrets.GITHUB_TOKEN }} + # check_id: ${{ steps.check.outputs.check_id }} + # conclusion: ${{ job.status }} + # sha: ${{github.event.workflow_run.head_sha}} + # output: | + # {"title": "${{ inputs.name }}", "summary":"finished ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"} From de24cf7b021444a837726aac0288523bf4462c5c Mon Sep 17 00:00:00 2001 From: Shwetha Gururaj Date: Tue, 5 Mar 2024 16:56:51 -0500 Subject: [PATCH 06/12] Revert workflow changes --- .../workflows/tests-integration-reusable.yml | 38 +++++++++---------- .github/workflows/tests-integration.yml | 21 +++++----- 2 files changed, 28 insertions(+), 31 deletions(-) diff --git a/.github/workflows/tests-integration-reusable.yml b/.github/workflows/tests-integration-reusable.yml index 7e195ca399f..ba7758ebacc 100644 --- a/.github/workflows/tests-integration-reusable.yml +++ b/.github/workflows/tests-integration-reusable.yml @@ -32,16 +32,16 @@ jobs: runs-on: ${{ inputs.os }} steps: - # - uses: LouisBrunner/checks-action@v1.6.2 - # if: always() - # id: check - # with: - # token: ${{ secrets.GITHUB_TOKEN }} - # name: "${{ inputs.name }}" - # status: in_progress - # sha: ${{github.event.workflow_run.head_sha}} - # output: | - # {"title": "${{ inputs.name }}", "summary":"started ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"} + - uses: LouisBrunner/checks-action@v1.6.2 + if: always() + id: check + with: + token: ${{ secrets.GITHUB_TOKEN }} + name: "${{ inputs.name }}" + status: in_progress + sha: ${{github.event.workflow_run.head_sha}} + output: | + {"title": "${{ inputs.name }}", "summary":"started ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"} - name: Checkout cli uses: actions/checkout@v4 with: @@ -301,12 +301,12 @@ jobs: grep -q -E "HTTP/[[:digit:]\.]{1,3} 404" headers.txt && echo $(echo "$output" | jq '.messages | join(", ")') && exit 2 grep -q -E "HTTP/[[:digit:]\.]{1,3} 202" headers.txt && break done - # - uses: LouisBrunner/checks-action@v1.6.2 - # if: always() - # with: - # token: ${{ secrets.GITHUB_TOKEN }} - # check_id: ${{ steps.check.outputs.check_id }} - # conclusion: ${{ job.status }} - # sha: ${{github.event.workflow_run.head_sha}} - # output: | - # {"title": "${{ inputs.name }}", "summary":"finished ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"} + - uses: LouisBrunner/checks-action@v1.6.2 + if: always() + with: + token: ${{ secrets.GITHUB_TOKEN }} + check_id: ${{ steps.check.outputs.check_id }} + conclusion: ${{ job.status }} + sha: ${{github.event.workflow_run.head_sha}} + output: | + {"title": "${{ inputs.name }}", "summary":"finished ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"} diff --git a/.github/workflows/tests-integration.yml b/.github/workflows/tests-integration.yml index ba850128a00..270e87cd38a 100644 --- a/.github/workflows/tests-integration.yml +++ b/.github/workflows/tests-integration.yml @@ -1,22 +1,19 @@ name: "Tests: Integration" -# run-name: "Integration [${{ github.event.workflow_run.head_branch }}]: ${{ github.event.workflow_run.head_commit.message }}" +run-name: "Integration [${{ github.event.workflow_run.head_branch }}]: ${{ github.event.workflow_run.head_commit.message }}" on: workflow_dispatch: - push: - branches: - - v8 - # workflow_run: - # workflows: - # - "Tests" - # types: - # - completed + workflow_run: + workflows: + - "Tests" + types: + - completed jobs: run-integration-tests-cf-env: name: Integration tests - # if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} uses: ./.github/workflows/tests-integration-reusable.yml with: capi-version: edge @@ -27,7 +24,7 @@ jobs: run-integration-tests-cf-env-with-client-creds: name: client creds - # if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} uses: ./.github/workflows/tests-integration-reusable.yml with: capi-version: edge @@ -38,7 +35,7 @@ jobs: run-integration-tests-cf-env-with-min-capi: name: MIN CAPI - # if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} uses: ./.github/workflows/tests-integration-reusable.yml with: capi-version: min From e1b63aa9a3503a09bb72e4634a7dd0a5d334260a Mon Sep 17 00:00:00 2001 From: Shwetha Gururaj Date: Tue, 12 Mar 2024 11:00:45 -0400 Subject: [PATCH 07/12] Update checks-action version --- .github/workflows/tests-integration-reusable.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests-integration-reusable.yml b/.github/workflows/tests-integration-reusable.yml index ba7758ebacc..856e88ee585 100644 --- a/.github/workflows/tests-integration-reusable.yml +++ b/.github/workflows/tests-integration-reusable.yml @@ -32,7 +32,7 @@ jobs: runs-on: ${{ inputs.os }} steps: - - uses: LouisBrunner/checks-action@v1.6.2 + - uses: LouisBrunner/checks-action@v2.0.0 if: always() id: check with: @@ -301,7 +301,7 @@ jobs: grep -q -E "HTTP/[[:digit:]\.]{1,3} 404" headers.txt && echo $(echo "$output" | jq '.messages | join(", ")') && exit 2 grep -q -E "HTTP/[[:digit:]\.]{1,3} 202" headers.txt && break done - - uses: LouisBrunner/checks-action@v1.6.2 + - uses: LouisBrunner/checks-action@v2.0.0 if: always() with: token: ${{ secrets.GITHUB_TOKEN }} From 78980c9d8efd7898f708c9955eae0241a0fdb24c Mon Sep 17 00:00:00 2001 From: Shwetha Gururaj Date: Wed, 27 Mar 2024 15:04:40 -0400 Subject: [PATCH 08/12] fix dependencies --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index c00c7e28b75..2cc8b5a3887 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.20.0/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= -github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= -github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= +github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= +github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/openzipkin/zipkin-go v0.4.2 h1:zjqfqHjUpPmB3c1GlCvvgsM1G4LkvqQbBDueDOCg/jA= From 7b8b510ca94c41500b2b94ac80e7b0945e329fff Mon Sep 17 00:00:00 2001 From: Shwetha Gururaj Date: Fri, 26 Apr 2024 13:53:15 -0400 Subject: [PATCH 09/12] Shepherd migration (#13) Use shepherd environments instead of toolsmiths to run integration tests. As part of this change, min capi job is disabled since we are still figuring out our versioning policy of how long CLI should support an old version of CAPI. --- .../workflows/tests-integration-reusable.yml | 99 +++++++++---------- .github/workflows/tests-integration.yml | 20 ++-- 2 files changed, 58 insertions(+), 61 deletions(-) diff --git a/.github/workflows/tests-integration-reusable.yml b/.github/workflows/tests-integration-reusable.yml index 856e88ee585..67aa8e95c86 100644 --- a/.github/workflows/tests-integration-reusable.yml +++ b/.github/workflows/tests-integration-reusable.yml @@ -2,9 +2,8 @@ # secrets.CLIENT_SECRET # secrets.GITHUB_TOKEN -# secrets.TOOLSMITHS_API_TOKEN -# vars.TOOLSMITHS_HOSTNAME -# vars.TOOLSMITHS_POOL_NAME +# secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN +# vars.SHEPHERD_POOL_NAME name: "pvt: run integration tests" @@ -30,7 +29,7 @@ jobs: run: shell: bash runs-on: ${{ inputs.os }} - + container: us-west2-docker.pkg.dev/shepherd-268822/shepherd2/concourse-resource:latest steps: - uses: LouisBrunner/checks-action@v2.0.0 if: always() @@ -42,6 +41,7 @@ jobs: sha: ${{github.event.workflow_run.head_sha}} output: | {"title": "${{ inputs.name }}", "summary":"started ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"} + - name: Checkout cli uses: actions/checkout@v4 with: @@ -56,6 +56,8 @@ jobs: - id: read-min-capi name: Read MIN CAPI run: | + wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq &&\ + chmod +x /usr/bin/yq version=$(yq '.capi-version-min' build_data.yml) echo "version=$version" >> $GITHUB_OUTPUT @@ -80,38 +82,36 @@ jobs: repository: cloudfoundry/cf-deployment-concourse-tasks path: cf-deployment-concourse-tasks - - id: claim-toolsmiths-env - name: Claim Toolsmiths Environment + - id: claim-env + name: Claim Environment env: - api_token: ${{ secrets.TOOLSMITHS_API_TOKEN }} - hostname: ${{ vars.TOOLSMITHS_HOSTNAME }} - notes: "GHA CF CLI Integration Tests ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - pool_name: ${{ vars.TOOLSMITHS_POOL_NAME }} + account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }} + pool_name: ${{ vars.SHEPHERD_POOL_NAME }} run: | - while true; do - curl -s --show-error -D headers.txt -H 'Accept: application/json' \ - -X POST "https://${hostname}/pooled_gcp_engineering_environments/claim" \ - --data-urlencode "api_token=${api_token}" \ - --data-urlencode "pool_name=${pool_name}" \ - --data-urlencode "notes=${notes}" > metadata.json \ - || echo "Unable to reach server, trying again in 30 seconds..." - - cat headers.txt - - ERR_500="Sorry, the Toolsmiths Environments app is currently encountering issues. Trying again in 30 seconds..." - ERR_429="Sorry, Toolsmiths are out of environments in your requested pool. New environments are on their way but you can stop by the Toolsmiths slack channel for more help." - ERR_409="Sorry, was not able to claim an environment. Trying again in 30 seconds..." - - grep -q -E "HTTP/[[:digit:]\.]{1,3} 401" headers.txt && exit 1 - grep -q -E "HTTP/[[:digit:]\.]{1,3} 404" headers.txt && exit 2 - grep -q -E "HTTP/[[:digit:]\.]{1,3} 500" headers.txt && echo "$ERR_500" - grep -q -E "HTTP/[[:digit:]\.]{1,3} 200" headers.txt && break - grep -q -E "HTTP/[[:digit:]\.]{1,3} 429" && echo "$ERR_429" - grep -q -E "HTTP/[[:digit:]\.]{1,3} 409" && echo "$ERR_409" + shepherd login service-account ${account_token} + lease_id=$(shepherd create lease --duration 8h --pool ${pool_name} --pool-namespace official --namespace tas-devex --json | jq -r .id) + # Give somtime for the lease to complete. Shepherd may take upto an 3 hours to create an env + # if the pool is empty. + count=0 + while [ $count -lt 360 ] ; do sleep 30 + status=$(shepherd get lease ${lease_id} --namespace tas-devex --json | jq -r .status) + if [ $status == "LEASED" ] ; then + shepherd get lease ${lease_id} --namespace tas-devex --json | jq .output > metadata.json + break + elif [ $status == "FAILED" -o $status == "EXPIRED" ] ; then + echo "There was an error obtaining the lease. Lease status is ${status}." + exit 1 + else + echo "Waiting for environment to be ready. Lease status is ${status}." + fi + count=$(($count+1)) done + + env_name=$(jq -r .name metadata.json) cat metadata.json | jq -r '.name' + echo "lease-id=$lease_id" >> "${GITHUB_OUTPUT}" - name: Install Tools run: | @@ -141,11 +141,16 @@ jobs: chmod +x $install_location/credhub credhub --version + apt-get update + apt-get install -y build-essential + - name: Deploy Isolation Segment and OIDC Provider if: ${{ inputs.capi-version == 'edge' }} env: CF_INT_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} run: | + env_name=$(jq -r .name metadata.json) + jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${env_name}.priv eval "$(bbl print-env --metadata-file metadata.json)" # deploy @@ -182,6 +187,8 @@ jobs: cp cli-ci/ci/infrastructure/operations/use-latest-ruby-buildpack.yml ops-files/ # Deletes CF-D + env_name=$(jq -r .name metadata.json) + jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${env_name}.priv eval "$(bbl print-env --metadata-file metadata.json)" bosh -d cf delete-deployment -n @@ -225,11 +232,13 @@ jobs: if: ${{ !inputs.run-with-client-creds }} run: | ENV=$(cat metadata.json | jq -r '.name') + jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${ENV}.priv eval "$(bbl print-env --metadata-file ./metadata.json)" export CF_INT_PASSWORD="$(credhub get -n /bosh-$ENV/cf/cf_admin_password | bosh interpolate --path /value -)" export CF_INT_OIDC_USERNAME="admin-oidc" export CF_INT_OIDC_PASSWORD=$(credhub get -n /bosh-$ENV/cf/uaa_oidc_admin_password | bosh interpolate --path /value -) - export CF_INT_API="https://api.${ENV}.cf-app.com" + API_URL="$(jq -r .cf.api_url metadata.json)" + export CF_INT_API="https://$API_URL" export CF_DIAL_TIMEOUT=15 export CF_USERNAME=admin export FLAKE_ATTEMPTS=2 @@ -255,11 +264,13 @@ jobs: CF_INT_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} run: | ENV=$(cat metadata.json | jq -r '.name') + jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${ENV}.priv eval "$(bbl print-env --metadata-file ./metadata.json)" export CF_INT_PASSWORD="$(credhub get -n /bosh-$ENV/cf/cf_admin_password | bosh interpolate --path /value -)" export CF_INT_OIDC_USERNAME="admin-oidc" export CF_INT_OIDC_PASSWORD=$(credhub get -n /bosh-$ENV/cf/uaa_oidc_admin_password | bosh interpolate --path /value -) - export CF_INT_API="https://api.${ENV}.cf-app.com" + API_URL="$(jq -r .cf.api_url metadata.json)" + export CF_INT_API="https://$API_URL" export CF_DIAL_TIMEOUT=15 export CF_USERNAME=admin export FLAKE_ATTEMPTS=2 @@ -278,29 +289,15 @@ jobs: make integration-tests-full-ci - - name: Unclaim environment if: always() env: - api_token: ${{ secrets.TOOLSMITHS_API_TOKEN }} - hostname: ${{ vars.TOOLSMITHS_HOSTNAME }} + account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }} run: | - ENV=$(cat metadata.json | jq -r '.name') - while true; do - output=$(curl -s --show-error -D headers.txt -H 'Accept: application/json' \ - -X POST "https://${hostname}/pooled_gcp_engineering_environments/unclaim" \ - --data-urlencode "api_token=${api_token}" \ - --data-urlencode "name=${ENV}") - - cat headers.txt - - ERR_500="Sorry, the Toolsmiths Environments app is currently encountering issues. Trying again in 30 seconds..." - - grep -q -E "HTTP/[[:digit:]\.]{1,3} 500" headers.txt && echo "$ERR_500" && sleep 30 && continue - grep -q -E "HTTP/[[:digit:]\.]{1,3} 401" headers.txt && echo $(echo "$output" | jq '.messages | join(", ")') && exit 1 - grep -q -E "HTTP/[[:digit:]\.]{1,3} 404" headers.txt && echo $(echo "$output" | jq '.messages | join(", ")') && exit 2 - grep -q -E "HTTP/[[:digit:]\.]{1,3} 202" headers.txt && break - done + shepherd login service-account ${account_token} + set -x + shepherd delete lease ${{ steps.claim-env.outputs.lease-id }} --namespace tas-devex + - uses: LouisBrunner/checks-action@v2.0.0 if: always() with: diff --git a/.github/workflows/tests-integration.yml b/.github/workflows/tests-integration.yml index 270e87cd38a..b49c93963c7 100644 --- a/.github/workflows/tests-integration.yml +++ b/.github/workflows/tests-integration.yml @@ -33,16 +33,16 @@ jobs: name: Integration client creds secrets: inherit - run-integration-tests-cf-env-with-min-capi: - name: MIN CAPI - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} - uses: ./.github/workflows/tests-integration-reusable.yml - with: - capi-version: min - run-with-client-creds: false - os: ubuntu-latest - name: Integration MIN CAPI - secrets: inherit + # run-integration-tests-cf-env-with-min-capi: + # name: MIN CAPI + # # if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + # uses: ./.github/workflows/tests-integration-reusable.yml + # with: + # capi-version: min + # run-with-client-creds: false + # os: ubuntu-latest + # name: Integration MIN CAPI + # secrets: inherit #run-integration-windows: # name: Windows From 5b3eee4ac9ee9f613b44a58ccf3efd10b96ff884 Mon Sep 17 00:00:00 2001 From: Shwetha Gururaj Date: Fri, 26 Apr 2024 14:12:37 -0400 Subject: [PATCH 10/12] Update go sum --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 2cc8b5a3887..c00c7e28b75 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.20.0/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= -github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= -github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= +github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= +github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/openzipkin/zipkin-go v0.4.2 h1:zjqfqHjUpPmB3c1GlCvvgsM1G4LkvqQbBDueDOCg/jA= From 6acf8f0b58e8445370a03ae6aa790d6d22954ae4 Mon Sep 17 00:00:00 2001 From: Shwetha Gururaj Date: Fri, 26 Apr 2024 15:34:39 -0400 Subject: [PATCH 11/12] Test v8 --- .../workflows/tests-integration-reusable.yml | 38 +++++++++---------- .github/workflows/tests-integration.yml | 14 +++---- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/tests-integration-reusable.yml b/.github/workflows/tests-integration-reusable.yml index 67aa8e95c86..16543571f8d 100644 --- a/.github/workflows/tests-integration-reusable.yml +++ b/.github/workflows/tests-integration-reusable.yml @@ -31,16 +31,16 @@ jobs: runs-on: ${{ inputs.os }} container: us-west2-docker.pkg.dev/shepherd-268822/shepherd2/concourse-resource:latest steps: - - uses: LouisBrunner/checks-action@v2.0.0 - if: always() - id: check - with: - token: ${{ secrets.GITHUB_TOKEN }} - name: "${{ inputs.name }}" - status: in_progress - sha: ${{github.event.workflow_run.head_sha}} - output: | - {"title": "${{ inputs.name }}", "summary":"started ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"} + # - uses: LouisBrunner/checks-action@v2.0.0 + # if: always() + # id: check + # with: + # token: ${{ secrets.GITHUB_TOKEN }} + # name: "${{ inputs.name }}" + # status: in_progress + # sha: ${{github.event.workflow_run.head_sha}} + # output: | + # {"title": "${{ inputs.name }}", "summary":"started ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"} - name: Checkout cli uses: actions/checkout@v4 @@ -298,12 +298,12 @@ jobs: set -x shepherd delete lease ${{ steps.claim-env.outputs.lease-id }} --namespace tas-devex - - uses: LouisBrunner/checks-action@v2.0.0 - if: always() - with: - token: ${{ secrets.GITHUB_TOKEN }} - check_id: ${{ steps.check.outputs.check_id }} - conclusion: ${{ job.status }} - sha: ${{github.event.workflow_run.head_sha}} - output: | - {"title": "${{ inputs.name }}", "summary":"finished ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"} + # - uses: LouisBrunner/checks-action@v2.0.0 + # if: always() + # with: + # token: ${{ secrets.GITHUB_TOKEN }} + # check_id: ${{ steps.check.outputs.check_id }} + # conclusion: ${{ job.status }} + # sha: ${{github.event.workflow_run.head_sha}} + # output: | + # {"title": "${{ inputs.name }}", "summary":"finished ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"} diff --git a/.github/workflows/tests-integration.yml b/.github/workflows/tests-integration.yml index b49c93963c7..9355edeaccc 100644 --- a/.github/workflows/tests-integration.yml +++ b/.github/workflows/tests-integration.yml @@ -4,16 +4,16 @@ run-name: "Integration [${{ github.event.workflow_run.head_branch }}]: ${{ githu on: workflow_dispatch: - workflow_run: - workflows: - - "Tests" - types: - - completed + # workflow_run: + # workflows: + # - "Tests" + # types: + # - completed jobs: run-integration-tests-cf-env: name: Integration tests - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + # if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} uses: ./.github/workflows/tests-integration-reusable.yml with: capi-version: edge @@ -24,7 +24,7 @@ jobs: run-integration-tests-cf-env-with-client-creds: name: client creds - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + # if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} uses: ./.github/workflows/tests-integration-reusable.yml with: capi-version: edge From 45c6f642c4801b230225f85687f444f3c0bd3189 Mon Sep 17 00:00:00 2001 From: Shwetha Gururaj Date: Mon, 29 Apr 2024 09:56:25 -0400 Subject: [PATCH 12/12] Disable manual trigger --- .../workflows/tests-integration-reusable.yml | 38 +++++++++---------- .github/workflows/tests-integration.yml | 14 +++---- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/tests-integration-reusable.yml b/.github/workflows/tests-integration-reusable.yml index 16543571f8d..67aa8e95c86 100644 --- a/.github/workflows/tests-integration-reusable.yml +++ b/.github/workflows/tests-integration-reusable.yml @@ -31,16 +31,16 @@ jobs: runs-on: ${{ inputs.os }} container: us-west2-docker.pkg.dev/shepherd-268822/shepherd2/concourse-resource:latest steps: - # - uses: LouisBrunner/checks-action@v2.0.0 - # if: always() - # id: check - # with: - # token: ${{ secrets.GITHUB_TOKEN }} - # name: "${{ inputs.name }}" - # status: in_progress - # sha: ${{github.event.workflow_run.head_sha}} - # output: | - # {"title": "${{ inputs.name }}", "summary":"started ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"} + - uses: LouisBrunner/checks-action@v2.0.0 + if: always() + id: check + with: + token: ${{ secrets.GITHUB_TOKEN }} + name: "${{ inputs.name }}" + status: in_progress + sha: ${{github.event.workflow_run.head_sha}} + output: | + {"title": "${{ inputs.name }}", "summary":"started ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"} - name: Checkout cli uses: actions/checkout@v4 @@ -298,12 +298,12 @@ jobs: set -x shepherd delete lease ${{ steps.claim-env.outputs.lease-id }} --namespace tas-devex - # - uses: LouisBrunner/checks-action@v2.0.0 - # if: always() - # with: - # token: ${{ secrets.GITHUB_TOKEN }} - # check_id: ${{ steps.check.outputs.check_id }} - # conclusion: ${{ job.status }} - # sha: ${{github.event.workflow_run.head_sha}} - # output: | - # {"title": "${{ inputs.name }}", "summary":"finished ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"} + - uses: LouisBrunner/checks-action@v2.0.0 + if: always() + with: + token: ${{ secrets.GITHUB_TOKEN }} + check_id: ${{ steps.check.outputs.check_id }} + conclusion: ${{ job.status }} + sha: ${{github.event.workflow_run.head_sha}} + output: | + {"title": "${{ inputs.name }}", "summary":"finished ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"} diff --git a/.github/workflows/tests-integration.yml b/.github/workflows/tests-integration.yml index 9355edeaccc..b49c93963c7 100644 --- a/.github/workflows/tests-integration.yml +++ b/.github/workflows/tests-integration.yml @@ -4,16 +4,16 @@ run-name: "Integration [${{ github.event.workflow_run.head_branch }}]: ${{ githu on: workflow_dispatch: - # workflow_run: - # workflows: - # - "Tests" - # types: - # - completed + workflow_run: + workflows: + - "Tests" + types: + - completed jobs: run-integration-tests-cf-env: name: Integration tests - # if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} uses: ./.github/workflows/tests-integration-reusable.yml with: capi-version: edge @@ -24,7 +24,7 @@ jobs: run-integration-tests-cf-env-with-client-creds: name: client creds - # if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} uses: ./.github/workflows/tests-integration-reusable.yml with: capi-version: edge