diff --git a/.codecov.yml b/.codecov.yml index 9b9f57ef04..2758a60933 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -8,13 +8,16 @@ coverage: default: threshold: 0.1 ignore: - - "pkg/apis/rollouts/v1alpha1" - - "test" - - "**/*.pb.go" - - "**/*.pb.gw.go" - - "**/*generated.go" - - "**/*generated.deepcopy.go" - - "**/*_test.go" - - "pkg/apis/client/.*" - - "pkg/client/.*" - - "vendor/.*" + - 'pkg/apis/rollouts/v1alpha1' + - 'test' + - '**/*.pb.go' + - '**/*.pb.gw.go' + - '**/*generated.go' + - '**/*generated.deepcopy.go' + - '**/*_test.go' + - 'pkg/apis/client/.*' + - 'pkg/client/.*' + - 'vendor/.*' + - '**/mocks/*' + - 'hack/gen-crd-spec/main.go' + - 'hack/gen-docs/main.go' diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 33da74335a..384a49caf5 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -2,7 +2,7 @@ Checklist: * [ ] Either (a) I've created an [enhancement proposal](https://github.com/argoproj/argo-rollouts/issues/new/choose) and discussed it with the community, (b) this is a bug fix, or (c) this is a chore. * [ ] The title of the PR is (a) [conventional](https://www.conventionalcommits.org/en/v1.0.0/) with a list of types and scopes found [here](https://github.com/argoproj/argo-rollouts/blob/master/.github/workflows/pr-title-check.yml), (b) states what changed, and (c) suffixes the related issues number. E.g. `"fix(controller): Updates such and such. Fixes #1234"`. -* [ ] I've signed my commits with [DCO](https://github.com/argoproj/argoproj) +* [ ] I've signed my commits with [DCO](https://github.com/argoproj/argoproj/blob/main/community/CONTRIBUTING.md#legal) * [ ] I have written unit and/or e2e tests for my change. PRs without these are unlikely to be merged. * [ ] My builds are green. Try syncing with master if they are not. -* [ ] My organization is added to [USERS.md](https://github.com/argoproj/argo-rollouts/blob/master/USERS.md). \ No newline at end of file +* [ ] My organization is added to [USERS.md](https://github.com/argoproj/argo-rollouts/blob/master/USERS.md). diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 326772bb06..c3c4e138e2 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -23,7 +23,7 @@ jobs: ./git-chglog --sort semver -o CHANGELOG.md v1.3.1.. rm git-chglog - name: Create Pull Request - uses: peter-evans/create-pull-request@v6 + uses: peter-evans/create-pull-request@v7 with: commit-message: update changelog title: "docs: Update Changelog" diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 04fd8f9efc..2ce05c728e 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -61,13 +61,13 @@ jobs: needs: [set-vars] permissions: contents: read - packages: write # for pushing packages to GHCR, which is used by cd.apps.argoproj.io to avoid polluting Quay with tags + packages: write # for pushing packages to GHCR, which is used by cd.apps.argoproj.io to avoid polluting Quay with tags id-token: write # for creating OIDC tokens for signing. uses: ./.github/workflows/image-reuse.yaml with: quay_image_name: ${{ needs.set-vars.outputs.controller-meta-tags }} # Note: cannot use env variables to set go-version (https://docs.github.com/en/actions/using-workflows/reusing-workflows#limitations) - go-version: '1.21' + go-version: '1.23' platforms: ${{ needs.set-vars.outputs.platforms }} push: ${{ github.event_name != 'pull_request' }} secrets: @@ -78,13 +78,13 @@ jobs: needs: [set-vars] permissions: contents: read - packages: write # for pushing packages to GHCR, which is used by cd.apps.argoproj.io to avoid polluting Quay with tags + packages: write # for pushing packages to GHCR, which is used by cd.apps.argoproj.io to avoid polluting Quay with tags id-token: write # for creating OIDC tokens for signing. uses: ./.github/workflows/image-reuse.yaml with: quay_image_name: ${{ needs.set-vars.outputs.plugin-meta-tags }} # Note: cannot use env variables to set go-version (https://docs.github.com/en/actions/using-workflows/reusing-workflows#limitations) - go-version: '1.21' + go-version: '1.23' platforms: ${{ needs.set-vars.outputs.platforms }} push: ${{ github.event_name != 'pull_request' }} target: kubectl-argo-rollouts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml deleted file mode 100644 index 36e4a26aa4..0000000000 --- a/.github/workflows/e2e.yaml +++ /dev/null @@ -1,100 +0,0 @@ -name: E2E Tests - -on: - push: - branches: - - 'master' - - 'release-*' - pull_request: - branches: - - 'master' - - 'release-*' - workflow_dispatch: - inputs: - debug_enabled: - description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' - required: false - default: false - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - event_file: - name: "Event File" - runs-on: ubuntu-latest - steps: - - name: Upload - uses: actions/upload-artifact@v4 - with: - name: Event File - path: ${{ github.event_path }} - test-e2e: - strategy: - fail-fast: false - matrix: - kubernetes-minor-version: - - 1.24 - - 1.25 - - 1.26 - - 1.27 - - 1.28 - - 1.29 - name: Run end-to-end tests - runs-on: ubuntu-latest - steps: - - name: Set up Go - uses: actions/setup-go@v5.0.0 - with: - go-version: '1.21' - - uses: actions/checkout@v4 - - name: Setup k3s - env: - INSTALL_K3S_CHANNEL: v${{ matrix.kubernetes-minor-version }} - run: | - curl -sfL https://get.k3s.io | sh - - sudo mkdir ~/.kube - sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config - sudo chmod 755 ~/.kube/config - kubectl version - kubectl create ns argo-rollouts - - uses: actions/cache@v4 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - name: Download cache - run: go mod download - - name: Install CRDs - run: | - kubectl apply -k manifests/crds - kubectl apply -f test/e2e/crds - - name: Start controller - run: make start-e2e 2>&1 | sed -r "s/[[:cntrl:]]\[[0-9]{1,3}m//g" > /tmp/e2e-controller.log & - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true'}} - - name: Run e2e tests - run: make test-e2e - if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true') }} - - name: Output Rerun Overview - run: | - [[ -f rerunreport.txt ]] && cat rerunreport.txt || echo "No rerun report found" - - name: Upload E2E Test Results - if: always() - uses: actions/upload-artifact@v4 - with: - name: E2E Test Results (k8s ${{ matrix.kubernetes-minor-version }}) - path: | - junit.xml - - name: Upload e2e-controller logs - uses: actions/upload-artifact@v4 - with: - name: e2e-controller-k8s-${{ matrix.kubernetes-minor-version }}.log - path: /tmp/e2e-controller.log - if: ${{ failure() }} diff --git a/.github/workflows/gh-pages.yaml b/.github/workflows/gh-pages.yaml index ffd8b736d0..831a81a3d3 100644 --- a/.github/workflows/gh-pages.yaml +++ b/.github/workflows/gh-pages.yaml @@ -15,7 +15,7 @@ permissions: jobs: deploy: permissions: - contents: write # for peaceiris/actions-gh-pages to push pages branch + contents: write # for peaceiris/actions-gh-pages to push pages branch runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -24,16 +24,16 @@ jobs: with: python-version: 3.x - name: Set up Go - uses: actions/setup-go@v5.0.0 + uses: actions/setup-go@v5.1.0 with: - go-version: '1.21' + go-version: '1.23' - name: build run: | pip install mkdocs mkdocs_material make docs mkdocs build - name: Deploy - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@v4 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./site diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 65e600d8d3..67b525a662 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -2,14 +2,14 @@ name: Go on: push: branches: - - "master" - - "release-*" + - 'master' + - 'release-*' pull_request: branches: - - "master" + - 'master' env: # Golang version to use across CI steps - GOLANG_VERSION: '1.21' + GOLANG_VERSION: '1.23' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -19,39 +19,30 @@ permissions: contents: read jobs: - event_file: - name: "Event File" - runs-on: ubuntu-latest - steps: - - name: Upload - uses: actions/upload-artifact@v4 - with: - name: Event File - path: ${{ github.event_path }} lint-go: permissions: - contents: read # for actions/checkout to fetch code - pull-requests: read # for golangci/golangci-lint-action to fetch pull requests + contents: read # for actions/checkout to fetch code + pull-requests: read # for golangci/golangci-lint-action to fetch pull requests name: Lint Go code runs-on: ubuntu-latest steps: - name: Set up Go - uses: actions/setup-go@v5.0.0 + uses: actions/setup-go@v5.1.0 with: go-version: ${{ env.GOLANG_VERSION }} - name: Checkout code uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v4 + uses: golangci/golangci-lint-action@v6 with: - version: v1.57.2 + version: v1.61.0 args: --timeout 6m build: name: Build runs-on: ubuntu-latest steps: - name: Set up Go - uses: actions/setup-go@v5.0.0 + uses: actions/setup-go@v5.1.0 with: go-version: ${{ env.GOLANG_VERSION }} id: go @@ -72,28 +63,6 @@ jobs: - name: Compile all packages run: make controller plugin - - name: Test - run: make test-unit - - - name: Upload Unit Test Results - if: always() - uses: actions/upload-artifact@v4 - with: - name: Unit Test Results - path: | - junit.xml - - - name: Generate code coverage artifacts - uses: actions/upload-artifact@v4 - with: - name: code-coverage - path: coverage.out - - - name: Upload code coverage information to codecov.io - uses: codecov/codecov-action@v4.1.1 - with: - file: coverage.out - codegen: name: Verify Codegen runs-on: ubuntu-latest @@ -103,7 +72,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Setup Golang - uses: actions/setup-go@v5.0.0 + uses: actions/setup-go@v5.1.0 with: go-version: ${{ env.GOLANG_VERSION }} # k8s codegen generates files into GOPATH location instead of the GitHub git checkout location diff --git a/.github/workflows/image-reuse.yaml b/.github/workflows/image-reuse.yaml index 970080e845..da828a7e39 100644 --- a/.github/workflows/image-reuse.yaml +++ b/.github/workflows/image-reuse.yaml @@ -69,17 +69,17 @@ jobs: if: ${{ github.ref_type != 'tag'}} - name: Setup Golang - uses: actions/setup-go@v5.0.0 # v3.5.0 + uses: actions/setup-go@v5.1.0 # v3.5.0 with: go-version: ${{ inputs.go-version }} - name: Install cosign - uses: sigstore/cosign-installer@e1523de7571e31dbe865fd2e80c5c7c23ae71eb4 # v3.4.0 + uses: sigstore/cosign-installer@dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da # v3.7.0 with: cosign-release: 'v2.2.0' - - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 - - uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0 + - uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 + - uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 - name: Setup tags for container image as a CSV type run: | @@ -106,7 +106,7 @@ jobs: echo 'EOF' >> $GITHUB_ENV - name: Login to Quay.io - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 with: registry: quay.io username: ${{ secrets.quay_username }} @@ -114,7 +114,7 @@ jobs: if: ${{ inputs.quay_image_name && inputs.push }} - name: Login to GitHub Container Registry - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 with: registry: ghcr.io username: ${{ secrets.ghcr_username }} @@ -122,7 +122,7 @@ jobs: if: ${{ inputs.ghcr_image_name && inputs.push }} - name: Login to dockerhub Container Registry - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 with: username: ${{ secrets.docker_username }} password: ${{ secrets.docker_password }} @@ -130,7 +130,7 @@ jobs: - name: Build and push container image id: image - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 #v5.3.0 + uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 #v6.10.0 with: context: . platforms: ${{ inputs.platforms }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a8c0781fbe..99b08b01fd 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -7,7 +7,7 @@ on: permissions: {} env: - GOLANG_VERSION: '1.21' # Note: go-version must also be set in job controller-image.with.go-version & plugin-image.with.go-version. + GOLANG_VERSION: '1.23' # Note: go-version must also be set in job controller-image.with.go-version & plugin-image.with.go-version. jobs: controller-image: @@ -19,7 +19,7 @@ jobs: with: quay_image_name: quay.io/argoproj/argo-rollouts:${{ github.ref_name }} # Note: cannot use env variables to set go-version (https://docs.github.com/en/actions/using-workflows/reusing-workflows#limitations) - go-version: '1.21' + go-version: '1.23' platforms: linux/amd64,linux/arm64 push: true secrets: @@ -35,7 +35,7 @@ jobs: with: quay_image_name: quay.io/argoproj/kubectl-argo-rollouts:${{ github.ref_name }} # Note: cannot use env variables to set go-version (https://docs.github.com/en/actions/using-workflows/reusing-workflows#limitations) - go-version: '1.21' + go-version: '1.23' platforms: linux/amd64,linux/arm64 push: true target: kubectl-argo-rollouts @@ -44,41 +44,40 @@ jobs: quay_password: ${{ secrets.QUAY_ROBOT_TOKEN }} controller-image-provenance: - needs: - - controller-image - permissions: - actions: read # for detecting the Github Actions environment. - id-token: write # for creating OIDC tokens for signing. - packages: write # for uploading attestations. (https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/container/README.md#known-issues) - # Must be refernced by a tag. https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/container/README.md#referencing-the-slsa-generator - uses: slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v1.10.0 - with: - image: quay.io/argoproj/argo-rollouts - digest: ${{ needs.controller-image.outputs.image-digest }} - secrets: - registry-username: ${{ secrets.QUAY_USERNAME }} - registry-password: ${{ secrets.QUAY_ROBOT_TOKEN }} + needs: + - controller-image + permissions: + actions: read # for detecting the Github Actions environment. + id-token: write # for creating OIDC tokens for signing. + packages: write # for uploading attestations. (https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/container/README.md#known-issues) + # Must be refernced by a tag. https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/container/README.md#referencing-the-slsa-generator + uses: slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v2.0.0 + with: + image: quay.io/argoproj/argo-rollouts + digest: ${{ needs.controller-image.outputs.image-digest }} + secrets: + registry-username: ${{ secrets.QUAY_USERNAME }} + registry-password: ${{ secrets.QUAY_ROBOT_TOKEN }} plugin-image-provenance: - needs: - - plugin-image - permissions: - actions: read # for detecting the Github Actions environment. - id-token: write # for creating OIDC tokens for signing. - packages: write # for uploading attestations. (https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/container/README.md#known-issues) - # Must be refernced by a tag. https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/container/README.md#referencing-the-slsa-generator - uses: slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v1.10.0 - with: - image: quay.io/argoproj/kubectl-argo-rollouts - digest: ${{ needs.plugin-image.outputs.image-digest }} - secrets: - registry-username: ${{ secrets.QUAY_USERNAME }} - registry-password: ${{ secrets.QUAY_ROBOT_TOKEN }} - + needs: + - plugin-image + permissions: + actions: read # for detecting the Github Actions environment. + id-token: write # for creating OIDC tokens for signing. + packages: write # for uploading attestations. (https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/container/README.md#known-issues) + # Must be refernced by a tag. https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/container/README.md#referencing-the-slsa-generator + uses: slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v2.0.0 + with: + image: quay.io/argoproj/kubectl-argo-rollouts + digest: ${{ needs.plugin-image.outputs.image-digest }} + secrets: + registry-username: ${{ secrets.QUAY_USERNAME }} + registry-password: ${{ secrets.QUAY_ROBOT_TOKEN }} release-artifacts: permissions: - contents: write # for softprops/action-gh-release to create GitHub release + contents: write # for softprops/action-gh-release to create GitHub release runs-on: ubuntu-latest outputs: hashes: ${{ steps.hash.outputs.hashes }} @@ -91,15 +90,15 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Golang - uses: actions/setup-go@v5.0.0 # v4.0.1 + uses: actions/setup-go@v5.1.0 # v4.0.1 with: go-version: ${{ env.GOLANG_VERSION }} - name: Set up QEMU - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0 + uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 - name: Generate release artifacts run: | @@ -108,7 +107,7 @@ jobs: make manifests IMAGE_TAG=${{ github.ref_name }} - name: Draft release - uses: softprops/action-gh-release@9d7c94cfd0a1f3ed45544c887983e9fa900f0564 # v0.1.15 + uses: softprops/action-gh-release@01570a1f39cb168c169c802c3bceb9e93fb10974 # v0.1.15 with: tag_name: ${{ github.event.inputs.tag }} draft: true @@ -132,7 +131,6 @@ jobs: run: | echo "hashes=$(sha256sum ./dist/kubectl-argo-rollouts-* ./manifests/*.yaml | base64 -w0)" >> "$GITHUB_OUTPUT" - release-artifacts-provenance: needs: - release-artifacts @@ -141,10 +139,10 @@ jobs: id-token: write # Needed for provenance signing and ID contents: write # Needed for release uploads # Must be refernced by a tag. https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/container/README.md#referencing-the-slsa-generator - uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v1.10.0 + uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.0.0 with: - base64-subjects: "${{ needs.release-artifacts.outputs.hashes }}" - provenance-name: "argo-rollouts.intoto.jsonl" + base64-subjects: '${{ needs.release-artifacts.outputs.hashes }}' + provenance-name: 'argo-rollouts.intoto.jsonl' upload-assets: true draft-release: true @@ -165,12 +163,12 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Golang - uses: actions/setup-go@v5.0.0 # v4.0.0 + uses: actions/setup-go@v5.1.0 # v4.0.0 with: go-version: ${{ env.GOLANG_VERSION }} - name: Install cosign - uses: sigstore/cosign-installer@e1523de7571e31dbe865fd2e80c5c7c23ae71eb4 # v3.4.0 + uses: sigstore/cosign-installer@dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da # v3.7.0 with: cosign-release: 'v2.2.0' @@ -183,7 +181,7 @@ jobs: SIGS_BOM_VERSION: v0.2.1 # comma delimited list of project relative folders to inspect for package # managers (gomod, yarn, npm). - PROJECT_FOLDERS: ".,./ui" + PROJECT_FOLDERS: '.,./ui' # full qualified name of the container image to be inspected CONTAINER_IMAGE: quay.io/argoproj/argo-rollouts:${{ github.event.inputs.tag }} @@ -214,7 +212,7 @@ jobs: /tmp/sbom.tar.gz - name: Upload SBOM and signature assets - uses: softprops/action-gh-release@9d7c94cfd0a1f3ed45544c887983e9fa900f0564 # v0.1.15 + uses: softprops/action-gh-release@01570a1f39cb168c169c802c3bceb9e93fb10974 # v0.1.15 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/e2e-test-results.yml b/.github/workflows/testing-results.yml similarity index 50% rename from .github/workflows/e2e-test-results.yml rename to .github/workflows/testing-results.yml index 0846ccd92b..f91ae48aa7 100644 --- a/.github/workflows/e2e-test-results.yml +++ b/.github/workflows/testing-results.yml @@ -1,9 +1,9 @@ # use separate workflow to support fork repositories and dependabot branches when publishing test results: see https://github.com/EnricoMi/publish-unit-test-result-action#support-fork-repositories-and-dependabot-branches -name: Test Results +name: Testing Results on: workflow_run: - workflows: ["E2E Tests", "Go"] + workflows: ["Testing"] types: - completed permissions: {} @@ -19,27 +19,31 @@ jobs: actions: read steps: - name: Download and Extract Artifacts - # TODO repace with native actions/download-artifact once it supports downloading from another workflow: https://github.com/actions/download-artifact/issues/3 - env: - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - run: | - mkdir -p artifacts && cd artifacts - artifacts_url=${{ github.event.workflow_run.artifacts_url }} - gh api "$artifacts_url" -q '.artifacts[] | [.name, .archive_download_url] | @tsv' | while read artifact - do - IFS=$'\t' read name url <<< "$artifact" - gh api $url > "$name.zip" - unzip -d "$name" "$name.zip" - done + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + run-id: ${{ github.event.workflow_run.id }} + path: artifacts + github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Publish Test Results + - name: Publish E2E Test Results uses: EnricoMi/publish-unit-test-result-action@v2 with: - check_name: "${{ github.event.workflow.name }} Published Test Results" + check_name: "Published E2E Test Results" + compare_to_earlier_commit: false + test_changes_limit: 0 + fail_on: "errors" commit: ${{ github.event.workflow_run.head_sha }} event_file: artifacts/Event File/event.json event_name: ${{ github.event.workflow_run.event }} - junit_files: "artifacts/**/junit.xml" + files: "artifacts/**/junit-e2e-test.xml" + - name: Publish Unit Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + check_name: "Published Unit Test Results" compare_to_earlier_commit: false test_changes_limit: 0 fail_on: "errors" + commit: ${{ github.event.workflow_run.head_sha }} + event_file: artifacts/Event File/event.json + event_name: ${{ github.event.workflow_run.event }} + files: "artifacts/**/junit-unit-test.xml" diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml new file mode 100644 index 0000000000..db09896ef8 --- /dev/null +++ b/.github/workflows/testing.yaml @@ -0,0 +1,195 @@ +name: Testing + +on: + push: + branches: + - 'master' + - 'release-*' + pull_request: + branches: + - 'master' + - 'release-*' + workflow_dispatch: + inputs: + debug_enabled: + description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' + required: false + default: false +env: + # Golang version to use across CI steps + GOLANG_VERSION: '1.23' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + event_file: + name: 'Event File' + runs-on: ubuntu-latest + steps: + - name: Upload + uses: actions/upload-artifact@v4 + with: + name: Event File + path: ${{ github.event_path }} + test-unit: + name: Run unit tests + runs-on: ubuntu-latest + steps: + - name: Set up Go + uses: actions/setup-go@v5.1.0 + with: + go-version: ${{ env.GOLANG_VERSION }} + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + + - name: Restore go build cache + uses: actions/cache@v4 + with: + path: ~/.cache/go-build + key: ${{ runner.os }}-go-build-v1-${{ github.run_id }} + + - name: Download all Go modules + run: | + go mod download + - name: Test + run: make test-unit + + - name: Upload Unit Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: Unit Test Results + path: | + junit-unit-test.xml + - name: Generate code coverage artifacts + uses: actions/upload-artifact@v4 + with: + name: coverage-output-unit + path: coverage-output-unit + + test-e2e: + strategy: + fail-fast: false + matrix: + kubernetes: + - version: '1.28' + latest: false + - version: '1.29' + latest: false + - version: '1.30' + latest: false + - version: '1.31' + latest: true + name: Run end-to-end tests + runs-on: ubuntu-latest + steps: + - name: Set up Go + uses: actions/setup-go@v5.1.0 + with: + go-version: '1.23' + - uses: actions/checkout@v4 + - name: Setup k3s + env: + INSTALL_K3S_CHANNEL: v${{ matrix.kubernetes.version }} + run: | + curl -sfL https://get.k3s.io | sh - + sudo mkdir ~/.kube + sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config + sudo chmod 755 ~/.kube/config + kubectl version + kubectl create ns argo-rollouts + - uses: actions/cache@v4 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Download cache + run: go mod download + - name: Install CRDs + run: | + kubectl apply -k manifests/crds + kubectl apply -f test/e2e/crds + - name: Start controller + run: make start-e2e 2>&1 | sed -r "s/[[:cntrl:]]\[[0-9]{1,3}m//g" > /tmp/e2e-controller.log & + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true'}} + - name: Run e2e tests + run: | + make test-e2e + if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true') }} + - name: Stop e2e tests controller + run: | + pgrep -f go-build -a + pkill -f go-build + sleep 5 + echo "done stopping process" + ls -lah coverage-output-e2e/ + if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true' && matrix.kubernetes.latest)}} + - name: Output Rerun Overview + run: | + [[ -f rerunreport.txt ]] && cat rerunreport.txt || echo "No rerun report found" + - name: Upload E2E Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: E2E Test Results (k8s ${{ matrix.kubernetes.version }}) + path: | + junit-e2e-test.xml + - name: Upload e2e-controller logs + uses: actions/upload-artifact@v4 + with: + name: e2e-controller-k8s-${{ matrix.kubernetes.version }}.log + path: /tmp/e2e-controller.log + if: ${{ always() }} + - name: Upload code coverage artifacts + uses: actions/upload-artifact@v4 + with: + name: coverage-output-e2e + path: coverage-output-e2e + if: ${{ matrix.kubernetes.latest }} + + coverage-process: + name: Process Coverage Files + runs-on: ubuntu-latest + needs: + - test-unit + - test-e2e + steps: + - name: Set up Go + uses: actions/setup-go@v5.1.0 + with: + go-version: ${{ env.GOLANG_VERSION }} + id: go + - uses: actions/checkout@v4 + - name: Get e2e code coverage + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: coverage-output-e2e + path: coverage-output-e2e + - name: Get unit test code coverage + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: coverage-output-unit + path: coverage-output-unit + - name: combine-go-coverage + run: | + go tool covdata textfmt -i=coverage-output-unit/,coverage-output-e2e/ -o full-coverage.out + - name: Upload code coverage information to codecov.io + uses: codecov/codecov-action@7f8b4b4bde536c465e797be725718b88c5d95e0e # v5.1.1 + with: + file: full-coverage.out + fail_ci_if_error: false + codecov_yml_path: .codecov.yml + disable_search: true + verbose: true + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1a22b8eef6..ce23e23261 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ plugin-bin/ # static server/static/* !server/static/.gitkeep +coverage-output-e2e/ +coverage-output-unit/ +junit-unit-test.xml \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index c59d635e9c..4bd278eb08 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,20 +1,21 @@ run: - deadline: 1m - skip-files: - - ".*\\.pb\\.go" - skip-dirs: - - pkg/client + timeout: 10m modules-download-mode: readonly -linter-settings: +issues: + exclude-dirs: + - pkg/client + exclude-files: + - ".*\\.pb\\.go" +linters-settings: goimports: local-prefixes: github.com/argoproj/argo-rollouts linters: enable: - - vet - gofmt - goimports - - unused + - govet - ineffassign - - unconvert - misspell + - unconvert + - unused disable-all: true diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 7b5d5d427f..aa5927f907 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,6 +1,7 @@ version: 2 formats: all mkdocs: + configuration: mkdocs.yml fail_on_warning: false python: install: diff --git a/CHANGELOG.md b/CHANGELOG.md index c92e20fed3..32f585e4e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,319 @@ + +## [v1.7.2](https://github.com/argoproj/argo-rollouts/compare/v1.7.1...v1.7.2) (2024-08-12) + +### Fix + +* replicaSet not scaled down due to incorrect annotations ([#3762](https://github.com/argoproj/argo-rollouts/issues/3762)) ([#3784](https://github.com/argoproj/argo-rollouts/issues/3784)) +* add update verb to ClusterRole permissions for scaleDown feature. Fixes [#3672](https://github.com/argoproj/argo-rollouts/issues/3672) ([#3675](https://github.com/argoproj/argo-rollouts/issues/3675)) +* **analysis:** explicitly set datadog aggregator to last only on v2 ([#3730](https://github.com/argoproj/argo-rollouts/issues/3730)) +* **analysis:** Take RollbackWindow into account when Reconciling Analysis Runs. Fixes [#3669](https://github.com/argoproj/argo-rollouts/issues/3669) ([#3670](https://github.com/argoproj/argo-rollouts/issues/3670)) +* **controller:** Get the right resourceName for traefik.io.Fixes [#3615](https://github.com/argoproj/argo-rollouts/issues/3615) ([#3759](https://github.com/argoproj/argo-rollouts/issues/3759)) +* **controller:** use the stableRS from the rollout context rather tha… ([#3664](https://github.com/argoproj/argo-rollouts/issues/3664)) +* **dashboard:** Update pod status logic to support native sidecars. Fixes [#3366](https://github.com/argoproj/argo-rollouts/issues/3366) ([#3639](https://github.com/argoproj/argo-rollouts/issues/3639)) +* **metricprovider:** reuse http.Transport for http.Client ([#3780](https://github.com/argoproj/argo-rollouts/issues/3780)) + + + +## [v1.7.1](https://github.com/argoproj/argo-rollouts/compare/v1.7.0...v1.7.1) (2024-06-22) + +### Fix + +* docs site version selector broken ([#3590](https://github.com/argoproj/argo-rollouts/issues/3590)) +* don't default datadog aggregator ([#3643](https://github.com/argoproj/argo-rollouts/issues/3643)) +* Add volume for plugin and tmp folder ([#3546](https://github.com/argoproj/argo-rollouts/issues/3546)) + + + +## [v1.7.0](https://github.com/argoproj/argo-rollouts/compare/v1.7.0-rc1...v1.7.0) (2024-06-12) + +### Fix + +* verify the weight of the alb at the end of the rollout ([#3627](https://github.com/argoproj/argo-rollouts/issues/3627)) +* when Rollout has pingpong and stable/canary service defined, only alb traffic management uses pingpong. ([#3628](https://github.com/argoproj/argo-rollouts/issues/3628)) +* protocol missing in ambassador canary mapping creation. Fixes [#3593](https://github.com/argoproj/argo-rollouts/issues/3593) ([#3603](https://github.com/argoproj/argo-rollouts/issues/3603)) +* rs conflict with fallback to patch ([#3559](https://github.com/argoproj/argo-rollouts/issues/3559)) +* **controller:** Corrects the logic of comparing sha256 has. Fixes [#3519](https://github.com/argoproj/argo-rollouts/issues/3519) ([#3520](https://github.com/argoproj/argo-rollouts/issues/3520)) + + + +## [v1.7.0-rc1](https://github.com/argoproj/argo-rollouts/compare/v1.6.6...v1.7.0-rc1) (2024-04-03) + +### Build + +* **deps:** always resolve momentjs version 2.29.4 ([#3182](https://github.com/argoproj/argo-rollouts/issues/3182)) + +### Chore + +* fix PodSecurity warning ([#3424](https://github.com/argoproj/argo-rollouts/issues/3424)) +* add WeLab Bank to users.md ([#2996](https://github.com/argoproj/argo-rollouts/issues/2996)) +* change file name for readthedocs compatibility ([#2999](https://github.com/argoproj/argo-rollouts/issues/2999)) +* Update users doc with CircleCI ([#3028](https://github.com/argoproj/argo-rollouts/issues/3028)) +* bump k8s versions to 1.29 ([#3494](https://github.com/argoproj/argo-rollouts/issues/3494)) +* updating getCanaryConfigId to be more efficient with better error handling ([#3070](https://github.com/argoproj/argo-rollouts/issues/3070)) +* add missing rollout fields ([#3062](https://github.com/argoproj/argo-rollouts/issues/3062)) +* upgrade cosign ([#3139](https://github.com/argoproj/argo-rollouts/issues/3139)) +* add OpenSSF Scorecard badge ([#3154](https://github.com/argoproj/argo-rollouts/issues/3154)) +* add test for reconcileEphemeralMetadata() ([#3163](https://github.com/argoproj/argo-rollouts/issues/3163)) +* leave the validation of setHeaderRoute to the plugin when plugins is not empty. ([#2898](https://github.com/argoproj/argo-rollouts/issues/2898)) +* fix lint errors reported by golangci-lint ([#3458](https://github.com/argoproj/argo-rollouts/issues/3458)) +* fix unit test data races ([#3478](https://github.com/argoproj/argo-rollouts/issues/3478)) ([#3479](https://github.com/argoproj/argo-rollouts/issues/3479)) +* added organization to users.md ([#3481](https://github.com/argoproj/argo-rollouts/issues/3481)) +* set webpack hashFunction to modern sha256, remove legacy-provider. Fixes [#2609](https://github.com/argoproj/argo-rollouts/issues/2609) ([#3475](https://github.com/argoproj/argo-rollouts/issues/3475)) +* remove year from codegen license ([#3282](https://github.com/argoproj/argo-rollouts/issues/3282)) +* update follow-redirects to 1.15.5 ([#3314](https://github.com/argoproj/argo-rollouts/issues/3314)) +* add logging context around replicaset updates ([#3326](https://github.com/argoproj/argo-rollouts/issues/3326)) +* bump notification engine lib ([#3327](https://github.com/argoproj/argo-rollouts/issues/3327)) +* change controller's deploy strategy to RollingUpdate due to leader election ([#3334](https://github.com/argoproj/argo-rollouts/issues/3334)) +* Add exception to `requireCanaryStableServices` to disable validation when using the `hashicorp/consul` plugin ([#3339](https://github.com/argoproj/argo-rollouts/issues/3339)) +* Update notifications engine to 7a06976 ([#3384](https://github.com/argoproj/argo-rollouts/issues/3384)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.30.4 to 1.30.5 ([#3491](https://github.com/argoproj/argo-rollouts/issues/3491)) +* **deps:** bump golang.org/x/oauth2 from 0.17.0 to 0.18.0 ([#3422](https://github.com/argoproj/argo-rollouts/issues/3422)) +* **deps:** bump softprops/action-gh-release from 2.0.3 to 2.0.4 ([#3442](https://github.com/argoproj/argo-rollouts/issues/3442)) +* **deps:** bump softprops/action-gh-release from 2.0.2 to 2.0.3 ([#3440](https://github.com/argoproj/argo-rollouts/issues/3440)) +* **deps:** bump softprops/action-gh-release from 1 to 2 ([#3438](https://github.com/argoproj/argo-rollouts/issues/3438)) +* **deps:** bump docker/build-push-action from 5.1.0 to 5.2.0 ([#3439](https://github.com/argoproj/argo-rollouts/issues/3439)) +* **deps:** bump docker/setup-buildx-action from 3.1.0 to 3.2.0 ([#3449](https://github.com/argoproj/argo-rollouts/issues/3449)) +* **deps:** bump google.golang.org/grpc from 1.62.0 to 1.62.1 ([#3426](https://github.com/argoproj/argo-rollouts/issues/3426)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.27.4 to 1.27.5 ([#3421](https://github.com/argoproj/argo-rollouts/issues/3421)) +* **deps:** bump github.com/stretchr/testify from 1.8.4 to 1.9.0 ([#3419](https://github.com/argoproj/argo-rollouts/issues/3419)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.27.0 to 1.27.4 ([#3410](https://github.com/argoproj/argo-rollouts/issues/3410)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.27.0 to 1.30.1 ([#3399](https://github.com/argoproj/argo-rollouts/issues/3399)) +* **deps:** bump google.golang.org/grpc from 1.61.0 to 1.62.0 ([#3404](https://github.com/argoproj/argo-rollouts/issues/3404)) +* **deps:** bump docker/setup-buildx-action from 3.0.0 to 3.1.0 ([#3406](https://github.com/argoproj/argo-rollouts/issues/3406)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.33.0 to 1.36.1 ([#3400](https://github.com/argoproj/argo-rollouts/issues/3400)) +* **deps:** bump codecov/codecov-action from 4.0.1 to 4.1.0 ([#3403](https://github.com/argoproj/argo-rollouts/issues/3403)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.30.1 to 1.30.3 ([#3447](https://github.com/argoproj/argo-rollouts/issues/3447)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.26.6 to 1.27.0 ([#3368](https://github.com/argoproj/argo-rollouts/issues/3368)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.32.2 to 1.33.0 ([#3363](https://github.com/argoproj/argo-rollouts/issues/3363)) +* **deps:** bump docker/login-action from 3.0.0 to 3.1.0 ([#3443](https://github.com/argoproj/argo-rollouts/issues/3443)) +* **deps:** bump golang.org/x/oauth2 from 0.16.0 to 0.17.0 ([#3357](https://github.com/argoproj/argo-rollouts/issues/3357)) +* **deps:** bump golangci/golangci-lint-action from 3 to 4 ([#3359](https://github.com/argoproj/argo-rollouts/issues/3359)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.26.7 to 1.27.0 ([#3341](https://github.com/argoproj/argo-rollouts/issues/3341)) +* **deps:** bump peter-evans/create-pull-request from 5 to 6 ([#3342](https://github.com/argoproj/argo-rollouts/issues/3342)) +* **deps:** bump sigstore/cosign-installer from 3.3.0 to 3.4.0 ([#3343](https://github.com/argoproj/argo-rollouts/issues/3343)) +* **deps:** bump codecov/codecov-action from 3.1.5 to 4.0.1 ([#3347](https://github.com/argoproj/argo-rollouts/issues/3347)) +* **deps:** bump github.com/evanphx/json-patch/v5 from 5.8.1 to 5.9.0 ([#3335](https://github.com/argoproj/argo-rollouts/issues/3335)) +* **deps:** bump docker/build-push-action from 5.2.0 to 5.3.0 ([#3448](https://github.com/argoproj/argo-rollouts/issues/3448)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.26.5 to 1.26.6 ([#3322](https://github.com/argoproj/argo-rollouts/issues/3322)) +* **deps:** bump github.com/evanphx/json-patch/v5 from 5.8.0 to 5.8.1 ([#3312](https://github.com/argoproj/argo-rollouts/issues/3312)) +* **deps:** bump codecov/codecov-action from 3.1.4 to 3.1.5 ([#3330](https://github.com/argoproj/argo-rollouts/issues/3330)) +* **deps:** bump slsa-framework/slsa-github-generator from 1.9.0 to 1.9.1 ([#3456](https://github.com/argoproj/argo-rollouts/issues/3456)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.36.1 to 1.36.3 ([#3452](https://github.com/argoproj/argo-rollouts/issues/3452)) +* **deps:** bump google.golang.org/grpc from 1.60.1 to 1.61.0 ([#3325](https://github.com/argoproj/argo-rollouts/issues/3325)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.26.4 to 1.26.5 ([#3319](https://github.com/argoproj/argo-rollouts/issues/3319)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.26.3 to 1.26.4 ([#3313](https://github.com/argoproj/argo-rollouts/issues/3313)) +* **deps:** bump actions/cache from 3 to 4 ([#3315](https://github.com/argoproj/argo-rollouts/issues/3315)) +* **deps:** bump slsa-framework/slsa-github-generator from 1.9.1 to 1.10.0 ([#3462](https://github.com/argoproj/argo-rollouts/issues/3462)) +* **deps:** bump github.com/evanphx/json-patch/v5 from 5.7.0 to 5.8.0 ([#3309](https://github.com/argoproj/argo-rollouts/issues/3309)) +* **deps:** bump golang.org/x/oauth2 from 0.15.0 to 0.16.0 ([#3294](https://github.com/argoproj/argo-rollouts/issues/3294)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.32.1 to 1.32.2 ([#3288](https://github.com/argoproj/argo-rollouts/issues/3288)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.26.2 to 1.26.3 ([#3289](https://github.com/argoproj/argo-rollouts/issues/3289)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.26.6 to 1.26.7 ([#3290](https://github.com/argoproj/argo-rollouts/issues/3290)) +* **deps:** bump github.com/aws/aws-sdk-go-v2 from 1.24.0 to 1.24.1 ([#3291](https://github.com/argoproj/argo-rollouts/issues/3291)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.30.3 to 1.30.4 ([#3461](https://github.com/argoproj/argo-rollouts/issues/3461)) +* **deps:** bump google.golang.org/protobuf from 1.31.0 to 1.32.0 ([#3273](https://github.com/argoproj/argo-rollouts/issues/3273)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.26.1 to 1.26.2 ([#3268](https://github.com/argoproj/argo-rollouts/issues/3268)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.26.5 to 1.26.6 ([#3269](https://github.com/argoproj/argo-rollouts/issues/3269)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.32.0 to 1.32.1 ([#3270](https://github.com/argoproj/argo-rollouts/issues/3270)) +* **deps:** bump google.golang.org/grpc from 1.60.0 to 1.60.1 ([#3260](https://github.com/argoproj/argo-rollouts/issues/3260)) +* **deps:** bump github/codeql-action from 2 to 3 ([#3252](https://github.com/argoproj/argo-rollouts/issues/3252)) +* **deps:** bump actions/upload-artifact from 3 to 4 ([#3255](https://github.com/argoproj/argo-rollouts/issues/3255)) +* **deps:** bump sigstore/cosign-installer from 3.2.0 to 3.3.0 ([#3245](https://github.com/argoproj/argo-rollouts/issues/3245)) +* **deps:** bump google.golang.org/grpc from 1.59.0 to 1.60.0 ([#3246](https://github.com/argoproj/argo-rollouts/issues/3246)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.26.0 to 1.26.1 ([#3241](https://github.com/argoproj/argo-rollouts/issues/3241)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.26.4 to 1.26.5 ([#3240](https://github.com/argoproj/argo-rollouts/issues/3240)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.31.4 to 1.32.0 ([#3239](https://github.com/argoproj/argo-rollouts/issues/3239)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.25.12 to 1.26.0 ([#3236](https://github.com/argoproj/argo-rollouts/issues/3236)) +* **deps:** bump codecov/codecov-action from 4.1.0 to 4.1.1 ([#3476](https://github.com/argoproj/argo-rollouts/issues/3476)) +* **deps:** bump github.com/influxdata/influxdb-client-go/v2 from 2.12.4 to 2.13.0 ([#3217](https://github.com/argoproj/argo-rollouts/issues/3217)) +* **deps:** bump actions/stale from 8 to 9 ([#3232](https://github.com/argoproj/argo-rollouts/issues/3232)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.31.3 to 1.31.4 ([#3235](https://github.com/argoproj/argo-rollouts/issues/3235)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.26.3 to 1.26.4 ([#3234](https://github.com/argoproj/argo-rollouts/issues/3234)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.25.11 to 1.25.12 ([#3230](https://github.com/argoproj/argo-rollouts/issues/3230)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.31.2 to 1.31.3 ([#3226](https://github.com/argoproj/argo-rollouts/issues/3226)) +* **deps:** bump actions/setup-python from 4 to 5 ([#3227](https://github.com/argoproj/argo-rollouts/issues/3227)) +* **deps:** bump actions/setup-go from 4.1.0 to 5.0.0 ([#3228](https://github.com/argoproj/argo-rollouts/issues/3228)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.26.2 to 1.26.3 ([#3229](https://github.com/argoproj/argo-rollouts/issues/3229)) +* **deps:** Bump k8s dependencies to v1.26.11 ([#3211](https://github.com/argoproj/argo-rollouts/issues/3211)) +* **deps:** bump argo-ui and fix browser console errors ([#3212](https://github.com/argoproj/argo-rollouts/issues/3212)) +* **deps:** bump docker/build-push-action from 5.0.0 to 5.1.0 ([#3178](https://github.com/argoproj/argo-rollouts/issues/3178)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.25.10 to 1.25.11 ([#3206](https://github.com/argoproj/argo-rollouts/issues/3206)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.26.1 to 1.26.2 ([#3207](https://github.com/argoproj/argo-rollouts/issues/3207)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.31.1 to 1.31.2 ([#3208](https://github.com/argoproj/argo-rollouts/issues/3208)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.30.5 to 1.31.1 ([#3201](https://github.com/argoproj/argo-rollouts/issues/3201)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.25.2 to 1.26.1 ([#3203](https://github.com/argoproj/argo-rollouts/issues/3203)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.25.8 to 1.25.10 ([#3204](https://github.com/argoproj/argo-rollouts/issues/3204)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.25.5 to 1.25.8 ([#3191](https://github.com/argoproj/argo-rollouts/issues/3191)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.24.3 to 1.25.2 ([#3192](https://github.com/argoproj/argo-rollouts/issues/3192)) +* **deps:** bump golang.org/x/oauth2 from 0.13.0 to 0.15.0 ([#3187](https://github.com/argoproj/argo-rollouts/issues/3187)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.30.3 to 1.30.5 ([#3193](https://github.com/argoproj/argo-rollouts/issues/3193)) +* **deps:** bump github.com/antonmedv/expr from 1.15.4 to 1.15.5 ([#3186](https://github.com/argoproj/argo-rollouts/issues/3186)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.30.1 to 1.30.3 ([#3179](https://github.com/argoproj/argo-rollouts/issues/3179)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.24.0 to 1.24.3 ([#3180](https://github.com/argoproj/argo-rollouts/issues/3180)) +* **deps:** bump github.com/influxdata/influxdb-client-go/v2 from 2.12.3 to 2.12.4 ([#3150](https://github.com/argoproj/argo-rollouts/issues/3150)) +* **deps:** bump github.com/antonmedv/expr from 1.15.3 to 1.15.4 ([#3184](https://github.com/argoproj/argo-rollouts/issues/3184)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.23.0 to 1.25.5 ([#3183](https://github.com/argoproj/argo-rollouts/issues/3183)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.30.0 to 1.30.1 ([#3166](https://github.com/argoproj/argo-rollouts/issues/3166)) +* **deps:** bump github.com/hashicorp/go-plugin from 1.5.2 to 1.6.0 ([#3167](https://github.com/argoproj/argo-rollouts/issues/3167)) +* **deps:** update golang to 1.21 ([#3482](https://github.com/argoproj/argo-rollouts/issues/3482)) +* **deps:** bump github.com/bombsimon/logrusr/v4 from 4.0.0 to 4.1.0 ([#3151](https://github.com/argoproj/argo-rollouts/issues/3151)) +* **deps:** bump github.com/spf13/cobra from 1.7.0 to 1.8.0 ([#3152](https://github.com/argoproj/argo-rollouts/issues/3152)) +* **deps:** bump sigstore/cosign-installer from 3.1.2 to 3.2.0 ([#3158](https://github.com/argoproj/argo-rollouts/issues/3158)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.22.0 to 1.23.0 ([#3161](https://github.com/argoproj/argo-rollouts/issues/3161)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.27.5 to 1.27.9 ([#3469](https://github.com/argoproj/argo-rollouts/issues/3469)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.28.0 to 1.30.0 ([#3144](https://github.com/argoproj/argo-rollouts/issues/3144)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.22.0 to 1.24.0 ([#3143](https://github.com/argoproj/argo-rollouts/issues/3143)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.20.0 to 1.22.0 ([#3149](https://github.com/argoproj/argo-rollouts/issues/3149)) +* **deps:** bump google.golang.org/protobuf from 1.32.0 to 1.33.0 ([#3429](https://github.com/argoproj/argo-rollouts/issues/3429)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.19.1 to 1.20.0 ([#3135](https://github.com/argoproj/argo-rollouts/issues/3135)) +* **deps:** bump github.com/aws/aws-sdk-go-v2 from 1.21.2 to 1.22.0 ([#3136](https://github.com/argoproj/argo-rollouts/issues/3136)) +* **deps:** bump sigs.k8s.io/yaml from 1.3.0 to 1.4.0 ([#3122](https://github.com/argoproj/argo-rollouts/issues/3122)) +* **deps:** bump google.golang.org/grpc from 1.58.3 to 1.59.0 ([#3113](https://github.com/argoproj/argo-rollouts/issues/3113)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.21.6 to 1.22.0 ([#3127](https://github.com/argoproj/argo-rollouts/issues/3127)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.19.0 to 1.19.1 ([#3123](https://github.com/argoproj/argo-rollouts/issues/3123)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.27.9 to 1.28.0 ([#3124](https://github.com/argoproj/argo-rollouts/issues/3124)) +* **deps:** bump golang.org/x/oauth2 from 0.10.0 to 0.13.0 ([#3107](https://github.com/argoproj/argo-rollouts/issues/3107)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.45 to 1.19.0 ([#3109](https://github.com/argoproj/argo-rollouts/issues/3109)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.44 to 1.18.45 ([#3101](https://github.com/argoproj/argo-rollouts/issues/3101)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.21.4 to 1.21.6 ([#3100](https://github.com/argoproj/argo-rollouts/issues/3100)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.27.8 to 1.27.9 ([#3102](https://github.com/argoproj/argo-rollouts/issues/3102)) +* **deps:** bump github.com/aws/aws-sdk-go-v2 from 1.21.1 to 1.21.2 ([#3103](https://github.com/argoproj/argo-rollouts/issues/3103)) +* **deps:** bump github.com/aws/smithy-go from 1.20.1 to 1.20.2 ([#3488](https://github.com/argoproj/argo-rollouts/issues/3488)) +* **deps:** bump google.golang.org/grpc from 1.58.2 to 1.58.3 ([#3098](https://github.com/argoproj/argo-rollouts/issues/3098)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.43 to 1.18.44 ([#3099](https://github.com/argoproj/argo-rollouts/issues/3099)) +* **deps:** bump github.com/aws/aws-sdk-go-v2 from 1.21.0 to 1.21.1 ([#3085](https://github.com/argoproj/argo-rollouts/issues/3085)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.27.7 to 1.27.8 ([#3086](https://github.com/argoproj/argo-rollouts/issues/3086)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.42 to 1.18.43 ([#3072](https://github.com/argoproj/argo-rollouts/issues/3072)) +* **deps:** bump github.com/hashicorp/go-plugin from 1.5.1 to 1.5.2 ([#3056](https://github.com/argoproj/argo-rollouts/issues/3056)) +* **deps:** bump github.com/prometheus/common from 0.42.0 to 0.51.1 ([#3468](https://github.com/argoproj/argo-rollouts/issues/3468)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.41 to 1.18.42 ([#3055](https://github.com/argoproj/argo-rollouts/issues/3055)) +* **deps:** bump github.com/antonmedv/expr from 1.15.2 to 1.15.3 ([#3046](https://github.com/argoproj/argo-rollouts/issues/3046)) +* **deps:** bump docker/setup-qemu-action from 2.2.0 to 3.0.0 ([#3031](https://github.com/argoproj/argo-rollouts/issues/3031)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.39 to 1.18.41 ([#3047](https://github.com/argoproj/argo-rollouts/issues/3047)) +* **deps:** bump google.golang.org/grpc from 1.58.0 to 1.58.2 ([#3050](https://github.com/argoproj/argo-rollouts/issues/3050)) +* **deps:** bump google.golang.org/grpc from 1.57.0 to 1.58.0 ([#3023](https://github.com/argoproj/argo-rollouts/issues/3023)) +* **deps:** bump github.com/evanphx/json-patch/v5 from 5.6.0 to 5.7.0 ([#3030](https://github.com/argoproj/argo-rollouts/issues/3030)) +* **deps:** bump docker/metadata-action from 4 to 5 ([#3032](https://github.com/argoproj/argo-rollouts/issues/3032)) +* **deps:** bump docker/build-push-action from 4.1.1 to 5.0.0 ([#3033](https://github.com/argoproj/argo-rollouts/issues/3033)) +* **deps:** bump docker/setup-buildx-action from 2.10.0 to 3.0.0 ([#3034](https://github.com/argoproj/argo-rollouts/issues/3034)) +* **deps:** bump docker/login-action from 2.2.0 to 3.0.0 ([#3035](https://github.com/argoproj/argo-rollouts/issues/3035)) +* **deps:** bump github.com/antonmedv/expr from 1.15.1 to 1.15.2 ([#3036](https://github.com/argoproj/argo-rollouts/issues/3036)) +* **deps:** bump github.com/aws/aws-sdk-go-v2 from 1.26.0 to 1.26.1 ([#3490](https://github.com/argoproj/argo-rollouts/issues/3490)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.21.3 to 1.21.4 ([#3025](https://github.com/argoproj/argo-rollouts/issues/3025)) +* **deps:** bump github.com/hashicorp/go-plugin from 1.5.0 to 1.5.1 ([#3017](https://github.com/argoproj/argo-rollouts/issues/3017)) +* **deps:** bump github.com/antonmedv/expr from 1.13.0 to 1.15.1 ([#3024](https://github.com/argoproj/argo-rollouts/issues/3024)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.38 to 1.18.39 ([#3018](https://github.com/argoproj/argo-rollouts/issues/3018)) +* **deps:** bump actions/checkout from 3 to 4 ([#3012](https://github.com/argoproj/argo-rollouts/issues/3012)) +* **deps:** bump sigstore/cosign-installer from 3.1.1 to 3.1.2 ([#3011](https://github.com/argoproj/argo-rollouts/issues/3011)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.37 to 1.18.38 ([#3002](https://github.com/argoproj/argo-rollouts/issues/3002)) +* **deps:** bump github.com/hashicorp/go-plugin from 1.4.10 to 1.5.0 ([#2995](https://github.com/argoproj/argo-rollouts/issues/2995)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.36.3 to 1.37.0 ([#3489](https://github.com/argoproj/argo-rollouts/issues/3489)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.27.9 to 1.27.10 ([#3492](https://github.com/argoproj/argo-rollouts/issues/3492)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.27.6 to 1.27.7 ([#2990](https://github.com/argoproj/argo-rollouts/issues/2990)) +* **deps:** bump docker/setup-buildx-action from 2.9.1 to 2.10.0 ([#2994](https://github.com/argoproj/argo-rollouts/issues/2994)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.21.0 to 1.21.3 ([#2977](https://github.com/argoproj/argo-rollouts/issues/2977)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.36 to 1.18.37 ([#2984](https://github.com/argoproj/argo-rollouts/issues/2984)) +* **deps:** bump slsa-framework/slsa-github-generator from 1.8.0 to 1.9.0 ([#2983](https://github.com/argoproj/argo-rollouts/issues/2983)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.33 to 1.18.36 ([#2978](https://github.com/argoproj/argo-rollouts/issues/2978)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.27.2 to 1.27.6 ([#2979](https://github.com/argoproj/argo-rollouts/issues/2979)) + +### Docs + +* more best practices ([#3484](https://github.com/argoproj/argo-rollouts/issues/3484)) +* typo in BlueGreen ([#3463](https://github.com/argoproj/argo-rollouts/issues/3463)) +* minor readability on migration ([#3427](https://github.com/argoproj/argo-rollouts/issues/3427)) +* added Consul plugin support to website ([#3362](https://github.com/argoproj/argo-rollouts/issues/3362)) +* Update shell autocompletion instructions ([#3377](https://github.com/argoproj/argo-rollouts/issues/3377)) +* Update Changelog ([#3365](https://github.com/argoproj/argo-rollouts/issues/3365)) +* Guides for popular use-cases ([#3346](https://github.com/argoproj/argo-rollouts/issues/3346)) +* Update Changelog ([#3328](https://github.com/argoproj/argo-rollouts/issues/3328)) +* Fixed the key for headers in prometheus based argo analysis ([#3306](https://github.com/argoproj/argo-rollouts/issues/3306)) +* mention archival of the SMI spec ([#3263](https://github.com/argoproj/argo-rollouts/issues/3263)) +* Update Changelog ([#3244](https://github.com/argoproj/argo-rollouts/issues/3244)) +* Update Changelog ([#3214](https://github.com/argoproj/argo-rollouts/issues/3214)) +* Update Changelog ([#2952](https://github.com/argoproj/argo-rollouts/issues/2952)) +* fix typo in smi.md ([#3160](https://github.com/argoproj/argo-rollouts/issues/3160)) +* Update Changelog ([#3148](https://github.com/argoproj/argo-rollouts/issues/3148)) +* add Gateway-API integration information to README.md ([#2985](https://github.com/argoproj/argo-rollouts/issues/2985)) +* add CONTRIBUTING.md at root of repo, directing to docs/ ([#3121](https://github.com/argoproj/argo-rollouts/issues/3121)) +* Ensure image not present between incomplete sentence. ([#3079](https://github.com/argoproj/argo-rollouts/issues/3079)) +* clarify external clusters ([#3058](https://github.com/argoproj/argo-rollouts/issues/3058)) +* Update Changelog ([#3021](https://github.com/argoproj/argo-rollouts/issues/3021)) +* replace `patchesStrategicMerge` with `patches` in tests/docs ([#3010](https://github.com/argoproj/argo-rollouts/issues/3010)) +* update all ingress objects to networking.k8s.io/v1 ([#3005](https://github.com/argoproj/argo-rollouts/issues/3005)) +* Remove rogue apostrophe in features/analysis.md ([#3001](https://github.com/argoproj/argo-rollouts/issues/3001)) +* add contour integration information to README.md ([#2980](https://github.com/argoproj/argo-rollouts/issues/2980)) +* **analysis:** Add note about availability of new datadog v2 functionality ([#3131](https://github.com/argoproj/argo-rollouts/issues/3131)) +* **deps:** Specify minimum kustomize version ([#3199](https://github.com/argoproj/argo-rollouts/issues/3199)) + +### Feat + +* Reference AnalysisTemplates inside an AnalysisTemplate ([#3353](https://github.com/argoproj/argo-rollouts/issues/3353)) +* add command args for plugin ([#2992](https://github.com/argoproj/argo-rollouts/issues/2992)) +* expose secrets for notification templates ([#3455](https://github.com/argoproj/argo-rollouts/issues/3455)) ([#3466](https://github.com/argoproj/argo-rollouts/issues/3466)) +* ping pong support for istio ([#3371](https://github.com/argoproj/argo-rollouts/issues/3371)) +* display init container images on the rollout dashboard ([#3473](https://github.com/argoproj/argo-rollouts/issues/3473)) +* add Analysis run to rollout notifications ([#3296](https://github.com/argoproj/argo-rollouts/issues/3296)) +* add the max traffic weight support for the traffic routing (nginx/plugins). ([#3215](https://github.com/argoproj/argo-rollouts/issues/3215)) +* allow analysis run to use separate kubeconfig for jobs ([#3350](https://github.com/argoproj/argo-rollouts/issues/3350)) +* Support AnalysisRunMetadata and Dryrun for experiments via Rollout ([#3213](https://github.com/argoproj/argo-rollouts/issues/3213)) +* allow setting traefik versions ([#3348](https://github.com/argoproj/argo-rollouts/issues/3348)) +* support ability to run only the analysis controller ([#3336](https://github.com/argoproj/argo-rollouts/issues/3336)) +* Support OAuth2 for prometheus and web providers ([#3038](https://github.com/argoproj/argo-rollouts/issues/3038)) +* Add support for aggregator type in DataDog metric provider ([#3293](https://github.com/argoproj/argo-rollouts/issues/3293)) +* add analysis modal ([#3174](https://github.com/argoproj/argo-rollouts/issues/3174)) +* automatically scale down Deployment after migrating to Rollout ([#3111](https://github.com/argoproj/argo-rollouts/issues/3111)) +* Rollouts UI List View Refresh ([#3118](https://github.com/argoproj/argo-rollouts/issues/3118)) +* **analysis:** add ttlStrategy on AnalysisRun for garbage collecting stale AnalysisRun automatically ([#3324](https://github.com/argoproj/argo-rollouts/issues/3324)) +* **dashboard:** improve pods visibility ([#3483](https://github.com/argoproj/argo-rollouts/issues/3483)) +* **trafficrouting:** use values array for multiple accepted values under same header name ([#2974](https://github.com/argoproj/argo-rollouts/issues/2974)) + +### Fix + +* set formatter for klog logger ([#3493](https://github.com/argoproj/argo-rollouts/issues/3493)) +* fix the issue that when max weight is 100000000, and the replicas> 20, the trafficWeightToReplicas will return negative value. ([#3474](https://github.com/argoproj/argo-rollouts/issues/3474)) +* analysis step should be ignored after promote ([#3016](https://github.com/argoproj/argo-rollouts/issues/3016)) +* job metrics owner ref when using custom job kubeconfig/ns ([#3425](https://github.com/argoproj/argo-rollouts/issues/3425)) +* Add the GOPATH to the go-to-protobuf command ([#3022](https://github.com/argoproj/argo-rollouts/issues/3022)) +* prevent hot loop when fully promoted rollout is aborted ([#3064](https://github.com/argoproj/argo-rollouts/issues/3064)) +* include the correct response error in the plugin init error message ([#3388](https://github.com/argoproj/argo-rollouts/issues/3388)) +* append weighted destination only when weight is mentioned ([#2734](https://github.com/argoproj/argo-rollouts/issues/2734)) +* stuck rollout when 2nd deployment happens before 1st finishes ([#3354](https://github.com/argoproj/argo-rollouts/issues/3354)) +* do not require pod readiness when switching desired service selector on abort ([#3338](https://github.com/argoproj/argo-rollouts/issues/3338)) +* log rs name when update fails ([#3318](https://github.com/argoproj/argo-rollouts/issues/3318)) +* keep rs inormer updated upon updating labels and annotations ([#3321](https://github.com/argoproj/argo-rollouts/issues/3321)) +* updates to replicas and pod template at the same time causes rollout to get stuck ([#3272](https://github.com/argoproj/argo-rollouts/issues/3272)) +* canary step analysis run wasn't terminated as keep running after promote action being called. Fixes [#3220](https://github.com/argoproj/argo-rollouts/issues/3220) ([#3221](https://github.com/argoproj/argo-rollouts/issues/3221)) +* make sure we use the updated rs when we write back to informer ([#3237](https://github.com/argoproj/argo-rollouts/issues/3237)) +* conflict on updates to replicaset revision ([#3216](https://github.com/argoproj/argo-rollouts/issues/3216)) +* rollouts getting stuck due to bad rs informer updates ([#3200](https://github.com/argoproj/argo-rollouts/issues/3200)) +* missing notification on error ([#3076](https://github.com/argoproj/argo-rollouts/issues/3076)) +* istio destionationrule subsets enforcement ([#3126](https://github.com/argoproj/argo-rollouts/issues/3126)) +* docs require build.os to be defined ([#3133](https://github.com/argoproj/argo-rollouts/issues/3133)) +* rollback to stable with dynamicStableScale could overwhelm stable pods ([#3077](https://github.com/argoproj/argo-rollouts/issues/3077)) +* inopportune scaling events would lose some status fields ([#3060](https://github.com/argoproj/argo-rollouts/issues/3060)) +* codegen was missed ([#3104](https://github.com/argoproj/argo-rollouts/issues/3104)) +* keep rs informer updated ([#3091](https://github.com/argoproj/argo-rollouts/issues/3091)) +* bump notification-engine to fix double send on self server notifications ([#3095](https://github.com/argoproj/argo-rollouts/issues/3095)) +* revert repo change to expr ([#3094](https://github.com/argoproj/argo-rollouts/issues/3094)) +* Replace antonmedv/expr with expr-lang/expr ([#3090](https://github.com/argoproj/argo-rollouts/issues/3090)) +* Revert "fix: istio destionationrule subsets enforcement ([#3126](https://github.com/argoproj/argo-rollouts/issues/3126))" ([#3147](https://github.com/argoproj/argo-rollouts/issues/3147)) +* sync notification controller configmaps/secrets first ([#3075](https://github.com/argoproj/argo-rollouts/issues/3075)) +* **controller:** don't timeout rollout when still waiting for scale down delay ([#3417](https://github.com/argoproj/argo-rollouts/issues/3417)) +* **controller:** treat spec.canary.analysis.template empty list as spec.canary.analysis not set ([#3446](https://github.com/argoproj/argo-rollouts/issues/3446)) +* **controller:** prevent negative vsvc weights on a replica scaledown following a canary abort for istio trafficrouting ([#3467](https://github.com/argoproj/argo-rollouts/issues/3467)) +* **controller:** rollback should skip all steps to active rs within RollbackWindow ([#2953](https://github.com/argoproj/argo-rollouts/issues/2953)) +* **controller:** typo fix ("Secrete" -> "Secret") in secret informer ([#2965](https://github.com/argoproj/argo-rollouts/issues/2965)) +* **metricprovider:** support Datadog v2 API Fixes [#2813](https://github.com/argoproj/argo-rollouts/issues/2813) ([#2997](https://github.com/argoproj/argo-rollouts/issues/2997)) + +### Refactor + +* rename interface{} => any ([#3000](https://github.com/argoproj/argo-rollouts/issues/3000)) + +### Test + +* add unit tests for maxSurge=0, replicas=1 ([#3375](https://github.com/argoproj/argo-rollouts/issues/3375)) + ## [v1.6.6](https://github.com/argoproj/argo-rollouts/compare/v1.6.5...v1.6.6) (2024-02-12) @@ -98,23 +414,21 @@ ### Chore -* bump gotestsum and fix flakey test causing nil channel send ([#2934](https://github.com/argoproj/argo-rollouts/issues/2934)) * quote golang version string to not use go 1.2.2 ([#2915](https://github.com/argoproj/argo-rollouts/issues/2915)) -* bump golang to 1.20 ([#2910](https://github.com/argoproj/argo-rollouts/issues/2910)) -* add make help cmd ([#2854](https://github.com/argoproj/argo-rollouts/issues/2854)) -* add unit test ([#2798](https://github.com/argoproj/argo-rollouts/issues/2798)) +* bump gotestsum and fix flakey test causing nil channel send ([#2934](https://github.com/argoproj/argo-rollouts/issues/2934)) * Update test and related docs for plugin name standard ([#2728](https://github.com/argoproj/argo-rollouts/issues/2728)) * bump k8s deps to v0.25.8 ([#2712](https://github.com/argoproj/argo-rollouts/issues/2712)) * add zachaller as lead in owers file ([#2759](https://github.com/argoproj/argo-rollouts/issues/2759)) +* add unit test ([#2798](https://github.com/argoproj/argo-rollouts/issues/2798)) +* add make help cmd ([#2854](https://github.com/argoproj/argo-rollouts/issues/2854)) * Add tests for pause functionality in rollout package ([#2772](https://github.com/argoproj/argo-rollouts/issues/2772)) -* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.26.0 to 1.26.1 ([#2840](https://github.com/argoproj/argo-rollouts/issues/2840)) +* bump golang to 1.20 ([#2910](https://github.com/argoproj/argo-rollouts/issues/2910)) +* **deps:** bump actions/setup-go from 4.0.1 to 4.1.0 ([#2947](https://github.com/argoproj/argo-rollouts/issues/2947)) * **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.30 to 1.18.31 ([#2924](https://github.com/argoproj/argo-rollouts/issues/2924)) -* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.27.0 to 1.27.1 ([#2927](https://github.com/argoproj/argo-rollouts/issues/2927)) -* **deps:** bump docker/build-push-action from 4.0.0 to 4.1.0 ([#2832](https://github.com/argoproj/argo-rollouts/issues/2832)) -* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.26.3 to 1.27.0 ([#2922](https://github.com/argoproj/argo-rollouts/issues/2922)) -* **deps:** bump github.com/sirupsen/logrus from 1.9.2 to 1.9.3 ([#2821](https://github.com/argoproj/argo-rollouts/issues/2821)) * **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.29 to 1.18.30 ([#2919](https://github.com/argoproj/argo-rollouts/issues/2919)) * **deps:** bump github.com/aws/aws-sdk-go-v2 from 1.19.0 to 1.19.1 ([#2920](https://github.com/argoproj/argo-rollouts/issues/2920)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.26.3 to 1.27.0 ([#2922](https://github.com/argoproj/argo-rollouts/issues/2922)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.31 to 1.18.32 ([#2928](https://github.com/argoproj/argo-rollouts/issues/2928)) * **deps:** bump google.golang.org/grpc from 1.56.2 to 1.57.0 ([#2908](https://github.com/argoproj/argo-rollouts/issues/2908)) * **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.28 to 1.18.29 ([#2907](https://github.com/argoproj/argo-rollouts/issues/2907)) * **deps:** bump github.com/antonmedv/expr from 1.12.6 to 1.12.7 ([#2894](https://github.com/argoproj/argo-rollouts/issues/2894)) @@ -130,42 +444,44 @@ * **deps:** bump google.golang.org/grpc from 1.56.1 to 1.56.2 ([#2872](https://github.com/argoproj/argo-rollouts/issues/2872)) * **deps:** bump sigstore/cosign-installer from 3.1.0 to 3.1.1 ([#2860](https://github.com/argoproj/argo-rollouts/issues/2860)) * **deps:** bump google.golang.org/protobuf from 1.30.0 to 1.31.0 ([#2859](https://github.com/argoproj/argo-rollouts/issues/2859)) -* **deps:** bump sigstore/cosign-installer from 3.0.5 to 3.1.0 ([#2858](https://github.com/argoproj/argo-rollouts/issues/2858)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.27.0 to 1.27.1 ([#2927](https://github.com/argoproj/argo-rollouts/issues/2927)) * **deps:** bump google.golang.org/grpc from 1.55.0 to 1.56.1 ([#2856](https://github.com/argoproj/argo-rollouts/issues/2856)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.19.14 to 1.20.1 ([#2926](https://github.com/argoproj/argo-rollouts/issues/2926)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.19.12 to 1.19.13 ([#2847](https://github.com/argoproj/argo-rollouts/issues/2847)) * **deps:** bump actions/setup-go from 3.5.0 to 4.0.1 ([#2849](https://github.com/argoproj/argo-rollouts/issues/2849)) * **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.26 to 1.18.27 ([#2844](https://github.com/argoproj/argo-rollouts/issues/2844)) * **deps:** bump github.com/prometheus/client_golang from 1.15.1 to 1.16.0 ([#2846](https://github.com/argoproj/argo-rollouts/issues/2846)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.26.1 to 1.26.2 ([#2848](https://github.com/argoproj/argo-rollouts/issues/2848)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.19.11 to 1.19.12 ([#2839](https://github.com/argoproj/argo-rollouts/issues/2839)) -* **deps:** bump slsa-framework/slsa-github-generator from 1.7.0 to 1.8.0 ([#2936](https://github.com/argoproj/argo-rollouts/issues/2936)) -* **deps:** bump docker/build-push-action from 4.1.0 to 4.1.1 ([#2837](https://github.com/argoproj/argo-rollouts/issues/2837)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.26.0 to 1.26.1 ([#2840](https://github.com/argoproj/argo-rollouts/issues/2840)) +* **deps:** bump sigstore/cosign-installer from 3.0.5 to 3.1.0 ([#2858](https://github.com/argoproj/argo-rollouts/issues/2858)) * **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.25 to 1.18.26 ([#2841](https://github.com/argoproj/argo-rollouts/issues/2841)) -* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.31 to 1.18.32 ([#2928](https://github.com/argoproj/argo-rollouts/issues/2928)) +* **deps:** bump docker/build-push-action from 4.0.0 to 4.1.0 ([#2832](https://github.com/argoproj/argo-rollouts/issues/2832)) +* **deps:** bump github.com/sirupsen/logrus from 1.9.2 to 1.9.3 ([#2821](https://github.com/argoproj/argo-rollouts/issues/2821)) * **deps:** bump github.com/hashicorp/go-plugin from 1.4.9 to 1.4.10 ([#2822](https://github.com/argoproj/argo-rollouts/issues/2822)) -* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.19.14 to 1.20.1 ([#2926](https://github.com/argoproj/argo-rollouts/issues/2926)) * **deps:** bump github.com/stretchr/testify from 1.8.3 to 1.8.4 ([#2817](https://github.com/argoproj/argo-rollouts/issues/2817)) * **deps:** bump github.com/sirupsen/logrus from 1.9.1 to 1.9.2 ([#2789](https://github.com/argoproj/argo-rollouts/issues/2789)) * **deps:** bump github.com/stretchr/testify from 1.8.2 to 1.8.3 ([#2796](https://github.com/argoproj/argo-rollouts/issues/2796)) +* **deps:** bump slsa-framework/slsa-github-generator from 1.7.0 to 1.8.0 ([#2936](https://github.com/argoproj/argo-rollouts/issues/2936)) * **deps:** bump sigstore/cosign-installer from 3.0.3 to 3.0.5 ([#2788](https://github.com/argoproj/argo-rollouts/issues/2788)) -* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.20.1 to 1.20.2 ([#2941](https://github.com/argoproj/argo-rollouts/issues/2941)) +* **deps:** bump docker/build-push-action from 4.1.0 to 4.1.1 ([#2837](https://github.com/argoproj/argo-rollouts/issues/2837)) * **deps:** bump github.com/sirupsen/logrus from 1.9.0 to 1.9.1 ([#2784](https://github.com/argoproj/argo-rollouts/issues/2784)) * **deps:** bump codecov/codecov-action from 3.1.3 to 3.1.4 ([#2782](https://github.com/argoproj/argo-rollouts/issues/2782)) * **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.24 to 1.18.25 ([#2770](https://github.com/argoproj/argo-rollouts/issues/2770)) * **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.23 to 1.18.24 ([#2768](https://github.com/argoproj/argo-rollouts/issues/2768)) * **deps:** bump google.golang.org/grpc from 1.54.0 to 1.55.0 ([#2763](https://github.com/argoproj/argo-rollouts/issues/2763)) * **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.22 to 1.18.23 ([#2756](https://github.com/argoproj/argo-rollouts/issues/2756)) -* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.27.1 to 1.27.2 ([#2944](https://github.com/argoproj/argo-rollouts/issues/2944)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.20.1 to 1.20.2 ([#2941](https://github.com/argoproj/argo-rollouts/issues/2941)) * **deps:** replace `github.com/ghodss/yaml` with `sigs.k8s.io/yaml` ([#2681](https://github.com/argoproj/argo-rollouts/issues/2681)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.25.10 to 1.26.0 ([#2755](https://github.com/argoproj/argo-rollouts/issues/2755)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.19.10 to 1.19.11 ([#2757](https://github.com/argoproj/argo-rollouts/issues/2757)) * **deps:** bump github.com/prometheus/client_golang from 1.15.0 to 1.15.1 ([#2754](https://github.com/argoproj/argo-rollouts/issues/2754)) * **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.21 to 1.18.22 ([#2746](https://github.com/argoproj/argo-rollouts/issues/2746)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.25.9 to 1.25.10 ([#2745](https://github.com/argoproj/argo-rollouts/issues/2745)) -* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.32 to 1.18.33 ([#2943](https://github.com/argoproj/argo-rollouts/issues/2943)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.27.1 to 1.27.2 ([#2944](https://github.com/argoproj/argo-rollouts/issues/2944)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.19.9 to 1.19.10 ([#2747](https://github.com/argoproj/argo-rollouts/issues/2747)) * **deps:** bump codecov/codecov-action from 3.1.2 to 3.1.3 ([#2735](https://github.com/argoproj/argo-rollouts/issues/2735)) -* **deps:** bump actions/setup-go from 4.0.1 to 4.1.0 ([#2947](https://github.com/argoproj/argo-rollouts/issues/2947)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.32 to 1.18.33 ([#2943](https://github.com/argoproj/argo-rollouts/issues/2943)) * **deps:** bump github.com/prometheus/client_golang from 1.14.0 to 1.15.0 ([#2721](https://github.com/argoproj/argo-rollouts/issues/2721)) * **deps:** bump codecov/codecov-action from 3.1.1 to 3.1.2 ([#2711](https://github.com/argoproj/argo-rollouts/issues/2711)) * **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.20 to 1.18.21 ([#2709](https://github.com/argoproj/argo-rollouts/issues/2709)) @@ -186,38 +502,38 @@ ### Docs -* support for Google Cloud Load balancers ([#2803](https://github.com/argoproj/argo-rollouts/issues/2803)) -* Update Changelog ([#2683](https://github.com/argoproj/argo-rollouts/issues/2683)) * mirroring support in Traefik is not implemented yet ([#2904](https://github.com/argoproj/argo-rollouts/issues/2904)) +* update contributions.md to include k3d as recommended cluster, add details on e2e test setup, and update kubectl install link. Fixes [#1750](https://github.com/argoproj/argo-rollouts/issues/1750) ([#1867](https://github.com/argoproj/argo-rollouts/issues/1867)) +* fix minor mistakes in Migrating to Deployments ([#2270](https://github.com/argoproj/argo-rollouts/issues/2270)) * Update docs of Rollout spec to add active/previewMetadata ([#2833](https://github.com/argoproj/argo-rollouts/issues/2833)) * Update datadog.md - clarify formulas [#2813](https://github.com/argoproj/argo-rollouts/issues/2813) ([#2819](https://github.com/argoproj/argo-rollouts/issues/2819)) * support for Kong ingress ([#2820](https://github.com/argoproj/argo-rollouts/issues/2820)) * Fix AWS App Mesh getting started documentation to avoid connection pooling problems ([#2814](https://github.com/argoproj/argo-rollouts/issues/2814)) * Update Changelog ([#2807](https://github.com/argoproj/argo-rollouts/issues/2807)) * use correct capitalization for "Datadog" in navigation sidebar ([#2809](https://github.com/argoproj/argo-rollouts/issues/2809)) -* Add gateway API link, fix Contour plugin naming ([#2787](https://github.com/argoproj/argo-rollouts/issues/2787)) -* fix minor mistakes in Migrating to Deployments ([#2270](https://github.com/argoproj/argo-rollouts/issues/2270)) -* Show how plugins are loaded ([#2801](https://github.com/argoproj/argo-rollouts/issues/2801)) * Fix typo in header routing specification docs ([#2808](https://github.com/argoproj/argo-rollouts/issues/2808)) +* support for Google Cloud Load balancers ([#2803](https://github.com/argoproj/argo-rollouts/issues/2803)) +* Show how plugins are loaded ([#2801](https://github.com/argoproj/argo-rollouts/issues/2801)) +* Add gateway API link, fix Contour plugin naming ([#2787](https://github.com/argoproj/argo-rollouts/issues/2787)) * Add some details around running locally to make things clearer new contributors ([#2786](https://github.com/argoproj/argo-rollouts/issues/2786)) * Add docs for Amazon Managed Prometheus ([#2777](https://github.com/argoproj/argo-rollouts/issues/2777)) * Update Changelog ([#2765](https://github.com/argoproj/argo-rollouts/issues/2765)) * copy argo cd docs drop down fix ([#2731](https://github.com/argoproj/argo-rollouts/issues/2731)) * Add contour trafficrouter plugin ([#2729](https://github.com/argoproj/argo-rollouts/issues/2729)) * fix link to plugins for traffic routers ([#2719](https://github.com/argoproj/argo-rollouts/issues/2719)) -* update contributions.md to include k3d as recommended cluster, add details on e2e test setup, and update kubectl install link. Fixes [#1750](https://github.com/argoproj/argo-rollouts/issues/1750) ([#1867](https://github.com/argoproj/argo-rollouts/issues/1867)) +* Update Changelog ([#2683](https://github.com/argoproj/argo-rollouts/issues/2683)) * **analysis:** fix use stringData in the examples ([#2715](https://github.com/argoproj/argo-rollouts/issues/2715)) -* **example:** interval requires count ([#2690](https://github.com/argoproj/argo-rollouts/issues/2690)) * **example:** Add example on how to execute subset of e2e tests ([#2867](https://github.com/argoproj/argo-rollouts/issues/2867)) +* **example:** interval requires count ([#2690](https://github.com/argoproj/argo-rollouts/issues/2690)) ### Feat +* Send informer add k8s event ([#2834](https://github.com/argoproj/argo-rollouts/issues/2834)) * enable self service notification support ([#2930](https://github.com/argoproj/argo-rollouts/issues/2930)) * support prometheus headers ([#2937](https://github.com/argoproj/argo-rollouts/issues/2937)) * Add insecure option for Prometheus. Fixes [#2913](https://github.com/argoproj/argo-rollouts/issues/2913) ([#2914](https://github.com/argoproj/argo-rollouts/issues/2914)) * Add prometheus timeout ([#2893](https://github.com/argoproj/argo-rollouts/issues/2893)) * Support Multiple ALB Ingresses ([#2639](https://github.com/argoproj/argo-rollouts/issues/2639)) -* Send informer add k8s event ([#2834](https://github.com/argoproj/argo-rollouts/issues/2834)) * add merge key to analysis template ([#2842](https://github.com/argoproj/argo-rollouts/issues/2842)) * retain TLS configuration for canary ingresses in the nginx integration. Fixes [#1134](https://github.com/argoproj/argo-rollouts/issues/1134) ([#2679](https://github.com/argoproj/argo-rollouts/issues/2679)) * **analysis:** Adds rollout Spec.Selector.MatchLabels to AnalysisRun. Fixes [#2888](https://github.com/argoproj/argo-rollouts/issues/2888) ([#2903](https://github.com/argoproj/argo-rollouts/issues/2903)) @@ -227,20 +543,20 @@ ### Fix -* istio dropping fields during removing of managed routes ([#2692](https://github.com/argoproj/argo-rollouts/issues/2692)) -* resolve args to metric in garbage collection function ([#2843](https://github.com/argoproj/argo-rollouts/issues/2843)) +* make new alb fullName field optional for backward compatability ([#2806](https://github.com/argoproj/argo-rollouts/issues/2806)) +* cloudwatch metrics provider multiple dimensions ([#2932](https://github.com/argoproj/argo-rollouts/issues/2932)) * rollout not modify the VirtualService whit setHeaderRoute step with workloadRef ([#2797](https://github.com/argoproj/argo-rollouts/issues/2797)) * get new httpRoutesI after removeRoute() to avoid duplicates. Fixes [#2769](https://github.com/argoproj/argo-rollouts/issues/2769) ([#2887](https://github.com/argoproj/argo-rollouts/issues/2887)) -* make new alb fullName field optional for backward compatability ([#2806](https://github.com/argoproj/argo-rollouts/issues/2806)) * change logic of analysis run to better handle errors ([#2695](https://github.com/argoproj/argo-rollouts/issues/2695)) -* cloudwatch metrics provider multiple dimensions ([#2932](https://github.com/argoproj/argo-rollouts/issues/2932)) -* add required ingress permission ([#2933](https://github.com/argoproj/argo-rollouts/issues/2933)) +* istio dropping fields during removing of managed routes ([#2692](https://github.com/argoproj/argo-rollouts/issues/2692)) +* resolve args to metric in garbage collection function ([#2843](https://github.com/argoproj/argo-rollouts/issues/2843)) * properly wrap Datadog API v2 request body ([#2771](https://github.com/argoproj/argo-rollouts/issues/2771)) ([#2775](https://github.com/argoproj/argo-rollouts/issues/2775)) +* add required ingress permission ([#2933](https://github.com/argoproj/argo-rollouts/issues/2933)) +* **analysis:** Adding field in YAML to provide region for Sigv4 signing. ([#2794](https://github.com/argoproj/argo-rollouts/issues/2794)) * **analysis:** Graphite query - remove whitespaces ([#2752](https://github.com/argoproj/argo-rollouts/issues/2752)) * **analysis:** Graphite metric provider - index out of range [0] with length 0 ([#2751](https://github.com/argoproj/argo-rollouts/issues/2751)) -* **analysis:** Adding field in YAML to provide region for Sigv4 signing. ([#2794](https://github.com/argoproj/argo-rollouts/issues/2794)) -* **controller:** Fix for rollouts getting stuck in loop ([#2689](https://github.com/argoproj/argo-rollouts/issues/2689)) * **controller:** Remove name label from some k8s client metrics on events and replicasets ([#2851](https://github.com/argoproj/argo-rollouts/issues/2851)) +* **controller:** Fix for rollouts getting stuck in loop ([#2689](https://github.com/argoproj/argo-rollouts/issues/2689)) * **controller:** Add klog logrus bridge. Fixes [#2707](https://github.com/argoproj/argo-rollouts/issues/2707). ([#2701](https://github.com/argoproj/argo-rollouts/issues/2701)) * **trafficrouting:** apply stable selectors on canary service on rollout abort [#2781](https://github.com/argoproj/argo-rollouts/issues/2781) ([#2818](https://github.com/argoproj/argo-rollouts/issues/2818)) @@ -315,8 +631,8 @@ types. * switch to distroless for cli/dashboard image ([#2596](https://github.com/argoproj/argo-rollouts/issues/2596)) * add Tuhu to users ([#2630](https://github.com/argoproj/argo-rollouts/issues/2630)) * bump deps for prisma ([#2643](https://github.com/argoproj/argo-rollouts/issues/2643)) -* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.13 to 1.18.14 ([#2614](https://github.com/argoproj/argo-rollouts/issues/2614)) -* **deps:** bump github.com/antonmedv/expr from 1.12.3 to 1.12.5 ([#2670](https://github.com/argoproj/argo-rollouts/issues/2670)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.25.6 to 1.25.7 ([#2682](https://github.com/argoproj/argo-rollouts/issues/2682)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.15 to 1.18.16 ([#2652](https://github.com/argoproj/argo-rollouts/issues/2652)) * **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.16 to 1.18.17 ([#2659](https://github.com/argoproj/argo-rollouts/issues/2659)) * **deps:** bump github.com/antonmedv/expr from 1.12.2 to 1.12.3 ([#2653](https://github.com/argoproj/argo-rollouts/issues/2653)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.19.5 to 1.19.6 ([#2654](https://github.com/argoproj/argo-rollouts/issues/2654)) @@ -331,20 +647,20 @@ types. * **deps:** bump google.golang.org/protobuf from 1.29.0 to 1.29.1 ([#2660](https://github.com/argoproj/argo-rollouts/issues/2660)) * **deps:** bump google.golang.org/protobuf from 1.29.1 to 1.30.0 ([#2665](https://github.com/argoproj/argo-rollouts/issues/2665)) * **deps:** bump github.com/stretchr/testify from 1.8.1 to 1.8.2 ([#2627](https://github.com/argoproj/argo-rollouts/issues/2627)) -* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.14 to 1.18.15 ([#2618](https://github.com/argoproj/argo-rollouts/issues/2618)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.19.6 to 1.19.7 ([#2672](https://github.com/argoproj/argo-rollouts/issues/2672)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.25.3 to 1.25.4 ([#2617](https://github.com/argoproj/argo-rollouts/issues/2617)) * **deps:** bump github.com/antonmedv/expr from 1.12.0 to 1.12.1 ([#2619](https://github.com/argoproj/argo-rollouts/issues/2619)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.19.4 to 1.19.5 ([#2616](https://github.com/argoproj/argo-rollouts/issues/2616)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.19.3 to 1.19.4 ([#2612](https://github.com/argoproj/argo-rollouts/issues/2612)) * **deps:** bump github.com/prometheus/common from 0.39.0 to 0.40.0 ([#2611](https://github.com/argoproj/argo-rollouts/issues/2611)) -* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.25.6 to 1.25.7 ([#2682](https://github.com/argoproj/argo-rollouts/issues/2682)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.13 to 1.18.14 ([#2614](https://github.com/argoproj/argo-rollouts/issues/2614)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.25.2 to 1.25.3 ([#2615](https://github.com/argoproj/argo-rollouts/issues/2615)) -* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 from 1.19.6 to 1.19.7 ([#2672](https://github.com/argoproj/argo-rollouts/issues/2672)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.14 to 1.18.15 ([#2618](https://github.com/argoproj/argo-rollouts/issues/2618)) * **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.17 to 1.18.19 ([#2673](https://github.com/argoproj/argo-rollouts/issues/2673)) * **deps:** bump imjasonh/setup-crane from 0.2 to 0.3 ([#2600](https://github.com/argoproj/argo-rollouts/issues/2600)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch from 1.25.5 to 1.25.6 ([#2671](https://github.com/argoproj/argo-rollouts/issues/2671)) * **deps:** bump github.com/aws/aws-sdk-go-v2/config ([#2593](https://github.com/argoproj/argo-rollouts/issues/2593)) -* **deps:** bump github.com/aws/aws-sdk-go-v2/config from 1.18.15 to 1.18.16 ([#2652](https://github.com/argoproj/argo-rollouts/issues/2652)) +* **deps:** bump github.com/antonmedv/expr from 1.12.3 to 1.12.5 ([#2670](https://github.com/argoproj/argo-rollouts/issues/2670)) * **deps:** bump google.golang.org/grpc from 1.52.3 to 1.53.0 ([#2574](https://github.com/argoproj/argo-rollouts/issues/2574)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 ([#2565](https://github.com/argoproj/argo-rollouts/issues/2565)) * **deps:** bump github.com/aws/aws-sdk-go-v2/config ([#2564](https://github.com/argoproj/argo-rollouts/issues/2564)) @@ -406,12 +722,12 @@ types. ### Fix +* update GetTargetGroupMetadata to call DescribeTags in batches ([#2570](https://github.com/argoproj/argo-rollouts/issues/2570)) * switch service selector back to stable on canary service when aborted ([#2540](https://github.com/argoproj/argo-rollouts/issues/2540)) * change log generator to only add CHANGELOG.md ([#2626](https://github.com/argoproj/argo-rollouts/issues/2626)) * Rollback change on service creation with weightless experiments ([#2624](https://github.com/argoproj/argo-rollouts/issues/2624)) * flakey TestWriteBackToInformer test ([#2621](https://github.com/argoproj/argo-rollouts/issues/2621)) * remove outdated ioutil package dependencies ([#2583](https://github.com/argoproj/argo-rollouts/issues/2583)) -* update GetTargetGroupMetadata to call DescribeTags in batches ([#2570](https://github.com/argoproj/argo-rollouts/issues/2570)) * analysis information box [#2530](https://github.com/argoproj/argo-rollouts/issues/2530) ([#2575](https://github.com/argoproj/argo-rollouts/issues/2575)) * support only tls in virtual services ([#2502](https://github.com/argoproj/argo-rollouts/issues/2502)) * **analysis:** Nil Pointer Fixes [#2458](https://github.com/argoproj/argo-rollouts/issues/2458) ([#2680](https://github.com/argoproj/argo-rollouts/issues/2680)) @@ -477,21 +793,21 @@ There was an unintentional change in behavior related to service creation with e ### Chore -* add optum to users list ([#2466](https://github.com/argoproj/argo-rollouts/issues/2466)) +* Add Yotpo to USERS.md * upgrade golang to 1.19 ([#2219](https://github.com/argoproj/argo-rollouts/issues/2219)) -* sign container images and checksum assets ([#2334](https://github.com/argoproj/argo-rollouts/issues/2334)) +* remove deprecated -i for go build ([#2047](https://github.com/argoproj/argo-rollouts/issues/2047)) +* rename the examples/trafffic-management directory to istio ([#2315](https://github.com/argoproj/argo-rollouts/issues/2315)) * update stable tag conditionally ([#2480](https://github.com/argoproj/argo-rollouts/issues/2480)) * fix checksum generation ([#2481](https://github.com/argoproj/argo-rollouts/issues/2481)) -* Add Yotpo to USERS.md +* add optum to users list ([#2466](https://github.com/argoproj/argo-rollouts/issues/2466)) * use docker login to sign images ([#2479](https://github.com/argoproj/argo-rollouts/issues/2479)) * use correct image for plugin container ([#2478](https://github.com/argoproj/argo-rollouts/issues/2478)) -* rename the examples/trafffic-management directory to istio ([#2315](https://github.com/argoproj/argo-rollouts/issues/2315)) * Add example for istio-subset-split ([#2318](https://github.com/argoproj/argo-rollouts/issues/2318)) * add deprecation notice for rollout_phase in docs ([#2377](https://github.com/argoproj/argo-rollouts/issues/2377)) ([#2378](https://github.com/argoproj/argo-rollouts/issues/2378)) -* remove deprecated -i for go build ([#2047](https://github.com/argoproj/argo-rollouts/issues/2047)) +* sign container images and checksum assets ([#2334](https://github.com/argoproj/argo-rollouts/issues/2334)) * **cli:** add darwin arm64 to build and release ([#2264](https://github.com/argoproj/argo-rollouts/issues/2264)) -* **deps:** upgrade ui deps to fix high security cve's ([#2345](https://github.com/argoproj/argo-rollouts/issues/2345)) -* **deps:** bump github.com/aws/aws-sdk-go-v2 from 1.17.0 to 1.17.1 ([#2369](https://github.com/argoproj/argo-rollouts/issues/2369)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch ([#2487](https://github.com/argoproj/argo-rollouts/issues/2487)) +* **deps:** bump github.com/prometheus/common from 0.37.0 to 0.38.0 ([#2468](https://github.com/argoproj/argo-rollouts/issues/2468)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch ([#2455](https://github.com/argoproj/argo-rollouts/issues/2455)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 ([#2454](https://github.com/argoproj/argo-rollouts/issues/2454)) * **deps:** bump github.com/aws/aws-sdk-go-v2/config ([#2452](https://github.com/argoproj/argo-rollouts/issues/2452)) @@ -507,23 +823,23 @@ There was an unintentional change in behavior related to service creation with e * **deps:** bump github.com/aws/aws-sdk-go-v2/config ([#2413](https://github.com/argoproj/argo-rollouts/issues/2413)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 ([#2412](https://github.com/argoproj/argo-rollouts/issues/2412)) * **deps:** bump github.com/aws/aws-sdk-go-v2/config ([#2409](https://github.com/argoproj/argo-rollouts/issues/2409)) -* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 ([#2406](https://github.com/argoproj/argo-rollouts/issues/2406)) -* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch ([#2404](https://github.com/argoproj/argo-rollouts/issues/2404)) * **deps:** bump github.com/prometheus/client_golang ([#2469](https://github.com/argoproj/argo-rollouts/issues/2469)) -* **deps:** bump codecov/codecov-action from 2.1.0 to 3.1.1 ([#2251](https://github.com/argoproj/argo-rollouts/issues/2251)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch ([#2404](https://github.com/argoproj/argo-rollouts/issues/2404)) * **deps:** bump notification engine ([#2470](https://github.com/argoproj/argo-rollouts/issues/2470)) +* **deps:** bump codecov/codecov-action from 2.1.0 to 3.1.1 ([#2251](https://github.com/argoproj/argo-rollouts/issues/2251)) * **deps:** bump github.com/prometheus/common from 0.38.0 to 0.39.0 ([#2476](https://github.com/argoproj/argo-rollouts/issues/2476)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch ([#2477](https://github.com/argoproj/argo-rollouts/issues/2477)) +* **deps:** bump github.com/aws/aws-sdk-go-v2 from 1.17.2 to 1.17.3 ([#2484](https://github.com/argoproj/argo-rollouts/issues/2484)) * **deps:** bump dependabot/fetch-metadata from 1.3.4 to 1.3.5 ([#2390](https://github.com/argoproj/argo-rollouts/issues/2390)) * **deps:** bump imjasonh/setup-crane from 0.1 to 0.2 ([#2387](https://github.com/argoproj/argo-rollouts/issues/2387)) -* **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch ([#2487](https://github.com/argoproj/argo-rollouts/issues/2487)) -* **deps:** bump github.com/aws/aws-sdk-go-v2 from 1.17.2 to 1.17.3 ([#2484](https://github.com/argoproj/argo-rollouts/issues/2484)) +* **deps:** upgrade ui deps to fix high security cve's ([#2345](https://github.com/argoproj/argo-rollouts/issues/2345)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 ([#2406](https://github.com/argoproj/argo-rollouts/issues/2406)) * **deps:** bump actions/upload-artifact from 2 to 3 ([#1973](https://github.com/argoproj/argo-rollouts/issues/1973)) * **deps:** bump github.com/influxdata/influxdb-client-go/v2 ([#2381](https://github.com/argoproj/argo-rollouts/issues/2381)) * **deps:** bump github.com/spf13/cobra from 1.6.0 to 1.6.1 ([#2370](https://github.com/argoproj/argo-rollouts/issues/2370)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/cloudwatch ([#2366](https://github.com/argoproj/argo-rollouts/issues/2366)) * **deps:** bump github.com/aws/aws-sdk-go-v2/config ([#2367](https://github.com/argoproj/argo-rollouts/issues/2367)) -* **deps:** bump github.com/prometheus/common from 0.37.0 to 0.38.0 ([#2468](https://github.com/argoproj/argo-rollouts/issues/2468)) +* **deps:** bump github.com/aws/aws-sdk-go-v2 from 1.17.0 to 1.17.1 ([#2369](https://github.com/argoproj/argo-rollouts/issues/2369)) * **deps:** bump github.com/stretchr/testify from 1.8.0 to 1.8.1 ([#2368](https://github.com/argoproj/argo-rollouts/issues/2368)) * **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 ([#2365](https://github.com/argoproj/argo-rollouts/issues/2365)) * **deps:** bump github.com/aws/aws-sdk-go-v2 from 1.16.16 to 1.17.0 ([#2364](https://github.com/argoproj/argo-rollouts/issues/2364)) @@ -546,7 +862,7 @@ There was an unintentional change in behavior related to service creation with e * **deps:** bump github.com/aws/aws-sdk-go-v2/config ([#2294](https://github.com/argoproj/argo-rollouts/issues/2294)) * **deps:** bump google.golang.org/grpc from 1.47.0 to 1.50.0 ([#2293](https://github.com/argoproj/argo-rollouts/issues/2293)) * **deps:** bump docker/metadata-action from 3 to 4 ([#2292](https://github.com/argoproj/argo-rollouts/issues/2292)) -* **deps:** bump github/codeql-action from 1 to 2 ([#2289](https://github.com/argoproj/argo-rollouts/issues/2289)) +* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 ([#2486](https://github.com/argoproj/argo-rollouts/issues/2486)) * **deps:** bump docker/login-action from 1 to 2 ([#2288](https://github.com/argoproj/argo-rollouts/issues/2288)) * **deps:** bump actions/setup-go from 2 to 3 ([#2287](https://github.com/argoproj/argo-rollouts/issues/2287)) * **deps:** bump dependabot/fetch-metadata from 1.3.3 to 1.3.4 ([#2286](https://github.com/argoproj/argo-rollouts/issues/2286)) @@ -555,8 +871,8 @@ There was an unintentional change in behavior related to service creation with e * **deps:** bump actions/cache from 2 to 3.0.1 ([#1940](https://github.com/argoproj/argo-rollouts/issues/1940)) * **deps:** bump docker/setup-qemu-action from 1 to 2 ([#2284](https://github.com/argoproj/argo-rollouts/issues/2284)) * **deps:** bump actions/checkout from 2 to 3.1.0 ([#2283](https://github.com/argoproj/argo-rollouts/issues/2283)) -* **deps:** bump github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 ([#2486](https://github.com/argoproj/argo-rollouts/issues/2486)) * **deps:** bump github.com/aws/aws-sdk-go-v2/config ([#2485](https://github.com/argoproj/argo-rollouts/issues/2485)) +* **deps:** bump github/codeql-action from 1 to 2 ([#2289](https://github.com/argoproj/argo-rollouts/issues/2289)) ### Ci @@ -575,23 +891,23 @@ There was an unintentional change in behavior related to service creation with e ### Docs -* common questions for Rollbacks ([#2027](https://github.com/argoproj/argo-rollouts/issues/2027)) +* Add traffic router support to readme ([#2444](https://github.com/argoproj/argo-rollouts/issues/2444)) +* fix typo in helm Argo rollouts ([#2442](https://github.com/argoproj/argo-rollouts/issues/2442)) * correct syntax of canary setMirrorRoute's value ([#2431](https://github.com/argoproj/argo-rollouts/issues/2431)) -* add artifact badge ([#2331](https://github.com/argoproj/argo-rollouts/issues/2331)) -* Use new Google Analytics 4 site tag ([#2299](https://github.com/argoproj/argo-rollouts/issues/2299)) +* Explain upgrade process ([#2424](https://github.com/argoproj/argo-rollouts/issues/2424)) * add progressive delivery with gitops example for openshift ([#2400](https://github.com/argoproj/argo-rollouts/issues/2400)) * fix !important block typo ([#2372](https://github.com/argoproj/argo-rollouts/issues/2372)) * mention supported versions ([#2163](https://github.com/argoproj/argo-rollouts/issues/2163)) * Added blog post for minimize impact in Kubernetes using Progressive Delivery and customer side impact ([#2355](https://github.com/argoproj/argo-rollouts/issues/2355)) -* add Opensurvey to USERS.md ([#2195](https://github.com/argoproj/argo-rollouts/issues/2195)) -* fix typo in helm Argo rollouts ([#2442](https://github.com/argoproj/argo-rollouts/issues/2442)) -* Explain upgrade process ([#2424](https://github.com/argoproj/argo-rollouts/issues/2424)) +* Update docs for new openapi kustomize support ([#2216](https://github.com/argoproj/argo-rollouts/issues/2216)) +* add artifact badge ([#2331](https://github.com/argoproj/argo-rollouts/issues/2331)) +* Use new Google Analytics 4 site tag ([#2299](https://github.com/argoproj/argo-rollouts/issues/2299)) * Fixed read the docs rendering ([#2277](https://github.com/argoproj/argo-rollouts/issues/2277)) -* Add traffic router support to readme ([#2444](https://github.com/argoproj/argo-rollouts/issues/2444)) +* common questions for Rollbacks ([#2027](https://github.com/argoproj/argo-rollouts/issues/2027)) * add OpsVerse as an official user (USERS.md) ([#2209](https://github.com/argoproj/argo-rollouts/issues/2209)) * Fix the controller annotation to enable data scrapping ([#2238](https://github.com/argoproj/argo-rollouts/issues/2238)) * Update release docs for versioned formula ([#2245](https://github.com/argoproj/argo-rollouts/issues/2245)) -* Update docs for new openapi kustomize support ([#2216](https://github.com/argoproj/argo-rollouts/issues/2216)) +* add Opensurvey to USERS.md ([#2195](https://github.com/argoproj/argo-rollouts/issues/2195)) * **trafficrouting:** fix docs warning to github style markdown ([#2342](https://github.com/argoproj/argo-rollouts/issues/2342)) ### Feat @@ -599,30 +915,30 @@ There was an unintentional change in behavior related to service creation with e * Implement Issue [#1779](https://github.com/argoproj/argo-rollouts/issues/1779): add rollout.Spec.Strategy.Canary.MinPodsPerReplicaSet ([#2448](https://github.com/argoproj/argo-rollouts/issues/2448)) * Apache APISIX support. Fixes [#2395](https://github.com/argoproj/argo-rollouts/issues/2395) ([#2437](https://github.com/argoproj/argo-rollouts/issues/2437)) * rollback windows. Fixes [#574](https://github.com/argoproj/argo-rollouts/issues/574) ([#2394](https://github.com/argoproj/argo-rollouts/issues/2394)) +* Support TCP routes traffic splitting for Istio VirtualService ([#1659](https://github.com/argoproj/argo-rollouts/issues/1659)) * add support for getting the replicaset name via templating ([#2396](https://github.com/argoproj/argo-rollouts/issues/2396)) * Allow Traffic shaping through header based routing for ALB ([#2214](https://github.com/argoproj/argo-rollouts/issues/2214)) * Add support for spec.ingressClassName ([#2178](https://github.com/argoproj/argo-rollouts/issues/2178)) -* Support TCP routes traffic splitting for Istio VirtualService ([#1659](https://github.com/argoproj/argo-rollouts/issues/1659)) * **cli:** dynamic shell completion for main resources names (rollouts, experiments, analysisrun) ([#2379](https://github.com/argoproj/argo-rollouts/issues/2379)) * **cli:** add port flag for dashboard command ([#2383](https://github.com/argoproj/argo-rollouts/issues/2383)) * **controller:** don't hardcode experiment ports; always create service ([#2397](https://github.com/argoproj/argo-rollouts/issues/2397)) ### Fix +* set gopath in makefile ([#2398](https://github.com/argoproj/argo-rollouts/issues/2398)) * dev build can set DEV_IMAGE=true ([#2440](https://github.com/argoproj/argo-rollouts/issues/2440)) * add patch verb to deployment resource ([#2407](https://github.com/argoproj/argo-rollouts/issues/2407)) * rootPath support so that it uses the embedded files system ([#2198](https://github.com/argoproj/argo-rollouts/issues/2198)) -* set gopath in makefile ([#2398](https://github.com/argoproj/argo-rollouts/issues/2398)) * change completed condition so it only triggers on pod hash changes also adds an event for when it does changes. ([#2203](https://github.com/argoproj/argo-rollouts/issues/2203)) * enable notifications without when condition ([#2231](https://github.com/argoproj/argo-rollouts/issues/2231)) * UI not redirecting on / ([#2252](https://github.com/argoproj/argo-rollouts/issues/2252)) * nil pointer while linting with basic canary and ingresses ([#2256](https://github.com/argoproj/argo-rollouts/issues/2256)) -* **analysis:** Make AR End When Only Dry-Run Metrics Are Defined ([#2230](https://github.com/argoproj/argo-rollouts/issues/2230)) * **analysis:** Fix Analysis Terminal Decision For Dry-Run Metrics ([#2399](https://github.com/argoproj/argo-rollouts/issues/2399)) +* **analysis:** Make AR End When Only Dry-Run Metrics Are Defined ([#2230](https://github.com/argoproj/argo-rollouts/issues/2230)) * **analysis:** Avoid Infinite Error Message Append For Failed Dry-Run Metrics ([#2182](https://github.com/argoproj/argo-rollouts/issues/2182)) * **cli:** nil pointer while linting ([#2324](https://github.com/argoproj/argo-rollouts/issues/2324)) -* **controller:** Fix k8s clientset controller metrics. Fixes [#2139](https://github.com/argoproj/argo-rollouts/issues/2139) ([#2261](https://github.com/argoproj/argo-rollouts/issues/2261)) * **controller:** leader election preventing two controllers running and gracefully shutting down ([#2291](https://github.com/argoproj/argo-rollouts/issues/2291)) +* **controller:** Fix k8s clientset controller metrics. Fixes [#2139](https://github.com/argoproj/argo-rollouts/issues/2139) ([#2261](https://github.com/argoproj/argo-rollouts/issues/2261)) * **dashboard:** correct mime type is returned. Fixes: [#2290](https://github.com/argoproj/argo-rollouts/issues/2290) ([#2303](https://github.com/argoproj/argo-rollouts/issues/2303)) * **example:** correct docs when metrics got result empty ([#2309](https://github.com/argoproj/argo-rollouts/issues/2309)) * **metricprovider:** Support jsonBody for web metric provider Fixes [#2275](https://github.com/argoproj/argo-rollouts/issues/2275) ([#2312](https://github.com/argoproj/argo-rollouts/issues/2312)) @@ -686,4 +1002,3 @@ There was an unintentional change in behavior related to service creation with e ### Fixes * **controller:** istio dropping fields not defined in type ([#2268](https://github.com/argoproj/argo-rollouts/issues/2268)) - diff --git a/Dockerfile b/Dockerfile index 4998a43454..ae5df9fa2e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ # Initial stage which pulls prepares build dependencies and CLI tooling we need for our final image # Also used as the image in CI jobs so needs all dependencies #################################################################################################### -FROM --platform=$BUILDPLATFORM golang:1.21 as builder +FROM --platform=$BUILDPLATFORM golang:1.23 AS builder RUN apt-get update && apt-get install -y \ wget \ @@ -12,7 +12,7 @@ RUN apt-get update && apt-get install -y \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # Install golangci-lint -RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.57.2 && \ +RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.61.0 && \ golangci-lint linters COPY .golangci.yml ${GOPATH}/src/dummy/.golangci.yml @@ -24,7 +24,7 @@ RUN cd ${GOPATH}/src/dummy && \ #################################################################################################### # UI build stage #################################################################################################### -FROM --platform=$BUILDPLATFORM docker.io/library/node:18 as argo-rollouts-ui +FROM --platform=$BUILDPLATFORM docker.io/library/node:18 AS argo-rollouts-ui WORKDIR /src ADD ["ui/package.json", "ui/yarn.lock", "./"] @@ -40,7 +40,7 @@ RUN NODE_ENV='production' yarn build #################################################################################################### # Rollout Controller Build stage which performs the actual build of argo-rollouts binaries #################################################################################################### -FROM --platform=$BUILDPLATFORM golang:1.21 as argo-rollouts-build +FROM --platform=$BUILDPLATFORM golang:1.23 AS argo-rollouts-build WORKDIR /go/src/github.com/argoproj/argo-rollouts @@ -69,7 +69,7 @@ RUN GOOS=$TARGETOS GOARCH=$TARGETARCH make ${MAKE_TARGET} #################################################################################################### # Kubectl plugin image #################################################################################################### -FROM gcr.io/distroless/static-debian11 as kubectl-argo-rollouts +FROM gcr.io/distroless/static-debian11 AS kubectl-argo-rollouts COPY --from=argo-rollouts-build /go/src/github.com/argoproj/argo-rollouts/dist/kubectl-argo-rollouts /bin/kubectl-argo-rollouts diff --git a/Dockerfile.dev b/Dockerfile.dev index 7c3fc3f30c..413f50ec7c 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,7 +1,7 @@ #################################################################################################### # argo-rollouts-dev #################################################################################################### -FROM golang:1.20 as builder +FROM golang:1.20 AS builder RUN apt-get update && apt-get install -y \ ca-certificates && \ @@ -12,9 +12,12 @@ FROM gcr.io/distroless/static-debian11 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY rollouts-controller-linux-amd64 /bin/rollouts-controller +COPY step-plugin-e2e-linux-amd64 /bin/plugin-files/step-plugin-e2e # Use numeric user, allows kubernetes to identify this user as being # non-root when we use a security context with runAsNonRoot: true USER 999 +WORKDIR /home/argo-rollouts + ENTRYPOINT [ "/bin/rollouts-controller" ] diff --git a/Makefile b/Makefile index a17bf33676..7df22e7cce 100644 --- a/Makefile +++ b/Makefile @@ -12,19 +12,26 @@ GIT_TREE_STATE=$(shell if [ -z "`git status --porcelain`" ]; then echo "clean" ; GIT_REMOTE_REPO=upstream VERSION=$(shell if [ ! -z "${GIT_TAG}" ] ; then echo "${GIT_TAG}" | sed -e "s/^v//" ; else cat VERSION ; fi) + +TARGET_ARCH?=linux/amd64 + # docker image publishing options -DOCKER_PUSH=false -IMAGE_TAG=latest +DOCKER_PUSH ?= false +IMAGE_TAG ?= latest # build development images DEV_IMAGE ?= false # E2E variables +E2E_K8S_CONTEXT ?= rancher-desktop E2E_INSTANCE_ID ?= argo-rollouts-e2e E2E_TEST_OPTIONS ?= E2E_PARALLEL ?= 1 -E2E_WAIT_TIMEOUT ?= 120 +E2E_WAIT_TIMEOUT ?= 90 GOPATH ?= $(shell go env GOPATH) +# Global toolchain configuration +NODE_OPTIONS="" + override LDFLAGS += \ -X ${PACKAGE}/utils/version.version=${VERSION} \ -X ${PACKAGE}/utils/version.buildDate=${BUILD_DATE} \ @@ -146,6 +153,9 @@ gen-crd: install-go-tools-local ## generate crd manifests gen-mocks: install-go-tools-local ## generate mock files ./hack/update-mocks.sh +gen-mocks-fast: + ./hack/update-mocks.sh + # generates openapi_generated.go .PHONY: gen-openapi gen-openapi: $(DIST_DIR)/openapi-gen ## generate openapi files @@ -197,10 +207,11 @@ builder-image: ## build builder image .PHONY: image image: ifeq ($(DEV_IMAGE), true) + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/step-plugin-e2e-linux-amd64 ./test/cmd/step-plugin-e2e CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/rollouts-controller-linux-amd64 ./cmd/rollouts-controller - DOCKER_BUILDKIT=1 docker build -t $(IMAGE_PREFIX)argo-rollouts:$(IMAGE_TAG) -f Dockerfile.dev ${DIST_DIR} + DOCKER_BUILDKIT=1 docker build --platform=$(TARGET_ARCH) -t $(IMAGE_PREFIX)argo-rollouts:$(IMAGE_TAG) -f Dockerfile.dev ${DIST_DIR} else - DOCKER_BUILDKIT=1 docker build -t $(IMAGE_PREFIX)argo-rollouts:$(IMAGE_TAG) . + DOCKER_BUILDKIT=1 docker build --platform=$(TARGET_ARCH) -t $(IMAGE_PREFIX)argo-rollouts:$(IMAGE_TAG) . endif @if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)argo-rollouts:$(IMAGE_TAG) ; fi @@ -209,15 +220,19 @@ endif # https://www.jetbrains.com/help/go/attach-to-running-go-processes-with-debugger.html .PHONY: build-sample-metric-plugin-debug build-sample-metric-plugin-debug: ## build sample metric plugin with debug info - go build -gcflags="all=-N -l" -o metric-plugin test/cmd/metrics-plugin-sample/main.go + go build -gcflags="all=-N -l" -o plugin-bin/metric-plugin test/cmd/metrics-plugin-sample/main.go .PHONY: build-sample-traffic-plugin-debug build-sample-traffic-plugin-debug: ## build sample traffic plugin with debug info - go build -gcflags="all=-N -l" -o traffic-plugin test/cmd/trafficrouter-plugin-sample/main.go + go build -gcflags="all=-N -l" -o plugin-bin/traffic-plugin test/cmd/trafficrouter-plugin-sample/main.go + +.PHONY: build-sample-step-plugin-debug +build-sample-step-plugin-debug: ## build sample traffic plugin with debug info + go build -gcflags="all=-N -l" -o plugin-bin/step-plugin test/cmd/step-plugin-sample/main.go .PHONY: plugin-image plugin-image: ## build plugin image - DOCKER_BUILDKIT=1 docker build --target kubectl-argo-rollouts -t $(IMAGE_PREFIX)kubectl-argo-rollouts:$(IMAGE_TAG) . + DOCKER_BUILDKIT=1 docker build --platform=$(TARGET_ARCH) --target kubectl-argo-rollouts -t $(IMAGE_PREFIX)kubectl-argo-rollouts:$(IMAGE_TAG) . if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)kubectl-argo-rollouts:$(IMAGE_TAG) ; fi ##@ Test @@ -230,17 +245,25 @@ test: test-kustomize ## run all tests test-kustomize: ## run kustomize tests ./test/kustomize/test.sh +setup-e2e: + @kubectl apply --context='${E2E_K8S_CONTEXT}' -f manifests/crds/rollout-crd.yaml + @kubectl apply --context='${E2E_K8S_CONTEXT}' -n argo-rollouts -f test/e2e/step-plugin/argo-rollouts-config.yaml + @rm -rf plugin-bin + @go build -gcflags="all=-N -l" -o plugin-bin/e2e-step-plugin test/cmd/step-plugin-e2e/main.go + .PHONY: start-e2e start-e2e: ## start e2e test environment - go run ./cmd/rollouts-controller/main.go --instance-id ${E2E_INSTANCE_ID} --loglevel debug --kloglevel 6 + mkdir -p coverage-output-e2e + GOCOVERDIR=coverage-output-e2e go run -cover ./cmd/rollouts-controller/main.go --instance-id ${E2E_INSTANCE_ID} --loglevel debug --kloglevel 6 .PHONY: test-e2e test-e2e: install-devtools-local - ${DIST_DIR}/gotestsum --rerun-fails-report=rerunreport.txt --junitfile=junit.xml --format=testname --packages="./test/e2e" --rerun-fails=5 -- -timeout 60m -count 1 --tags e2e -p ${E2E_PARALLEL} -parallel ${E2E_PARALLEL} -v --short ./test/e2e ${E2E_TEST_OPTIONS} + ${DIST_DIR}/gotestsum --rerun-fails-report=rerunreport.txt --junitfile=junit-e2e-test.xml --format=testname --packages="./test/e2e" --rerun-fails=5 -- -timeout 60m -count 1 --tags e2e -p ${E2E_PARALLEL} -parallel ${E2E_PARALLEL} -v --short ./test/e2e ${E2E_TEST_OPTIONS} .PHONY: test-unit test-unit: install-devtools-local ## run unit tests - ${DIST_DIR}/gotestsum --junitfile=junit.xml --format=testname -- -covermode=count -coverprofile=coverage.out `go list ./... | grep -v ./test/cmd/metrics-plugin-sample` + mkdir -p coverage-output-unit + ${DIST_DIR}/gotestsum --junitfile=junit-unit-test.xml --format=testname -- `go list ./... | grep -v ./test/cmd/metrics-plugin-sample` -cover -test.gocoverdir=$(CURDIR)/coverage-output-unit .PHONY: coverage diff --git a/USERS.md b/USERS.md index fc8217db6a..211b65ab32 100644 --- a/USERS.md +++ b/USERS.md @@ -1,16 +1,21 @@ ## Who uses Argo Rollouts? + Organizations below are **officially** using Argo Rollouts. Please send a PR with your organization name if you are using Argo Rollouts. +1. [7shifts](https://www.7shifts.com) 1. [Ada](https://www.ada.cx) 1. [ADP](https://www.adp.com) 1. [Akuity](https://akuity.io/) 1. [Alibaba Group](https://www.alibabagroup.com/) +1. [Amadeus IT Group](https://amadeus.com/) 1. [Ambassador Labs](https://www.getambassador.io) 1. [Ant Group](https://www.antgroup.com/) 1. [Bucketplace](https://www.bucketplace.co.kr/) 1. [BukuKas](https://bukukas.co.id/) 1. [Calm](https://www.calm.com/) +1. [CarGurus](https://www.cargurus.com/) 1. [CircleCI](https://circleci.com/) +1. [Cloudflare](https://cloudflare.com/) 1. [Codefresh](https://codefresh.io/) 1. [Credit Karma](https://creditkarma.com/) 1. [DaoCloud](https://daocloud.io) @@ -21,7 +26,9 @@ Organizations below are **officially** using Argo Rollouts. Please send a PR wit 1. [Flipkart](https://flipkart.com) 1. [GetYourGuide](https://www.getyourguide.com) 1. [Gllue](https://gllue.com) +1. [Groww](https://groww.in/) 1. [HashiCorp](https://www.hashicorp.com/) +1. [Hepsiburada](https://hepsiburada.com/) 1. [Ibotta](https://home.ibotta.com/) 1. [Intuit](https://www.intuit.com/) 1. [New Relic](https://newrelic.com/) @@ -35,6 +42,8 @@ Organizations below are **officially** using Argo Rollouts. Please send a PR wit 1. [PagerDuty](https://www.pagerduty.com/) 1. [PayPal](https://www.paypal.com/) 1. [PayPay](https://paypay.ne.jp/) +1. [Plaid](https://plaid.com/) +1. [Priceline](https://priceline.com) 1. [Productboard](https://www.productboard.com) 1. [Quipper](https://www.quipper.com/) 1. [Quizlet](https://quizlet.com) @@ -44,6 +53,9 @@ Organizations below are **officially** using Argo Rollouts. Please send a PR wit 1. [Skillz](https://www.skillz.com) 1. [Spotify](https://www.spotify.com/) 1. [Synamedia](https://www.synamedia.com) +1. [Taboola](https://www.taboola.com) +1. [TBC Bank](https://tbcbank.ge/) +1. [Trustly](https://www.trustly.com/) 1. [Tuhu](https://www.tuhu.cn/) 1. [Twilio SendGrid](https://sendgrid.com) 1. [Ubie](https://ubie.life/) @@ -51,4 +63,3 @@ Organizations below are **officially** using Argo Rollouts. Please send a PR wit 1. [WeLab Bank](https://www.welab.bank/) 1. [Yotpo](https://www.yotpo.com/) 1. [VGS](https://www.vgs.io) -2. [Groww](https://groww.in/) diff --git a/analysis/analysis.go b/analysis/analysis.go index 97497c4c4d..567998149c 100644 --- a/analysis/analysis.go +++ b/analysis/analysis.go @@ -332,7 +332,7 @@ func (c *Controller) runMeasurements(run *v1alpha1.AnalysisRun, tasks []metricTa logger := logutil.WithRedactor(*logutil.WithAnalysisRun(run).WithField("metric", t.metric.Name), secrets) var newMeasurement v1alpha1.Measurement - provider, providerErr := c.newProvider(*logger, t.metric) + provider, providerErr := c.newProvider(*logger, run.Namespace, t.metric) if providerErr != nil { log.Errorf("Error in getting metric provider :%v", providerErr) if t.incompleteMeasurement != nil { @@ -387,17 +387,21 @@ func (c *Controller) runMeasurements(run *v1alpha1.AnalysisRun, tasks []metricTa metricResult.Successful++ metricResult.Count++ metricResult.ConsecutiveError = 0 + metricResult.ConsecutiveSuccess++ case v1alpha1.AnalysisPhaseFailed: metricResult.Failed++ metricResult.Count++ metricResult.ConsecutiveError = 0 + metricResult.ConsecutiveSuccess = 0 case v1alpha1.AnalysisPhaseInconclusive: metricResult.Inconclusive++ metricResult.Count++ metricResult.ConsecutiveError = 0 + metricResult.ConsecutiveSuccess = 0 case v1alpha1.AnalysisPhaseError: metricResult.Error++ metricResult.ConsecutiveError++ + metricResult.ConsecutiveSuccess = 0 logger.Warnf("Measurement had error: %s", newMeasurement.Message) } } @@ -599,11 +603,47 @@ func assessMetricStatus(metric v1alpha1.Metric, result v1alpha1.MetricResult, te return phaseFailureInconclusiveOrError } + // Check if consecutiveSuccessLimit is applicable and was reached. + if metric.ConsecutiveSuccessLimit != nil && metric.ConsecutiveSuccessLimit.IntValue() > 0 && result.ConsecutiveSuccess >= int32(metric.ConsecutiveSuccessLimit.IntValue()) { + logger.Infof("Metric Assessment Result - %s: ConsecutiveSuccessLimit (%s) Reached", v1alpha1.AnalysisPhaseSuccessful, metric.ConsecutiveSuccessLimit.String()) + return v1alpha1.AnalysisPhaseSuccessful + } + // If a count was specified, and we reached that count, then metric is considered Successful. // The Error, Failed, Inconclusive counters are ignored because those checks have already been // taken into consideration above, and we do not want to fail if failures < failureLimit. effectiveCount := metric.EffectiveCount() if effectiveCount != nil && result.Count >= int32(effectiveCount.IntValue()) { + + failureApplicable := (metric.FailureLimit != nil && metric.FailureLimit.IntValue() >= 0) || metric.FailureLimit == nil + successApplicable := metric.ConsecutiveSuccessLimit != nil && metric.ConsecutiveSuccessLimit.IntValue() > 0 + + if failureApplicable && successApplicable { + + // failureLimit was checked above and not reached. + // consecutiveSuccessLimit was checked above and not reached. + + failureLimit := "0" + if metric.FailureLimit != nil { + failureLimit = metric.FailureLimit.String() + } + + logger.Infof("Metric Assessment Result - %s: ConsecutiveSuccessLimit (%s) Not Reached and FailureLimit (%s) Not Violated", v1alpha1.AnalysisPhaseInconclusive, metric.ConsecutiveSuccessLimit.String(), failureLimit) + return v1alpha1.AnalysisPhaseInconclusive + + } else if successApplicable { + + logger.Infof("Metric Assessment Result - %s: ConsecutiveSuccessLimit (%s) Not Reached", v1alpha1.AnalysisPhaseFailed, metric.ConsecutiveSuccessLimit.String()) + return v1alpha1.AnalysisPhaseFailed + + } else if failureApplicable { + // failureLimit was not reached in assessMetricFailureInconclusiveOrError above. + // AnalysisPhaseSuccessful below. + } else { + // This cannot happen, since one of failureLimit or consecutiveSuccessLimit will be applicable + // We validate that failureLimit >= 0 when consecutiveSuccessLimit == 0 + } + logger.Infof("Metric Assessment Result - %s: Count (%s) Reached", v1alpha1.AnalysisPhaseSuccessful, effectiveCount.String()) return v1alpha1.AnalysisPhaseSuccessful } @@ -623,7 +663,8 @@ func assessMetricFailureInconclusiveOrError(metric v1alpha1.Metric, result v1alp if metric.FailureLimit != nil { failureLimit = int32(metric.FailureLimit.IntValue()) } - if result.Failed > failureLimit { + // If failureLimit is negative, that means it isn't applicable. + if failureLimit >= 0 && result.Failed > failureLimit { phase = v1alpha1.AnalysisPhaseFailed message = fmt.Sprintf("failed (%d) > failureLimit (%d)", result.Failed, failureLimit) } @@ -744,7 +785,7 @@ func (c *Controller) garbageCollectMeasurements(run *v1alpha1.AnalysisRun, measu continue } logger := logutil.WithAnalysisRun(run).WithField("metric", metric.Name) - provider, err := c.newProvider(*logger, metric) + provider, err := c.newProvider(*logger, run.Namespace, metric) if err != nil { errors = append(errors, err) continue diff --git a/analysis/analysis_test.go b/analysis/analysis_test.go index 9e7d3fe5f8..07bacd5dd8 100644 --- a/analysis/analysis_test.go +++ b/analysis/analysis_test.go @@ -594,6 +594,208 @@ func TestAssessMetricStatusFailureLimit(t *testing.T) { // max failures assert.Equal(t, v1alpha1.AnalysisPhaseSuccessful, assessMetricStatus(metric, result, true)) } +func TestAssessMetricStatusConsecutiveSuccessLimit(t *testing.T) { + failureLimit := intstr.FromInt(-1) + consecutiveSuccessLimit := intstr.FromInt(3) + metric := v1alpha1.Metric{ + Name: "success-rate", + ConsecutiveSuccessLimit: &consecutiveSuccessLimit, + FailureLimit: &failureLimit, + Interval: "60s", + } + result := v1alpha1.MetricResult{ + Failed: 3, + Successful: 4, + ConsecutiveSuccess: 3, + Count: 7, + Measurements: []v1alpha1.Measurement{{ + Value: "99", + Phase: v1alpha1.AnalysisPhaseFailed, + StartedAt: timePtr(metav1.NewTime(time.Now().Add(-60 * time.Second))), + FinishedAt: timePtr(metav1.NewTime(time.Now().Add(-60 * time.Second))), + }}, + } + + ///////////////////////////////////////////////////////////////////////// + // For indefinite analysis (count is not set) + + // When ConsecutiveSuccess == ConsecutiveSuccessLimit + assert.Equal(t, v1alpha1.AnalysisPhaseSuccessful, assessMetricStatus(metric, result, false)) + assert.Equal(t, v1alpha1.AnalysisPhaseSuccessful, assessMetricStatus(metric, result, true)) + + // When ConsecutiveSuccess < ConsecutiveSuccessLimit + consecutiveSuccessLimit = intstr.FromInt(5) + metric.ConsecutiveSuccessLimit = &consecutiveSuccessLimit + assert.Equal(t, v1alpha1.AnalysisPhaseRunning, assessMetricStatus(metric, result, false)) + assert.Equal(t, v1alpha1.AnalysisPhaseSuccessful, assessMetricStatus(metric, result, true)) + + ///////////////////////////////////////////////////////////////////////// + // For limited analysis (count is >= 1) + + /// When metric.Count is reached + metricCount := intstr.FromInt(7) + metric.Count = &metricCount + + //// ConsecutiveSuccess=3 < ConsecutiveSuccessLimit=5 + assert.Equal(t, v1alpha1.AnalysisPhaseFailed, assessMetricStatus(metric, result, false)) + assert.Equal(t, v1alpha1.AnalysisPhaseFailed, assessMetricStatus(metric, result, true)) + + //// ConsecutiveSuccess = ConsecutiveSuccessLimit = 3 + consecutiveSuccessLimit = intstr.FromInt(3) + metric.ConsecutiveSuccessLimit = &consecutiveSuccessLimit + assert.Equal(t, v1alpha1.AnalysisPhaseSuccessful, assessMetricStatus(metric, result, false)) + assert.Equal(t, v1alpha1.AnalysisPhaseSuccessful, assessMetricStatus(metric, result, true)) + + /// When metric.Count is not reached + metricCount = intstr.FromInt(9) + metric.Count = &metricCount + + //// ConsecutiveSuccess = ConsecutiveSuccessLimit = 3 + assert.Equal(t, v1alpha1.AnalysisPhaseSuccessful, assessMetricStatus(metric, result, false)) + assert.Equal(t, v1alpha1.AnalysisPhaseSuccessful, assessMetricStatus(metric, result, true)) + + //// ConsecutiveSuccess=3 < ConsecutiveSuccessLimit=5 + consecutiveSuccessLimit = intstr.FromInt(5) + metric.ConsecutiveSuccessLimit = &consecutiveSuccessLimit + assert.Equal(t, v1alpha1.AnalysisPhaseRunning, assessMetricStatus(metric, result, false)) + assert.Equal(t, v1alpha1.AnalysisPhaseSuccessful, assessMetricStatus(metric, result, true)) +} + +func TestAssessMetricStatusFailureLimitAndConsecutiveSuccessLimit(t *testing.T) { + failureLimit := intstr.FromInt(4) + consecutiveSuccessLimit := intstr.FromInt(4) + metric := v1alpha1.Metric{ + Name: "success-rate", + ConsecutiveSuccessLimit: &consecutiveSuccessLimit, + FailureLimit: &failureLimit, + Interval: "60s", + } + result := v1alpha1.MetricResult{ + Failed: 3, + Successful: 4, + ConsecutiveSuccess: 3, + Count: 7, + Measurements: []v1alpha1.Measurement{{ + Value: "99", + Phase: v1alpha1.AnalysisPhaseFailed, + StartedAt: timePtr(metav1.NewTime(time.Now().Add(-60 * time.Second))), + FinishedAt: timePtr(metav1.NewTime(time.Now().Add(-60 * time.Second))), + }}, + } + + ///////////////////////////////////////////////////////////////////////// + // For indefinite analysis (count is not set) + + // FailureLimit is not violated and consecutiveSuccessLimit not yet satisfied. + assert.Equal(t, v1alpha1.AnalysisPhaseRunning, assessMetricStatus(metric, result, false)) + assert.Equal(t, v1alpha1.AnalysisPhaseSuccessful, assessMetricStatus(metric, result, true)) + + // FailureLimit is violated and consecutiveSuccessLimit is not yet satisfied. + result.Failed = 5 + result.Successful = 9 + result.Count = 9 + result.ConsecutiveSuccess = 0 + assert.Equal(t, v1alpha1.AnalysisPhaseFailed, assessMetricStatus(metric, result, false)) + assert.Equal(t, v1alpha1.AnalysisPhaseFailed, assessMetricStatus(metric, result, true)) + + // FailureLimit is not violated and consecutiveSuccessLimit is satisfied. + result.Failed = 3 + result.Successful = 5 + result.Count = 8 + result.ConsecutiveSuccess = 4 + assert.Equal(t, v1alpha1.AnalysisPhaseSuccessful, assessMetricStatus(metric, result, false)) + assert.Equal(t, v1alpha1.AnalysisPhaseSuccessful, assessMetricStatus(metric, result, true)) + + // FailureLimit is violated and consecutiveSuccessLimit is satisfied. + result.Failed = 5 + result.Successful = 5 + result.Count = 10 + result.ConsecutiveSuccess = 4 + assert.Equal(t, v1alpha1.AnalysisPhaseFailed, assessMetricStatus(metric, result, false)) + assert.Equal(t, v1alpha1.AnalysisPhaseFailed, assessMetricStatus(metric, result, true)) + + ///////////////////////////////////////////////////////////////////////// + // For limited analysis (count is >= 1) + metricCount := intstr.FromInt(10) + metric.Count = &metricCount + + /// When metric.Count is reached + + //// FailureLimit is not violated and consecutiveSuccessLimit not yet satisfied. + result.Failed = 4 + result.Successful = 6 + result.Count = 10 + result.ConsecutiveSuccess = 3 + + assert.Equal(t, v1alpha1.AnalysisPhaseInconclusive, assessMetricStatus(metric, result, false)) + assert.Equal(t, v1alpha1.AnalysisPhaseInconclusive, assessMetricStatus(metric, result, true)) + + //// FailureLimit is violated and consecutiveSuccessLimit is not yet satisfied. + result.Failed = 5 + result.Successful = 5 + result.Count = 10 + result.ConsecutiveSuccess = 3 + + assert.Equal(t, v1alpha1.AnalysisPhaseFailed, assessMetricStatus(metric, result, false)) + assert.Equal(t, v1alpha1.AnalysisPhaseFailed, assessMetricStatus(metric, result, true)) + + //// FailureLimit is not violated and consecutiveSuccessLimit is satisfied. + result.Failed = 4 + result.Successful = 6 + result.Count = 10 + result.ConsecutiveSuccess = 4 + + assert.Equal(t, v1alpha1.AnalysisPhaseSuccessful, assessMetricStatus(metric, result, false)) + assert.Equal(t, v1alpha1.AnalysisPhaseSuccessful, assessMetricStatus(metric, result, true)) + + //// FailureLimit is violated and consecutiveSuccessLimit is satisfied. + result.Failed = 5 + result.Successful = 5 + result.Count = 10 + result.ConsecutiveSuccess = 4 + + assert.Equal(t, v1alpha1.AnalysisPhaseFailed, assessMetricStatus(metric, result, false)) + assert.Equal(t, v1alpha1.AnalysisPhaseFailed, assessMetricStatus(metric, result, true)) + + /// When metric.Count is not yet reached + + //// FailureLimit is not violated and consecutiveSuccessLimit not yet satisfied. + result.Failed = 3 + result.Successful = 5 + result.Count = 8 + result.ConsecutiveSuccess = 3 + + assert.Equal(t, v1alpha1.AnalysisPhaseRunning, assessMetricStatus(metric, result, false)) + assert.Equal(t, v1alpha1.AnalysisPhaseSuccessful, assessMetricStatus(metric, result, true)) + + //// FailureLimit is violated and consecutiveSuccessLimit is not yet satisfied. + result.Failed = 5 + result.Successful = 3 + result.Count = 8 + result.ConsecutiveSuccess = 3 + + assert.Equal(t, v1alpha1.AnalysisPhaseFailed, assessMetricStatus(metric, result, false)) + assert.Equal(t, v1alpha1.AnalysisPhaseFailed, assessMetricStatus(metric, result, true)) + + //// FailureLimit is not violated and consecutiveSuccessLimit is satisfied. + result.Failed = 3 + result.Successful = 5 + result.Count = 8 + result.ConsecutiveSuccess = 4 + + assert.Equal(t, v1alpha1.AnalysisPhaseSuccessful, assessMetricStatus(metric, result, false)) + assert.Equal(t, v1alpha1.AnalysisPhaseSuccessful, assessMetricStatus(metric, result, true)) + + //// FailureLimit is violated and consecutiveSuccessLimit is satisfied. + result.Failed = 5 + result.Successful = 4 + result.Count = 9 + result.ConsecutiveSuccess = 4 + + assert.Equal(t, v1alpha1.AnalysisPhaseFailed, assessMetricStatus(metric, result, false)) + assert.Equal(t, v1alpha1.AnalysisPhaseFailed, assessMetricStatus(metric, result, true)) +} + func TestAssessMetricStatusInconclusiveLimit(t *testing.T) { inconclusiveLimit := intstr.FromInt(2) metric := v1alpha1.Metric{ @@ -1113,7 +1315,7 @@ func TestGarbageCollectArgResolution(t *testing.T) { defer f.Close() c, _, _ := f.newController(noResyncPeriodFunc) - c.newProvider = func(logCtx log.Entry, metric v1alpha1.Metric) (metric.Provider, error) { + c.newProvider = func(logCtx log.Entry, namespace string, metric v1alpha1.Metric) (metric.Provider, error) { assert.Equal(t, "https://prometheus.kubeaddons:8080", metric.Provider.Prometheus.Address) return f.provider, nil } diff --git a/analysis/controller.go b/analysis/controller.go index 1afa6ab556..804ca7bd34 100644 --- a/analysis/controller.go +++ b/analysis/controller.go @@ -5,12 +5,13 @@ import ( "sync" "time" - "github.com/argoproj/argo-rollouts/metric" - jobProvider "github.com/argoproj/argo-rollouts/metricproviders/job" "github.com/aws/smithy-go/ptr" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "github.com/argoproj/argo-rollouts/metric" + jobProvider "github.com/argoproj/argo-rollouts/metricproviders/job" + unstructuredutil "github.com/argoproj/argo-rollouts/utils/unstructured" log "github.com/sirupsen/logrus" @@ -54,7 +55,7 @@ type Controller struct { metricsServer *metrics.MetricsServer - newProvider func(logCtx log.Entry, metric v1alpha1.Metric) (metric.Provider, error) + newProvider func(logCtx log.Entry, namespace string, metric v1alpha1.Metric) (metric.Provider, error) // used for unit testing enqueueAnalysis func(obj any) diff --git a/analysis/controller_test.go b/analysis/controller_test.go index 601139a6d4..ad6730807c 100644 --- a/analysis/controller_test.go +++ b/analysis/controller_test.go @@ -143,7 +143,7 @@ func (f *fixture) newController(resync resyncFunc) (*Controller, informers.Share c.enqueueAnalysis(obj) } f.provider = &mocks.Provider{} - c.newProvider = func(logCtx log.Entry, metric v1alpha1.Metric) (metric.Provider, error) { + c.newProvider = func(logCtx log.Entry, namespace string, metric v1alpha1.Metric) (metric.Provider, error) { return f.provider, nil } @@ -370,7 +370,7 @@ func TestFailedToCreateProviderError(t *testing.T) { f.objects = append(f.objects, ar) c, i, k8sI := f.newController(noResyncPeriodFunc) - c.newProvider = func(logCtx log.Entry, metric v1alpha1.Metric) (metric.Provider, error) { + c.newProvider = func(logCtx log.Entry, namespace string, metric v1alpha1.Metric) (metric.Provider, error) { return nil, fmt.Errorf("failed to create provider") } diff --git a/cmd/kubectl-argo-rollouts/main.go b/cmd/kubectl-argo-rollouts/main.go index b7d9a02d0a..f81b506ead 100644 --- a/cmd/kubectl-argo-rollouts/main.go +++ b/cmd/kubectl-argo-rollouts/main.go @@ -3,7 +3,6 @@ package main import ( "os" - logutil "github.com/argoproj/argo-rollouts/utils/log" log "github.com/sirupsen/logrus" "k8s.io/cli-runtime/pkg/genericclioptions" _ "k8s.io/client-go/plugin/pkg/client/auth/azure" @@ -11,6 +10,8 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" "k8s.io/klog/v2" + logutil "github.com/argoproj/argo-rollouts/utils/log" + "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/cmd" "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/options" ) diff --git a/cmd/rollouts-controller/main.go b/cmd/rollouts-controller/main.go index c8deab3ea0..370ce55aa9 100644 --- a/cmd/rollouts-controller/main.go +++ b/cmd/rollouts-controller/main.go @@ -2,12 +2,11 @@ package main import ( "fmt" + "net/http" "os" "strings" "time" - "github.com/argoproj/argo-rollouts/metricproviders" - "github.com/argoproj/argo-rollouts/utils/record" "github.com/argoproj/pkg/kubeclientmetrics" smiclientset "github.com/servicemeshinterface/smi-sdk-go/pkg/gen/client/split/clientset/versioned" log "github.com/sirupsen/logrus" @@ -23,6 +22,9 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" "k8s.io/client-go/tools/clientcmd" + "github.com/argoproj/argo-rollouts/metricproviders" + "github.com/argoproj/argo-rollouts/utils/record" + "github.com/argoproj/argo-rollouts/controller" "github.com/argoproj/argo-rollouts/controller/metrics" jobprovider "github.com/argoproj/argo-rollouts/metricproviders/job" @@ -79,6 +81,7 @@ func newCommand() *cobra.Command { printVersion bool selfServiceNotificationEnabled bool controllersEnabled []string + pprofAddress string ) electOpts := controller.NewLeaderElectionOptions() var command = cobra.Command{ @@ -204,6 +207,11 @@ func newCommand() *cobra.Command { ingressWrapper, err := ingressutil.NewIngressWrapper(mode, kubeClient, kubeInformerFactory) checkError(err) + if pprofAddress != "" { + mux := controller.NewPProfServer() + go func() { log.Println(http.ListenAndServe(pprofAddress, mux)) }() + } + var cm *controller.Manager enabledControllers, err := getEnabledControllers(controllersEnabled) @@ -310,6 +318,7 @@ func newCommand() *cobra.Command { command.Flags().DurationVar(&electOpts.LeaderElectionRetryPeriod, "leader-election-retry-period", controller.DefaultLeaderElectionRetryPeriod, "The duration the clients should wait between attempting acquisition and renewal of a leadership. This is only applicable if leader election is enabled.") command.Flags().BoolVar(&selfServiceNotificationEnabled, "self-service-notification-enabled", false, "Allows rollouts controller to pull notification config from the namespace that the rollout resource is in. This is useful for self-service notification.") command.Flags().StringSliceVar(&controllersEnabled, "controllers", nil, "Explicitly specify the list of controllers to run, currently only supports 'analysis', eg. --controller=analysis. Default: all controllers are enabled") + command.Flags().StringVar(&pprofAddress, "enable-pprof-address", "", "Enable pprof profiling on controller by providing a server address.") return &command } diff --git a/controller/controller.go b/controller/controller.go index e9509cd78e..5d7105c281 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -13,9 +13,10 @@ import ( istioutil "github.com/argoproj/argo-rollouts/utils/istio" - rolloutsConfig "github.com/argoproj/argo-rollouts/utils/config" goPlugin "github.com/hashicorp/go-plugin" + rolloutsConfig "github.com/argoproj/argo-rollouts/utils/config" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/dynamic/dynamicinformer" @@ -238,7 +239,7 @@ func NewAnalysisManager( log.Fatalf("Failed to init config: %v", err) } - err = plugin.DownloadPlugins(plugin.FileDownloaderImpl{}) + err = plugin.DownloadPlugins(plugin.FileDownloaderImpl{}, kubeclientset) if err != nil { log.Fatalf("Failed to download plugins: %v", err) } @@ -444,7 +445,7 @@ func NewManager( log.Fatalf("Failed to init config: %v", err) } - err = plugin.DownloadPlugins(plugin.FileDownloaderImpl{}) + err = plugin.DownloadPlugins(plugin.FileDownloaderImpl{}, kubeclientset) if err != nil { log.Fatalf("Failed to download plugins: %v", err) } diff --git a/controller/controller_test.go b/controller/controller_test.go index 0f94131c96..8bd9582fba 100644 --- a/controller/controller_test.go +++ b/controller/controller_test.go @@ -274,6 +274,40 @@ func TestNewManager(t *testing.T) { assert.NotNil(t, cm) } +func TestNewAnalysisManager(t *testing.T) { + f := newFixture(t) + + i := informers.NewSharedInformerFactory(f.client, noResyncPeriodFunc()) + k8sI := kubeinformers.NewSharedInformerFactory(f.kubeclient, noResyncPeriodFunc()) + + scheme := runtime.NewScheme() + listMapping := map[schema.GroupVersionResource]string{} + dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, listMapping) + dynamicInformerFactory := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, 0) + + k8sRequestProvider := &metrics.K8sRequestsCountProvider{} + cm := NewAnalysisManager( + "default", + f.kubeclient, + f.client, + k8sI.Batch().V1().Jobs(), + i.Argoproj().V1alpha1().AnalysisRuns(), + i.Argoproj().V1alpha1().AnalysisTemplates(), + i.Argoproj().V1alpha1().ClusterAnalysisTemplates(), + noResyncPeriodFunc(), + 8090, + 8080, + k8sRequestProvider, + nil, + dynamicInformerFactory, + false, + nil, + nil, + ) + + assert.NotNil(t, cm) +} + func TestPrimaryController(t *testing.T) { f := newFixture(t) diff --git a/controller/metrics/experiment_test.go b/controller/metrics/experiment_test.go index 5694fc9830..9362dab4f7 100644 --- a/controller/metrics/experiment_test.go +++ b/controller/metrics/experiment_test.go @@ -10,9 +10,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/yaml" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" ) const ( diff --git a/controller/metrics/metrics.go b/controller/metrics/metrics.go index 9691b73a40..d06fbbfe37 100644 --- a/controller/metrics/metrics.go +++ b/controller/metrics/metrics.go @@ -2,6 +2,7 @@ package metrics import ( "net/http" + "runtime" "time" "github.com/argoproj/argo-rollouts/utils/defaults" @@ -16,6 +17,7 @@ import ( "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" rolloutlister "github.com/argoproj/argo-rollouts/pkg/client/listers/rollouts/v1alpha1" "github.com/argoproj/argo-rollouts/utils/log" + "github.com/argoproj/argo-rollouts/utils/version" ) type MetricsServer struct { @@ -39,6 +41,16 @@ const ( MetricsPath = "/metrics" ) +var ( + buildInfo = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "build_info", + Help: "A metric with a constant '1' value labeled by version from which Argo-Rollouts was built.", + }, + []string{"version", "goversion", "goarch", "commit"}, + ) +) + type ServerConfig struct { Addr string RolloutLister rolloutlister.RolloutLister @@ -74,6 +86,9 @@ func NewMetricsServer(cfg ServerConfig) *MetricsServer { reg.MustRegister(MetricNotificationFailedTotal) reg.MustRegister(MetricNotificationSend) reg.MustRegister(MetricVersionGauge) + reg.MustRegister(buildInfo) + + recordBuildInfo() mux.Handle(MetricsPath, promhttp.HandlerFor(prometheus.Gatherers{ // contains app controller specific metrics @@ -174,6 +189,12 @@ func (m *MetricsServer) Remove(namespace string, name string, kind string) { } +// recordBuildInfo publishes information about Argo-Rollouts version and runtime info through an info metric (gauge). +func recordBuildInfo() { + vers := version.GetVersion() + buildInfo.WithLabelValues(vers.Version, runtime.Version(), runtime.GOARCH, vers.GitCommit).Set(1) +} + func boolFloat64(b bool) float64 { if b { return 1 diff --git a/controller/metrics/metrics_test.go b/controller/metrics/metrics_test.go index ced10b2442..43a52230f9 100644 --- a/controller/metrics/metrics_test.go +++ b/controller/metrics/metrics_test.go @@ -9,11 +9,12 @@ import ( "testing" "time" - "github.com/argoproj/argo-rollouts/utils/defaults" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/cache" + "github.com/argoproj/argo-rollouts/utils/defaults" + "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/fake" informerfactory "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions" logutil "github.com/argoproj/argo-rollouts/utils/log" diff --git a/controller/metrics/prommetrics.go b/controller/metrics/prommetrics.go index c0d3b89f62..e2e5aaaac3 100644 --- a/controller/metrics/prommetrics.go +++ b/controller/metrics/prommetrics.go @@ -1,8 +1,9 @@ package metrics import ( - "github.com/argoproj/argo-rollouts/utils/version" "github.com/prometheus/client_golang/prometheus" + + "github.com/argoproj/argo-rollouts/utils/version" ) // Follow Prometheus naming practices diff --git a/controller/metrics/rollout_test.go b/controller/metrics/rollout_test.go index 17f86b011f..9cb0013822 100644 --- a/controller/metrics/rollout_test.go +++ b/controller/metrics/rollout_test.go @@ -10,10 +10,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/yaml" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" - "github.com/argoproj/argo-rollouts/utils/conditions" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/utils/conditions" ) const ( diff --git a/controller/profiling.go b/controller/profiling.go new file mode 100644 index 0000000000..cf5af5e60f --- /dev/null +++ b/controller/profiling.go @@ -0,0 +1,21 @@ +package controller + +import ( + "net/http" + "net/http/pprof" +) + +// NewPProfServer returns a new pprof server to gather runtime profiling data +func NewPProfServer() *http.ServeMux { + mux := http.NewServeMux() + + // TODO: Remove enumerating all pprof endpoints if/when a more ergonomic + // attachment solution is introduced. See: https://github.com/golang/go/issues/71213. + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline/", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile/", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol/", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace/", pprof.Trace) + + return mux +} diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index f727834d8b..a73c76d557 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -1,24 +1,27 @@ # Contributing + ## Before You Start + Argo Rollouts is written in Golang. If you do not have a good grounding in Go, try out [the tutorial](https://tour.golang.org/). ## Pre-requisites + Install: -* [docker](https://docs.docker.com/install/#supported-platforms) -* [golang](https://golang.org/) -* [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) -* [kustomize](https://github.com/kubernetes-sigs/kustomize/releases) >= 4.5.5 -* [k3d](https://k3d.io/) recommended +- [docker](https://docs.docker.com/install/#supported-platforms) +- [golang](https://golang.org/) +- [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) +- [kustomize](https://github.com/kubernetes-sigs/kustomize/releases) >= 4.5.5 +- [k3d](https://k3d.io/) recommended Kustomize is required for unit tests (`make test` is using it), so you [must install it](https://kubectl.docs.kubernetes.io/installation/kustomize/) locally if you wish to make code contributions to Argo Rollouts. Argo Rollout additionally uses the following tools -* `golangci-lint` to lint the project. -* `protoc` and `swagger-codegen` to generate proto related files -* `yarn` to build the UI +- `golangci-lint` to lint the project. +- `protoc` and `swagger-codegen` to generate proto related files +- `yarn` to build the UI Run the following commands to install them: @@ -56,10 +59,9 @@ cd ~/go/src/github.com/argoproj/argo-rollouts The `make controller` command will build the controller. -* `make install-tools-local` - Runs scripts to install codegen utility CLIs necessary for codegen. - -* `make codegen` - Runs the code generator that creates the informers, client, lister, and deepcopies from the types.go and modifies the open-api spec. +- `make install-tools-local` - Runs scripts to install codegen utility CLIs necessary for codegen. +- `make codegen` - Runs the code generator that creates the informers, client, lister, and deepcopies from the types.go and modifies the open-api spec. ## Running Controller Locally @@ -83,11 +85,7 @@ make test ## Running E2E tests The end-to-end tests need to run against a kubernetes cluster with the Argo Rollouts controller -running. The rollout controller can be started with the command: - -``` -make start-e2e -``` +running. Start and prepare your cluster for e2e tests: @@ -98,6 +96,12 @@ kubectl apply -k manifests/crds kubectl apply -f test/e2e/crds ``` +The rollout controller can be started with the command: + +``` +make start-e2e +``` + Then run the e2e tests: ``` @@ -112,9 +116,10 @@ E2E_TEST_OPTIONS="-run 'TestCanarySuite' -testify.m 'TestCanaryScaleDownOnAbortN ## Running the UI -If you'd like to run the UI locally, you first need a running Rollouts controller. This can be a locally running controller with a k3d cluster, as described above, or a controller running in a remote Kubernetes cluster. +If you'd like to run the UI locally, you first need a running Rollouts controller. This can be a locally running controller with a k3d cluster, as described above, or a controller running in a remote Kubernetes cluster. In order for the local React app to communicate with the controller and Kubernetes API, run the following to open a port forward to the dashboard: + ```bash kubectl argo rollouts dashboard ``` @@ -127,12 +132,30 @@ make plugin ``` In another terminal, run the following to start the UI: + ```bash cd ui yarn install yarn start ``` +## Getting your feature accepted + +To be eligible for inclusion in a minor release, a new feature must meet the following criteria before the release’s RC +date. + +If it is a large feature that involves significant design decisions, that feature must be described in a Proposal. + +The feature PR must include: + +* Tests (passing) +* Documentation +* If necessary, a note in the Upgrading docs for the planned minor release +* The PR must be reviewed, approved, and merged by an Approver. + +If these criteria are not met by the RC date, the feature will be ineligible for inclusion in the RC series or GA for +that minor release. It will have to wait for the next minor release. + ## Controller architecture Argo Rollouts is actually a collection of individual controllers @@ -142,11 +165,11 @@ that handle a specific aspect of Progressive Delivery. The controllers are: -* [Rollout Controller](https://github.com/argoproj/argo-rollouts/blob/master/rollout/controller.go) -* [Service Controller](https://github.com/argoproj/argo-rollouts/blob/master/service/service.go) -* [Ingress Controller](https://github.com/argoproj/argo-rollouts/blob/master/ingress/ingress.go) -* [Experiment Controller](https://github.com/argoproj/argo-rollouts/blob/master/experiments/controller.go) -* [AnalysisRun Controller](https://github.com/argoproj/argo-rollouts/blob/master/analysis/controller.go) +- [Rollout Controller](https://github.com/argoproj/argo-rollouts/blob/master/rollout/controller.go) +- [Service Controller](https://github.com/argoproj/argo-rollouts/blob/master/service/service.go) +- [Ingress Controller](https://github.com/argoproj/argo-rollouts/blob/master/ingress/ingress.go) +- [Experiment Controller](https://github.com/argoproj/argo-rollouts/blob/master/experiments/controller.go) +- [AnalysisRun Controller](https://github.com/argoproj/argo-rollouts/blob/master/analysis/controller.go) ### Tips @@ -158,36 +181,35 @@ KUBECONFIG=~/.kube/minikube make test-e2e ``` 2. To run a specific e2e test, set the `E2E_TEST_OPTIONS` environment variable to specify the test -(or test regex): + (or test regex): ```shell make test-e2e E2E_TEST_OPTIONS="-testify.m ^TestRolloutRestart$" ``` 3. The e2e tests are designed to run as quickly as possible, eliminating readiness and termination -delays. However, it is often desired to artificially slow down the tests for debugging purposes, -as well as to understand what the test is doing. To delay startup and termination of pods, set the -`E2E_POD_DELAY` to an integer value in seconds. This environment variable is often coupled with -`E2E_TEST_OPTIONS` to debug and slow down a specific test. + delays. However, it is often desired to artificially slow down the tests for debugging purposes, + as well as to understand what the test is doing. To delay startup and termination of pods, set the + `E2E_POD_DELAY` to an integer value in seconds. This environment variable is often coupled with + `E2E_TEST_OPTIONS` to debug and slow down a specific test. ```shell make test-e2e E2E_POD_DELAY=10 ``` 4. Increasing the timeout. The E2E tests time out waiting on conditions to be met within 60 seconds. -If debugging the rollout controller, it may be useful to increase this timeout while say sitting -at a debugger breakpoint: + If debugging the rollout controller, it may be useful to increase this timeout while say sitting + at a debugger breakpoint: ```shell make test-e2e E2E_WAIT_TIMEOUT=999999 ``` - 5. The e2e tests leverage a feature of the controller allowing the controller to be sharded with -a user-specific "instance id" label. This allows the tests to operate only on rollouts with the -specified label, and prevents any other controllers (including the system rollout controller), -from also operating on the same set of rollouts. This value can be changed (from the default of -`argo-rollouts-e2e`), using the `E2E_INSTANCE_ID` environment variable: + a user-specific "instance id" label. This allows the tests to operate only on rollouts with the + specified label, and prevents any other controllers (including the system rollout controller), + from also operating on the same set of rollouts. This value can be changed (from the default of + `argo-rollouts-e2e`), using the `E2E_INSTANCE_ID` environment variable: ```shell make start-e2e E2E_INSTANCE_ID=foo @@ -200,8 +222,7 @@ Alternatively, the e2e tests can be run against the system controller (i.e. with make start-e2e E2E_INSTANCE_ID='' ``` - -6. Working on CRDs? While editing them directly works when you are finding the shape of things you want, the final CRDs are autogenerated. Make sure to regenerate them before submitting PRs. They are controlled by the relevant annotations in the types file: +6. Working on CRDs? While editing them directly works when you are finding the shape of things you want, the final CRDs are autogenerated. Make sure to regenerate them by running `make gen-crd` before submitting PRs. They are controlled by the relevant annotations in the types file: eg: Analysis Templates are controlled by annotations in `pkg/apis/rollouts/v1alpha1/analysis_types.go`. @@ -242,14 +263,16 @@ kubectl -n argo-rollouts apply -f manifests/install.yaml ``` ## Upgrading Kubernetes Libraries + Argo Rollouts has a dependency on the kubernetes/kubernetes repo for some of the functionality that has not been pushed into the other kubernetes repositories yet. In order to import the kubernetes/kubernetes repo, all of the associated repos have to pinned to the correct version specified by the kubernetes/kubernetes release. The `./hack/update-k8s-dependencies.sh` updates all the dependencies to the those correct versions. ## Upgrading Notifications Engine -Argo Rollouts has a dependency on the [argoproj/notifications-engines](https://github.com/argoproj/notifications-engine) repo -for the notifications functionality and related documentation. + +Argo Rollouts has a dependency on the [argoproj/notifications-engines](https://github.com/argoproj/notifications-engine) repo +for the notifications functionality and related documentation. This is updated by upgrading the Go library in `go.mod` by running the commands: diff --git a/docs/FAQ.md b/docs/FAQ.md index 861e4b3a80..caa9df475f 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -1,5 +1,7 @@ # FAQ +Be sure to read the [Best practices page](../best-practices) as well. + ## General ### Does Argo Rollouts depend on Argo CD or any other Argo project? diff --git a/docs/analysis/datadog.md b/docs/analysis/datadog.md index 7bca70be10..fe58c1565c 100644 --- a/docs/analysis/datadog.md +++ b/docs/analysis/datadog.md @@ -41,6 +41,48 @@ stringData: `apiVersion` here is different from the `apiVersion` from the Datadog configuration above. +!!! important + ###### Namespaced secret + Datadog integration supports referring to secrets inside the same namespace as argo-rollouts (by default) + or referring to a secret in the same namespace as the `AnalysisTemplate`. + + To use a secret from the `AnalysisTemplate` namespace, include a `secretRef` section in the template, specifying the `name` of the secret and setting the `namespaced` property to `true`. + + The process for retrieving Datadog credentials is as follows: + 1. **If a `secretRef` is defined in the `AnalysisTemplate`:** Argo Rollouts will search for the secret with the specified name in the namespace where the template resides. + 2. **If the secret is not found in the specified namespace:** Argo Rollouts will then check the environment variables. + 3. **If the credentials are not found in environment variables:** Argo Rollouts will look for a secret named "Datadog" in the namespace where Argo Rollouts itself is deployed. + +--- + +Let me know if there's anything else you'd like to adjust! + + ```yaml + apiVersion: argoproj.io/v1alpha1 + kind: AnalysisTemplate + metadata: + name: loq-error-rate + spec: + args: + - name: service-name + metrics: + - name: error-rate + interval: 5m + successCondition: result <= 0.01 + failureLimit: 3 + provider: + datadog: + apiVersion: v2 + interval: 5m + secretRef: + name: "mysecret" + namespaced: true + query: | + sum:requests.error.rate{service:{{args.service-name}}} + ``` + + + ### Working with Datadog API v2 !!! important diff --git a/docs/analysis/job.md b/docs/analysis/job.md index eae1e077ca..27c16db66f 100644 --- a/docs/analysis/job.md +++ b/docs/analysis/job.md @@ -4,7 +4,7 @@ A Kubernetes Job can be used to run analysis. When a Job is used, the metric is successful if the Job completes and had an exit code of zero, otherwise it is failed. ```yaml - metrics: +metrics: - name: test provider: job: @@ -18,8 +18,19 @@ successful if the Job completes and had an exit code of zero, otherwise it is fa template: spec: containers: - - name: test - image: my-image:latest - command: [my-test-script, my-service.default.svc.cluster.local] + - name: test + image: my-image:latest + command: + [my-test-script, my-service.default.svc.cluster.local] restartPolicy: Never ``` + +## Control where the jobs run + +Argo Rollouts allows you some control over where your metric job runs. + +The following env vars can be set on the Rollouts controller: + +`ARGO_ROLLOUTS_ANALYSIS_JOB_NAMESPACE` will allow you to run your metric jobs in a namespace other than the default (which can vary depending on if you are running Rollouts in cluster mode or not). + +`ARGO_ROLLOUTS_ANALYSIS_JOB_KUBECONFIG` will allow running metric jobs in a different cluster entirely. This should be a path to the kubeconfig you want to use. diff --git a/docs/analysis/newrelic.md b/docs/analysis/newrelic.md index 81afad9b75..38416c6e2b 100644 --- a/docs/analysis/newrelic.md +++ b/docs/analysis/newrelic.md @@ -21,6 +21,7 @@ spec: profile: my-newrelic-secret # optional, defaults to 'newrelic' query: | FROM Transaction SELECT percentage(count(*), WHERE httpResponseCode != 500) as successRate where appName = '{{ args.application-name }}' + timeout: 10 # NRQL query timeout in seconds. Optional, defaults to 5 ``` The `result` evaluated for the condition will always be map or list of maps. The name will follow the pattern of either `function` or `function.field`, e.g. `SELECT average(duration) from Transaction` will yield `average.duration`. In this case the field result cannot be accessed with dot notation and instead should be accessed like `result['average.duration']`. Query results can be renamed using the [NRQL clause `AS`](https://docs.newrelic.com/docs/query-your-data/nrql-new-relic-query-language/get-started/nrql-syntax-clauses-functions#sel-as) as seen above. @@ -54,3 +55,11 @@ stringData: base-url-rest: base-url-nerdgraph: ``` + +## Additional Metadata + +The New Relic provider returns the below metadata under the `Metadata` map in the `MetricsResult` object of `AnalysisRun`. + +| KEY | Description | +|-----------------------|-------------| +| ResolvedNewRelicQuery | Resolved query after substituting the template's arguments | diff --git a/docs/analysis/plugins.md b/docs/analysis/plugins.md index a7fd159a4a..e1356bf447 100644 --- a/docs/analysis/plugins.md +++ b/docs/analysis/plugins.md @@ -1,14 +1,16 @@ # Metric Plugins -!!! important - Available since v1.5 - Status: Alpha +!!! warning "Alpha Feature (Since 1.5.0)" -Argo Rollouts supports getting analysis metrics via 3rd party plugin system. This allows users to extend the capabilities of Rollouts + This is an experimental, [alpha-quality](https://github.com/argoproj/argoproj/blob/main/community/feature-status.md#alpha) + feature that allows you to support metric providers that are not natively supported. + +Argo Rollouts supports getting analysis metrics via 3rd party [plugin system](../plugins.md). This allows users to extend the capabilities of Rollouts to support metric providers that are not natively supported. Rollout's uses a plugin library called [go-plugin](https://github.com/hashicorp/go-plugin) to do this. You can find a sample plugin here: [rollouts-plugin-metric-sample-prometheus](https://github.com/argoproj-labs/rollouts-plugin-metric-sample-prometheus) -## Using a Metric Plugin +## Installing There are two methods of installing and using an argo rollouts plugin. The first method is to mount up the plugin executable into the rollouts controller container. The second method is to use a HTTP(S) server to host the plugin executable. @@ -18,9 +20,9 @@ into the rollouts controller container. The second method is to use a HTTP(S) se There are a few ways to mount the plugin executable into the rollouts controller container. Some of these will depend on your particular infrastructure. Here are a few methods: -* Using an init container to download the plugin executable -* Using a Kubernetes volume mount with a shared volume such as NFS, EBS, etc. -* Building the plugin into the rollouts controller container +- Using an init container to download the plugin executable +- Using a Kubernetes volume mount with a shared volume such as NFS, EBS, etc. +- Building the plugin into the rollouts controller container Then you can use the configmap to point to the plugin executable file location. Example: @@ -37,6 +39,19 @@ data: ### Using a HTTP(S) server to host the plugin executable +!!! warning "Installing a plugin with http(s)" + + Depending on which method you use to install and the plugin, there are some things to be aware of. + The rollouts controller will not start if it can not download or find the plugin executable. This means that if you are using + a method of installation that requires a download of the plugin and the server hosting the plugin for some reason is not available and the rollouts + controllers pod got deleted while the server was down or is coming up for the first time, it will not be able to start until + the server hosting the plugin is available again. + + Argo Rollouts will download the plugin at startup only once but if the pod is deleted it will need to download the plugin again on next startup. Running + Argo Rollouts in HA mode can help a little with this situation because each pod will download the plugin at startup. So if a single pod gets + deleted during a server outage, the other pods will still be able to take over because there will already be a plugin executable available to it. It is the + responsibility of the Argo Rollouts administrator to define the plugin installation method considering the risks of each approach. + Argo Rollouts supports downloading the plugin executable from a HTTP(S) server. To use this method, you will need to configure the controller via the `argo-rollouts-config` configmap and set `pluginLocation` to a http(s) url. Example: @@ -50,8 +65,21 @@ data: - name: "argoproj-labs/sample-prometheus" # name of the plugin, it must match the name required by the plugin so it can find it's configuration location: "https://github.com/argoproj-labs/rollouts-plugin-metric-sample-prometheus/releases/download/v0.0.4/metric-plugin-linux-amd64" # supports http(s):// urls and file:// sha256: "dac10cbf57633c9832a17f8c27d2ca34aa97dd3d" #optional sha256 checksum of the plugin executable + headersFrom: #optional headers for the download via http request + - secretRef: + name: secret-name +--- +apiVersion: v1 +kind: Secret +metadata: + name: secret-name +stringData: + Authorization: Basic + My-Header: value ``` + + ## Some words of caution Depending on which method you use to install and the plugin, there are some things to be aware of. @@ -67,8 +95,13 @@ responsibility of the Argo Rollouts administrator to define the plugin installat ## List of Available Plugins (alphabetical order) -#### Add Your Plugin Here - * If you have created a plugin, please submit a PR to add it to this list. -#### [rollouts-plugin-metric-sample-prometheus](https://github.com/argoproj-labs/rollouts-plugin-metric-sample-prometheus) - * This is just a sample plugin that can be used as a starting point for creating your own plugin. -It is not meant to be used in production. It is based on the built-in prometheus provider. +If you have created a plugin, please submit a PR to add it to this list. + +### [rollouts-plugin-metric-opensearch](https://github.com/argoproj-labs/rollouts-plugin-metric-opensearch) + +- The application is an OpenSearch plugin designed for use with the Argo Rollouts plugin system. This plugin enables the integration of OpenSearch metrics into Argo Rollouts, allowing for advanced metric analysis and monitoring during application rollouts. + +### [rollouts-plugin-metric-sample-prometheus](https://github.com/argoproj-labs/rollouts-plugin-metric-sample-prometheus) + +- This is just a sample plugin that can be used as a starting point for creating your own plugin. + It is not meant to be used in production. It is based on the built-in prometheus provider. diff --git a/docs/analysis/prometheus.md b/docs/analysis/prometheus.md index cb93bf0e5d..0fb2fb6a65 100644 --- a/docs/analysis/prometheus.md +++ b/docs/analysis/prometheus.md @@ -39,6 +39,47 @@ you validate your [PromQL expression](https://prometheus.io/docs/prometheus/late See the [Analysis Overview page](../../features/analysis) for more details on the available options. +## Range queries + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: AnalysisTemplate +metadata: + name: range-query-example +spec: + args: + - name: service-name + - name: lookback-duration + value: 5m + metrics: + - name: success-rate + # checks that all returned values are under 1000ms + successCondition: "all(result, # < 1000)" + failureLimit: 3 + provider: + prometheus: + rangeQuery: + # See https://expr-lang.org/docs/language-definition#date-functions + # for value date functions + # The start point to query from + start: 'now() - duration("{{args.lookback-duration}}")' + # The end point to query to + end: 'now()' + # Query resolution width + step: 1m + address: http://prometheus.example.com:9090 + query: http_latency_ms{service="{{args.service-name}}"} +``` + +### Range query and successCondition/failureCondition + +Since range queries will usually return multiple values from prometheus. It is important to assert on every value returned. See the following examples: + +* ❌ `result[0] < 1000` - this will only check the first value returned +* ✅ `all(result, # < 1000)` - checks every value returns from prometheus + +See [expr](https://github.com/expr-lang/expr) for more expression options. + ## Authorization ### Utilizing Amazon Managed Prometheus diff --git a/docs/assets/versions.js b/docs/assets/versions.js index 1336443d1a..fe0aeb1d3c 100644 --- a/docs/assets/versions.js +++ b/docs/assets/versions.js @@ -1,15 +1,40 @@ -setTimeout(function() { - const callbackName = 'callback_' + new Date().getTime(); - window[callbackName] = function (response) { - const div = document.createElement('div'); - div.innerHTML = response.html; - document.querySelector(".md-header__inner > .md-header__title").appendChild(div); - const container = div.querySelector('.rst-versions'); - var caret = document.createElement('div'); - caret.innerHTML = "" - caret.classList.add('dropdown-caret') - div.querySelector('.rst-current-version').appendChild(caret); +const targetNode = document.querySelector('.md-header__inner'); +const observerOptions = { + childList: true, + subtree: true +}; + +const observerCallback = function(mutationsList, observer) { + for (let mutation of mutationsList) { + if (mutation.type === 'childList') { + const titleElement = document.querySelector('.md-header__inner > .md-header__title'); + if (titleElement) { + initializeVersionDropdown(); + observer.disconnect(); + } + } } +}; + +const observer = new MutationObserver(observerCallback); +observer.observe(targetNode, observerOptions); + +function initializeVersionDropdown() { + const callbackName = 'callback_' + new Date().getTime(); + window[callbackName] = function(response) { + const div = document.createElement('div'); + div.innerHTML = response.html; + document.querySelector(".md-header__inner > .md-header__title").appendChild(div); + const container = div.querySelector('.rst-versions'); + var caret = document.createElement('div'); + caret.innerHTML = ""; + caret.classList.add('dropdown-caret'); + div.querySelector('.rst-current-version').appendChild(caret); + + div.querySelector('.rst-current-version').addEventListener('click', function() { + container.classList.toggle('shift-up'); + }); + }; var CSSLink = document.createElement('link'); CSSLink.rel='stylesheet'; @@ -20,6 +45,31 @@ setTimeout(function() { script.src = 'https://argo-rollouts.readthedocs.io/_/api/v2/footer_html/?'+ 'callback=' + callbackName + '&project=argo-rollouts&page=&theme=mkdocs&format=jsonp&docroot=docs&source_suffix=.md&version=' + (window['READTHEDOCS_DATA'] || { version: 'latest' }).version; document.getElementsByTagName('head')[0].appendChild(script); -}, 0); - +} +// VERSION WARNINGS +window.addEventListener("DOMContentLoaded", function() { + var currentVersion = window.location.href.match(/\/en\/(release-(?:v\d+|\w+)|latest|stable)\//); + var margin = 30; + var headerHeight = document.getElementsByClassName("md-header")[0].offsetHeight; + if (currentVersion && currentVersion.length > 1) { + currentVersion = currentVersion[1]; + if (currentVersion === "latest") { + document.querySelector("div[data-md-component=announce]").innerHTML = "
You are viewing the docs for an unreleased version of Argo Rollouts, click here to go to the latest stable version.
"; + var bannerHeight = document.getElementById('announce-msg').offsetHeight + margin; + document.querySelector("header.md-header").style.top = bannerHeight + "px"; + document.querySelector('style').textContent += + "@media screen and (min-width: 76.25em){ .md-sidebar { height: 0; top:" + (bannerHeight + headerHeight) + "px !important; }}"; + document.querySelector('style').textContent += + "@media screen and (min-width: 60em){ .md-sidebar--secondary { height: 0; top:" + (bannerHeight + headerHeight) + "px !important; }}"; + } else if (currentVersion !== "stable") { + document.querySelector("div[data-md-component=announce]").innerHTML = "
You are viewing the docs for a previous version of Argo Rollouts, click here to go to the latest stable version.
"; + var bannerHeight = document.getElementById('announce-msg').offsetHeight + margin; + document.querySelector("header.md-header").style.top = bannerHeight + "px"; + document.querySelector('style').textContent += + "@media screen and (min-width: 76.25em){ .md-sidebar { height: 0; top:" + (bannerHeight + headerHeight) + "px !important; }}"; + document.querySelector('style').textContent += + "@media screen and (min-width: 60em){ .md-sidebar--secondary { height: 0; top:" + (bannerHeight + headerHeight) + "px !important; }}"; + } + } +}); \ No newline at end of file diff --git a/docs/best-practices.md b/docs/best-practices.md index 2f7908f4ba..0ed32efbec 100644 --- a/docs/best-practices.md +++ b/docs/best-practices.md @@ -29,7 +29,23 @@ You should *NOT* use Argo Rollouts for preview/ephemeral environments. For that The recommended way to use Argo Rollouts is for brief deployments that take 15-20 minutes or maximum 1-2 hours. If you want to run new versions for days or weeks before deciding to promote, then Argo Rollouts is probably not the best solution for you. -Also, if you want to run a wave of multiple versions at the same time (i.e. have 1.1 and 1.2 and 1.3 running at the same time), know that Argo Rollouts was not designed for this scenario. +Keeping parallel releases for long times, complicates the deployment process a lot and opens several questions where different people have different views on how Argo Rollouts should work. + +For example let's say that you are testing for a week version 1.3 as stable and 1.4 as preview. +Then somebody deploys 1.5 + +1. Some people believe that the new state should be 1.3 stable and 1.5 as preview +1. Some people believe that the new state should be 1.4 stable and 1.5 as preview + +Currently Argo Rollouts follows the first approach, under the assumption that something was really wrong with 1.4 and 1.5 is the hotfix. + +And then let's say that 1.5 has an issue. Some people believe that Argo rollouts should "rollback" to 1.3 while other people think it should rollback to 1.4 + +Currently Argo Rollouts assumes that the version to rollback is always 1.3 regardless of how many "hotfixes" have been previewed in-between. + +All these problems are not present if you make the assumption that each release stays active only for a minimal time and you always create one new version when the previous one has finished. + +Also, if you want to run a wave of multiple versions at the same time (i.e. have 1.1 and 1.2 and 1.3 running at the same time), know that Argo Rollouts was not designed for this scenario. Argo Rollouts always works with the assumption that there is one stable/previous version and one preview/next version. A version that has just been promoted is assumed to be ready for production and has already passed all your tests (either manual or automated). @@ -41,6 +57,8 @@ While Argo Rollouts supports manual promotions and other manual pauses, these ar Ideally you should have proper metrics that tell you in 5-15 minutes if a deployment is successful or not. If you don't have those metrics, then you will miss a lot of value from Argo Rollouts. +If you are doing a deployment right now and then have an actual human looking at logs/metrics/traces for the next 2 hours, adopting Argo Rollouts is not going to help you a lot with automated deployments. + Get your [metrics](../features/analysis) in place first and test them with dry-runs before applying them to production deployments. @@ -139,3 +157,34 @@ One user of Argo Rollouts saw a 27% reduction in memory usage for a cluster with 1290 rollouts by changing `RevisionHistoryLimit` from 10 to 0. + +## Rollout a ConfigMap change + +Argo Rollouts is meant to work on a Kubernetes Deployment. When a ConfigMap is mounted inside one the Deployment container and a change occurs inside the ConfigMap, it won't trigger a new Rollout by default. + +One technique to trigger the Rollout it to name dynamically the ConfigMap. +For example, adding a hash of its content at the end of the name: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-config-7270e14e6 +``` + +Each time a change occurs in the ConfigMap, its name will change in the Deployment reference as well, triggering a Rollout. + +However, it's not enough to perform correctly progressive rollouts, as the old ConfigMap might get deleted as soon as the new one is created. This would prevent Experiments and rollbacks in case of rollout failure to work correctly. + +While no magical solution exist to work aroud that, you can tweak your deployment tool to remove the ConfigMap only when the Rollout is completed successfully. + +Example with Argo CD: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-config-7270e14e6 + annotations: + argocd.argoproj.io/sync-options: PruneLast=true +``` diff --git a/docs/concepts.md b/docs/concepts.md index 26cd7f5cef..6f8312cdbf 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -3,7 +3,7 @@ ## Rollout A Rollout is Kubernetes workload resource which is equivalent to a Kubernetes Deployment object. -It is intended to replace a Deployment object in scenarios when more advanced deployment or +It is intended to replace a Deployment object in scenarios when more advanced deployment or progressive delivery functionality is needed. A Rollout provides the following features which a Kubernetes Deployment cannot: @@ -17,17 +17,17 @@ a Kubernetes Deployment cannot: Progressive delivery is the process of releasing updates of a product in a controlled and gradual manner, thereby reducing the risk of the release, typically coupling automation and metric analysis -to drive the automated promotion or rollback of the update. +to drive the automated promotion or rollback of the update. Progressive delivery is often described as an evolution of continuous delivery, extending the speed benefits made in CI/CD to the deployment process. This is accomplished by limiting the -exposure of the new version to a subset of users, observing and analyzing for correct behavior, +exposure of the new version to a subset of users, observing and analyzing for correct behavior, then progressively increasing the exposure to a broader and wider audience while continuously verifying correctness. ## Deployment Strategies -While the industry has used a consistent terminology to describe various deployment strategies, the implementations of these strategies tend to differ across tooling. To make it clear how the Argo Rollouts will behave, here are the descriptions of the various deployment strategies implementations offered by the Argo Rollouts. +While the industry has used a consistent terminology to describe various deployment strategies, the implementations of these strategies tend to differ across tooling. To make it clear how the Argo Rollouts will behave, here are the descriptions of the various deployment strategy implementations. Argo Rollouts only supports Blue-Green and Canary. ### Rolling Update A `RollingUpdate` slowly replaces the old version with the new version. As the new version comes up, the old version is scaled down in order to maintain the overall count of the application. This is the default strategy of the Deployment object. @@ -46,4 +46,33 @@ A Canary deployment exposes a subset of users to the new version of the applicat [![How Canary deployments work](concepts-assets/canary-deployments.png)](concepts-assets/canary-deployments.png) The picture above shows a canary with two stages (10% and 33% of traffic goes to new version) but this is just an example. With Argo Rollouts you can define the exact number of stages -and percentages of traffic according to your use case. \ No newline at end of file +and percentages of traffic according to your use case. + +## Which strategy to choose + +In general Blue/Green is the easier strategy to start with, but also the more limited. We recommend you start with Blue/Green deployments first and as you gain confidence for your metrics and applications switch to Canaries. + +You also need to examine if your application can handle canaries or not. + +* Blue/Green always works because only one application is active at a time. Not all applications can have different versions running in parallel at the same time (which is what canaries are doing). This can be a showstopper for adopting canary deployments especially for legacy applications. +* Blue/Green is simpler because you can get their full value WITHOUT a traffic manager. While canaries can also work without a traffic manager, most of their advanced features assume a fine-grained way to control traffic. If you don't have a traffic manager, then you can easily get the full value +of blue/green deployments but only the basic capabilities of canaries. +* Blue/Green also works with services that use queues and databases (workers that fetch tasks). Argo Rollouts doesn't control traffic flow for +connections it doesn't understand (i.e. binary/queue channels). + +Here is a summary table for the possible approaches. + +| | Blue/Green | Basic Canary | Canary with Traffic manager | +|--------------------------:|:-------------------------:|:--------------------------:| :-----------------------------:| +| Adoption Complexity | Low | Medium | High | +| Flexibility | Low | High | Maximum | +| Needs traffic provider | No | No | Yes | +| Works with queue workers | Yes | No | No | +| Works with shared/locked resources | Yes | No | No | +| Traffic switch | All or nothing | Gradual percentage | Gradual percentage | +| Traffic control | 0% or 100% | coarse grained | fine grained | +| Traffic depends on | deployment state | number of canary pods | Any split option is possible | +| Advanced routing scenarios | No | No | Yes | +| Failure Blast Radius | Massive impact | Low impact | Low impact | + +Note that that traffic manager can be any compatible Service Mesh or Ingress Controller or Gateway API implementation (via a plugin). diff --git a/docs/features/analysis.md b/docs/features/analysis.md index 4029164425..3afbb42b68 100644 --- a/docs/features/analysis.md +++ b/docs/features/analysis.md @@ -154,8 +154,7 @@ spec: )) ``` -Multiple measurements can be performed over a longer duration period, by specifying the `count` and -`interval` fields: +Multiple measurements can be performed over a longer duration period, by specifying the `count` and `interval` fields: ```yaml hl_lines="4 5" metrics: @@ -169,6 +168,13 @@ Multiple measurements can be performed over a longer duration period, by specify query: ... ``` +!!! note + The `count` can have 0 as value which means that it will run until + the end of the Rollout execution for background analysis (outside + of steps). However if the `count` has value 0 and the analysis is + defined in the steps, the analysis won't be executed. + + ## ClusterAnalysisTemplates !!! important @@ -482,9 +488,9 @@ Or without additional metrics: clusterScope: false ``` - The result in the AnalysisRun will have the aggregation of metrics of each template: +The result in the AnalysisRun will have the aggregation of metrics of each template: - === "AnalysisRun" +=== "AnalysisRun" ```yaml # NOTE: Generated AnalysisRun from a single template referencing several templates @@ -753,6 +759,67 @@ The entire analysis run is considered as Failed after three failed measurements. )) ``` +## ConsecutiveSuccessLimit and FailureLimit + +!!! important + `consecutiveSuccessLimit` available since v1.8 + +You can use either `failureLimit` to define a limit for the number of failures before the analysis is considered failed, `consecutiveSuccessLimit` to define the required consecutive number of successes for the analysis to succeed, or both together. One of them has to be applicable (i.e. not disabled, see below for more), otherwise a validation error is thrown. + +To disable: + + * `failureLimit`, set the field to `-1`. + * `consecutiveSuccessLimit`, set the field to `0` (the default value). + +The default value for both is `0`, the meaning of which differs for each one of them. A value of `0` for `failureLimit` means its logic _is_ applicable and no failures are tolerated. However, a value of `0` for `consecutiveSuccessLimit` means it's inapplicable or disabled. + +Let's go through each case and show what the behavior would look like. + +### Only FailureLimit applicable + +The behavior is shown above in the [Failure Conditions and Failure Limit](#failure-conditions-and-failure-limit) section. This is the default behavior if you set nothing of the two fields (with `failureLimit` having a default value of `0`, so no failures are tolerated). + + +### Only ConsecutiveSuccessLimit applicable + +To have this behavior, you need to have something like +```yaml +failureLimit: -1 +consecutiveSuccessLimit: 4 # Any value > 0 +``` + +This behavior is essentially waiting for a condition to hold, or an event to happen. That is, keep measuring a metric and keep failing until you measure `N` consecutive successful measurements, at which point the analysis concludes successfully. This can be useful as an event-driven way of promoting a rollout when used in an inline analysis. + + +### Both FailureLimit and ConsecutiveSuccessLimit applicable + +To have this behavior, you need to have something like +```yaml +failureLimit: 3 # Any value >= 0 +consecutiveSuccessLimit: 4 # Any value > 0 +``` + +The behavior is simply waiting to measure `N` consecutive successful measurements, _while_ being limited by the number of overall failures specified by `failureLimit`. Above, we need to have at most 3 failures before we get 4 consecutive successful measurements for the analysis to be considered successful. + + +In case of an analysis that has `count` specified (that is, runs for a specific amount of time) and that `count` is reached, the evaluation of success is as follows: + + * `failureLimit` is violated and `consecutiveSuccessLimit` is satisfied: Failure. + * `failureLimit` is violated and `consecutiveSuccessLimit` is not satisfied: Failure. + * `failureLimit` is not violated and `consecutiveSuccessLimit` is satisfied: Success. + * `failureLimit` is not violated and `consecutiveSuccessLimit` is not satisfied: Inconclusive State. + +As illustrated, `failureLimit` takes priority if violated. However, if neither is violated/satisfied, the analysis reaches an inconclusive state. + + +!!! note + When terminating analyses prematurely, they are always terminated successfully, unless it happens that `failureLimit` is enabled and violated, then they terminate in failure. `consecutiveSuccessLimit`, if enabled, doesn't affect the termination status. + + For more clarity, examples of analyses terminated "prematurely": + + * A background analysis with `count` not specified when terminated at the end of the rollout. + * Any analysis with `count` specified and not yet reached when the rollout is aborted. + ## Dry-Run Mode !!! important diff --git a/docs/features/bluegreen.md b/docs/features/bluegreen.md index cc3e819adf..8002ae7b4a 100644 --- a/docs/features/bluegreen.md +++ b/docs/features/bluegreen.md @@ -1,6 +1,6 @@ # BlueGreen Deployment Strategy -A Blue Green Deployment allows users to reduce the amount of time multiple versions running at the same time. +A Blue Green Deployment allows users to reduce the amount of time multiple versions are running at the same time. ## Overview @@ -93,7 +93,7 @@ The AutoPromotionEnabled will make the rollout automatically promote the new Rep Defaults to true ### autoPromotionSeconds -The AutoPromotionSeconds will make the rollout automatically promote the new ReplicaSet to active Service after the AutoPromotionSeconds time has passed since the rollout has entered a paused state. If the `AutoPromotionEnabled` field is set to false, this field will be ignored +Setting a positive non-zero value here would make the rollout automatically promote the new `ReplicaSet` to active Service after this much time has been elapsed since the rollout has entered a paused state. If the `AutoPromotionEnabled` field is set to **false**, this field would be ignored. Defaults to nil diff --git a/docs/features/canary.md b/docs/features/canary/index.md similarity index 81% rename from docs/features/canary.md rename to docs/features/canary/index.md index 4e3dab50c8..d14967ae69 100644 --- a/docs/features/canary.md +++ b/docs/features/canary/index.md @@ -1,15 +1,19 @@ # Canary Deployment Strategy + A canary rollout is a deployment strategy where the operator releases a new version of their application to a small percentage of the production traffic. ## Overview + Since there is no agreed upon standard for a canary deployment, the rollouts controller allows users to outline how they want to run their canary deployment. Users can define a list of steps the controller uses to manipulate the ReplicaSets when there is a change to the `.spec.template`. Each step will be evaluated before the new ReplicaSet is promoted to the stable version, and the old version is completely scaled down. -Each step can have one of two fields. The `setWeight` field dictates the percentage of traffic that should be sent to the canary, and the `pause` struct instructs the rollout to pause. When the controller reaches a `pause` step for a rollout, it will add a `PauseCondition` struct to the `.status.PauseConditions` field. If the `duration` field within the `pause` struct is set, the rollout will not progress to the next step until it has waited for the value of the `duration` field. Otherwise, the rollout will wait indefinitely until that Pause condition is removed. By using the `setWeight` and the `pause` fields, a user can declaratively describe how they want to progress to the new version. Below is an example of a canary strategy. +There are multiple steps available, the most basic ones are `setWeight` and `pause`. The `setWeight` field dictates the percentage of traffic that should be sent to the canary, and the `pause` step instructs the rollout to pause. When the controller reaches a `pause` step for a rollout, it will add a `PauseCondition` struct to the `.status.PauseConditions` field. If the `duration` field within the `pause` struct is set, the rollout will not progress to the next step until it has waited for the value of the `duration` field. Otherwise, the rollout will wait indefinitely until that Pause condition is removed. By using the `setWeight` and the `pause` fields, a user can declaratively describe how they want to progress to the new version. Below is an example of a canary strategy. !!! important - If the canary Rollout does not use [traffic management](traffic-management/index.md), the Rollout makes a best effort attempt to achieve the percentage listed in the last `setWeight` step between the new and old version. For example, if a Rollout has 10 Replicas and 10% for the first `setWeight` step, the controller will scale the new desired ReplicaSet to 1 replicas and the old stable ReplicaSet to 9. In the case where the setWeight is 41%, the Rollout attempts to get there by finding the whole number with the smallest delta, rounding up the calculation if the deltas are equals (i.e. the new ReplicaSet has 4 pods since 41% of 10 is closer to 4/10 than 5/10, and the old ReplicaSet has 6 pods). If a user wants to have more fine-grained control of the percentages without a large number of Replicas, that user should use the [traffic management](#trafficrouting) functionality. + + If the canary Rollout does not use [traffic management](traffic-management/index.md), the Rollout makes a best effort attempt to achieve the percentage listed in the last `setWeight` step between the new and old version. For example, if a Rollout has 10 Replicas and 10% for the first `setWeight` step, the controller will scale the new desired ReplicaSet to 1 replicas and the old stable ReplicaSet to 9. In the case where the setWeight is 41%, the Rollout attempts to get there by finding the whole number with the smallest delta, rounding up the calculation if the deltas are equals (i.e. the new ReplicaSet has 4 pods since 41% of 10 is closer to 4/10 than 5/10, and the old ReplicaSet has 6 pods). If a user wants to have more fine-grained control of the percentages without a large number of Replicas, that user should use the [traffic management](#trafficrouting) functionality. ## Example + ```yaml apiVersion: argoproj.io/v1alpha1 kind: Rollout @@ -26,25 +30,26 @@ spec: app: nginx spec: containers: - - name: nginx - image: nginx:1.15.4 - ports: - - containerPort: 80 + - name: nginx + image: nginx:1.15.4 + ports: + - containerPort: 80 minReadySeconds: 30 revisionHistoryLimit: 3 strategy: canary: #Indicates that the rollout should use the Canary strategy - maxSurge: "25%" + maxSurge: '25%' maxUnavailable: 0 steps: - - setWeight: 10 - - pause: - duration: 1h # 1 hour - - setWeight: 20 - - pause: {} # pause indefinitely + - setWeight: 10 + - pause: + duration: 1h # 1 hour + - setWeight: 20 + - pause: {} # pause indefinitely ``` ## Pause Duration + Pause duration can be specified with an optional time unit suffix. Valid time units are "s", "m", "h". Defaults to "s" if not specified. ```yaml @@ -52,11 +57,11 @@ spec: strategy: canary: steps: - - pause: { duration: 10 } # 10 seconds + - pause: { duration: 10 } # 10 seconds - pause: { duration: 10s } # 10 seconds - pause: { duration: 10m } # 10 minutes - pause: { duration: 10h } # 10 hours - - pause: {} # pause indefinitely + - pause: {} # pause indefinitely ``` If no `duration` is specified for a pause step, the rollout will be paused indefinitely. To unpause, use the [argo kubectl plugin](kubectl-plugin.md) `promote` command. @@ -81,31 +86,31 @@ match the traffic weight. Some use cases for this: the canary, while setWeight is still set to 0. 3. You wish to scale the canary up to 100%, in order to facilitate traffic shadowing. - !!! important + Setting canary scale is only available when using the canary strategy with a traffic router, since the basic canary needs to control canary scale in order to approximate canary weight. To control canary scales and weights during steps, use the `setCanaryScale` step and indicate which scale the the canary should use: -* explicit replica count without changing traffic weight (`replicas`) -* explicit weight percentage of total spec.replicas without changing traffic weight(`weight`) -* to or not to match current canary's `setWeight` step (`matchTrafficWeight: true or false`) +- explicit replica count without changing traffic weight (`replicas`) +- explicit weight percentage of total spec.replicas without changing traffic weight(`weight`) +- to or not to match current canary's `setWeight` step (`matchTrafficWeight: true or false`) ```yaml spec: strategy: canary: steps: - # explicit count - - setCanaryScale: - replicas: 3 - # a percentage of spec.replicas - - setCanaryScale: - weight: 25 - # matchTrafficWeight returns to the default behavior of matching the canary traffic weight - - setCanaryScale: - matchTrafficWeight: true + # explicit count + - setCanaryScale: + replicas: 3 + # a percentage of spec.replicas + - setCanaryScale: + weight: 25 + # matchTrafficWeight returns to the default behavior of matching the canary traffic weight + - setCanaryScale: + matchTrafficWeight: true ``` When using `setCanaryScale` with explicit values for either replicas or weight, one must be careful @@ -119,12 +124,12 @@ spec: strategy: canary: steps: - # 1 canary pod (10% of spec.replicas) - - setCanaryScale: - weight: 10 - # 90% of traffic to the 1 canary pod - - setWeight: 90 - - pause: {} + # 1 canary pod (10% of spec.replicas) + - setCanaryScale: + weight: 10 + # 90% of traffic to the 1 canary pod + - setWeight: 90 + - pause: {} ``` The above situation is caused by the changed behvaior of `setWeight` after `setCanaryScale`. To reset, set `matchTrafficWeight: true` and the `setWeight` behavior will be restored, i.e., subsequent `setWeight` will create canary replicas matching the traffic weight. @@ -132,6 +137,7 @@ The above situation is caused by the changed behvaior of `setWeight` after `setC ## Dynamic Stable Scale (with Traffic Routing) !!! important + Available since v1.1 When using traffic routing, by default the stable ReplicaSet is left scaled to 100% during the update. @@ -167,11 +173,12 @@ spec: abortScaleDownDelaySeconds: 600 ``` - ## Mimicking Rolling Update + If the `steps` field is omitted, the canary strategy will mimic the rolling update behavior. Similar to the deployment, the canary strategy has the `maxSurge` and `maxUnavailable` fields to configure how the Rollout should progress to the new version. ## Other Configurable Features + Here are the optional fields that will modify the behavior of canary strategy: ```yaml @@ -188,36 +195,43 @@ spec: ``` ### analysis + Configure the background [Analysis](analysis.md) to execute during the rollout. If the analysis is unsuccessful the rollout will be aborted. Defaults to nil ### antiAffinity + Check out the [Anti Affinity document](anti-affinity/anti-affinity.md) document for more information. Defaults to nil ### canaryService + `canaryService` references a Service that will be modified to send traffic to only the canary ReplicaSet. This allows users to only hit the canary ReplicaSet. Defaults to an empty string ### stableService + `stableService` the name of a Service which selects pods with stable version and doesn't select any pods with canary version. This allows users to only hit the stable ReplicaSet. Defaults to an empty string ### maxSurge + `maxSurge` defines the maximum number of replicas the rollout can create to move to the correct ratio set by the last setWeight. Max Surge can either be an integer or percentage as a string (i.e. "20%") Defaults to "25%". ### maxUnavailable + The maximum number of pods that can be unavailable during the update. Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). This can not be 0 if MaxSurge is 0. Defaults to 25% ### trafficRouting + The [traffic management](traffic-management/index.md) rules to apply to control the flow of traffic between the active and canary versions. If not set, the default weighted pod replica based routing will be used. Defaults to nil diff --git a/docs/features/canary/plugins.md b/docs/features/canary/plugins.md new file mode 100644 index 0000000000..a344801c40 --- /dev/null +++ b/docs/features/canary/plugins.md @@ -0,0 +1,168 @@ +# Canary Step Plugins + +!!! warning "Alpha Feature (Since 1.8.0)" + + This is an experimental, [alpha-quality](https://github.com/argoproj/argoproj/blob/main/community/feature-status.md#alpha) + feature that allows you to execute plugins during canary steps. + +Argo Rollouts supports getting step plugins via 3rd party [plugin system](../../plugins.md). This allows users to extend the capabilities of Rollouts +to support executing arbitrary steps during the canary. Rollout's uses a plugin library called +[go-plugin](https://github.com/hashicorp/go-plugin) to do this. + +## Installing + +There are two methods of installing and using an Argo Rollouts plugin. The first method is to mount up the plugin executable +into the rollouts controller container. The second method is to use a HTTP(S) server to host the plugin executable. + +### Mounting the plugin executable into the rollouts controller container + +There are a few ways to mount the plugin executable into the rollouts controller container. Some of these will depend on your +particular infrastructure. Here are a few methods: + +- Using an init container to download the plugin executable +- Using a Kubernetes volume mount with a shared volume such as NFS, EBS, etc. +- Building the plugin into the rollouts controller container + +Then you can use the ConfigMap to point to the plugin executable file location. Example: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: argo-rollouts-config +data: + stepPlugins: |- + - name: "argoproj-labs/sample-step" # name of the plugin, it must match the name required by the plugin so it can find it's configuration + location: "file://./my-custom-plugin" # supports http(s):// urls and file:// +``` + +### Using a HTTP(S) server to host the plugin executable + +!!! warning "Installing a plugin with http(s)" + + Depending on which method you use to install and the plugin, there are some things to be aware of. + The rollouts controller will not start if it can not download or find the plugin executable. This means that if you are using + a method of installation that requires a download of the plugin and the server hosting the plugin for some reason is not available and the rollouts + controllers pod got deleted while the server was down or is coming up for the first time, it will not be able to start until + the server hosting the plugin is available again. + + Argo Rollouts will download the plugin at startup only once but if the pod is deleted it will need to download the plugin again on next startup. Running + Argo Rollouts in HA mode can help a little with this situation because each pod will download the plugin at startup. So if a single pod gets + deleted during a server outage, the other pods will still be able to take over because there will already be a plugin executable available to it. It is the + responsibility of the Argo Rollouts administrator to define the plugin installation method considering the risks of each approach. + +Argo Rollouts supports downloading the plugin executable from a HTTP(S) server. To use this method, you will need to +configure the controller via the `argo-rollouts-config` ConfigMap and set `pluginLocation` to a http(s) url. Example: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: argo-rollouts-config +data: + stepPlugins: |- + - name: "argoproj-labs/sample-nginx" # name of the plugin, it must match the name required by the plugin so it can find it's configuration + location: "https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-sample-nginx/releases/download/v0.0.1/metric-plugin-linux-amd64" # supports http(s):// urls and file:// + sha256: "08f588b1c799a37bbe8d0fc74cc1b1492dd70b2c" # optional sha256 checksum of the plugin executable +``` + +### Disabling a plugin + +A step plugin that will execute during your Rollouts will fail the canary deployment whenever there is an unhandled error. +If a step plugin is used in multiple rollouts and is suddenly unstable, none of the rollouts will be able to progress. +To make the plugin less disruptive and the upgrades easier, you can use the `disabled` flag in the plugin configuration to +disable it globally. This will skip the plugin execution in every Rollout where it is configured, and progress to the next canary step. + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: argo-rollouts-config +data: + stepPlugins: |- + - name: "argoproj-labs/sample-nginx" + location: "https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-sample-nginx/releases/download/v0.0.1/metric-plugin-linux-amd64" + disabled: true # Skip all canary steps using this plugin because it may be faulty. +``` + +## Usage + +You can execute a configured step plugin at any point during your canary steps. +The plugin will be executed and the rollout step will be progressing until the plugin execution returns a status of +`Successful`, `Failed` or `Error`. Once completed, the rollout will progress to the next configured step. + +For the available plugin `config`, refer to each specific plugin documentation. + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: example-plugin-ro +spec: + strategy: + canary: + steps: + - plugin: + name: argoproj-labs/step-exec + config: + command: echo "hello world" +``` + +### Plugin Statuses + +To know the result of your plugin, you can use the `status.stepPluginStatuses[]` property to find the status that correspond to +your execution. Each status item is unique by `index`, `name` and `operation`. The `operation` can be one of the following: + +- `Run`: The main operation that execute the plugin. +- `Terminate`: The operation called on your plugin when the `Run` operation is still ongoing, but your rollout is aborted. +- `Abort`: The operation called on your plugin when it is aborted. This will be called for every `Successful` `Run` operation. + +## Implementation + +As a plugin developer, your step plugin should follow some conventions to make it predictable and easier to use. + +### Run operation + +The run operation is the method called on your plugin when executed. The operation can be called +**multiple times**. It is the responsibility of the plugin's implementation to validate if the desired +plugin actions were already taken or not. + +#### Long-running operations + +If the plugin needs to execute an operation that may take a long time, or poll for a result, it can return +early with a `Running` phase and a `RequeueAfter` duration. The controller will requeue the rollout and call the `Run` operation +again after the `RequeueAfter` has expired. The `Status` property on the return object can hold any information that would be +necessary to retrieve the state in subsequent executions. + +### Terminate operation + +If the `Run` operation returns with a `Running` phase and the rollout needs to cancel the execution, the controller will call the plugin's terminate method +with the state of the ongoing `Running` operation. The plugin can use this method to cancel any ongoing information. +This is often called if the rollout is fully promoted during a plugin execution. + +If the terminate operation has an error and fails, it will not be retried. The plugin should have a mechanism to cancel +suspiciously long-running operations if necessary. + +### Abort operation + +The abort operation will be called whenever a rollout is aborted and plugin step `Run` operation was `Successful` or currently `Running`. +The operation will be called in the reverse execution order with the existing state of the operation it is aborting. + +If the abort operation has an error and fails, it will not be retried. The plugin should have a mechanism to cancel +suspiciously long-running operations if necessary. + +### Returning errors + +The plugin can return an error for unhandled operations. In that case, the rollout will handle that error and apply a +backoff mechanism to retry the execution of the plugin until it returns `Successful` or `Failed` phase. When an error happens, the +`Status` returned by the plugin is not persisted, allowing it to retry later on the last known valid status. + +The controller will keep retrying until it succeeds, or the rollout is aborted. + +## List of Available Plugins (alphabetical order) + +If you have created a plugin, please submit a PR to add it to this list. + +### [plugin-name](#plugin-name) + +- Brief plugin description diff --git a/docs/features/ephemeral-metadata.md b/docs/features/ephemeral-metadata.md index 0a2d0f50e9..346aac31d8 100644 --- a/docs/features/ephemeral-metadata.md +++ b/docs/features/ephemeral-metadata.md @@ -12,7 +12,7 @@ One use case is for a Rollout to label or annotate the desired/stable pods with labels/annotations, for _only_ the duration which they are the desired or stable set, and for the labels to be updated/removed as soon as the ReplicaSet switches roles (e.g. from desired to stable). The use case which this enables, is to allow prometheus, wavefront, datadog queries and dashboards -to be built, which can rely on a consistent labels, rather than the `rollouts-pod-template-hash` +to be built, which can rely on a consistent list of labels, rather than the `rollouts-pod-template-hash` which is unpredictable and changing from revision to revision. A Rollout using the canary strategy has the ability to attach ephemeral metadata to the stable or diff --git a/docs/features/kustomize.md b/docs/features/kustomize.md index e1a691b511..568d9d2a78 100644 --- a/docs/features/kustomize.md +++ b/docs/features/kustomize.md @@ -28,6 +28,13 @@ configurations: - https://argoproj.github.io/argo-rollouts/features/kustomize/rollout-transform.yaml ``` +- With Kustomize 5 it is possible to reference the configuration directly from a remote resource: + +```yaml +configurations: + - https://argoproj.github.io/argo-rollouts/features/kustomize/rollout-transform-kustomize-v5.yaml +``` + - With Kustomize 4.5.5 kustomize can use kubernetes OpenAPI data to get merge key and patch strategy information about [resource types](https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/openapi). For example, given the following rollout: ```yaml diff --git a/docs/features/kustomize/rollout-transform-kustomize-v5.yaml b/docs/features/kustomize/rollout-transform-kustomize-v5.yaml new file mode 100644 index 0000000000..2084a04554 --- /dev/null +++ b/docs/features/kustomize/rollout-transform-kustomize-v5.yaml @@ -0,0 +1,274 @@ +# https://github.com/kubernetes-sigs/kustomize/blob/master/api/konfig/builtinpluginconsts/namereference.go +nameReference: +- kind: ConfigMap + version: v1 + fieldSpecs: + - path: spec/template/spec/volumes/configMap/name + kind: Rollout + - path: spec/template/spec/containers/env/valueFrom/configMapKeyRef/name + kind: Rollout + - path: spec/template/spec/initContainers/env/valueFrom/configMapKeyRef/name + kind: Rollout + - path: spec/template/spec/containers/envFrom/configMapRef/name + kind: Rollout + - path: spec/template/spec/initContainers/envFrom/configMapRef/name + kind: Rollout + - path: spec/template/spec/volumes/projected/sources/configMap/name + kind: Rollout + - path: spec/templates/template/spec/volumes/configMap/name + kind: Experiment + - path: spec/templates/template/spec/containers/env/valueFrom/configMapKeyRef/name + kind: Experiment + - path: spec/templates/template/spec/initContainers/env/valueFrom/configMapKeyRef/name + kind: Experiment + - path: spec/templates/template/spec/containers/envFrom/configMapRef/name + kind: Experiment + - path: spec/templates/template/spec/initContainers/envFrom/configMapRef/name + kind: Experiment + - path: spec/templates/template/spec/volumes/projected/sources/configMap/name + kind: Experiment + - path: spec/metrics/provider/job/spec/template/spec/volumes/configMap/name + kind: AnalysisTemplate + - path: spec/metrics/provider/job/spec/template/spec/containers/env/valueFrom/configMapKeyRef/name + kind: AnalysisTemplate + - path: spec/metrics/provider/job/spec/template/spec/initContainers/env/valueFrom/configMapKeyRef/name + kind: AnalysisTemplate + - path: spec/metrics/provider/job/spec/template/spec/containers/envFrom/configMapRef/name + kind: AnalysisTemplate + - path: spec/metrics/provider/job/spec/template/spec/initContainers/envFrom/configMapRef/name + kind: AnalysisTemplate + - path: spec/metrics/provider/job/spec/template/spec/volumes/projected/sources/configMap/name + kind: AnalysisTemplate +- kind: Secret + version: v1 + fieldSpecs: + - path: spec/template/spec/volumes/secret/secretName + kind: Rollout + - path: spec/template/spec/containers/env/valueFrom/secretKeyRef/name + kind: Rollout + - path: spec/template/spec/initContainers/env/valueFrom/secretKeyRef/name + kind: Rollout + - path: spec/template/spec/containers/envFrom/secretRef/name + kind: Rollout + - path: spec/template/spec/initContainers/envFrom/secretRef/name + kind: Rollout + - path: spec/template/spec/imagePullSecrets/name + kind: Rollout + - path: spec/template/spec/volumes/projected/sources/secret/name + kind: Rollout + - path: spec/templates/template/spec/volumes/secret/secretName + kind: Experiment + - path: spec/templates/template/spec/containers/env/valueFrom/secretKeyRef/name + kind: Experiment + - path: spec/templates/template/spec/initContainers/env/valueFrom/secretKeyRef/name + kind: Experiment + - path: spec/templates/template/spec/containers/envFrom/secretRef/name + kind: Experiment + - path: spec/templates/template/spec/initContainers/envFrom/secretRef/name + kind: Experiment + - path: spec/templates/template/spec/imagePullSecrets/name + kind: Experiment + - path: spec/templates/template/spec/volumes/projected/sources/secret/name + kind: Experiment + - path: spec/metrics/provider/job/spec/template/spec/volumes/secret/secretName + kind: AnalysisTemplate + - path: spec/metrics/provider/job/spec/template/spec/containers/env/valueFrom/secretKeyRef/name + kind: AnalysisTemplate + - path: spec/metrics/provider/job/spec/template/spec/initContainers/env/valueFrom/secretKeyRef/name + kind: AnalysisTemplate + - path: spec/metrics/provider/job/spec/template/spec/containers/envFrom/secretRef/name + kind: AnalysisTemplate + - path: spec/metrics/provider/job/spec/template/spec/initContainers/envFrom/secretRef/name + kind: AnalysisTemplate + - path: spec/metrics/provider/job/spec/template/spec/imagePullSecrets/name + kind: AnalysisTemplate + - path: spec/metrics/provider/job/spec/template/spec/volumes/projected/sources/secret/name + kind: AnalysisTemplate +- kind: ServiceAccount + version: v1 + fieldSpecs: + - path: spec/template/spec/serviceAccountName + kind: Rollout + - path: spec/templates/template/spec/serviceAccountName + kind: Experiment + - path: spec/metrics/provider/job/spec/template/spec/serviceAccountName + kind: AnalysisTemplate +- kind: PersistentVolumeClaim + version: v1 + fieldSpecs: + - path: spec/template/spec/volumes/persistentVolumeClaim/claimName + kind: Rollout + - path: spec/templates/template/spec/volumes/persistentVolumeClaim/claimName + kind: Experiment + - path: spec/metrics/provider/job/spec/template/spec/volumes/persistentVolumeClaim/claimName + kind: AnalysisTemplate +- kind: PriorityClass + version: v1 + group: scheduling.k8s.io + fieldSpecs: + - path: spec/template/spec/priorityClassName + kind: Rollout + - path: spec/templates/template/spec/priorityClassName + kind: Experiment + - path: spec/metrics/provider/job/spec/template/spec/priorityClassName + kind: AnalysisTemplate + +# The name references below are unique to Rollouts and not applicable to Deployment +- kind: Service + version: v1 + fieldSpecs: + - path: spec/strategy/blueGreen/activeService + kind: Rollout + - path: spec/strategy/blueGreen/previewService + kind: Rollout + - path: spec/strategy/canary/canaryService + kind: Rollout + - path: spec/strategy/canary/stableService + kind: Rollout + - path: spec/strategy/canary/trafficRouting/alb/rootService + kind: Rollout +- kind: VirtualService + group: networking.istio.io + fieldSpecs: + - path: spec/strategy/canary/trafficRouting/istio/virtualService/name + kind: Rollout +- kind: DestinationRule + group: networking.istio.io + fieldSpecs: + - path: spec/strategy/canary/trafficRouting/istio/destinationRule/name + kind: Rollout +- kind: Ingress + group: networking.k8s.io + fieldSpecs: + - path: spec/strategy/canary/trafficRouting/alb/ingress + kind: Rollout + - path: spec/strategy/canary/trafficRouting/nginx/stableIngress + kind: Rollout +- kind: Ingress + group: extensions + fieldSpecs: + - path: spec/strategy/canary/trafficRouting/alb/ingress + kind: Rollout + - path: spec/strategy/canary/trafficRouting/nginx/stableIngress + kind: Rollout +- kind: AnalysisTemplate + group: argoproj.io + fieldSpecs: + - path: spec/strategy/blueGreen/prePromotionAnalysis/templates/templateName + kind: Rollout + - path: spec/strategy/blueGreen/postPromotionAnalysis/templates/templateName + kind: Rollout + - path: spec/strategy/canary/analysis/templates/templateName + kind: Rollout + - path: spec/strategy/canary/steps/analysis/templates/templateName + kind: Rollout + - path: spec/strategy/canary/steps/experiment/analyses/templateName + kind: Rollout + - path: spec/analyses/templateName + kind: Experiment +- kind: Rollout + fieldSpecs: + - path: spec/scaleTargetRef/name + kind: HorizontalPodAutoscaler +- kind: Deployment + version: v1 + group: apps + fieldSpecs: + - path: spec/workloadRef/name + kind: Rollout +- kind: Mapping + group: getambassador.io + fieldSpecs: + - path: spec/strategy/canary/trafficRouting/ambassador/mappings + kind: Rollout + +# https://github.com/kubernetes-sigs/kustomize/blob/master/api/konfig/builtinpluginconsts/commonlabels.go +commonLabels: +- path: spec/selector/matchLabels + create: true + kind: Rollout +- path: spec/template/metadata/labels + create: true + kind: Rollout +- path: spec/template/spec/affinity/podAffinity/preferredDuringSchedulingIgnoredDuringExecution/podAffinityTerm/labelSelector/matchLabels + create: false + kind: Rollout +- path: spec/template/spec/affinity/podAffinity/requiredDuringSchedulingIgnoredDuringExecution/labelSelector/matchLabels + create: false + kind: Rollout +- path: spec/template/spec/affinity/podAntiAffinity/preferredDuringSchedulingIgnoredDuringExecution/podAffinityTerm/labelSelector/matchLabels + create: false + kind: Rollout +- path: spec/template/spec/affinity/podAntiAffinity/requiredDuringSchedulingIgnoredDuringExecution/labelSelector/matchLabels + create: false + kind: Rollout + +templateLabels: +- path: spec/template/metadata/labels + create: true + kind: Rollout + +# https://github.com/kubernetes-sigs/kustomize/blob/master/api/konfig/builtinpluginconsts/commonannotations.go +commonAnnotations: +- path: spec/template/metadata/annotations + create: true + kind: Rollout + +# https://github.com/kubernetes-sigs/kustomize/blob/master/api/konfig/builtinpluginconsts/varreference.go +varReference: +- path: spec/template/spec/containers/args + kind: Rollout +- path: spec/template/spec/containers/command + kind: Rollout +- path: spec/template/spec/containers/env/value + kind: Rollout +- path: spec/template/spec/containers/volumeMounts/mountPath + kind: Rollout +- path: spec/template/spec/initContainers/args + kind: Rollout +- path: spec/template/spec/initContainers/command + kind: Rollout +- path: spec/template/spec/initContainers/env/value + kind: Rollout +- path: spec/template/spec/initContainers/volumeMounts/mountPath + kind: Rollout +- path: spec/templates/template/spec/containers/args + kind: Experiment +- path: spec/templates/template/spec/containers/command + kind: Experiment +- path: spec/templates/template/spec/containers/env/value + kind: Experiment +- path: spec/templates/template/spec/containers/volumeMounts/mountPath + kind: Experiment +- path: spec/templates/template/spec/initContainers/args + kind: Experiment +- path: spec/templates/template/spec/initContainers/command + kind: Experiment +- path: spec/templates/template/spec/initContainers/env/value + kind: Experiment +- path: spec/templates/template/spec/initContainers/volumeMounts/mountPath + kind: Experiment +- path: spec/metrics/provider/job/spec/template/spec/containers/args + kind: AnalysisTemplate +- path: spec/metrics/provider/job/spec/template/spec/containers/command + kind: AnalysisTemplate +- path: spec/metrics/provider/job/spec/template/spec/containers/env/value + kind: AnalysisTemplate +- path: spec/metrics/provider/job/spec/template/spec/containers/volumeMounts/mountPath + kind: AnalysisTemplate +- path: spec/metrics/provider/job/spec/template/spec/initContainers/args + kind: AnalysisTemplate +- path: spec/metrics/provider/job/spec/template/spec/initContainers/command + kind: AnalysisTemplate +- path: spec/metrics/provider/job/spec/template/spec/initContainers/env/value + kind: AnalysisTemplate +- path: spec/metrics/provider/job/spec/template/spec/initContainers/volumeMounts/mountPath + kind: AnalysisTemplate +- path: spec/metrics/provider/job/spec/template/spec/volumes/nfs/server + kind: AnalysisTemplate + +# https://github.com/kubernetes-sigs/kustomize/blob/master/api/konfig/builtinpluginconsts/replicas.go +replicas: +- path: spec/replicas + create: true + kind: Rollout diff --git a/docs/features/kustomize/rollout_cr_schema.json b/docs/features/kustomize/rollout_cr_schema.json index 038ebebf89..edff0c7931 100644 --- a/docs/features/kustomize/rollout_cr_schema.json +++ b/docs/features/kustomize/rollout_cr_schema.json @@ -105,6 +105,17 @@ ], "x-kubernetes-int-or-string": true }, + "consecutiveSuccessLimit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "x-kubernetes-int-or-string": true + }, "count": { "anyOf": [ { @@ -244,7 +255,6 @@ "datadog": { "properties": { "aggregator": { - "default": "last", "enum": [ "avg", "min", @@ -281,6 +291,17 @@ }, "query": { "type": "string" + }, + "secretRef": { + "properties": { + "name": { + "type": "string" + }, + "namespaced": { + "type": "boolean" + } + }, + "type": "object" } }, "type": "object" @@ -4607,6 +4628,10 @@ }, "query": { "type": "string" + }, + "timeout": { + "format": "int64", + "type": "integer" } }, "required": [ @@ -4686,6 +4711,20 @@ "query": { "type": "string" }, + "rangeQuery": { + "properties": { + "end": { + "type": "string" + }, + "start": { + "type": "string" + }, + "step": { + "type": "string" + } + }, + "type": "object" + }, "timeout": { "format": "int64", "type": "integer" @@ -4943,6 +4982,17 @@ ], "x-kubernetes-int-or-string": true }, + "consecutiveSuccessLimit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "x-kubernetes-int-or-string": true + }, "count": { "anyOf": [ { @@ -5082,7 +5132,6 @@ "datadog": { "properties": { "aggregator": { - "default": "last", "enum": [ "avg", "min", @@ -5119,6 +5168,17 @@ }, "query": { "type": "string" + }, + "secretRef": { + "properties": { + "name": { + "type": "string" + }, + "namespaced": { + "type": "boolean" + } + }, + "type": "object" } }, "type": "object" @@ -9445,6 +9505,10 @@ }, "query": { "type": "string" + }, + "timeout": { + "format": "int64", + "type": "integer" } }, "required": [ @@ -9524,6 +9588,20 @@ "query": { "type": "string" }, + "rangeQuery": { + "properties": { + "end": { + "type": "string" + }, + "start": { + "type": "string" + }, + "step": { + "type": "string" + } + }, + "type": "object" + }, "timeout": { "format": "int64", "type": "integer" @@ -9794,6 +9872,17 @@ ], "x-kubernetes-int-or-string": true }, + "consecutiveSuccessLimit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "x-kubernetes-int-or-string": true + }, "count": { "anyOf": [ { @@ -9933,7 +10022,6 @@ "datadog": { "properties": { "aggregator": { - "default": "last", "enum": [ "avg", "min", @@ -9970,6 +10058,17 @@ }, "query": { "type": "string" + }, + "secretRef": { + "properties": { + "name": { + "type": "string" + }, + "namespaced": { + "type": "boolean" + } + }, + "type": "object" } }, "type": "object" @@ -14296,6 +14395,10 @@ }, "query": { "type": "string" + }, + "timeout": { + "format": "int64", + "type": "integer" } }, "required": [ @@ -14375,6 +14478,20 @@ "query": { "type": "string" }, + "rangeQuery": { + "properties": { + "end": { + "type": "string" + }, + "start": { + "type": "string" + }, + "step": { + "type": "string" + } + }, + "type": "object" + }, "timeout": { "format": "int64", "type": "integer" diff --git a/docs/features/notifications.md b/docs/features/notifications.md index 5574ab566f..2fc4d33f1b 100644 --- a/docs/features/notifications.md +++ b/docs/features/notifications.md @@ -70,7 +70,12 @@ To enable you need to add a flag to the controller `--self-service-notification- Currently the following triggers have [built-in templates](https://github.com/argoproj/argo-rollouts/tree/master/manifests/notifications). +* `on-analysis-run-error` when an error occurs during the execution of an analysis run +* `on-analysis-run-failed` when an analysis run fails +* `on-analysis-run-running` when an analysis run is running +* `on-rollout-aborted` when a rollout process is aborted before completion. * `on-rollout-completed` when a rollout is finished and all its steps are completed +* `on-rollout-paused` when a rollout is paused * `on-rollout-step-completed` when an individual step inside a rollout definition is completed * `on-rollout-updated` when a rollout definition is changed * `on-scaling-replica-set` when the number of replicas in a rollout is changed @@ -141,8 +146,8 @@ add blocks and attachments for Slack, subject for Email or URL path, and body fo In addition to custom notification template administrator and configure custom triggers. Custom trigger defines the condition when the notification should be sent. The definition includes name, condition and notification templates reference. The condition is a predicate expression that returns true if the notification should be sent. The trigger condition -evaluation is powered by [antonmedv/expr](https://github.com/antonmedv/expr). -The condition language syntax is described at [Language-Definition.md](https://github.com/antonmedv/expr/blob/master/docs/Language-Definition.md). +evaluation is powered by [expr-lang/expr](https://github.com/expr-lang/expr). +The condition language syntax is described at [Language-Definition.md](https://github.com/expr-lang/expr/blob/master/docs/language-definition.md). The trigger is configured in `argo-rollouts-notification-configmap` ConfigMap. For example the following trigger sends a notification when rollout pod spec uses `argoproj/rollouts-demo:purple` image: @@ -165,4 +170,4 @@ Each condition might use several templates. Typically each template is responsib The following prometheus metrics are emitted when notifications are enabled in argo-rollouts. - notification_send_success is a counter that measures how many times the notification is sent successfully. - notification_send_error is a counter that measures how many times the notification failed to send. -- notification_send is a histogram that measures performance of sending notification. \ No newline at end of file +- notification_send is a histogram that measures performance of sending notification. diff --git a/docs/features/specification.md b/docs/features/specification.md index 607367c0c8..c99ff3d627 100644 --- a/docs/features/specification.md +++ b/docs/features/specification.md @@ -15,7 +15,7 @@ spec: # limits the number of successful analysis runs and experiments to be stored in a history # Defaults to 5. successfulRunHistoryLimit: 10 - # limits the number of unsuccessful analysis runs and experiments to be stored in a history. + # limits the number of unsuccessful analysis runs and experiments to be stored in a history. # Stages for unsuccessful: "Error", "Failed", "Inconclusive" # Defaults to 5. unsuccessfulRunHistoryLimit: 10 @@ -27,9 +27,9 @@ spec: matchLabels: app: guestbook - # WorkloadRef holds a references to a workload that provides Pod template + # WorkloadRef holds a references to a workload that provides Pod template # (e.g. Deployment). If used, then do not use Rollout template property. - workloadRef: + workloadRef: apiVersion: apps/v1 kind: Deployment name: rollout-ref-deployment @@ -42,12 +42,12 @@ spec: scaleDown: never|onsuccess|progressively # Template describes the pods that will be created. Same as deployment. - # If used, then do not use Rollout workloadRef property. + # If used, then do not use Rollout workloadRef property. template: spec: containers: - - name: guestbook - image: argoproj/rollouts-demo:blue + - name: guestbook + image: argoproj/rollouts-demo:blue # Minimum number of seconds for which a newly created pod should be ready # without any of its container crashing, for it to be considered available. @@ -83,7 +83,7 @@ spec: # its pods. Used by the `kubectl argo rollouts restart ROLLOUT` command. # The controller will ensure all pods have a creationTimestamp greater # than or equal to this value. - restartAt: "2020-03-30T21:19:35Z" + restartAt: '2020-03-30T21:19:35Z' # The rollback window provides a way to fast track deployments to # previously deployed versions. @@ -92,10 +92,8 @@ spec: revisions: 3 strategy: - # Blue-green update strategy blueGreen: - # Reference to service that the rollout modifies as the active service. # Required. activeService: active-service @@ -104,19 +102,19 @@ spec: # cutover. +optional prePromotionAnalysis: templates: - - templateName: success-rate + - templateName: success-rate args: - - name: service-name - value: guestbook-svc.default.svc.cluster.local + - name: service-name + value: guestbook-svc.default.svc.cluster.local # Post-promotion analysis run which performs analysis after the service # cutover. +optional postPromotionAnalysis: templates: - - templateName: success-rate + - templateName: success-rate args: - - name: service-name - value: guestbook-svc.default.svc.cluster.local + - name: service-name + value: guestbook-svc.default.svc.cluster.local # Name of the service that the rollout modifies as the preview service. # +optional @@ -158,13 +156,13 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: {} preferredDuringSchedulingIgnoredDuringExecution: weight: 1 # Between 1 - 100 - + # activeMetadata will be merged and updated in-place into the ReplicaSet's spec.template.metadata # of the active pods. +optional activeMetadata: labels: role: active - + # Metadata which will be attached to the preview pods only during their preview phase. # +optional previewMetadata: @@ -173,7 +171,6 @@ spec: # Canary update strategy canary: - # Reference to a service which the controller will update to select # canary pods. Required for traffic routing. canaryService: canary-service @@ -220,7 +217,7 @@ spec: # killed, new RC can be scaled up further, ensuring that total number # of pods running at any time during the update is at most 130% of # original pods. +optional - maxSurge: "20%" + maxSurge: '20%' # Adds a delay before scaling down the previous ReplicaSet when the # canary strategy is used with traffic routing (default 30 seconds). @@ -244,135 +241,141 @@ spec: # initial deploy of a rollout. +optional analysis: templates: - - templateName: success-rate + - templateName: success-rate args: - - name: service-name - value: guestbook-svc.default.svc.cluster.local - - # valueFrom.podTemplateHashValue is a convenience to supply the - # rollouts-pod-template-hash value of either the Stable ReplicaSet - # or the Latest ReplicaSet - - name: stable-hash - valueFrom: - podTemplateHashValue: Stable - - name: latest-hash - valueFrom: - podTemplateHashValue: Latest - - # valueFrom.fieldRef allows metadata about the rollout to be - # supplied as arguments to analysis. - - name: region - valueFrom: - fieldRef: - fieldPath: metadata.labels['region'] + - name: service-name + value: guestbook-svc.default.svc.cluster.local + + # valueFrom.podTemplateHashValue is a convenience to supply the + # rollouts-pod-template-hash value of either the Stable ReplicaSet + # or the Latest ReplicaSet + - name: stable-hash + valueFrom: + podTemplateHashValue: Stable + - name: latest-hash + valueFrom: + podTemplateHashValue: Latest + + # valueFrom.fieldRef allows metadata about the rollout to be + # supplied as arguments to analysis. + - name: region + valueFrom: + fieldRef: + fieldPath: metadata.labels['region'] # Steps define sequence of steps to take during an update of the # canary. Skipped upon initial deploy of a rollout. +optional steps: - - # Sets the ratio of canary ReplicaSet to 20% - - setWeight: 20 - - # Pauses the rollout for an hour. Supported units: s, m, h - - pause: - duration: 1h - - # Pauses indefinitely until manually resumed - - pause: {} - - # set canary scale to a explicit count without changing traffic weight - # (supported only with trafficRouting) - - setCanaryScale: - replicas: 3 - - # set canary scale to a percentage of spec.replicas without changing traffic weight - # (supported only with trafficRouting) - - setCanaryScale: - weight: 25 - - # set canary scale to match the canary traffic weight (default behavior) - - setCanaryScale: - matchTrafficWeight: true - - # Sets header based route with specified header values - # Setting header based route will send all traffic to the canary for the requests - # with a specified header, in this case request header "version":"2" - # (supported only with trafficRouting, for Istio only at the moment) - - setHeaderRoute: - # Name of the route that will be created by argo rollouts this must also be configured - # in spec.strategy.canary.trafficRouting.managedRoutes - name: "header-route-1" - # The matching rules for the header route, if this is missing it acts as a removal of the route. - match: - # headerName The name of the header to apply the match rules to. - - headerName: "version" - # headerValue must contain exactly one field of exact, regex, or prefix. Not all traffic routers support - # all types - headerValue: - # Exact will only match if the header value is exactly the same - exact: "2" - # Will match the rule if the regular expression matches - regex: "2.0.(.*)" - # prefix will be a prefix match of the header value - prefix: "2.0" - - # Sets up a mirror/shadow based route with the specified match rules - # The traffic will be mirrored at the configured percentage to the canary service - # during the rollout + # Sets the ratio of canary ReplicaSet to 20% + - setWeight: 20 + + # Pauses the rollout for an hour. Supported units: s, m, h + - pause: + duration: 1h + + # Pauses indefinitely until manually resumed + - pause: {} + + # set canary scale to a explicit count without changing traffic weight + # (supported only with trafficRouting) + - setCanaryScale: + replicas: 3 + + # set canary scale to spec.Replica * (setweight / maxTrafficWeight) without changing traffic weight + # if maxTrafficWeight unspecified, it defaults to 100 + # (supported only with trafficRouting) + - setCanaryScale: + weight: 25 + + # set canary scale to match the canary traffic weight (default behavior) + - setCanaryScale: + matchTrafficWeight: true + + # executes the configured plugin by name with the provided configuration + - plugin: + name: example + config: + key: value + + # Sets header based route with specified header values + # Setting header based route will send all traffic to the canary for the requests + # with a specified header, in this case request header "version":"2" # (supported only with trafficRouting, for Istio only at the moment) - - setMirrorRoute: - # Name of the route that will be created by argo rollouts this must also be configured - # in spec.strategy.canary.trafficRouting.managedRoutes - name: "header-route-1" - # The percentage of the matched traffic to mirror to the canary - percentage: 100 - # The matching rules for the header route, if this is missing it acts as a removal of the route. - # All conditions inside a single match block have AND semantics, while the list of match blocks have OR semantics. - # Each type within a match (method, path, headers) must have one and only one match type (exact, regex, prefix) - # Not all match types (exact, regex, prefix) will be supported by all traffic routers. - match: - - method: # What HTTP method to match - exact: "GET" - regex: "P.*" - prefix: "POST" - path: # What HTTP url paths to match. - exact: "/test" - regex: "/test/.*" - prefix: "/" - headers: - agent-1b: # What HTTP header name to use in the match. - exact: "firefox" - regex: "firefox2(.*)" - prefix: "firefox" - - # an inline analysis step - - analysis: - templates: - - templateName: success-rate - - # an inline experiment step - - experiment: - duration: 1h - templates: - - name: baseline - specRef: stable - # optional, creates a service for the experiment if set - service: - # optional, service: {} is also acceptable if name is not included - name: test-service - - name: canary - specRef: canary - # optional, set the weight of traffic routed to this version - weight: 10 - analyses: - - name : mann-whitney - templateName: mann-whitney - # Metadata which will be attached to the AnalysisRun. - analysisRunMetadata: - labels: - app.service.io/analysisType: smoke-test - annotations: - link.argocd.argoproj.io/external-link: http://my-loggin-platform.com/pre-generated-link + - setHeaderRoute: + # Name of the route that will be created by argo rollouts this must also be configured + # in spec.strategy.canary.trafficRouting.managedRoutes + name: 'header-route-1' + # The matching rules for the header route, if this is missing it acts as a removal of the route. + match: + # headerName The name of the header to apply the match rules to. + - headerName: 'version' + # headerValue must contain exactly one field of exact, regex, or prefix. Not all traffic routers support + # all types + headerValue: + # Exact will only match if the header value is exactly the same + exact: '2' + # Will match the rule if the regular expression matches + regex: '2.0.(.*)' + # prefix will be a prefix match of the header value + prefix: '2.0' + + # Sets up a mirror/shadow based route with the specified match rules + # The traffic will be mirrored at the configured percentage to the canary service + # during the rollout + # (supported only with trafficRouting, for Istio only at the moment) + - setMirrorRoute: + # Name of the route that will be created by argo rollouts this must also be configured + # in spec.strategy.canary.trafficRouting.managedRoutes + name: 'header-route-1' + # The percentage of the matched traffic to mirror to the canary + percentage: 100 + # The matching rules for the header route, if this is missing it acts as a removal of the route. + # All conditions inside a single match block have AND semantics, while the list of match blocks have OR semantics. + # Each type within a match (method, path, headers) must have one and only one match type (exact, regex, prefix) + # Not all match types (exact, regex, prefix) will be supported by all traffic routers. + match: + - method: # What HTTP method to match + exact: 'GET' + regex: 'P.*' + prefix: 'POST' + path: # What HTTP url paths to match. + exact: '/test' + regex: '/test/.*' + prefix: '/' + headers: + agent-1b: # What HTTP header name to use in the match. + exact: 'firefox' + regex: 'firefox2(.*)' + prefix: 'firefox' + + # an inline analysis step + - analysis: + templates: + - templateName: success-rate + + # an inline experiment step + - experiment: + duration: 1h + templates: + - name: baseline + specRef: stable + # optional, creates a service for the experiment if set + service: + # optional, service: {} is also acceptable if name is not included + name: test-service + - name: canary + specRef: canary + # optional, set the weight of traffic routed to this version + weight: 10 + analyses: + - name: mann-whitney + templateName: mann-whitney + # Metadata which will be attached to the AnalysisRun. + analysisRunMetadata: + labels: + app.service.io/analysisType: smoke-test + annotations: + link.argocd.argoproj.io/external-link: http://my-loggin-platform.com/pre-generated-link # Anti-affinity configuration between desired and previous ReplicaSet. # Only one must be specified. @@ -392,7 +395,7 @@ spec: # This is a list of routes that Argo Rollouts has the rights to manage it is currently only required for # setMirrorRoute and setHeaderRoute. The order of managedRoutes array also sets the precedence of the route # in the traffic router. Argo Rollouts will place these routes in the order specified above any routes already - # defined in the used traffic router if something exists. The names here must match the names from the + # defined in the used traffic router if something exists. The names here must match the names from the # setHeaderRoute and setMirrorRoute steps. managedRoutes: - name: set-header @@ -400,18 +403,18 @@ spec: # Istio traffic routing configuration istio: # Either virtualService or virtualServices can be configured. - virtualService: - name: rollout-vsvc # required - routes: - - primary # optional if there is a single route in VirtualService, required otherwise - virtualServices: - # One or more virtualServices can be configured - - name: rollouts-vsvc1 # required + virtualService: + name: rollout-vsvc # required routes: - primary # optional if there is a single route in VirtualService, required otherwise - - name: rollouts-vsvc2 # required - routes: - - secondary # optional if there is a single route in VirtualService, required otherwise + virtualServices: + # One or more virtualServices can be configured + - name: rollouts-vsvc1 # required + routes: + - primary # optional if there is a single route in VirtualService, required otherwise + - name: rollouts-vsvc2 # required + routes: + - secondary # optional if there is a single route in VirtualService, required otherwise # NGINX Ingress Controller routing configuration nginx: @@ -422,14 +425,16 @@ spec: - secondary-ingress - tertiary-ingress annotationPrefix: customingress.nginx.ingress.kubernetes.io # optional - additionalIngressAnnotations: # optional + additionalIngressAnnotations: # optional canary-by-header: X-Canary canary-by-header-value: iwantsit + canaryIngressAnnotations: # optional + my-custom-annotation.mygroup.com/key: value # ALB Ingress Controller routing configuration alb: - ingress: ingress # required - servicePort: 443 # required + ingress: ingress # required + servicePort: 443 # required annotationPrefix: custom.alb.ingress.kubernetes.io # optional # Service Mesh Interface routing configuration @@ -442,18 +447,24 @@ spec: # 0 means canary pods are not scaled down. Default is 30 seconds. abortScaleDownDelaySeconds: 30 + # Automatically reduce the number of stable pods as the number of canary pods increases + # Only available when traffic routing is used. Default value is false meaning that as more canary pods + # are created the number of stable pods stays the same. + dynamicStableScale: false + status: pauseConditions: - - reason: StepPause - startTime: 2019-10-00T1234 - - reason: BlueGreenPause - startTime: 2019-10-00T1234 - - reason: AnalysisRunInconclusive - startTime: 2019-10-00T1234 + - reason: StepPause + startTime: 2019-10-00T1234 + - reason: BlueGreenPause + startTime: 2019-10-00T1234 + - reason: AnalysisRunInconclusive + startTime: 2019-10-00T1234 ``` + ## Examples You can find examples of Rollouts at: - * The [example directory](https://github.com/argoproj/argo-rollouts/tree/master/examples) - * The [Argo Rollouts Demo application](https://github.com/argoproj/rollouts-demo) +- The [example directory](https://github.com/argoproj/argo-rollouts/tree/master/examples) +- The [Argo Rollouts Demo application](https://github.com/argoproj/rollouts-demo) diff --git a/docs/features/traffic-management/nginx.md b/docs/features/traffic-management/nginx.md index dc7c2ccdb8..220ad7fd84 100644 --- a/docs/features/traffic-management/nginx.md +++ b/docs/features/traffic-management/nginx.md @@ -7,7 +7,8 @@ The Rollout controller will always set the following two annotations on the cana - `canary: true` to indicate that this is the canary Ingress - `canary-weight: ` to indicate what percentage of traffic to send to the canary. If all traffic is routed to the stable Service, this is set to `0` -You can provide additional annotations to add to the canary Ingress via the `additionalIngressAnnotations` field to enable features like routing by header or cookie. +You can provide additional annotations to add to the canary Ingress via the `additionalIngressAnnotations` or `canaryIngressAnnotations` field to enable features like routing by header or cookie. +While the former offers the possibility of reusing a common, injectable nginx annotation prefix, the latter allows defining custom full annotations of any group, which may be relevant for other, independent mechanisms, such as a third-party metrics scraper or some other infrastructure integration. ## Integration with Argo Rollouts @@ -34,6 +35,8 @@ spec: additionalIngressAnnotations: # optional canary-by-header: X-Canary canary-by-header-value: iwantsit + canaryIngressAnnotations: # optional + my-custom-annotation.mygroup.com/key: value ``` The stable Ingress field is a reference to an Ingress in the same namespace of the Rollout. The Rollout requires the primary Ingress routes traffic to the stable Service. The Rollout checks that condition by confirming the Ingress has a backend that matches the Rollout's stableService. @@ -42,6 +45,8 @@ The controller routes traffic to the canary Service by creating a second Ingress Since the Nginx Ingress controller allows users to configure the annotation prefix used by the Ingress controller, Rollouts can specify the optional `annotationPrefix` field. The canary Ingress uses that prefix instead of the default `nginx.ingress.kubernetes.io` if the field set. +If full annotations, [as defined in the Kubernetes docs](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set), perhaps from different groups, need to be declared instead, the `canaryIngressAnnotations` field can be used, which accepts a similar key-value structure, but performs no prefix injection. +Note that, in case of collision with `additionalIngressAnnotations`, the value under `canaryIngressAnnotations` prevails. ## Using Argo Rollouts with multiple NGINX ingress controllers per service Starting with v1.5, argo rollouts supports multiple Nginx ingress controllers pointing at one service with canary deployments. If only one ingress controller is needed, utilize the existing key `stableIngress`. If multiple ingress controllers are needed (e.g., separating internal vs external traffic), use the key `stableIngresses` instead. It takes an array of string values that are the names of the ingress controllers. Canary steps are applied identically across all ingress controllers. diff --git a/docs/features/traffic-management/plugins.md b/docs/features/traffic-management/plugins.md index 7f8d8e1b11..1d726ff67f 100644 --- a/docs/features/traffic-management/plugins.md +++ b/docs/features/traffic-management/plugins.md @@ -1,14 +1,16 @@ # Traffic Router Plugins -!!! important - Available since v1.5 - Status: Alpha +!!! warning "Alpha Feature (Since 1.5.0)" -Argo Rollouts supports getting analysis metrics via 3rd party plugin system. This allows users to extend the capabilities of Rollouts -to support metric providers that are not natively supported. Rollout's uses a plugin library called + This is an experimental, [alpha-quality](https://github.com/argoproj/argoproj/blob/main/community/feature-status.md#alpha) + feature that allows you to supporttraffic router that are not natively supported. + +Argo Rollouts supports getting traffic router via 3rd party [plugin system](../../plugins.md). This allows users to extend the capabilities of Rollouts +to support traffic router that are not natively supported. Rollout's uses a plugin library called [go-plugin](https://github.com/hashicorp/go-plugin) to do this. You can find a sample plugin here: [rollouts-plugin-trafficrouter-sample-nginx](https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-sample-nginx) -## Using a Traffic Router Plugin +## Installing There are two methods of installing and using an argo rollouts plugin. The first method is to mount up the plugin executable into the rollouts controller container. The second method is to use a HTTP(S) server to host the plugin executable. @@ -18,9 +20,9 @@ into the rollouts controller container. The second method is to use a HTTP(S) se There are a few ways to mount the plugin executable into the rollouts controller container. Some of these will depend on your particular infrastructure. Here are a few methods: -* Using an init container to download the plugin executable -* Using a Kubernetes volume mount with a shared volume such as NFS, EBS, etc. -* Building the plugin into the rollouts controller container +- Using an init container to download the plugin executable +- Using a Kubernetes volume mount with a shared volume such as NFS, EBS, etc. +- Building the plugin into the rollouts controller container Then you can use the configmap to point to the plugin executable file location. Example: @@ -37,6 +39,19 @@ data: ### Using a HTTP(S) server to host the plugin executable +!!! warning "Installing a plugin with http(s)" + + Depending on which method you use to install and the plugin, there are some things to be aware of. + The rollouts controller will not start if it can not download or find the plugin executable. This means that if you are using + a method of installation that requires a download of the plugin and the server hosting the plugin for some reason is not available and the rollouts + controllers pod got deleted while the server was down or is coming up for the first time, it will not be able to start until + the server hosting the plugin is available again. + + Argo Rollouts will download the plugin at startup only once but if the pod is deleted it will need to download the plugin again on next startup. Running + Argo Rollouts in HA mode can help a little with this situation because each pod will download the plugin at startup. So if a single pod gets + deleted during a server outage, the other pods will still be able to take over because there will already be a plugin executable available to it. It is the + responsibility of the Argo Rollouts administrator to define the plugin installation method considering the risks of each approach. + Argo Rollouts supports downloading the plugin executable from a HTTP(S) server. To use this method, you will need to configure the controller via the `argo-rollouts-config` configmap and set `pluginLocation` to a http(s) url. Example: @@ -50,34 +65,36 @@ data: - name: "argoproj-labs/sample-nginx" # name of the plugin, it must match the name required by the plugin so it can find it's configuration location: "https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-sample-nginx/releases/download/v0.0.1/metric-plugin-linux-amd64" # supports http(s):// urls and file:// sha256: "08f588b1c799a37bbe8d0fc74cc1b1492dd70b2c" #optional sha256 checksum of the plugin executable + headersFrom: #optional headers for the download via http request + - secretRef: + name: secret-name +--- +apiVersion: v1 +kind: Secret +metadata: + name: secret-name +stringData: + Authorization: Basic + My-Header: value ``` -## Some words of caution - -Depending on which method you use to install and the plugin, there are some things to be aware of. -The rollouts controller will not start if it can not download or find the plugin executable. This means that if you are using -a method of installation that requires a download of the plugin and the server hosting the plugin for some reason is not available and the rollouts -controllers pod got deleted while the server was down or is coming up for the first time, it will not be able to start until -the server hosting the plugin is available again. +## List of Available Plugins (alphabetical order) -Argo Rollouts will download the plugin at startup only once but if the pod is deleted it will need to download the plugin again on next startup. Running -Argo Rollouts in HA mode can help a little with this situation because each pod will download the plugin at startup. So if a single pod gets -deleted during a server outage, the other pods will still be able to take over because there will already be a plugin executable available to it. It is the -responsibility of the Argo Rollouts administrator to define the plugin installation method considering the risks of each approach. +If you have created a plugin, please submit a PR to add it to this list. -## List of Available Plugins (alphabetical order) +### [rollouts-plugin-trafficrouter-sample-nginx](https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-sample-nginx) -#### Add Your Plugin Here -* If you have created a plugin, please submit a PR to add it to this list. -#### [rollouts-plugin-trafficrouter-sample-nginx](https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-sample-nginx) -* This is just a sample plugin that can be used as a starting point for creating your own plugin. +- This is just a sample plugin that can be used as a starting point for creating your own plugin. It is not meant to be used in production. It is based on the built-in prometheus provider. -#### [Consul](https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-consul) -* This is a plugin that allows argo-rollouts to work with Consul's service mesh for traffic shaping patterns. +### [Consul](https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-consul) + +- This is a plugin that allows argo-rollouts to work with Consul's service mesh for traffic shaping patterns. + +### [Contour](https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-contour) + +- This is a plugin that allows argo-rollouts to work with contour's resource: HTTPProxy. It enables traffic shaping patterns such as canary releases and more. -#### [Contour](https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-contour) -* This is a plugin that allows argo-rollouts to work with contour's resource: HTTPProxy. It enables traffic shaping patterns such as canary releases and more. +### [Gateway API](https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-gatewayapi/) -#### [Gateway API](https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-gatewayapi/) -* Provide support for Gateway API, which includes Kuma, Traefix, cilium, Contour, GloodMesh, HAProxy, and [many others](https://gateway-api.sigs.k8s.io/implementations/#implementation-status). \ No newline at end of file +- Provide support for Gateway API, which includes Kuma, Traefix, cilium, Contour, GloodMesh, HAProxy, and [many others](https://gateway-api.sigs.k8s.io/implementations/#implementation-status). diff --git a/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_dashboard.md b/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_dashboard.md index eb6ee9e6fc..ad99bfb5ca 100644 --- a/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_dashboard.md +++ b/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_dashboard.md @@ -10,6 +10,16 @@ Start UI dashboard kubectl argo rollouts dashboard [flags] ``` +## Examples + +```shell +# Start UI dashboard +kubectl argo rollouts dashboard + +# Start UI dashboard on a specific port +kubectl argo rollouts dashboard --port 8080 +``` + ## Options ``` diff --git a/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_get_rollout.md b/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_get_rollout.md index 77b04d2217..4654f2f9c5 100644 --- a/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_get_rollout.md +++ b/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_get_rollout.md @@ -30,6 +30,9 @@ kubectl argo rollouts get rollout guestbook # Watch progress of a rollout kubectl argo rollouts get rollout guestbook -w + +# Watch the rollout, fail if it takes more than 60 seconds +kubectl argo rollouts get rollout guestbook -w --timeout-seconds 60 ``` ## Options diff --git a/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_list_experiments.md b/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_list_experiments.md index 4add56c247..39d9f902bb 100644 --- a/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_list_experiments.md +++ b/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_list_experiments.md @@ -26,7 +26,7 @@ kubectl argo rollouts list experiments --watch ## Options ``` - --all-namespaces Include all namespaces + -A, --all-namespaces Include all namespaces -h, --help help for experiments ``` diff --git a/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_list_rollouts.md b/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_list_rollouts.md index e861ca20c8..7382d88185 100644 --- a/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_list_rollouts.md +++ b/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_list_rollouts.md @@ -16,6 +16,9 @@ kubectl argo rollouts list rollouts [flags] # List rollouts kubectl argo rollouts list rollouts +# List rollouts with a specific name +kubectl argo rollouts list rollouts --name my-rollout + # List rollouts from all namespaces kubectl argo rollouts list rollouts --all-namespaces diff --git a/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_set_image.md b/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_set_image.md index 5f89a6a506..a25acaa730 100644 --- a/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_set_image.md +++ b/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_set_image.md @@ -13,8 +13,11 @@ kubectl argo rollouts set image ROLLOUT_NAME CONTAINER=IMAGE [flags] ## Examples ```shell -# Set rollout image -kubectl argo rollouts set image my-rollout www=image:v2 +# Set rollout image (containers contains 'initContainer', 'container', 'ephemeralContainer') +kubectl argo rollouts set image my-rollout containerName=imageName + +# Set rollout image for all containers +kubectl argo rollouts set image my-rollout *=imageName ``` ## Options diff --git a/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_status.md b/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_status.md index e1221d4056..4bfff8b586 100644 --- a/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_status.md +++ b/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_status.md @@ -17,6 +17,9 @@ kubectl argo rollouts status ROLLOUT_NAME [flags] # Watch the rollout until it succeeds kubectl argo rollouts status guestbook +# Show the rollout status +kubectl argo rollouts status guestbook --watch false + # Watch the rollout until it succeeds, fail if it takes more than 60 seconds kubectl argo rollouts status --timeout 60s guestbook diff --git a/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_terminate.md b/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_terminate.md index 4491cca5fe..21881976dc 100644 --- a/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_terminate.md +++ b/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_terminate.md @@ -13,7 +13,7 @@ kubectl argo rollouts terminate RESOURCE_NAME [flags] ## Examples ```shell -# Terminate an analysisRun +# Terminate an AnalysisRun kubectl argo rollouts terminate analysisrun guestbook-877894d5b-4-success-rate.1 # Terminate a failed experiment diff --git a/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_terminate_analysisrun.md b/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_terminate_analysisrun.md index 268b7a0150..3ff552e97a 100644 --- a/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_terminate_analysisrun.md +++ b/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_terminate_analysisrun.md @@ -14,7 +14,7 @@ kubectl argo rollouts terminate analysisrun ANALYSISRUN_NAME [flags] ```shell # Terminate an AnalysisRun -kubectl argo rollouts terminate analysis guestbook-877894d5b-4-success-rate.1 +kubectl argo rollouts terminate analysisrun guestbook-877894d5b-4-success-rate.1 ``` ## Options diff --git a/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_undo.md b/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_undo.md index 0ddf539dc9..e6d95a203b 100644 --- a/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_undo.md +++ b/docs/generated/kubectl-argo-rollouts/kubectl-argo-rollouts_undo.md @@ -16,7 +16,7 @@ kubectl argo rollouts undo ROLLOUT_NAME [flags] # Undo a rollout kubectl argo rollouts undo guestbook -# Undo a rollout revision 3 +# Undo a rollout to revision 3 kubectl argo rollouts undo guestbook --to-revision=3 ``` diff --git a/docs/installation.md b/docs/installation.md index 7fae1d6d61..c2d9853290 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -116,7 +116,7 @@ docker run quay.io/argoproj/kubectl-argo-rollouts:master version ## Supported versions -Check [e2e testing file]( https://github.com/argoproj/argo-rollouts/blob/master/.github/workflows/e2e.yaml#L40-L44) to see what the Kubernetes version is being fully tested. +Check [e2e testing file](https://github.com/argoproj/argo-rollouts/blob/master/.github/workflows/testing.yaml#L82-L89) to see what the Kubernetes version is being fully tested. You can switch to different tags to see what relevant Kubernetes versions were being tested for the respective version. diff --git a/docs/plugins.md b/docs/plugins.md index 901bc79878..bc6eca7d1e 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -11,7 +11,6 @@ Here is an overview of how plugins are loaded: [![Loading of plugins](contributing-assets/plugin-loading.png)](contributing-assets/plugin-loading.png) - The communication protocol uses golang built in net/rpc library so plugins have to be written in golang. ## Plugin Repository @@ -28,7 +27,7 @@ types to exist such as `/nginx` and `/nginx`. These names could be b as `argoproj-labs/rollouts-plugin-metric-sample-prometheus` but it is not a requirement. There will also be a standard for naming repositories under argoproj-labs in the form of `rollouts-plugin--` -where `` is say `metric`, or `trafficrouter` and `` is the software the plugin is for say nginx. +where `` is one of `metric`, `step`, or `trafficrouter` and `` is the software the plugin is for, say nginx. ## Plugin Name @@ -55,13 +54,28 @@ data: args: - "--log-level" - "debug" + stepPlugins: |- + - name: "argoproj-labs/canary-step" + location: "file:///tmp/argo-rollouts/canary-step-plugin" + disabled: false + args: + - "--log-level" + - "debug" ``` -As you can see there is a field called `name:` under both `metrics` or `trafficrouters` this is the first place where your +As you can see there is a field called `name:` under each plugin type. This is the first place where your end users will need to configure the name of the plugin. The second `location` is either in the rollout object or the analysis template which you can see the examples below. The third `args` holds the command line arguments of the plugin. -#### AnalysisTemplate Example +### Configuration Examples + +#### AnalysisTemplate + +You can see that we use the plugin name under `spec.metrics[].provider.plugin` for analysis template. + +You, as a plugin author, can then put any configuration you need under `argoproj-labs/metrics` and you will be able to +look up that config in your plugin via the plugin name key. You will also want to document what configuration options your plugin supports. + ```yaml apiVersion: argoproj.io/v1alpha1 kind: AnalysisTemplate @@ -77,7 +91,13 @@ spec: address: http://prometheus.local ``` -#### Traffic Router Example +#### Traffic Router + +You can see that we use the plugin name under `spec.strategy.canary.trafficRouting.plugins` for traffic routers. + +You, as a plugin author, can then put any configuration you need under `argoproj-labs/metrics` and you will be able to +look up that config in your plugin via the plugin name key. You will also want to document what configuration options your plugin supports. + ```yaml apiVersion: argoproj.io/v1alpha1 kind: Rollout @@ -94,18 +114,36 @@ spec: stableIngress: canary-demo ``` -You can see that we use the plugin name under `spec.metrics[].provider.plugin` for analysis template and `spec.strategy.canary.trafficRouting.plugins` -for traffic routers. You as a plugin author can then put any configuration you need under `argoproj-labs/nginx` and you will be able to -look up that config in your plugin via the plugin name key. You will also want to document what configuration options your plugin supports. +#### Step Plugin + +You can see that we use the plugin name under `spec.strategy.canary.steps[].plugin.name` for canary steps. + +You, as a plugin author, can then put any configuration you need in the plugin object under `config` property and you will receive it +as an arguments when your plugin is called. You will also want to document what configuration options your plugin supports. + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: example-plugin-ro +spec: + strategy: + canary: + steps: + - plugin: + name: argoproj-labs/step-exec + config: + command: echo "hello world" +``` ## Plugin Interfaces -Argo Rollouts currently supports two plugin systems as a plugin author your end goal is to implement these interfaces as -a hashicorp go-plugin. The two interfaces are `MetricsPlugin` and `TrafficRouterPlugin` for each of the respective plugins: +Argo Rollouts currently supports three plugin systems. As a plugin author, your end goal is to implement at least one of these interfaces as +a hashicorp go-plugin. The interfaces are `MetricsPlugin`, `TrafficRouterPlugin` and `StepPlugin` for each of the respective plugins: ```go type MetricProviderPlugin interface { - // InitPlugin initializes the traffic router plugin this gets called once when the plugin is loaded. + // InitPlugin initializes the traffic router plugin. This gets called once when the plugin is loaded. InitPlugin() RpcError // Run start a new external system call for a measurement // Should be idempotent and do nothing if a call has already been started @@ -124,7 +162,7 @@ type MetricProviderPlugin interface { } type TrafficRouterPlugin interface { - // InitPlugin initializes the traffic router plugin this gets called once when the plugin is loaded. + // InitPlugin initializes the traffic router plugin. This gets called once when the plugin is loaded. InitPlugin() RpcError // UpdateHash informs a traffic routing reconciler about new canary, stable, and additionalDestination(s) pod hashes UpdateHash(rollout *v1alpha1.Rollout, canaryHash, stableHash string, additionalDestinations []v1alpha1.WeightDestination) RpcError @@ -142,6 +180,19 @@ type TrafficRouterPlugin interface { // Type returns the type of the traffic routing reconciler Type() string } + +type StepPlugin interface { + // InitPlugin initializes the canary step plugin. This gets called once when the plugin is loaded. + InitPlugin() RpcError + // Run executes a step plugin for the RpcStepContext and returns the result to the controller or an RpcError for unexpeted failures + Run(*v1alpha1.Rollout, *RpcStepContext) (RpcStepResult, RpcError) + // Terminate stops an uncompleted operation started by the Run operation + Terminate(*v1alpha1.Rollout, *RpcStepContext) (RpcStepResult, RpcError) + // Abort reverts the actions performed during the Run operation if necessary + Abort(*v1alpha1.Rollout, *RpcStepContext) (RpcStepResult, RpcError) + // Type returns the type of the step plugin + Type() string +} ``` ## Plugin Init Function @@ -161,7 +212,8 @@ for the plugin to use. This will probably affect traffic router plugins more tha ## Sample Plugins -There are two sample plugins within the argo-rollouts repo that you can use as a reference for creating your own plugin. +There are sample plugins within the argo-rollouts repo that you can use as a reference for creating your own plugin. -* [Metrics Plugin Sample](https://github.com/argoproj/argo-rollouts/tree/master/test/cmd/metrics-plugin-sample) -* [Traffic Router Plugin Sample](https://github.com/argoproj/argo-rollouts/tree/master/test/cmd/trafficrouter-plugin-sample) +- [Metrics Plugin Sample](https://github.com/argoproj/argo-rollouts/tree/master/test/cmd/metrics-plugin-sample) +- [Traffic Router Plugin Sample](https://github.com/argoproj/argo-rollouts/tree/master/test/cmd/trafficrouter-plugin-sample) +- [Step Plugin Sample](https://github.com/argoproj/argo-rollouts/tree/master/test/cmd/step-plugin-sample) diff --git a/docs/proposals/step-plugin.md b/docs/proposals/step-plugin.md new file mode 100644 index 0000000000..0c170b7fe7 --- /dev/null +++ b/docs/proposals/step-plugin.md @@ -0,0 +1,334 @@ +--- +title: Step Plugin +authors: + - '@agaudreault' +sponsors: + - '@zaller' +creation-date: 2024-03-27 +--- + +# Step Plugin + +Step plugins can be used to call code built outside of argo-rollout's codebase to execute actions during a canary rollout. + +This document provides technical implementation proposals to avoid implementing rollout steps in Argo Rollout’s codebase. + +## Summary + +Rollout steps need to be implemented natively in Argo Rollout source code. +It makes it difficult for the community to add new rollout steps because their implementation +is coupled with Rollout release cycle. The Rollout maintainers also have to acquire +knowledge on the different technologies used in the steps and validate them on each release. + +## Motivation + +This section is for explicitly listing the motivation, goals and non-goals of this proposal. +Describe why the change is important and the benefits to users. + +### Goals + +The goals of this proposal are: + +- Update steps outside Rollout’s release cycle +- Allow the community experts to maintain their steps +- Allow for a faster step development iteration +- Allow for more features without increasing complexity on the controller +- Allow users to use proprietary steps with Argo Rollouts + +### Non-Goals + +Implement plugins. + +## Proposal + +Rollout already has a plugin mechanism for metric providers and traffic routers as +documented in https://argoproj.github.io/argo-rollouts/plugins/. The implementation is based on [hashicorp/go-plugin](https://github.com/hashicorp/go-plugin). +This mechanism can be extended to support the steps plugin. + +- Consistent with existing behavior. +- Add a `stepPluginStatuses` array to the `.status.canary` field. +- Users can consult the Rollout object after the execution to get details on their status. + +### Use cases + +More details in https://github.com/argoproj/argo-rollouts/issues/2685 + +### Implementation Details/Notes/Constraints + +#### Configuration + +The plugin will be configured alongside existing plugins. + +```yaml +kind: ConfigMap +metadata: + name: argo-rollouts-config +data: + metricProviderPlugins: |- + - name: "argoproj-labs/metrics" + location: "file:///tmp/argo-rollouts/metric-plugin" + args: + - "--log-level" + - "debug" + stepPlugins: |- + - name: "argoproj-labs/curl/v2" + disabled: false + location: "file:///tmp/argo-rollouts/step-plugin" + sha256: "08f588b1c799a37bbe8d0fc74cc1b1492dd70abc" + args: + - "--log-level" + - "debug" +``` + +#### Interface + +_The interface implementation details related to go-plugin and rpc calls have been omitted for clarity._ + +```go +type Phase string + +const ( + PhaseRunning Phase = "Running" + PhaseSuccessful Phase = "Successful" + PhaseFailed Phase = "Failed" + PhaseError Phase = "Error" +) + +type StepContext struct { + PluginName string + Config map[string]interface{} + Status map[string]interface{} +} + +type StepStatus struct { + Index int + Name string + Phase Phase + Message string + StartedAt Time + FinishedAt Time + Status map[string]interface{} +} + +type StepResult struct { + Phase Phase + Message string + RequeueAfter Duration + Status map[string]interface{} +} + +type StepPlugin interface { + Init() error + Run(Rollout, StepContext) (StepResult, error) + Terminate(Rollout, StepContext) (StepResult, error) + Abort(Rollout, StepContext) (StepResult, error) +} +``` + +#### Rollout object (plugin input) + +The step will provide rollout specific configuration defined by the users +as a map of key value pairs allowing the user to pass any declarative configs to the plugin. + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: example-rollout +spec: + strategy: + canary: + ... + steps: + - setWeight: 20 + - plugin: + name: argoproj/curl + abortOnFailure: false # <--- example + progressDeadline: 30s # <--- example + config: + url: https://example.com/ + some_key: some_value + - pause: {} + - setWeight: 40 + +``` + +#### Rollout Status (plugin output) + +The controller will write a new object to the status. +This object will be used to persist the plugin state after the step execution and to allow other steps to use it. + +```yaml +status: + canary: + stepPluginStatuses: + - index: 2 + name: argoproj/curl + message: Call completed with status code 302 + phase: Successful + startedAt: '2024-02-15T20:05:40Z' + finishedAt: '2024-02-15T20:21:40Z' + status: {} + - index: 4 + name: argoproj/async-task + message: Waiting for result + phase: Running + startedAt: '2024-02-15T20:05:40Z' + finishedAt: null + status: + id: 12 + path: /an/example/property + validated: false +``` + +### Detailed execution flow + +1. **At initialization** + + The plugins are loaded based on configuration and started as processes with the provided arguments in the configuration. + The `Init()` method is called for plugins to perform one-time initialization such as creating clients and establishing connections. + +2. **A rollout reach the plugin step for the first time** + + During the rollout, the controller will use the `currentStepIndex` to find the step to run. + If the step is a plugin, the controller will create a StepContext object based + on the configuration on the Rollout’s step object. + + The controller will try to retrieve the value of the current `stepPluginStatuses`, + and if it matches the current step index, it will add the persisted state to the context. + + It will create a StepStatus object otherwise, set the name and index to the current step, + set the startedAt value to the current time and the phase as Running. + +3. **Running the step plugin** + + The plugin `Run()` method is called with the StepContext. + The plugin implementation will use the StepContext to perform the necessary logic. + +4. **Status is updated** + + Based on the return of the Run command, the StepStatus object is updated. + Then, the StepStatus is assigned to the stepPluginStatuses property. + + The status is persisted in the object. + +5. **Validate step completed** + + If the controller current step is of type plugin, the controller will check if the phase is successful, + and if so, go to the next step. + + If the step is still running, the controller will requeue a reconcile operation based on the value of `RequeueAfter`. + + If the phase is failed, it will update the status and conditions, **aborting** the rollout. + +#### Scenarios + +##### Step plugin completes successfully + +The `stepPluginStatuses` will not contain any object for the current step index, the plugin will perform the desired action successfully and return a successful state. + +The state will be persisted and the controller wil execute the next step. + +##### Step plugin completes with failed phase + +If the step plugin returns a failed phase, the controller will set the rollout to **aborted** and persist the state. + +The user will receive the feedback based on the Progressing condition status. This behavior is consistent with existing mechanisms aborting a rollout. + +##### Step plugin completes with running phase + +If the step plugin returns a running phase, the controller will persist the state, but will not increment the current step index. + +The controller will requeue a reconcile operation based on the value of `RequeueAfter` and terminate the current reconciliation. + +On the next reconciliation, the persisted state will be passed to the plugin `Run()` method. + +##### Step plugin is called multiple times + +If an external error happens causing the controller to crash after it called the `Run()` method and before it could persist the status, +the controller will replay the current plugin step, with a new context, like if it was the first time it is called. + +> **A step plugin can be called multiple times and operations should be idempotent.** + +##### Rollout is fully promoted during a step plugin + +If a Rollout is forcefully considered fully promoted while the current step is in a Running phase, +the plugin will call the `Terminate()` method with the current context on the next reconciliation and update the status based on the result. + +##### Rollout aborted during a step plugin + +If a Rollout is aborted while the current step is in a Running phase, the plugin will call the `Abort()` method with the current context on the next reconciliation. +The controller will call the `Abort()` operation for each step that were executed in the reverse order. +The steps may or may not perform any action during the Abort. +The result of the abort will be saved in the status, overriding the state persisted during the `Run()`. +If the Abort operation has an error, the error is propagated to the controller. +The controller has the responsibility to retry the Abort operation, and eventually proceed with the next step if it never succeeds. + +##### The step plugin reports an error + +Before returning the error, the status is persisted with the error phase and message. Other properties of the current status remain unmodified so the step can be re-executed with the last known valid status. + +For an expected retryable error, the plugin should return a Running phase with a RequeueAfter value to retry the execution. +For an expected un-retryable error, the plugin should return a Failed status. +After the state is persisted, the error is propagated to the controller and the controller error-handling logic will handle the error. + +##### The step plugin uses Rollout information + +The step plugin may need to have access to the current state of the Rollout. The step plugin will receive a deep copy of the rollout object in parameter. + +##### I can investigate my step plugin execution + +A user wants to know what happened during their custom step plugin after the execution. They can use the status in the Rollout object. + +##### State is shared between plugins + +A user wants to use an API to publish information about the rollout. The plugin first calls the API that returns a conversation ID. +Other steps need to use the conversationID during their execution. +The plugin step receives in parameters the full rollout object and the pluginName. This information can be used to retrieve the status of other plugin execution. + +An utility function such as `PluginHelper.GetStatuses(rollout, pluginName)` can be implemented and made available to the plugins. + +##### I want my rollout to continue event if my plugin failed + +A parameter such as `abortOnFailure` can be added to the Rollout plugin step configuration object. +When specified, the controller can use the value to modify the default logic. + +##### I dont want my plugin execution time to count towards the progress deadline + +A parameter such as `ignoreProgressDeadline` can be added to the Rollout plugin step configuration object. +When specified, the controller can use the value to modify the default logic. + +### Security Considerations + +- Plugins binary can be validated with the configured `sha256`. + +### Risks and Mitigations + +- Rollout status size can grow more than expected based on plugins hygiene. + - Object size can be validated with `unsafe.Sizeof(struct)` and a size limit can be imposed. +- Plugins that are failing or causing problems cannot be removed without updating all the Rollouts. + - A `disabled` config can be added globally and ignore the plugin execution if true. + +### Upgrade / Downgrade Strategy + +It is expected that plugins will be compiled with different versions than the running argo-rollout controller. The plugins version could be either newer or older than the controller. + +The hashicorp/go-plugin uses gob encoding with rpc. + +> “The source and destination values/types need not correspond exactly. For structs, fields (identified by name) that are in the source but absent from the receiving variable will be ignored. Fields that are in the receiving variable but missing from the transmitted type or value will be ignored in the destination. If a field with the same name is present in both, their types must be compatible. Both the receiver and transmitter will do all necessary indirection and dereferencing to convert between gobs and actual Go values.” - package [encoding/gob](https://pkg.go.dev/encoding/gob) + +Plugins should validate the objects they receive in parameters such as the `Rollout` and user confiuguration. If they expect a property to be set and it is not, it is highly probably that the controller's object version does not have that property. + +For breaking changes, hashicorp/go-plugin has a `ProtocolVersion` property that can be used in the future. + +Plugins can also be added with different names, which would require update to the Rollout CR objects as well. + +## Drawbacks + +- Gives a lot of power to the plugin and bad plugins could destabilize the rollouts +- If plugins need more permission, the access needs to be given using the Rollout service account. + +## Alternatives + +- Only implement vetted code in the argo-rollout codebase. +- Create a plugin that calls other containers diff --git a/docs/roadmap.md b/docs/roadmap.md deleted file mode 100644 index 7c3e52dcb9..0000000000 --- a/docs/roadmap.md +++ /dev/null @@ -1,55 +0,0 @@ -# Roadmap - -The Argo Rollouts roadmap is maintained in Github Milestones on the Github repository. - -## Release Cycle - -### Schedule - -These are the upcoming releases date estimates: - -| Release | Release Planning Meeting | Release Candidate 1 | General Availability | -|---------|--------------------------|-----------------------|--------------------------| -| v1.4 | TBD | Monday, Dec. 19, 2022 | Tuesday, Jan. 9, 2023 | -| v1.5 | Monday, Mar. 6, 2023 | Monday, Mar. 20, 2023 | Monday, Apr. 10, 2023 | -| v1.6 | Monday, Jun. 5, 2023 | Monday, Jun. 19, 2023 | Wednesday, Jul. 12, 2023 | -| v1.7 | Monday, Sep. 4, 2023 | Monday, Sep. 18, 2023 | Monday, Oct. 9, 2023 | - -### Release Process - -#### Minor Releases (e.g. 1.x.0) - -A minor Argo Rollouts release occurs four times a year, once every three months. Each General Availability (GA) release is -preceded by several Release Candidates (RCs). The first RC is released three weeks before the scheduled GA date. - -These are the approximate release dates: - -* The first Monday of January -* The first Monday of April -* The first Monday of July -* The first Monday of October - -Dates may be shifted slightly to accommodate holidays. Those shifts should be minimal. - -#### Patch Releases (e.g. 1.4.x) - -Argo Rollouts patch releases occur on an as-needed basis. Only the three most recent minor versions are eligible for patch -releases. Versions older than the three most recent minor versions are considered EOL and will not receive bug fixes or -security updates. - -### Feature Acceptance Criteria - -To be eligible for inclusion in a minor release, a new feature must meet the following criteria before the release’s RC -date. - -If it is a large feature that involves significant design decisions, that feature must be described in a Proposal. - -The feature PR must include: - -* Tests (passing) -* Documentation -* If necessary, a note in the Upgrading docs for the planned minor release -* The PR must be reviewed, approved, and merged by an Approver. - -If these criteria are not met by the RC date, the feature will be ineligible for inclusion in the RC series or GA for -that minor release. It will have to wait for the next minor release. diff --git a/experiments/controller.go b/experiments/controller.go index 3aa1519e46..b3c0c56329 100644 --- a/experiments/controller.go +++ b/experiments/controller.go @@ -5,6 +5,8 @@ import ( "sync" "time" + corev1 "k8s.io/api/core/v1" + log "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -318,6 +320,7 @@ func (ec *Controller) syncHandler(ctx context.Context, key string) error { } func (ec *Controller) persistExperimentStatus(orig *v1alpha1.Experiment, newStatus *v1alpha1.ExperimentStatus) error { + prevStatus := orig.Status ctx := context.TODO() logCtx := logutil.WithExperiment(orig) patch, modified, err := diff.CreateTwoWayMergePatch( @@ -336,15 +339,27 @@ func (ec *Controller) persistExperimentStatus(orig *v1alpha1.Experiment, newStat return nil } logCtx.Debugf("Experiment Patch: %s", patch) - _, err = ec.argoProjClientset.ArgoprojV1alpha1().Experiments(orig.Namespace).Patch(ctx, orig.Name, patchtypes.MergePatchType, patch, metav1.PatchOptions{}) + patched, err := ec.argoProjClientset.ArgoprojV1alpha1().Experiments(orig.Namespace).Patch(ctx, orig.Name, patchtypes.MergePatchType, patch, metav1.PatchOptions{}) if err != nil { logCtx.Warningf("Error updating experiment: %v", err) return err } logCtx.Info("Patch status successfully") + ec.recordEvent(patched, prevStatus, newStatus) return nil } +func (ec *Controller) recordEvent(ex *v1alpha1.Experiment, prevStatus v1alpha1.ExperimentStatus, newStatus *v1alpha1.ExperimentStatus) { + if prevStatus.Phase != newStatus.Phase { + eventType := corev1.EventTypeNormal + switch newStatus.Phase { + case v1alpha1.AnalysisPhaseError, v1alpha1.AnalysisPhaseFailed, v1alpha1.AnalysisPhaseInconclusive: + eventType = corev1.EventTypeWarning + } + ec.recorder.Eventf(ex, record.EventOptions{EventType: eventType, EventReason: "Experiment" + string(newStatus.Phase)}, "Experiment transitioned from %s -> %s", prevStatus.Phase, newStatus.Phase) + } +} + // enqueueIfCompleted conditionally enqueues the AnalysisRun's Experiment if the run is complete func (ec *Controller) enqueueIfCompleted(obj any) { run := unstructuredutil.ObjectToAnalysisRun(obj) diff --git a/experiments/experiment.go b/experiments/experiment.go index 1b40464015..b27ed5326a 100644 --- a/experiments/experiment.go +++ b/experiments/experiment.go @@ -523,7 +523,6 @@ func (ec *experimentContext) ResolveAnalysisRunArgs(args []v1alpha1.Argument) ([ } func (ec *experimentContext) calculateStatus() *v1alpha1.ExperimentStatus { - prevStatus := ec.newStatus.DeepCopy() switch ec.newStatus.Phase { case "": ec.newStatus.Phase = v1alpha1.AnalysisPhasePending @@ -568,14 +567,6 @@ func (ec *experimentContext) calculateStatus() *v1alpha1.ExperimentStatus { } } ec.newStatus = calculateExperimentConditions(ec.ex, *ec.newStatus) - if prevStatus.Phase != ec.newStatus.Phase { - eventType := corev1.EventTypeNormal - switch ec.newStatus.Phase { - case v1alpha1.AnalysisPhaseError, v1alpha1.AnalysisPhaseFailed, v1alpha1.AnalysisPhaseInconclusive: - eventType = corev1.EventTypeWarning - } - ec.recorder.Eventf(ec.ex, record.EventOptions{EventType: eventType, EventReason: "Experiment" + string(ec.newStatus.Phase)}, "Experiment transitioned from %s -> %s", prevStatus.Phase, ec.newStatus.Phase) - } return ec.newStatus } diff --git a/experiments/service.go b/experiments/service.go index afab417679..aaac3d3fee 100644 --- a/experiments/service.go +++ b/experiments/service.go @@ -6,12 +6,13 @@ import ( "k8s.io/apimachinery/pkg/api/errors" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" - logutil "github.com/argoproj/argo-rollouts/utils/log" log "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + logutil "github.com/argoproj/argo-rollouts/utils/log" ) var experimentKind = v1alpha1.SchemeGroupVersion.WithKind("Experiment") diff --git a/go.mod b/go.mod index 422327627b..bc3db1a35e 100644 --- a/go.mod +++ b/go.mod @@ -1,48 +1,47 @@ module github.com/argoproj/argo-rollouts -go 1.21 - -toolchain go1.21.8 +go 1.23 require ( github.com/antonmedv/expr v1.15.5 github.com/argoproj/notifications-engine v0.4.1-0.20240219110818-7a069766e954 github.com/argoproj/pkg v0.13.6 - github.com/aws/aws-sdk-go-v2 v1.26.1 - github.com/aws/aws-sdk-go-v2/config v1.27.10 - github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.37.0 - github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.30.5 - github.com/aws/smithy-go v1.20.2 + github.com/aws/aws-sdk-go-v2 v1.32.6 + github.com/aws/aws-sdk-go-v2/config v1.28.6 + github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.43.3 + github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.31.3 + github.com/aws/smithy-go v1.22.1 github.com/blang/semver v3.5.1+incompatible github.com/bombsimon/logrusr/v4 v4.1.0 github.com/evanphx/json-patch/v5 v5.9.0 github.com/gogo/protobuf v1.3.2 github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.5.4 + github.com/google/uuid v1.6.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 - github.com/hashicorp/go-plugin v1.6.0 - github.com/influxdata/influxdb-client-go/v2 v2.13.0 + github.com/hashicorp/go-plugin v1.6.2 + github.com/influxdata/influxdb-client-go/v2 v2.14.0 github.com/juju/ansiterm v1.0.0 github.com/machinebox/graphql v0.2.2 github.com/mitchellh/mapstructure v1.5.0 - github.com/newrelic/newrelic-client-go v1.1.0 + github.com/newrelic/newrelic-client-go/v2 v2.51.3 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.18.0 - github.com/prometheus/client_model v0.6.0 + github.com/prometheus/client_model v0.6.1 github.com/prometheus/common v0.47.0 github.com/prometheus/common/sigv4 v0.1.0 github.com/servicemeshinterface/smi-sdk-go v0.5.0 github.com/sirupsen/logrus v1.9.3 github.com/soheilhy/cmux v0.1.5 github.com/spaceapegames/go-wavefront v1.8.1 - github.com/spf13/cobra v1.8.0 - github.com/stretchr/testify v1.9.0 + github.com/spf13/cobra v1.8.1 + github.com/stretchr/testify v1.10.0 github.com/tj/assert v0.0.3 github.com/valyala/fasttemplate v1.2.2 - golang.org/x/oauth2 v0.18.0 - google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 - google.golang.org/grpc v1.62.1 - google.golang.org/protobuf v1.33.0 + golang.org/x/oauth2 v0.24.0 + google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 + google.golang.org/grpc v1.68.1 + google.golang.org/protobuf v1.35.2 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.29.3 k8s.io/apiextensions-apiserver v0.29.3 @@ -61,8 +60,7 @@ require ( ) require ( - cloud.google.com/go/compute v1.23.3 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/compute/metadata v0.5.0 // indirect github.com/PagerDuty/go-pagerduty v1.7.0 // indirect github.com/bradleyfalzon/ghinstallation/v2 v2.5.0 // indirect github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 // indirect @@ -83,21 +81,21 @@ require ( github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/aws/aws-sdk-go v1.44.116 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.10 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.47 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect github.com/aws/aws-sdk-go-v2/service/sqs v1.29.7 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.20.4 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/cloudflare/circl v1.3.3 // indirect github.com/coreos/go-semver v0.3.1 // indirect @@ -107,18 +105,18 @@ require ( github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect - github.com/fatih/color v1.7.0 // indirect - github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/fatih/color v1.16.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect github.com/go-errors/errors v1.4.2 // indirect - github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/golang/glog v1.2.0 // indirect + github.com/golang/glog v1.2.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/btree v1.1.2 // indirect github.com/google/cel-go v0.17.7 // indirect @@ -129,7 +127,6 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect @@ -138,11 +135,11 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-hclog v0.14.1 // indirect - github.com/hashicorp/go-retryablehttp v0.7.1 // indirect + github.com/hashicorp/go-hclog v1.6.3 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/yamux v0.1.1 // indirect github.com/huandu/xstrings v1.3.3 // indirect - github.com/imdario/mergo v0.3.13 // indirect + github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -153,9 +150,8 @@ require ( github.com/lunixbochs/vtclean v1.0.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect - github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/spdystream v0.2.0 // indirect @@ -182,42 +178,41 @@ require ( github.com/stretchr/objx v0.5.2 // indirect github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fastjson v1.6.3 // indirect + github.com/valyala/fastjson v1.6.4 // indirect github.com/whilp/git-urls v0.0.0-20191001220047-6db9661140c0 // indirect github.com/xlab/treeprint v1.2.0 // indirect go.etcd.io/etcd/api/v3 v3.5.10 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect go.etcd.io/etcd/client/v3 v3.5.10 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect - go.opentelemetry.io/otel v1.19.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect + go.opentelemetry.io/otel v1.22.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect - go.opentelemetry.io/otel/metric v1.19.0 // indirect - go.opentelemetry.io/otel/sdk v1.19.0 // indirect - go.opentelemetry.io/otel/trace v1.19.0 // indirect + go.opentelemetry.io/otel/metric v1.22.0 // indirect + go.opentelemetry.io/otel/sdk v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.22.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.19.0 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.27.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect - golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.22.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.16.1 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/term v0.24.0 // indirect + golang.org/x/text v0.18.0 // indirect + golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect gomodules.xyz/envconfig v1.3.1-0.20190308184047-426f31af0d45 // indirect gomodules.xyz/notify v0.1.1 // indirect - google.golang.org/api v0.149.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/api v0.162.0 // indirect + google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 3aff874a74..c4df8353ed 100644 --- a/go.sum +++ b/go.sum @@ -19,11 +19,9 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= -cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= +cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -98,38 +96,38 @@ github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2z github.com/aws/aws-sdk-go v1.44.39/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.44.116 h1:NpLIhcvLWXJZAEwvPj3TDHeqp7DleK6ZUVYyW01WNHY= github.com/aws/aws-sdk-go v1.44.116/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= -github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= -github.com/aws/aws-sdk-go-v2/config v1.27.10 h1:PS+65jThT0T/snC5WjyfHHyUgG+eBoupSDV+f838cro= -github.com/aws/aws-sdk-go-v2/config v1.27.10/go.mod h1:BePM7Vo4OBpHreKRUMuDXX+/+JWP38FLkzl5m27/Jjs= -github.com/aws/aws-sdk-go-v2/credentials v1.17.10 h1:qDZ3EA2lv1KangvQB6y258OssCHD0xvaGiEDkG4X/10= -github.com/aws/aws-sdk-go-v2/credentials v1.17.10/go.mod h1:6t3sucOaYDwDssHQa0ojH1RpmVmF5/jArkye1b2FKMI= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= -github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.37.0 h1:sGGUnU/pUSzjrcCvQgN2pEc3aTQILyK2rRsWVY5CSt0= -github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.37.0/go.mod h1:U12sr6Lt14X96f16t+rR52+2BdqtydwN7DjEEHRMjO0= -github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.30.5 h1:/x2u/TOx+n17U+gz98TOw1HKJom0EOqrhL4SjrHr0cQ= -github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.30.5/go.mod h1:e1McVqsud0JOERidvppLEHnuCdh/X6MRyL5L0LseAUk= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk= +github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4= +github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2/config v1.28.6 h1:D89IKtGrs/I3QXOLNTH93NJYtDhm8SYa9Q5CsPShmyo= +github.com/aws/aws-sdk-go-v2/config v1.28.6/go.mod h1:GDzxJ5wyyFSCoLkS+UhGB0dArhb9mI+Co4dHtoTxbko= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47 h1:48bA+3/fCdi2yAwVt+3COvmatZ6jUDNkDTIsqDiMUdw= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47/go.mod h1:+KdckOejLW3Ks3b0E3b5rHsr2f9yuORBum0WPnE5o5w= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 h1:AmoU1pziydclFT/xRV+xXE/Vb8fttJCLRPv8oAkprc0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21/go.mod h1:AjUdLYe4Tgs6kpH4Bv7uMZo7pottoyHMn4eTcIcneaY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25/go.mod h1:DBdPrgeocww+CSl1C8cEV8PN1mHMBhuCDLpXezyvWkE= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.43.3 h1:nQLG9irjDGUFXVPDHzjCGEEwh0hZ6BcxTvHOod1YsP4= +github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.43.3/go.mod h1:URs8sqsyaxiAZkKP6tOEmhcs9j2ynFIomqOKY/CAHJc= +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.31.3 h1:Avh8YS+sgb2OKRht0wdNwY8tqtsCzVrmc8dG8Wfy9LI= +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.31.3/go.mod h1:HbtHaw/hnNPaiqcyYnheILVyn81wOZiX9n2gYF5tPmM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 h1:50+XsN70RS7dwJ2CkVNXzj7U2L1HKP8nqTd3XWEXBN4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6/go.mod h1:WqgLmwY7so32kG01zD8CPTJWVWM+TzJoOVHwTg4aPug= github.com/aws/aws-sdk-go-v2/service/sqs v1.29.7 h1:tRNrFDGRm81e6nTX5Q4CFblea99eAfm0dxXazGpLceU= github.com/aws/aws-sdk-go-v2/service/sqs v1.29.7/go.mod h1:8GWUDux5Z2h6z2efAtr54RdHXtLm8sq7Rg85ZNY/CZM= -github.com/aws/aws-sdk-go-v2/service/sso v1.20.4 h1:WzFol5Cd+yDxPAdnzTA5LmpHYSWinhmSj4rQChV0ee8= -github.com/aws/aws-sdk-go-v2/service/sso v1.20.4/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak= -github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU= -github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw= -github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= -github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 h1:rLnYAfXQ3YAccocshIH5mzNNwZBkBo+bP6EhIxak6Hw= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7/go.mod h1:ZHtuQJ6t9A/+YDuxOLnbryAmITtr8UysSny3qcyvJTc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 h1:JnhTZR3PiYDNKlXy50/pNeix9aGMo6lLpXwJ1mw8MD4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6/go.mod h1:URronUEGfXZN1VpdktPSD1EkAL9mfrV+2F4sjH38qOY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 h1:s4074ZO1Hk8qv65GqNXqDjmkf4HSQqJukaLuuW0TpDA= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2/go.mod h1:mVggCnIWoM09jP71Wh+ea7+5gAp53q+49wDFs1SW5z8= +github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= +github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beevik/ntp v0.2.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -156,8 +154,8 @@ github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqy github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -168,8 +166,8 @@ github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtM github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= -github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= +github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI= +github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27/go.mod h1:VQx0hjo2oUeQkQUET7wRwradO6f+fN5jzXgB/zROxxE= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= @@ -177,7 +175,7 @@ github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03V github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -194,8 +192,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/elazarl/goproxy v0.0.0-20220417044921-416226498f94 h1:VIy7cdK7ufs7ctpTFkXJHm1uP3dJSnCGSPysEICB1so= -github.com/elazarl/goproxy v0.0.0-20220417044921-416226498f94/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/elazarl/goproxy v0.0.0-20231117061959-7cc037d33fb5 h1:m62nsMU279qRD9PQSWD1l66kmkXzuYcnVJqL4XLeV2M= +github.com/elazarl/goproxy v0.0.0-20231117061959-7cc037d33fb5/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= @@ -204,8 +202,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= -github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= +github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM= +github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= github.com/evanphx/json-patch v0.0.0-20200808040245-162e5629780b/go.mod h1:NAJj0yf/KaRKURN6nyi7A9IZydMivZEm9oQLWNjfKDc= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= @@ -217,10 +215,13 @@ github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSY github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= -github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -246,8 +247,9 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= @@ -284,8 +286,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= -github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= +github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -408,17 +410,15 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v0.14.1 h1:nQcJDQwIAGnmoUWp8ubocEX40cCml/17YkF6csQLReU= -github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-plugin v1.6.0 h1:wgd4KxHJTVGGqWBq4QPB1i5BZNEx9BR8+OFmHDmTk8A= -github.com/hashicorp/go-plugin v1.6.0/go.mod h1:lBS5MtSSBZk0SHc66KACcjjlU6WzEVP/8pwz68aMkCI= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-plugin v1.6.2 h1:zdGAEd0V1lCaU0u+MxWQhtSDQmahpkwOun8U8EiRVog= +github.com/hashicorp/go-plugin v1.6.2/go.mod h1:CkgLQ5CZqNmdL9U9JzM532t8ZiYQ35+pj3b1FD37R0Q= github.com/hashicorp/go-retryablehttp v0.5.1/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ= -github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= @@ -430,13 +430,13 @@ github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= -github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/influxdata/influxdb-client-go/v2 v2.13.0 h1:ioBbLmR5NMbAjP4UVA5r9b5xGjpABD7j65pI8kFphDM= -github.com/influxdata/influxdb-client-go/v2 v2.13.0/go.mod h1:k+spCbt9hcvqvUiz0sr5D8LolXHqAAOfPw9v/RIRHl4= +github.com/influxdata/influxdb-client-go/v2 v2.14.0 h1:AjbBfJuq+QoaXNcrova8smSjwJdUHnwvfjMF71M1iI4= +github.com/influxdata/influxdb-client-go/v2 v2.14.0/go.mod h1:Ahpm3QXKMJslpXl3IftVLVezreAUtBOTZssDrjZEFHI= github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf h1:7JTmneyiNEwVBOHSjoMxiWAqB992atOeepeFYegn5RU= github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= github.com/jaytaylor/html2text v0.0.0-20190408195923-01ec452cbe43/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= @@ -505,17 +505,17 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.10/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= @@ -525,8 +525,6 @@ github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFW github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -556,8 +554,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/newrelic/newrelic-client-go v1.1.0 h1:aflNjzQ21c+2GwBVh+UbAf9lznkRfCcVABoc5UM4IXw= -github.com/newrelic/newrelic-client-go v1.1.0/go.mod h1:RYMXt7hgYw7nzuXIGd2BH0F1AivgWw7WrBhNBQZEB4k= +github.com/newrelic/newrelic-client-go/v2 v2.51.3 h1:Bu/cUs6nfMjQMPBcxxHt4Xm30tKDT7ttYy/XRDsWP6Y= +github.com/newrelic/newrelic-client-go/v2 v2.51.3/go.mod h1:+RRjI3nDGWT3kLm9Oi3QxpBm70uu8q1upEHBVWCZFpo= github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oapi-codegen/runtime v1.0.0 h1:P4rqFX5fMFWqRzY9M/3YF9+aPSPPB06IzP2P7oOxrWo= @@ -606,8 +604,8 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1: github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= -github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= @@ -659,8 +657,8 @@ github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -686,8 +684,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= @@ -696,8 +694,8 @@ github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJ github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc= -github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= +github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= +github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/whilp/git-urls v0.0.0-20191001220047-6db9661140c0 h1:qqllXPzXh+So+mmANlX/gCJrgo+1kQyshMoQ+NASzm0= @@ -735,22 +733,22 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 h1:ZOLJc06r4CB42laIXg/7udr0pbZyuAihN10A/XuiQRY= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0/go.mod h1:5z+/ZWJQKXa9YT34fQNx5K8Hd1EoIhvtUygUQPqEOgQ= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 h1:KfYpVmrjI7JuToy5k8XV3nkapjWx48k4E4JOtVstzQI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0/go.mod h1:SeQhzAEccGVZVEy7aH87Nh0km+utSpo1pTv6eMMop48= -go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= -go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 h1:UNQQKPfTDe1J81ViolILjTKPr9WetKW6uei2hFgJmFs= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw= +go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= +go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= -go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= -go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= -go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= -go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= -go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= -go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg= +go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= +go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= +go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= @@ -783,8 +781,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -820,8 +818,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -869,8 +867,8 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -878,8 +876,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= -golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= -golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= +golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -893,8 +891,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -916,7 +914,6 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -950,8 +947,10 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -961,8 +960,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -971,8 +970,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -981,18 +980,17 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1045,8 +1043,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= -golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1072,8 +1070,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.149.0 h1:b2CqT6kG+zqJIVKRQ3ELJVLN1PwHZ6DJ3dW8yl82rgY= -google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9I7qI= +google.golang.org/api v0.162.0 h1:Vhs54HkaEpkMBdgGdOT2P6F0csGG/vxDS0hWHJzmmps= +google.golang.org/api v0.162.0/go.mod h1:6SulDkfoBIg4NFmCuZ39XeeAgSHCPecfSUuDyYlAHs0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1081,8 +1079,6 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1113,12 +1109,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= -google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= -google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU= -google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1:hjSy6tcFQZ171igDaN5QHOw2n6vx40juYbC/x67CEhc= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1133,8 +1129,8 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= -google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= +google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1148,8 +1144,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= +google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= @@ -1180,7 +1176,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/hack/gen-crd-spec/main.go b/hack/gen-crd-spec/main.go index a3e31d92b8..60c1bde9d7 100644 --- a/hack/gen-crd-spec/main.go +++ b/hack/gen-crd-spec/main.go @@ -96,11 +96,12 @@ func NewCustomResourceDefinition() []*extensionsobj.CustomResourceDefinition { // clean up stuff left by controller-gen deleteFile("config/webhook/manifests.yaml") deleteFile("config/webhook") - deleteFile("config/argoproj.io_analysisruns.yaml") - deleteFile("config/argoproj.io_analysistemplates.yaml") - deleteFile("config/argoproj.io_clusteranalysistemplates.yaml") - deleteFile("config/argoproj.io_experiments.yaml") - deleteFile("config/argoproj.io_rollouts.yaml") + deleteFile("config/crd/argoproj.io_analysisruns.yaml") + deleteFile("config/crd/argoproj.io_analysistemplates.yaml") + deleteFile("config/crd/argoproj.io_clusteranalysistemplates.yaml") + deleteFile("config/crd/argoproj.io_experiments.yaml") + deleteFile("config/crd/argoproj.io_rollouts.yaml") + deleteFile("config/crd") deleteFile("config") crds := []*extensionsobj.CustomResourceDefinition{} diff --git a/hack/installers/install-codegen-go-tools.sh b/hack/installers/install-codegen-go-tools.sh index 9630fd615b..501cb535e3 100755 --- a/hack/installers/install-codegen-go-tools.sh +++ b/hack/installers/install-codegen-go-tools.sh @@ -45,13 +45,13 @@ go_mod_install k8s.io/code-generator/cmd/lister-gen go_mod_install k8s.io/kube-openapi/cmd/openapi-gen # controller-gen is run by ./hack/gen-crd-spec to generate the CRDs -go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.13.0 +go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.14.0 # swagger cli is used to generate swagger docs go install github.com/go-swagger/go-swagger/cmd/swagger@v0.30.5 # goimports is used to auto-format generated code -go install golang.org/x/tools/cmd/goimports@v0.11.0 +go install golang.org/x/tools/cmd/goimports@v0.20.0 # mockery is used for generating mock -go install github.com/vektra/mockery/v2@v2.32.0 +go install github.com/vektra/mockery/v2@v2.42.2 diff --git a/hack/installers/install-dev-tools.sh b/hack/installers/install-dev-tools.sh index 16ead66083..89a88deed0 100755 --- a/hack/installers/install-dev-tools.sh +++ b/hack/installers/install-dev-tools.sh @@ -7,7 +7,7 @@ PATH="${DIST_PATH}:${PATH}" mkdir -p ${DIST_PATH} -gotestsum_version=1.10.1 +gotestsum_version=1.11.0 OS=$(go env GOOS) ARCH=$(go env GOARCH) diff --git a/hack/installers/install-protoc.sh b/hack/installers/install-protoc.sh index 1f3dd8e7bc..1c557be08f 100755 --- a/hack/installers/install-protoc.sh +++ b/hack/installers/install-protoc.sh @@ -5,7 +5,7 @@ PROJECT_ROOT=$(cd $(dirname ${BASH_SOURCE})/../..; pwd) DIST_PATH="${PROJECT_ROOT}/dist" PATH="${DIST_PATH}:${PATH}" -protoc_version=3.17.3 +protoc_version=26.1 OS=$(go env GOOS) ARCH=$(go env GOARCH) diff --git a/hack/swagger-codegen.sh b/hack/swagger-codegen.sh index 9ba2cf6634..96c1728f0c 100755 --- a/hack/swagger-codegen.sh +++ b/hack/swagger-codegen.sh @@ -1,9 +1,9 @@ #!/bin/bash -export SWAGGER_CODEGEN_VERSION=3.0.25 +export SWAGGER_CODEGEN_VERSION=3.0.54 PROJECT_ROOT=$(cd $(dirname ${BASH_SOURCE})/..; pwd) test -f "$PROJECT_ROOT/dist/swagger-codegen-cli-${SWAGGER_CODEGEN_VERSION}.jar" || \ curl https://repo1.maven.org/maven2/io/swagger/codegen/v3/swagger-codegen-cli/${SWAGGER_CODEGEN_VERSION}/swagger-codegen-cli-${SWAGGER_CODEGEN_VERSION}.jar -o "$PROJECT_ROOT/dist/swagger-codegen-cli-${SWAGGER_CODEGEN_VERSION}.jar" -docker run --rm -v $PROJECT_ROOT:/src -w /src/ui -t maven:3-jdk-8 java -jar /src/dist/swagger-codegen-cli-${SWAGGER_CODEGEN_VERSION}.jar $@ \ No newline at end of file +docker run --rm -v "$PROJECT_ROOT:/src" -w /src/ui -t maven:3-jdk-11-slim java -jar /src/dist/swagger-codegen-cli-${SWAGGER_CODEGEN_VERSION}.jar "$@" diff --git a/hack/update-mocks.sh b/hack/update-mocks.sh index 6f1bcfad4a..9e4f397db1 100755 --- a/hack/update-mocks.sh +++ b/hack/update-mocks.sh @@ -21,3 +21,13 @@ mockery \ --dir "${PROJECT_ROOT}"/rollout/trafficrouting \ --name TrafficRoutingReconciler \ --output "${PROJECT_ROOT}"/rollout/mocks + +mockery \ + --dir "${PROJECT_ROOT}"/rollout/steps/plugin \ + --name "Resolver|StepPlugin" \ + --output "${PROJECT_ROOT}"/rollout/steps/plugin/mocks + +mockery \ + --dir "${PROJECT_ROOT}"/rollout/steps/plugin/rpc \ + --name "StepPlugin" \ + --output "${PROJECT_ROOT}"/rollout/steps/plugin/rpc/mocks diff --git a/ingress/ingress_test.go b/ingress/ingress_test.go index be588dd51a..0336c0e3f3 100644 --- a/ingress/ingress_test.go +++ b/ingress/ingress_test.go @@ -18,12 +18,13 @@ import ( k8sfake "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/util/workqueue" + "k8s.io/client-go/tools/cache" + "github.com/argoproj/argo-rollouts/controller/metrics" "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/fake" informers "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions" ingressutil "github.com/argoproj/argo-rollouts/utils/ingress" - "k8s.io/client-go/tools/cache" ) const stableService string = "test-stable-service" diff --git a/manifests/base/argo-rollouts-deployment.yaml b/manifests/base/argo-rollouts-deployment.yaml index 046a1a0bba..285617fde4 100644 --- a/manifests/base/argo-rollouts-deployment.yaml +++ b/manifests/base/argo-rollouts-deployment.yaml @@ -52,7 +52,20 @@ spec: readOnlyRootFilesystem: true seccompProfile: type: RuntimeDefault + resources: + limits: + ephemeral-storage: 1Gi + volumeMounts: + - name: plugin-bin + mountPath: /home/argo-rollouts/plugin-bin + - name: tmp + mountPath: /tmp securityContext: runAsNonRoot: true + volumes: + - name: plugin-bin + emptyDir: {} + - name: tmp + emptyDir: {} strategy: type: RollingUpdate diff --git a/manifests/crds/analysis-run-crd.yaml b/manifests/crds/analysis-run-crd.yaml index c0199f2de2..e6256b151d 100644 --- a/manifests/crds/analysis-run-crd.yaml +++ b/manifests/crds/analysis-run-crd.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: analysisruns.argoproj.io spec: group: argoproj.io @@ -98,6 +98,11 @@ spec: - type: integer - type: string x-kubernetes-int-or-string: true + consecutiveSuccessLimit: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true count: anyOf: - type: integer @@ -179,7 +184,6 @@ spec: datadog: properties: aggregator: - default: last enum: - avg - min @@ -208,6 +212,13 @@ spec: type: object query: type: string + secretRef: + properties: + name: + type: string + namespaced: + type: boolean + type: object type: object graphite: properties: @@ -3016,6 +3027,9 @@ spec: type: string query: type: string + timeout: + format: int64 + type: integer required: - query type: object @@ -3067,6 +3081,15 @@ spec: type: boolean query: type: string + rangeQuery: + properties: + end: + type: string + start: + type: string + step: + type: string + type: object timeout: format: int64 type: integer @@ -3201,6 +3224,9 @@ spec: consecutiveError: format: int32 type: integer + consecutiveSuccess: + format: int32 + type: integer count: format: int32 type: integer diff --git a/manifests/crds/analysis-template-crd.yaml b/manifests/crds/analysis-template-crd.yaml index aaf34277bd..0b9f22d2cb 100644 --- a/manifests/crds/analysis-template-crd.yaml +++ b/manifests/crds/analysis-template-crd.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: analysistemplates.argoproj.io spec: group: argoproj.io @@ -94,6 +94,11 @@ spec: - type: integer - type: string x-kubernetes-int-or-string: true + consecutiveSuccessLimit: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true count: anyOf: - type: integer @@ -175,7 +180,6 @@ spec: datadog: properties: aggregator: - default: last enum: - avg - min @@ -204,6 +208,13 @@ spec: type: object query: type: string + secretRef: + properties: + name: + type: string + namespaced: + type: boolean + type: object type: object graphite: properties: @@ -3012,6 +3023,9 @@ spec: type: string query: type: string + timeout: + format: int64 + type: integer required: - query type: object @@ -3063,6 +3077,15 @@ spec: type: boolean query: type: string + rangeQuery: + properties: + end: + type: string + start: + type: string + step: + type: string + type: object timeout: format: int64 type: integer diff --git a/manifests/crds/cluster-analysis-template-crd.yaml b/manifests/crds/cluster-analysis-template-crd.yaml index ed75ada312..f966d4f8b0 100644 --- a/manifests/crds/cluster-analysis-template-crd.yaml +++ b/manifests/crds/cluster-analysis-template-crd.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: clusteranalysistemplates.argoproj.io spec: group: argoproj.io @@ -94,6 +94,11 @@ spec: - type: integer - type: string x-kubernetes-int-or-string: true + consecutiveSuccessLimit: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true count: anyOf: - type: integer @@ -175,7 +180,6 @@ spec: datadog: properties: aggregator: - default: last enum: - avg - min @@ -204,6 +208,13 @@ spec: type: object query: type: string + secretRef: + properties: + name: + type: string + namespaced: + type: boolean + type: object type: object graphite: properties: @@ -3012,6 +3023,9 @@ spec: type: string query: type: string + timeout: + format: int64 + type: integer required: - query type: object @@ -3063,6 +3077,15 @@ spec: type: boolean query: type: string + rangeQuery: + properties: + end: + type: string + start: + type: string + step: + type: string + type: object timeout: format: int64 type: integer diff --git a/manifests/crds/experiment-crd.yaml b/manifests/crds/experiment-crd.yaml index e4287b51e0..0dae59cb3b 100644 --- a/manifests/crds/experiment-crd.yaml +++ b/manifests/crds/experiment-crd.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: experiments.argoproj.io spec: group: argoproj.io diff --git a/manifests/crds/rollout-crd.yaml b/manifests/crds/rollout-crd.yaml index cd244b8b98..aa06634f9b 100755 --- a/manifests/crds/rollout-crd.yaml +++ b/manifests/crds/rollout-crd.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: rollouts.argoproj.io spec: group: argoproj.io @@ -662,6 +662,16 @@ spec: - type: string x-kubernetes-int-or-string: true type: object + plugin: + properties: + config: + type: object + x-kubernetes-preserve-unknown-fields: true + name: + type: string + required: + - name + type: object setCanaryScale: properties: matchTrafficWeight: @@ -934,6 +944,10 @@ spec: type: object annotationPrefix: type: string + canaryIngressAnnotations: + additionalProperties: + type: string + type: object stableIngress: type: string stableIngresses: @@ -3717,6 +3731,45 @@ spec: type: object stablePingPong: type: string + stepPluginStatuses: + items: + properties: + backoff: + type: string + disabled: + type: boolean + executions: + format: int32 + type: integer + finishedAt: + format: date-time + type: string + index: + format: int32 + type: integer + message: + type: string + name: + type: string + operation: + type: string + phase: + type: string + startedAt: + format: date-time + type: string + status: + type: object + x-kubernetes-preserve-unknown-fields: true + updatedAt: + format: date-time + type: string + required: + - index + - name + - operation + type: object + type: array weights: properties: additional: diff --git a/manifests/install.yaml b/manifests/install.yaml index 1b5013ef4a..acaf436f2d 100755 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: analysisruns.argoproj.io spec: group: argoproj.io @@ -99,6 +99,11 @@ spec: - type: integer - type: string x-kubernetes-int-or-string: true + consecutiveSuccessLimit: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true count: anyOf: - type: integer @@ -180,7 +185,6 @@ spec: datadog: properties: aggregator: - default: last enum: - avg - min @@ -209,6 +213,13 @@ spec: type: object query: type: string + secretRef: + properties: + name: + type: string + namespaced: + type: boolean + type: object type: object graphite: properties: @@ -3017,6 +3028,9 @@ spec: type: string query: type: string + timeout: + format: int64 + type: integer required: - query type: object @@ -3068,6 +3082,15 @@ spec: type: boolean query: type: string + rangeQuery: + properties: + end: + type: string + start: + type: string + step: + type: string + type: object timeout: format: int64 type: integer @@ -3202,6 +3225,9 @@ spec: consecutiveError: format: int32 type: integer + consecutiveSuccess: + format: int32 + type: integer count: format: int32 type: integer @@ -3297,7 +3323,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: analysistemplates.argoproj.io spec: group: argoproj.io @@ -3389,6 +3415,11 @@ spec: - type: integer - type: string x-kubernetes-int-or-string: true + consecutiveSuccessLimit: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true count: anyOf: - type: integer @@ -3470,7 +3501,6 @@ spec: datadog: properties: aggregator: - default: last enum: - avg - min @@ -3499,6 +3529,13 @@ spec: type: object query: type: string + secretRef: + properties: + name: + type: string + namespaced: + type: boolean + type: object type: object graphite: properties: @@ -6307,6 +6344,9 @@ spec: type: string query: type: string + timeout: + format: int64 + type: integer required: - query type: object @@ -6358,6 +6398,15 @@ spec: type: boolean query: type: string + rangeQuery: + properties: + end: + type: string + start: + type: string + step: + type: string + type: object timeout: format: int64 type: integer @@ -6465,7 +6514,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: clusteranalysistemplates.argoproj.io spec: group: argoproj.io @@ -6557,6 +6606,11 @@ spec: - type: integer - type: string x-kubernetes-int-or-string: true + consecutiveSuccessLimit: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true count: anyOf: - type: integer @@ -6638,7 +6692,6 @@ spec: datadog: properties: aggregator: - default: last enum: - avg - min @@ -6667,6 +6720,13 @@ spec: type: object query: type: string + secretRef: + properties: + name: + type: string + namespaced: + type: boolean + type: object type: object graphite: properties: @@ -9475,6 +9535,9 @@ spec: type: string query: type: string + timeout: + format: int64 + type: integer required: - query type: object @@ -9526,6 +9589,15 @@ spec: type: boolean query: type: string + rangeQuery: + properties: + end: + type: string + start: + type: string + step: + type: string + type: object timeout: format: int64 type: integer @@ -9633,7 +9705,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: experiments.argoproj.io spec: group: argoproj.io @@ -12492,7 +12564,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: rollouts.argoproj.io spec: group: argoproj.io @@ -13152,6 +13224,16 @@ spec: - type: string x-kubernetes-int-or-string: true type: object + plugin: + properties: + config: + type: object + x-kubernetes-preserve-unknown-fields: true + name: + type: string + required: + - name + type: object setCanaryScale: properties: matchTrafficWeight: @@ -13424,6 +13506,10 @@ spec: type: object annotationPrefix: type: string + canaryIngressAnnotations: + additionalProperties: + type: string + type: object stableIngress: type: string stableIngresses: @@ -16207,6 +16293,45 @@ spec: type: object stablePingPong: type: string + stepPluginStatuses: + items: + properties: + backoff: + type: string + disabled: + type: boolean + executions: + format: int32 + type: integer + finishedAt: + format: date-time + type: string + index: + format: int32 + type: integer + message: + type: string + name: + type: string + operation: + type: string + phase: + type: string + startedAt: + format: date-time + type: string + status: + type: object + x-kubernetes-preserve-unknown-fields: true + updatedAt: + format: date-time + type: string + required: + - index + - name + - operation + type: object + type: array weights: properties: additional: @@ -16420,6 +16545,7 @@ rules: - get - list - watch + - update - apiGroups: - "" resources: @@ -16757,6 +16883,9 @@ spec: periodSeconds: 5 successThreshold: 1 timeoutSeconds: 4 + resources: + limits: + ephemeral-storage: 1Gi securityContext: allowPrivilegeEscalation: false capabilities: @@ -16765,6 +16894,16 @@ spec: readOnlyRootFilesystem: true seccompProfile: type: RuntimeDefault + volumeMounts: + - mountPath: /home/argo-rollouts/plugin-bin + name: plugin-bin + - mountPath: /tmp + name: tmp securityContext: runAsNonRoot: true serviceAccountName: argo-rollouts + volumes: + - emptyDir: {} + name: plugin-bin + - emptyDir: {} + name: tmp diff --git a/manifests/namespace-install.yaml b/manifests/namespace-install.yaml index 0a7adf80ed..16ea68f99c 100644 --- a/manifests/namespace-install.yaml +++ b/manifests/namespace-install.yaml @@ -75,6 +75,7 @@ rules: - get - list - watch + - update - apiGroups: - "" resources: @@ -413,6 +414,9 @@ spec: periodSeconds: 5 successThreshold: 1 timeoutSeconds: 4 + resources: + limits: + ephemeral-storage: 1Gi securityContext: allowPrivilegeEscalation: false capabilities: @@ -421,6 +425,16 @@ spec: readOnlyRootFilesystem: true seccompProfile: type: RuntimeDefault + volumeMounts: + - mountPath: /home/argo-rollouts/plugin-bin + name: plugin-bin + - mountPath: /tmp + name: tmp securityContext: runAsNonRoot: true serviceAccountName: argo-rollouts + volumes: + - emptyDir: {} + name: plugin-bin + - emptyDir: {} + name: tmp diff --git a/manifests/role/argo-rollouts-clusterrole.yaml b/manifests/role/argo-rollouts-clusterrole.yaml index aab253e5ea..b43ed1adef 100644 --- a/manifests/role/argo-rollouts-clusterrole.yaml +++ b/manifests/role/argo-rollouts-clusterrole.yaml @@ -67,6 +67,7 @@ rules: - get - list - watch + - update # services patch needed to update selector of canary/stable/active/preview services # services create needed to create and delete services for experiments - apiGroups: diff --git a/metricproviders/datadog/datadog.go b/metricproviders/datadog/datadog.go index 48b60fc0f3..65b96e10d9 100644 --- a/metricproviders/datadog/datadog.go +++ b/metricproviders/datadog/datadog.go @@ -2,17 +2,14 @@ package datadog import ( "bytes" - "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" - "os" "reflect" "strconv" - "strings" "time" "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" @@ -22,7 +19,6 @@ import ( timeutil "github.com/argoproj/argo-rollouts/utils/time" log "github.com/sirupsen/logrus" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) @@ -71,7 +67,7 @@ type datadogResponseV2 struct { Data struct { Attributes struct { Columns []struct { - Values []float64 + Values []*float64 } } Errors string @@ -224,6 +220,10 @@ func (p *Provider) createRequestV2(queries map[string]string, formula string, no "formula": formula, }} } + // we cannot leave aggregator empty as it will be passed as such to datadog API and fail + if aggregator == "" { + aggregator = "last" + } attribs := datadogQueryAttributes{ // Datadog requires milliseconds for v2 api @@ -320,7 +320,7 @@ func (p *Provider) parseResponseV2(metric v1alpha1.Metric, response *http.Respon } // Handle an empty query result - if reflect.ValueOf(res.Data.Attributes).IsZero() || len(res.Data.Attributes.Columns) == 0 || len(res.Data.Attributes.Columns[0].Values) == 0 { + if reflect.ValueOf(res.Data.Attributes).IsZero() || len(res.Data.Attributes.Columns) == 0 || len(res.Data.Attributes.Columns[0].Values) == 0 || res.Data.Attributes.Columns[0].Values[0] == nil { var nilFloat64 *float64 status, err := evaluate.EvaluateResult(nilFloat64, metric, p.logCtx) @@ -343,7 +343,7 @@ func (p *Provider) parseResponseV2(metric v1alpha1.Metric, response *http.Respon // Handle a populated query result column := res.Data.Attributes.Columns[0] - value := column.Values[0] + value := *column.Values[0] status, err := evaluate.EvaluateResult(value, metric, p.logCtx) return strconv.FormatFloat(value, 'f', -1, 64), status, err } @@ -365,18 +365,6 @@ func (p *Provider) GarbageCollect(run *v1alpha1.AnalysisRun, metric v1alpha1.Met return nil } -func lookupKeysInEnv(keys []string) map[string]string { - valuesByKey := make(map[string]string) - for i := range keys { - key := keys[i] - formattedKey := strings.ToUpper(strings.ReplaceAll(key, "-", "_")) - if value, ok := os.LookupEnv(fmt.Sprintf("DD_%s", formattedKey)); ok { - valuesByKey[key] = value - } - } - return valuesByKey -} - // The current gen tooling we are using can't generate CRD with all the validations we need. // This is unfortunate, user has more ways to deliver an invalid Analysis Template vs // being rejected on delivery by k8s (and allowing for a validation step if desired in CI/CD). @@ -418,32 +406,13 @@ func validateIncomingProps(dd *v1alpha1.DatadogMetric) error { return nil } -func NewDatadogProvider(logCtx log.Entry, kubeclientset kubernetes.Interface, metric v1alpha1.Metric) (*Provider, error) { - ns := defaults.Namespace() - - apiKey := "" - appKey := "" - address := "" - secretKeys := []string{DatadogApiKey, DatadogAppKey, DatadogAddress} - envValuesByKey := lookupKeysInEnv(secretKeys) - if len(envValuesByKey) == len(secretKeys) { - apiKey = envValuesByKey[DatadogApiKey] - appKey = envValuesByKey[DatadogAppKey] - address = envValuesByKey[DatadogAddress] - } else { - secret, err := kubeclientset.CoreV1().Secrets(ns).Get(context.TODO(), DatadogTokensSecretName, metav1.GetOptions{}) - if err != nil { - return nil, err - } - apiKey = string(secret.Data[DatadogApiKey]) - appKey = string(secret.Data[DatadogAppKey]) - if _, hasAddress := secret.Data[DatadogAddress]; hasAddress { - address = string(secret.Data[DatadogAddress]) - } +func NewDatadogProvider(logCtx log.Entry, kubeclientset kubernetes.Interface, namespace string, metric v1alpha1.Metric) (*Provider, error) { + address, apiKey, appKey, err := findCredentials(logCtx, kubeclientset, namespace, metric) + if err != nil { + return nil, err } if apiKey != "" && appKey != "" { - err := validateIncomingProps(metric.Provider.Datadog) if err != nil { return nil, err @@ -461,3 +430,31 @@ func NewDatadogProvider(logCtx log.Entry, kubeclientset kubernetes.Interface, me return nil, errors.New("API or App token not found") } } + +func findCredentials(logCtx log.Entry, kubeclientset kubernetes.Interface, namespace string, metric v1alpha1.Metric) (string, string, string, error) { + finders := []CredentialsFinder{} + secretName := metric.Provider.Datadog.SecretRef.Name + namespaced := metric.Provider.Datadog.SecretRef.Namespaced + credentialsNs := defaults.Namespace() + + if namespaced { + credentialsNs = namespace + if secretName == "" { + return "", "", "", errors.New("secret name is required for namespaced credentials") + } + } + + if secretName != "" { + finders = append(finders, NewSecretFinder(kubeclientset, secretName, credentialsNs)) + } else { + finders = append(finders, NewEnvVariablesFinder(), NewSecretFinder(kubeclientset, DatadogTokensSecretName, defaults.Namespace())) + } + for _, finder := range finders { + address, apiKey, appKey := finder.FindCredentials(logCtx) + if apiKey != "" && appKey != "" { + return address, apiKey, appKey, nil + } + } + + return "", "", "", errors.New("failed to find the credentials for datadog metrics provider") +} diff --git a/metricproviders/datadog/datadogV1_test.go b/metricproviders/datadog/datadogV1_test.go index af58d47617..06d261faf7 100644 --- a/metricproviders/datadog/datadogV1_test.go +++ b/metricproviders/datadog/datadogV1_test.go @@ -8,7 +8,6 @@ import ( "strconv" "testing" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" @@ -16,6 +15,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" k8sfake "k8s.io/client-go/kubernetes/fake" kubetesting "k8s.io/client-go/testing" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" ) func TestRunSuite(t *testing.T) { @@ -29,7 +30,6 @@ func TestRunSuite(t *testing.T) { Query: "avg:kubernetes.cpu.user.total{*}", }, } - ddProviderInterval10m := v1alpha1.MetricProvider{ Datadog: &v1alpha1.DatadogMetric{ Query: "avg:kubernetes.cpu.user.total{*}", @@ -37,8 +37,20 @@ func TestRunSuite(t *testing.T) { }, } + ddProviderNamespacedSecret := v1alpha1.MetricProvider{ + Datadog: &v1alpha1.DatadogMetric{ + Query: "avg:kubernetes.cpu.user.total{*}", + Interval: "10m", + SecretRef: v1alpha1.SecretRef{ + Name: "secret", + Namespaced: true, + }, + }, + } + // Test Cases tests := []struct { + name string serverURL string webServerStatus int webServerResponse string @@ -47,6 +59,7 @@ func TestRunSuite(t *testing.T) { expectedValue string expectedPhase v1alpha1.AnalysisPhase expectedErrorMessage string + expectedErrorProvider bool useEnvVarForKeys bool }{ // When last value of time series matches condition then succeed. @@ -125,7 +138,7 @@ func TestRunSuite(t *testing.T) { useEnvVarForKeys: false, }, - // Expect success with default() and data + // Expect success with Default and data { webServerStatus: 200, webServerResponse: `{"status":"ok","series":[{"pointlist":[[1598867910000,0.0020008318672513122],[1598867925000,0.006121378742186943]]}]}`, @@ -140,7 +153,7 @@ func TestRunSuite(t *testing.T) { useEnvVarForKeys: false, }, - // Expect error with no default() and no data + // Expect error with no Default and no data { webServerStatus: 200, webServerResponse: `{"status":"ok","series":[]}`, @@ -155,7 +168,7 @@ func TestRunSuite(t *testing.T) { useEnvVarForKeys: false, }, - // Expect success with default() and no data + // Expect success with Default and no data { webServerStatus: 200, webServerResponse: `{"status":"ok","series":[]}`, @@ -170,7 +183,7 @@ func TestRunSuite(t *testing.T) { useEnvVarForKeys: false, }, - // Expect failure with bad default() and no data + // Expect failure with bad Default and no data { webServerStatus: 200, webServerResponse: `{"status":"ok","series":[]}`, @@ -185,7 +198,7 @@ func TestRunSuite(t *testing.T) { useEnvVarForKeys: false, }, - // Expect success with bad default() and good data + // Expect success with bad Default and good data { webServerStatus: 200, webServerResponse: `{"status":"ok","series":[{"pointlist":[[1598867910000,0.0020008318672513122],[1598867925000,0.006121378742186943]]}]}`, @@ -226,6 +239,22 @@ func TestRunSuite(t *testing.T) { expectedErrorMessage: "parse \"://wrong.schema\": missing protocol scheme", useEnvVarForKeys: false, }, + // When secret passed name passed in the analysis template, expect success + { + webServerStatus: 200, + webServerResponse: `{"status":"ok","series":[{"pointlist":[[1598867910000,0.0020008318672513122],[1598867925000,0.0003332881882246533]]}]}`, + metric: v1alpha1.Metric{ + Name: "foo", + SuccessCondition: "result < 0.001", + FailureCondition: "result >= 0.001", + Provider: ddProviderNamespacedSecret, + }, + expectedIntervalSeconds: 600, + expectedValue: "0.0003332881882246533", + expectedPhase: v1alpha1.AnalysisPhaseSuccessful, + useEnvVarForKeys: false, + expectedErrorProvider: false, + }, } // Run @@ -320,8 +349,9 @@ func TestRunSuite(t *testing.T) { if test.metric.Provider.Datadog.Interval == "" { test.metric.Provider.Datadog.Interval = "5m" } + namespace := "namespace" - provider, _ := NewDatadogProvider(*logCtx, fakeClient, test.metric) + provider, _ := NewDatadogProvider(*logCtx, fakeClient, namespace, test.metric) metricsMetadata := provider.GetMetadata(test.metric) assert.Nil(t, metricsMetadata) diff --git a/metricproviders/datadog/datadogV2_test.go b/metricproviders/datadog/datadogV2_test.go index f3e481160e..6b752bd1bb 100644 --- a/metricproviders/datadog/datadogV2_test.go +++ b/metricproviders/datadog/datadogV2_test.go @@ -8,7 +8,6 @@ import ( "os" "testing" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" @@ -16,6 +15,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" k8sfake "k8s.io/client-go/kubernetes/fake" kubetesting "k8s.io/client-go/testing" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" ) func newQueryDefaultProvider() v1alpha1.MetricProvider { @@ -52,6 +53,31 @@ func newQueryProviderInterval10m() v1alpha1.MetricProvider { } } +func newQueryProviderSumAggregator() v1alpha1.MetricProvider { + return v1alpha1.MetricProvider{ + Datadog: &v1alpha1.DatadogMetric{ + Query: "avg:kubernetes.cpu.user.total{*}", + Interval: "5m", + Aggregator: "sum", + ApiVersion: "v2", + }, + } +} + +func newNamespacedSecretProvider() v1alpha1.MetricProvider { + return v1alpha1.MetricProvider{ + Datadog: &v1alpha1.DatadogMetric{ + Query: "avg:kubernetes.cpu.user.total{*}", + Interval: "5m", + Aggregator: "sum", + ApiVersion: "v2", + SecretRef: v1alpha1.SecretRef{ + Name: "secret", + Namespaced: true, + }, + }, + } +} func TestRunSuiteV2(t *testing.T) { const expectedApiKey = "0123456789abcdef0123456789abcdef" const expectedAppKey = "0123456789abcdef0123456789abcdef01234567" @@ -68,6 +94,7 @@ func TestRunSuiteV2(t *testing.T) { expectedValue string expectedPhase v1alpha1.AnalysisPhase expectedErrorMessage string + expectedAggregator string useEnvVarForKeys bool }{ { @@ -222,7 +249,7 @@ func TestRunSuiteV2(t *testing.T) { }, expectedIntervalSeconds: 300, expectedPhase: v1alpha1.AnalysisPhaseError, - expectedErrorMessage: "Could not parse JSON body: json: cannot unmarshal string into Go struct field .Data.Attributes.Columns.Values of type []float64", + expectedErrorMessage: "Could not parse JSON body: json: cannot unmarshal string into Go struct field .Data.Attributes.Columns.Values of type []*float64", useEnvVarForKeys: false, }, @@ -252,6 +279,35 @@ func TestRunSuiteV2(t *testing.T) { expectedPhase: v1alpha1.AnalysisPhaseSuccessful, useEnvVarForKeys: false, }, + + { + webServerStatus: 200, + webServerResponse: `{"data": {"attributes": {"columns": [ {"values": [0.006121378742186943]}]}}}`, + metric: v1alpha1.Metric{ + Name: "success with default and data", + SuccessCondition: "default(result, 0) < 0.05", + Provider: newQueryProviderSumAggregator(), + }, + expectedIntervalSeconds: 300, + expectedValue: "0.006121378742186943", + expectedPhase: v1alpha1.AnalysisPhaseSuccessful, + expectedAggregator: "sum", + useEnvVarForKeys: false, + }, + { + webServerStatus: 200, + webServerResponse: `{"data": {"attributes": {"columns": [ {"values": [0.006121378742186943]}]}}}`, + metric: v1alpha1.Metric{ + Name: "success with default and data", + SuccessCondition: "default(result, 0) < 0.05", + Provider: newNamespacedSecretProvider(), + }, + expectedIntervalSeconds: 300, + expectedValue: "0.006121378742186943", + expectedPhase: v1alpha1.AnalysisPhaseSuccessful, + expectedAggregator: "sum", + useEnvVarForKeys: false, + }, } // Run @@ -298,6 +354,16 @@ func TestRunSuiteV2(t *testing.T) { t.Errorf("\nExpected formula but no Formulas in request: %+v", actualFormulas) } } + // Check query aggregation being set + expectedAggregator := test.expectedAggregator + if expectedAggregator == "" { + expectedAggregator = "last" + } + for _, query := range actualQueries { + if query["aggregator"] != expectedAggregator { + t.Errorf("\naggregator expected %s but got %s", expectedAggregator, query["aggregator"]) + } + } if actualFrom != (unixNow()-test.expectedIntervalSeconds)*1000 { t.Errorf("\nfrom %d expected be equal to %d", actualFrom, (unixNow()-test.expectedIntervalSeconds)*1000) @@ -365,8 +431,9 @@ func TestRunSuiteV2(t *testing.T) { } return true, tokenSecret, nil }) + namespace := "namespace" - provider, _ := NewDatadogProvider(*logCtx, fakeClient, test.metric) + provider, _ := NewDatadogProvider(*logCtx, fakeClient, namespace, test.metric) metricsMetadata := provider.GetMetadata(test.metric) assert.Nil(t, metricsMetadata) diff --git a/metricproviders/datadog/datadog_test.go b/metricproviders/datadog/datadog_test.go index fe817f385e..ea995f4804 100644 --- a/metricproviders/datadog/datadog_test.go +++ b/metricproviders/datadog/datadog_test.go @@ -3,14 +3,19 @@ package datadog import ( - "log" "os" "testing" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + k8sfake "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/kubernetes/scheme" + kubetesting "k8s.io/client-go/testing" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" ) func TestDatadogSpecDefaults(t *testing.T) { @@ -66,12 +71,6 @@ func TestDatadogSpecDefaults(t *testing.T) { assert.Equal(t, "\"l2norm\"", string(aggregatorEnums[7].Raw), "\"l2norm\" expected, got %s", string(aggregatorEnums[7].Raw)) assert.Equal(t, "\"area\"", string(aggregatorEnums[8].Raw), "\"area\" expected, got %s", string(aggregatorEnums[8].Raw)) }) - - t.Run("aggregator: Validate default is last", func(t *testing.T) { - defaultAggregator := string(ddSpec.Properties["aggregator"].Default.Raw) - assert.Equal(t, "\"last\"", defaultAggregator, "Default aggregator should be \"last\" ") - }) - } func TestValidateIncomingProps(t *testing.T) { @@ -185,3 +184,78 @@ func TestValidateIncomingProps(t *testing.T) { }) } } + +func TestFindCredentials(t *testing.T) { + + testCases := []struct { + name string + secret *corev1.Secret + expectsError bool + expectsEmptyAddress bool + metric v1alpha1.Metric + }{ + { + name: "when secretRef is set and secret found, should success", + secret: NewSecretBuilderDefaultData().Build(), + metric: newMetric("datadog", true), + }, + { + name: "when secretRef without address is set and secret found, should success", + secret: NewSecretBuilder().WithData("api-key", []byte("apiKey")).WithData("app-key", []byte("appKey")).Build(), + metric: newMetric("datadog", true), + expectsEmptyAddress: true, + }, + { + name: "when secretRef is set but secret not found, should fail", + secret: NewSecretBuilder().Build(), + metric: newMetric("datadog", true), + expectsError: true, + }, + { + name: "when secretRef name is not set but namespaced is true, should fail", + secret: NewSecretBuilder().Build(), + metric: newMetric("", true), + expectsError: true, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + logCtx := *log.WithField("test", "test") + fakeClient := k8sfake.NewSimpleClientset() + fakeClient.PrependReactor("get", "*", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + return true, testCase.secret, nil + }) + + address, apiKey, appKey, err := findCredentials(logCtx, fakeClient, "namespace", testCase.metric) + assert.Equal(t, err != nil, testCase.expectsError) + if !testCase.expectsError { + if testCase.expectsEmptyAddress { + assert.Empty(t, address) + } else { + assert.Equal(t, string(testCase.secret.Data["address"]), address) + } + assert.Equal(t, string(testCase.secret.Data["api-key"]), apiKey) + assert.Equal(t, string(testCase.secret.Data["app-key"]), appKey) + } + }) + + } +} + +func newMetric(name string, namespaced bool) v1alpha1.Metric { + return v1alpha1.Metric{ + Provider: v1alpha1.MetricProvider{ + Datadog: &v1alpha1.DatadogMetric{ + Query: "avg:kubernetes.cpu.user.total{*}", + Interval: "5m", + Aggregator: "sum", + ApiVersion: "v2", + SecretRef: v1alpha1.SecretRef{ + Name: name, + Namespaced: namespaced, + }, + }, + }, + } +} diff --git a/metricproviders/datadog/finders.go b/metricproviders/datadog/finders.go new file mode 100644 index 0000000000..3e66121c09 --- /dev/null +++ b/metricproviders/datadog/finders.go @@ -0,0 +1,79 @@ +package datadog + +import ( + "context" + "fmt" + "os" + "strings" + + log "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +type CredentialsFinder interface { + FindCredentials(logCtx log.Entry) (string, string, string) +} + +func NewSecretFinder(kubeclientset kubernetes.Interface, secretName string, namespace string) *secretFinder { + return &secretFinder{ + kubeclientset: kubeclientset, + secretName: secretName, + namespace: namespace, + } +} + +type secretFinder struct { + kubeclientset kubernetes.Interface + secretName string + namespace string +} + +func (sf *secretFinder) FindCredentials(logCtx log.Entry) (string, string, string) { + address := "" + secret, err := sf.kubeclientset.CoreV1().Secrets(sf.namespace).Get(context.TODO(), sf.secretName, metav1.GetOptions{}) + if err != nil { + logCtx.Debugf("error searching for secret %s in namespace %s: %s", sf.secretName, sf.namespace, err.Error()) + return "", "", "" + } + + apiKey := string(secret.Data[DatadogApiKey]) + appKey := string(secret.Data[DatadogAppKey]) + + if apiKey == "" || appKey == "" { + logCtx.Debugf("credentials missing in secret %s in namespace %s", sf.secretName, sf.namespace) + return "", "", "" + } + if _, hasAddress := secret.Data[DatadogAddress]; hasAddress { + address = string(secret.Data[DatadogAddress]) + } + return address, apiKey, appKey +} + +type envVariablesFinder struct { +} + +func NewEnvVariablesFinder() *envVariablesFinder { + return &envVariablesFinder{} +} +func (evf *envVariablesFinder) FindCredentials(logCtx log.Entry) (string, string, string) { + secretKeys := []string{DatadogApiKey, DatadogAppKey, DatadogAddress} + envValuesByKey := lookupKeysInEnv(secretKeys) + if len(envValuesByKey) == len(secretKeys) { + return envValuesByKey[DatadogAddress], envValuesByKey[DatadogApiKey], envValuesByKey[DatadogAppKey] + } + logCtx.Debug("credentials not found as env variables") + return "", "", "" +} + +func lookupKeysInEnv(keys []string) map[string]string { + valuesByKey := make(map[string]string) + for i := range keys { + key := keys[i] + formattedKey := strings.ToUpper(strings.ReplaceAll(key, "-", "_")) + if value, ok := os.LookupEnv(fmt.Sprintf("DD_%s", formattedKey)); ok { + valuesByKey[key] = value + } + } + return valuesByKey +} diff --git a/metricproviders/datadog/finders_test.go b/metricproviders/datadog/finders_test.go new file mode 100644 index 0000000000..62f2a8b2e5 --- /dev/null +++ b/metricproviders/datadog/finders_test.go @@ -0,0 +1,122 @@ +package datadog + +import ( + "fmt" + "testing" + + log "github.com/sirupsen/logrus" + "github.com/tj/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + k8sfake "k8s.io/client-go/kubernetes/fake" + kubetesting "k8s.io/client-go/testing" +) + +const ( + DatadogUrl = "datadog.com" + ApiKey = "apiKey" + AppKey = "appKey" +) + +func TestRunSuit(t *testing.T) { + + testCases := []struct { + name string + secret *corev1.Secret + expectedApiKey string + expectedAppKey string + expectedAddress string + expectError bool + }{ + { + name: "When secret valid, should be successful", + secret: NewSecretBuilder(). + WithName("DatadogTokensSecretName"). + WithData("address", []byte(DatadogUrl)). + WithData("api-key", []byte(ApiKey)). + WithData("app-key", []byte(AppKey)). + Build(), + expectedApiKey: ApiKey, + expectedAppKey: AppKey, + expectedAddress: DatadogUrl, + }, + { + name: "When secret is found but no data, should return empty values", + secret: NewSecretBuilder(). + WithName("DatadogTokensSecretName"). + Build(), + }, + { + name: "When secret not found, should return empty values", + expectError: true, + }, + } + + for _, tc := range testCases { + fakeClient := k8sfake.NewSimpleClientset() + fakeClient.PrependReactor("get", "*", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + if tc.expectError { + return true, nil, fmt.Errorf("api error") + } + return true, tc.secret, nil + }) + + secretFinder := NewSecretFinder(fakeClient, "", "") + t.Run(tc.name, func(t *testing.T) { + logCtx := *log.WithField("test", "test") + + address, apiKey, appKey := secretFinder.FindCredentials(logCtx) + assert.Equal(t, tc.expectedAddress, address) + assert.Equal(t, tc.expectedApiKey, apiKey) + assert.Equal(t, tc.expectedAppKey, appKey) + }) + } + +} + +// SecretBuilder helps in constructing a corev1.Secret object +type SecretBuilder struct { + name string + data map[string][]byte +} + +// NewSecretBuilder creates a new SecretBuilder +func NewSecretBuilder() *SecretBuilder { + return &SecretBuilder{ + data: make(map[string][]byte), + } +} + +// WithName sets the name for the Secret +func (b *SecretBuilder) WithName(name string) *SecretBuilder { + b.name = name + return b +} + +// WithData sets the data for the Secret +func (b *SecretBuilder) WithData(key string, value []byte) *SecretBuilder { + b.data[key] = value + return b +} + +// Build constructs the corev1.Secret object +func (b *SecretBuilder) Build() *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: b.name, + }, + Data: b.data, + } +} + +// NewSecretBuilderDefaultData creates a new SecretBuilder with default values +func NewSecretBuilderDefaultData() *SecretBuilder { + return &SecretBuilder{ + data: map[string][]byte{ + "address": []byte("datadog.com"), + "api-key": []byte("apiKey"), + "app-key": []byte("appKey"), + }, + } +} diff --git a/metricproviders/graphite/api_test.go b/metricproviders/graphite/api_test.go index 74f0ef1176..8fba5c4374 100644 --- a/metricproviders/graphite/api_test.go +++ b/metricproviders/graphite/api_test.go @@ -8,9 +8,10 @@ import ( "testing" "time" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" ) func testGraphiteMetric(addr string) v1alpha1.Metric { diff --git a/metricproviders/influxdb/influxdb.go b/metricproviders/influxdb/influxdb.go index 4266d2e2f6..9bd38598b5 100644 --- a/metricproviders/influxdb/influxdb.go +++ b/metricproviders/influxdb/influxdb.go @@ -6,14 +6,15 @@ import ( "fmt" "time" - "github.com/argoproj/argo-rollouts/utils/defaults" - "github.com/argoproj/argo-rollouts/utils/evaluate" influxdb2 "github.com/influxdata/influxdb-client-go/v2" influxapi "github.com/influxdata/influxdb-client-go/v2/api" log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "github.com/argoproj/argo-rollouts/utils/defaults" + "github.com/argoproj/argo-rollouts/utils/evaluate" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" metricutil "github.com/argoproj/argo-rollouts/utils/metric" ) diff --git a/metricproviders/influxdb/influxdb_test.go b/metricproviders/influxdb/influxdb_test.go index 028d66aed4..0ba80f4be8 100644 --- a/metricproviders/influxdb/influxdb_test.go +++ b/metricproviders/influxdb/influxdb_test.go @@ -7,7 +7,6 @@ import ( "strings" "testing" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" influxdb2 "github.com/influxdata/influxdb-client-go/v2/api" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -16,6 +15,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" k8sfake "k8s.io/client-go/kubernetes/fake" kubetesting "k8s.io/client-go/testing" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" ) func newAnalysisRun() *v1alpha1.AnalysisRun { diff --git a/metricproviders/job/job_test.go b/metricproviders/job/job_test.go index 80fdb28342..eb7a6a1629 100644 --- a/metricproviders/job/job_test.go +++ b/metricproviders/job/job_test.go @@ -2,6 +2,7 @@ package job import ( "context" + "errors" "fmt" "testing" "time" @@ -178,7 +179,7 @@ func TestRunCreateFail(t *testing.T) { // The following causes the Create call to fail fakeClient := p.kubeclientset.(*k8sfake.Clientset) fakeClient.PrependReactor("create", "*", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, fmt.Errorf(errMsg) + return true, nil, errors.New(errMsg) }) metricsMetadata := p.GetMetadata(run.Spec.Metrics[0]) assert.Nil(t, metricsMetadata) @@ -269,7 +270,7 @@ func TestTerminateError(t *testing.T) { errMsg := "random delete error" fakeClient := p.kubeclientset.(*k8sfake.Clientset) fakeClient.PrependReactor("delete", "*", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, fmt.Errorf(errMsg) + return true, nil, errors.New(errMsg) }) measurement = p.Terminate(run, run.Spec.Metrics[0], measurement) diff --git a/metricproviders/metricproviders.go b/metricproviders/metricproviders.go index 916fc85aa2..ef1381d668 100644 --- a/metricproviders/metricproviders.go +++ b/metricproviders/metricproviders.go @@ -4,11 +4,12 @@ import ( "fmt" "os" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "github.com/argoproj/argo-rollouts/metric" "github.com/argoproj/argo-rollouts/metricproviders/influxdb" "github.com/argoproj/argo-rollouts/metricproviders/skywalking" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" "github.com/argoproj/argo-rollouts/metricproviders/cloudwatch" "github.com/argoproj/argo-rollouts/metricproviders/datadog" @@ -43,7 +44,7 @@ type ProviderFactory struct { type ProviderFactoryFunc func(logCtx log.Entry, metric v1alpha1.Metric) (metric.Provider, error) // NewProvider creates the correct provider based on the provider type of the Metric -func (f *ProviderFactory) NewProvider(logCtx log.Entry, metric v1alpha1.Metric) (metric.Provider, error) { +func (f *ProviderFactory) NewProvider(logCtx log.Entry, namespace string, metric v1alpha1.Metric) (metric.Provider, error) { switch provider := Type(metric); provider { case prometheus.ProviderType: api, err := prometheus.NewPrometheusAPI(metric) @@ -72,7 +73,7 @@ func (f *ProviderFactory) NewProvider(logCtx log.Entry, metric v1alpha1.Metric) } return webmetric.NewWebMetricProvider(logCtx, c, p), nil case datadog.ProviderType: - return datadog.NewDatadogProvider(logCtx, f.KubeClient, metric) + return datadog.NewDatadogProvider(logCtx, f.KubeClient, namespace, metric) case wavefront.ProviderType: client, err := wavefront.NewWavefrontAPI(metric, f.KubeClient) if err != nil { diff --git a/metricproviders/mocks/Provider.go b/metricproviders/mocks/Provider.go index 1e647a943d..b0fd342ec5 100644 --- a/metricproviders/mocks/Provider.go +++ b/metricproviders/mocks/Provider.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.42.2. DO NOT EDIT. package mocks @@ -16,6 +16,10 @@ type Provider struct { func (_m *Provider) GarbageCollect(_a0 *v1alpha1.AnalysisRun, _a1 v1alpha1.Metric, _a2 int) error { ret := _m.Called(_a0, _a1, _a2) + if len(ret) == 0 { + panic("no return value specified for GarbageCollect") + } + var r0 error if rf, ok := ret.Get(0).(func(*v1alpha1.AnalysisRun, v1alpha1.Metric, int) error); ok { r0 = rf(_a0, _a1, _a2) @@ -30,6 +34,10 @@ func (_m *Provider) GarbageCollect(_a0 *v1alpha1.AnalysisRun, _a1 v1alpha1.Metri func (_m *Provider) GetMetadata(_a0 v1alpha1.Metric) map[string]string { ret := _m.Called(_a0) + if len(ret) == 0 { + panic("no return value specified for GetMetadata") + } + var r0 map[string]string if rf, ok := ret.Get(0).(func(v1alpha1.Metric) map[string]string); ok { r0 = rf(_a0) @@ -46,6 +54,10 @@ func (_m *Provider) GetMetadata(_a0 v1alpha1.Metric) map[string]string { func (_m *Provider) Resume(_a0 *v1alpha1.AnalysisRun, _a1 v1alpha1.Metric, _a2 v1alpha1.Measurement) v1alpha1.Measurement { ret := _m.Called(_a0, _a1, _a2) + if len(ret) == 0 { + panic("no return value specified for Resume") + } + var r0 v1alpha1.Measurement if rf, ok := ret.Get(0).(func(*v1alpha1.AnalysisRun, v1alpha1.Metric, v1alpha1.Measurement) v1alpha1.Measurement); ok { r0 = rf(_a0, _a1, _a2) @@ -60,6 +72,10 @@ func (_m *Provider) Resume(_a0 *v1alpha1.AnalysisRun, _a1 v1alpha1.Metric, _a2 v func (_m *Provider) Run(_a0 *v1alpha1.AnalysisRun, _a1 v1alpha1.Metric) v1alpha1.Measurement { ret := _m.Called(_a0, _a1) + if len(ret) == 0 { + panic("no return value specified for Run") + } + var r0 v1alpha1.Measurement if rf, ok := ret.Get(0).(func(*v1alpha1.AnalysisRun, v1alpha1.Metric) v1alpha1.Measurement); ok { r0 = rf(_a0, _a1) @@ -74,6 +90,10 @@ func (_m *Provider) Run(_a0 *v1alpha1.AnalysisRun, _a1 v1alpha1.Metric) v1alpha1 func (_m *Provider) Terminate(_a0 *v1alpha1.AnalysisRun, _a1 v1alpha1.Metric, _a2 v1alpha1.Measurement) v1alpha1.Measurement { ret := _m.Called(_a0, _a1, _a2) + if len(ret) == 0 { + panic("no return value specified for Terminate") + } + var r0 v1alpha1.Measurement if rf, ok := ret.Get(0).(func(*v1alpha1.AnalysisRun, v1alpha1.Metric, v1alpha1.Measurement) v1alpha1.Measurement); ok { r0 = rf(_a0, _a1, _a2) @@ -88,6 +108,10 @@ func (_m *Provider) Terminate(_a0 *v1alpha1.AnalysisRun, _a1 v1alpha1.Metric, _a func (_m *Provider) Type() string { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for Type") + } + var r0 string if rf, ok := ret.Get(0).(func() string); ok { r0 = rf() diff --git a/metricproviders/newrelic/mock_test.go b/metricproviders/newrelic/mock_test.go index a96b4febbc..0aad3208fc 100644 --- a/metricproviders/newrelic/mock_test.go +++ b/metricproviders/newrelic/mock_test.go @@ -1,15 +1,65 @@ package newrelic -import "github.com/newrelic/newrelic-client-go/pkg/nrdb" +import ( + "reflect" + + "github.com/newrelic/newrelic-client-go/v2/pkg/nrdb" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" +) type mockAPI struct { response []nrdb.NRDBResult err error } -func (m *mockAPI) Query(query string) ([]nrdb.NRDBResult, error) { +func (m *mockAPI) Query(metric v1alpha1.Metric) ([]nrdb.NRDBResult, error) { if m.err != nil { return nil, m.err } return m.response, nil } + +type mockNerdGraphClient struct { + response []nrdb.NRDBResult + lastArgs map[string]interface{} + err error +} + +func (m *mockNerdGraphClient) QueryWithResponse(query string, variables map[string]interface{}, respBody interface{}) error { + m.lastArgs = variables + + if m.err != nil { + return m.err + } + + r := gqlNrglQueryResponse{} + r.Actor.Account.NRQL.Results = m.response + + rVal := reflect.ValueOf(r) + reflect.ValueOf(respBody).Elem().Set(rVal) + + return nil +} + +// Response sets the response the mock client will return +func (m *mockNerdGraphClient) Response(response []nrdb.NRDBResult) { + m.response = response +} + +// LastArgs returns the variables used when calling the NerdGraph API +func (m *mockNerdGraphClient) LastArgs() map[string]any { + return m.lastArgs +} + +// Err sets the error to be returned when calling the NerdGraph API +func (m *mockNerdGraphClient) Err(err error) { + m.err = err +} + +// Clear removes all configured mock behavior +func (m *mockNerdGraphClient) Clear() { + m.err = nil + m.response = nil + m.lastArgs = nil +} diff --git a/metricproviders/newrelic/newrelic.go b/metricproviders/newrelic/newrelic.go index 0c971e5c60..7f0ed2dee4 100644 --- a/metricproviders/newrelic/newrelic.go +++ b/metricproviders/newrelic/newrelic.go @@ -7,8 +7,8 @@ import ( "fmt" "strconv" - "github.com/newrelic/newrelic-client-go/newrelic" - "github.com/newrelic/newrelic-client-go/pkg/nrdb" + "github.com/newrelic/newrelic-client-go/v2/newrelic" + "github.com/newrelic/newrelic-client-go/v2/pkg/nrdb" log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -22,31 +22,82 @@ import ( ) const ( - //ProviderType indicates the provider is wavefront + // ProviderType indicates the provider is newrelic ProviderType = "NewRelic" DefaultNewRelicProfileSecretName = "newrelic" repoURL = "https://github.com/argoproj/argo-rollouts" + resolvedNewRelicQuery = "ResolvedNewRelicQuery" + defaultNrqlTimeout = 5 ) +var ( + ErrNegativeTimeout = errors.New("timeout value needs to be a positive value") +) + +type gqlNrglQueryResponse struct { + Actor struct { + Account struct { + NRQL nrdb.NRDBResultContainer + } + } +} + +const gqlNrqlQuery = `query ( + $query: Nrql!, + $accountId: Int!, + $timeout: Seconds! +) +{ + actor { + account(id: $accountId) { + nrql(query: $query, timeout: $timeout) { + results + } + } + } +} +` + var userAgent = fmt.Sprintf("argo-rollouts/%s (%s)", version.GetVersion(), repoURL) type NewRelicClientAPI interface { - Query(query string) ([]nrdb.NRDBResult, error) + Query(metric v1alpha1.Metric) ([]nrdb.NRDBResult, error) +} + +type nerdGraphClient interface { + QueryWithResponse(query string, variables map[string]interface{}, respBody interface{}) error } type NewRelicClient struct { - *newrelic.NewRelic - AccountID int + NerdGraphClient nerdGraphClient + AccountID int } // Query executes a NRQL query against the given New Relic account -func (n *NewRelicClient) Query(query string) ([]nrdb.NRDBResult, error) { - results, err := n.Nrdb.Query(n.AccountID, nrdb.NRQL(query)) - if err != nil { +func (n *NewRelicClient) Query(metric v1alpha1.Metric) ([]nrdb.NRDBResult, error) { + var timeout int64 = defaultNrqlTimeout + respBody := gqlNrglQueryResponse{} + + if metric.Provider.NewRelic.Timeout != nil { + timeout = *metric.Provider.NewRelic.Timeout + } + + if timeout < 0 { + return nil, ErrNegativeTimeout + } + + args := map[string]any{ + "accountId": n.AccountID, + "query": metric.Provider.NewRelic.Query, + "timeout": timeout, + } + + if err := n.NerdGraphClient.QueryWithResponse(gqlNrqlQuery, args, &respBody); err != nil { return nil, err } + // TODO(jwelch) return metadata from NRDBResultContainer to include on the measurement - return results.Results, nil + return respBody.Actor.Account.NRQL.Results, nil } type Provider struct { @@ -61,7 +112,7 @@ func (p *Provider) Run(run *v1alpha1.AnalysisRun, metric v1alpha1.Metric) v1alph StartedAt: &startTime, } - results, err := p.api.Query(metric.Provider.NewRelic.Query) + results, err := p.api.Query(metric) if err != nil { return metricutil.MarkMeasurementError(newMeasurement, err) } @@ -133,7 +184,11 @@ func (p *Provider) Type() string { // GetMetadata returns any additional metadata which needs to be stored & displayed as part of the metrics result. func (p *Provider) GetMetadata(metric v1alpha1.Metric) map[string]string { - return nil + metricsMetadata := make(map[string]string) + if metric.Provider.NewRelic.Query != "" { + metricsMetadata[resolvedNewRelicQuery] = metric.Provider.NewRelic.Query + } + return metricsMetadata } // NewNewRelicProvider creates a new NewRelic provider @@ -186,7 +241,7 @@ func NewNewRelicAPIClient(metric v1alpha1.Metric, kubeclientset kubernetes.Inter if err != nil { return nil, fmt.Errorf("could not parse account ID: %w", err) } - return &NewRelicClient{NewRelic: nrClient, AccountID: accID}, nil + return &NewRelicClient{NerdGraphClient: &nrClient.NerdGraph, AccountID: accID}, nil } else { return nil, errors.New("account ID or personal API key not found") } diff --git a/metricproviders/newrelic/newrelic_test.go b/metricproviders/newrelic/newrelic_test.go index 8f49049815..9b1cf31156 100644 --- a/metricproviders/newrelic/newrelic_test.go +++ b/metricproviders/newrelic/newrelic_test.go @@ -5,7 +5,7 @@ import ( "fmt" "testing" - "github.com/newrelic/newrelic-client-go/pkg/nrdb" + "github.com/newrelic/newrelic-client-go/v2/pkg/nrdb" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" @@ -40,12 +40,13 @@ func TestRunSuccessfully(t *testing.T) { FailureCondition: "result.count != 10", Provider: v1alpha1.MetricProvider{ NewRelic: &v1alpha1.NewRelicMetric{ - Query: "test", + Query: "resolved-query", }, }, } metricsMetadata := p.GetMetadata(metric) - assert.Nil(t, metricsMetadata) + assert.NotNil(t, metricsMetadata) + assert.Equal(t, "resolved-query", metricsMetadata[resolvedNewRelicQuery]) measurement := p.Run(newAnalysisRun(), metric) assert.NotNil(t, measurement.StartedAt) @@ -60,7 +61,8 @@ func TestRunWithTimeseries(t *testing.T) { response: []nrdb.NRDBResult{ map[string]any{"count": 10}, map[string]any{"count": 20}, - map[string]any{"count": 30}}, + map[string]any{"count": 30}, + }, } p := NewNewRelicProvider(mock, *e) metric := v1alpha1.Metric{ @@ -69,12 +71,13 @@ func TestRunWithTimeseries(t *testing.T) { FailureCondition: "result[2].count < 20", Provider: v1alpha1.MetricProvider{ NewRelic: &v1alpha1.NewRelicMetric{ - Query: "test", + Query: "resolved-query", }, }, } metricsMetadata := p.GetMetadata(metric) - assert.Nil(t, metricsMetadata) + assert.NotNil(t, metricsMetadata) + assert.Equal(t, "resolved-query", metricsMetadata[resolvedNewRelicQuery]) measurement := p.Run(newAnalysisRun(), metric) assert.NotNil(t, measurement.StartedAt) @@ -95,12 +98,13 @@ func TestRunWithFacet(t *testing.T) { FailureCondition: "result.count != 10 or result['average.duration'] >= 15.0", Provider: v1alpha1.MetricProvider{ NewRelic: &v1alpha1.NewRelicMetric{ - Query: "test", + Query: "resolved-query", }, }, } metricsMetadata := p.GetMetadata(metric) - assert.Nil(t, metricsMetadata) + assert.NotNil(t, metricsMetadata) + assert.Equal(t, "resolved-query", metricsMetadata[resolvedNewRelicQuery]) measurement := p.Run(newAnalysisRun(), metric) assert.NotNil(t, measurement.StartedAt) @@ -121,12 +125,13 @@ func TestRunWithMultipleSelectTerms(t *testing.T) { FailureCondition: "result.count != 10", Provider: v1alpha1.MetricProvider{ NewRelic: &v1alpha1.NewRelicMetric{ - Query: "test", + Query: "resolved-query", }, }, } metricsMetadata := p.GetMetadata(metric) - assert.Nil(t, metricsMetadata) + assert.NotNil(t, metricsMetadata) + assert.Equal(t, "resolved-query", metricsMetadata[resolvedNewRelicQuery]) measurement := p.Run(newAnalysisRun(), metric) assert.NotNil(t, measurement.StartedAt) @@ -402,3 +407,73 @@ func TestNewNewRelicAPIClient(t *testing.T) { assert.NotNil(t, err) }) } + +func TestNewRelicClient_Query(t *testing.T) { + accountId := 1234567 + sevenTo := int64(7) + negativeTo := int64(-1) + defaultTo := int64(defaultNrqlTimeout) + theQuery := "FROM K8sContainerSample SELECT percentile(`cpuCoresUtilization`, 95)" + + mockNGC := &mockNerdGraphClient{} + nrc := &NewRelicClient{NerdGraphClient: mockNGC, AccountID: accountId} + + tests := map[string]struct { + timeoutProvided *int64 + timeoutUsed *int64 + query string + want []nrdb.NRDBResult + errMsg string + gqlErr error + }{ + `returns results`: { + timeoutUsed: &defaultTo, + query: theQuery, + want: []nrdb.NRDBResult{map[string]any{"count": 10}}, + }, + `uses default timeout when one is not provided`: { + timeoutUsed: &defaultTo, + query: theQuery, + }, + `uses provided timeout`: { + timeoutUsed: &sevenTo, + timeoutProvided: &sevenTo, + query: theQuery, + }, + `errors when timeout is negative`: { + timeoutProvided: &negativeTo, + query: theQuery, + errMsg: ErrNegativeTimeout.Error(), + }, + `errors when nerdgraph returns error`: { + timeoutUsed: &defaultTo, + query: theQuery, + errMsg: "boom", + gqlErr: errors.New("boom"), + }, + } + for testName, tc := range tests { + t.Run(testName, func(t *testing.T) { + defer mockNGC.Clear() + mockNGC.Err(tc.gqlErr) + mockNGC.Response(tc.want) + metric := v1alpha1.Metric{ + Provider: v1alpha1.MetricProvider{ + NewRelic: &v1alpha1.NewRelicMetric{ + Timeout: tc.timeoutProvided, + Query: tc.query, + }, + }, + } + results, err := nrc.Query(metric) + if len(tc.errMsg) > 0 { + assert.EqualError(t, err, tc.errMsg) + return + } + assert.Equal(t, *tc.timeoutUsed, mockNGC.LastArgs()["timeout"]) + assert.Equal(t, tc.query, mockNGC.LastArgs()["query"]) + assert.Equal(t, accountId, mockNGC.LastArgs()["accountId"]) + assert.Equal(t, tc.want, results) + }) + } +} diff --git a/metricproviders/plugin/client/client.go b/metricproviders/plugin/client/client.go index 9fc82ed244..b661073e1e 100644 --- a/metricproviders/plugin/client/client.go +++ b/metricproviders/plugin/client/client.go @@ -5,10 +5,12 @@ import ( "os/exec" "sync" + goPlugin "github.com/hashicorp/go-plugin" + "github.com/argoproj/argo-rollouts/metricproviders/plugin/rpc" "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" "github.com/argoproj/argo-rollouts/utils/plugin" - goPlugin "github.com/hashicorp/go-plugin" + "github.com/argoproj/argo-rollouts/utils/plugin/types" ) type metricPlugin struct { @@ -54,7 +56,7 @@ func (m *metricPlugin) startPluginSystem(metric v1alpha1.Metric) (rpc.MetricProv // There should only ever be one plugin defined in metric.Provider.Plugin per analysis template this gets checked // during validation for pluginName := range metric.Provider.Plugin { - pluginPath, args, err := plugin.GetPluginInfo(pluginName) + pluginPath, args, err := plugin.GetPluginInfo(pluginName, types.PluginTypeMetricProvider) if err != nil { return nil, fmt.Errorf("unable to find plugin (%s): %w", pluginName, err) } diff --git a/metricproviders/plugin/rpc/rpc.go b/metricproviders/plugin/rpc/rpc.go index a709693976..d4331c7096 100644 --- a/metricproviders/plugin/rpc/rpc.go +++ b/metricproviders/plugin/rpc/rpc.go @@ -7,9 +7,10 @@ import ( "github.com/argoproj/argo-rollouts/utils/plugin/types" + "github.com/hashicorp/go-plugin" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" metricutil "github.com/argoproj/argo-rollouts/utils/metric" - "github.com/hashicorp/go-plugin" ) type RunArgs struct { diff --git a/metricproviders/plugin/rpc/rpc_test.go b/metricproviders/plugin/rpc/rpc_test.go index ad351149c3..fbcb6b37c1 100644 --- a/metricproviders/plugin/rpc/rpc_test.go +++ b/metricproviders/plugin/rpc/rpc_test.go @@ -7,9 +7,10 @@ import ( "github.com/argoproj/argo-rollouts/utils/plugin/types" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" goPlugin "github.com/hashicorp/go-plugin" "github.com/tj/assert" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" ) var testHandshake = goPlugin.HandshakeConfig{ diff --git a/metricproviders/plugin/rpc/rpc_test_implementation.go b/metricproviders/plugin/rpc/rpc_test_implementation.go index 2dd18d6434..5ff6380f28 100644 --- a/metricproviders/plugin/rpc/rpc_test_implementation.go +++ b/metricproviders/plugin/rpc/rpc_test_implementation.go @@ -6,10 +6,11 @@ import ( "github.com/argoproj/argo-rollouts/utils/plugin/types" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" metricutil "github.com/argoproj/argo-rollouts/utils/metric" timeutil "github.com/argoproj/argo-rollouts/utils/time" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type testRpcPlugin struct{} diff --git a/metricproviders/prometheus/headers_round_tripper_test.go b/metricproviders/prometheus/headers_round_tripper_test.go index a9983747a1..1725e050da 100644 --- a/metricproviders/prometheus/headers_round_tripper_test.go +++ b/metricproviders/prometheus/headers_round_tripper_test.go @@ -8,9 +8,10 @@ import ( "net/url" "testing" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" "github.com/prometheus/client_golang/api" "github.com/stretchr/testify/assert" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" ) type testHttpHeaderRoundTripper struct { diff --git a/metricproviders/prometheus/mock_test.go b/metricproviders/prometheus/mock_test.go index e16a42902d..9636828d1e 100644 --- a/metricproviders/prometheus/mock_test.go +++ b/metricproviders/prometheus/mock_test.go @@ -9,17 +9,20 @@ import ( ) type mockAPI struct { - value model.Value - err error - warnings v1.Warnings + value model.Value + err error + warnings v1.Warnings + startTimeSent time.Time + endTimeSent time.Time + stepSent time.Duration } -func (m mockAPI) WalReplay(ctx context.Context) (v1.WalReplayStatus, error) { +func (m *mockAPI) WalReplay(ctx context.Context) (v1.WalReplayStatus, error) { panic("Not used") } // Query performs a query for the given time. -func (m mockAPI) Query(ctx context.Context, query string, ts time.Time, opt ...v1.Option) (model.Value, v1.Warnings, error) { +func (m *mockAPI) Query(ctx context.Context, query string, ts time.Time, opt ...v1.Option) (model.Value, v1.Warnings, error) { if m.err != nil { return nil, m.warnings, m.err } @@ -28,78 +31,84 @@ func (m mockAPI) Query(ctx context.Context, query string, ts time.Time, opt ...v // Below methods are not used but required for the interface implementation -func (m mockAPI) Metadata(ctx context.Context, metric string, limit string) (map[string][]v1.Metadata, error) { +func (m *mockAPI) Metadata(ctx context.Context, metric string, limit string) (map[string][]v1.Metadata, error) { panic("Not used") } -func (m mockAPI) CleanTombstones(ctx context.Context) error { +func (m *mockAPI) CleanTombstones(ctx context.Context) error { panic("Not used") } -func (m mockAPI) DeleteSeries(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) error { +func (m *mockAPI) DeleteSeries(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) error { panic("Not used") } -func (m mockAPI) LabelNames(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) ([]string, v1.Warnings, error) { +func (m *mockAPI) LabelNames(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) ([]string, v1.Warnings, error) { panic("Not used") } -func (m mockAPI) LabelValues(ctx context.Context, label string, matches []string, startTime time.Time, endTime time.Time) (model.LabelValues, v1.Warnings, error) { +func (m *mockAPI) LabelValues(ctx context.Context, label string, matches []string, startTime time.Time, endTime time.Time) (model.LabelValues, v1.Warnings, error) { panic("Not used") } -func (m mockAPI) QueryRange(ctx context.Context, query string, r v1.Range, opt ...v1.Option) (model.Value, v1.Warnings, error) { - panic("Not used") +func (m *mockAPI) QueryRange(ctx context.Context, query string, r v1.Range, opt ...v1.Option) (model.Value, v1.Warnings, error) { + m.startTimeSent = r.Start + m.endTimeSent = r.End + m.stepSent = r.Step + if m.err != nil { + return nil, m.warnings, m.err + } + return m.value, m.warnings, nil } -func (m mockAPI) Series(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) ([]model.LabelSet, v1.Warnings, error) { +func (m *mockAPI) Series(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) ([]model.LabelSet, v1.Warnings, error) { panic("Not used") } -func (m mockAPI) Targets(ctx context.Context) (v1.TargetsResult, error) { +func (m *mockAPI) Targets(ctx context.Context) (v1.TargetsResult, error) { panic("Not used") } -func (m mockAPI) Alerts(ctx context.Context) (v1.AlertsResult, error) { +func (m *mockAPI) Alerts(ctx context.Context) (v1.AlertsResult, error) { panic("Not used") } -func (m mockAPI) AlertManagers(ctx context.Context) (v1.AlertManagersResult, error) { +func (m *mockAPI) AlertManagers(ctx context.Context) (v1.AlertManagersResult, error) { panic("Not used") } -func (m mockAPI) Config(ctx context.Context) (v1.ConfigResult, error) { +func (m *mockAPI) Config(ctx context.Context) (v1.ConfigResult, error) { panic("Not used") } -func (m mockAPI) Flags(ctx context.Context) (v1.FlagsResult, error) { +func (m *mockAPI) Flags(ctx context.Context) (v1.FlagsResult, error) { panic("Not used") } -func (m mockAPI) Snapshot(ctx context.Context, skipHead bool) (v1.SnapshotResult, error) { +func (m *mockAPI) Snapshot(ctx context.Context, skipHead bool) (v1.SnapshotResult, error) { panic("Not used") } -func (m mockAPI) Rules(ctx context.Context) (v1.RulesResult, error) { +func (m *mockAPI) Rules(ctx context.Context) (v1.RulesResult, error) { panic("Not used") } -func (m mockAPI) TargetsMetadata(ctx context.Context, matchTarget string, metric string, limit string) ([]v1.MetricMetadata, error) { +func (m *mockAPI) TargetsMetadata(ctx context.Context, matchTarget string, metric string, limit string) ([]v1.MetricMetadata, error) { panic("Not used") } -func (m mockAPI) Runtimeinfo(ctx context.Context) (v1.RuntimeinfoResult, error) { +func (m *mockAPI) Runtimeinfo(ctx context.Context) (v1.RuntimeinfoResult, error) { panic("Not used") } -func (m mockAPI) TSDB(ctx context.Context) (v1.TSDBResult, error) { +func (m *mockAPI) TSDB(ctx context.Context) (v1.TSDBResult, error) { panic("Not used") } -func (m mockAPI) Buildinfo(ctx context.Context) (v1.BuildinfoResult, error) { +func (m *mockAPI) Buildinfo(ctx context.Context) (v1.BuildinfoResult, error) { panic("Not used") } -func (m mockAPI) QueryExemplars(ctx context.Context, query string, startTime time.Time, endTime time.Time) ([]v1.ExemplarQueryResult, error) { +func (m *mockAPI) QueryExemplars(ctx context.Context, query string, startTime time.Time, endTime time.Time) ([]v1.ExemplarQueryResult, error) { panic("Not used") } diff --git a/metricproviders/prometheus/prometheus.go b/metricproviders/prometheus/prometheus.go index 7473693525..f369187f9d 100644 --- a/metricproviders/prometheus/prometheus.go +++ b/metricproviders/prometheus/prometheus.go @@ -56,6 +56,30 @@ func (p *Provider) GetMetadata(metric v1alpha1.Metric) map[string]string { return metricsMetadata } +func (p *Provider) executeQuery(ctx context.Context, metric v1alpha1.Metric) (model.Value, v1.Warnings, error) { + if metric.Provider.Prometheus.RangeQuery != nil { + start, err := evaluate.EvalTime(metric.Provider.Prometheus.RangeQuery.Start) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse rangeQuery.start as time: %w", err) + } + end, err := evaluate.EvalTime(metric.Provider.Prometheus.RangeQuery.End) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse rangeQuery.end as time: %w", err) + } + stepDuration, err := metric.Provider.Prometheus.RangeQuery.Step.Duration() + if err != nil { + return nil, nil, fmt.Errorf("failed to parse rangeQuery.step as duration: %w", err) + } + return p.api.QueryRange(ctx, metric.Provider.Prometheus.Query, v1.Range{ + Start: start, + End: end, + Step: stepDuration, + }) + } else { + return p.api.Query(ctx, metric.Provider.Prometheus.Query, time.Now()) + } +} + // Run queries prometheus for the metric func (p *Provider) Run(run *v1alpha1.AnalysisRun, metric v1alpha1.Metric) v1alpha1.Measurement { startTime := timeutil.MetaNow() @@ -66,7 +90,7 @@ func (p *Provider) Run(run *v1alpha1.AnalysisRun, metric v1alpha1.Metric) v1alph ctx, cancel := context.WithTimeout(context.Background(), p.timeout) defer cancel() - response, warnings, err := p.api.Query(ctx, metric.Provider.Prometheus.Query, time.Now()) + response, warnings, err := p.executeQuery(ctx, metric) if err != nil { return metricutil.MarkMeasurementError(newMeasurement, err) } @@ -112,6 +136,22 @@ func (p *Provider) GarbageCollect(run *v1alpha1.AnalysisRun, metric v1alpha1.Met return nil } +func sampleValuesToFloatSlice(sampleValues []model.SampleValue) []float64 { + results := make([]float64, 0, len(sampleValues)) + for _, s := range sampleValues { + results = append(results, float64(s)) + } + return results +} + +func sampleValuesToResultStr(sampleValues []model.SampleValue) string { + results := []string{} + for _, s := range sampleValues { + results = append(results, s.String()) + } + return fmt.Sprintf("[%s]", strings.Join(results, ",")) +} + func (p *Provider) processResponse(metric v1alpha1.Metric, response model.Value) (string, v1alpha1.AnalysisPhase, error) { switch value := response.(type) { case *model.Scalar: @@ -119,22 +159,28 @@ func (p *Provider) processResponse(metric v1alpha1.Metric, response model.Value) result := float64(value.Value) newStatus, err := evaluate.EvaluateResult(result, metric, p.logCtx) return valueStr, newStatus, err + case model.Matrix: + sampleValues := []model.SampleValue{} + for _, sample := range value { + if sample != nil { + for _, s := range sample.Values { + sampleValues = append(sampleValues, s.Value) + } + } + } + floatResults := sampleValuesToFloatSlice(sampleValues) + newStatus, err := evaluate.EvaluateResult(floatResults, metric, p.logCtx) + return sampleValuesToResultStr(sampleValues), newStatus, err case model.Vector: - results := make([]float64, 0, len(value)) - valueStr := "[" + sampleValues := []model.SampleValue{} for _, s := range value { if s != nil { - valueStr = valueStr + s.Value.String() + "," - results = append(results, float64(s.Value)) + sampleValues = append(sampleValues, s.Value) } } - // if we appended to the string, we should remove the last comma on the string - if len(valueStr) > 1 { - valueStr = valueStr[:len(valueStr)-1] - } - valueStr = valueStr + "]" - newStatus, err := evaluate.EvaluateResult(results, metric, p.logCtx) - return valueStr, newStatus, err + floatResults := sampleValuesToFloatSlice(sampleValues) + newStatus, err := evaluate.EvaluateResult(floatResults, metric, p.logCtx) + return sampleValuesToResultStr(sampleValues), newStatus, err //TODO(dthomson) add other response types default: return "", v1alpha1.AnalysisPhaseError, fmt.Errorf("Prometheus metric type not supported") @@ -163,6 +209,21 @@ func NewPrometheusProvider(api v1.API, logCtx log.Entry, metric v1alpha1.Metric) return provider, nil } +func newHTTPTransport(insecureSkipVerify bool) *http.Transport { + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + TLSHandshakeTimeout: 10 * time.Second, + TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureSkipVerify}, + } +} + +var secureTransport *http.Transport = newHTTPTransport(false) +var insecureTransport *http.Transport = newHTTPTransport(true) + // NewPrometheusAPI generates a prometheus API from the metric configuration func NewPrometheusAPI(metric v1alpha1.Metric) (v1.API, error) { envValuesByKey := make(map[string]string) @@ -186,15 +247,10 @@ func NewPrometheusAPI(metric v1alpha1.Metric) (v1.API, error) { } var roundTripper http.RoundTripper - - roundTripper = &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }).DialContext, - TLSHandshakeTimeout: 10 * time.Second, - TLSClientConfig: &tls.Config{InsecureSkipVerify: metric.Provider.Prometheus.Insecure}, + if metric.Provider.Prometheus.Insecure { + roundTripper = insecureTransport + } else { + roundTripper = secureTransport } // attach custom headers to api requests, if specified diff --git a/metricproviders/prometheus/prometheus_test.go b/metricproviders/prometheus/prometheus_test.go index f54d6d6c61..519de88759 100644 --- a/metricproviders/prometheus/prometheus_test.go +++ b/metricproviders/prometheus/prometheus_test.go @@ -10,12 +10,13 @@ import ( "testing" "time" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" ) const ( @@ -28,6 +29,35 @@ type OAuthResponse struct { Expiry string `json:"expires_in,omitempty"` } +func newMatrix(baseline float64) model.Matrix { + return model.Matrix{ + &model.SampleStream{ + Values: []model.SamplePair{ + { + Timestamp: 1234, + Value: model.SampleValue(baseline + 1.0), + }, + { + Timestamp: 1234, + Value: model.SampleValue(baseline + 2.0), + }, + }, + }, + &model.SampleStream{ + Values: []model.SamplePair{ + { + Timestamp: 1234, + Value: model.SampleValue(baseline + 3.0), + }, + { + Timestamp: 1234, + Value: model.SampleValue(baseline + 4.0), + }, + }, + }, + } +} + func newScalar(f float64) model.Value { return &model.Scalar{ Value: model.SampleValue(f), @@ -41,7 +71,7 @@ func newAnalysisRun() *v1alpha1.AnalysisRun { func TestType(t *testing.T) { e := log.Entry{} - mock := mockAPI{ + mock := &mockAPI{ value: newScalar(10), } timeout := int64(5) @@ -64,7 +94,7 @@ func TestType(t *testing.T) { func TestRunSuccessfully(t *testing.T) { e := log.Entry{} - mock := mockAPI{ + mock := &mockAPI{ value: newScalar(10), } metric := v1alpha1.Metric{ @@ -87,9 +117,142 @@ func TestRunSuccessfully(t *testing.T) { assert.Equal(t, v1alpha1.AnalysisPhaseSuccessful, measurement.Phase) } +func TestRunSuccessfullyWithRangeQuery(t *testing.T) { + e := log.Entry{} + mock := &mockAPI{ + value: newMatrix(10), + } + metric := v1alpha1.Metric{ + Name: "foo", + SuccessCondition: "all(result, # > 10)", + FailureCondition: "all(result, # < 10)", + Provider: v1alpha1.MetricProvider{ + Prometheus: &v1alpha1.PrometheusMetric{ + Query: "test", + RangeQuery: &v1alpha1.PrometheusRangeQueryArgs{ + Start: `date("2023-08-14 00:00:00", "2006-01-02 15:04:05", "UTC") - duration("1h")`, + End: `date("2023-08-14 00:00:00", "2006-01-02 15:04:05", "UTC")`, + Step: "1m", + }, + }, + }, + } + + p, err := NewPrometheusProvider(mock, e, metric) + + measurement := p.Run(newAnalysisRun(), metric) + assert.NotNil(t, measurement.StartedAt) + assert.NoError(t, err) + assert.Equal(t, "[11,12,13,14]", measurement.Value) + assert.NotNil(t, measurement.FinishedAt) + assert.Equal(t, v1alpha1.AnalysisPhaseSuccessful, measurement.Phase) + assert.Equal(t, "2023-08-13 23:00:00 +0000 UTC", mock.startTimeSent.String()) + assert.Equal(t, "2023-08-14 00:00:00 +0000 UTC", mock.endTimeSent.String()) + assert.Equal(t, "1m0s", mock.stepSent.String()) +} + +func TestRunUnparsableStartTime(t *testing.T) { + e := log.Entry{} + mock := &mockAPI{ + value: newMatrix(10), + } + metric := v1alpha1.Metric{ + Name: "foo", + SuccessCondition: "all(result, # > 10)", + FailureCondition: "all(result, # < 10)", + Provider: v1alpha1.MetricProvider{ + Prometheus: &v1alpha1.PrometheusMetric{ + Query: "test", + RangeQuery: &v1alpha1.PrometheusRangeQueryArgs{ + Start: `now() - duration("??")`, + End: `date("2023-08-14 00:00:00", "2006-01-02 15:04:05", "UTC")`, + Step: "1m", + }, + }, + }, + } + expectedErr := fmt.Errorf(`failed to parse rangeQuery.start as time: time: invalid duration "??"`) + + p, err := NewPrometheusProvider(mock, e, metric) + + measurement := p.Run(newAnalysisRun(), metric) + assert.NotNil(t, measurement.StartedAt) + assert.NoError(t, err) + assert.Equal(t, expectedErr.Error(), measurement.Message) + assert.Equal(t, "", measurement.Value) + assert.NotNil(t, measurement.FinishedAt) + assert.Equal(t, v1alpha1.AnalysisPhaseError, measurement.Phase) +} + +func TestRunUnparsableEndTime(t *testing.T) { + e := log.Entry{} + mock := &mockAPI{ + value: newMatrix(10), + } + metric := v1alpha1.Metric{ + Name: "foo", + SuccessCondition: "all(result, # > 10)", + FailureCondition: "all(result, # < 10)", + Provider: v1alpha1.MetricProvider{ + Prometheus: &v1alpha1.PrometheusMetric{ + Query: "test", + RangeQuery: &v1alpha1.PrometheusRangeQueryArgs{ + Start: `date("2023-08-14 00:00:00", "2006-01-02 15:04:05", "UTC") - duration("1h")`, + End: `now() - duration("??")`, + Step: "1m", + }, + }, + }, + } + expectedErr := fmt.Errorf(`failed to parse rangeQuery.end as time: time: invalid duration "??"`) + + p, err := NewPrometheusProvider(mock, e, metric) + + measurement := p.Run(newAnalysisRun(), metric) + assert.NotNil(t, measurement.StartedAt) + assert.NoError(t, err) + assert.Equal(t, expectedErr.Error(), measurement.Message) + assert.Equal(t, "", measurement.Value) + assert.NotNil(t, measurement.FinishedAt) + assert.Equal(t, v1alpha1.AnalysisPhaseError, measurement.Phase) +} + +func TestRunUnparsableStep(t *testing.T) { + e := log.Entry{} + mock := &mockAPI{ + value: newMatrix(10), + } + metric := v1alpha1.Metric{ + Name: "foo", + SuccessCondition: "all(result, # > 10)", + FailureCondition: "all(result, # < 10)", + Provider: v1alpha1.MetricProvider{ + Prometheus: &v1alpha1.PrometheusMetric{ + Query: "test", + RangeQuery: &v1alpha1.PrometheusRangeQueryArgs{ + Start: `date("2023-08-14 00:00:00", "2006-01-02 15:04:05", "UTC") - duration("1h")`, + End: `date("2023-08-14 00:00:00", "2006-01-02 15:04:05", "UTC")`, + Step: "??", + }, + }, + }, + } + expectedErr := fmt.Errorf("failed to parse rangeQuery.step as duration: time: invalid duration \"??\"") + + p, err := NewPrometheusProvider(mock, e, metric) + + measurement := p.Run(newAnalysisRun(), metric) + assert.NotNil(t, measurement.StartedAt) + assert.NoError(t, err) + assert.Equal(t, expectedErr.Error(), measurement.Message) + assert.Equal(t, "", measurement.Value) + assert.NotNil(t, measurement.FinishedAt) + assert.Equal(t, v1alpha1.AnalysisPhaseError, measurement.Phase) +} + func TestRunSuccessfullyWithEnv(t *testing.T) { e := log.Entry{} - mock := mockAPI{ + mock := &mockAPI{ value: newScalar(10), } address := "http://127.0.0.1:9090" @@ -115,7 +278,7 @@ func TestRunSuccessfullyWithEnv(t *testing.T) { func TestRunSuccessfullyWithWarning(t *testing.T) { e := log.NewEntry(log.New()) - mock := mockAPI{ + mock := &mockAPI{ value: newScalar(10), warnings: v1.Warnings([]string{"warning", "warning2"}), } @@ -141,7 +304,7 @@ func TestRunSuccessfullyWithWarning(t *testing.T) { func TestRunSuccessfullyWithWarningWithEnv(t *testing.T) { e := log.NewEntry(log.New()) - mock := mockAPI{ + mock := &mockAPI{ value: newScalar(10), warnings: v1.Warnings([]string{"warning", "warning2"}), } @@ -168,7 +331,7 @@ func TestRunSuccessfullyWithWarningWithEnv(t *testing.T) { func TestRunWithQueryError(t *testing.T) { e := log.NewEntry(log.New()) expectedErr := fmt.Errorf("bad big bug :(") - mock := mockAPI{ + mock := &mockAPI{ err: expectedErr, } metric := v1alpha1.Metric{ @@ -194,7 +357,7 @@ func TestRunWithQueryError(t *testing.T) { func TestRunWithResolveArgsError(t *testing.T) { e := log.Entry{} expectedErr := fmt.Errorf("failed to resolve {{args.var}}") - mock := mockAPI{ + mock := &mockAPI{ err: expectedErr, } metric := v1alpha1.Metric{ @@ -217,7 +380,7 @@ func TestRunWithResolveArgsError(t *testing.T) { func TestGetStatusReturnsResolvedQuery(t *testing.T) { e := log.Entry{} - mock := mockAPI{} + mock := &mockAPI{} metric := v1alpha1.Metric{ Name: "foo", Provider: v1alpha1.MetricProvider{ @@ -235,7 +398,7 @@ func TestGetStatusReturnsResolvedQuery(t *testing.T) { func TestRunWithEvaluationError(t *testing.T) { e := log.WithField("", "") - mock := mockAPI{} + mock := &mockAPI{} metric := v1alpha1.Metric{ Name: "foo", SuccessCondition: "result == 10", @@ -258,7 +421,7 @@ func TestRunWithEvaluationError(t *testing.T) { func TestResume(t *testing.T) { e := log.WithField("", "") - mock := mockAPI{} + mock := &mockAPI{} metric := v1alpha1.Metric{ Name: "foo", SuccessCondition: "result == 10", @@ -282,7 +445,7 @@ func TestResume(t *testing.T) { func TestTerminate(t *testing.T) { e := log.NewEntry(log.New()) - mock := mockAPI{} + mock := &mockAPI{} metric := v1alpha1.Metric{} p, err := NewPrometheusProvider(mock, *e, metric) assert.NoError(t, err) @@ -297,7 +460,7 @@ func TestTerminate(t *testing.T) { func TestGarbageCollect(t *testing.T) { e := log.NewEntry(log.New()) - mock := mockAPI{} + mock := &mockAPI{} metric := v1alpha1.Metric{} p, err := NewPrometheusProvider(mock, *e, metric) assert.NoError(t, err) @@ -510,7 +673,7 @@ func TestNewPrometheusAddressNotConfigured(t *testing.T) { func TestNewPrometheusNegativeTimeout(t *testing.T) { e := log.Entry{} - mock := mockAPI{ + mock := &mockAPI{ value: newScalar(10), } timeout := int64(-20) diff --git a/metricproviders/wavefront/wavefront_test.go b/metricproviders/wavefront/wavefront_test.go index af8bd53e7f..49f86448e2 100644 --- a/metricproviders/wavefront/wavefront_test.go +++ b/metricproviders/wavefront/wavefront_test.go @@ -4,7 +4,6 @@ import ( "fmt" "testing" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" log "github.com/sirupsen/logrus" wavefrontapi "github.com/spaceapegames/go-wavefront" "github.com/stretchr/testify/assert" @@ -13,6 +12,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" k8sfake "k8s.io/client-go/kubernetes/fake" kubetesting "k8s.io/client-go/testing" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" ) func newAnalysisRun() *v1alpha1.AnalysisRun { diff --git a/metricproviders/webmetric/webmetric.go b/metricproviders/webmetric/webmetric.go index 6561e72290..6043885f72 100644 --- a/metricproviders/webmetric/webmetric.go +++ b/metricproviders/webmetric/webmetric.go @@ -176,6 +176,10 @@ func (p *Provider) GarbageCollect(run *v1alpha1.AnalysisRun, metric v1alpha1.Met return nil } +var insecureTransport *http.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, +} + func NewWebMetricHttpClient(metric v1alpha1.Metric) (*http.Client, error) { var timeout time.Duration var oauthCfg clientcredentials.Config @@ -191,10 +195,7 @@ func NewWebMetricHttpClient(metric v1alpha1.Metric) (*http.Client, error) { Timeout: timeout, } if metric.Provider.Web.Insecure { - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - c.Transport = tr + c.Transport = insecureTransport } if metric.Provider.Web.Authentication.OAuth2.TokenURL != "" { if metric.Provider.Web.Authentication.OAuth2.ClientID == "" || metric.Provider.Web.Authentication.OAuth2.ClientSecret == "" { diff --git a/metricproviders/webmetric/webmetric_test.go b/metricproviders/webmetric/webmetric_test.go index 1f8556a0eb..69e867d6fc 100644 --- a/metricproviders/webmetric/webmetric_test.go +++ b/metricproviders/webmetric/webmetric_test.go @@ -10,9 +10,10 @@ import ( "strings" "testing" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" ) const ( diff --git a/mkdocs.yml b/mkdocs.yml index 8506973d4f..d0ef5b0cd9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -32,8 +32,9 @@ nav: - Rollout: - Deployment Strategies: - BlueGreen: features/bluegreen.md - - Canary: features/canary.md + - Canary: features/canary/index.md - Rollout Spec: features/specification.md + - Canary Step Plugin: features/canary/plugins.md - HPA: features/hpa-support.md - VPA: features/vpa-support.md - Ephemeral Metadata: features/ephemeral-metadata.md @@ -136,7 +137,6 @@ nav: - Security: - Security Policy: security/security.md - Verify Release Artifacts: security/signed-release-assets.md -- Roadmap/Release Schedule: roadmap.md - Contributing: - Contribution Guide: CONTRIBUTING.md - Environment Setup: getting-started/setup/index.md diff --git a/pkg/apiclient/rollout/rollout.swagger.json b/pkg/apiclient/rollout/rollout.swagger.json index b3c10e03f8..ce2d2fe036 100755 --- a/pkg/apiclient/rollout/rollout.swagger.json +++ b/pkg/apiclient/rollout/rollout.swagger.json @@ -978,6 +978,13 @@ "stablePingPong": { "type": "string", "title": "StablePingPong For the ping-pong feature holds the current stable service, ping or pong" + }, + "stepPluginStatuses": { + "type": "array", + "items": { + "$ref": "#/definitions/github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.StepPluginStatus" + }, + "title": "StepPluginStatuses holds the status of the step plugins executed" } }, "title": "CanaryStatus status fields that only pertain to the canary rollout" @@ -1013,6 +1020,10 @@ "setMirrorRoute": { "$ref": "#/definitions/github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.SetMirrorRoute", "title": "SetMirrorRoutes Mirrors traffic that matches rules to a particular destination\n+optional" + }, + "plugin": { + "$ref": "#/definitions/github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.PluginStep", + "title": "Plugin defines a plugin to execute for a step" } }, "description": "CanaryStep defines a step of a canary deployment." @@ -1205,7 +1216,11 @@ }, "aggregator": { "type": "string", - "title": "+kubebuilder:default=\"last\"\n+kubebuilder:validation:Enum=avg;min;max;sum;last;percentile;mean;l2norm;area\nAggregator is a type of aggregator to use for metrics-based queries (default: last). Used for v2" + "title": "+kubebuilder:validation:Enum=avg;min;max;sum;last;percentile;mean;l2norm;area\nAggregator is a type of aggregator to use for metrics-based queries (default: \"\"). Used for v2" + }, + "secretRef": { + "$ref": "#/definitions/github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.SecretRef", + "title": "Secret refers to the name of the secret that should be used for an analysis and should exists in the namespace where the controller is.\n+optional" } } }, @@ -1499,7 +1514,7 @@ }, "failureLimit": { "$ref": "#/definitions/k8s.io.apimachinery.pkg.util.intstr.IntOrString", - "title": "FailureLimit is the maximum number of times the measurement is allowed to fail, before the\nentire metric is considered Failed (default: 0)" + "title": "FailureLimit is the maximum number of times the measurement is allowed to fail, before the\nentire metric is considered Failed (default: 0)\n-1 for making it disabled (when opting to use ConsecutiveSuccessLimit solely)" }, "inconclusiveLimit": { "$ref": "#/definitions/k8s.io.apimachinery.pkg.util.intstr.IntOrString", @@ -1512,6 +1527,10 @@ "provider": { "$ref": "#/definitions/github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.MetricProvider", "title": "Provider configuration to the external system to use to verify the analysis" + }, + "consecutiveSuccessLimit": { + "$ref": "#/definitions/k8s.io.apimachinery.pkg.util.intstr.IntOrString", + "title": "ConsecutiveSuccessLimit is the number of consecutive times the measurement must succeed for the\nentire metric to be considered Successful (default: 0, which means it's disabled)" } }, "title": "Metric defines a metric in which to perform analysis" @@ -1636,6 +1655,11 @@ "type": "string" }, "description": "Metadata stores additional metadata about this metric. It is used by different providers to store\nthe final state which gets used while taking measurements. For example, Prometheus uses this field\nto store the final resolved query after substituting the template arguments." + }, + "consecutiveSuccess": { + "type": "integer", + "format": "int32", + "title": "ConsecutiveSuccess is the number of times a measurement was successful in succession\nResets to zero when failures, inconclusive measurements, or errors are encountered" } }, "title": "MetricResult contain a list of the most recent measurements for a single metric along with\ncounters on how often the measurement" @@ -1650,6 +1674,11 @@ "query": { "type": "string", "title": "Query is a raw newrelic NRQL query to perform" + }, + "timeout": { + "type": "string", + "format": "int64", + "title": "Timeout represents the duration limit in seconds that will apply to the NRQL query\n+optional" } }, "title": "NewRelicMetric defines the newrelic query to perform canary analysis" @@ -1678,6 +1707,13 @@ "type": "string" }, "title": "StableIngresses refers to the names of `Ingress` resources in the same namespace as the `Rollout` in a multi ingress scenario\n+optional" + }, + "canaryIngressAnnotations": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "title": "+optional" } }, "title": "NginxTrafficRouting configuration for Nginx ingress controller to control traffic routing" @@ -1754,6 +1790,20 @@ }, "description": "PingPongSpec holds the ping and pong service name." }, + "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.PluginStep": { + "type": "object", + "properties": { + "name": { + "type": "string", + "title": "Name of the hashicorp go-plugin step to query" + }, + "config": { + "type": "string", + "format": "byte", + "title": "+kubebuilder:validation:Schemaless\n+kubebuilder:pruning:PreserveUnknownFields\n+kubebuilder:validation:Type=object\nConfig is the configuration object for the specified plugin" + } + } + }, "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.PodTemplateMetadata": { "type": "object", "properties": { @@ -1815,10 +1865,32 @@ "$ref": "#/definitions/github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.WebMetricHeader" }, "title": "Headers are optional HTTP headers to use in the request\n+optional\n+patchMergeKey=key\n+patchStrategy=merge" + }, + "rangeQuery": { + "$ref": "#/definitions/github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.PrometheusRangeQueryArgs", + "title": "Arguments for prometheus\n+optional" } }, "title": "PrometheusMetric defines the prometheus query to perform canary analysis" }, + "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.PrometheusRangeQueryArgs": { + "type": "object", + "properties": { + "start": { + "type": "string", + "title": "The start time to query in expr format e.g. now(), now() - duration(\"1h\"), now() - duration(\"{{args.lookback_duration}}\")" + }, + "end": { + "type": "string", + "title": "The end time to query in expr format e.g. now(), now() - duration(\"1h\"), now() - duration(\"{{args.lookback_duration}}\")" + }, + "step": { + "type": "string", + "description": "The maximum time between two slices from the start to end (e.g. 30s, 5m, 1h)." + } + }, + "title": "Arguments to perform a prometheus range query" + }, "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.RequiredDuringSchedulingIgnoredDuringExecution": { "type": "object", "title": "RequiredDuringSchedulingIgnoredDuringExecution defines inter-pod scheduling rule to be RequiredDuringSchedulingIgnoredDuringExecution" @@ -2411,6 +2483,19 @@ } } }, + "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.SecretRef": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name refers to the name of the secret that should be used to integrate with Datadog." + }, + "namespaced": { + "type": "boolean", + "title": "Namespaced indicates whether the secret is in the namespace where rollouts it installed or in the namespace where the metric was found" + } + } + }, "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.SetCanaryScale": { "type": "object", "properties": { @@ -2499,6 +2584,62 @@ } } }, + "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.StepPluginStatus": { + "type": "object", + "properties": { + "index": { + "type": "integer", + "format": "int32", + "title": "Index is the matching step index of the executed plugin" + }, + "name": { + "type": "string", + "title": "Name is the matching step name of the executed plugin" + }, + "operation": { + "type": "string", + "title": "Operation is the name of the operation that produced this status" + }, + "phase": { + "type": "string", + "title": "Phase is the resulting phase of the operation" + }, + "message": { + "type": "string", + "title": "Message provides details on why the plugin is in its current phase" + }, + "startedAt": { + "$ref": "#/definitions/k8s.io.apimachinery.pkg.apis.meta.v1.Time", + "title": "StartedAt indicates when the plugin was first called for the operation" + }, + "updatedAt": { + "$ref": "#/definitions/k8s.io.apimachinery.pkg.apis.meta.v1.Time", + "title": "UpdatedAt indicates when the plugin was last called for the operation" + }, + "finishedAt": { + "$ref": "#/definitions/k8s.io.apimachinery.pkg.apis.meta.v1.Time", + "title": "FinishedAt indicates when the operation was completed" + }, + "backoff": { + "type": "string", + "title": "Backoff is a duration to wait before trying to execute the operation again if it was not completed" + }, + "executions": { + "type": "integer", + "format": "int32", + "title": "Executions is the number of time the operation was executed" + }, + "disabled": { + "type": "boolean", + "title": "Disabled indicates if the plugin is globally disabled" + }, + "status": { + "type": "string", + "format": "byte", + "title": "+kubebuilder:validation:Schemaless\n+kubebuilder:pruning:PreserveUnknownFields\n+kubebuilder:validation:Type=object\nStatus holds the internal status of the plugin for this operation" + } + } + }, "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.StickinessConfig": { "type": "object", "properties": { diff --git a/pkg/apis/api-rules/violation_exceptions.list b/pkg/apis/api-rules/violation_exceptions.list index ea9a241837..c4c0952a74 100644 --- a/pkg/apis/api-rules/violation_exceptions.list +++ b/pkg/apis/api-rules/violation_exceptions.list @@ -12,6 +12,7 @@ API rule violation: list_type_missing,github.com/argoproj/argo-rollouts/pkg/apis API rule violation: list_type_missing,github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1,AnalysisTemplateSpec,Templates API rule violation: list_type_missing,github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1,ApisixRoute,Rules API rule violation: list_type_missing,github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1,AppMeshVirtualService,Routes +API rule violation: list_type_missing,github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1,CanaryStatus,StepPluginStatuses API rule violation: list_type_missing,github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1,CanaryStrategy,Steps API rule violation: list_type_missing,github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1,CloudWatchMetric,MetricDataQueries API rule violation: list_type_missing,github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1,CloudWatchMetricStatMetric,Dimensions diff --git a/pkg/apis/rollouts/v1alpha1/analysis_types.go b/pkg/apis/rollouts/v1alpha1/analysis_types.go index 08ef18071f..20c50f881f 100644 --- a/pkg/apis/rollouts/v1alpha1/analysis_types.go +++ b/pkg/apis/rollouts/v1alpha1/analysis_types.go @@ -112,6 +112,7 @@ type Metric struct { FailureCondition string `json:"failureCondition,omitempty" protobuf:"bytes,6,opt,name=failureCondition"` // FailureLimit is the maximum number of times the measurement is allowed to fail, before the // entire metric is considered Failed (default: 0) + // -1 for making it disabled (when opting to use ConsecutiveSuccessLimit solely) FailureLimit *intstrutil.IntOrString `json:"failureLimit,omitempty" protobuf:"bytes,7,opt,name=failureLimit"` // InconclusiveLimit is the maximum number of times the measurement is allowed to measure // Inconclusive, before the entire metric is considered Inconclusive (default: 0) @@ -121,6 +122,9 @@ type Metric struct { ConsecutiveErrorLimit *intstrutil.IntOrString `json:"consecutiveErrorLimit,omitempty" protobuf:"bytes,9,opt,name=consecutiveErrorLimit"` // Provider configuration to the external system to use to verify the analysis Provider MetricProvider `json:"provider" protobuf:"bytes,10,opt,name=provider"` + // ConsecutiveSuccessLimit is the number of consecutive times the measurement must succeed for the + // entire metric to be considered Successful (default: 0, which means it's disabled) + ConsecutiveSuccessLimit *intstrutil.IntOrString `json:"consecutiveSuccessLimit,omitempty" protobuf:"bytes,11,opt,name=consecutiveSuccessLimit"` } // DryRun defines the settings for running the analysis in Dry-Run mode. @@ -218,6 +222,16 @@ func (as AnalysisPhase) Completed() bool { return false } +// Arguments to perform a prometheus range query +type PrometheusRangeQueryArgs struct { + // The start time to query in expr format e.g. now(), now() - duration("1h"), now() - duration("{{args.lookback_duration}}") + Start string `json:"start,omitempty" protobuf:"bytes,1,opt,name=start"` + // The end time to query in expr format e.g. now(), now() - duration("1h"), now() - duration("{{args.lookback_duration}}") + End string `json:"end,omitempty" protobuf:"bytes,2,opt,name=end"` + // The maximum time between two slices from the start to end (e.g. 30s, 5m, 1h). + Step DurationString `json:"step,omitempty" protobuf:"bytes,3,opt,name=step,casttype=DurationString"` +} + // PrometheusMetric defines the prometheus query to perform canary analysis type PrometheusMetric struct { // Address is the HTTP address and port of the prometheus server @@ -237,6 +251,9 @@ type PrometheusMetric struct { // +patchMergeKey=key // +patchStrategy=merge Headers []WebMetricHeader `json:"headers,omitempty" patchStrategy:"merge" patchMergeKey:"key" protobuf:"bytes,6,opt,name=headers"` + // Arguments for prometheus + // +optional + RangeQuery *PrometheusRangeQueryArgs `json:"rangeQuery,omitempty" protobuf:"bytes,7,opt,name=rangeQuery"` } // Authentication method @@ -284,6 +301,9 @@ type NewRelicMetric struct { Profile string `json:"profile,omitempty" protobuf:"bytes,1,opt,name=profile"` // Query is a raw newrelic NRQL query to perform Query string `json:"query" protobuf:"bytes,2,opt,name=query"` + // Timeout represents the duration limit in seconds that will apply to the NRQL query + // +optional + Timeout *int64 `json:"timeout,omitempty" protobuf:"bytes,3,opt,name=timeout"` } // JobMetric defines a job to run which acts as a metric @@ -489,6 +509,9 @@ type MetricResult struct { // the final state which gets used while taking measurements. For example, Prometheus uses this field // to store the final resolved query after substituting the template arguments. Metadata map[string]string `json:"metadata,omitempty" protobuf:"bytes,12,rep,name=metadata"` + // ConsecutiveSuccess is the number of times a measurement was successful in succession + // Resets to zero when failures, inconclusive measurements, or errors are encountered + ConsecutiveSuccess int32 `json:"consecutiveSuccess,omitempty" protobuf:"varint,13,opt,name=consecutiveSuccess"` } // Measurement is a point in time result value of a single metric, and the time it was measured @@ -601,8 +624,17 @@ type DatadogMetric struct { // +kubebuilder:validation:Enum=v1;v2 // +kubebuilder:default=v1 ApiVersion string `json:"apiVersion,omitempty" protobuf:"bytes,5,opt,name=apiVersion"` - // +kubebuilder:default="last" // +kubebuilder:validation:Enum=avg;min;max;sum;last;percentile;mean;l2norm;area - // Aggregator is a type of aggregator to use for metrics-based queries (default: last). Used for v2 + // Aggregator is a type of aggregator to use for metrics-based queries (default: ""). Used for v2 Aggregator string `json:"aggregator,omitempty" protobuf:"bytes,6,opt,name=aggregator"` + // Secret refers to the name of the secret that should be used for an analysis and should exists in the namespace where the controller is. + // +optional + SecretRef SecretRef `json:"secretRef,omitempty" protobuf:"bytes,7,opt,name=secretRef"` +} + +type SecretRef struct { + // Name refers to the name of the secret that should be used to integrate with Datadog. + Name string `json:"name,omitempty" protobuf:"bytes,1,opt,name=name"` + // Namespaced indicates whether the secret is in the namespace where rollouts it installed or in the namespace where the metric was found + Namespaced bool `json:"namespaced,omitempty" protobuf:"varint,2,opt,namespaced=dryRun"` } diff --git a/pkg/apis/rollouts/v1alpha1/generated.pb.go b/pkg/apis/rollouts/v1alpha1/generated.pb.go index cab169a9d0..7caa8b0aaa 100644 --- a/pkg/apis/rollouts/v1alpha1/generated.pb.go +++ b/pkg/apis/rollouts/v1alpha1/generated.pb.go @@ -1979,10 +1979,38 @@ func (m *PingPongSpec) XXX_DiscardUnknown() { var xxx_messageInfo_PingPongSpec proto.InternalMessageInfo +func (m *PluginStep) Reset() { *m = PluginStep{} } +func (*PluginStep) ProtoMessage() {} +func (*PluginStep) Descriptor() ([]byte, []int) { + return fileDescriptor_e0e705f843545fab, []int{69} +} +func (m *PluginStep) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *PluginStep) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil +} +func (m *PluginStep) XXX_Merge(src proto.Message) { + xxx_messageInfo_PluginStep.Merge(m, src) +} +func (m *PluginStep) XXX_Size() int { + return m.Size() +} +func (m *PluginStep) XXX_DiscardUnknown() { + xxx_messageInfo_PluginStep.DiscardUnknown(m) +} + +var xxx_messageInfo_PluginStep proto.InternalMessageInfo + func (m *PodTemplateMetadata) Reset() { *m = PodTemplateMetadata{} } func (*PodTemplateMetadata) ProtoMessage() {} func (*PodTemplateMetadata) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{69} + return fileDescriptor_e0e705f843545fab, []int{70} } func (m *PodTemplateMetadata) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2012,7 +2040,7 @@ func (m *PreferredDuringSchedulingIgnoredDuringExecution) Reset() { } func (*PreferredDuringSchedulingIgnoredDuringExecution) ProtoMessage() {} func (*PreferredDuringSchedulingIgnoredDuringExecution) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{70} + return fileDescriptor_e0e705f843545fab, []int{71} } func (m *PreferredDuringSchedulingIgnoredDuringExecution) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2040,7 +2068,7 @@ var xxx_messageInfo_PreferredDuringSchedulingIgnoredDuringExecution proto.Intern func (m *PrometheusMetric) Reset() { *m = PrometheusMetric{} } func (*PrometheusMetric) ProtoMessage() {} func (*PrometheusMetric) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{71} + return fileDescriptor_e0e705f843545fab, []int{72} } func (m *PrometheusMetric) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2065,12 +2093,40 @@ func (m *PrometheusMetric) XXX_DiscardUnknown() { var xxx_messageInfo_PrometheusMetric proto.InternalMessageInfo +func (m *PrometheusRangeQueryArgs) Reset() { *m = PrometheusRangeQueryArgs{} } +func (*PrometheusRangeQueryArgs) ProtoMessage() {} +func (*PrometheusRangeQueryArgs) Descriptor() ([]byte, []int) { + return fileDescriptor_e0e705f843545fab, []int{73} +} +func (m *PrometheusRangeQueryArgs) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *PrometheusRangeQueryArgs) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil +} +func (m *PrometheusRangeQueryArgs) XXX_Merge(src proto.Message) { + xxx_messageInfo_PrometheusRangeQueryArgs.Merge(m, src) +} +func (m *PrometheusRangeQueryArgs) XXX_Size() int { + return m.Size() +} +func (m *PrometheusRangeQueryArgs) XXX_DiscardUnknown() { + xxx_messageInfo_PrometheusRangeQueryArgs.DiscardUnknown(m) +} + +var xxx_messageInfo_PrometheusRangeQueryArgs proto.InternalMessageInfo + func (m *RequiredDuringSchedulingIgnoredDuringExecution) Reset() { *m = RequiredDuringSchedulingIgnoredDuringExecution{} } func (*RequiredDuringSchedulingIgnoredDuringExecution) ProtoMessage() {} func (*RequiredDuringSchedulingIgnoredDuringExecution) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{72} + return fileDescriptor_e0e705f843545fab, []int{74} } func (m *RequiredDuringSchedulingIgnoredDuringExecution) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2098,7 +2154,7 @@ var xxx_messageInfo_RequiredDuringSchedulingIgnoredDuringExecution proto.Interna func (m *RollbackWindowSpec) Reset() { *m = RollbackWindowSpec{} } func (*RollbackWindowSpec) ProtoMessage() {} func (*RollbackWindowSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{73} + return fileDescriptor_e0e705f843545fab, []int{75} } func (m *RollbackWindowSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2126,7 +2182,7 @@ var xxx_messageInfo_RollbackWindowSpec proto.InternalMessageInfo func (m *Rollout) Reset() { *m = Rollout{} } func (*Rollout) ProtoMessage() {} func (*Rollout) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{74} + return fileDescriptor_e0e705f843545fab, []int{76} } func (m *Rollout) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2154,7 +2210,7 @@ var xxx_messageInfo_Rollout proto.InternalMessageInfo func (m *RolloutAnalysis) Reset() { *m = RolloutAnalysis{} } func (*RolloutAnalysis) ProtoMessage() {} func (*RolloutAnalysis) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{75} + return fileDescriptor_e0e705f843545fab, []int{77} } func (m *RolloutAnalysis) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2182,7 +2238,7 @@ var xxx_messageInfo_RolloutAnalysis proto.InternalMessageInfo func (m *RolloutAnalysisBackground) Reset() { *m = RolloutAnalysisBackground{} } func (*RolloutAnalysisBackground) ProtoMessage() {} func (*RolloutAnalysisBackground) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{76} + return fileDescriptor_e0e705f843545fab, []int{78} } func (m *RolloutAnalysisBackground) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2210,7 +2266,7 @@ var xxx_messageInfo_RolloutAnalysisBackground proto.InternalMessageInfo func (m *RolloutAnalysisRunStatus) Reset() { *m = RolloutAnalysisRunStatus{} } func (*RolloutAnalysisRunStatus) ProtoMessage() {} func (*RolloutAnalysisRunStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{77} + return fileDescriptor_e0e705f843545fab, []int{79} } func (m *RolloutAnalysisRunStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2238,7 +2294,7 @@ var xxx_messageInfo_RolloutAnalysisRunStatus proto.InternalMessageInfo func (m *RolloutCondition) Reset() { *m = RolloutCondition{} } func (*RolloutCondition) ProtoMessage() {} func (*RolloutCondition) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{78} + return fileDescriptor_e0e705f843545fab, []int{80} } func (m *RolloutCondition) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2266,7 +2322,7 @@ var xxx_messageInfo_RolloutCondition proto.InternalMessageInfo func (m *RolloutExperimentStep) Reset() { *m = RolloutExperimentStep{} } func (*RolloutExperimentStep) ProtoMessage() {} func (*RolloutExperimentStep) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{79} + return fileDescriptor_e0e705f843545fab, []int{81} } func (m *RolloutExperimentStep) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2296,7 +2352,7 @@ func (m *RolloutExperimentStepAnalysisTemplateRef) Reset() { } func (*RolloutExperimentStepAnalysisTemplateRef) ProtoMessage() {} func (*RolloutExperimentStepAnalysisTemplateRef) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{80} + return fileDescriptor_e0e705f843545fab, []int{82} } func (m *RolloutExperimentStepAnalysisTemplateRef) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2324,7 +2380,7 @@ var xxx_messageInfo_RolloutExperimentStepAnalysisTemplateRef proto.InternalMessa func (m *RolloutExperimentTemplate) Reset() { *m = RolloutExperimentTemplate{} } func (*RolloutExperimentTemplate) ProtoMessage() {} func (*RolloutExperimentTemplate) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{81} + return fileDescriptor_e0e705f843545fab, []int{83} } func (m *RolloutExperimentTemplate) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2352,7 +2408,7 @@ var xxx_messageInfo_RolloutExperimentTemplate proto.InternalMessageInfo func (m *RolloutList) Reset() { *m = RolloutList{} } func (*RolloutList) ProtoMessage() {} func (*RolloutList) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{82} + return fileDescriptor_e0e705f843545fab, []int{84} } func (m *RolloutList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2380,7 +2436,7 @@ var xxx_messageInfo_RolloutList proto.InternalMessageInfo func (m *RolloutPause) Reset() { *m = RolloutPause{} } func (*RolloutPause) ProtoMessage() {} func (*RolloutPause) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{83} + return fileDescriptor_e0e705f843545fab, []int{85} } func (m *RolloutPause) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2408,7 +2464,7 @@ var xxx_messageInfo_RolloutPause proto.InternalMessageInfo func (m *RolloutSpec) Reset() { *m = RolloutSpec{} } func (*RolloutSpec) ProtoMessage() {} func (*RolloutSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{84} + return fileDescriptor_e0e705f843545fab, []int{86} } func (m *RolloutSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2436,7 +2492,7 @@ var xxx_messageInfo_RolloutSpec proto.InternalMessageInfo func (m *RolloutStatus) Reset() { *m = RolloutStatus{} } func (*RolloutStatus) ProtoMessage() {} func (*RolloutStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{85} + return fileDescriptor_e0e705f843545fab, []int{87} } func (m *RolloutStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2464,7 +2520,7 @@ var xxx_messageInfo_RolloutStatus proto.InternalMessageInfo func (m *RolloutStrategy) Reset() { *m = RolloutStrategy{} } func (*RolloutStrategy) ProtoMessage() {} func (*RolloutStrategy) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{86} + return fileDescriptor_e0e705f843545fab, []int{88} } func (m *RolloutStrategy) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2492,7 +2548,7 @@ var xxx_messageInfo_RolloutStrategy proto.InternalMessageInfo func (m *RolloutTrafficRouting) Reset() { *m = RolloutTrafficRouting{} } func (*RolloutTrafficRouting) ProtoMessage() {} func (*RolloutTrafficRouting) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{87} + return fileDescriptor_e0e705f843545fab, []int{89} } func (m *RolloutTrafficRouting) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2520,7 +2576,7 @@ var xxx_messageInfo_RolloutTrafficRouting proto.InternalMessageInfo func (m *RouteMatch) Reset() { *m = RouteMatch{} } func (*RouteMatch) ProtoMessage() {} func (*RouteMatch) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{88} + return fileDescriptor_e0e705f843545fab, []int{90} } func (m *RouteMatch) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2548,7 +2604,7 @@ var xxx_messageInfo_RouteMatch proto.InternalMessageInfo func (m *RunSummary) Reset() { *m = RunSummary{} } func (*RunSummary) ProtoMessage() {} func (*RunSummary) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{89} + return fileDescriptor_e0e705f843545fab, []int{91} } func (m *RunSummary) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2576,7 +2632,7 @@ var xxx_messageInfo_RunSummary proto.InternalMessageInfo func (m *SMITrafficRouting) Reset() { *m = SMITrafficRouting{} } func (*SMITrafficRouting) ProtoMessage() {} func (*SMITrafficRouting) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{90} + return fileDescriptor_e0e705f843545fab, []int{92} } func (m *SMITrafficRouting) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2604,7 +2660,7 @@ var xxx_messageInfo_SMITrafficRouting proto.InternalMessageInfo func (m *ScopeDetail) Reset() { *m = ScopeDetail{} } func (*ScopeDetail) ProtoMessage() {} func (*ScopeDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{91} + return fileDescriptor_e0e705f843545fab, []int{93} } func (m *ScopeDetail) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2632,7 +2688,7 @@ var xxx_messageInfo_ScopeDetail proto.InternalMessageInfo func (m *SecretKeyRef) Reset() { *m = SecretKeyRef{} } func (*SecretKeyRef) ProtoMessage() {} func (*SecretKeyRef) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{92} + return fileDescriptor_e0e705f843545fab, []int{94} } func (m *SecretKeyRef) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2657,10 +2713,38 @@ func (m *SecretKeyRef) XXX_DiscardUnknown() { var xxx_messageInfo_SecretKeyRef proto.InternalMessageInfo +func (m *SecretRef) Reset() { *m = SecretRef{} } +func (*SecretRef) ProtoMessage() {} +func (*SecretRef) Descriptor() ([]byte, []int) { + return fileDescriptor_e0e705f843545fab, []int{95} +} +func (m *SecretRef) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SecretRef) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil +} +func (m *SecretRef) XXX_Merge(src proto.Message) { + xxx_messageInfo_SecretRef.Merge(m, src) +} +func (m *SecretRef) XXX_Size() int { + return m.Size() +} +func (m *SecretRef) XXX_DiscardUnknown() { + xxx_messageInfo_SecretRef.DiscardUnknown(m) +} + +var xxx_messageInfo_SecretRef proto.InternalMessageInfo + func (m *SetCanaryScale) Reset() { *m = SetCanaryScale{} } func (*SetCanaryScale) ProtoMessage() {} func (*SetCanaryScale) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{93} + return fileDescriptor_e0e705f843545fab, []int{96} } func (m *SetCanaryScale) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2688,7 +2772,7 @@ var xxx_messageInfo_SetCanaryScale proto.InternalMessageInfo func (m *SetHeaderRoute) Reset() { *m = SetHeaderRoute{} } func (*SetHeaderRoute) ProtoMessage() {} func (*SetHeaderRoute) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{94} + return fileDescriptor_e0e705f843545fab, []int{97} } func (m *SetHeaderRoute) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2716,7 +2800,7 @@ var xxx_messageInfo_SetHeaderRoute proto.InternalMessageInfo func (m *SetMirrorRoute) Reset() { *m = SetMirrorRoute{} } func (*SetMirrorRoute) ProtoMessage() {} func (*SetMirrorRoute) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{95} + return fileDescriptor_e0e705f843545fab, []int{98} } func (m *SetMirrorRoute) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2744,7 +2828,7 @@ var xxx_messageInfo_SetMirrorRoute proto.InternalMessageInfo func (m *Sigv4Config) Reset() { *m = Sigv4Config{} } func (*Sigv4Config) ProtoMessage() {} func (*Sigv4Config) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{96} + return fileDescriptor_e0e705f843545fab, []int{99} } func (m *Sigv4Config) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2772,7 +2856,7 @@ var xxx_messageInfo_Sigv4Config proto.InternalMessageInfo func (m *SkyWalkingMetric) Reset() { *m = SkyWalkingMetric{} } func (*SkyWalkingMetric) ProtoMessage() {} func (*SkyWalkingMetric) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{97} + return fileDescriptor_e0e705f843545fab, []int{100} } func (m *SkyWalkingMetric) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2797,10 +2881,38 @@ func (m *SkyWalkingMetric) XXX_DiscardUnknown() { var xxx_messageInfo_SkyWalkingMetric proto.InternalMessageInfo +func (m *StepPluginStatus) Reset() { *m = StepPluginStatus{} } +func (*StepPluginStatus) ProtoMessage() {} +func (*StepPluginStatus) Descriptor() ([]byte, []int) { + return fileDescriptor_e0e705f843545fab, []int{101} +} +func (m *StepPluginStatus) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *StepPluginStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil +} +func (m *StepPluginStatus) XXX_Merge(src proto.Message) { + xxx_messageInfo_StepPluginStatus.Merge(m, src) +} +func (m *StepPluginStatus) XXX_Size() int { + return m.Size() +} +func (m *StepPluginStatus) XXX_DiscardUnknown() { + xxx_messageInfo_StepPluginStatus.DiscardUnknown(m) +} + +var xxx_messageInfo_StepPluginStatus proto.InternalMessageInfo + func (m *StickinessConfig) Reset() { *m = StickinessConfig{} } func (*StickinessConfig) ProtoMessage() {} func (*StickinessConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{98} + return fileDescriptor_e0e705f843545fab, []int{102} } func (m *StickinessConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2828,7 +2940,7 @@ var xxx_messageInfo_StickinessConfig proto.InternalMessageInfo func (m *StringMatch) Reset() { *m = StringMatch{} } func (*StringMatch) ProtoMessage() {} func (*StringMatch) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{99} + return fileDescriptor_e0e705f843545fab, []int{103} } func (m *StringMatch) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2856,7 +2968,7 @@ var xxx_messageInfo_StringMatch proto.InternalMessageInfo func (m *TCPRoute) Reset() { *m = TCPRoute{} } func (*TCPRoute) ProtoMessage() {} func (*TCPRoute) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{100} + return fileDescriptor_e0e705f843545fab, []int{104} } func (m *TCPRoute) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2884,7 +2996,7 @@ var xxx_messageInfo_TCPRoute proto.InternalMessageInfo func (m *TLSRoute) Reset() { *m = TLSRoute{} } func (*TLSRoute) ProtoMessage() {} func (*TLSRoute) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{101} + return fileDescriptor_e0e705f843545fab, []int{105} } func (m *TLSRoute) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2912,7 +3024,7 @@ var xxx_messageInfo_TLSRoute proto.InternalMessageInfo func (m *TTLStrategy) Reset() { *m = TTLStrategy{} } func (*TTLStrategy) ProtoMessage() {} func (*TTLStrategy) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{102} + return fileDescriptor_e0e705f843545fab, []int{106} } func (m *TTLStrategy) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2940,7 +3052,7 @@ var xxx_messageInfo_TTLStrategy proto.InternalMessageInfo func (m *TemplateService) Reset() { *m = TemplateService{} } func (*TemplateService) ProtoMessage() {} func (*TemplateService) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{103} + return fileDescriptor_e0e705f843545fab, []int{107} } func (m *TemplateService) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2968,7 +3080,7 @@ var xxx_messageInfo_TemplateService proto.InternalMessageInfo func (m *TemplateSpec) Reset() { *m = TemplateSpec{} } func (*TemplateSpec) ProtoMessage() {} func (*TemplateSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{104} + return fileDescriptor_e0e705f843545fab, []int{108} } func (m *TemplateSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2996,7 +3108,7 @@ var xxx_messageInfo_TemplateSpec proto.InternalMessageInfo func (m *TemplateStatus) Reset() { *m = TemplateStatus{} } func (*TemplateStatus) ProtoMessage() {} func (*TemplateStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{105} + return fileDescriptor_e0e705f843545fab, []int{109} } func (m *TemplateStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3024,7 +3136,7 @@ var xxx_messageInfo_TemplateStatus proto.InternalMessageInfo func (m *TraefikTrafficRouting) Reset() { *m = TraefikTrafficRouting{} } func (*TraefikTrafficRouting) ProtoMessage() {} func (*TraefikTrafficRouting) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{106} + return fileDescriptor_e0e705f843545fab, []int{110} } func (m *TraefikTrafficRouting) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3052,7 +3164,7 @@ var xxx_messageInfo_TraefikTrafficRouting proto.InternalMessageInfo func (m *TrafficWeights) Reset() { *m = TrafficWeights{} } func (*TrafficWeights) ProtoMessage() {} func (*TrafficWeights) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{107} + return fileDescriptor_e0e705f843545fab, []int{111} } func (m *TrafficWeights) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3080,7 +3192,7 @@ var xxx_messageInfo_TrafficWeights proto.InternalMessageInfo func (m *ValueFrom) Reset() { *m = ValueFrom{} } func (*ValueFrom) ProtoMessage() {} func (*ValueFrom) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{108} + return fileDescriptor_e0e705f843545fab, []int{112} } func (m *ValueFrom) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3108,7 +3220,7 @@ var xxx_messageInfo_ValueFrom proto.InternalMessageInfo func (m *WavefrontMetric) Reset() { *m = WavefrontMetric{} } func (*WavefrontMetric) ProtoMessage() {} func (*WavefrontMetric) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{109} + return fileDescriptor_e0e705f843545fab, []int{113} } func (m *WavefrontMetric) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3136,7 +3248,7 @@ var xxx_messageInfo_WavefrontMetric proto.InternalMessageInfo func (m *WebMetric) Reset() { *m = WebMetric{} } func (*WebMetric) ProtoMessage() {} func (*WebMetric) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{110} + return fileDescriptor_e0e705f843545fab, []int{114} } func (m *WebMetric) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3164,7 +3276,7 @@ var xxx_messageInfo_WebMetric proto.InternalMessageInfo func (m *WebMetricHeader) Reset() { *m = WebMetricHeader{} } func (*WebMetricHeader) ProtoMessage() {} func (*WebMetricHeader) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{111} + return fileDescriptor_e0e705f843545fab, []int{115} } func (m *WebMetricHeader) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3192,7 +3304,7 @@ var xxx_messageInfo_WebMetricHeader proto.InternalMessageInfo func (m *WeightDestination) Reset() { *m = WeightDestination{} } func (*WeightDestination) ProtoMessage() {} func (*WeightDestination) Descriptor() ([]byte, []int) { - return fileDescriptor_e0e705f843545fab, []int{112} + return fileDescriptor_e0e705f843545fab, []int{116} } func (m *WeightDestination) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3290,15 +3402,18 @@ func init() { proto.RegisterType((*NewRelicMetric)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.NewRelicMetric") proto.RegisterType((*NginxTrafficRouting)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.NginxTrafficRouting") proto.RegisterMapType((map[string]string)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.NginxTrafficRouting.AdditionalIngressAnnotationsEntry") + proto.RegisterMapType((map[string]string)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.NginxTrafficRouting.CanaryIngressAnnotationsEntry") proto.RegisterType((*OAuth2Config)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.OAuth2Config") proto.RegisterType((*ObjectRef)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.ObjectRef") proto.RegisterType((*PauseCondition)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.PauseCondition") proto.RegisterType((*PingPongSpec)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.PingPongSpec") + proto.RegisterType((*PluginStep)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.PluginStep") proto.RegisterType((*PodTemplateMetadata)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.PodTemplateMetadata") proto.RegisterMapType((map[string]string)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.PodTemplateMetadata.AnnotationsEntry") proto.RegisterMapType((map[string]string)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.PodTemplateMetadata.LabelsEntry") proto.RegisterType((*PreferredDuringSchedulingIgnoredDuringExecution)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.PreferredDuringSchedulingIgnoredDuringExecution") proto.RegisterType((*PrometheusMetric)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.PrometheusMetric") + proto.RegisterType((*PrometheusRangeQueryArgs)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.PrometheusRangeQueryArgs") proto.RegisterType((*RequiredDuringSchedulingIgnoredDuringExecution)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.RequiredDuringSchedulingIgnoredDuringExecution") proto.RegisterType((*RollbackWindowSpec)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.RollbackWindowSpec") proto.RegisterType((*Rollout)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.Rollout") @@ -3322,11 +3437,13 @@ func init() { proto.RegisterType((*SMITrafficRouting)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.SMITrafficRouting") proto.RegisterType((*ScopeDetail)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.ScopeDetail") proto.RegisterType((*SecretKeyRef)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.SecretKeyRef") + proto.RegisterType((*SecretRef)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.SecretRef") proto.RegisterType((*SetCanaryScale)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.SetCanaryScale") proto.RegisterType((*SetHeaderRoute)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.SetHeaderRoute") proto.RegisterType((*SetMirrorRoute)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.SetMirrorRoute") proto.RegisterType((*Sigv4Config)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.Sigv4Config") proto.RegisterType((*SkyWalkingMetric)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.SkyWalkingMetric") + proto.RegisterType((*StepPluginStatus)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.StepPluginStatus") proto.RegisterType((*StickinessConfig)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.StickinessConfig") proto.RegisterType((*StringMatch)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.StringMatch") proto.RegisterType((*TCPRoute)(nil), "github.com.argoproj.argo_rollouts.pkg.apis.rollouts.v1alpha1.TCPRoute") @@ -3349,543 +3466,571 @@ func init() { } var fileDescriptor_e0e705f843545fab = []byte{ - // 8574 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x7d, 0x6d, 0x8c, 0x24, 0xd7, - 0x71, 0x98, 0x7a, 0x67, 0x67, 0x77, 0xa6, 0x66, 0xbf, 0xee, 0xdd, 0x1d, 0xb9, 0x5c, 0x92, 0x37, - 0x54, 0xd3, 0x61, 0x48, 0x8b, 0xda, 0x95, 0xf8, 0x91, 0x50, 0xa2, 0xc2, 0x64, 0x66, 0xf7, 0x8e, - 0xb7, 0xc7, 0xdd, 0xbb, 0x65, 0xcd, 0x1e, 0xcf, 0x96, 0x44, 0x5b, 0xbd, 0x33, 0x6f, 0x67, 0xfb, - 0x76, 0xa6, 0x7b, 0xd4, 0xdd, 0xb3, 0x77, 0x4b, 0x11, 0x16, 0x25, 0x81, 0xb2, 0xac, 0x48, 0xb0, - 0x12, 0x5b, 0x08, 0x82, 0x04, 0x81, 0x62, 0x18, 0x70, 0x12, 0xfb, 0x47, 0x60, 0x38, 0x48, 0x7e, - 0x18, 0x88, 0x11, 0xc5, 0x86, 0x02, 0xc4, 0x81, 0xfc, 0x23, 0x91, 0x13, 0xc0, 0xeb, 0x68, 0x9d, - 0x3f, 0x31, 0x12, 0x08, 0x09, 0x14, 0x18, 0xe1, 0x8f, 0x20, 0x78, 0x9f, 0xfd, 0xba, 0xa7, 0x67, - 0xbf, 0xa6, 0xf7, 0x48, 0x27, 0xfe, 0x37, 0xf3, 0xaa, 0x5e, 0x55, 0xf5, 0xfb, 0xac, 0x57, 0xaf, - 0xaa, 0x1e, 0xac, 0xb5, 0xdd, 0x68, 0xa7, 0xbf, 0xb5, 0xd8, 0xf4, 0xbb, 0x4b, 0x4e, 0xd0, 0xf6, - 0x7b, 0x81, 0x7f, 0x97, 0xff, 0xf8, 0x68, 0xe0, 0x77, 0x3a, 0x7e, 0x3f, 0x0a, 0x97, 0x7a, 0xbb, - 0xed, 0x25, 0xa7, 0xe7, 0x86, 0x4b, 0xba, 0x64, 0xef, 0xe3, 0x4e, 0xa7, 0xb7, 0xe3, 0x7c, 0x7c, - 0xa9, 0x4d, 0x3d, 0x1a, 0x38, 0x11, 0x6d, 0x2d, 0xf6, 0x02, 0x3f, 0xf2, 0xc9, 0xa7, 0x62, 0x6a, - 0x8b, 0x8a, 0x1a, 0xff, 0xf1, 0xb3, 0xaa, 0xee, 0x62, 0x6f, 0xb7, 0xbd, 0xc8, 0xa8, 0x2d, 0xea, - 0x12, 0x45, 0x6d, 0xe1, 0xa3, 0x86, 0x2c, 0x6d, 0xbf, 0xed, 0x2f, 0x71, 0xa2, 0x5b, 0xfd, 0x6d, - 0xfe, 0x8f, 0xff, 0xe1, 0xbf, 0x04, 0xb3, 0x85, 0x27, 0x77, 0x5f, 0x0a, 0x17, 0x5d, 0x9f, 0xc9, - 0xb6, 0xb4, 0xe5, 0x44, 0xcd, 0x9d, 0xa5, 0xbd, 0x01, 0x89, 0x16, 0x6c, 0x03, 0xa9, 0xe9, 0x07, - 0x34, 0x0b, 0xe7, 0x85, 0x18, 0xa7, 0xeb, 0x34, 0x77, 0x5c, 0x8f, 0x06, 0xfb, 0xf1, 0x57, 0x77, - 0x69, 0xe4, 0x64, 0xd5, 0x5a, 0x1a, 0x56, 0x2b, 0xe8, 0x7b, 0x91, 0xdb, 0xa5, 0x03, 0x15, 0xfe, - 0xca, 0x71, 0x15, 0xc2, 0xe6, 0x0e, 0xed, 0x3a, 0x03, 0xf5, 0x9e, 0x1f, 0x56, 0xaf, 0x1f, 0xb9, - 0x9d, 0x25, 0xd7, 0x8b, 0xc2, 0x28, 0x48, 0x57, 0xb2, 0x7f, 0x54, 0x80, 0x72, 0x6d, 0xad, 0xde, - 0x88, 0x9c, 0xa8, 0x1f, 0x92, 0xaf, 0x5a, 0x30, 0xd5, 0xf1, 0x9d, 0x56, 0xdd, 0xe9, 0x38, 0x5e, - 0x93, 0x06, 0xf3, 0xd6, 0x13, 0xd6, 0xd3, 0x95, 0xe7, 0xd6, 0x16, 0x47, 0xe9, 0xaf, 0xc5, 0xda, - 0xbd, 0x10, 0x69, 0xe8, 0xf7, 0x83, 0x26, 0x45, 0xba, 0x5d, 0xbf, 0xf4, 0xbd, 0x83, 0xea, 0x87, - 0x0e, 0x0f, 0xaa, 0x53, 0x6b, 0x06, 0x27, 0x4c, 0xf0, 0x25, 0xdf, 0xb6, 0xe0, 0x42, 0xd3, 0xf1, - 0x9c, 0x60, 0x7f, 0xd3, 0x09, 0xda, 0x34, 0x7a, 0x35, 0xf0, 0xfb, 0xbd, 0xf9, 0xb1, 0x73, 0x90, - 0xe6, 0x11, 0x29, 0xcd, 0x85, 0xe5, 0x34, 0x3b, 0x1c, 0x94, 0x80, 0xcb, 0x15, 0x46, 0xce, 0x56, - 0x87, 0x9a, 0x72, 0x15, 0xce, 0x53, 0xae, 0x46, 0x9a, 0x1d, 0x0e, 0x4a, 0x40, 0x9e, 0x81, 0x49, - 0xd7, 0x6b, 0x07, 0x34, 0x0c, 0xe7, 0xc7, 0x9f, 0xb0, 0x9e, 0x2e, 0xd7, 0x67, 0x65, 0xf5, 0xc9, - 0x55, 0x51, 0x8c, 0x0a, 0x6e, 0xff, 0x66, 0x01, 0x2e, 0xd4, 0xd6, 0xea, 0x9b, 0x81, 0xb3, 0xbd, - 0xed, 0x36, 0xd1, 0xef, 0x47, 0xae, 0xd7, 0x36, 0x09, 0x58, 0x47, 0x13, 0x20, 0x2f, 0x42, 0x25, - 0xa4, 0xc1, 0x9e, 0xdb, 0xa4, 0x1b, 0x7e, 0x10, 0xf1, 0x4e, 0x29, 0xd6, 0x2f, 0x4a, 0xf4, 0x4a, - 0x23, 0x06, 0xa1, 0x89, 0xc7, 0xaa, 0x05, 0xbe, 0x1f, 0x49, 0x38, 0x6f, 0xb3, 0x72, 0x5c, 0x0d, - 0x63, 0x10, 0x9a, 0x78, 0x64, 0x05, 0xe6, 0x1c, 0xcf, 0xf3, 0x23, 0x27, 0x72, 0x7d, 0x6f, 0x23, - 0xa0, 0xdb, 0xee, 0x7d, 0xf9, 0x89, 0xf3, 0xb2, 0xee, 0x5c, 0x2d, 0x05, 0xc7, 0x81, 0x1a, 0xe4, - 0x5b, 0x16, 0xcc, 0x85, 0x91, 0xdb, 0xdc, 0x75, 0x3d, 0x1a, 0x86, 0xcb, 0xbe, 0xb7, 0xed, 0xb6, - 0xe7, 0x8b, 0xbc, 0xdb, 0x6e, 0x8e, 0xd6, 0x6d, 0x8d, 0x14, 0xd5, 0xfa, 0x25, 0x26, 0x52, 0xba, - 0x14, 0x07, 0xb8, 0x93, 0x8f, 0x40, 0x59, 0xb6, 0x28, 0x0d, 0xe7, 0x27, 0x9e, 0x28, 0x3c, 0x5d, - 0xae, 0x4f, 0x1f, 0x1e, 0x54, 0xcb, 0xab, 0xaa, 0x10, 0x63, 0xb8, 0xbd, 0x02, 0xf3, 0xb5, 0xee, - 0x96, 0x13, 0x86, 0x4e, 0xcb, 0x0f, 0x52, 0x5d, 0xf7, 0x34, 0x94, 0xba, 0x4e, 0xaf, 0xe7, 0x7a, - 0x6d, 0xd6, 0x77, 0x8c, 0xce, 0xd4, 0xe1, 0x41, 0xb5, 0xb4, 0x2e, 0xcb, 0x50, 0x43, 0xed, 0xff, - 0x38, 0x06, 0x95, 0x9a, 0xe7, 0x74, 0xf6, 0x43, 0x37, 0xc4, 0xbe, 0x47, 0x3e, 0x07, 0x25, 0xb6, - 0x6a, 0xb5, 0x9c, 0xc8, 0x91, 0x33, 0xfd, 0x63, 0x8b, 0x62, 0x11, 0x59, 0x34, 0x17, 0x91, 0xf8, - 0xf3, 0x19, 0xf6, 0xe2, 0xde, 0xc7, 0x17, 0x6f, 0x6d, 0xdd, 0xa5, 0xcd, 0x68, 0x9d, 0x46, 0x4e, - 0x9d, 0xc8, 0x5e, 0x80, 0xb8, 0x0c, 0x35, 0x55, 0xe2, 0xc3, 0x78, 0xd8, 0xa3, 0x4d, 0x39, 0x73, - 0xd7, 0x47, 0x9c, 0x21, 0xb1, 0xe8, 0x8d, 0x1e, 0x6d, 0xd6, 0xa7, 0x24, 0xeb, 0x71, 0xf6, 0x0f, - 0x39, 0x23, 0x72, 0x0f, 0x26, 0x42, 0xbe, 0x96, 0xc9, 0x49, 0x79, 0x2b, 0x3f, 0x96, 0x9c, 0x6c, - 0x7d, 0x46, 0x32, 0x9d, 0x10, 0xff, 0x51, 0xb2, 0xb3, 0xff, 0x93, 0x05, 0x17, 0x0d, 0xec, 0x5a, - 0xd0, 0xee, 0x77, 0xa9, 0x17, 0x91, 0x27, 0x60, 0xdc, 0x73, 0xba, 0x54, 0xce, 0x2a, 0x2d, 0xf2, - 0x4d, 0xa7, 0x4b, 0x91, 0x43, 0xc8, 0x93, 0x50, 0xdc, 0x73, 0x3a, 0x7d, 0xca, 0x1b, 0xa9, 0x5c, - 0x9f, 0x96, 0x28, 0xc5, 0x37, 0x58, 0x21, 0x0a, 0x18, 0x79, 0x1b, 0xca, 0xfc, 0xc7, 0xb5, 0xc0, - 0xef, 0xe6, 0xf4, 0x69, 0x52, 0xc2, 0x37, 0x14, 0x59, 0x31, 0xfc, 0xf4, 0x5f, 0x8c, 0x19, 0xda, - 0x7f, 0x6c, 0xc1, 0xac, 0xf1, 0x71, 0x6b, 0x6e, 0x18, 0x91, 0xcf, 0x0e, 0x0c, 0x9e, 0xc5, 0x93, - 0x0d, 0x1e, 0x56, 0x9b, 0x0f, 0x9d, 0x39, 0xf9, 0xa5, 0x25, 0x55, 0x62, 0x0c, 0x1c, 0x0f, 0x8a, - 0x6e, 0x44, 0xbb, 0xe1, 0xfc, 0xd8, 0x13, 0x85, 0xa7, 0x2b, 0xcf, 0xad, 0xe6, 0xd6, 0x8d, 0x71, - 0xfb, 0xae, 0x32, 0xfa, 0x28, 0xd8, 0xd8, 0xbf, 0x55, 0x48, 0x74, 0xdf, 0xba, 0x92, 0xe3, 0x5d, - 0x0b, 0x26, 0x3a, 0xce, 0x16, 0xed, 0x88, 0xb9, 0x55, 0x79, 0xee, 0xcd, 0xdc, 0x24, 0x51, 0x3c, - 0x16, 0xd7, 0x38, 0xfd, 0xab, 0x5e, 0x14, 0xec, 0xc7, 0xc3, 0x4b, 0x14, 0xa2, 0x64, 0x4e, 0xfe, - 0xae, 0x05, 0x95, 0x78, 0x55, 0x53, 0xcd, 0xb2, 0x95, 0xbf, 0x30, 0xf1, 0x62, 0x2a, 0x25, 0xd2, - 0x4b, 0xb4, 0x01, 0x41, 0x53, 0x96, 0x85, 0x4f, 0x40, 0xc5, 0xf8, 0x04, 0x32, 0x07, 0x85, 0x5d, - 0xba, 0x2f, 0x06, 0x3c, 0xb2, 0x9f, 0xe4, 0x52, 0x62, 0x84, 0xcb, 0x21, 0xfd, 0xc9, 0xb1, 0x97, - 0xac, 0x85, 0x57, 0x60, 0x2e, 0xcd, 0xf0, 0x34, 0xf5, 0xed, 0x7f, 0x5a, 0x4c, 0x0c, 0x4c, 0xb6, - 0x10, 0x10, 0x1f, 0x26, 0xbb, 0x34, 0x0a, 0xdc, 0xa6, 0xea, 0xb2, 0x95, 0xd1, 0x5a, 0x69, 0x9d, - 0x13, 0x8b, 0x37, 0x44, 0xf1, 0x3f, 0x44, 0xc5, 0x85, 0xec, 0xc0, 0xb8, 0x13, 0xb4, 0x55, 0x9f, - 0x5c, 0xcb, 0x67, 0x5a, 0xc6, 0x4b, 0x45, 0x2d, 0x68, 0x87, 0xc8, 0x39, 0x90, 0x25, 0x28, 0x47, - 0x34, 0xe8, 0xba, 0x9e, 0x13, 0x89, 0x1d, 0xb4, 0x54, 0xbf, 0x20, 0xd1, 0xca, 0x9b, 0x0a, 0x80, - 0x31, 0x0e, 0xe9, 0xc0, 0x44, 0x2b, 0xd8, 0xc7, 0xbe, 0x37, 0x3f, 0x9e, 0x47, 0x53, 0xac, 0x70, - 0x5a, 0xf1, 0x20, 0x15, 0xff, 0x51, 0xf2, 0x20, 0xbf, 0x6a, 0xc1, 0xa5, 0x2e, 0x75, 0xc2, 0x7e, - 0x40, 0xd9, 0x27, 0x20, 0x8d, 0xa8, 0xc7, 0x3a, 0x76, 0xbe, 0xc8, 0x99, 0xe3, 0xa8, 0xfd, 0x30, - 0x48, 0xb9, 0xfe, 0x98, 0x14, 0xe5, 0x52, 0x16, 0x14, 0x33, 0xa5, 0x21, 0x6f, 0x43, 0x25, 0x8a, - 0x3a, 0x8d, 0x88, 0xe9, 0xc1, 0xed, 0xfd, 0xf9, 0x09, 0xbe, 0x78, 0x8d, 0xb8, 0xc2, 0x6c, 0x6e, - 0xae, 0x29, 0x82, 0xf5, 0x59, 0x36, 0x5b, 0x8c, 0x02, 0x34, 0xd9, 0xd9, 0xff, 0xa2, 0x08, 0x17, - 0x06, 0xb6, 0x15, 0xf2, 0x02, 0x14, 0x7b, 0x3b, 0x4e, 0xa8, 0xf6, 0x89, 0x2b, 0x6a, 0x91, 0xda, - 0x60, 0x85, 0xef, 0x1d, 0x54, 0xa7, 0x55, 0x15, 0x5e, 0x80, 0x02, 0x99, 0x69, 0x6d, 0x5d, 0x1a, - 0x86, 0x4e, 0x5b, 0x6d, 0x1e, 0xc6, 0x20, 0xe5, 0xc5, 0xa8, 0xe0, 0xe4, 0xe7, 0x2d, 0x98, 0x16, - 0x03, 0x16, 0x69, 0xd8, 0xef, 0x44, 0x6c, 0x83, 0x64, 0x9d, 0x72, 0x23, 0x8f, 0xc9, 0x21, 0x48, - 0xd6, 0x2f, 0x4b, 0xee, 0xd3, 0x66, 0x69, 0x88, 0x49, 0xbe, 0xe4, 0x0e, 0x94, 0xc3, 0xc8, 0x09, - 0x22, 0xda, 0xaa, 0x45, 0x5c, 0x95, 0xab, 0x3c, 0xf7, 0x93, 0x27, 0xdb, 0x39, 0x36, 0xdd, 0x2e, - 0x15, 0xbb, 0x54, 0x43, 0x11, 0xc0, 0x98, 0x16, 0x79, 0x1b, 0x20, 0xe8, 0x7b, 0x8d, 0x7e, 0xb7, - 0xeb, 0x04, 0xfb, 0x52, 0xbb, 0xbb, 0x3e, 0xda, 0xe7, 0xa1, 0xa6, 0x17, 0x2b, 0x3a, 0x71, 0x19, - 0x1a, 0xfc, 0xc8, 0x97, 0x2c, 0x98, 0x16, 0xf3, 0x40, 0x49, 0x30, 0x91, 0xb3, 0x04, 0x17, 0x58, - 0xd3, 0xae, 0x98, 0x2c, 0x30, 0xc9, 0x91, 0xbc, 0x09, 0x95, 0xa6, 0xdf, 0xed, 0x75, 0xa8, 0x68, - 0xdc, 0xc9, 0x53, 0x37, 0x2e, 0x1f, 0xba, 0xcb, 0x31, 0x09, 0x34, 0xe9, 0xd9, 0xff, 0x3e, 0xa9, - 0xe3, 0xa8, 0x21, 0x4d, 0x3e, 0x03, 0x8f, 0x84, 0xfd, 0x66, 0x93, 0x86, 0xe1, 0x76, 0xbf, 0x83, - 0x7d, 0xef, 0xba, 0x1b, 0x46, 0x7e, 0xb0, 0xbf, 0xe6, 0x76, 0xdd, 0x88, 0x0f, 0xe8, 0x62, 0xfd, - 0xf1, 0xc3, 0x83, 0xea, 0x23, 0x8d, 0x61, 0x48, 0x38, 0xbc, 0x3e, 0x71, 0xe0, 0xd1, 0xbe, 0x37, - 0x9c, 0xbc, 0x38, 0x7e, 0x54, 0x0f, 0x0f, 0xaa, 0x8f, 0xde, 0x1e, 0x8e, 0x86, 0x47, 0xd1, 0xb0, - 0xff, 0xd4, 0x62, 0xdb, 0x90, 0xf8, 0xae, 0x4d, 0xda, 0xed, 0x75, 0xd8, 0xd2, 0x79, 0xfe, 0xca, - 0x71, 0x94, 0x50, 0x8e, 0x31, 0x9f, 0xbd, 0x5c, 0xc9, 0x3f, 0x4c, 0x43, 0xb6, 0xff, 0xab, 0x05, - 0x97, 0xd2, 0xc8, 0x0f, 0x40, 0xa1, 0x0b, 0x93, 0x0a, 0xdd, 0xcd, 0x7c, 0xbf, 0x76, 0x88, 0x56, - 0xf7, 0x0b, 0xc6, 0x80, 0x55, 0xa8, 0x48, 0xb7, 0xc9, 0x4b, 0x30, 0x15, 0xc9, 0xbf, 0x37, 0x63, - 0xe5, 0x5c, 0x1b, 0x26, 0x36, 0x0d, 0x18, 0x26, 0x30, 0x59, 0xcd, 0x66, 0xa7, 0x1f, 0x46, 0x34, - 0x68, 0x34, 0xfd, 0x9e, 0x58, 0x76, 0x4b, 0x71, 0xcd, 0x65, 0x03, 0x86, 0x09, 0x4c, 0xfb, 0x6f, - 0x16, 0x07, 0xdb, 0xfd, 0xff, 0x75, 0x7d, 0x25, 0x56, 0x3f, 0x0a, 0xef, 0xa7, 0xfa, 0x31, 0xfe, - 0x81, 0x52, 0x3f, 0xbe, 0x6c, 0x31, 0x2d, 0x4e, 0x0c, 0x80, 0x50, 0xaa, 0x46, 0xaf, 0xe7, 0x3b, - 0x1d, 0x90, 0x6e, 0x9b, 0x8a, 0xa1, 0xe4, 0x85, 0x31, 0x5b, 0xfb, 0x1f, 0x8d, 0xc3, 0x54, 0xcd, - 0x8b, 0xdc, 0xda, 0xf6, 0xb6, 0xeb, 0xb9, 0xd1, 0x3e, 0xf9, 0xc6, 0x18, 0x2c, 0xf5, 0x02, 0xba, - 0x4d, 0x83, 0x80, 0xb6, 0x56, 0xfa, 0x81, 0xeb, 0xb5, 0x1b, 0xcd, 0x1d, 0xda, 0xea, 0x77, 0x5c, - 0xaf, 0xbd, 0xda, 0xf6, 0x7c, 0x5d, 0x7c, 0xf5, 0x3e, 0x6d, 0xf6, 0x79, 0xbb, 0x8a, 0x55, 0xa2, - 0x3b, 0x9a, 0xec, 0x1b, 0xa7, 0x63, 0x5a, 0x7f, 0xfe, 0xf0, 0xa0, 0xba, 0x74, 0xca, 0x4a, 0x78, - 0xda, 0x4f, 0x23, 0x5f, 0x1b, 0x83, 0xc5, 0x80, 0x7e, 0xbe, 0xef, 0x9e, 0xbc, 0x35, 0xc4, 0x32, - 0xde, 0x19, 0x71, 0xbb, 0x3f, 0x15, 0xcf, 0xfa, 0x73, 0x87, 0x07, 0xd5, 0x53, 0xd6, 0xc1, 0x53, - 0x7e, 0x97, 0xbd, 0x01, 0x95, 0x5a, 0xcf, 0x0d, 0xdd, 0xfb, 0xe8, 0xf7, 0x23, 0x7a, 0x02, 0x83, - 0x46, 0x15, 0x8a, 0x41, 0xbf, 0x43, 0xc5, 0x02, 0x53, 0xae, 0x97, 0xd9, 0xb2, 0x8c, 0xac, 0x00, - 0x45, 0xb9, 0xfd, 0x65, 0xb6, 0x05, 0x71, 0x92, 0x29, 0x53, 0xd6, 0x5d, 0x28, 0x06, 0x8c, 0x89, - 0x1c, 0x59, 0xa3, 0x9e, 0xfa, 0x63, 0xa9, 0xa5, 0x10, 0xec, 0x27, 0x0a, 0x16, 0xf6, 0x77, 0xc7, - 0xe0, 0x72, 0xad, 0xd7, 0x5b, 0xa7, 0xe1, 0x4e, 0x4a, 0x8a, 0x5f, 0xb4, 0x60, 0x66, 0xcf, 0x0d, - 0xa2, 0xbe, 0xd3, 0x51, 0xd6, 0x4a, 0x21, 0x4f, 0x63, 0x54, 0x79, 0x38, 0xb7, 0x37, 0x12, 0xa4, - 0xeb, 0xe4, 0xf0, 0xa0, 0x3a, 0x93, 0x2c, 0xc3, 0x14, 0x7b, 0xf2, 0x77, 0x2c, 0x98, 0x93, 0x45, - 0x37, 0xfd, 0x16, 0x35, 0xad, 0xe1, 0xb7, 0xf3, 0x94, 0x49, 0x13, 0x17, 0x56, 0xcc, 0x74, 0x29, - 0x0e, 0x08, 0x61, 0xff, 0xf7, 0x31, 0x78, 0x78, 0x08, 0x0d, 0xf2, 0x6b, 0x16, 0x5c, 0x12, 0x26, - 0x74, 0x03, 0x84, 0x74, 0x5b, 0xb6, 0xe6, 0x4f, 0xe7, 0x2d, 0x39, 0xb2, 0x29, 0x4e, 0xbd, 0x26, - 0xad, 0xcf, 0xb3, 0x25, 0x79, 0x39, 0x83, 0x35, 0x66, 0x0a, 0xc4, 0x25, 0x15, 0x46, 0xf5, 0x94, - 0xa4, 0x63, 0x0f, 0x44, 0xd2, 0x46, 0x06, 0x6b, 0xcc, 0x14, 0xc8, 0xfe, 0xeb, 0xf0, 0xe8, 0x11, - 0xe4, 0x8e, 0x9f, 0x9c, 0xf6, 0x9b, 0x7a, 0xd4, 0x27, 0xc7, 0xdc, 0x09, 0xe6, 0xb5, 0x0d, 0x13, - 0x7c, 0xea, 0xa8, 0x89, 0x0d, 0x6c, 0x0f, 0xe6, 0x73, 0x2a, 0x44, 0x09, 0xb1, 0xbf, 0x6b, 0x41, - 0xe9, 0x14, 0xb6, 0xcf, 0x6a, 0xd2, 0xf6, 0x59, 0x1e, 0xb0, 0x7b, 0x46, 0x83, 0x76, 0xcf, 0x57, - 0x47, 0xeb, 0x8d, 0x93, 0xd8, 0x3b, 0x7f, 0x64, 0xc1, 0x85, 0x01, 0xfb, 0x28, 0xd9, 0x81, 0x4b, - 0x3d, 0xbf, 0xa5, 0xb6, 0xd3, 0xeb, 0x4e, 0xb8, 0xc3, 0x61, 0xf2, 0xf3, 0x5e, 0x60, 0x3d, 0xb9, - 0x91, 0x01, 0x7f, 0xef, 0xa0, 0x3a, 0xaf, 0x89, 0xa4, 0x10, 0x30, 0x93, 0x22, 0xe9, 0x41, 0x69, - 0xdb, 0xa5, 0x9d, 0x56, 0x3c, 0x04, 0x47, 0xd4, 0xd2, 0xae, 0x49, 0x6a, 0xe2, 0x6a, 0x40, 0xfd, - 0x43, 0xcd, 0xc5, 0xfe, 0xb1, 0x05, 0x33, 0xb5, 0x7e, 0xb4, 0xc3, 0x74, 0x94, 0x26, 0xb7, 0xc6, - 0x11, 0x0f, 0x8a, 0xa1, 0xdb, 0xde, 0x7b, 0x21, 0x9f, 0xc5, 0xb8, 0xc1, 0x48, 0xc9, 0x2b, 0x12, - 0xad, 0xac, 0xf3, 0x42, 0x14, 0x6c, 0x48, 0x00, 0x13, 0xbe, 0xd3, 0x8f, 0x76, 0x9e, 0x93, 0x9f, - 0x3c, 0xa2, 0x65, 0xe2, 0x16, 0xfb, 0x9c, 0xe7, 0x24, 0x47, 0xad, 0x32, 0x8a, 0x52, 0x94, 0x9c, - 0xec, 0x2f, 0xc2, 0x4c, 0xf2, 0xde, 0xed, 0x04, 0x63, 0xf6, 0x71, 0x28, 0x38, 0x81, 0x27, 0x47, - 0x6c, 0x45, 0x22, 0x14, 0x6a, 0x78, 0x13, 0x59, 0x39, 0x79, 0x16, 0x4a, 0xdb, 0xfd, 0x4e, 0x87, - 0x9f, 0x2b, 0xc4, 0x25, 0x97, 0x3e, 0x16, 0x5d, 0x93, 0xe5, 0xa8, 0x31, 0xec, 0xff, 0x3d, 0x0e, - 0xb3, 0xf5, 0x4e, 0x9f, 0xbe, 0x1a, 0x50, 0xaa, 0x6c, 0x41, 0x35, 0x98, 0xed, 0x05, 0x74, 0xcf, - 0xa5, 0xf7, 0x1a, 0xb4, 0x43, 0x9b, 0x91, 0x1f, 0x48, 0x69, 0x1e, 0x96, 0x84, 0x66, 0x37, 0x92, - 0x60, 0x4c, 0xe3, 0x93, 0x57, 0x60, 0xc6, 0x69, 0x46, 0xee, 0x1e, 0xd5, 0x14, 0x84, 0xb8, 0x0f, - 0x49, 0x0a, 0x33, 0xb5, 0x04, 0x14, 0x53, 0xd8, 0xe4, 0xb3, 0x30, 0x1f, 0x36, 0x9d, 0x0e, 0xbd, - 0xdd, 0x93, 0xac, 0x96, 0x77, 0x68, 0x73, 0x77, 0xc3, 0x77, 0xbd, 0x48, 0xda, 0x1d, 0x9f, 0x90, - 0x94, 0xe6, 0x1b, 0x43, 0xf0, 0x70, 0x28, 0x05, 0xf2, 0x2f, 0x2d, 0x78, 0xbc, 0x17, 0xd0, 0x8d, - 0xc0, 0xef, 0xfa, 0x6c, 0xa8, 0x0d, 0x98, 0xc3, 0xa4, 0x59, 0xe8, 0x8d, 0x11, 0x75, 0x29, 0x51, - 0x32, 0x78, 0x87, 0xf3, 0xe1, 0xc3, 0x83, 0xea, 0xe3, 0x1b, 0x47, 0x09, 0x80, 0x47, 0xcb, 0x47, - 0xfe, 0x95, 0x05, 0x57, 0x7a, 0x7e, 0x18, 0x1d, 0xf1, 0x09, 0xc5, 0x73, 0xfd, 0x04, 0xfb, 0xf0, - 0xa0, 0x7a, 0x65, 0xe3, 0x48, 0x09, 0xf0, 0x18, 0x09, 0xed, 0xc3, 0x0a, 0x5c, 0x30, 0xc6, 0x9e, - 0x34, 0xe6, 0xbc, 0x0c, 0xd3, 0x6a, 0x30, 0xc4, 0xba, 0x4f, 0x39, 0xb6, 0xed, 0xd5, 0x4c, 0x20, - 0x26, 0x71, 0xd9, 0xb8, 0xd3, 0x43, 0x51, 0xd4, 0x4e, 0x8d, 0xbb, 0x8d, 0x04, 0x14, 0x53, 0xd8, - 0x64, 0x15, 0x2e, 0xca, 0x12, 0xa4, 0xbd, 0x8e, 0xdb, 0x74, 0x96, 0xfd, 0xbe, 0x1c, 0x72, 0xc5, - 0xfa, 0xc3, 0x87, 0x07, 0xd5, 0x8b, 0x1b, 0x83, 0x60, 0xcc, 0xaa, 0x43, 0xd6, 0xe0, 0x92, 0xd3, - 0x8f, 0x7c, 0xfd, 0xfd, 0x57, 0x3d, 0xb6, 0x9d, 0xb6, 0xf8, 0xd0, 0x2a, 0x89, 0x7d, 0xb7, 0x96, - 0x01, 0xc7, 0xcc, 0x5a, 0x64, 0x23, 0x45, 0xad, 0x41, 0x9b, 0xbe, 0xd7, 0x12, 0xbd, 0x5c, 0x8c, - 0x8f, 0x81, 0xb5, 0x0c, 0x1c, 0xcc, 0xac, 0x49, 0x3a, 0x30, 0xd3, 0x75, 0xee, 0xdf, 0xf6, 0x9c, - 0x3d, 0xc7, 0xed, 0x30, 0x26, 0xd2, 0x5e, 0x38, 0xdc, 0xca, 0xd4, 0x8f, 0xdc, 0xce, 0xa2, 0xf0, - 0xe3, 0x58, 0x5c, 0xf5, 0xa2, 0x5b, 0x41, 0x23, 0x62, 0x9a, 0xba, 0xd0, 0x20, 0xd7, 0x13, 0xb4, - 0x30, 0x45, 0x9b, 0xdc, 0x82, 0xcb, 0x7c, 0x3a, 0xae, 0xf8, 0xf7, 0xbc, 0x15, 0xda, 0x71, 0xf6, - 0xd5, 0x07, 0x4c, 0xf2, 0x0f, 0x78, 0xe4, 0xf0, 0xa0, 0x7a, 0xb9, 0x91, 0x85, 0x80, 0xd9, 0xf5, - 0x88, 0x03, 0x8f, 0x26, 0x01, 0x48, 0xf7, 0xdc, 0xd0, 0xf5, 0x3d, 0x61, 0x96, 0x2b, 0xc5, 0x66, - 0xb9, 0xc6, 0x70, 0x34, 0x3c, 0x8a, 0x06, 0xf9, 0x7b, 0x16, 0x5c, 0xca, 0x9a, 0x86, 0xf3, 0xe5, - 0x3c, 0x6e, 0x93, 0x53, 0x53, 0x4b, 0x8c, 0x88, 0xcc, 0x45, 0x21, 0x53, 0x08, 0xf2, 0x8e, 0x05, - 0x53, 0x8e, 0x71, 0x82, 0x9e, 0x87, 0x3c, 0x76, 0x2d, 0xf3, 0x4c, 0x5e, 0x9f, 0x3b, 0x3c, 0xa8, - 0x26, 0x4e, 0xe9, 0x98, 0xe0, 0x48, 0xfe, 0x81, 0x05, 0x97, 0x33, 0xe7, 0xf8, 0x7c, 0xe5, 0x3c, - 0x5a, 0x88, 0x0f, 0x92, 0xec, 0x35, 0x27, 0x5b, 0x0c, 0xf2, 0x2d, 0x4b, 0x6f, 0x65, 0xea, 0x82, - 0x71, 0x7e, 0x8a, 0x8b, 0x36, 0xa2, 0xc1, 0xc3, 0x50, 0xa3, 0x14, 0xe1, 0xfa, 0x45, 0x63, 0x67, - 0x54, 0x85, 0x98, 0x66, 0x4f, 0xbe, 0x69, 0xa9, 0xad, 0x51, 0x4b, 0x34, 0x7d, 0x5e, 0x12, 0x91, - 0x78, 0xa7, 0xd5, 0x02, 0xa5, 0x98, 0x93, 0x9f, 0x81, 0x05, 0x67, 0xcb, 0x0f, 0xa2, 0xcc, 0xc9, - 0x37, 0x3f, 0xc3, 0xa7, 0xd1, 0x95, 0xc3, 0x83, 0xea, 0x42, 0x6d, 0x28, 0x16, 0x1e, 0x41, 0xc1, - 0xfe, 0x8d, 0x22, 0x4c, 0x89, 0x93, 0x90, 0xdc, 0xba, 0x7e, 0xdb, 0x82, 0xc7, 0x9a, 0xfd, 0x20, - 0xa0, 0x5e, 0xd4, 0x88, 0x68, 0x6f, 0x70, 0xe3, 0xb2, 0xce, 0x75, 0xe3, 0x7a, 0xe2, 0xf0, 0xa0, - 0xfa, 0xd8, 0xf2, 0x11, 0xfc, 0xf1, 0x48, 0xe9, 0xc8, 0xbf, 0xb3, 0xc0, 0x96, 0x08, 0x75, 0xa7, - 0xb9, 0xdb, 0x0e, 0xfc, 0xbe, 0xd7, 0x1a, 0xfc, 0x88, 0xb1, 0x73, 0xfd, 0x88, 0xa7, 0x0e, 0x0f, - 0xaa, 0xf6, 0xf2, 0xb1, 0x52, 0xe0, 0x09, 0x24, 0x25, 0xaf, 0xc2, 0x05, 0x89, 0x75, 0xf5, 0x7e, - 0x8f, 0x06, 0x2e, 0x3b, 0x73, 0x48, 0xc5, 0x31, 0xf6, 0x4d, 0x4b, 0x23, 0xe0, 0x60, 0x1d, 0x12, - 0xc2, 0xe4, 0x3d, 0xea, 0xb6, 0x77, 0x22, 0xa5, 0x3e, 0x8d, 0xe8, 0x90, 0x26, 0xad, 0x22, 0x77, - 0x04, 0xcd, 0x7a, 0xe5, 0xf0, 0xa0, 0x3a, 0x29, 0xff, 0xa0, 0xe2, 0x44, 0x6e, 0xc2, 0x8c, 0x38, - 0xa7, 0x6e, 0xb8, 0x5e, 0x7b, 0xc3, 0xf7, 0x84, 0x57, 0x55, 0xb9, 0xfe, 0x94, 0xda, 0xf0, 0x1b, - 0x09, 0xe8, 0x7b, 0x07, 0xd5, 0x29, 0xf5, 0x7b, 0x73, 0xbf, 0x47, 0x31, 0x55, 0xdb, 0xfe, 0xbd, - 0x09, 0x00, 0x35, 0x5c, 0x69, 0x8f, 0x7c, 0x04, 0xca, 0x21, 0x8d, 0x04, 0x57, 0x79, 0x93, 0x24, - 0xee, 0xff, 0x54, 0x21, 0xc6, 0x70, 0xb2, 0x0b, 0xc5, 0x9e, 0xd3, 0x0f, 0x69, 0x3e, 0xe7, 0x07, - 0xd9, 0xf9, 0x1b, 0x8c, 0xa2, 0x38, 0x98, 0xf2, 0x9f, 0x28, 0x78, 0x90, 0xaf, 0x58, 0x00, 0x34, - 0xd9, 0x61, 0x23, 0x1b, 0x88, 0x24, 0xcb, 0xb8, 0x4f, 0x59, 0x1b, 0xd4, 0x67, 0x0e, 0x0f, 0xaa, - 0x60, 0x74, 0xbd, 0xc1, 0x96, 0xdc, 0x83, 0x92, 0xa3, 0xd6, 0xfc, 0xf1, 0xf3, 0x58, 0xf3, 0xf9, - 0x79, 0x51, 0x0f, 0x5a, 0xcd, 0x8c, 0x7c, 0xcd, 0x82, 0x99, 0x90, 0x46, 0xb2, 0xab, 0xd8, 0xca, - 0x23, 0x15, 0xde, 0x11, 0x07, 0x5d, 0x23, 0x41, 0x53, 0xac, 0xa0, 0xc9, 0x32, 0x4c, 0xf1, 0x55, - 0xa2, 0x5c, 0xa7, 0x4e, 0x8b, 0x06, 0xdc, 0x1c, 0x21, 0x35, 0xa9, 0xd1, 0x45, 0x31, 0x68, 0x6a, - 0x51, 0x8c, 0x32, 0x4c, 0xf1, 0x55, 0xa2, 0xac, 0xbb, 0x41, 0xe0, 0x4b, 0x51, 0x4a, 0x39, 0x89, - 0x62, 0xd0, 0xd4, 0xa2, 0x18, 0x65, 0x98, 0xe2, 0x6b, 0x7f, 0x67, 0x1a, 0x66, 0xd4, 0x44, 0x8a, - 0x35, 0x7b, 0x61, 0xfd, 0x1a, 0xa2, 0xd9, 0x2f, 0x9b, 0x40, 0x4c, 0xe2, 0xb2, 0xca, 0x62, 0xaa, - 0x26, 0x15, 0x7b, 0x5d, 0xb9, 0x61, 0x02, 0x31, 0x89, 0x4b, 0xba, 0x50, 0x0c, 0x23, 0xda, 0x53, - 0x3e, 0x07, 0x23, 0x5e, 0x89, 0xc7, 0xeb, 0x83, 0x61, 0x49, 0x60, 0xe4, 0x51, 0x70, 0xe1, 0x06, - 0xdc, 0x28, 0x61, 0xd3, 0x95, 0x93, 0x23, 0x9f, 0xf9, 0x99, 0x34, 0x17, 0x8b, 0xde, 0x48, 0x96, - 0x61, 0x8a, 0x7d, 0x86, 0xb2, 0x5f, 0x3c, 0x47, 0x65, 0xff, 0xd3, 0x50, 0xea, 0x3a, 0xf7, 0x1b, - 0xfd, 0xa0, 0x7d, 0xf6, 0x43, 0x85, 0xf4, 0x21, 0x15, 0x54, 0x50, 0xd3, 0x23, 0x5f, 0xb2, 0x8c, - 0x25, 0x47, 0x38, 0x18, 0xdc, 0xc9, 0x77, 0xc9, 0xd1, 0x7b, 0xe5, 0xd0, 0xc5, 0x67, 0x40, 0xf5, - 0x2e, 0x3d, 0x70, 0xd5, 0x9b, 0xa9, 0x91, 0x62, 0x82, 0x68, 0x35, 0xb2, 0x7c, 0xae, 0x6a, 0xe4, - 0x72, 0x82, 0x19, 0xa6, 0x98, 0x73, 0x79, 0xc4, 0x9c, 0xd3, 0xf2, 0xc0, 0xb9, 0xca, 0xd3, 0x48, - 0x30, 0xc3, 0x14, 0xf3, 0xe1, 0xe7, 0xcd, 0xca, 0xf9, 0x9c, 0x37, 0xa7, 0x72, 0x38, 0x6f, 0x1e, - 0xad, 0x8a, 0x4f, 0x8f, 0xaa, 0x8a, 0x93, 0x1b, 0x40, 0x5a, 0xfb, 0x9e, 0xd3, 0x75, 0x9b, 0x72, - 0xb1, 0xe4, 0xdb, 0xe6, 0x0c, 0xb7, 0x47, 0x2c, 0xc8, 0x85, 0x8c, 0xac, 0x0c, 0x60, 0x60, 0x46, - 0x2d, 0x12, 0x41, 0xa9, 0xa7, 0x34, 0xae, 0xd9, 0x3c, 0x46, 0xbf, 0xd2, 0xc0, 0x84, 0xdf, 0x08, - 0x9b, 0x78, 0xaa, 0x04, 0x35, 0x27, 0xb2, 0x06, 0x97, 0xba, 0xae, 0xb7, 0xe1, 0xb7, 0xc2, 0x0d, - 0x1a, 0x48, 0x6b, 0x4b, 0x83, 0x46, 0xf3, 0x73, 0xbc, 0x6d, 0xf8, 0x09, 0x7a, 0x3d, 0x03, 0x8e, - 0x99, 0xb5, 0xec, 0xff, 0x65, 0xc1, 0xdc, 0x72, 0xc7, 0xef, 0xb7, 0xee, 0x38, 0x51, 0x73, 0x47, - 0xb8, 0x29, 0x90, 0x57, 0xa0, 0xe4, 0x7a, 0x11, 0x0d, 0xf6, 0x9c, 0x8e, 0xdc, 0x9f, 0x6c, 0x65, - 0x3e, 0x5d, 0x95, 0xe5, 0xef, 0x1d, 0x54, 0x67, 0x56, 0xfa, 0x01, 0xb7, 0x52, 0x8b, 0xd5, 0x0a, - 0x75, 0x1d, 0xf2, 0x1d, 0x0b, 0x2e, 0x08, 0x47, 0x87, 0x15, 0x27, 0x72, 0x5e, 0xef, 0xd3, 0xc0, - 0xa5, 0xca, 0xd5, 0x61, 0xc4, 0x85, 0x2a, 0x2d, 0xab, 0x62, 0xb0, 0x1f, 0x2b, 0xea, 0xeb, 0x69, - 0xce, 0x38, 0x28, 0x8c, 0xfd, 0x4b, 0x05, 0x78, 0x64, 0x28, 0x2d, 0xb2, 0x00, 0x63, 0x6e, 0x4b, - 0x7e, 0x3a, 0x48, 0xba, 0x63, 0xab, 0x2d, 0x1c, 0x73, 0x5b, 0x64, 0x91, 0xeb, 0x9c, 0x01, 0x0d, - 0x43, 0x75, 0xe1, 0x5c, 0xd6, 0xea, 0xa1, 0x2c, 0x45, 0x03, 0x83, 0x54, 0xa1, 0xc8, 0xfd, 0x87, - 0xe5, 0x79, 0x82, 0x6b, 0xb1, 0xdc, 0x55, 0x17, 0x45, 0x39, 0xf9, 0xb2, 0x05, 0x20, 0x04, 0x64, - 0xa7, 0x11, 0xb9, 0x4b, 0x62, 0xbe, 0xcd, 0xc4, 0x28, 0x0b, 0x29, 0xe3, 0xff, 0x68, 0x70, 0x25, - 0x9b, 0x30, 0xc1, 0x14, 0x5a, 0xbf, 0x75, 0xe6, 0x4d, 0x91, 0xdf, 0x44, 0x6d, 0x70, 0x1a, 0x28, - 0x69, 0xb1, 0xb6, 0x0a, 0x68, 0xd4, 0x0f, 0x3c, 0xd6, 0xb4, 0x7c, 0x1b, 0x2c, 0x09, 0x29, 0x50, - 0x97, 0xa2, 0x81, 0x61, 0xff, 0xf3, 0x31, 0xb8, 0x94, 0x25, 0x3a, 0xdb, 0x6d, 0x26, 0x84, 0xb4, - 0xf2, 0x68, 0xfc, 0x53, 0xf9, 0xb7, 0x8f, 0xf4, 0xd9, 0xd1, 0xd7, 0x14, 0xd2, 0x81, 0x52, 0xf2, - 0x25, 0x3f, 0xa5, 0x5b, 0x68, 0xec, 0x8c, 0x2d, 0xa4, 0x29, 0xa7, 0x5a, 0xe9, 0x09, 0x18, 0x0f, - 0x59, 0xcf, 0x17, 0x92, 0xd7, 0x1d, 0xbc, 0x8f, 0x38, 0x84, 0x61, 0xf4, 0x3d, 0x37, 0x92, 0x41, - 0x37, 0x1a, 0xe3, 0xb6, 0xe7, 0x46, 0xc8, 0x21, 0xf6, 0xb7, 0xc7, 0x60, 0x61, 0xf8, 0x47, 0x91, - 0x6f, 0x5b, 0x00, 0x2d, 0x76, 0x5c, 0x09, 0xb9, 0xe7, 0xba, 0xf0, 0x71, 0x72, 0xce, 0xab, 0x0d, - 0x57, 0x14, 0xa7, 0xd8, 0xf9, 0x4e, 0x17, 0x85, 0x68, 0x08, 0x42, 0x9e, 0x53, 0x43, 0x9f, 0x5f, - 0xd5, 0x88, 0xc9, 0xa4, 0xeb, 0xac, 0x6b, 0x08, 0x1a, 0x58, 0xec, 0x3c, 0xea, 0x39, 0x5d, 0x1a, - 0xf6, 0x1c, 0x1d, 0xc2, 0xc4, 0xcf, 0xa3, 0x37, 0x55, 0x21, 0xc6, 0x70, 0xbb, 0x03, 0x4f, 0x9e, - 0x40, 0xce, 0x9c, 0x22, 0x44, 0xec, 0xff, 0x61, 0xc1, 0xc3, 0xd2, 0xfd, 0xec, 0xff, 0x1b, 0x5f, - 0xc6, 0x3f, 0xb3, 0xe0, 0xd1, 0x21, 0xdf, 0xfc, 0x00, 0x5c, 0x1a, 0xdf, 0x4a, 0xba, 0x34, 0xde, - 0x1e, 0x75, 0x48, 0x67, 0x7e, 0xc7, 0x10, 0xcf, 0xc6, 0x7f, 0x53, 0x80, 0x69, 0xb6, 0x6c, 0xb5, - 0xfc, 0x76, 0x4e, 0x1b, 0xe7, 0x93, 0x50, 0xfc, 0x3c, 0xdb, 0x80, 0xd2, 0x83, 0x8c, 0xef, 0x4a, - 0x28, 0x60, 0xe4, 0x2b, 0x16, 0x4c, 0x7e, 0x5e, 0xee, 0xa9, 0xe2, 0x2c, 0x37, 0xe2, 0x62, 0x98, - 0xf8, 0x86, 0x45, 0xb9, 0x43, 0x8a, 0xc0, 0x13, 0xed, 0xc0, 0xa8, 0xb6, 0x52, 0xc5, 0x99, 0x3c, - 0x03, 0x93, 0xdb, 0x7e, 0xd0, 0xed, 0x77, 0x9c, 0x74, 0xb4, 0xe3, 0x35, 0x51, 0x8c, 0x0a, 0xce, - 0x26, 0xb9, 0xd3, 0x73, 0xdf, 0xa0, 0x41, 0x28, 0xe2, 0x10, 0x12, 0x93, 0xbc, 0xa6, 0x21, 0x68, - 0x60, 0xf1, 0x3a, 0xed, 0x76, 0x40, 0xdb, 0x4e, 0xe4, 0x07, 0x7c, 0xe7, 0x30, 0xeb, 0x68, 0x08, - 0x1a, 0x58, 0x0b, 0x9f, 0x84, 0x29, 0x53, 0xf8, 0x53, 0x05, 0xb1, 0x7c, 0x0a, 0xa4, 0x27, 0x63, - 0x6a, 0x49, 0xb2, 0x4e, 0xb2, 0x24, 0xd9, 0xff, 0x61, 0x0c, 0x0c, 0xeb, 0xd0, 0x03, 0x98, 0xea, - 0x5e, 0x62, 0xaa, 0x8f, 0x68, 0xd9, 0x30, 0x6c, 0x5d, 0xc3, 0x42, 0xfa, 0xf6, 0x52, 0x21, 0x7d, - 0x37, 0x73, 0xe3, 0x78, 0x74, 0x44, 0xdf, 0x0f, 0x2c, 0x78, 0x34, 0x46, 0x1e, 0x34, 0xdc, 0x1e, - 0xbf, 0x6e, 0xbf, 0x08, 0x15, 0x27, 0xae, 0x26, 0x27, 0x96, 0x11, 0x4f, 0xa5, 0x41, 0x68, 0xe2, - 0xc5, 0xb1, 0x20, 0x85, 0x33, 0xc6, 0x82, 0x8c, 0x1f, 0x1d, 0x0b, 0x62, 0xff, 0x78, 0x0c, 0x1e, - 0x1f, 0xfc, 0x32, 0xd3, 0x41, 0xfa, 0xf8, 0x6f, 0x4b, 0xbb, 0x50, 0x8f, 0x9d, 0xd9, 0x85, 0xba, - 0x70, 0x52, 0x17, 0x6a, 0xed, 0xb8, 0x3c, 0x7e, 0xee, 0x8e, 0xcb, 0x0d, 0xb8, 0xac, 0xbc, 0x24, - 0xaf, 0xf9, 0x81, 0x0c, 0x88, 0x50, 0x2b, 0x48, 0xa9, 0xfe, 0xb8, 0xac, 0x72, 0x19, 0xb3, 0x90, - 0x30, 0xbb, 0xae, 0xfd, 0x83, 0x02, 0x5c, 0x8c, 0x9b, 0x7d, 0xd9, 0xf7, 0x5a, 0x2e, 0x77, 0xb4, - 0x79, 0x19, 0xc6, 0xa3, 0xfd, 0x9e, 0x6a, 0xec, 0xbf, 0xac, 0xc4, 0xd9, 0xdc, 0xef, 0xb1, 0xde, - 0x7e, 0x38, 0xa3, 0x0a, 0x37, 0x9d, 0xf3, 0x4a, 0x64, 0x4d, 0xcf, 0x0e, 0xd1, 0x03, 0x2f, 0x24, - 0x47, 0xf3, 0x7b, 0x07, 0xd5, 0x8c, 0xd4, 0x06, 0x8b, 0x9a, 0x52, 0x72, 0xcc, 0x93, 0xbb, 0x30, - 0xd3, 0x71, 0xc2, 0xe8, 0x76, 0xaf, 0xe5, 0x44, 0x74, 0xd3, 0x95, 0x2e, 0x2c, 0xa7, 0x8b, 0x21, - 0xd1, 0x77, 0xfd, 0x6b, 0x09, 0x4a, 0x98, 0xa2, 0x4c, 0xf6, 0x80, 0xb0, 0x92, 0xcd, 0xc0, 0xf1, - 0x42, 0xf1, 0x55, 0x8c, 0xdf, 0xe9, 0x03, 0x82, 0xf4, 0xd1, 0x79, 0x6d, 0x80, 0x1a, 0x66, 0x70, - 0x20, 0x4f, 0xc1, 0x44, 0x40, 0x9d, 0x50, 0x6f, 0x07, 0x7a, 0xfe, 0x23, 0x2f, 0x45, 0x09, 0x35, - 0x27, 0xd4, 0xc4, 0x31, 0x13, 0xea, 0x8f, 0x2c, 0x98, 0x89, 0xbb, 0xe9, 0x01, 0xa8, 0x1e, 0xdd, - 0xa4, 0xea, 0x71, 0x3d, 0xaf, 0x25, 0x71, 0x88, 0xb6, 0xf1, 0xa7, 0x93, 0xe6, 0xf7, 0xf1, 0xa8, - 0x85, 0x2f, 0x98, 0x4e, 0xec, 0x56, 0x1e, 0xa1, 0x64, 0x09, 0x6d, 0xef, 0x48, 0xef, 0x75, 0xa6, - 0xeb, 0xb4, 0xa4, 0x1e, 0x23, 0x87, 0xbd, 0xd6, 0x75, 0x94, 0x7e, 0x93, 0xa5, 0xeb, 0xa8, 0x3a, - 0xe4, 0x36, 0x3c, 0xdc, 0x0b, 0x7c, 0x1e, 0x5c, 0xbf, 0x42, 0x9d, 0x56, 0xc7, 0xf5, 0xa8, 0x32, - 0xf3, 0x08, 0x57, 0x93, 0x47, 0x0f, 0x0f, 0xaa, 0x0f, 0x6f, 0x64, 0xa3, 0xe0, 0xb0, 0xba, 0xc9, - 0xf0, 0xcc, 0xf1, 0x13, 0x84, 0x67, 0xfe, 0x82, 0x36, 0xa6, 0xea, 0x48, 0x80, 0xcf, 0xe4, 0xd5, - 0x95, 0x59, 0x31, 0x01, 0x7a, 0x48, 0xd5, 0x24, 0x53, 0xd4, 0xec, 0x87, 0x5b, 0xec, 0x26, 0xce, - 0x68, 0xb1, 0x8b, 0x83, 0x3f, 0x26, 0xdf, 0xcf, 0xe0, 0x8f, 0xd2, 0x07, 0x2a, 0xf8, 0xe3, 0x3b, - 0x16, 0x5c, 0x74, 0x06, 0xc3, 0xae, 0xf3, 0x31, 0x1e, 0x67, 0xc4, 0x73, 0xd7, 0x1f, 0x95, 0x42, - 0x66, 0x45, 0xb7, 0x63, 0x96, 0x28, 0xf6, 0xbb, 0x45, 0x98, 0x4b, 0x2b, 0x49, 0xe7, 0x1f, 0x9f, - 0xfa, 0xb7, 0x2d, 0x98, 0x53, 0x13, 0x5c, 0xf0, 0xd4, 0x47, 0x8c, 0xb5, 0x9c, 0xd6, 0x15, 0xa1, - 0xee, 0xe9, 0xb4, 0x21, 0x9b, 0x29, 0x6e, 0x38, 0xc0, 0x9f, 0xbc, 0x09, 0x15, 0x7d, 0xab, 0x72, - 0xa6, 0x60, 0x55, 0x1e, 0x4f, 0x59, 0x8b, 0x49, 0xa0, 0x49, 0x8f, 0xbc, 0x6b, 0x01, 0x34, 0xd5, - 0x4e, 0x9c, 0x53, 0x28, 0x50, 0x86, 0xb6, 0x10, 0xeb, 0xf3, 0xba, 0x28, 0x44, 0x83, 0x31, 0xf9, - 0x25, 0x7e, 0x9f, 0xa2, 0x47, 0x82, 0x48, 0x47, 0x32, 0xb2, 0xdb, 0xfb, 0x11, 0xba, 0x73, 0xac, - 0xed, 0x19, 0xa0, 0x10, 0x13, 0x42, 0xd8, 0x2f, 0x83, 0x76, 0x54, 0x66, 0x2b, 0x2b, 0x77, 0x55, - 0xde, 0x70, 0xa2, 0x1d, 0x39, 0x04, 0xf5, 0xca, 0x7a, 0x4d, 0x01, 0x30, 0xc6, 0xb1, 0x3f, 0x07, - 0x33, 0xaf, 0x06, 0x4e, 0x6f, 0xc7, 0xe5, 0xf7, 0x16, 0xec, 0x7c, 0xfc, 0x0c, 0x4c, 0x3a, 0xad, - 0x56, 0x56, 0x86, 0x9b, 0x9a, 0x28, 0x46, 0x05, 0x3f, 0xd1, 0x51, 0xd8, 0xfe, 0x3d, 0x0b, 0x48, - 0x7c, 0xf7, 0xeb, 0x7a, 0xed, 0x75, 0x27, 0x6a, 0xee, 0xb0, 0x23, 0xdc, 0x0e, 0x2f, 0xcd, 0x3a, - 0xc2, 0x5d, 0xd7, 0x10, 0x34, 0xb0, 0xc8, 0xdb, 0x50, 0x11, 0xff, 0xde, 0xd0, 0x07, 0xc4, 0xd1, - 0xfd, 0xad, 0xf9, 0x9e, 0xc7, 0x65, 0x12, 0xa3, 0xf0, 0x7a, 0xcc, 0x01, 0x4d, 0x76, 0xac, 0xa9, - 0x56, 0xbd, 0xed, 0x4e, 0xff, 0x7e, 0x6b, 0x2b, 0x6e, 0xaa, 0x5e, 0xe0, 0x6f, 0xbb, 0x1d, 0x9a, - 0x6e, 0xaa, 0x0d, 0x51, 0x8c, 0x0a, 0x7e, 0xb2, 0xa6, 0xfa, 0xd7, 0x16, 0x5c, 0x5a, 0x0d, 0x23, - 0xd7, 0x5f, 0xa1, 0x61, 0xc4, 0x76, 0x3e, 0xb6, 0x3e, 0xf6, 0x3b, 0x27, 0x89, 0x39, 0x58, 0x81, - 0x39, 0x79, 0x0f, 0xdd, 0xdf, 0x0a, 0x69, 0x64, 0x1c, 0x35, 0xf4, 0x3c, 0x5e, 0x4e, 0xc1, 0x71, - 0xa0, 0x06, 0xa3, 0x22, 0x2f, 0xa4, 0x63, 0x2a, 0x85, 0x24, 0x95, 0x46, 0x0a, 0x8e, 0x03, 0x35, - 0xec, 0xef, 0x17, 0xe0, 0x22, 0xff, 0x8c, 0x54, 0xbc, 0xd0, 0x37, 0x87, 0xc5, 0x0b, 0x8d, 0x38, - 0x95, 0x39, 0xaf, 0x33, 0x44, 0x0b, 0xfd, 0x2d, 0x0b, 0x66, 0x5b, 0xc9, 0x96, 0xce, 0xc7, 0x2e, - 0x97, 0xd5, 0x87, 0xc2, 0xed, 0x2e, 0x55, 0x88, 0x69, 0xfe, 0xe4, 0x97, 0x2d, 0x98, 0x4d, 0x8a, - 0xa9, 0x56, 0xf7, 0x73, 0x68, 0x24, 0xed, 0x27, 0x9f, 0x2c, 0x0f, 0x31, 0x2d, 0x82, 0xfd, 0xfb, - 0x63, 0xb2, 0x4b, 0xcf, 0x23, 0x18, 0x86, 0xdc, 0x83, 0x72, 0xd4, 0x09, 0x45, 0xa1, 0xfc, 0xda, - 0x11, 0x0f, 0xad, 0x9b, 0x6b, 0x0d, 0xe1, 0x02, 0x12, 0xeb, 0x95, 0xb2, 0x84, 0xe9, 0xc7, 0x8a, - 0x17, 0x67, 0xdc, 0xec, 0x49, 0xc6, 0xb9, 0x9c, 0x96, 0x37, 0x97, 0x37, 0xd2, 0x8c, 0x65, 0x09, - 0x63, 0xac, 0x78, 0xd9, 0xbf, 0x6e, 0x41, 0xf9, 0x86, 0xaf, 0xd6, 0x91, 0x9f, 0xc9, 0xc1, 0x16, - 0xa5, 0x55, 0x56, 0xad, 0xb4, 0xc4, 0xa7, 0xa0, 0x57, 0x12, 0x96, 0xa8, 0xc7, 0x0c, 0xda, 0x8b, - 0x3c, 0xd1, 0x1f, 0x23, 0x75, 0xc3, 0xdf, 0x1a, 0x6a, 0x3e, 0xfe, 0x95, 0x22, 0x4c, 0xbf, 0xe6, - 0xec, 0x53, 0x2f, 0x72, 0x4e, 0xbf, 0x49, 0xbc, 0x08, 0x15, 0xa7, 0xc7, 0xef, 0x32, 0x8d, 0x63, - 0x48, 0x6c, 0xdc, 0x89, 0x41, 0x68, 0xe2, 0xc5, 0x0b, 0x9a, 0x88, 0x4c, 0xc9, 0x5a, 0x8a, 0x96, - 0x53, 0x70, 0x1c, 0xa8, 0x41, 0x6e, 0x00, 0x91, 0xd1, 0xdc, 0xb5, 0x66, 0xd3, 0xef, 0x7b, 0x62, - 0x49, 0x13, 0x76, 0x1f, 0x7d, 0x1e, 0x5e, 0x1f, 0xc0, 0xc0, 0x8c, 0x5a, 0xe4, 0xb3, 0x30, 0xdf, - 0xe4, 0x94, 0xe5, 0xe9, 0xc8, 0xa4, 0x28, 0x4e, 0xc8, 0x3a, 0xd6, 0x63, 0x79, 0x08, 0x1e, 0x0e, - 0xa5, 0xc0, 0x24, 0x0d, 0x23, 0x3f, 0x70, 0xda, 0xd4, 0xa4, 0x3b, 0x91, 0x94, 0xb4, 0x31, 0x80, - 0x81, 0x19, 0xb5, 0xc8, 0x17, 0xa1, 0x1c, 0xed, 0x04, 0x34, 0xdc, 0xf1, 0x3b, 0x2d, 0xe9, 0x7b, - 0x32, 0xa2, 0x31, 0x50, 0xf6, 0xfe, 0xa6, 0xa2, 0x6a, 0x0c, 0x6f, 0x55, 0x84, 0x31, 0x4f, 0x12, - 0xc0, 0x44, 0xd8, 0xf4, 0x7b, 0x34, 0x94, 0xa7, 0x8a, 0x1b, 0xb9, 0x70, 0xe7, 0xc6, 0x2d, 0xc3, - 0x0c, 0xc9, 0x39, 0xa0, 0xe4, 0x64, 0xff, 0xee, 0x18, 0x4c, 0x99, 0x88, 0x27, 0x58, 0x9b, 0xbe, - 0x62, 0xc1, 0x54, 0xd3, 0xf7, 0xa2, 0xc0, 0xef, 0xc4, 0x59, 0x0a, 0x46, 0xd7, 0x28, 0x18, 0xa9, - 0x15, 0x1a, 0x39, 0x6e, 0xc7, 0xb0, 0xd6, 0x19, 0x6c, 0x30, 0xc1, 0x94, 0x7c, 0xc3, 0x82, 0xd9, - 0xd8, 0x55, 0x31, 0xb6, 0xf5, 0xe5, 0x2a, 0x88, 0x5e, 0xea, 0xaf, 0x26, 0x39, 0x61, 0x9a, 0xb5, - 0xbd, 0x05, 0x73, 0xe9, 0xde, 0x66, 0x4d, 0xd9, 0x73, 0xe4, 0x5c, 0x2f, 0xc4, 0x4d, 0xb9, 0xe1, - 0x84, 0x21, 0x72, 0x08, 0x79, 0x16, 0x4a, 0x5d, 0x27, 0x68, 0xbb, 0x9e, 0xd3, 0xe1, 0xad, 0x58, - 0x30, 0x16, 0x24, 0x59, 0x8e, 0x1a, 0xc3, 0xfe, 0x18, 0x4c, 0xad, 0x3b, 0x5e, 0x9b, 0xb6, 0xe4, - 0x3a, 0x7c, 0x7c, 0x38, 0xe6, 0x9f, 0x8c, 0x43, 0xc5, 0x38, 0x3e, 0x9e, 0xff, 0x39, 0x2b, 0x91, - 0x7d, 0xa7, 0x90, 0x63, 0xf6, 0x9d, 0x4f, 0x03, 0x6c, 0xbb, 0x9e, 0x1b, 0xee, 0x9c, 0x31, 0xaf, - 0x0f, 0xbf, 0x9b, 0xbf, 0xa6, 0x29, 0xa0, 0x41, 0x2d, 0xbe, 0x00, 0x2d, 0x1e, 0x91, 0x22, 0xef, - 0x5d, 0xcb, 0xd8, 0x6e, 0x26, 0xf2, 0x70, 0xf8, 0x30, 0x3a, 0x66, 0x51, 0x6d, 0x3f, 0xe2, 0x6e, - 0xea, 0xa8, 0x5d, 0x69, 0x13, 0x4a, 0x01, 0x0d, 0xfb, 0x5d, 0x7a, 0xa6, 0x0c, 0x3c, 0xdc, 0xf5, - 0x06, 0x65, 0x7d, 0xd4, 0x94, 0x16, 0x5e, 0x86, 0xe9, 0x84, 0x08, 0xa7, 0xba, 0x61, 0xf2, 0x21, - 0xd3, 0x46, 0x71, 0x96, 0xfb, 0x26, 0xd6, 0x17, 0x1d, 0x23, 0xf3, 0x8e, 0xee, 0x0b, 0xe1, 0x60, - 0x25, 0x60, 0xf6, 0x8f, 0x27, 0x40, 0xfa, 0x30, 0x9c, 0x60, 0xb9, 0x32, 0x6f, 0x2e, 0xc7, 0xce, - 0x70, 0x73, 0x79, 0x03, 0xa6, 0x5c, 0xcf, 0x8d, 0x5c, 0xa7, 0xc3, 0xed, 0x4f, 0x72, 0x3b, 0x55, - 0x1e, 0xe8, 0x53, 0xab, 0x06, 0x2c, 0x83, 0x4e, 0xa2, 0x2e, 0x79, 0x1d, 0x8a, 0x7c, 0xbf, 0x91, - 0x03, 0xf8, 0xf4, 0x8e, 0x16, 0xdc, 0xc7, 0x46, 0x84, 0xa5, 0x09, 0x4a, 0xfc, 0xf0, 0x21, 0x52, - 0x0f, 0xe9, 0xe3, 0xb7, 0x1c, 0xc7, 0xf1, 0xe1, 0x23, 0x05, 0xc7, 0x81, 0x1a, 0x8c, 0xca, 0xb6, - 0xe3, 0x76, 0xfa, 0x01, 0x8d, 0xa9, 0x4c, 0x24, 0xa9, 0x5c, 0x4b, 0xc1, 0x71, 0xa0, 0x06, 0xd9, - 0x86, 0x29, 0x59, 0x26, 0xdc, 0xe6, 0x26, 0xcf, 0xf8, 0x95, 0xdc, 0x3d, 0xf2, 0x9a, 0x41, 0x09, - 0x13, 0x74, 0x49, 0x1f, 0x2e, 0xb8, 0x5e, 0xd3, 0xf7, 0x9a, 0x9d, 0x7e, 0xe8, 0xee, 0xd1, 0x38, - 0x26, 0xec, 0x2c, 0xcc, 0x2e, 0x1f, 0x1e, 0x54, 0x2f, 0xac, 0xa6, 0xc9, 0xe1, 0x20, 0x07, 0xf2, - 0x25, 0x0b, 0x2e, 0x37, 0x7d, 0x2f, 0xe4, 0xa9, 0x2b, 0xf6, 0xe8, 0xd5, 0x20, 0xf0, 0x03, 0xc1, - 0xbb, 0x7c, 0x46, 0xde, 0xdc, 0xec, 0xb9, 0x9c, 0x45, 0x12, 0xb3, 0x39, 0x91, 0xb7, 0xa0, 0xd4, - 0x0b, 0xfc, 0x3d, 0xb7, 0x45, 0x03, 0xe9, 0x82, 0xb9, 0x96, 0x47, 0x3e, 0x9f, 0x0d, 0x49, 0x33, - 0x5e, 0x7a, 0x54, 0x09, 0x6a, 0x7e, 0xf6, 0xff, 0xa9, 0xc0, 0x4c, 0x12, 0x9d, 0xfc, 0x1c, 0x40, - 0x2f, 0xf0, 0xbb, 0x34, 0xda, 0xa1, 0x3a, 0xb6, 0xe7, 0xe6, 0xa8, 0x19, 0x5b, 0x14, 0x3d, 0xe5, - 0xb6, 0xc4, 0x96, 0x8b, 0xb8, 0x14, 0x0d, 0x8e, 0x24, 0x80, 0xc9, 0x5d, 0xb1, 0xed, 0x4a, 0x2d, - 0xe4, 0xb5, 0x5c, 0x74, 0x26, 0xc9, 0x99, 0x07, 0xa5, 0xc8, 0x22, 0x54, 0x8c, 0xc8, 0x16, 0x14, - 0xee, 0xd1, 0xad, 0x7c, 0xd2, 0x05, 0xdc, 0xa1, 0xf2, 0x34, 0x53, 0x9f, 0x3c, 0x3c, 0xa8, 0x16, - 0xee, 0xd0, 0x2d, 0x64, 0xc4, 0xd9, 0x77, 0xb5, 0x84, 0xef, 0x82, 0x5c, 0x2a, 0x5e, 0xcb, 0xd1, - 0x11, 0x42, 0x7c, 0x97, 0x2c, 0x42, 0xc5, 0x88, 0xbc, 0x05, 0xe5, 0x7b, 0xce, 0x1e, 0xdd, 0x0e, - 0x7c, 0x2f, 0x92, 0xbe, 0x72, 0x23, 0x86, 0x7b, 0xdc, 0x51, 0xe4, 0x24, 0x5f, 0xbe, 0xbd, 0xeb, - 0x42, 0x8c, 0xd9, 0x91, 0x3d, 0x28, 0x79, 0xf4, 0x1e, 0xd2, 0x8e, 0xdb, 0xcc, 0x27, 0xbc, 0xe2, - 0xa6, 0xa4, 0x26, 0x39, 0xf3, 0x7d, 0x4f, 0x95, 0xa1, 0xe6, 0xc5, 0xfa, 0xf2, 0xae, 0xbf, 0x25, - 0x17, 0xaa, 0x11, 0xfb, 0x52, 0x9f, 0x4c, 0x45, 0x5f, 0xde, 0xf0, 0xb7, 0x90, 0x11, 0x67, 0x73, - 0xa4, 0xa9, 0x1d, 0xb5, 0xe4, 0x32, 0x75, 0x33, 0x5f, 0x07, 0x35, 0x31, 0x47, 0xe2, 0x52, 0x34, - 0x38, 0xb2, 0xb6, 0x6d, 0x4b, 0x63, 0xa5, 0x5c, 0xa8, 0x46, 0x6c, 0xdb, 0xa4, 0xe9, 0x53, 0xb4, - 0xad, 0x2a, 0x43, 0xcd, 0x8b, 0xf1, 0x75, 0xa5, 0xe5, 0x2f, 0x9f, 0xa5, 0x2a, 0x69, 0x47, 0x14, - 0x7c, 0x55, 0x19, 0x6a, 0x5e, 0xac, 0xbd, 0xc3, 0xdd, 0xfd, 0x7b, 0x4e, 0x67, 0xd7, 0xf5, 0xda, - 0x32, 0x56, 0x75, 0xd4, 0x34, 0xdc, 0xbb, 0xfb, 0x77, 0x04, 0x3d, 0xb3, 0xbd, 0xe3, 0x52, 0x34, - 0x38, 0x92, 0xbf, 0x6f, 0xc1, 0x44, 0xaf, 0xd3, 0x6f, 0xbb, 0xde, 0xfc, 0x54, 0x1e, 0x4e, 0x4c, - 0xc9, 0x25, 0x77, 0x71, 0x83, 0x93, 0x16, 0x8a, 0xe2, 0x4f, 0x6a, 0xbf, 0x4b, 0x5e, 0xf8, 0xf5, - 0x3f, 0xae, 0xce, 0x53, 0xaf, 0xe9, 0xb7, 0x5c, 0xaf, 0xbd, 0x74, 0x37, 0xf4, 0xbd, 0x45, 0x74, - 0xee, 0x29, 0x1d, 0x5d, 0xca, 0xb4, 0xf0, 0x09, 0xa8, 0x18, 0x24, 0x8e, 0x53, 0xf4, 0xa6, 0x4c, - 0x45, 0xef, 0xd7, 0x27, 0x60, 0xca, 0x4c, 0xbe, 0x79, 0x02, 0xed, 0x4b, 0x9f, 0x38, 0xc6, 0x4e, - 0x73, 0xe2, 0x60, 0x47, 0x4c, 0xe3, 0x82, 0x4b, 0x99, 0xb7, 0x56, 0x73, 0x53, 0xb8, 0xe3, 0x23, - 0xa6, 0x51, 0x18, 0x62, 0x82, 0xe9, 0x29, 0x7c, 0x5e, 0x98, 0xda, 0x2a, 0x14, 0xbb, 0x62, 0x52, - 0x6d, 0x4d, 0xa8, 0x6a, 0xcf, 0x01, 0xc4, 0x59, 0x22, 0xe5, 0xc5, 0xa7, 0xd6, 0x87, 0x8d, 0xec, - 0x95, 0x06, 0x16, 0x79, 0x0a, 0x26, 0x98, 0xea, 0x43, 0x5b, 0x32, 0x94, 0x5e, 0x9f, 0xe3, 0xaf, - 0xf1, 0x52, 0x94, 0x50, 0xf2, 0x12, 0xd3, 0x52, 0x63, 0x85, 0x45, 0x46, 0xc8, 0x5f, 0x8a, 0xb5, - 0xd4, 0x18, 0x86, 0x09, 0x4c, 0x26, 0x3a, 0x65, 0xfa, 0x05, 0x5f, 0x1b, 0x0c, 0xd1, 0xb9, 0xd2, - 0x81, 0x02, 0xc6, 0xed, 0x4a, 0x29, 0x7d, 0x84, 0xcf, 0xe9, 0xa2, 0x61, 0x57, 0x4a, 0xc1, 0x71, - 0xa0, 0x06, 0xfb, 0x18, 0x79, 0x67, 0x5b, 0x11, 0x0e, 0xd3, 0x43, 0x6e, 0x5b, 0xbf, 0x6a, 0x9e, - 0xb5, 0x72, 0x9c, 0x43, 0x62, 0xd4, 0x9e, 0xfc, 0xb0, 0x35, 0xda, 0xb1, 0xe8, 0x73, 0x30, 0x93, - 0xdc, 0x85, 0x72, 0xbf, 0xf9, 0xf8, 0xda, 0x38, 0x5c, 0xbc, 0xd9, 0x76, 0xbd, 0x74, 0xa2, 0xb3, - 0xac, 0x57, 0x0d, 0xac, 0x53, 0xbf, 0x6a, 0xa0, 0x63, 0xf2, 0xe4, 0x9b, 0x01, 0xd9, 0x31, 0x79, - 0xea, 0x01, 0x87, 0x24, 0x2e, 0xf9, 0x23, 0x0b, 0x1e, 0x73, 0x5a, 0xe2, 0x5c, 0xe0, 0x74, 0x64, - 0xa9, 0x91, 0x8c, 0x5b, 0xce, 0xe8, 0x70, 0xc4, 0x5d, 0x7e, 0xf0, 0xe3, 0x17, 0x6b, 0x47, 0x70, - 0x15, 0x3d, 0xfe, 0x13, 0xf2, 0x0b, 0x1e, 0x3b, 0x0a, 0x15, 0x8f, 0x14, 0x9f, 0xfc, 0x35, 0x98, - 0x4d, 0x7c, 0xb0, 0xb4, 0x84, 0x97, 0xc5, 0x85, 0x45, 0x23, 0x09, 0xc2, 0x34, 0xee, 0xc2, 0x2d, - 0xf8, 0xf0, 0xb1, 0x72, 0x9e, 0x6a, 0xb0, 0x7d, 0xcf, 0x82, 0x29, 0x33, 0x27, 0x11, 0x79, 0x16, - 0x4a, 0x91, 0xbf, 0x4b, 0xbd, 0xdb, 0x81, 0x72, 0xd8, 0xd5, 0x03, 0x7d, 0x93, 0x97, 0xe3, 0x1a, - 0x6a, 0x0c, 0x86, 0xdd, 0xec, 0xb8, 0xd4, 0x8b, 0x56, 0x5b, 0xb2, 0x9b, 0x35, 0xf6, 0xb2, 0x28, - 0x5f, 0x41, 0x8d, 0x21, 0x7c, 0xec, 0xd8, 0xef, 0x06, 0x6d, 0x06, 0x54, 0xb9, 0xf7, 0x1b, 0x3e, - 0x76, 0x31, 0x0c, 0x13, 0x98, 0xc4, 0xd6, 0x26, 0xce, 0xf1, 0xf8, 0x5e, 0x23, 0x65, 0x92, 0xfc, - 0x2d, 0x0b, 0xca, 0xc2, 0x44, 0x8f, 0x74, 0x3b, 0xe5, 0x62, 0x9b, 0x32, 0x22, 0xd4, 0x36, 0x56, - 0xb3, 0x5c, 0x6c, 0x9f, 0x80, 0xf1, 0x5d, 0xd7, 0x53, 0x5f, 0xa2, 0xb7, 0xa5, 0xd7, 0x5c, 0xaf, - 0x85, 0x1c, 0xa2, 0x37, 0xae, 0xc2, 0xd0, 0x8d, 0x6b, 0x09, 0xca, 0xda, 0xf1, 0x44, 0x2e, 0xff, - 0xda, 0x7a, 0xab, 0x1d, 0x55, 0x30, 0xc6, 0xb1, 0x7f, 0xd5, 0x82, 0x19, 0x1e, 0xc3, 0x1d, 0x9f, - 0x87, 0x5f, 0xd4, 0xbe, 0x60, 0x42, 0xee, 0xc7, 0x93, 0xbe, 0x60, 0xef, 0x1d, 0x54, 0x2b, 0x22, - 0xea, 0x3b, 0xe9, 0x1a, 0xf6, 0x19, 0x69, 0x44, 0xe3, 0x1e, 0x6b, 0x63, 0xa7, 0xb6, 0xf1, 0xc4, - 0x62, 0x2a, 0x22, 0x18, 0xd3, 0xb3, 0xdf, 0x86, 0x29, 0x33, 0x18, 0x8b, 0xbc, 0x08, 0x95, 0x9e, - 0xeb, 0xb5, 0x93, 0x41, 0xbb, 0xfa, 0xa2, 0x61, 0x23, 0x06, 0xa1, 0x89, 0xc7, 0xab, 0xf9, 0x71, - 0xb5, 0xd4, 0xfd, 0xc4, 0x86, 0x6f, 0x56, 0x8b, 0xff, 0xf0, 0x87, 0x10, 0x32, 0x82, 0xfe, 0x72, - 0x7f, 0x08, 0x21, 0x83, 0xc7, 0xfb, 0xf7, 0x10, 0x42, 0x96, 0x30, 0x7f, 0xbe, 0x1e, 0x42, 0xf8, - 0x69, 0x38, 0x6d, 0x4e, 0x54, 0xb6, 0xd7, 0xdf, 0x33, 0x13, 0x2b, 0xe8, 0x16, 0x97, 0x99, 0x15, - 0x24, 0xd4, 0xfe, 0x9d, 0x02, 0xcc, 0xa5, 0x8f, 0xfc, 0x79, 0x7b, 0x53, 0x90, 0x6f, 0x58, 0x30, - 0xe3, 0x24, 0xf2, 0xcf, 0xe5, 0xf4, 0xaa, 0x52, 0x82, 0xa6, 0x91, 0xff, 0x2c, 0x51, 0x8e, 0x29, - 0xde, 0xe4, 0x2f, 0xc1, 0x64, 0xe4, 0x76, 0xa9, 0xdf, 0x17, 0x86, 0xc0, 0x82, 0x38, 0x90, 0x6f, - 0x8a, 0x22, 0x54, 0x30, 0xb6, 0x28, 0xbb, 0x5c, 0x83, 0x0a, 0xa8, 0xf4, 0x0c, 0x9e, 0x8b, 0x2d, - 0x97, 0xa2, 0x1c, 0x35, 0x06, 0xb9, 0x0f, 0x93, 0xc2, 0xef, 0x42, 0x39, 0xd8, 0xac, 0xe7, 0x64, - 0x9a, 0x10, 0xae, 0x1d, 0x71, 0x17, 0x88, 0xff, 0x21, 0x2a, 0x76, 0xf6, 0xc7, 0xe0, 0x94, 0x49, - 0x62, 0xed, 0xab, 0x40, 0xd0, 0xef, 0x74, 0xb6, 0x9c, 0xe6, 0xee, 0x1d, 0xd7, 0x6b, 0xf9, 0xf7, - 0xf8, 0x52, 0xb4, 0x04, 0xe5, 0x40, 0x86, 0xcc, 0x86, 0x72, 0xd4, 0xe8, 0xb5, 0x4c, 0xc5, 0xd2, - 0x86, 0x18, 0xe3, 0xd8, 0xbf, 0x3f, 0x06, 0x93, 0x32, 0xbe, 0xfb, 0x01, 0x44, 0x26, 0xec, 0x26, - 0xee, 0x83, 0x57, 0x73, 0x09, 0x4b, 0x1f, 0x1a, 0x96, 0x10, 0xa6, 0xc2, 0x12, 0x5e, 0xcb, 0x87, - 0xdd, 0xd1, 0x31, 0x09, 0xdf, 0x2d, 0xc2, 0x6c, 0x2a, 0x5e, 0x3e, 0x95, 0x4f, 0xda, 0x7a, 0x5f, - 0xf2, 0x49, 0x93, 0x30, 0x91, 0x53, 0x3c, 0x3f, 0x3f, 0xc6, 0xbf, 0x48, 0x2f, 0x9e, 0x97, 0x87, - 0x69, 0xf1, 0x83, 0xe3, 0x61, 0xfa, 0x5f, 0x2c, 0x78, 0x64, 0x68, 0xd6, 0x07, 0x9e, 0x34, 0x2c, - 0x48, 0x42, 0xe5, 0x7a, 0x91, 0x73, 0x6e, 0x1b, 0x7d, 0x77, 0x9c, 0xce, 0xf3, 0x94, 0x66, 0x4f, - 0x5e, 0x80, 0x29, 0xae, 0x9f, 0xb1, 0x95, 0x33, 0xa2, 0x3d, 0x79, 0xf5, 0xc5, 0x2f, 0x41, 0x1a, - 0x46, 0x39, 0x26, 0xb0, 0xec, 0xef, 0x58, 0x30, 0x3f, 0x2c, 0x85, 0xd4, 0x09, 0x0c, 0x33, 0x7f, - 0x35, 0x15, 0xd9, 0x51, 0x1d, 0x88, 0xec, 0x48, 0x99, 0x66, 0x54, 0x10, 0x87, 0x61, 0x15, 0x29, - 0x1c, 0x13, 0xb8, 0xf0, 0x07, 0x05, 0x98, 0x93, 0x22, 0xc6, 0x4a, 0xf1, 0x4b, 0x89, 0x78, 0x94, - 0x9f, 0x48, 0xc5, 0xa3, 0x5c, 0x4a, 0xe3, 0xff, 0x45, 0x30, 0xca, 0x07, 0x2b, 0x18, 0xe5, 0xeb, - 0x45, 0xb8, 0x9c, 0x99, 0x49, 0x8a, 0x7c, 0x2d, 0x63, 0xa7, 0xb8, 0x93, 0x73, 0xca, 0x2a, 0x1d, - 0xb7, 0x7a, 0xbe, 0x11, 0x1c, 0xbf, 0x6c, 0x46, 0x4e, 0x88, 0xd5, 0x7f, 0xfb, 0x1c, 0x92, 0x6f, - 0x9d, 0x36, 0x88, 0xe2, 0xc1, 0xbe, 0xb7, 0xf5, 0xe7, 0x60, 0xa9, 0xff, 0x7a, 0x01, 0x9e, 0x3e, - 0x69, 0xcb, 0x7e, 0x40, 0xa3, 0x0e, 0xc3, 0x44, 0xd4, 0xe1, 0x03, 0x52, 0x6d, 0xce, 0x25, 0x00, - 0xf1, 0x1f, 0x8e, 0xeb, 0x7d, 0x77, 0x70, 0xc2, 0x9e, 0xc8, 0x51, 0x63, 0x92, 0xa9, 0xbe, 0x2a, - 0x2b, 0x79, 0xbc, 0x37, 0x4c, 0x36, 0x44, 0xf1, 0x7b, 0x07, 0xd5, 0x0b, 0x71, 0x82, 0x17, 0x59, - 0x88, 0xaa, 0x12, 0x79, 0x1a, 0x4a, 0x81, 0x80, 0xaa, 0x38, 0x2b, 0xe9, 0xed, 0x22, 0xca, 0x50, - 0x43, 0xc9, 0x17, 0x8d, 0xb3, 0xc2, 0xf8, 0x79, 0xe5, 0x31, 0x3a, 0xca, 0x89, 0xe7, 0x4d, 0x28, - 0x85, 0x2a, 0x75, 0xb6, 0x98, 0x4e, 0xcf, 0x9f, 0x30, 0x7c, 0xcf, 0xd9, 0xa2, 0x1d, 0x95, 0x47, - 0x5b, 0x7c, 0x9f, 0xce, 0xb2, 0xad, 0x49, 0x12, 0x5b, 0x9f, 0xbd, 0xc5, 0x25, 0x03, 0x0c, 0x9e, - 0xbb, 0x49, 0x04, 0x93, 0xf2, 0xfd, 0x5c, 0x79, 0xfb, 0xb9, 0x9e, 0x53, 0x1c, 0x8c, 0xf4, 0x92, - 0xe6, 0x47, 0x5a, 0x65, 0x03, 0x52, 0xac, 0xec, 0x1f, 0x58, 0x50, 0x91, 0x63, 0xe4, 0x01, 0xc4, - 0x31, 0xde, 0x4d, 0xc6, 0x31, 0x5e, 0xcd, 0x65, 0x09, 0x1f, 0x12, 0xc4, 0x78, 0x17, 0xa6, 0xcc, - 0x9c, 0x8e, 0xe4, 0xd3, 0xc6, 0x16, 0x64, 0x8d, 0x92, 0x25, 0x4d, 0x6d, 0x52, 0xf1, 0xf6, 0x64, - 0xff, 0x46, 0x59, 0xb7, 0x22, 0x3f, 0x38, 0x9b, 0x23, 0xdf, 0x3a, 0x72, 0xe4, 0x9b, 0x03, 0x6f, - 0x2c, 0xff, 0x81, 0xf7, 0x3a, 0x94, 0xd4, 0xb2, 0x28, 0xb5, 0xa9, 0x27, 0x4d, 0xb7, 0x69, 0xa6, - 0x92, 0x31, 0x62, 0xc6, 0x74, 0xe1, 0x07, 0xe0, 0xd8, 0x32, 0xad, 0x96, 0x6b, 0x4d, 0x86, 0xbc, - 0x05, 0x95, 0x7b, 0x7e, 0xb0, 0xdb, 0xf1, 0x1d, 0xfe, 0x5e, 0x01, 0xe4, 0x71, 0x53, 0xaf, 0xad, - 0xcb, 0x22, 0x76, 0xe5, 0x4e, 0x4c, 0x1f, 0x4d, 0x66, 0xa4, 0x06, 0xb3, 0x5d, 0xd7, 0x43, 0xea, - 0xb4, 0x74, 0xb8, 0xe2, 0xb8, 0xc8, 0x15, 0xae, 0x74, 0xfb, 0xf5, 0x24, 0x18, 0xd3, 0xf8, 0xdc, - 0xf2, 0x14, 0x24, 0x4c, 0x1d, 0x32, 0x21, 0xf0, 0xc6, 0xe8, 0x83, 0x31, 0x69, 0x3e, 0x11, 0xc1, - 0x1b, 0xc9, 0x72, 0x4c, 0xf1, 0x26, 0x5f, 0x80, 0x52, 0xa8, 0x5e, 0xa6, 0x2c, 0xe6, 0x78, 0xea, - 0xd1, 0xaf, 0x53, 0xea, 0xae, 0xd4, 0xcf, 0x53, 0x6a, 0x86, 0x64, 0x0d, 0x2e, 0x29, 0xdb, 0x4d, - 0xe2, 0x91, 0xbd, 0x89, 0x38, 0xbf, 0x17, 0x66, 0xc0, 0x31, 0xb3, 0x16, 0xd3, 0x6d, 0x79, 0xae, - 0x54, 0x71, 0x33, 0x6a, 0x5c, 0x26, 0xf2, 0xf9, 0xd7, 0x42, 0x09, 0x3d, 0x2a, 0x1a, 0xb7, 0x34, - 0x42, 0x34, 0x6e, 0x03, 0x2e, 0xa7, 0x41, 0x3c, 0x71, 0x1b, 0xcf, 0x15, 0x67, 0x6c, 0xa1, 0x1b, - 0x59, 0x48, 0x98, 0x5d, 0x97, 0xdc, 0x81, 0x72, 0x40, 0xf9, 0x29, 0xaf, 0xa6, 0x9c, 0xca, 0x4e, - 0xed, 0x3e, 0x8b, 0x8a, 0x00, 0xc6, 0xb4, 0x58, 0xbf, 0x3b, 0xc9, 0xec, 0xdd, 0xaf, 0xe7, 0xf8, - 0x74, 0xb5, 0xec, 0xfb, 0x21, 0x09, 0x15, 0xed, 0x7f, 0x3b, 0x0b, 0xd3, 0x09, 0x03, 0x14, 0x79, - 0x12, 0x8a, 0x3c, 0x93, 0x1d, 0x5f, 0xad, 0x4a, 0xf1, 0x8a, 0x2a, 0x1a, 0x47, 0xc0, 0xc8, 0x2f, - 0x5a, 0x30, 0xdb, 0x4b, 0x5c, 0xa8, 0xa8, 0x85, 0x7c, 0x44, 0xab, 0x6d, 0xf2, 0x96, 0xc6, 0x78, - 0xf7, 0x22, 0xc9, 0x0c, 0xd3, 0xdc, 0xd9, 0x7a, 0x20, 0x7d, 0xd0, 0x3b, 0x34, 0xe0, 0xd8, 0x52, - 0xd1, 0xd3, 0x24, 0x96, 0x93, 0x60, 0x4c, 0xe3, 0xb3, 0x1e, 0xe6, 0x5f, 0x37, 0xca, 0xf3, 0xa4, - 0x35, 0x45, 0x00, 0x63, 0x5a, 0xe4, 0x15, 0x98, 0x91, 0x49, 0x9b, 0x37, 0xfc, 0xd6, 0x75, 0x27, - 0xdc, 0x91, 0x47, 0x3e, 0x7d, 0x44, 0x5d, 0x4e, 0x40, 0x31, 0x85, 0xcd, 0xbf, 0x2d, 0xce, 0x8c, - 0xcd, 0x09, 0x4c, 0x24, 0x9f, 0x05, 0x59, 0x4e, 0x82, 0x31, 0x8d, 0x4f, 0x9e, 0x35, 0xb6, 0x21, - 0xe1, 0xad, 0xa0, 0x57, 0x83, 0x8c, 0xad, 0xa8, 0x06, 0xb3, 0x7d, 0x7e, 0x42, 0x6e, 0x29, 0xa0, - 0x9c, 0x8f, 0x9a, 0xe1, 0xed, 0x24, 0x18, 0xd3, 0xf8, 0xe4, 0x65, 0x98, 0x0e, 0xd8, 0x62, 0xab, - 0x09, 0x08, 0x17, 0x06, 0x7d, 0x43, 0x8d, 0x26, 0x10, 0x93, 0xb8, 0xe4, 0x55, 0xb8, 0x10, 0xe7, - 0x38, 0x55, 0x04, 0x84, 0x4f, 0x83, 0x4e, 0xb8, 0x57, 0x4b, 0x23, 0xe0, 0x60, 0x1d, 0xf2, 0x37, - 0x60, 0xce, 0x68, 0x89, 0x55, 0xaf, 0x45, 0xef, 0xcb, 0x3c, 0x94, 0xfc, 0x99, 0xab, 0xe5, 0x14, - 0x0c, 0x07, 0xb0, 0xc9, 0x27, 0x61, 0xa6, 0xe9, 0x77, 0x3a, 0x7c, 0x8d, 0x13, 0x4f, 0x52, 0x88, - 0x84, 0x93, 0x22, 0x35, 0x67, 0x02, 0x82, 0x29, 0x4c, 0x72, 0x03, 0x88, 0xbf, 0xc5, 0xd4, 0x2b, - 0xda, 0x7a, 0x95, 0x7a, 0x54, 0x6a, 0x1c, 0xd3, 0xc9, 0x08, 0x98, 0x5b, 0x03, 0x18, 0x98, 0x51, - 0x8b, 0xe7, 0xeb, 0x33, 0x22, 0x86, 0x67, 0xf2, 0x78, 0x4b, 0x33, 0x6d, 0xcf, 0x39, 0x36, 0x5c, - 0x38, 0x80, 0x09, 0x11, 0x90, 0x94, 0x4f, 0xe6, 0x49, 0x33, 0x3b, 0x7d, 0xbc, 0x47, 0x88, 0x52, - 0x94, 0x9c, 0xc8, 0xcf, 0x41, 0x79, 0x4b, 0x3d, 0x55, 0xc2, 0xd3, 0x4d, 0x8e, 0xbc, 0x2f, 0xa6, - 0x5e, 0xdd, 0x89, 0xed, 0x15, 0x1a, 0x80, 0x31, 0x4b, 0xf2, 0x14, 0x54, 0xae, 0x6f, 0xd4, 0xf4, - 0x28, 0xbc, 0xc0, 0x7b, 0x7f, 0x9c, 0x55, 0x41, 0x13, 0xc0, 0x66, 0x98, 0x56, 0xdf, 0x48, 0xf2, - 0x9a, 0x3e, 0x43, 0x1b, 0x63, 0xd8, 0xdc, 0xef, 0x00, 0x1b, 0xf3, 0x17, 0x53, 0xd8, 0xb2, 0x1c, - 0x35, 0x06, 0x79, 0x13, 0x2a, 0x72, 0xbf, 0xe0, 0x6b, 0xd3, 0xa5, 0xb3, 0x45, 0xa3, 0x63, 0x4c, - 0x02, 0x4d, 0x7a, 0xfc, 0xc2, 0x98, 0xbf, 0xe0, 0x40, 0xaf, 0xf5, 0x3b, 0x9d, 0xf9, 0xcb, 0x7c, - 0xdd, 0x8c, 0x2f, 0x8c, 0x63, 0x10, 0x9a, 0x78, 0xe4, 0x79, 0xe5, 0x3f, 0xf6, 0x50, 0xe2, 0x06, - 0x5d, 0xfb, 0x8f, 0x69, 0xa5, 0x7b, 0x48, 0xc0, 0xca, 0xc3, 0xc7, 0x38, 0x6e, 0x6d, 0xc1, 0x82, - 0xd2, 0xf8, 0x06, 0x27, 0xc9, 0xfc, 0x7c, 0xc2, 0x76, 0xb4, 0x70, 0x67, 0x28, 0x26, 0x1e, 0x41, - 0x85, 0x6c, 0x41, 0xc1, 0xe9, 0x6c, 0xcd, 0x3f, 0x92, 0x87, 0xea, 0x5a, 0x5b, 0xab, 0xcb, 0x11, - 0xc5, 0x9d, 0x4c, 0x6b, 0x6b, 0x75, 0x64, 0xc4, 0x89, 0x0b, 0xe3, 0x4e, 0x67, 0x2b, 0x9c, 0x5f, - 0xe0, 0x73, 0x36, 0x37, 0x26, 0xb1, 0xf1, 0x60, 0xad, 0x1e, 0x22, 0x67, 0x61, 0x7f, 0x69, 0x4c, - 0xdf, 0x12, 0xe9, 0xe4, 0xdf, 0x6f, 0x9b, 0x13, 0x48, 0x1c, 0x77, 0x6e, 0xe5, 0x36, 0x81, 0xa4, - 0x7a, 0x31, 0x3d, 0x74, 0xfa, 0xf4, 0xf4, 0x92, 0x91, 0x4b, 0xd6, 0xb0, 0x64, 0x62, 0x73, 0x71, - 0x7a, 0x4e, 0x2e, 0x18, 0xf6, 0x97, 0x2b, 0xda, 0x0a, 0x9a, 0xf2, 0xbd, 0x0a, 0xa0, 0xe8, 0x86, - 0x91, 0xeb, 0xe7, 0x18, 0xa4, 0x9d, 0xca, 0x08, 0xce, 0x63, 0x40, 0x38, 0x00, 0x05, 0x2b, 0xc6, - 0xd3, 0x6b, 0xbb, 0xde, 0x7d, 0xf9, 0xf9, 0xaf, 0xe7, 0xee, 0x54, 0x25, 0x78, 0x72, 0x00, 0x0a, - 0x56, 0xe4, 0xae, 0x18, 0xd4, 0x85, 0x3c, 0xfa, 0xba, 0xb6, 0x56, 0x4f, 0xf1, 0x4b, 0x0e, 0xee, - 0xbb, 0x50, 0x08, 0xbb, 0xae, 0x54, 0x97, 0x46, 0xe4, 0xd5, 0x58, 0x5f, 0xcd, 0xe2, 0xd5, 0x58, - 0x5f, 0x45, 0xc6, 0x84, 0x7c, 0xd5, 0x02, 0x70, 0xba, 0x5b, 0x4e, 0x18, 0x3a, 0x2d, 0x6d, 0x9d, - 0x19, 0xf1, 0xa5, 0x8f, 0x9a, 0xa6, 0x97, 0x62, 0xcd, 0xdd, 0x88, 0x63, 0x28, 0x1a, 0x9c, 0xc9, - 0x5b, 0x30, 0xe9, 0x88, 0xa7, 0x14, 0xa5, 0x47, 0x7c, 0x3e, 0xef, 0x83, 0xa6, 0x24, 0xe0, 0x66, - 0x1a, 0x09, 0x42, 0xc5, 0x90, 0xf1, 0x8e, 0x02, 0x87, 0x6e, 0xbb, 0xbb, 0xd2, 0x38, 0xd4, 0x18, - 0xf9, 0xb1, 0x0f, 0x46, 0x2c, 0x8b, 0xb7, 0x04, 0xa1, 0x62, 0x48, 0x7e, 0xde, 0x82, 0xe9, 0xae, - 0xe3, 0x39, 0x3a, 0xce, 0x31, 0x9f, 0x68, 0x58, 0x33, 0x72, 0x32, 0xd6, 0x10, 0xd7, 0x4d, 0x46, - 0x98, 0xe4, 0x4b, 0xf6, 0x60, 0xc2, 0xe1, 0x8f, 0xbc, 0xca, 0xa3, 0x18, 0xe6, 0xf1, 0x60, 0x6c, - 0xaa, 0x0d, 0xf8, 0xe2, 0x22, 0x9f, 0x92, 0x95, 0xdc, 0xc8, 0xaf, 0x59, 0x30, 0x29, 0x9c, 0xb5, - 0x99, 0x42, 0xca, 0xbe, 0xfd, 0x73, 0xe7, 0xf0, 0xb2, 0x80, 0x74, 0x24, 0x97, 0xee, 0x47, 0x1f, - 0xd1, 0x9e, 0xa8, 0xa2, 0xf4, 0x48, 0x57, 0x72, 0x25, 0x1d, 0x53, 0x7d, 0xbb, 0xce, 0xfd, 0xc4, - 0x53, 0x2e, 0xa6, 0xea, 0xbb, 0x9e, 0x82, 0xe1, 0x00, 0xf6, 0xc2, 0x27, 0x61, 0xca, 0x94, 0xe3, - 0x54, 0xee, 0xe8, 0x3f, 0x2a, 0x00, 0xf0, 0xae, 0x12, 0xb9, 0x51, 0xba, 0x3c, 0x91, 0xf2, 0x8e, - 0xdf, 0xca, 0xe9, 0x49, 0x49, 0x23, 0xc5, 0x09, 0xc8, 0xac, 0xc9, 0x3b, 0x7e, 0x0b, 0x25, 0x13, - 0xd2, 0x86, 0xf1, 0x9e, 0x13, 0xed, 0xe4, 0x9f, 0x4f, 0xa5, 0x24, 0x82, 0x84, 0xa3, 0x1d, 0xe4, - 0x0c, 0xc8, 0x3b, 0x56, 0xec, 0xd9, 0x53, 0xc8, 0x23, 0x17, 0x6c, 0xdc, 0x66, 0x8b, 0xd2, 0x97, - 0x27, 0x95, 0x12, 0x35, 0xed, 0xe1, 0xb3, 0xf0, 0xae, 0x05, 0x53, 0x26, 0x6a, 0x46, 0x37, 0xfd, - 0xac, 0xd9, 0x4d, 0x79, 0xb6, 0x87, 0xd9, 0xe3, 0xff, 0xcd, 0x02, 0xc0, 0xbe, 0xd7, 0xe8, 0x77, - 0xbb, 0x4c, 0x6d, 0xd7, 0x5e, 0xf7, 0xd6, 0x89, 0xbd, 0xee, 0xc7, 0x4e, 0xe9, 0x75, 0x5f, 0x38, - 0x95, 0xd7, 0xfd, 0xf8, 0xe9, 0xbd, 0xee, 0x8b, 0xc3, 0xbd, 0xee, 0xed, 0x6f, 0x59, 0x70, 0x61, - 0x60, 0xbf, 0x62, 0x9a, 0x74, 0xe0, 0xfb, 0xd1, 0x10, 0x8f, 0x4d, 0x8c, 0x41, 0x68, 0xe2, 0x91, - 0x15, 0x98, 0x93, 0xcf, 0x86, 0x34, 0x7a, 0x1d, 0x37, 0x33, 0xd7, 0xcd, 0x66, 0x0a, 0x8e, 0x03, - 0x35, 0xec, 0xdf, 0xb1, 0xa0, 0x62, 0x44, 0xc8, 0xb3, 0xef, 0xe0, 0x6e, 0xbb, 0x52, 0x8c, 0xf8, - 0xc5, 0x14, 0x7e, 0xd5, 0x25, 0x60, 0xe2, 0x1a, 0xba, 0x6d, 0x24, 0x95, 0x8f, 0xaf, 0xa1, 0x59, - 0x29, 0x4a, 0xa8, 0x48, 0x17, 0x4e, 0x7b, 0xbc, 0xd1, 0x0b, 0x66, 0xba, 0x70, 0xda, 0x43, 0x0e, - 0xe1, 0xec, 0xd8, 0x91, 0x42, 0x7a, 0xe4, 0x1a, 0x0f, 0xb4, 0x38, 0x41, 0x84, 0x02, 0x46, 0x1e, - 0x87, 0x02, 0xf5, 0x5a, 0xd2, 0xfe, 0xa1, 0x9f, 0x50, 0xbd, 0xea, 0xb5, 0x90, 0x95, 0xdb, 0xb7, - 0x60, 0x4a, 0x78, 0x23, 0xbf, 0x46, 0xf7, 0x4f, 0xfc, 0x26, 0x2b, 0x1b, 0xed, 0xa9, 0x37, 0x59, - 0x59, 0x75, 0x56, 0x6e, 0xff, 0x13, 0x0b, 0x52, 0xaf, 0x08, 0x19, 0x37, 0x30, 0xd6, 0xd0, 0x1b, - 0x18, 0xd3, 0x6a, 0x3f, 0x76, 0xa4, 0xd5, 0xfe, 0x06, 0x90, 0x2e, 0x9b, 0x0a, 0xc9, 0x85, 0xb6, - 0x90, 0x7c, 0xda, 0x61, 0x7d, 0x00, 0x03, 0x33, 0x6a, 0xd9, 0xff, 0x58, 0x08, 0x6b, 0xbe, 0x2b, - 0x74, 0x7c, 0x03, 0xf4, 0xa1, 0xc8, 0x49, 0x49, 0xfb, 0xdb, 0x88, 0xb6, 0xeb, 0xc1, 0xbc, 0x56, - 0x71, 0x47, 0xca, 0x29, 0xcf, 0xb9, 0xd9, 0x7f, 0x20, 0x64, 0x35, 0x1e, 0x1e, 0x3a, 0x81, 0xac, - 0xdd, 0xa4, 0xac, 0xd7, 0xf3, 0x5a, 0x2b, 0xb3, 0x65, 0x24, 0x8b, 0x00, 0x3d, 0x1a, 0x34, 0xa9, - 0x17, 0xa9, 0x38, 0xa1, 0xa2, 0x8c, 0x58, 0xd5, 0xa5, 0x68, 0x60, 0xd8, 0xdf, 0x64, 0x13, 0x28, - 0x7e, 0xad, 0x98, 0x3c, 0x9d, 0x76, 0x75, 0x4d, 0x4f, 0x0e, 0xed, 0xe9, 0x6a, 0x44, 0x8f, 0x8c, - 0x1d, 0x13, 0x3d, 0xf2, 0x0c, 0x4c, 0x06, 0x7e, 0x87, 0xd6, 0x02, 0x2f, 0xed, 0xa3, 0x83, 0xac, - 0x18, 0x6f, 0xa2, 0x82, 0xdb, 0xbf, 0x62, 0xc1, 0x5c, 0x3a, 0xbc, 0x2d, 0x77, 0xff, 0x5b, 0x33, - 0x06, 0xbf, 0x70, 0xfa, 0x18, 0x7c, 0xfb, 0x1d, 0x26, 0x64, 0xe4, 0x36, 0x77, 0x5d, 0x4f, 0x84, - 0xad, 0xb3, 0x96, 0x7b, 0x06, 0x26, 0xa9, 0x7c, 0x75, 0x55, 0x98, 0x91, 0xb5, 0x90, 0xea, 0xb1, - 0x55, 0x05, 0x27, 0x35, 0x98, 0x55, 0x97, 0x67, 0xca, 0xf6, 0x2f, 0xd2, 0x6d, 0x68, 0x5b, 0xe3, - 0x4a, 0x12, 0x8c, 0x69, 0x7c, 0xfb, 0x8b, 0x50, 0x31, 0x36, 0x25, 0xbe, 0x7e, 0xdf, 0x77, 0x9a, - 0x51, 0x7a, 0xdd, 0xbb, 0xca, 0x0a, 0x51, 0xc0, 0xf8, 0x15, 0x85, 0x88, 0xbe, 0x49, 0xad, 0x7b, - 0x32, 0xe6, 0x46, 0x42, 0x19, 0xb1, 0x80, 0xb6, 0xe9, 0x7d, 0x95, 0xf3, 0x5f, 0x11, 0x43, 0x56, - 0x88, 0x02, 0x66, 0x3f, 0x0b, 0x25, 0x95, 0x14, 0x89, 0x67, 0x16, 0x51, 0xe6, 0x73, 0x33, 0xb3, - 0x88, 0x1f, 0x44, 0xc8, 0x21, 0xf6, 0x1b, 0x50, 0x52, 0xb9, 0x9b, 0x8e, 0xc7, 0x66, 0x4b, 0x51, - 0xe8, 0xb9, 0xd7, 0xfd, 0x30, 0x52, 0x09, 0xa7, 0xc4, 0x0d, 0xdf, 0xcd, 0x55, 0x5e, 0x86, 0x1a, - 0x6a, 0xff, 0x99, 0x05, 0x95, 0xcd, 0xcd, 0x35, 0x7d, 0xf0, 0x47, 0x78, 0x28, 0x14, 0x2d, 0x54, - 0xdb, 0x8e, 0xa8, 0xe9, 0x4a, 0x20, 0x16, 0xbe, 0x85, 0xc3, 0x83, 0xea, 0x43, 0x8d, 0x4c, 0x0c, - 0x1c, 0x52, 0x93, 0xac, 0xc2, 0x45, 0x13, 0x22, 0x13, 0x01, 0xc8, 0x35, 0x92, 0x3f, 0xd3, 0xdb, - 0x18, 0x04, 0x63, 0x56, 0x9d, 0x34, 0x29, 0xb9, 0xdd, 0x9b, 0x2f, 0xfe, 0x36, 0x06, 0xc1, 0x98, - 0x55, 0xc7, 0x7e, 0x1e, 0x66, 0x53, 0x77, 0xdc, 0x27, 0x48, 0xc0, 0xf2, 0xbb, 0x05, 0x98, 0x32, - 0xaf, 0x3a, 0x4f, 0xb0, 0x7e, 0x9d, 0x7c, 0x5b, 0xc8, 0xb8, 0x9e, 0x2c, 0x9c, 0xf2, 0x7a, 0xd2, - 0xbc, 0x0f, 0x1e, 0x3f, 0xdf, 0xfb, 0xe0, 0x62, 0x3e, 0xf7, 0xc1, 0x86, 0xdf, 0xc2, 0xc4, 0x83, - 0xf3, 0x5b, 0xf8, 0xed, 0x22, 0xcc, 0x24, 0x33, 0x7a, 0x9e, 0xa0, 0x27, 0x9f, 0x1d, 0xe8, 0xc9, - 0x53, 0xde, 0x87, 0x14, 0x46, 0xbd, 0x0f, 0x19, 0x1f, 0xf5, 0x3e, 0xa4, 0x78, 0x86, 0xfb, 0x90, - 0xc1, 0xdb, 0x8c, 0x89, 0x13, 0xdf, 0x66, 0x7c, 0x4a, 0xbb, 0x78, 0x4e, 0x26, 0x5c, 0x80, 0x62, - 0x17, 0x4f, 0x92, 0xec, 0x86, 0x65, 0xbf, 0x95, 0xe9, 0x9a, 0x5a, 0x3a, 0xc6, 0xee, 0x1b, 0x64, - 0x7a, 0x64, 0x9e, 0xfe, 0xca, 0xf5, 0xa1, 0x53, 0x78, 0x63, 0xbe, 0x08, 0x15, 0x39, 0x9e, 0xb8, - 0xf2, 0x0d, 0x49, 0xc5, 0xbd, 0x11, 0x83, 0xd0, 0xc4, 0xe3, 0x0f, 0xf6, 0xc7, 0x13, 0x84, 0xdf, - 0xcc, 0x55, 0x52, 0x0f, 0xf6, 0x27, 0xc1, 0x98, 0xc6, 0xb7, 0xbf, 0x00, 0x97, 0x33, 0x4d, 0x30, - 0xdc, 0xfc, 0xcd, 0xf5, 0x42, 0xda, 0x92, 0x08, 0x86, 0x18, 0xa9, 0x87, 0x3e, 0x16, 0xee, 0x0c, - 0xc5, 0xc4, 0x23, 0xa8, 0xd8, 0xbf, 0x59, 0x80, 0x99, 0xe4, 0x6b, 0xaf, 0xe4, 0x9e, 0x36, 0xd8, - 0xe6, 0x62, 0x2b, 0x16, 0x64, 0x8d, 0x2c, 0x91, 0x43, 0x2f, 0x7a, 0xee, 0xf1, 0xf1, 0xb5, 0xa5, - 0x53, 0x56, 0x9e, 0x1f, 0x63, 0x79, 0xc3, 0x22, 0xd9, 0xf1, 0x07, 0x5d, 0xe3, 0x80, 0x52, 0x79, - 0x8e, 0xcf, 0x9d, 0x7b, 0x1c, 0x18, 0xa9, 0x59, 0xa1, 0xc1, 0x96, 0xed, 0x2d, 0x7b, 0x34, 0x70, - 0xb7, 0x5d, 0xfd, 0x52, 0x3d, 0x5f, 0xb9, 0xdf, 0x90, 0x65, 0xa8, 0xa1, 0xf6, 0x3b, 0x63, 0x50, - 0xe6, 0xf9, 0xaf, 0xae, 0x05, 0x7e, 0x97, 0x3f, 0x89, 0x18, 0x1a, 0x67, 0x26, 0xd9, 0x6d, 0x37, - 0x46, 0x7d, 0x77, 0x34, 0xa6, 0x28, 0xdd, 0xdd, 0x8d, 0x12, 0x4c, 0x70, 0x24, 0x3d, 0x28, 0x6d, - 0xcb, 0x7c, 0xbd, 0xb2, 0xef, 0x46, 0xcc, 0x39, 0xa9, 0xb2, 0xff, 0x8a, 0x26, 0x50, 0xff, 0x50, - 0x73, 0xb1, 0x1d, 0x98, 0x4d, 0x25, 0x30, 0xc9, 0x3d, 0xcb, 0xef, 0xff, 0x1c, 0x87, 0xb2, 0x8e, - 0xb3, 0x22, 0x9f, 0x48, 0x18, 0xb0, 0xca, 0xf5, 0x0f, 0x1b, 0xef, 0x75, 0xed, 0xf8, 0xad, 0xf7, - 0x0e, 0xaa, 0xb3, 0x1a, 0x39, 0x65, 0x8c, 0x7a, 0x1c, 0x0a, 0xfd, 0xa0, 0x93, 0x3e, 0xa1, 0xde, - 0xc6, 0x35, 0x64, 0xe5, 0x66, 0x6c, 0x58, 0xe1, 0x81, 0xc6, 0x86, 0xb1, 0x5d, 0x72, 0xcb, 0x6f, - 0xed, 0xa7, 0xdf, 0xf7, 0xaa, 0xfb, 0xad, 0x7d, 0xe4, 0x10, 0xf2, 0x0a, 0xcc, 0xc8, 0x80, 0x37, - 0xa5, 0xc4, 0x14, 0xb9, 0x9e, 0xaa, 0x1d, 0x17, 0x36, 0x13, 0x50, 0x4c, 0x61, 0xb3, 0x5d, 0xf6, - 0x6e, 0xe8, 0x7b, 0x3c, 0x77, 0xf3, 0x44, 0xf2, 0x96, 0xf3, 0x46, 0xe3, 0xd6, 0x4d, 0x6e, 0x48, - 0xd3, 0x18, 0x89, 0x98, 0xba, 0xc9, 0x63, 0x63, 0xea, 0x56, 0x04, 0x6d, 0x26, 0x2d, 0xdf, 0x51, - 0xa6, 0xea, 0x4f, 0x2b, 0xba, 0xac, 0xec, 0xbd, 0x83, 0x23, 0x8c, 0xa4, 0xba, 0x66, 0x56, 0xf4, - 0x61, 0xf9, 0xfd, 0x8b, 0x3e, 0xb4, 0x6f, 0xc3, 0x6c, 0xaa, 0xff, 0x94, 0x81, 0xc3, 0xca, 0x36, - 0x70, 0x9c, 0xec, 0x85, 0xb0, 0x7f, 0x66, 0xc1, 0x85, 0x81, 0x15, 0xe9, 0xa4, 0x61, 0xa0, 0xe9, - 0xbd, 0x71, 0xec, 0xec, 0x7b, 0x63, 0xe1, 0x74, 0x7b, 0x63, 0x7d, 0xeb, 0x7b, 0x3f, 0xbc, 0xf2, - 0xa1, 0xef, 0xff, 0xf0, 0xca, 0x87, 0xfe, 0xf0, 0x87, 0x57, 0x3e, 0xf4, 0xce, 0xe1, 0x15, 0xeb, - 0x7b, 0x87, 0x57, 0xac, 0xef, 0x1f, 0x5e, 0xb1, 0xfe, 0xf0, 0xf0, 0x8a, 0xf5, 0x9f, 0x0f, 0xaf, - 0x58, 0xdf, 0xfa, 0x93, 0x2b, 0x1f, 0xfa, 0xf4, 0xa7, 0xe2, 0x9e, 0x5a, 0x52, 0x3d, 0xc5, 0x7f, - 0x7c, 0x54, 0xf5, 0xcb, 0x52, 0x6f, 0xb7, 0xbd, 0xc4, 0x7a, 0x6a, 0x49, 0x97, 0xa8, 0x9e, 0xfa, - 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x4c, 0x7d, 0x66, 0x21, 0xe0, 0xa6, 0x00, 0x00, + // 9012 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0xbd, 0x5d, 0x6c, 0x24, 0xd9, + 0x75, 0x18, 0xac, 0x62, 0xb3, 0xc9, 0xee, 0xd3, 0x1c, 0x92, 0x73, 0x67, 0x66, 0x87, 0xcb, 0xdd, + 0x19, 0x8e, 0x6a, 0xfd, 0xed, 0x37, 0x6b, 0xad, 0x48, 0x69, 0x76, 0xd7, 0x59, 0x69, 0x95, 0x4d, + 0xba, 0xc9, 0x99, 0x1d, 0xce, 0x92, 0x33, 0xdc, 0xd3, 0x9c, 0x1d, 0xeb, 0x67, 0x6d, 0x15, 0xbb, + 0x2f, 0x9b, 0x35, 0xec, 0xae, 0x6a, 0x55, 0x55, 0x73, 0x86, 0xab, 0x85, 0xb5, 0x92, 0xb0, 0x96, + 0xac, 0x48, 0xb0, 0x62, 0x5b, 0x08, 0xf2, 0x83, 0x40, 0x31, 0x1c, 0x38, 0x7f, 0x0f, 0x81, 0xa1, + 0x20, 0x79, 0x30, 0x90, 0x20, 0x8a, 0x03, 0x19, 0x88, 0x02, 0xf9, 0x21, 0x91, 0x13, 0xc0, 0x74, + 0x44, 0xe7, 0x25, 0x46, 0x02, 0xc1, 0x81, 0x03, 0x21, 0xfb, 0x60, 0x04, 0xf7, 0xb7, 0x6e, 0x55, + 0x57, 0xf3, 0xaf, 0x8b, 0xb3, 0xeb, 0xc4, 0x6f, 0xdd, 0xf7, 0x9c, 0x7b, 0xce, 0xa9, 0xfb, 0x7b, + 0xee, 0xb9, 0xe7, 0x9c, 0x0b, 0x2b, 0x2d, 0x37, 0xda, 0xea, 0x6d, 0xcc, 0x37, 0xfc, 0xce, 0x82, + 0x13, 0xb4, 0xfc, 0x6e, 0xe0, 0xdf, 0xe7, 0x3f, 0x3e, 0x1c, 0xf8, 0xed, 0xb6, 0xdf, 0x8b, 0xc2, + 0x85, 0xee, 0x76, 0x6b, 0xc1, 0xe9, 0xba, 0xe1, 0x82, 0x2e, 0xd9, 0xf9, 0xa8, 0xd3, 0xee, 0x6e, + 0x39, 0x1f, 0x5d, 0x68, 0x51, 0x8f, 0x06, 0x4e, 0x44, 0x9b, 0xf3, 0xdd, 0xc0, 0x8f, 0x7c, 0xf2, + 0x89, 0x98, 0xda, 0xbc, 0xa2, 0xc6, 0x7f, 0xfc, 0xbc, 0xaa, 0x3b, 0xdf, 0xdd, 0x6e, 0xcd, 0x33, + 0x6a, 0xf3, 0xba, 0x44, 0x51, 0x9b, 0xfd, 0xb0, 0x21, 0x4b, 0xcb, 0x6f, 0xf9, 0x0b, 0x9c, 0xe8, + 0x46, 0x6f, 0x93, 0xff, 0xe3, 0x7f, 0xf8, 0x2f, 0xc1, 0x6c, 0xf6, 0xa9, 0xed, 0x17, 0xc3, 0x79, + 0xd7, 0x67, 0xb2, 0x2d, 0x6c, 0x38, 0x51, 0x63, 0x6b, 0x61, 0xa7, 0x4f, 0xa2, 0x59, 0xdb, 0x40, + 0x6a, 0xf8, 0x01, 0xcd, 0xc2, 0x79, 0x3e, 0xc6, 0xe9, 0x38, 0x8d, 0x2d, 0xd7, 0xa3, 0xc1, 0x6e, + 0xfc, 0xd5, 0x1d, 0x1a, 0x39, 0x59, 0xb5, 0x16, 0x06, 0xd5, 0x0a, 0x7a, 0x5e, 0xe4, 0x76, 0x68, + 0x5f, 0x85, 0x9f, 0x39, 0xac, 0x42, 0xd8, 0xd8, 0xa2, 0x1d, 0xa7, 0xaf, 0xde, 0x73, 0x83, 0xea, + 0xf5, 0x22, 0xb7, 0xbd, 0xe0, 0x7a, 0x51, 0x18, 0x05, 0xe9, 0x4a, 0xf6, 0x8f, 0x0b, 0x50, 0xae, + 0xae, 0xd4, 0xea, 0x91, 0x13, 0xf5, 0x42, 0xf2, 0x8b, 0x16, 0x4c, 0xb4, 0x7d, 0xa7, 0x59, 0x73, + 0xda, 0x8e, 0xd7, 0xa0, 0xc1, 0x8c, 0x75, 0xc5, 0xba, 0x5a, 0xb9, 0xb6, 0x32, 0x3f, 0x4c, 0x7f, + 0xcd, 0x57, 0x1f, 0x84, 0x48, 0x43, 0xbf, 0x17, 0x34, 0x28, 0xd2, 0xcd, 0xda, 0xf9, 0xef, 0xed, + 0xcd, 0x7d, 0x60, 0x7f, 0x6f, 0x6e, 0x62, 0xc5, 0xe0, 0x84, 0x09, 0xbe, 0xe4, 0x5b, 0x16, 0x9c, + 0x6d, 0x38, 0x9e, 0x13, 0xec, 0xae, 0x3b, 0x41, 0x8b, 0x46, 0xaf, 0x04, 0x7e, 0xaf, 0x3b, 0x33, + 0x72, 0x0a, 0xd2, 0x3c, 0x2e, 0xa5, 0x39, 0xbb, 0x98, 0x66, 0x87, 0xfd, 0x12, 0x70, 0xb9, 0xc2, + 0xc8, 0xd9, 0x68, 0x53, 0x53, 0xae, 0xc2, 0x69, 0xca, 0x55, 0x4f, 0xb3, 0xc3, 0x7e, 0x09, 0xc8, + 0x33, 0x30, 0xee, 0x7a, 0xad, 0x80, 0x86, 0xe1, 0xcc, 0xe8, 0x15, 0xeb, 0x6a, 0xb9, 0x36, 0x25, + 0xab, 0x8f, 0x2f, 0x8b, 0x62, 0x54, 0x70, 0xfb, 0xb7, 0x0a, 0x70, 0xb6, 0xba, 0x52, 0x5b, 0x0f, + 0x9c, 0xcd, 0x4d, 0xb7, 0x81, 0x7e, 0x2f, 0x72, 0xbd, 0x96, 0x49, 0xc0, 0x3a, 0x98, 0x00, 0x79, + 0x01, 0x2a, 0x21, 0x0d, 0x76, 0xdc, 0x06, 0x5d, 0xf3, 0x83, 0x88, 0x77, 0x4a, 0xb1, 0x76, 0x4e, + 0xa2, 0x57, 0xea, 0x31, 0x08, 0x4d, 0x3c, 0x56, 0x2d, 0xf0, 0xfd, 0x48, 0xc2, 0x79, 0x9b, 0x95, + 0xe3, 0x6a, 0x18, 0x83, 0xd0, 0xc4, 0x23, 0x4b, 0x30, 0xed, 0x78, 0x9e, 0x1f, 0x39, 0x91, 0xeb, + 0x7b, 0x6b, 0x01, 0xdd, 0x74, 0x1f, 0xca, 0x4f, 0x9c, 0x91, 0x75, 0xa7, 0xab, 0x29, 0x38, 0xf6, + 0xd5, 0x20, 0xdf, 0xb4, 0x60, 0x3a, 0x8c, 0xdc, 0xc6, 0xb6, 0xeb, 0xd1, 0x30, 0x5c, 0xf4, 0xbd, + 0x4d, 0xb7, 0x35, 0x53, 0xe4, 0xdd, 0x76, 0x7b, 0xb8, 0x6e, 0xab, 0xa7, 0xa8, 0xd6, 0xce, 0x33, + 0x91, 0xd2, 0xa5, 0xd8, 0xc7, 0x9d, 0x7c, 0x08, 0xca, 0xb2, 0x45, 0x69, 0x38, 0x33, 0x76, 0xa5, + 0x70, 0xb5, 0x5c, 0x3b, 0xb3, 0xbf, 0x37, 0x57, 0x5e, 0x56, 0x85, 0x18, 0xc3, 0xed, 0x25, 0x98, + 0xa9, 0x76, 0x36, 0x9c, 0x30, 0x74, 0x9a, 0x7e, 0x90, 0xea, 0xba, 0xab, 0x50, 0xea, 0x38, 0xdd, + 0xae, 0xeb, 0xb5, 0x58, 0xdf, 0x31, 0x3a, 0x13, 0xfb, 0x7b, 0x73, 0xa5, 0x55, 0x59, 0x86, 0x1a, + 0x6a, 0xff, 0xa7, 0x11, 0xa8, 0x54, 0x3d, 0xa7, 0xbd, 0x1b, 0xba, 0x21, 0xf6, 0x3c, 0xf2, 0x59, + 0x28, 0xb1, 0x55, 0xab, 0xe9, 0x44, 0x8e, 0x9c, 0xe9, 0x1f, 0x99, 0x17, 0x8b, 0xc8, 0xbc, 0xb9, + 0x88, 0xc4, 0x9f, 0xcf, 0xb0, 0xe7, 0x77, 0x3e, 0x3a, 0x7f, 0x67, 0xe3, 0x3e, 0x6d, 0x44, 0xab, + 0x34, 0x72, 0x6a, 0x44, 0xf6, 0x02, 0xc4, 0x65, 0xa8, 0xa9, 0x12, 0x1f, 0x46, 0xc3, 0x2e, 0x6d, + 0xc8, 0x99, 0xbb, 0x3a, 0xe4, 0x0c, 0x89, 0x45, 0xaf, 0x77, 0x69, 0xa3, 0x36, 0x21, 0x59, 0x8f, + 0xb2, 0x7f, 0xc8, 0x19, 0x91, 0x07, 0x30, 0x16, 0xf2, 0xb5, 0x4c, 0x4e, 0xca, 0x3b, 0xf9, 0xb1, + 0xe4, 0x64, 0x6b, 0x93, 0x92, 0xe9, 0x98, 0xf8, 0x8f, 0x92, 0x9d, 0xfd, 0x9f, 0x2d, 0x38, 0x67, + 0x60, 0x57, 0x83, 0x56, 0xaf, 0x43, 0xbd, 0x88, 0x5c, 0x81, 0x51, 0xcf, 0xe9, 0x50, 0x39, 0xab, + 0xb4, 0xc8, 0xb7, 0x9d, 0x0e, 0x45, 0x0e, 0x21, 0x4f, 0x41, 0x71, 0xc7, 0x69, 0xf7, 0x28, 0x6f, + 0xa4, 0x72, 0xed, 0x8c, 0x44, 0x29, 0xbe, 0xce, 0x0a, 0x51, 0xc0, 0xc8, 0x5b, 0x50, 0xe6, 0x3f, + 0x6e, 0x04, 0x7e, 0x27, 0xa7, 0x4f, 0x93, 0x12, 0xbe, 0xae, 0xc8, 0x8a, 0xe1, 0xa7, 0xff, 0x62, + 0xcc, 0xd0, 0xfe, 0x43, 0x0b, 0xa6, 0x8c, 0x8f, 0x5b, 0x71, 0xc3, 0x88, 0x7c, 0xa6, 0x6f, 0xf0, + 0xcc, 0x1f, 0x6d, 0xf0, 0xb0, 0xda, 0x7c, 0xe8, 0x4c, 0xcb, 0x2f, 0x2d, 0xa9, 0x12, 0x63, 0xe0, + 0x78, 0x50, 0x74, 0x23, 0xda, 0x09, 0x67, 0x46, 0xae, 0x14, 0xae, 0x56, 0xae, 0x2d, 0xe7, 0xd6, + 0x8d, 0x71, 0xfb, 0x2e, 0x33, 0xfa, 0x28, 0xd8, 0xd8, 0xdf, 0x29, 0x24, 0xba, 0x6f, 0x55, 0xc9, + 0xf1, 0x8e, 0x05, 0x63, 0x6d, 0x67, 0x83, 0xb6, 0xc5, 0xdc, 0xaa, 0x5c, 0x7b, 0x23, 0x37, 0x49, + 0x14, 0x8f, 0xf9, 0x15, 0x4e, 0xff, 0xba, 0x17, 0x05, 0xbb, 0xf1, 0xf0, 0x12, 0x85, 0x28, 0x99, + 0x93, 0xbf, 0x69, 0x41, 0x25, 0x5e, 0xd5, 0x54, 0xb3, 0x6c, 0xe4, 0x2f, 0x4c, 0xbc, 0x98, 0x4a, + 0x89, 0xf4, 0x12, 0x6d, 0x40, 0xd0, 0x94, 0x65, 0xf6, 0x63, 0x50, 0x31, 0x3e, 0x81, 0x4c, 0x43, + 0x61, 0x9b, 0xee, 0x8a, 0x01, 0x8f, 0xec, 0x27, 0x39, 0x9f, 0x18, 0xe1, 0x72, 0x48, 0x7f, 0x7c, + 0xe4, 0x45, 0x6b, 0xf6, 0x65, 0x98, 0x4e, 0x33, 0x3c, 0x4e, 0x7d, 0xfb, 0x9f, 0x16, 0x13, 0x03, + 0x93, 0x2d, 0x04, 0xc4, 0x87, 0xf1, 0x0e, 0x8d, 0x02, 0xb7, 0xa1, 0xba, 0x6c, 0x69, 0xb8, 0x56, + 0x5a, 0xe5, 0xc4, 0xe2, 0x0d, 0x51, 0xfc, 0x0f, 0x51, 0x71, 0x21, 0x5b, 0x30, 0xea, 0x04, 0x2d, + 0xd5, 0x27, 0x37, 0xf2, 0x99, 0x96, 0xf1, 0x52, 0x51, 0x0d, 0x5a, 0x21, 0x72, 0x0e, 0x64, 0x01, + 0xca, 0x11, 0x0d, 0x3a, 0xae, 0xe7, 0x44, 0x62, 0x07, 0x2d, 0xd5, 0xce, 0x4a, 0xb4, 0xf2, 0xba, + 0x02, 0x60, 0x8c, 0x43, 0xda, 0x30, 0xd6, 0x0c, 0x76, 0xb1, 0xe7, 0xcd, 0x8c, 0xe6, 0xd1, 0x14, + 0x4b, 0x9c, 0x56, 0x3c, 0x48, 0xc5, 0x7f, 0x94, 0x3c, 0xc8, 0x6f, 0x58, 0x70, 0xbe, 0x43, 0x9d, + 0xb0, 0x17, 0x50, 0xf6, 0x09, 0x48, 0x23, 0xea, 0xb1, 0x8e, 0x9d, 0x29, 0x72, 0xe6, 0x38, 0x6c, + 0x3f, 0xf4, 0x53, 0xae, 0x3d, 0x29, 0x45, 0x39, 0x9f, 0x05, 0xc5, 0x4c, 0x69, 0xc8, 0x5b, 0x50, + 0x89, 0xa2, 0x76, 0x3d, 0x62, 0x7a, 0x70, 0x6b, 0x77, 0x66, 0x8c, 0x2f, 0x5e, 0x43, 0xae, 0x30, + 0xeb, 0xeb, 0x2b, 0x8a, 0x60, 0x6d, 0x8a, 0xcd, 0x16, 0xa3, 0x00, 0x4d, 0x76, 0xf6, 0xbf, 0x28, + 0xc2, 0xd9, 0xbe, 0x6d, 0x85, 0x3c, 0x0f, 0xc5, 0xee, 0x96, 0x13, 0xaa, 0x7d, 0xe2, 0xb2, 0x5a, + 0xa4, 0xd6, 0x58, 0xe1, 0xbb, 0x7b, 0x73, 0x67, 0x54, 0x15, 0x5e, 0x80, 0x02, 0x99, 0x69, 0x6d, + 0x1d, 0x1a, 0x86, 0x4e, 0x4b, 0x6d, 0x1e, 0xc6, 0x20, 0xe5, 0xc5, 0xa8, 0xe0, 0xe4, 0x2b, 0x16, + 0x9c, 0x11, 0x03, 0x16, 0x69, 0xd8, 0x6b, 0x47, 0x6c, 0x83, 0x64, 0x9d, 0x72, 0x2b, 0x8f, 0xc9, + 0x21, 0x48, 0xd6, 0x2e, 0x48, 0xee, 0x67, 0xcc, 0xd2, 0x10, 0x93, 0x7c, 0xc9, 0x3d, 0x28, 0x87, + 0x91, 0x13, 0x44, 0xb4, 0x59, 0x8d, 0xb8, 0x2a, 0x57, 0xb9, 0xf6, 0xd3, 0x47, 0xdb, 0x39, 0xd6, + 0xdd, 0x0e, 0x15, 0xbb, 0x54, 0x5d, 0x11, 0xc0, 0x98, 0x16, 0x79, 0x0b, 0x20, 0xe8, 0x79, 0xf5, + 0x5e, 0xa7, 0xe3, 0x04, 0xbb, 0x52, 0xbb, 0xbb, 0x39, 0xdc, 0xe7, 0xa1, 0xa6, 0x17, 0x2b, 0x3a, + 0x71, 0x19, 0x1a, 0xfc, 0xc8, 0x17, 0x2d, 0x38, 0x23, 0xe6, 0x81, 0x92, 0x60, 0x2c, 0x67, 0x09, + 0xce, 0xb2, 0xa6, 0x5d, 0x32, 0x59, 0x60, 0x92, 0x23, 0x79, 0x03, 0x2a, 0x0d, 0xbf, 0xd3, 0x6d, + 0x53, 0xd1, 0xb8, 0xe3, 0xc7, 0x6e, 0x5c, 0x3e, 0x74, 0x17, 0x63, 0x12, 0x68, 0xd2, 0xb3, 0xff, + 0x43, 0x52, 0xc7, 0x51, 0x43, 0x9a, 0x7c, 0x1a, 0x1e, 0x0f, 0x7b, 0x8d, 0x06, 0x0d, 0xc3, 0xcd, + 0x5e, 0x1b, 0x7b, 0xde, 0x4d, 0x37, 0x8c, 0xfc, 0x60, 0x77, 0xc5, 0xed, 0xb8, 0x11, 0x1f, 0xd0, + 0xc5, 0xda, 0xa5, 0xfd, 0xbd, 0xb9, 0xc7, 0xeb, 0x83, 0x90, 0x70, 0x70, 0x7d, 0xe2, 0xc0, 0x13, + 0x3d, 0x6f, 0x30, 0x79, 0x71, 0xfc, 0x98, 0xdb, 0xdf, 0x9b, 0x7b, 0xe2, 0xee, 0x60, 0x34, 0x3c, + 0x88, 0x86, 0xfd, 0xc7, 0x16, 0xdb, 0x86, 0xc4, 0x77, 0xad, 0xd3, 0x4e, 0xb7, 0xcd, 0x96, 0xce, + 0xd3, 0x57, 0x8e, 0xa3, 0x84, 0x72, 0x8c, 0xf9, 0xec, 0xe5, 0x4a, 0xfe, 0x41, 0x1a, 0xb2, 0xfd, + 0xdf, 0x2c, 0x38, 0x9f, 0x46, 0x7e, 0x04, 0x0a, 0x5d, 0x98, 0x54, 0xe8, 0x6e, 0xe7, 0xfb, 0xb5, + 0x03, 0xb4, 0xba, 0x5f, 0x32, 0x06, 0xac, 0x42, 0x45, 0xba, 0x49, 0x5e, 0x84, 0x89, 0x48, 0xfe, + 0xbd, 0x1d, 0x2b, 0xe7, 0xda, 0x30, 0xb1, 0x6e, 0xc0, 0x30, 0x81, 0xc9, 0x6a, 0x36, 0xda, 0xbd, + 0x30, 0xa2, 0x41, 0xbd, 0xe1, 0x77, 0xc5, 0xb2, 0x5b, 0x8a, 0x6b, 0x2e, 0x1a, 0x30, 0x4c, 0x60, + 0xda, 0x7f, 0xad, 0xd8, 0xdf, 0xee, 0xff, 0xb7, 0xeb, 0x2b, 0xb1, 0xfa, 0x51, 0x78, 0x2f, 0xd5, + 0x8f, 0xd1, 0xf7, 0x95, 0xfa, 0xf1, 0x25, 0x8b, 0x69, 0x71, 0x62, 0x00, 0x84, 0x52, 0x35, 0x7a, + 0x2d, 0xdf, 0xe9, 0x80, 0x74, 0xd3, 0x54, 0x0c, 0x25, 0x2f, 0x8c, 0xd9, 0xda, 0xff, 0x60, 0x14, + 0x26, 0xaa, 0x5e, 0xe4, 0x56, 0x37, 0x37, 0x5d, 0xcf, 0x8d, 0x76, 0xc9, 0xd7, 0x47, 0x60, 0xa1, + 0x1b, 0xd0, 0x4d, 0x1a, 0x04, 0xb4, 0xb9, 0xd4, 0x0b, 0x5c, 0xaf, 0x55, 0x6f, 0x6c, 0xd1, 0x66, + 0xaf, 0xed, 0x7a, 0xad, 0xe5, 0x96, 0xe7, 0xeb, 0xe2, 0xeb, 0x0f, 0x69, 0xa3, 0xc7, 0xdb, 0x55, + 0xac, 0x12, 0x9d, 0xe1, 0x64, 0x5f, 0x3b, 0x1e, 0xd3, 0xda, 0x73, 0xfb, 0x7b, 0x73, 0x0b, 0xc7, + 0xac, 0x84, 0xc7, 0xfd, 0x34, 0xf2, 0xd5, 0x11, 0x98, 0x0f, 0xe8, 0xe7, 0x7a, 0xee, 0xd1, 0x5b, + 0x43, 0x2c, 0xe3, 0xed, 0x21, 0xb7, 0xfb, 0x63, 0xf1, 0xac, 0x5d, 0xdb, 0xdf, 0x9b, 0x3b, 0x66, + 0x1d, 0x3c, 0xe6, 0x77, 0xd9, 0x6b, 0x50, 0xa9, 0x76, 0xdd, 0xd0, 0x7d, 0x88, 0x7e, 0x2f, 0xa2, + 0x47, 0x30, 0x68, 0xcc, 0x41, 0x31, 0xe8, 0xb5, 0xa9, 0x58, 0x60, 0xca, 0xb5, 0x32, 0x5b, 0x96, + 0x91, 0x15, 0xa0, 0x28, 0xb7, 0xbf, 0xc4, 0xb6, 0x20, 0x4e, 0x32, 0x65, 0xca, 0xba, 0x0f, 0xc5, + 0x80, 0x31, 0x91, 0x23, 0x6b, 0xd8, 0x53, 0x7f, 0x2c, 0xb5, 0x14, 0x82, 0xfd, 0x44, 0xc1, 0xc2, + 0xfe, 0xee, 0x08, 0x5c, 0xa8, 0x76, 0xbb, 0xab, 0x34, 0xdc, 0x4a, 0x49, 0xf1, 0xcb, 0x16, 0x4c, + 0xee, 0xb8, 0x41, 0xd4, 0x73, 0xda, 0xca, 0x5a, 0x29, 0xe4, 0xa9, 0x0f, 0x2b, 0x0f, 0xe7, 0xf6, + 0x7a, 0x82, 0x74, 0x8d, 0xec, 0xef, 0xcd, 0x4d, 0x26, 0xcb, 0x30, 0xc5, 0x9e, 0xfc, 0x0d, 0x0b, + 0xa6, 0x65, 0xd1, 0x6d, 0xbf, 0x49, 0x4d, 0x6b, 0xf8, 0xdd, 0x3c, 0x65, 0xd2, 0xc4, 0x85, 0x15, + 0x33, 0x5d, 0x8a, 0x7d, 0x42, 0xd8, 0xff, 0x63, 0x04, 0x2e, 0x0e, 0xa0, 0x41, 0x7e, 0xd3, 0x82, + 0xf3, 0xc2, 0x84, 0x6e, 0x80, 0x90, 0x6e, 0xca, 0xd6, 0xfc, 0x64, 0xde, 0x92, 0x23, 0x9b, 0xe2, + 0xd4, 0x6b, 0xd0, 0xda, 0x0c, 0x5b, 0x92, 0x17, 0x33, 0x58, 0x63, 0xa6, 0x40, 0x5c, 0x52, 0x61, + 0x54, 0x4f, 0x49, 0x3a, 0xf2, 0x48, 0x24, 0xad, 0x67, 0xb0, 0xc6, 0x4c, 0x81, 0xec, 0xbf, 0x02, + 0x4f, 0x1c, 0x40, 0xee, 0xf0, 0xc9, 0x69, 0xbf, 0xa1, 0x47, 0x7d, 0x72, 0xcc, 0x1d, 0x61, 0x5e, + 0xdb, 0x30, 0xc6, 0xa7, 0x8e, 0x9a, 0xd8, 0xc0, 0xf6, 0x60, 0x3e, 0xa7, 0x42, 0x94, 0x10, 0xfb, + 0xbb, 0x16, 0x94, 0x8e, 0x61, 0xfb, 0x9c, 0x4b, 0xda, 0x3e, 0xcb, 0x7d, 0x76, 0xcf, 0xa8, 0xdf, + 0xee, 0xf9, 0xca, 0x70, 0xbd, 0x71, 0x14, 0x7b, 0xe7, 0x8f, 0x2d, 0x38, 0xdb, 0x67, 0x1f, 0x25, + 0x5b, 0x70, 0xbe, 0xeb, 0x37, 0xd5, 0x76, 0x7a, 0xd3, 0x09, 0xb7, 0x38, 0x4c, 0x7e, 0xde, 0xf3, + 0xac, 0x27, 0xd7, 0x32, 0xe0, 0xef, 0xee, 0xcd, 0xcd, 0x68, 0x22, 0x29, 0x04, 0xcc, 0xa4, 0x48, + 0xba, 0x50, 0xda, 0x74, 0x69, 0xbb, 0x19, 0x0f, 0xc1, 0x21, 0xb5, 0xb4, 0x1b, 0x92, 0x9a, 0xb8, + 0x1a, 0x50, 0xff, 0x50, 0x73, 0xb1, 0xff, 0xd4, 0x82, 0xc9, 0x6a, 0x2f, 0xda, 0x62, 0x3a, 0x4a, + 0x83, 0x5b, 0xe3, 0x88, 0x07, 0xc5, 0xd0, 0x6d, 0xed, 0x3c, 0x9f, 0xcf, 0x62, 0x5c, 0x67, 0xa4, + 0xe4, 0x15, 0x89, 0x56, 0xd6, 0x79, 0x21, 0x0a, 0x36, 0x24, 0x80, 0x31, 0xdf, 0xe9, 0x45, 0x5b, + 0xd7, 0xe4, 0x27, 0x0f, 0x69, 0x99, 0xb8, 0xc3, 0x3e, 0xe7, 0x9a, 0xe4, 0xa8, 0x55, 0x46, 0x51, + 0x8a, 0x92, 0x93, 0xfd, 0x05, 0x98, 0x4c, 0xde, 0xbb, 0x1d, 0x61, 0xcc, 0x5e, 0x82, 0x82, 0x13, + 0x78, 0x72, 0xc4, 0x56, 0x24, 0x42, 0xa1, 0x8a, 0xb7, 0x91, 0x95, 0x93, 0x67, 0xa1, 0xb4, 0xd9, + 0x6b, 0xb7, 0xf9, 0xb9, 0x42, 0x5c, 0x72, 0xe9, 0x63, 0xd1, 0x0d, 0x59, 0x8e, 0x1a, 0xc3, 0xfe, + 0xdf, 0xa3, 0x30, 0x55, 0x6b, 0xf7, 0xe8, 0x2b, 0x01, 0xa5, 0xca, 0x16, 0x54, 0x85, 0xa9, 0x6e, + 0x40, 0x77, 0x5c, 0xfa, 0xa0, 0x4e, 0xdb, 0xb4, 0x11, 0xf9, 0x81, 0x94, 0xe6, 0xa2, 0x24, 0x34, + 0xb5, 0x96, 0x04, 0x63, 0x1a, 0x9f, 0xbc, 0x0c, 0x93, 0x4e, 0x23, 0x72, 0x77, 0xa8, 0xa6, 0x20, + 0xc4, 0x7d, 0x4c, 0x52, 0x98, 0xac, 0x26, 0xa0, 0x98, 0xc2, 0x26, 0x9f, 0x81, 0x99, 0xb0, 0xe1, + 0xb4, 0xe9, 0xdd, 0xae, 0x64, 0xb5, 0xb8, 0x45, 0x1b, 0xdb, 0x6b, 0xbe, 0xeb, 0x45, 0xd2, 0xee, + 0x78, 0x45, 0x52, 0x9a, 0xa9, 0x0f, 0xc0, 0xc3, 0x81, 0x14, 0xc8, 0xbf, 0xb4, 0xe0, 0x52, 0x37, + 0xa0, 0x6b, 0x81, 0xdf, 0xf1, 0xd9, 0x50, 0xeb, 0x33, 0x87, 0x49, 0xb3, 0xd0, 0xeb, 0x43, 0xea, + 0x52, 0xa2, 0xa4, 0xff, 0x0e, 0xe7, 0x83, 0xfb, 0x7b, 0x73, 0x97, 0xd6, 0x0e, 0x12, 0x00, 0x0f, + 0x96, 0x8f, 0xfc, 0x6b, 0x0b, 0x2e, 0x77, 0xfd, 0x30, 0x3a, 0xe0, 0x13, 0x8a, 0xa7, 0xfa, 0x09, + 0xf6, 0xfe, 0xde, 0xdc, 0xe5, 0xb5, 0x03, 0x25, 0xc0, 0x43, 0x24, 0xb4, 0xf7, 0x2b, 0x70, 0xd6, + 0x18, 0x7b, 0xd2, 0x98, 0xf3, 0x12, 0x9c, 0x51, 0x83, 0x21, 0xd6, 0x7d, 0xca, 0xb1, 0x6d, 0xaf, + 0x6a, 0x02, 0x31, 0x89, 0xcb, 0xc6, 0x9d, 0x1e, 0x8a, 0xa2, 0x76, 0x6a, 0xdc, 0xad, 0x25, 0xa0, + 0x98, 0xc2, 0x26, 0xcb, 0x70, 0x4e, 0x96, 0x20, 0xed, 0xb6, 0xdd, 0x86, 0xb3, 0xe8, 0xf7, 0xe4, + 0x90, 0x2b, 0xd6, 0x2e, 0xee, 0xef, 0xcd, 0x9d, 0x5b, 0xeb, 0x07, 0x63, 0x56, 0x1d, 0xb2, 0x02, + 0xe7, 0x9d, 0x5e, 0xe4, 0xeb, 0xef, 0xbf, 0xee, 0xb1, 0xed, 0xb4, 0xc9, 0x87, 0x56, 0x49, 0xec, + 0xbb, 0xd5, 0x0c, 0x38, 0x66, 0xd6, 0x22, 0x6b, 0x29, 0x6a, 0x75, 0xda, 0xf0, 0xbd, 0xa6, 0xe8, + 0xe5, 0x62, 0x7c, 0x0c, 0xac, 0x66, 0xe0, 0x60, 0x66, 0x4d, 0xd2, 0x86, 0xc9, 0x8e, 0xf3, 0xf0, + 0xae, 0xe7, 0xec, 0x38, 0x6e, 0x9b, 0x31, 0x91, 0xf6, 0xc2, 0xc1, 0x56, 0xa6, 0x5e, 0xe4, 0xb6, + 0xe7, 0x85, 0x1f, 0xc7, 0xfc, 0xb2, 0x17, 0xdd, 0x09, 0xea, 0x11, 0xd3, 0xd4, 0x85, 0x06, 0xb9, + 0x9a, 0xa0, 0x85, 0x29, 0xda, 0xe4, 0x0e, 0x5c, 0xe0, 0xd3, 0x71, 0xc9, 0x7f, 0xe0, 0x2d, 0xd1, + 0xb6, 0xb3, 0xab, 0x3e, 0x60, 0x9c, 0x7f, 0xc0, 0xe3, 0xfb, 0x7b, 0x73, 0x17, 0xea, 0x59, 0x08, + 0x98, 0x5d, 0x8f, 0x38, 0xf0, 0x44, 0x12, 0x80, 0x74, 0xc7, 0x0d, 0x5d, 0xdf, 0x13, 0x66, 0xb9, + 0x52, 0x6c, 0x96, 0xab, 0x0f, 0x46, 0xc3, 0x83, 0x68, 0x90, 0xbf, 0x6d, 0xc1, 0xf9, 0xac, 0x69, + 0x38, 0x53, 0xce, 0xe3, 0x36, 0x39, 0x35, 0xb5, 0xc4, 0x88, 0xc8, 0x5c, 0x14, 0x32, 0x85, 0x20, + 0x6f, 0x5b, 0x30, 0xe1, 0x18, 0x27, 0xe8, 0x19, 0xc8, 0x63, 0xd7, 0x32, 0xcf, 0xe4, 0xb5, 0xe9, + 0xfd, 0xbd, 0xb9, 0xc4, 0x29, 0x1d, 0x13, 0x1c, 0xc9, 0xdf, 0xb5, 0xe0, 0x42, 0xe6, 0x1c, 0x9f, + 0xa9, 0x9c, 0x46, 0x0b, 0xf1, 0x41, 0x92, 0xbd, 0xe6, 0x64, 0x8b, 0x41, 0xbe, 0x69, 0xe9, 0xad, + 0x4c, 0x5d, 0x30, 0xce, 0x4c, 0x70, 0xd1, 0x86, 0x34, 0x78, 0x18, 0x6a, 0x94, 0x22, 0x5c, 0x3b, + 0x67, 0xec, 0x8c, 0xaa, 0x10, 0xd3, 0xec, 0xc9, 0x37, 0x2c, 0xb5, 0x35, 0x6a, 0x89, 0xce, 0x9c, + 0x96, 0x44, 0x24, 0xde, 0x69, 0xb5, 0x40, 0x29, 0xe6, 0xe4, 0xe7, 0x60, 0xd6, 0xd9, 0xf0, 0x83, + 0x28, 0x73, 0xf2, 0xcd, 0x4c, 0xf2, 0x69, 0x74, 0x79, 0x7f, 0x6f, 0x6e, 0xb6, 0x3a, 0x10, 0x0b, + 0x0f, 0xa0, 0x60, 0xff, 0xee, 0x18, 0x4c, 0x88, 0x93, 0x90, 0xdc, 0xba, 0x7e, 0xdb, 0x82, 0x27, + 0x1b, 0xbd, 0x20, 0xa0, 0x5e, 0x54, 0x8f, 0x68, 0xb7, 0x7f, 0xe3, 0xb2, 0x4e, 0x75, 0xe3, 0xba, + 0xb2, 0xbf, 0x37, 0xf7, 0xe4, 0xe2, 0x01, 0xfc, 0xf1, 0x40, 0xe9, 0xc8, 0xbf, 0xb7, 0xc0, 0x96, + 0x08, 0x35, 0xa7, 0xb1, 0xdd, 0x0a, 0xfc, 0x9e, 0xd7, 0xec, 0xff, 0x88, 0x91, 0x53, 0xfd, 0x88, + 0xa7, 0xf7, 0xf7, 0xe6, 0xec, 0xc5, 0x43, 0xa5, 0xc0, 0x23, 0x48, 0x4a, 0x5e, 0x81, 0xb3, 0x12, + 0xeb, 0xfa, 0xc3, 0x2e, 0x0d, 0x5c, 0x76, 0xe6, 0x90, 0x8a, 0x63, 0xec, 0x9b, 0x96, 0x46, 0xc0, + 0xfe, 0x3a, 0x24, 0x84, 0xf1, 0x07, 0xd4, 0x6d, 0x6d, 0x45, 0x4a, 0x7d, 0x1a, 0xd2, 0x21, 0x4d, + 0x5a, 0x45, 0xee, 0x09, 0x9a, 0xb5, 0xca, 0xfe, 0xde, 0xdc, 0xb8, 0xfc, 0x83, 0x8a, 0x13, 0xb9, + 0x0d, 0x93, 0xe2, 0x9c, 0xba, 0xe6, 0x7a, 0xad, 0x35, 0xdf, 0x13, 0x5e, 0x55, 0xe5, 0xda, 0xd3, + 0x6a, 0xc3, 0xaf, 0x27, 0xa0, 0xef, 0xee, 0xcd, 0x4d, 0xa8, 0xdf, 0xeb, 0xbb, 0x5d, 0x8a, 0xa9, + 0xda, 0xe4, 0x6f, 0x59, 0x40, 0xc2, 0x88, 0x76, 0xd7, 0xda, 0xbd, 0x96, 0x2b, 0x9b, 0x48, 0xfa, + 0x47, 0xe5, 0xe0, 0xaa, 0x95, 0xa4, 0x5b, 0x9b, 0x95, 0x42, 0x92, 0x7a, 0x1f, 0x47, 0xcc, 0x90, + 0xc2, 0xfe, 0xce, 0x38, 0x80, 0x9a, 0x4b, 0xb4, 0x4b, 0x3e, 0x04, 0xe5, 0x90, 0x46, 0xa2, 0x49, + 0xe4, 0x35, 0x97, 0xb8, 0x9c, 0x54, 0x85, 0x18, 0xc3, 0xc9, 0x36, 0x14, 0xbb, 0x4e, 0x2f, 0xa4, + 0xf9, 0x1c, 0x6e, 0xe4, 0xc8, 0x5c, 0x63, 0x14, 0xc5, 0xa9, 0x99, 0xff, 0x44, 0xc1, 0x83, 0x7c, + 0xd9, 0x02, 0xa0, 0xc9, 0xd1, 0x34, 0xb4, 0xf5, 0x4a, 0xb2, 0x8c, 0x07, 0x1c, 0x6b, 0x83, 0xda, + 0xe4, 0xfe, 0xde, 0x1c, 0x18, 0xe3, 0xd2, 0x60, 0x4b, 0x1e, 0x40, 0xc9, 0x51, 0x1b, 0xd2, 0xe8, + 0x69, 0x6c, 0x48, 0xfc, 0x30, 0xab, 0x67, 0x94, 0x66, 0x46, 0xbe, 0x6a, 0xc1, 0x64, 0x48, 0x23, + 0xd9, 0x55, 0x6c, 0x59, 0x94, 0xda, 0xf8, 0x90, 0x33, 0xa2, 0x9e, 0xa0, 0x29, 0x96, 0xf7, 0x64, + 0x19, 0xa6, 0xf8, 0x2a, 0x51, 0x6e, 0x52, 0xa7, 0x49, 0x03, 0x6e, 0x2b, 0x91, 0x6a, 0xde, 0xf0, + 0xa2, 0x18, 0x34, 0xb5, 0x28, 0x46, 0x19, 0xa6, 0xf8, 0x2a, 0x51, 0x56, 0xdd, 0x20, 0xf0, 0xa5, + 0x28, 0xa5, 0x9c, 0x44, 0x31, 0x68, 0x6a, 0x51, 0x8c, 0x32, 0x4c, 0xf1, 0x25, 0x6d, 0x18, 0xeb, + 0xf2, 0xa9, 0x25, 0x55, 0xb9, 0x21, 0xef, 0xc8, 0xd5, 0x34, 0xa5, 0x5d, 0x61, 0x93, 0x12, 0xff, + 0x51, 0xf2, 0xb0, 0xbf, 0x7d, 0x06, 0x26, 0xd5, 0xb4, 0x8d, 0x0f, 0x39, 0xc2, 0x10, 0x38, 0xe0, + 0x90, 0xb3, 0x68, 0x02, 0x31, 0x89, 0xcb, 0x2a, 0x8b, 0x55, 0x2b, 0x79, 0xc6, 0xd1, 0x95, 0xeb, + 0x26, 0x10, 0x93, 0xb8, 0xa4, 0x03, 0x45, 0xb6, 0xb2, 0x28, 0xf7, 0x8b, 0x21, 0xbf, 0x3c, 0x5e, + 0x8d, 0x0c, 0xa3, 0x0a, 0x23, 0x8f, 0x82, 0x0b, 0xb7, 0x65, 0x47, 0x09, 0xf3, 0xb6, 0x9c, 0x8a, + 0xf9, 0xac, 0x06, 0x49, 0xcb, 0xb9, 0xe8, 0xfb, 0x64, 0x19, 0xa6, 0xd8, 0x67, 0x9c, 0x7b, 0x8a, + 0xa7, 0x78, 0xee, 0xf9, 0x14, 0x94, 0x3a, 0xce, 0xc3, 0x7a, 0x2f, 0x68, 0x9d, 0xfc, 0x7c, 0x25, + 0xdd, 0x69, 0x05, 0x15, 0xd4, 0xf4, 0xc8, 0x17, 0x2d, 0x63, 0x81, 0x13, 0xbe, 0x16, 0xf7, 0xf2, + 0x5d, 0xe0, 0xb4, 0xda, 0x30, 0x70, 0xa9, 0xeb, 0x3b, 0x85, 0x94, 0x1e, 0xf9, 0x29, 0x84, 0x69, + 0xd4, 0x62, 0x82, 0x68, 0x8d, 0xba, 0x7c, 0xaa, 0x1a, 0xf5, 0x62, 0x82, 0x19, 0xa6, 0x98, 0x73, + 0x79, 0xc4, 0x9c, 0xd3, 0xf2, 0xc0, 0xa9, 0xca, 0x53, 0x4f, 0x30, 0xc3, 0x14, 0xf3, 0xc1, 0x47, + 0xef, 0xca, 0xe9, 0x1c, 0xbd, 0x27, 0x72, 0x38, 0x7a, 0x1f, 0x7c, 0x2a, 0x39, 0x33, 0xec, 0xa9, + 0x84, 0xdc, 0x02, 0xd2, 0xdc, 0xf5, 0x9c, 0x8e, 0xdb, 0x90, 0x8b, 0x25, 0xdf, 0xa4, 0x27, 0xb9, + 0x69, 0x46, 0x6b, 0x65, 0x4b, 0x7d, 0x18, 0x98, 0x51, 0x8b, 0x44, 0x50, 0xea, 0x2a, 0xe5, 0x73, + 0x2a, 0x8f, 0xd1, 0xaf, 0x94, 0x51, 0xe1, 0x42, 0xc3, 0x26, 0x9e, 0x2a, 0x41, 0xcd, 0x89, 0xac, + 0xc0, 0xf9, 0x8e, 0xeb, 0xad, 0xf9, 0xcd, 0x70, 0x8d, 0x06, 0xd2, 0xf0, 0x54, 0xa7, 0xd1, 0xcc, + 0x34, 0x6f, 0x1b, 0x6e, 0x4c, 0x58, 0xcd, 0x80, 0x63, 0x66, 0x2d, 0xfb, 0x7f, 0x59, 0x30, 0xbd, + 0xd8, 0xf6, 0x7b, 0xcd, 0x7b, 0x4e, 0xd4, 0xd8, 0x12, 0x1e, 0x1b, 0xe4, 0x65, 0x28, 0xb9, 0x5e, + 0x44, 0x83, 0x1d, 0xa7, 0x2d, 0xf7, 0x27, 0x5b, 0x59, 0x92, 0x97, 0x65, 0xf9, 0xbb, 0x7b, 0x73, + 0x93, 0x4b, 0xbd, 0x80, 0x1b, 0xec, 0xc5, 0x6a, 0x85, 0xba, 0x0e, 0xf9, 0xb6, 0x05, 0x67, 0x85, + 0xcf, 0xc7, 0x92, 0x13, 0x39, 0xaf, 0xf5, 0x68, 0xe0, 0x52, 0xe5, 0xf5, 0x31, 0xe4, 0x42, 0x95, + 0x96, 0x55, 0x31, 0xd8, 0x8d, 0xcf, 0x2c, 0xab, 0x69, 0xce, 0xd8, 0x2f, 0x8c, 0xfd, 0xab, 0x05, + 0x78, 0x7c, 0x20, 0x2d, 0x32, 0x0b, 0x23, 0x6e, 0x53, 0x7e, 0x3a, 0x48, 0xba, 0x23, 0xcb, 0x4d, + 0x1c, 0x71, 0x9b, 0x64, 0x9e, 0x6b, 0xb8, 0x01, 0x0d, 0x43, 0x75, 0xf7, 0x5e, 0xd6, 0xca, 0xa8, + 0x2c, 0x45, 0x03, 0x83, 0xcc, 0x41, 0x91, 0xbb, 0x52, 0xcb, 0xa3, 0x15, 0xd7, 0x99, 0xb9, 0xd7, + 0x32, 0x8a, 0x72, 0xf2, 0x25, 0x0b, 0x40, 0x08, 0xc8, 0xf4, 0x7d, 0xb9, 0x4b, 0x62, 0xbe, 0xcd, + 0xc4, 0x28, 0x0b, 0x29, 0xe3, 0xff, 0x68, 0x70, 0x25, 0xeb, 0x30, 0xc6, 0xd4, 0x67, 0xbf, 0x79, + 0xe2, 0x4d, 0x51, 0x28, 0x40, 0x9c, 0x06, 0x4a, 0x5a, 0xac, 0xad, 0x02, 0x1a, 0xf5, 0x02, 0x8f, + 0x35, 0x2d, 0xdf, 0x06, 0x4b, 0x42, 0x0a, 0xd4, 0xa5, 0x68, 0x60, 0xd8, 0xff, 0x7c, 0x04, 0xce, + 0x67, 0x89, 0xce, 0x76, 0x9b, 0x31, 0x21, 0xad, 0xb4, 0x12, 0xfc, 0x6c, 0xfe, 0xed, 0x23, 0xdd, + 0x97, 0xf4, 0x8d, 0x8d, 0xf4, 0x25, 0x95, 0x7c, 0xc9, 0xcf, 0xea, 0x16, 0x1a, 0x39, 0x61, 0x0b, + 0x69, 0xca, 0xa9, 0x56, 0xba, 0x02, 0xa3, 0x21, 0xeb, 0xf9, 0x42, 0xf2, 0xe6, 0x87, 0xf7, 0x11, + 0x87, 0x30, 0x8c, 0x9e, 0xe7, 0x46, 0x32, 0xfe, 0x48, 0x63, 0xdc, 0xf5, 0xdc, 0x08, 0x39, 0xc4, + 0xfe, 0xd6, 0x08, 0xcc, 0x0e, 0xfe, 0x28, 0xf2, 0x2d, 0x0b, 0xa0, 0xc9, 0x0e, 0x47, 0x21, 0x77, + 0xe2, 0x17, 0xee, 0x5e, 0xce, 0x69, 0xb5, 0xe1, 0x92, 0xe2, 0x14, 0xfb, 0x21, 0xea, 0xa2, 0x10, + 0x0d, 0x41, 0xc8, 0x35, 0x35, 0xf4, 0xf9, 0xad, 0x95, 0x98, 0x4c, 0xba, 0xce, 0xaa, 0x86, 0xa0, + 0x81, 0xc5, 0x4e, 0xbf, 0x9e, 0xd3, 0xa1, 0x61, 0xd7, 0xd1, 0xd1, 0x5c, 0xfc, 0xf4, 0x7b, 0x5b, + 0x15, 0x62, 0x0c, 0xb7, 0xdb, 0xf0, 0xd4, 0x11, 0xe4, 0xcc, 0x29, 0x58, 0xc6, 0xfe, 0x13, 0x0b, + 0x2e, 0x4a, 0x4f, 0xbc, 0xff, 0x67, 0xdc, 0x3a, 0x7f, 0x62, 0xc1, 0x13, 0x03, 0xbe, 0xf9, 0x11, + 0x78, 0x77, 0xbe, 0x99, 0xf4, 0xee, 0xbc, 0x3b, 0xec, 0x90, 0xce, 0xfc, 0x8e, 0x01, 0x4e, 0x9e, + 0xdf, 0x1d, 0x85, 0x33, 0x6c, 0xd9, 0x6a, 0xfa, 0xad, 0x9c, 0x36, 0xce, 0xa7, 0xa0, 0xf8, 0x39, + 0xb6, 0x01, 0xa5, 0x07, 0x19, 0xdf, 0x95, 0x50, 0xc0, 0xc8, 0x97, 0x2d, 0x18, 0xff, 0x9c, 0xdc, + 0x53, 0xc5, 0x59, 0x6e, 0xc8, 0xc5, 0x30, 0xf1, 0x0d, 0xf3, 0x72, 0x87, 0x14, 0x31, 0x38, 0xda, + 0x97, 0x53, 0x6d, 0xa5, 0x8a, 0x33, 0x79, 0x06, 0xc6, 0x37, 0xfd, 0xa0, 0xd3, 0x6b, 0x3b, 0xe9, + 0xc0, 0xcf, 0x1b, 0xa2, 0x18, 0x15, 0x9c, 0x4d, 0x72, 0xa7, 0xeb, 0xbe, 0x4e, 0x83, 0x50, 0x84, + 0x64, 0x24, 0x26, 0x79, 0x55, 0x43, 0xd0, 0xc0, 0xe2, 0x75, 0x5a, 0xad, 0x80, 0xb6, 0x9c, 0xc8, + 0x0f, 0xf8, 0xce, 0x61, 0xd6, 0xd1, 0x10, 0x34, 0xb0, 0xc8, 0x43, 0x28, 0x87, 0xb4, 0x11, 0xd0, + 0x08, 0xe9, 0xa6, 0x3c, 0x16, 0xbd, 0x32, 0xac, 0x85, 0x41, 0x92, 0x8b, 0x9d, 0x1a, 0x75, 0x11, + 0xc6, 0xcc, 0x66, 0x3f, 0x0e, 0x13, 0x66, 0xb3, 0x1d, 0x2b, 0x92, 0xe8, 0x13, 0x20, 0xdd, 0x49, + 0x53, 0x8b, 0xa1, 0x75, 0x94, 0xc5, 0xd0, 0xfe, 0x8f, 0x23, 0x60, 0x58, 0xc1, 0x1e, 0xc1, 0x22, + 0xe3, 0x25, 0x16, 0x99, 0x21, 0x2d, 0x38, 0x86, 0x4d, 0x6f, 0x50, 0x5c, 0xe5, 0x4e, 0x2a, 0xae, + 0xf2, 0x76, 0x6e, 0x1c, 0x0f, 0x0e, 0xab, 0xfc, 0xa1, 0x05, 0x4f, 0xc4, 0xc8, 0xfd, 0xd6, 0xf3, + 0xc3, 0x77, 0x8c, 0x17, 0xa0, 0xe2, 0xc4, 0xd5, 0xe4, 0x94, 0x36, 0x82, 0xda, 0x34, 0x08, 0x4d, + 0xbc, 0x38, 0x20, 0xa7, 0x70, 0xc2, 0x80, 0x9c, 0xd1, 0x83, 0x03, 0x72, 0xec, 0x3f, 0x1d, 0x81, + 0x4b, 0xfd, 0x5f, 0x66, 0x7a, 0xa9, 0x1f, 0xfe, 0x6d, 0x69, 0x3f, 0xf6, 0x91, 0x13, 0xfb, 0xb1, + 0x17, 0x8e, 0xea, 0xc7, 0xae, 0xbd, 0xc7, 0x47, 0x4f, 0xdd, 0x7b, 0xbc, 0x0e, 0x17, 0x94, 0xab, + 0xea, 0x0d, 0x3f, 0x90, 0x51, 0x29, 0x6a, 0xed, 0x2a, 0xd5, 0x2e, 0xc9, 0x2a, 0x17, 0x30, 0x0b, + 0x09, 0xb3, 0xeb, 0xda, 0x3f, 0x2c, 0xc0, 0xb9, 0xb8, 0xd9, 0x17, 0x7d, 0xaf, 0xe9, 0x72, 0x6f, + 0xa7, 0x97, 0x60, 0x34, 0xda, 0xed, 0xaa, 0xc6, 0xfe, 0xff, 0x95, 0x38, 0xeb, 0xbb, 0x5d, 0xd6, + 0xdb, 0x17, 0x33, 0xaa, 0xf0, 0xfb, 0x0b, 0x5e, 0x89, 0xac, 0xe8, 0xd9, 0x21, 0x7a, 0xe0, 0xf9, + 0xe4, 0x68, 0x7e, 0x77, 0x6f, 0x2e, 0x23, 0xbf, 0xc4, 0xbc, 0xa6, 0x94, 0x1c, 0xf3, 0xe4, 0x3e, + 0x4c, 0xb6, 0x9d, 0x30, 0xba, 0xdb, 0x6d, 0x3a, 0x11, 0x5d, 0x77, 0xa5, 0x1f, 0xd1, 0xf1, 0x02, + 0x79, 0xb4, 0xc3, 0xc5, 0x4a, 0x82, 0x12, 0xa6, 0x28, 0x93, 0x1d, 0x20, 0xac, 0x64, 0x3d, 0x70, + 0xbc, 0x50, 0x7c, 0x15, 0xe3, 0x77, 0xfc, 0xa8, 0x2c, 0x7d, 0x68, 0x5f, 0xe9, 0xa3, 0x86, 0x19, + 0x1c, 0xc8, 0xd3, 0x30, 0x16, 0x50, 0x27, 0xd4, 0x1b, 0x91, 0x9e, 0xff, 0xc8, 0x4b, 0x51, 0x42, + 0xcd, 0x09, 0x35, 0x76, 0xc8, 0x84, 0xfa, 0x03, 0x0b, 0x26, 0xe3, 0x6e, 0x7a, 0x04, 0x4a, 0x4f, + 0x27, 0xa9, 0xf4, 0xdc, 0xcc, 0x6b, 0x49, 0x1c, 0xa0, 0xe7, 0xfc, 0xf1, 0xb8, 0xf9, 0x7d, 0x3c, + 0x74, 0xe4, 0xf3, 0x66, 0x24, 0x81, 0x95, 0x47, 0x3c, 0x5f, 0x42, 0xcf, 0x3c, 0x30, 0x84, 0x80, + 0x69, 0x59, 0x4d, 0xa9, 0x41, 0xc9, 0x61, 0xaf, 0xb5, 0x2c, 0xa5, 0x59, 0x65, 0x69, 0x59, 0xaa, + 0x0e, 0xb9, 0x0b, 0x17, 0xbb, 0x81, 0xcf, 0x33, 0x1c, 0x2c, 0x51, 0xa7, 0xd9, 0x76, 0x3d, 0xaa, + 0x0c, 0x4c, 0xc2, 0xdf, 0xe7, 0x89, 0xfd, 0xbd, 0xb9, 0x8b, 0x6b, 0xd9, 0x28, 0x38, 0xa8, 0x6e, + 0x32, 0x46, 0x76, 0xf4, 0x08, 0x31, 0xb2, 0xbf, 0xa4, 0xcd, 0xb8, 0x3a, 0x1c, 0xe3, 0xd3, 0x79, + 0x75, 0x65, 0x56, 0x60, 0x86, 0x1e, 0x52, 0x55, 0xc9, 0x14, 0x35, 0xfb, 0xc1, 0xb6, 0xc2, 0xb1, + 0x13, 0xda, 0x0a, 0xe3, 0x08, 0x9c, 0xf1, 0xf7, 0x32, 0x02, 0xa7, 0xf4, 0xbe, 0x8a, 0xc0, 0xf9, + 0xb6, 0x05, 0xe7, 0x9c, 0xfe, 0xd8, 0xf7, 0x7c, 0xcc, 0xd6, 0x19, 0x41, 0xf5, 0xb5, 0x27, 0xa4, + 0x90, 0x59, 0x29, 0x06, 0x30, 0x4b, 0x14, 0xfb, 0x9d, 0x22, 0x4c, 0xa7, 0x95, 0xa4, 0xd3, 0x0f, + 0x12, 0xfe, 0x15, 0x0b, 0xa6, 0xd5, 0x04, 0xd7, 0x77, 0xef, 0xe2, 0x70, 0xb3, 0x92, 0xd3, 0xba, + 0x22, 0xd4, 0x3d, 0x9d, 0xbb, 0x65, 0x3d, 0xc5, 0x0d, 0xfb, 0xf8, 0x93, 0x37, 0xa0, 0xa2, 0xef, + 0x73, 0x4e, 0x14, 0x31, 0xcc, 0x83, 0x5a, 0xab, 0x31, 0x09, 0x34, 0xe9, 0x91, 0x77, 0x2c, 0x80, + 0x86, 0xda, 0x89, 0x73, 0x8a, 0xc7, 0xca, 0xd0, 0x16, 0x62, 0x7d, 0x5e, 0x17, 0x85, 0x68, 0x30, + 0x26, 0xbf, 0xca, 0x6f, 0x72, 0xf4, 0x48, 0x50, 0x3e, 0x0f, 0x9f, 0xcc, 0x7b, 0x29, 0x8a, 0xbd, + 0x58, 0xb4, 0xb6, 0x67, 0x80, 0x42, 0x4c, 0x08, 0x61, 0xbf, 0x04, 0xda, 0x5b, 0x9c, 0xad, 0xac, + 0xdc, 0x5f, 0x7c, 0xcd, 0x89, 0xb6, 0xe4, 0x10, 0xd4, 0x2b, 0xeb, 0x0d, 0x05, 0xc0, 0x18, 0xc7, + 0xfe, 0x2c, 0x4c, 0xbe, 0x12, 0x38, 0xdd, 0x2d, 0x97, 0xdf, 0x98, 0xb0, 0x93, 0xf9, 0x33, 0x30, + 0xee, 0x34, 0x9b, 0x59, 0x69, 0x86, 0xaa, 0xa2, 0x18, 0x15, 0xfc, 0x48, 0x87, 0x70, 0xfb, 0xdf, + 0x5a, 0x40, 0xe2, 0x3b, 0x6e, 0xd7, 0x6b, 0xad, 0x3a, 0x51, 0x63, 0x8b, 0x1d, 0xe1, 0xb6, 0x78, + 0x69, 0xd6, 0x11, 0xee, 0xa6, 0x86, 0xa0, 0x81, 0x45, 0xde, 0x82, 0x8a, 0xf8, 0xf7, 0xba, 0x3e, + 0x20, 0x0e, 0xef, 0xf4, 0xce, 0xf7, 0x3c, 0x2e, 0x93, 0x18, 0x85, 0x37, 0x63, 0x0e, 0x68, 0xb2, + 0x63, 0x4d, 0xb5, 0xec, 0x6d, 0xb6, 0x7b, 0x0f, 0x9b, 0x1b, 0x71, 0x53, 0x75, 0x03, 0x7f, 0xd3, + 0x6d, 0xd3, 0x74, 0x53, 0xad, 0x89, 0x62, 0x54, 0xf0, 0xa3, 0x35, 0xd5, 0xbf, 0xb1, 0xe0, 0xfc, + 0x72, 0x18, 0xb9, 0xfe, 0x12, 0x0d, 0x23, 0xb6, 0xf3, 0xb1, 0xf5, 0xb1, 0xd7, 0x3e, 0x4a, 0xe0, + 0xc7, 0x12, 0x4c, 0xcb, 0x1b, 0xf0, 0xde, 0x46, 0x48, 0x23, 0xe3, 0xa8, 0xa1, 0xe7, 0xf1, 0x62, + 0x0a, 0x8e, 0x7d, 0x35, 0x18, 0x15, 0x79, 0x15, 0x1e, 0x53, 0x29, 0x24, 0xa9, 0xd4, 0x53, 0x70, + 0xec, 0xab, 0x61, 0xff, 0xa0, 0x00, 0xe7, 0xf8, 0x67, 0xa4, 0x82, 0xb6, 0xbe, 0x31, 0x28, 0x68, + 0x6b, 0xc8, 0xa9, 0xcc, 0x79, 0x9d, 0x20, 0x64, 0xeb, 0xaf, 0x5b, 0x30, 0xd5, 0x4c, 0xb6, 0x74, + 0x3e, 0x16, 0xc1, 0xac, 0x3e, 0x14, 0xbe, 0x8f, 0xa9, 0x42, 0x4c, 0xf3, 0x27, 0xbf, 0x66, 0xc1, + 0x54, 0x52, 0x4c, 0xb5, 0xba, 0x9f, 0x42, 0x23, 0xe9, 0x60, 0x85, 0x64, 0x79, 0x88, 0x69, 0x11, + 0xec, 0xef, 0x8f, 0xc8, 0x2e, 0x3d, 0x8d, 0x88, 0x24, 0xf2, 0x00, 0xca, 0x51, 0x3b, 0x14, 0x85, + 0xf2, 0x6b, 0x87, 0x3c, 0xb4, 0xae, 0xaf, 0xd4, 0x85, 0xab, 0x4b, 0xac, 0x57, 0xca, 0x12, 0xa6, + 0x1f, 0x2b, 0x5e, 0x9c, 0x71, 0xa3, 0x2b, 0x19, 0xe7, 0x72, 0x5a, 0x5e, 0x5f, 0x5c, 0x4b, 0x33, + 0x96, 0x25, 0x8c, 0xb1, 0xe2, 0x65, 0xff, 0x63, 0x0b, 0xca, 0xb7, 0x7c, 0xb5, 0x8e, 0xfc, 0x5c, + 0x0e, 0xb6, 0x28, 0xad, 0xb2, 0x6a, 0xa5, 0x25, 0x3e, 0x05, 0xbd, 0x9c, 0xb0, 0x44, 0x3d, 0x69, + 0xd0, 0x9e, 0xe7, 0xd9, 0x16, 0x19, 0xa9, 0x5b, 0xfe, 0xc6, 0x40, 0xc3, 0xf5, 0xaf, 0x17, 0xe1, + 0xcc, 0xab, 0xce, 0x2e, 0xf5, 0x22, 0xe7, 0xf8, 0x9b, 0xc4, 0x0b, 0x50, 0x71, 0xba, 0xfc, 0x16, + 0xd5, 0x38, 0x86, 0xc4, 0xc6, 0x9d, 0x18, 0x84, 0x26, 0x5e, 0xbc, 0xa0, 0x89, 0xf0, 0xa0, 0xac, + 0xa5, 0x68, 0x31, 0x05, 0xc7, 0xbe, 0x1a, 0xe4, 0x16, 0x10, 0x19, 0x52, 0x5f, 0x6d, 0x34, 0xfc, + 0x9e, 0x27, 0x96, 0x34, 0x61, 0xf7, 0xd1, 0xe7, 0xe1, 0xd5, 0x3e, 0x0c, 0xcc, 0xa8, 0x45, 0x3e, + 0x03, 0x33, 0x0d, 0x4e, 0x59, 0x9e, 0x8e, 0x4c, 0x8a, 0xe2, 0x84, 0xac, 0x03, 0x6e, 0x16, 0x07, + 0xe0, 0xe1, 0x40, 0x0a, 0x4c, 0xd2, 0x30, 0xf2, 0x03, 0xa7, 0x45, 0x4d, 0xba, 0x63, 0x49, 0x49, + 0xeb, 0x7d, 0x18, 0x98, 0x51, 0x8b, 0x7c, 0x01, 0xca, 0xd1, 0x56, 0x40, 0xc3, 0x2d, 0xbf, 0xdd, + 0x94, 0xe6, 0xdd, 0x21, 0x8d, 0x81, 0xb2, 0xf7, 0xd7, 0x15, 0x55, 0x63, 0x78, 0xab, 0x22, 0x8c, + 0x79, 0x92, 0x00, 0xc6, 0xc2, 0x86, 0xdf, 0xa5, 0xa1, 0x3c, 0x55, 0xdc, 0xca, 0x85, 0x3b, 0x37, + 0x6e, 0x19, 0x66, 0x48, 0xce, 0x01, 0x25, 0x27, 0xfb, 0x77, 0x46, 0x60, 0xc2, 0x44, 0x3c, 0xc2, + 0xda, 0xf4, 0x65, 0x0b, 0x26, 0x1a, 0xbe, 0x17, 0x05, 0x7e, 0x3b, 0x4e, 0x15, 0x31, 0xbc, 0x46, + 0xc1, 0x48, 0x2d, 0xd1, 0xc8, 0x71, 0xdb, 0x86, 0xb5, 0xce, 0x60, 0x83, 0x09, 0xa6, 0xe4, 0xeb, + 0x16, 0x4c, 0xc5, 0x2e, 0x99, 0xb1, 0xad, 0x2f, 0x57, 0x41, 0xf4, 0x52, 0x7f, 0x3d, 0xc9, 0x09, + 0xd3, 0xac, 0xed, 0x0d, 0x98, 0x4e, 0xf7, 0x36, 0x6b, 0xca, 0xae, 0x23, 0xe7, 0x7a, 0x21, 0x6e, + 0xca, 0x35, 0x27, 0x0c, 0x91, 0x43, 0xc8, 0xb3, 0x50, 0xea, 0x38, 0x41, 0xcb, 0xf5, 0x9c, 0x36, + 0x6f, 0xc5, 0x82, 0xb1, 0x20, 0xc9, 0x72, 0xd4, 0x18, 0xf6, 0x47, 0x60, 0x62, 0xd5, 0xf1, 0x5a, + 0xb4, 0x29, 0xd7, 0xe1, 0xc3, 0x63, 0x62, 0xff, 0x68, 0x14, 0x2a, 0xc6, 0xf1, 0xf1, 0xf4, 0xcf, + 0x59, 0x89, 0x14, 0x48, 0x85, 0x1c, 0x53, 0x20, 0x7d, 0x0a, 0x60, 0xd3, 0xf5, 0xdc, 0x70, 0xeb, + 0x84, 0xc9, 0x95, 0xb8, 0x57, 0xc0, 0x0d, 0x4d, 0x01, 0x0d, 0x6a, 0xf1, 0xd5, 0x6b, 0xf1, 0x80, + 0x3c, 0x85, 0xef, 0x58, 0xc6, 0x76, 0x33, 0x96, 0x87, 0xab, 0x89, 0xd1, 0x31, 0xf3, 0x6a, 0xfb, + 0x11, 0xb7, 0x62, 0x07, 0xed, 0x4a, 0xeb, 0x50, 0x0a, 0x68, 0xd8, 0xeb, 0xd0, 0x13, 0xa5, 0x41, + 0xe2, 0x4e, 0x3f, 0x28, 0xeb, 0xa3, 0xa6, 0x34, 0xfb, 0x12, 0x9c, 0x49, 0x88, 0x70, 0xac, 0x1b, + 0x26, 0x1f, 0x32, 0x6d, 0x14, 0x27, 0xb9, 0x6f, 0x62, 0x7d, 0xd1, 0x36, 0xd2, 0x1f, 0xe9, 0xbe, + 0x10, 0xae, 0x5d, 0x02, 0x66, 0xff, 0xd9, 0x38, 0x48, 0xef, 0x89, 0x23, 0x2c, 0x57, 0xe6, 0x9d, + 0xe9, 0xc8, 0x09, 0xee, 0x4c, 0x6f, 0xc1, 0x84, 0xeb, 0xb9, 0x91, 0xeb, 0xb4, 0xb9, 0xfd, 0x49, + 0x6e, 0xa7, 0x2a, 0x0c, 0x60, 0x62, 0xd9, 0x80, 0x65, 0xd0, 0x49, 0xd4, 0x25, 0xaf, 0x41, 0x91, + 0xef, 0x37, 0x72, 0x00, 0x1f, 0xdf, 0xc5, 0x83, 0x7b, 0xf7, 0x88, 0xd8, 0x40, 0x41, 0x89, 0x1f, + 0x3e, 0x44, 0xfe, 0x27, 0x7d, 0xfc, 0x96, 0xe3, 0x38, 0x3e, 0x7c, 0xa4, 0xe0, 0xd8, 0x57, 0x83, + 0x51, 0xd9, 0x74, 0xdc, 0x76, 0x2f, 0xa0, 0x31, 0x95, 0xb1, 0x24, 0x95, 0x1b, 0x29, 0x38, 0xf6, + 0xd5, 0x20, 0x9b, 0x30, 0x21, 0xcb, 0x84, 0xc3, 0xde, 0xf8, 0x09, 0xbf, 0x92, 0x3b, 0x66, 0xde, + 0x30, 0x28, 0x61, 0x82, 0x2e, 0xe9, 0xc1, 0x59, 0xd7, 0x6b, 0xf8, 0x5e, 0xa3, 0xdd, 0x0b, 0xdd, + 0x1d, 0x1a, 0x07, 0xe6, 0x9d, 0x84, 0xd9, 0x85, 0xfd, 0xbd, 0xb9, 0xb3, 0xcb, 0x69, 0x72, 0xd8, + 0xcf, 0x81, 0x7c, 0xd1, 0x82, 0x0b, 0x0d, 0xdf, 0x0b, 0x79, 0xfe, 0x90, 0x1d, 0x7a, 0x3d, 0x08, + 0xfc, 0x40, 0xf0, 0x2e, 0x9f, 0x90, 0x37, 0x37, 0x7b, 0x2e, 0x66, 0x91, 0xc4, 0x6c, 0x4e, 0xe4, + 0x4d, 0x28, 0x75, 0x03, 0x7f, 0xc7, 0x6d, 0xd2, 0x40, 0x3a, 0x7f, 0xae, 0xe4, 0x91, 0x54, 0x69, + 0x4d, 0xd2, 0x8c, 0x97, 0x1e, 0x55, 0x82, 0x9a, 0x1f, 0xf9, 0x8a, 0x05, 0x17, 0x0d, 0xa9, 0xe4, + 0xb0, 0x12, 0x2d, 0x50, 0x39, 0x61, 0x0b, 0x70, 0x53, 0xf8, 0x62, 0x36, 0x51, 0x1c, 0xc4, 0xcd, + 0xfe, 0xb3, 0x0a, 0x4c, 0x26, 0x05, 0x27, 0xbf, 0x00, 0xd0, 0x0d, 0xfc, 0x0e, 0x8d, 0xb6, 0xa8, + 0x0e, 0xf5, 0xba, 0x3d, 0x6c, 0x02, 0x1f, 0x45, 0x4f, 0xb9, 0x6e, 0xb1, 0x85, 0x2b, 0x2e, 0x45, + 0x83, 0x23, 0x09, 0x60, 0x7c, 0x5b, 0x28, 0x00, 0x52, 0x1f, 0x7a, 0x35, 0x17, 0xed, 0x4d, 0x72, + 0xe6, 0x31, 0x4a, 0xb2, 0x08, 0x15, 0x23, 0xb2, 0x01, 0x85, 0x07, 0x74, 0x23, 0x9f, 0xec, 0x11, + 0xf7, 0xa8, 0x3c, 0x57, 0xd5, 0xc6, 0xf7, 0xf7, 0xe6, 0x0a, 0xf7, 0xe8, 0x06, 0x32, 0xe2, 0xec, + 0xbb, 0x9a, 0xc2, 0x7f, 0x43, 0x2e, 0x5a, 0xaf, 0xe6, 0xe8, 0x0c, 0x22, 0xbe, 0x4b, 0x16, 0xa1, + 0x62, 0x44, 0xde, 0x84, 0xf2, 0x03, 0x67, 0x87, 0x6e, 0x06, 0xbe, 0x17, 0x49, 0x7f, 0xc1, 0x21, + 0x03, 0x6c, 0xee, 0x29, 0x72, 0x92, 0x2f, 0x57, 0x34, 0x74, 0x21, 0xc6, 0xec, 0xc8, 0x0e, 0x94, + 0x3c, 0xfa, 0x00, 0x69, 0xdb, 0x6d, 0xe4, 0x13, 0xd0, 0x72, 0x5b, 0x52, 0x93, 0x9c, 0xf9, 0x0e, + 0xac, 0xca, 0x50, 0xf3, 0x62, 0x7d, 0x79, 0xdf, 0xdf, 0xc8, 0xc7, 0xad, 0x44, 0x9f, 0x91, 0x45, + 0x5f, 0xde, 0xf2, 0x37, 0x90, 0x11, 0x67, 0x73, 0xa4, 0xa1, 0x9d, 0xd5, 0xe4, 0x82, 0x79, 0x3b, + 0x5f, 0x27, 0x3d, 0x31, 0x47, 0xe2, 0x52, 0x34, 0x38, 0xb2, 0xb6, 0x6d, 0x49, 0xb3, 0xa9, 0x5c, + 0x32, 0x87, 0x6c, 0xdb, 0xa4, 0x11, 0x56, 0xb4, 0xad, 0x2a, 0x43, 0xcd, 0x8b, 0xf1, 0x75, 0xa5, + 0x0d, 0x32, 0x9f, 0x45, 0x33, 0x69, 0xd1, 0x14, 0x7c, 0x55, 0x19, 0x6a, 0x5e, 0xac, 0xbd, 0xc3, + 0xed, 0xdd, 0x07, 0x4e, 0x7b, 0xdb, 0xf5, 0x5a, 0x72, 0x89, 0x1c, 0x36, 0xd4, 0x6f, 0x7b, 0xf7, + 0x9e, 0xa0, 0x67, 0xb6, 0x77, 0x5c, 0x8a, 0x06, 0x47, 0xf2, 0x77, 0x2c, 0x1d, 0x8e, 0x34, 0x91, + 0x87, 0x23, 0x57, 0x72, 0xc9, 0x95, 0xd1, 0x49, 0x42, 0x65, 0xfd, 0x69, 0xed, 0x7b, 0xca, 0x0b, + 0xbf, 0xf6, 0x87, 0x73, 0x33, 0xd4, 0x6b, 0xf8, 0x4d, 0xd7, 0x6b, 0x2d, 0xdc, 0x0f, 0x7d, 0x6f, + 0x1e, 0x9d, 0x07, 0xea, 0xb4, 0x20, 0x65, 0x9a, 0xfd, 0x18, 0x54, 0x0c, 0x12, 0x87, 0xa9, 0x9c, + 0x13, 0xa6, 0xca, 0xf9, 0x93, 0x31, 0x98, 0x30, 0x73, 0xb1, 0x1e, 0x41, 0x0f, 0xd4, 0x67, 0x9f, + 0x91, 0xe3, 0x9c, 0x7d, 0xd8, 0x61, 0xd7, 0xb8, 0x6a, 0x53, 0x86, 0xb6, 0xe5, 0xdc, 0x54, 0xff, + 0xf8, 0xb0, 0x6b, 0x14, 0x86, 0x98, 0x60, 0x7a, 0x0c, 0xef, 0x1b, 0xa6, 0x40, 0x0b, 0x15, 0xb3, + 0x98, 0x54, 0xa0, 0x13, 0x4a, 0xe3, 0x35, 0x80, 0x38, 0x69, 0xa8, 0xbc, 0x82, 0xd5, 0x9a, 0xb9, + 0x91, 0xcc, 0xd4, 0xc0, 0x22, 0x4f, 0xc3, 0x18, 0x53, 0xc2, 0x68, 0x53, 0x66, 0x56, 0xd0, 0x16, + 0x85, 0x1b, 0xbc, 0x14, 0x25, 0x94, 0xbc, 0xc8, 0xf4, 0xe5, 0x58, 0x75, 0x92, 0x09, 0x13, 0xce, + 0xc7, 0xfa, 0x72, 0x0c, 0xc3, 0x04, 0x26, 0x13, 0x9d, 0x32, 0x4d, 0x87, 0xaf, 0x0d, 0x86, 0xe8, + 0x5c, 0xfd, 0x41, 0x01, 0xe3, 0x16, 0xae, 0x94, 0x66, 0xc4, 0xe7, 0x74, 0xd1, 0xb0, 0x70, 0xa5, + 0xe0, 0xd8, 0x57, 0x83, 0x7d, 0x8c, 0xbc, 0x3d, 0xae, 0x08, 0xa7, 0xf1, 0x01, 0xf7, 0xbe, 0xbf, + 0x68, 0x9e, 0xfa, 0x72, 0x9c, 0x43, 0x62, 0xd4, 0x1e, 0xe3, 0xd8, 0x77, 0x0b, 0x48, 0xbf, 0x32, + 0x24, 0xe3, 0x55, 0xb4, 0xa1, 0xab, 0x5f, 0x8f, 0xc2, 0x8c, 0x5a, 0xc3, 0x1d, 0xf6, 0xbe, 0x62, + 0xc1, 0x64, 0x72, 0x4b, 0xcb, 0xfb, 0x42, 0x87, 0xfc, 0x7f, 0x30, 0x1e, 0xb9, 0x1d, 0xea, 0xf7, + 0x84, 0x09, 0xa1, 0x20, 0xb4, 0x84, 0x75, 0x51, 0x84, 0x0a, 0x66, 0xff, 0xfd, 0x31, 0x38, 0x77, + 0xbb, 0xe5, 0x7a, 0xe9, 0x5c, 0x7b, 0x59, 0x0f, 0x6b, 0x58, 0xc7, 0x7e, 0x58, 0x43, 0xc7, 0x42, + 0xca, 0x67, 0x2b, 0xb2, 0x63, 0x21, 0xd5, 0x1b, 0x22, 0x49, 0x5c, 0xf2, 0x07, 0x16, 0x3c, 0xe9, + 0x34, 0xc5, 0xa9, 0xc8, 0x69, 0xcb, 0x52, 0x23, 0x1f, 0xbc, 0x5c, 0x45, 0xc2, 0x21, 0x35, 0x8b, + 0xfe, 0x8f, 0x9f, 0xaf, 0x1e, 0xc0, 0x55, 0x8c, 0xb2, 0x9f, 0x92, 0x5f, 0xf0, 0xe4, 0x41, 0xa8, + 0x78, 0xa0, 0xf8, 0xe4, 0x2f, 0xc3, 0x54, 0xe2, 0x83, 0xe5, 0x3d, 0x40, 0x59, 0x5c, 0xd7, 0xd4, + 0x93, 0x20, 0x4c, 0xe3, 0x92, 0xef, 0x5b, 0x30, 0x23, 0x8c, 0xce, 0x19, 0x4d, 0x23, 0xee, 0xa9, + 0xfd, 0xfc, 0x9b, 0x66, 0x71, 0x00, 0x47, 0xd1, 0x2c, 0xb1, 0x15, 0x7a, 0x00, 0x1a, 0x0e, 0x14, + 0x79, 0xf6, 0x0e, 0x7c, 0xf0, 0xd0, 0x76, 0x3f, 0xd6, 0xeb, 0x01, 0xaf, 0xc2, 0xa5, 0x03, 0xa5, + 0x3d, 0xd6, 0x8c, 0xfd, 0x9e, 0x05, 0x13, 0x66, 0xce, 0x30, 0xf2, 0x2c, 0x94, 0x22, 0x7f, 0x9b, + 0x7a, 0x77, 0x03, 0xe5, 0x45, 0xae, 0x57, 0x9e, 0x75, 0x5e, 0x8e, 0x2b, 0xa8, 0x31, 0x18, 0x76, + 0xa3, 0xed, 0x52, 0x2f, 0x5a, 0x6e, 0xca, 0x39, 0xa0, 0xb1, 0x17, 0x45, 0xf9, 0x12, 0x6a, 0x0c, + 0xe1, 0x7e, 0xc9, 0x7e, 0x0b, 0x3f, 0x66, 0x69, 0x2d, 0x31, 0xdc, 0x2f, 0x63, 0x18, 0x26, 0x30, + 0x89, 0xad, 0xad, 0xdf, 0xa3, 0xf1, 0x95, 0x57, 0xca, 0x5a, 0xfd, 0x1d, 0x0b, 0xca, 0xe2, 0xf6, + 0x06, 0xe9, 0x66, 0xca, 0xef, 0x3b, 0x65, 0x5f, 0xaa, 0xae, 0x2d, 0x67, 0xf9, 0x7d, 0x5f, 0x81, + 0xd1, 0x6d, 0xd7, 0x53, 0x5f, 0xa2, 0xf5, 0x84, 0x57, 0x5d, 0xaf, 0x89, 0x1c, 0xa2, 0x35, 0x89, + 0xc2, 0x40, 0x4d, 0x62, 0x01, 0xca, 0xda, 0x27, 0x49, 0xee, 0xc7, 0xb1, 0xfb, 0xb6, 0x02, 0x60, + 0x8c, 0x63, 0xff, 0x86, 0x05, 0x93, 0x3c, 0x8d, 0x41, 0x6c, 0x2a, 0x79, 0x41, 0xbb, 0x09, 0x0a, + 0xb9, 0x2f, 0x25, 0xdd, 0x04, 0xdf, 0xdd, 0x9b, 0xab, 0x88, 0xc4, 0x07, 0x49, 0xaf, 0xc1, 0x4f, + 0x4b, 0xfb, 0x2a, 0x77, 0x66, 0x1c, 0x39, 0xb6, 0xf9, 0x2f, 0x16, 0x53, 0x11, 0xc1, 0x98, 0x9e, + 0xfd, 0x16, 0x4c, 0x98, 0x11, 0x82, 0xe4, 0x05, 0xa8, 0x74, 0x5d, 0xaf, 0x95, 0x8c, 0x24, 0xd7, + 0x77, 0x50, 0x6b, 0x31, 0x08, 0x4d, 0x3c, 0x5e, 0xcd, 0x8f, 0xab, 0xa5, 0xae, 0xae, 0xd6, 0x7c, + 0xb3, 0x5a, 0xfc, 0xc7, 0xf6, 0x00, 0xe2, 0x70, 0xf7, 0x23, 0xd9, 0xf5, 0xc6, 0xc4, 0xb5, 0x90, + 0xd0, 0x0e, 0x79, 0xea, 0x92, 0x31, 0x31, 0xc2, 0xdf, 0xdd, 0x3b, 0x48, 0xfb, 0x14, 0xb5, 0xf8, + 0xc3, 0x28, 0x19, 0x91, 0xaf, 0xb9, 0x3f, 0x8c, 0x92, 0xc1, 0xe3, 0xbd, 0x7b, 0x18, 0x25, 0x4b, + 0x98, 0x3f, 0x5f, 0x0f, 0xa3, 0x7c, 0x12, 0x8e, 0x9b, 0x23, 0x99, 0x29, 0x7b, 0x0f, 0xcc, 0x5c, + 0x26, 0xba, 0xc5, 0x65, 0x32, 0x13, 0x09, 0xb5, 0x7f, 0x77, 0x14, 0xa6, 0xd3, 0x36, 0x9f, 0xbc, + 0x1d, 0x7b, 0xc8, 0xd7, 0x2d, 0x98, 0x74, 0x12, 0xf9, 0x28, 0x73, 0x7a, 0x65, 0x2d, 0x41, 0xd3, + 0xc8, 0x87, 0x98, 0x28, 0xc7, 0x14, 0x6f, 0x53, 0xd7, 0x1a, 0x1d, 0xac, 0x6b, 0xb1, 0x4d, 0xc0, + 0xe5, 0x7a, 0x64, 0x40, 0xa5, 0x93, 0xfa, 0x74, 0x6c, 0x44, 0x17, 0xe5, 0xa8, 0x31, 0xc8, 0x43, + 0x18, 0x17, 0x2e, 0x40, 0xca, 0xd7, 0x6b, 0x35, 0x27, 0xdb, 0x94, 0xf0, 0x32, 0x8a, 0xbb, 0x40, + 0xfc, 0x0f, 0x51, 0xb1, 0x63, 0xfa, 0x3a, 0x04, 0x8e, 0xd7, 0xa2, 0xbc, 0xcd, 0xa5, 0x35, 0xe5, + 0xf5, 0xbc, 0xcc, 0x80, 0xa8, 0x29, 0x57, 0x83, 0x56, 0x28, 0x23, 0x4d, 0x75, 0x19, 0x1a, 0x9c, + 0xed, 0x5f, 0xb1, 0x60, 0x66, 0x50, 0x45, 0x36, 0x50, 0xf8, 0xaa, 0x2b, 0x47, 0x94, 0x91, 0xe0, + 0xc2, 0x09, 0x22, 0x14, 0x30, 0x72, 0x09, 0x0a, 0x54, 0x6f, 0x54, 0x3a, 0x1b, 0xe7, 0x75, 0xaf, + 0x89, 0xac, 0x9c, 0x5c, 0x83, 0xd1, 0x30, 0xa2, 0xdd, 0x54, 0x14, 0xc7, 0x28, 0x5b, 0x3c, 0x33, + 0xae, 0x21, 0x38, 0xae, 0xfd, 0x11, 0x38, 0x66, 0x4a, 0x6d, 0xfb, 0x3a, 0x10, 0xf4, 0xdb, 0xed, + 0x0d, 0xa7, 0xb1, 0x7d, 0xcf, 0xf5, 0x9a, 0xfe, 0x03, 0xbe, 0x31, 0x2c, 0x40, 0x39, 0x90, 0x51, + 0xf5, 0xa1, 0x9c, 0x53, 0x7a, 0x67, 0x51, 0xe1, 0xf6, 0x21, 0xc6, 0x38, 0xf6, 0xf7, 0x47, 0x60, + 0x5c, 0xa6, 0x80, 0x78, 0x04, 0x21, 0x44, 0xdb, 0x09, 0xc7, 0x8d, 0xe5, 0x5c, 0x32, 0x57, 0x0c, + 0x8c, 0x1f, 0x0a, 0x53, 0xf1, 0x43, 0xaf, 0xe6, 0xc3, 0xee, 0xe0, 0xe0, 0xa1, 0xef, 0x16, 0x61, + 0x2a, 0x95, 0x52, 0x23, 0x95, 0x7d, 0xdf, 0x7a, 0x4f, 0xb2, 0xef, 0x93, 0x30, 0xf1, 0x02, 0x43, + 0x7e, 0x0e, 0xc7, 0x7f, 0xf1, 0x18, 0x43, 0x5e, 0xae, 0xe0, 0xc5, 0xf7, 0x8f, 0x2b, 0xf8, 0x7f, + 0xb5, 0xe0, 0xf1, 0x81, 0x89, 0x61, 0x78, 0x8a, 0xc5, 0x20, 0x09, 0x95, 0xeb, 0x45, 0xce, 0xc9, + 0xb6, 0xb4, 0x93, 0x47, 0x3a, 0x2b, 0x5e, 0x9a, 0x3d, 0x79, 0x1e, 0x26, 0xf8, 0xda, 0xcc, 0x56, + 0x4e, 0xb6, 0xf6, 0x8a, 0x3b, 0x6a, 0x7e, 0x5b, 0x59, 0x37, 0xca, 0x31, 0x81, 0x65, 0x7f, 0xdb, + 0x82, 0x99, 0x41, 0x09, 0xf7, 0x8e, 0xa0, 0xe7, 0xfe, 0xa5, 0x54, 0x08, 0xd6, 0x5c, 0x5f, 0x08, + 0x56, 0xca, 0x72, 0xa9, 0xa2, 0xad, 0x0c, 0xa3, 0x61, 0xe1, 0x90, 0x08, 0xa3, 0xdf, 0x2b, 0xc0, + 0xb4, 0x14, 0x31, 0x3e, 0xa2, 0xbc, 0x98, 0x08, 0x1c, 0xfb, 0xa9, 0x54, 0xe0, 0xd8, 0xf9, 0x34, + 0xfe, 0x5f, 0x44, 0x8d, 0xbd, 0xbf, 0xa2, 0xc6, 0xbe, 0x56, 0x84, 0x0b, 0x99, 0xa9, 0xed, 0xc8, + 0x57, 0x33, 0x76, 0x8a, 0x7b, 0x39, 0xe7, 0xd0, 0xd3, 0xa1, 0xed, 0xa7, 0x1b, 0x6a, 0xf5, 0x6b, + 0x66, 0x88, 0x93, 0x58, 0xfd, 0x37, 0x4f, 0x21, 0x1b, 0xe0, 0x71, 0xa3, 0x9d, 0x1e, 0xed, 0xeb, + 0x84, 0x7f, 0x0e, 0x96, 0xfa, 0xaf, 0x15, 0xe0, 0xea, 0x51, 0x5b, 0xf6, 0x7d, 0x1a, 0x1e, 0x1c, + 0x26, 0xc2, 0x83, 0x1f, 0x91, 0x6a, 0x73, 0x2a, 0x91, 0xc2, 0x7f, 0x6f, 0x54, 0xef, 0xbb, 0xfd, + 0x13, 0xf6, 0x48, 0x96, 0x97, 0x71, 0xa6, 0xfa, 0xaa, 0x37, 0x1c, 0xe2, 0xbd, 0x61, 0xbc, 0x2e, + 0x8a, 0xdf, 0xdd, 0x9b, 0x3b, 0x1b, 0xe7, 0x80, 0x92, 0x85, 0xa8, 0x2a, 0x91, 0xab, 0x50, 0x0a, + 0x04, 0x54, 0x05, 0x44, 0x4a, 0xb7, 0x34, 0x51, 0x86, 0x1a, 0x4a, 0xbe, 0x60, 0x9c, 0x15, 0x46, + 0x4f, 0x2b, 0xd5, 0xd9, 0x41, 0xd7, 0x2e, 0x6f, 0x40, 0x29, 0x54, 0x0f, 0x0d, 0x88, 0xe9, 0xf4, + 0xdc, 0x11, 0xe3, 0x6c, 0x9d, 0x0d, 0xda, 0x56, 0xaf, 0x0e, 0x88, 0xef, 0xd3, 0x6f, 0x12, 0x68, + 0x92, 0xc4, 0xd6, 0x96, 0x09, 0x71, 0x07, 0x07, 0xfd, 0x56, 0x09, 0x12, 0xc1, 0xb8, 0x7c, 0x6d, + 0x5c, 0x1e, 0x67, 0x57, 0x73, 0x0a, 0x58, 0x93, 0xe1, 0x0c, 0xfc, 0xc0, 0xaf, 0x2c, 0x72, 0x8a, + 0x95, 0xfd, 0x43, 0x0b, 0x2a, 0x72, 0x8c, 0x3c, 0x82, 0x80, 0xe3, 0xfb, 0xc9, 0x80, 0xe3, 0xeb, + 0xb9, 0x2c, 0xe1, 0x03, 0xa2, 0x8d, 0xef, 0xc3, 0x84, 0x99, 0x64, 0x96, 0x7c, 0xca, 0xd8, 0x82, + 0xac, 0x61, 0x12, 0x29, 0xaa, 0x4d, 0x2a, 0xde, 0x9e, 0xec, 0x7f, 0x52, 0xd6, 0xad, 0xc8, 0x0f, + 0xce, 0xe6, 0xc8, 0xb7, 0x0e, 0x1c, 0xf9, 0xe6, 0xc0, 0x1b, 0xc9, 0x7f, 0xe0, 0xbd, 0x06, 0x25, + 0xb5, 0x2c, 0x4a, 0x6d, 0xea, 0x29, 0x33, 0xbe, 0x81, 0xa9, 0x64, 0x8c, 0x98, 0x31, 0x5d, 0xf8, + 0x01, 0x38, 0xbe, 0x27, 0x50, 0xcb, 0xb5, 0x26, 0x43, 0xde, 0x84, 0xca, 0x03, 0x3f, 0xd8, 0x6e, + 0xfb, 0x0e, 0x7f, 0xdd, 0x05, 0xf2, 0x70, 0x64, 0xd1, 0xb6, 0x7e, 0x11, 0x64, 0x76, 0x2f, 0xa6, + 0x8f, 0x26, 0x33, 0x52, 0x85, 0xa9, 0x8e, 0xeb, 0x21, 0x75, 0x9a, 0x3a, 0xae, 0x78, 0x54, 0xbc, + 0xac, 0xa0, 0x74, 0xfb, 0xd5, 0x24, 0x18, 0xd3, 0xf8, 0xdc, 0x2e, 0x17, 0x24, 0x4c, 0x1d, 0x32, + 0x7d, 0xfa, 0xda, 0xf0, 0x83, 0x31, 0x69, 0x3e, 0x11, 0x51, 0x56, 0xc9, 0x72, 0x4c, 0xf1, 0x26, + 0x9f, 0x87, 0x52, 0xa8, 0xde, 0xf1, 0x2d, 0xe6, 0x78, 0xea, 0xd1, 0x6f, 0xf9, 0xea, 0xae, 0xd4, + 0x8f, 0xf9, 0x6a, 0x86, 0x64, 0x05, 0xce, 0x2b, 0xdb, 0x4d, 0xe2, 0x49, 0xd2, 0xb1, 0x38, 0x05, + 0x20, 0x66, 0xc0, 0x31, 0xb3, 0x16, 0xd3, 0x6d, 0x79, 0xf2, 0x66, 0xe1, 0x38, 0x60, 0xdc, 0xb5, + 0xf3, 0xf9, 0xd7, 0x44, 0x09, 0x3d, 0x28, 0x6c, 0xbe, 0x34, 0x44, 0xd8, 0x7c, 0x1d, 0x2e, 0xa4, + 0x41, 0x3c, 0xb7, 0x23, 0x4f, 0x27, 0x69, 0x6c, 0xa1, 0x6b, 0x59, 0x48, 0x98, 0x5d, 0x97, 0xdc, + 0x83, 0x72, 0x40, 0xf9, 0x29, 0xaf, 0xaa, 0xbc, 0x3f, 0x8f, 0xed, 0xe7, 0x8e, 0x8a, 0x00, 0xc6, + 0xb4, 0x58, 0xbf, 0x3b, 0xc9, 0xb7, 0x0e, 0x5e, 0xcb, 0xf1, 0xa1, 0x7f, 0xd9, 0xf7, 0x03, 0x72, + 0xae, 0xda, 0xff, 0x6e, 0x0a, 0xce, 0x24, 0x0c, 0x50, 0xe4, 0x29, 0x28, 0xf2, 0x64, 0x97, 0x7c, + 0xb5, 0x2a, 0xc5, 0x2b, 0xaa, 0x68, 0x1c, 0x01, 0x23, 0xbf, 0x6c, 0xc1, 0x54, 0x37, 0x71, 0xbd, + 0xa5, 0x16, 0xf2, 0x21, 0x6d, 0xda, 0xc9, 0x3b, 0x33, 0xe3, 0x95, 0xa0, 0x24, 0x33, 0x4c, 0x73, + 0x67, 0xeb, 0x81, 0x0c, 0x16, 0x69, 0xd3, 0x80, 0x63, 0x4b, 0x45, 0x4f, 0x93, 0x58, 0x4c, 0x82, + 0x31, 0x8d, 0xcf, 0x7a, 0x98, 0x7f, 0xdd, 0x30, 0x8f, 0x39, 0x57, 0x15, 0x01, 0x8c, 0x69, 0x91, + 0x97, 0x61, 0x52, 0xa6, 0xb8, 0x5f, 0xf3, 0x9b, 0x37, 0x9d, 0x70, 0x4b, 0x1e, 0xf9, 0xf4, 0x11, + 0x75, 0x31, 0x01, 0xc5, 0x14, 0x36, 0xff, 0xb6, 0xf8, 0x1d, 0x01, 0x4e, 0x60, 0x2c, 0xf9, 0x88, + 0xd2, 0x62, 0x12, 0x8c, 0x69, 0x7c, 0xf2, 0xac, 0xb1, 0x0d, 0x09, 0x67, 0x1e, 0xbd, 0x1a, 0x64, + 0x6c, 0x45, 0x55, 0x98, 0xea, 0xf1, 0x13, 0x72, 0x53, 0x01, 0xe5, 0x7c, 0xd4, 0x0c, 0xef, 0x26, + 0xc1, 0x98, 0xc6, 0x27, 0x2f, 0xc1, 0x99, 0x80, 0x2d, 0xb6, 0x9a, 0x80, 0xf0, 0xf0, 0xd1, 0xce, + 0x14, 0x68, 0x02, 0x31, 0x89, 0x4b, 0x5e, 0x81, 0xb3, 0x71, 0x1a, 0x64, 0x45, 0x40, 0xb8, 0xfc, + 0xe8, 0x9c, 0x9c, 0xd5, 0x34, 0x02, 0xf6, 0xd7, 0x21, 0x7f, 0x15, 0xa6, 0x8d, 0x96, 0x58, 0xf6, + 0x9a, 0xf4, 0xa1, 0x4c, 0x55, 0xcb, 0x1f, 0x05, 0x5c, 0x4c, 0xc1, 0xb0, 0x0f, 0x9b, 0x7c, 0x1c, + 0x26, 0x1b, 0x7e, 0xbb, 0xcd, 0xd7, 0x38, 0xf1, 0x80, 0x8f, 0xc8, 0x49, 0x2b, 0xb2, 0xf7, 0x26, + 0x20, 0x98, 0xc2, 0x24, 0xb7, 0x80, 0xf8, 0x1b, 0x4c, 0xbd, 0xa2, 0xcd, 0x57, 0xa8, 0x47, 0xa5, + 0xc6, 0x71, 0x26, 0x19, 0xaa, 0x76, 0xa7, 0x0f, 0x03, 0x33, 0x6a, 0xf1, 0x94, 0x9e, 0x46, 0x68, + 0xff, 0x64, 0x1e, 0x8f, 0x08, 0xa4, 0xed, 0x39, 0x87, 0xc6, 0xf5, 0x07, 0x30, 0x26, 0x3c, 0x22, + 0xf2, 0x49, 0x4e, 0x6b, 0xbe, 0xe5, 0x11, 0xef, 0x11, 0xa2, 0x14, 0x25, 0x27, 0xf2, 0x0b, 0x50, + 0xde, 0x50, 0x0f, 0x3b, 0xf1, 0x8c, 0xb4, 0x43, 0xef, 0x8b, 0xa9, 0x37, 0xca, 0x62, 0x7b, 0x85, + 0x06, 0x60, 0xcc, 0x92, 0x3c, 0x0d, 0x95, 0x9b, 0x6b, 0x55, 0x3d, 0x0a, 0xcf, 0xf2, 0xde, 0x1f, + 0x65, 0x55, 0xd0, 0x04, 0xb0, 0x19, 0xa6, 0xd5, 0x37, 0x92, 0x74, 0x9a, 0xc8, 0xd0, 0xc6, 0x18, + 0x36, 0x77, 0x91, 0xc1, 0xfa, 0xcc, 0xb9, 0x14, 0xb6, 0x2c, 0x47, 0x8d, 0x41, 0xde, 0x80, 0x8a, + 0xdc, 0x2f, 0xf8, 0xda, 0x74, 0xfe, 0x64, 0x69, 0x23, 0x30, 0x26, 0x81, 0x26, 0x3d, 0x7e, 0x7d, + 0xcf, 0xdf, 0xbb, 0xa1, 0x37, 0x7a, 0xed, 0xf6, 0xcc, 0x05, 0xbe, 0x6e, 0xc6, 0xd7, 0xf7, 0x31, + 0x08, 0x4d, 0x3c, 0xf2, 0x9c, 0x72, 0xaf, 0x7c, 0x2c, 0xe1, 0xcf, 0xa0, 0xdd, 0x2b, 0xb5, 0xd2, + 0x3d, 0x20, 0xb2, 0xec, 0xe2, 0x21, 0x7e, 0x8d, 0x1b, 0x30, 0xab, 0x34, 0xbe, 0xfe, 0x49, 0x32, + 0x33, 0x93, 0xb0, 0x1d, 0xcd, 0xde, 0x1b, 0x88, 0x89, 0x07, 0x50, 0x21, 0x1b, 0x50, 0x70, 0xda, + 0x1b, 0x33, 0x8f, 0xe7, 0xa1, 0xba, 0x56, 0x57, 0x6a, 0x72, 0x44, 0x71, 0x1f, 0xec, 0xea, 0x4a, + 0x0d, 0x19, 0x71, 0xe2, 0xc2, 0xa8, 0xd3, 0xde, 0x08, 0x67, 0x66, 0xf9, 0x9c, 0xcd, 0x8d, 0x49, + 0x6c, 0x3c, 0x58, 0xa9, 0x85, 0xc8, 0x59, 0xd8, 0x5f, 0x1c, 0xd1, 0xb7, 0x44, 0xfa, 0x7d, 0x80, + 0xb7, 0xcc, 0x09, 0x24, 0x8e, 0x3b, 0x77, 0x72, 0x9b, 0x40, 0x52, 0xbd, 0x38, 0x33, 0x70, 0xfa, + 0x74, 0xf5, 0x92, 0x91, 0x4b, 0x7a, 0xbf, 0xe4, 0xdb, 0x07, 0xe2, 0xf4, 0x9c, 0x5c, 0x30, 0xec, + 0x2f, 0x55, 0xb4, 0x15, 0x34, 0xe5, 0x26, 0x18, 0x40, 0xd1, 0x0d, 0x23, 0xd7, 0xcf, 0x31, 0x9b, + 0x42, 0xea, 0xd1, 0x00, 0x1e, 0xac, 0xc5, 0x01, 0x28, 0x58, 0x31, 0x9e, 0x5e, 0xcb, 0xf5, 0x1e, + 0xca, 0xcf, 0x7f, 0x2d, 0x77, 0x27, 0x37, 0xc1, 0x93, 0x03, 0x50, 0xb0, 0x22, 0xf7, 0xc5, 0xa0, + 0x2e, 0xe4, 0xd1, 0xd7, 0xd5, 0x95, 0x5a, 0x8a, 0x5f, 0x72, 0x70, 0xdf, 0x87, 0x42, 0xd8, 0x71, + 0xa5, 0xba, 0x34, 0x24, 0xaf, 0xfa, 0xea, 0x72, 0x16, 0xaf, 0xfa, 0xea, 0x32, 0x32, 0x26, 0xfc, + 0xaa, 0xdf, 0xe9, 0x6c, 0x38, 0x61, 0xe8, 0x34, 0xb5, 0x75, 0x66, 0xc8, 0xab, 0xfe, 0xaa, 0xa6, + 0x97, 0x62, 0xcd, 0xaf, 0xfa, 0x63, 0x28, 0x1a, 0x9c, 0xc9, 0x9b, 0x30, 0xee, 0x88, 0x87, 0x67, + 0x65, 0xc0, 0x48, 0x3e, 0xaf, 0x29, 0xa7, 0x24, 0xe0, 0x66, 0x1a, 0x09, 0x42, 0xc5, 0x90, 0xf1, + 0x8e, 0x02, 0x87, 0x6e, 0xba, 0xdb, 0xd2, 0x38, 0x54, 0x1f, 0xfa, 0x69, 0x24, 0x46, 0x2c, 0x8b, + 0xb7, 0x04, 0xa1, 0x62, 0x48, 0xbe, 0x62, 0xc1, 0x99, 0x8e, 0xe3, 0x39, 0x3a, 0x20, 0x39, 0x9f, + 0xb0, 0x75, 0x33, 0xc4, 0x39, 0xd6, 0x10, 0x57, 0x4d, 0x46, 0x98, 0xe4, 0x4b, 0x76, 0x60, 0xcc, + 0xe1, 0x4f, 0x62, 0xcb, 0xa3, 0x18, 0xe6, 0xf1, 0xbc, 0x76, 0xaa, 0x0d, 0xf8, 0xe2, 0x22, 0x1f, + 0xde, 0x96, 0xdc, 0xc8, 0x6f, 0x5a, 0x30, 0x2e, 0x62, 0x19, 0x98, 0x42, 0xca, 0xbe, 0xfd, 0xb3, + 0xa7, 0xf0, 0xf8, 0x88, 0x8c, 0xb3, 0x90, 0xce, 0x59, 0x1f, 0xd2, 0xbe, 0xd5, 0xa2, 0xf4, 0xc0, + 0x48, 0x0b, 0x25, 0x1d, 0x53, 0x7d, 0x3b, 0xce, 0xc3, 0xc4, 0xc3, 0x57, 0xa6, 0xea, 0xbb, 0x9a, + 0x82, 0x61, 0x1f, 0xf6, 0xec, 0xc7, 0x61, 0xc2, 0x94, 0xe3, 0x58, 0xd1, 0x1a, 0x3f, 0x2e, 0x00, + 0xf0, 0xae, 0x12, 0x49, 0x8c, 0x3a, 0x3c, 0xd7, 0xfa, 0x96, 0xdf, 0xcc, 0xe9, 0x01, 0x5e, 0x23, + 0x17, 0x11, 0xc8, 0xc4, 0xea, 0x5b, 0x7e, 0x13, 0x25, 0x13, 0xd2, 0x82, 0xd1, 0xae, 0x13, 0x6d, + 0xe5, 0x9f, 0xf8, 0xa8, 0x24, 0xa2, 0xf9, 0xa3, 0x2d, 0xe4, 0x0c, 0xc8, 0xdb, 0x56, 0xec, 0xf7, + 0x54, 0xc8, 0x23, 0x5d, 0x74, 0xdc, 0x66, 0xf3, 0xd2, 0xd3, 0x29, 0x95, 0x35, 0x39, 0xed, 0xff, + 0x34, 0xfb, 0x8e, 0x05, 0x13, 0x26, 0x6a, 0x46, 0x37, 0xfd, 0xbc, 0xd9, 0x4d, 0x79, 0xb6, 0x87, + 0xd9, 0xe3, 0xff, 0xdd, 0x02, 0xc0, 0x9e, 0x57, 0xef, 0x75, 0x3a, 0x4c, 0x6d, 0xd7, 0x41, 0x29, + 0xd6, 0x91, 0x83, 0x52, 0x46, 0x8e, 0x19, 0x94, 0x52, 0x38, 0x56, 0x50, 0xca, 0xe8, 0xf1, 0x83, + 0x52, 0x8a, 0x83, 0x83, 0x52, 0xec, 0x6f, 0x5a, 0x70, 0xb6, 0x6f, 0xbf, 0x62, 0x9a, 0x74, 0xe0, + 0xfb, 0xd1, 0x00, 0xff, 0x59, 0x8c, 0x41, 0x68, 0xe2, 0x91, 0x25, 0x98, 0x96, 0x2f, 0x0b, 0xd5, + 0xbb, 0x6d, 0x37, 0x33, 0x29, 0xd5, 0x7a, 0x0a, 0x8e, 0x7d, 0x35, 0xec, 0x7f, 0x65, 0x41, 0xc5, + 0x48, 0x65, 0xc1, 0x7d, 0xce, 0xf8, 0x8d, 0x57, 0xda, 0xe7, 0x8c, 0x5f, 0x75, 0x09, 0x98, 0xb8, + 0x86, 0x6e, 0x19, 0xef, 0x4e, 0xc4, 0xd7, 0xd0, 0xac, 0x14, 0x25, 0x54, 0xbc, 0x28, 0x20, 0x9d, + 0xcf, 0x0a, 0xe6, 0x8b, 0x02, 0xb4, 0x2b, 0x5c, 0xcd, 0x62, 0x17, 0xb7, 0xd1, 0xc3, 0x5d, 0xdc, + 0x8a, 0xd9, 0x2e, 0x6e, 0xf6, 0x1d, 0x98, 0x10, 0xbe, 0xe1, 0xaf, 0xd2, 0xdd, 0x23, 0xbf, 0x60, + 0xcd, 0x46, 0x7b, 0xca, 0x67, 0x8e, 0x55, 0x67, 0xe5, 0xb6, 0x03, 0x71, 0x7a, 0xed, 0x23, 0x50, + 0xbb, 0x06, 0xa0, 0x13, 0xfd, 0x0b, 0x47, 0xbc, 0x52, 0x3c, 0x20, 0xf5, 0x6b, 0x00, 0x4d, 0x34, + 0xb0, 0xec, 0x7f, 0x64, 0x41, 0xea, 0xe5, 0x34, 0xe3, 0x92, 0xc7, 0x1a, 0x78, 0xc9, 0x63, 0x5e, + 0x0c, 0x8c, 0x1c, 0x78, 0x31, 0x70, 0x0b, 0x48, 0x87, 0xcd, 0xb6, 0xe4, 0x5a, 0x5e, 0x48, 0x3e, + 0x30, 0xb3, 0xda, 0x87, 0x81, 0x19, 0xb5, 0xec, 0x7f, 0x28, 0x84, 0x35, 0xdf, 0x52, 0x3b, 0xbc, + 0x55, 0x7a, 0x50, 0xe4, 0xa4, 0xa4, 0x89, 0x6f, 0x48, 0xf3, 0x78, 0x7f, 0x8e, 0xbb, 0x78, 0xac, + 0xc8, 0x55, 0x85, 0x73, 0xb3, 0x7f, 0x4f, 0xc8, 0x6a, 0x3e, 0xb6, 0x76, 0xb8, 0xac, 0x9d, 0xa4, + 0xac, 0x37, 0xf3, 0x5a, 0x8e, 0xb3, 0x65, 0x24, 0xf3, 0x00, 0x5d, 0x1a, 0x34, 0xa8, 0x17, 0xa9, + 0x48, 0xbd, 0xa2, 0x8c, 0x19, 0xd7, 0xa5, 0x68, 0x60, 0xd8, 0xdf, 0x60, 0x73, 0x34, 0x7e, 0x3e, + 0x9e, 0x5c, 0x4d, 0xfb, 0x1a, 0xa7, 0xe7, 0x9f, 0x76, 0x35, 0x36, 0x42, 0xae, 0x46, 0x0e, 0x09, + 0xb9, 0x7a, 0x06, 0xc6, 0x03, 0xbf, 0x4d, 0xab, 0x81, 0x97, 0x76, 0x03, 0x42, 0x56, 0x8c, 0xb7, + 0x51, 0xc1, 0xed, 0x5f, 0xb7, 0x60, 0x3a, 0x1d, 0x60, 0x9a, 0xbb, 0x03, 0xb4, 0x99, 0x8f, 0xa3, + 0x70, 0xfc, 0x7c, 0x1c, 0xf6, 0x9f, 0x14, 0x61, 0x3a, 0xfd, 0xac, 0x25, 0xe3, 0xec, 0x72, 0x7b, + 0x5e, 0x6a, 0x83, 0x11, 0x86, 0x3c, 0x01, 0xd3, 0xe3, 0x65, 0x64, 0xe0, 0x78, 0xb9, 0x01, 0x65, + 0xbf, 0xab, 0x6c, 0x0a, 0x42, 0xb8, 0xab, 0xca, 0x1e, 0x74, 0x47, 0x01, 0xde, 0xdd, 0x9b, 0x3b, + 0x17, 0x0b, 0xa0, 0x8b, 0x31, 0xae, 0x4a, 0x7e, 0x46, 0x19, 0x43, 0x46, 0x13, 0x19, 0xae, 0xb4, + 0x31, 0x64, 0x2a, 0xae, 0x3f, 0xc8, 0x1e, 0x52, 0x3c, 0x4e, 0xa6, 0x9d, 0xb1, 0x1c, 0x33, 0xed, + 0xdc, 0x83, 0xb2, 0x34, 0xdf, 0x9e, 0x28, 0xc3, 0x0c, 0x27, 0x7c, 0x57, 0x11, 0xc0, 0x98, 0x56, + 0x2a, 0x85, 0x4f, 0x29, 0xd7, 0x14, 0x3e, 0x2f, 0xc1, 0xf8, 0x86, 0xd3, 0xd8, 0xf6, 0x37, 0x37, + 0xf9, 0x11, 0xa0, 0x5c, 0xfb, 0xa0, 0x6a, 0xb8, 0x9a, 0x28, 0xce, 0x18, 0x52, 0xaa, 0x06, 0x5b, + 0xe7, 0xa9, 0xf2, 0x78, 0x56, 0x96, 0x65, 0xbd, 0xce, 0x6b, 0x5f, 0xe8, 0x10, 0x0d, 0x2c, 0xf2, + 0x2c, 0x94, 0x9a, 0x6e, 0x28, 0x1e, 0x5e, 0xaf, 0x24, 0x1d, 0xe2, 0x97, 0x64, 0x39, 0x6a, 0x0c, + 0xf2, 0xb2, 0x76, 0x88, 0x9b, 0x88, 0x63, 0x55, 0xb4, 0x33, 0xdc, 0x01, 0xb1, 0x2a, 0xd2, 0xdf, + 0xf7, 0x6d, 0x36, 0x31, 0x23, 0xb7, 0xb1, 0xed, 0x7a, 0x22, 0x6d, 0x0b, 0x5b, 0x2d, 0x9e, 0x81, + 0x71, 0x2a, 0x9f, 0x7e, 0x17, 0xb7, 0x33, 0x7a, 0xb0, 0xa8, 0x17, 0xdf, 0x15, 0x9c, 0x54, 0x61, + 0x4a, 0xdd, 0x49, 0xab, 0x2b, 0x35, 0x91, 0x6e, 0x4a, 0x9b, 0xf0, 0x97, 0x92, 0x60, 0x4c, 0xe3, + 0xdb, 0x5f, 0x80, 0x8a, 0xa1, 0xeb, 0x71, 0xb5, 0xe8, 0xa1, 0xd3, 0xe8, 0x73, 0x61, 0xbf, 0xce, + 0x0a, 0x51, 0xc0, 0xf8, 0xcd, 0x9f, 0x88, 0xbf, 0x4c, 0xa9, 0x13, 0x32, 0xea, 0x52, 0x42, 0x19, + 0xb1, 0x80, 0xb6, 0xe8, 0x43, 0xf5, 0xda, 0x8e, 0x22, 0x86, 0xac, 0x10, 0x05, 0xcc, 0x7e, 0x16, + 0x4a, 0x2a, 0x29, 0x20, 0xcf, 0xac, 0xa5, 0x6e, 0xa5, 0xcc, 0xcc, 0x5a, 0x7e, 0x10, 0x21, 0x87, + 0xd8, 0xaf, 0x43, 0x49, 0xe5, 0x2e, 0x3c, 0x1c, 0x9b, 0x6d, 0xbf, 0xa1, 0xe7, 0xde, 0xf4, 0xc3, + 0x48, 0x25, 0x5c, 0x14, 0x17, 0xe7, 0xb7, 0x97, 0x79, 0x19, 0x6a, 0xa8, 0xfd, 0x13, 0x0b, 0x2a, + 0xeb, 0xeb, 0x2b, 0xda, 0x9e, 0x86, 0xf0, 0x58, 0x28, 0x5a, 0xa8, 0xba, 0x19, 0x51, 0xd3, 0x43, + 0x47, 0xac, 0x44, 0xb3, 0xfb, 0x7b, 0x73, 0x8f, 0xd5, 0x33, 0x31, 0x70, 0x40, 0x4d, 0xb2, 0x0c, + 0xe7, 0x4c, 0x88, 0x4c, 0x84, 0x23, 0xf5, 0x82, 0x8b, 0xfb, 0x6c, 0xf9, 0xe9, 0x07, 0x63, 0x56, + 0x9d, 0x34, 0x29, 0x15, 0x37, 0x5c, 0xc8, 0x26, 0xa5, 0x82, 0x86, 0xb3, 0xea, 0xd8, 0xcf, 0xc1, + 0x54, 0xca, 0x75, 0xe4, 0x08, 0x09, 0xc8, 0x7e, 0xa7, 0x00, 0x13, 0xa6, 0x07, 0xc1, 0x11, 0xf6, + 0xec, 0xa3, 0xab, 0x42, 0x19, 0xb7, 0xfe, 0x85, 0x63, 0xde, 0xfa, 0x9b, 0x6e, 0x16, 0xa3, 0xa7, + 0xeb, 0x66, 0x51, 0xcc, 0xc7, 0xcd, 0xc2, 0x70, 0x07, 0x1a, 0x7b, 0x74, 0xee, 0x40, 0xbf, 0x5d, + 0x84, 0xc9, 0x64, 0x46, 0xeb, 0x23, 0xf4, 0xe4, 0xb3, 0x7d, 0x3d, 0x79, 0xcc, 0x6b, 0xc6, 0xc2, + 0xb0, 0xd7, 0x8c, 0xa3, 0xc3, 0x5e, 0x33, 0x16, 0x4f, 0x70, 0xcd, 0xd8, 0x7f, 0x49, 0x38, 0x76, + 0xe4, 0x4b, 0xc2, 0x4f, 0xe8, 0x8d, 0x62, 0x3c, 0xe1, 0x59, 0x17, 0x6f, 0x16, 0x24, 0xd9, 0x0d, + 0x8b, 0x7e, 0x33, 0xd3, 0xe3, 0xbb, 0x74, 0x88, 0xfa, 0x10, 0x64, 0x3a, 0x3a, 0x1f, 0xdf, 0x93, + 0xe1, 0xb1, 0x63, 0x38, 0x39, 0xbf, 0x00, 0x15, 0x39, 0x9e, 0xf8, 0x99, 0x16, 0x92, 0xe7, 0xe1, + 0x7a, 0x0c, 0x42, 0x13, 0x8f, 0x0d, 0x8c, 0x6e, 0x3c, 0x41, 0xf8, 0x85, 0x77, 0x25, 0x79, 0xe1, + 0xbd, 0x96, 0x04, 0x63, 0x1a, 0xdf, 0xfe, 0x3c, 0x5c, 0xc8, 0xb4, 0x6c, 0xf2, 0x5b, 0x25, 0x7e, + 0x16, 0xa2, 0x4d, 0x89, 0x60, 0x88, 0x91, 0x7a, 0x62, 0x6b, 0xf6, 0xde, 0x40, 0x4c, 0x3c, 0x80, + 0x8a, 0xfd, 0x5b, 0x05, 0x98, 0x4c, 0x3e, 0x39, 0x4f, 0x1e, 0xe8, 0x7b, 0x90, 0x5c, 0xae, 0x60, + 0x04, 0x59, 0x23, 0x4b, 0xf2, 0xc0, 0xfb, 0xd3, 0x07, 0x7c, 0x7c, 0x6d, 0xe8, 0x94, 0xcd, 0xa7, + 0xc7, 0x58, 0x5e, 0x5c, 0x4a, 0x76, 0xfc, 0xe1, 0xf6, 0x38, 0xa5, 0x80, 0x34, 0x8f, 0xe5, 0xce, + 0x3d, 0x8e, 0xfe, 0xd6, 0xac, 0xd0, 0x60, 0xcb, 0xf6, 0x96, 0x1d, 0x1a, 0xb8, 0x9b, 0x2e, 0x6d, + 0xca, 0x17, 0x34, 0xf8, 0xca, 0xfd, 0xba, 0x2c, 0x43, 0x0d, 0xb5, 0xdf, 0x1e, 0x81, 0x32, 0xcf, + 0xff, 0x78, 0x23, 0xf0, 0x3b, 0xfc, 0x31, 0xe2, 0xd0, 0x30, 0x45, 0xc8, 0x6e, 0xbb, 0x95, 0xc7, + 0xeb, 0x5f, 0x82, 0xa2, 0x8c, 0x22, 0x31, 0x4a, 0x30, 0xc1, 0x91, 0x74, 0xa1, 0xb4, 0x29, 0xf3, + 0xd5, 0xcb, 0xbe, 0x1b, 0x32, 0xe7, 0xb2, 0xca, 0x7e, 0x2f, 0x9a, 0x40, 0xfd, 0x43, 0xcd, 0xc5, + 0x76, 0x60, 0x2a, 0x95, 0x36, 0x2b, 0xf7, 0x2c, 0xf7, 0xff, 0x73, 0x14, 0xca, 0x3a, 0xb8, 0x93, + 0x7c, 0x2c, 0x61, 0x17, 0x8e, 0x75, 0x78, 0x69, 0xd0, 0x65, 0xe7, 0x26, 0x8d, 0x9c, 0xb2, 0xf1, + 0x5e, 0x82, 0x42, 0x2f, 0x68, 0xa7, 0x0d, 0x3f, 0x77, 0x71, 0x05, 0x59, 0xb9, 0x19, 0x90, 0x5a, + 0x78, 0xb4, 0x01, 0xa9, 0x57, 0x60, 0x74, 0xc3, 0x6f, 0xee, 0xa6, 0x5f, 0xd6, 0xac, 0xf9, 0xcd, + 0x5d, 0xe4, 0x10, 0xf2, 0x32, 0x4c, 0xca, 0x28, 0x5b, 0xa5, 0xc4, 0x14, 0xb9, 0x9e, 0xaa, 0xfd, + 0x81, 0xd6, 0x13, 0x50, 0x4c, 0x61, 0xb3, 0x5d, 0x96, 0x1d, 0x1b, 0xf8, 0xdb, 0x05, 0x63, 0x49, + 0xe7, 0x81, 0x5b, 0xf5, 0x3b, 0xb7, 0xb9, 0x7d, 0x5a, 0x63, 0x24, 0x02, 0x79, 0xc7, 0x0f, 0x0d, + 0xe4, 0x5d, 0x12, 0xb4, 0x99, 0xb4, 0x7c, 0x47, 0x99, 0xa8, 0x5d, 0x55, 0x74, 0x59, 0xd9, 0x81, + 0x67, 0x17, 0x5d, 0x33, 0x2b, 0xe4, 0xb9, 0xfc, 0xde, 0x85, 0x3c, 0xdb, 0x77, 0x61, 0x2a, 0xd5, + 0x7f, 0xca, 0x6e, 0x68, 0x65, 0xdb, 0x0d, 0x8f, 0xf6, 0x36, 0xe7, 0x3f, 0xb3, 0xe0, 0x6c, 0xdf, + 0x8a, 0x74, 0xd4, 0xd8, 0xf3, 0xf4, 0xde, 0x38, 0x72, 0xf2, 0xbd, 0xb1, 0x70, 0xbc, 0xbd, 0xb1, + 0xb6, 0xf1, 0xbd, 0x1f, 0x5d, 0xfe, 0xc0, 0x0f, 0x7e, 0x74, 0xf9, 0x03, 0xbf, 0xff, 0xa3, 0xcb, + 0x1f, 0x78, 0x7b, 0xff, 0xb2, 0xf5, 0xbd, 0xfd, 0xcb, 0xd6, 0x0f, 0xf6, 0x2f, 0x5b, 0xbf, 0xbf, + 0x7f, 0xd9, 0xfa, 0x2f, 0xfb, 0x97, 0xad, 0x6f, 0xfe, 0xd1, 0xe5, 0x0f, 0x7c, 0xea, 0x13, 0x71, + 0x4f, 0x2d, 0xa8, 0x9e, 0xe2, 0x3f, 0x3e, 0xac, 0xfa, 0x65, 0xa1, 0xbb, 0xdd, 0x5a, 0x60, 0x3d, + 0xb5, 0xa0, 0x4b, 0x54, 0x4f, 0xfd, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x12, 0x87, 0x61, 0xb2, + 0x65, 0xaf, 0x00, 0x00, } func (m *ALBStatus) Marshal() (dAtA []byte, err error) { @@ -5371,6 +5516,20 @@ func (m *CanaryStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.StepPluginStatuses) > 0 { + for iNdEx := len(m.StepPluginStatuses) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.StepPluginStatuses[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x32 + } + } i -= len(m.StablePingPong) copy(dAtA[i:], m.StablePingPong) i = encodeVarintGenerated(dAtA, i, uint64(len(m.StablePingPong))) @@ -5440,6 +5599,18 @@ func (m *CanaryStep) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Plugin != nil { + { + size, err := m.Plugin.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x4a + } if m.SetMirrorRoute != nil { { size, err := m.SetMirrorRoute.MarshalToSizedBuffer(dAtA[:i]) @@ -6056,6 +6227,16 @@ func (m *DatadogMetric) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + { + size, err := m.SecretRef.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x3a i -= len(m.Aggregator) copy(dAtA[i:], m.Aggregator) i = encodeVarintGenerated(dAtA, i, uint64(len(m.Aggregator))) @@ -7276,6 +7457,18 @@ func (m *Metric) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.ConsecutiveSuccessLimit != nil { + { + size, err := m.ConsecutiveSuccessLimit.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x5a + } { size, err := m.Provider.MarshalToSizedBuffer(dAtA[:i]) if err != nil { @@ -7563,6 +7756,9 @@ func (m *MetricResult) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + i = encodeVarintGenerated(dAtA, i, uint64(m.ConsecutiveSuccess)) + i-- + dAtA[i] = 0x68 if len(m.Metadata) > 0 { keysForMetadata := make([]string, 0, len(m.Metadata)) for k := range m.Metadata { @@ -7665,6 +7861,11 @@ func (m *NewRelicMetric) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Timeout != nil { + i = encodeVarintGenerated(dAtA, i, uint64(*m.Timeout)) + i-- + dAtA[i] = 0x18 + } i -= len(m.Query) copy(dAtA[i:], m.Query) i = encodeVarintGenerated(dAtA, i, uint64(len(m.Query))) @@ -7698,6 +7899,30 @@ func (m *NginxTrafficRouting) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.CanaryIngressAnnotations) > 0 { + keysForCanaryIngressAnnotations := make([]string, 0, len(m.CanaryIngressAnnotations)) + for k := range m.CanaryIngressAnnotations { + keysForCanaryIngressAnnotations = append(keysForCanaryIngressAnnotations, string(k)) + } + github_com_gogo_protobuf_sortkeys.Strings(keysForCanaryIngressAnnotations) + for iNdEx := len(keysForCanaryIngressAnnotations) - 1; iNdEx >= 0; iNdEx-- { + v := m.CanaryIngressAnnotations[string(keysForCanaryIngressAnnotations[iNdEx])] + baseI := i + i -= len(v) + copy(dAtA[i:], v) + i = encodeVarintGenerated(dAtA, i, uint64(len(v))) + i-- + dAtA[i] = 0x12 + i -= len(keysForCanaryIngressAnnotations[iNdEx]) + copy(dAtA[i:], keysForCanaryIngressAnnotations[iNdEx]) + i = encodeVarintGenerated(dAtA, i, uint64(len(keysForCanaryIngressAnnotations[iNdEx]))) + i-- + dAtA[i] = 0xa + i = encodeVarintGenerated(dAtA, i, uint64(baseI-i)) + i-- + dAtA[i] = 0x2a + } + } if len(m.StableIngresses) > 0 { for iNdEx := len(m.StableIngresses) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.StableIngresses[iNdEx]) @@ -7905,6 +8130,41 @@ func (m *PingPongSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *PluginStep) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PluginStep) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *PluginStep) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Config != nil { + i -= len(m.Config) + copy(dAtA[i:], m.Config) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Config))) + i-- + dAtA[i] = 0x12 + } + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + func (m *PodTemplateMetadata) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -8022,6 +8282,18 @@ func (m *PrometheusMetric) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.RangeQuery != nil { + { + size, err := m.RangeQuery.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x3a + } if len(m.Headers) > 0 { for iNdEx := len(m.Headers) - 1; iNdEx >= 0; iNdEx-- { { @@ -8072,7 +8344,7 @@ func (m *PrometheusMetric) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *RequiredDuringSchedulingIgnoredDuringExecution) Marshal() (dAtA []byte, err error) { +func (m *PrometheusRangeQueryArgs) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -8082,20 +8354,58 @@ func (m *RequiredDuringSchedulingIgnoredDuringExecution) Marshal() (dAtA []byte, return dAtA[:n], nil } -func (m *RequiredDuringSchedulingIgnoredDuringExecution) MarshalTo(dAtA []byte) (int, error) { +func (m *PrometheusRangeQueryArgs) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *RequiredDuringSchedulingIgnoredDuringExecution) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *PrometheusRangeQueryArgs) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l - return len(dAtA) - i, nil -} - -func (m *RollbackWindowSpec) Marshal() (dAtA []byte, err error) { + i -= len(m.Step) + copy(dAtA[i:], m.Step) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Step))) + i-- + dAtA[i] = 0x1a + i -= len(m.End) + copy(dAtA[i:], m.End) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.End))) + i-- + dAtA[i] = 0x12 + i -= len(m.Start) + copy(dAtA[i:], m.Start) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Start))) + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + +func (m *RequiredDuringSchedulingIgnoredDuringExecution) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RequiredDuringSchedulingIgnoredDuringExecution) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *RequiredDuringSchedulingIgnoredDuringExecution) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + return len(dAtA) - i, nil +} + +func (m *RollbackWindowSpec) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -9510,6 +9820,42 @@ func (m *SecretKeyRef) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *SecretRef) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SecretRef) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SecretRef) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + i-- + if m.Namespaced { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x10 + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + func (m *SetCanaryScale) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -9716,6 +10062,111 @@ func (m *SkyWalkingMetric) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *StepPluginStatus) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StepPluginStatus) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *StepPluginStatus) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Status != nil { + i -= len(m.Status) + copy(dAtA[i:], m.Status) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Status))) + i-- + dAtA[i] = 0x62 + } + i-- + if m.Disabled { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x58 + i = encodeVarintGenerated(dAtA, i, uint64(m.Executions)) + i-- + dAtA[i] = 0x50 + i -= len(m.Backoff) + copy(dAtA[i:], m.Backoff) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Backoff))) + i-- + dAtA[i] = 0x4a + if m.FinishedAt != nil { + { + size, err := m.FinishedAt.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x42 + } + if m.UpdatedAt != nil { + { + size, err := m.UpdatedAt.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x3a + } + if m.StartedAt != nil { + { + size, err := m.StartedAt.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x32 + } + i -= len(m.Message) + copy(dAtA[i:], m.Message) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Message))) + i-- + dAtA[i] = 0x2a + i -= len(m.Phase) + copy(dAtA[i:], m.Phase) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Phase))) + i-- + dAtA[i] = 0x22 + i -= len(m.Operation) + copy(dAtA[i:], m.Operation) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Operation))) + i-- + dAtA[i] = 0x1a + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0x12 + i = encodeVarintGenerated(dAtA, i, uint64(m.Index)) + i-- + dAtA[i] = 0x8 + return len(dAtA) - i, nil +} + func (m *StickinessConfig) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -10971,6 +11422,12 @@ func (m *CanaryStatus) Size() (n int) { } l = len(m.StablePingPong) n += 1 + l + sovGenerated(uint64(l)) + if len(m.StepPluginStatuses) > 0 { + for _, e := range m.StepPluginStatuses { + l = e.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + } return n } @@ -11007,6 +11464,10 @@ func (m *CanaryStep) Size() (n int) { l = m.SetMirrorRoute.Size() n += 1 + l + sovGenerated(uint64(l)) } + if m.Plugin != nil { + l = m.Plugin.Size() + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -11226,6 +11687,8 @@ func (m *DatadogMetric) Size() (n int) { n += 1 + l + sovGenerated(uint64(l)) l = len(m.Aggregator) n += 1 + l + sovGenerated(uint64(l)) + l = m.SecretRef.Size() + n += 1 + l + sovGenerated(uint64(l)) return n } @@ -11689,6 +12152,10 @@ func (m *Metric) Size() (n int) { } l = m.Provider.Size() n += 1 + l + sovGenerated(uint64(l)) + if m.ConsecutiveSuccessLimit != nil { + l = m.ConsecutiveSuccessLimit.Size() + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -11790,6 +12257,7 @@ func (m *MetricResult) Size() (n int) { n += mapEntrySize + 1 + sovGenerated(uint64(mapEntrySize)) } } + n += 1 + sovGenerated(uint64(m.ConsecutiveSuccess)) return n } @@ -11803,6 +12271,9 @@ func (m *NewRelicMetric) Size() (n int) { n += 1 + l + sovGenerated(uint64(l)) l = len(m.Query) n += 1 + l + sovGenerated(uint64(l)) + if m.Timeout != nil { + n += 1 + sovGenerated(uint64(*m.Timeout)) + } return n } @@ -11830,6 +12301,14 @@ func (m *NginxTrafficRouting) Size() (n int) { n += 1 + l + sovGenerated(uint64(l)) } } + if len(m.CanaryIngressAnnotations) > 0 { + for k, v := range m.CanaryIngressAnnotations { + _ = k + _ = v + mapEntrySize := 1 + len(k) + sovGenerated(uint64(len(k))) + 1 + len(v) + sovGenerated(uint64(len(v))) + n += mapEntrySize + 1 + sovGenerated(uint64(mapEntrySize)) + } + } return n } @@ -11897,6 +12376,21 @@ func (m *PingPongSpec) Size() (n int) { return n } +func (m *PluginStep) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Name) + n += 1 + l + sovGenerated(uint64(l)) + if m.Config != nil { + l = len(m.Config) + n += 1 + l + sovGenerated(uint64(l)) + } + return n +} + func (m *PodTemplateMetadata) Size() (n int) { if m == nil { return 0 @@ -11954,6 +12448,25 @@ func (m *PrometheusMetric) Size() (n int) { n += 1 + l + sovGenerated(uint64(l)) } } + if m.RangeQuery != nil { + l = m.RangeQuery.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + return n +} + +func (m *PrometheusRangeQueryArgs) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Start) + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.End) + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.Step) + n += 1 + l + sovGenerated(uint64(l)) return n } @@ -12468,6 +12981,18 @@ func (m *SecretKeyRef) Size() (n int) { return n } +func (m *SecretRef) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Name) + n += 1 + l + sovGenerated(uint64(l)) + n += 2 + return n +} + func (m *SetCanaryScale) Size() (n int) { if m == nil { return 0 @@ -12551,6 +13076,44 @@ func (m *SkyWalkingMetric) Size() (n int) { return n } +func (m *StepPluginStatus) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + n += 1 + sovGenerated(uint64(m.Index)) + l = len(m.Name) + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.Operation) + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.Phase) + n += 1 + l + sovGenerated(uint64(l)) + l = len(m.Message) + n += 1 + l + sovGenerated(uint64(l)) + if m.StartedAt != nil { + l = m.StartedAt.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + if m.UpdatedAt != nil { + l = m.UpdatedAt.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + if m.FinishedAt != nil { + l = m.FinishedAt.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + l = len(m.Backoff) + n += 1 + l + sovGenerated(uint64(l)) + n += 1 + sovGenerated(uint64(m.Executions)) + n += 2 + if m.Status != nil { + l = len(m.Status) + n += 1 + l + sovGenerated(uint64(l)) + } + return n +} + func (m *StickinessConfig) Size() (n int) { if m == nil { return 0 @@ -13229,12 +13792,18 @@ func (this *CanaryStatus) String() string { if this == nil { return "nil" } + repeatedStringForStepPluginStatuses := "[]StepPluginStatus{" + for _, f := range this.StepPluginStatuses { + repeatedStringForStepPluginStatuses += strings.Replace(strings.Replace(f.String(), "StepPluginStatus", "StepPluginStatus", 1), `&`, ``, 1) + "," + } + repeatedStringForStepPluginStatuses += "}" s := strings.Join([]string{`&CanaryStatus{`, `CurrentStepAnalysisRunStatus:` + strings.Replace(this.CurrentStepAnalysisRunStatus.String(), "RolloutAnalysisRunStatus", "RolloutAnalysisRunStatus", 1) + `,`, `CurrentBackgroundAnalysisRunStatus:` + strings.Replace(this.CurrentBackgroundAnalysisRunStatus.String(), "RolloutAnalysisRunStatus", "RolloutAnalysisRunStatus", 1) + `,`, `CurrentExperiment:` + fmt.Sprintf("%v", this.CurrentExperiment) + `,`, `Weights:` + strings.Replace(this.Weights.String(), "TrafficWeights", "TrafficWeights", 1) + `,`, `StablePingPong:` + fmt.Sprintf("%v", this.StablePingPong) + `,`, + `StepPluginStatuses:` + repeatedStringForStepPluginStatuses + `,`, `}`, }, "") return s @@ -13251,6 +13820,7 @@ func (this *CanaryStep) String() string { `SetCanaryScale:` + strings.Replace(this.SetCanaryScale.String(), "SetCanaryScale", "SetCanaryScale", 1) + `,`, `SetHeaderRoute:` + strings.Replace(this.SetHeaderRoute.String(), "SetHeaderRoute", "SetHeaderRoute", 1) + `,`, `SetMirrorRoute:` + strings.Replace(this.SetMirrorRoute.String(), "SetMirrorRoute", "SetMirrorRoute", 1) + `,`, + `Plugin:` + strings.Replace(this.Plugin.String(), "PluginStep", "PluginStep", 1) + `,`, `}`, }, "") return s @@ -13405,6 +13975,7 @@ func (this *DatadogMetric) String() string { `Formula:` + fmt.Sprintf("%v", this.Formula) + `,`, `ApiVersion:` + fmt.Sprintf("%v", this.ApiVersion) + `,`, `Aggregator:` + fmt.Sprintf("%v", this.Aggregator) + `,`, + `SecretRef:` + strings.Replace(strings.Replace(this.SecretRef.String(), "SecretRef", "SecretRef", 1), `&`, ``, 1) + `,`, `}`, }, "") return s @@ -13775,6 +14346,7 @@ func (this *Metric) String() string { `InconclusiveLimit:` + strings.Replace(fmt.Sprintf("%v", this.InconclusiveLimit), "IntOrString", "intstr.IntOrString", 1) + `,`, `ConsecutiveErrorLimit:` + strings.Replace(fmt.Sprintf("%v", this.ConsecutiveErrorLimit), "IntOrString", "intstr.IntOrString", 1) + `,`, `Provider:` + strings.Replace(strings.Replace(this.Provider.String(), "MetricProvider", "MetricProvider", 1), `&`, ``, 1) + `,`, + `ConsecutiveSuccessLimit:` + strings.Replace(fmt.Sprintf("%v", this.ConsecutiveSuccessLimit), "IntOrString", "intstr.IntOrString", 1) + `,`, `}`, }, "") return s @@ -13842,6 +14414,7 @@ func (this *MetricResult) String() string { `ConsecutiveError:` + fmt.Sprintf("%v", this.ConsecutiveError) + `,`, `DryRun:` + fmt.Sprintf("%v", this.DryRun) + `,`, `Metadata:` + mapStringForMetadata + `,`, + `ConsecutiveSuccess:` + fmt.Sprintf("%v", this.ConsecutiveSuccess) + `,`, `}`, }, "") return s @@ -13853,6 +14426,7 @@ func (this *NewRelicMetric) String() string { s := strings.Join([]string{`&NewRelicMetric{`, `Profile:` + fmt.Sprintf("%v", this.Profile) + `,`, `Query:` + fmt.Sprintf("%v", this.Query) + `,`, + `Timeout:` + valueToStringGenerated(this.Timeout) + `,`, `}`, }, "") return s @@ -13871,11 +14445,22 @@ func (this *NginxTrafficRouting) String() string { mapStringForAdditionalIngressAnnotations += fmt.Sprintf("%v: %v,", k, this.AdditionalIngressAnnotations[k]) } mapStringForAdditionalIngressAnnotations += "}" + keysForCanaryIngressAnnotations := make([]string, 0, len(this.CanaryIngressAnnotations)) + for k := range this.CanaryIngressAnnotations { + keysForCanaryIngressAnnotations = append(keysForCanaryIngressAnnotations, k) + } + github_com_gogo_protobuf_sortkeys.Strings(keysForCanaryIngressAnnotations) + mapStringForCanaryIngressAnnotations := "map[string]string{" + for _, k := range keysForCanaryIngressAnnotations { + mapStringForCanaryIngressAnnotations += fmt.Sprintf("%v: %v,", k, this.CanaryIngressAnnotations[k]) + } + mapStringForCanaryIngressAnnotations += "}" s := strings.Join([]string{`&NginxTrafficRouting{`, `AnnotationPrefix:` + fmt.Sprintf("%v", this.AnnotationPrefix) + `,`, `StableIngress:` + fmt.Sprintf("%v", this.StableIngress) + `,`, `AdditionalIngressAnnotations:` + mapStringForAdditionalIngressAnnotations + `,`, `StableIngresses:` + fmt.Sprintf("%v", this.StableIngresses) + `,`, + `CanaryIngressAnnotations:` + mapStringForCanaryIngressAnnotations + `,`, `}`, }, "") return s @@ -13928,6 +14513,17 @@ func (this *PingPongSpec) String() string { }, "") return s } +func (this *PluginStep) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&PluginStep{`, + `Name:` + fmt.Sprintf("%v", this.Name) + `,`, + `Config:` + valueToStringGenerated(this.Config) + `,`, + `}`, + }, "") + return s +} func (this *PodTemplateMetadata) String() string { if this == nil { return "nil" @@ -13985,11 +14581,24 @@ func (this *PrometheusMetric) String() string { `Timeout:` + valueToStringGenerated(this.Timeout) + `,`, `Insecure:` + fmt.Sprintf("%v", this.Insecure) + `,`, `Headers:` + repeatedStringForHeaders + `,`, + `RangeQuery:` + strings.Replace(this.RangeQuery.String(), "PrometheusRangeQueryArgs", "PrometheusRangeQueryArgs", 1) + `,`, `}`, }, "") return s } -func (this *RequiredDuringSchedulingIgnoredDuringExecution) String() string { +func (this *PrometheusRangeQueryArgs) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&PrometheusRangeQueryArgs{`, + `Start:` + fmt.Sprintf("%v", this.Start) + `,`, + `End:` + fmt.Sprintf("%v", this.End) + `,`, + `Step:` + fmt.Sprintf("%v", this.Step) + `,`, + `}`, + }, "") + return s +} +func (this *RequiredDuringSchedulingIgnoredDuringExecution) String() string { if this == nil { return "nil" } @@ -14372,6 +14981,17 @@ func (this *SecretKeyRef) String() string { }, "") return s } +func (this *SecretRef) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&SecretRef{`, + `Name:` + fmt.Sprintf("%v", this.Name) + `,`, + `Namespaced:` + fmt.Sprintf("%v", this.Namespaced) + `,`, + `}`, + }, "") + return s +} func (this *SetCanaryScale) String() string { if this == nil { return "nil" @@ -14441,6 +15061,27 @@ func (this *SkyWalkingMetric) String() string { }, "") return s } +func (this *StepPluginStatus) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&StepPluginStatus{`, + `Index:` + fmt.Sprintf("%v", this.Index) + `,`, + `Name:` + fmt.Sprintf("%v", this.Name) + `,`, + `Operation:` + fmt.Sprintf("%v", this.Operation) + `,`, + `Phase:` + fmt.Sprintf("%v", this.Phase) + `,`, + `Message:` + fmt.Sprintf("%v", this.Message) + `,`, + `StartedAt:` + strings.Replace(fmt.Sprintf("%v", this.StartedAt), "Time", "v1.Time", 1) + `,`, + `UpdatedAt:` + strings.Replace(fmt.Sprintf("%v", this.UpdatedAt), "Time", "v1.Time", 1) + `,`, + `FinishedAt:` + strings.Replace(fmt.Sprintf("%v", this.FinishedAt), "Time", "v1.Time", 1) + `,`, + `Backoff:` + fmt.Sprintf("%v", this.Backoff) + `,`, + `Executions:` + fmt.Sprintf("%v", this.Executions) + `,`, + `Disabled:` + fmt.Sprintf("%v", this.Disabled) + `,`, + `Status:` + valueToStringGenerated(this.Status) + `,`, + `}`, + }, "") + return s +} func (this *StickinessConfig) String() string { if this == nil { return "nil" @@ -19189,6 +19830,40 @@ func (m *CanaryStatus) Unmarshal(dAtA []byte) error { } m.StablePingPong = PingPongType(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field StepPluginStatuses", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.StepPluginStatuses = append(m.StepPluginStatuses, StepPluginStatus{}) + if err := m.StepPluginStatuses[len(m.StepPluginStatuses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -19475,6 +20150,42 @@ func (m *CanaryStep) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 9: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Plugin", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Plugin == nil { + m.Plugin = &PluginStep{} + } + if err := m.Plugin.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -21381,6 +22092,39 @@ func (m *DatadogMetric) Unmarshal(dAtA []byte) error { } m.Aggregator = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SecretRef", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.SecretRef.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -25426,6 +26170,42 @@ func (m *Metric) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 11: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ConsecutiveSuccessLimit", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ConsecutiveSuccessLimit == nil { + m.ConsecutiveSuccessLimit = &intstr.IntOrString{} + } + if err := m.ConsecutiveSuccessLimit.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -26441,6 +27221,25 @@ func (m *MetricResult) Unmarshal(dAtA []byte) error { } m.Metadata[mapkey] = mapvalue iNdEx = postIndex + case 13: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ConsecutiveSuccess", wireType) + } + m.ConsecutiveSuccess = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ConsecutiveSuccess |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -26555,6 +27354,26 @@ func (m *NewRelicMetric) Unmarshal(dAtA []byte) error { } m.Query = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Timeout", wireType) + } + var v int64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Timeout = &v default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -26828,61 +27647,11 @@ func (m *NginxTrafficRouting) Unmarshal(dAtA []byte) error { } m.StableIngresses = append(m.StableIngresses, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipGenerated(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthGenerated - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *OAuth2Config) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowGenerated - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: OAuth2Config: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: OAuth2Config: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: + case 5: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field TokenURL", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field CanaryIngressAnnotations", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowGenerated @@ -26892,16 +27661,193 @@ func (m *OAuth2Config) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLengthGenerated } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.CanaryIngressAnnotations == nil { + m.CanaryIngressAnnotations = make(map[string]string) + } + var mapkey string + var mapvalue string + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthGenerated + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey < 0 { + return ErrInvalidLengthGenerated + } + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var stringLenmapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapvalue |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapvalue := int(stringLenmapvalue) + if intStringLenmapvalue < 0 { + return ErrInvalidLengthGenerated + } + postStringIndexmapvalue := iNdEx + intStringLenmapvalue + if postStringIndexmapvalue < 0 { + return ErrInvalidLengthGenerated + } + if postStringIndexmapvalue > l { + return io.ErrUnexpectedEOF + } + mapvalue = string(dAtA[iNdEx:postStringIndexmapvalue]) + iNdEx = postStringIndexmapvalue + } else { + iNdEx = entryPreIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.CanaryIngressAnnotations[mapkey] = mapvalue + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *OAuth2Config) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: OAuth2Config: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: OAuth2Config: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TokenURL", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthGenerated } @@ -27434,7 +28380,7 @@ func (m *PingPongSpec) Unmarshal(dAtA []byte) error { } return nil } -func (m *PodTemplateMetadata) Unmarshal(dAtA []byte) error { +func (m *PluginStep) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -27457,17 +28403,17 @@ func (m *PodTemplateMetadata) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: PodTemplateMetadata: wiretype end group for non-group") + return fmt.Errorf("proto: PluginStep: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: PodTemplateMetadata: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: PluginStep: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } - var msglen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowGenerated @@ -27477,81 +28423,197 @@ func (m *PodTemplateMetadata) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return ErrInvalidLengthGenerated } - postIndex := iNdEx + msglen + postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthGenerated } if postIndex > l { return io.ErrUnexpectedEOF } - if m.Labels == nil { - m.Labels = make(map[string]string) + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Config", wireType) } - var mapkey string - var mapvalue string - for iNdEx < postIndex { - entryPreIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowGenerated - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated } - fieldNum := int32(wire >> 3) - if fieldNum == 1 { - var stringLenmapkey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowGenerated - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLenmapkey |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLenmapkey := int(stringLenmapkey) - if intStringLenmapkey < 0 { - return ErrInvalidLengthGenerated - } - postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey < 0 { - return ErrInvalidLengthGenerated - } - if postStringIndexmapkey > l { - return io.ErrUnexpectedEOF - } - mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) - iNdEx = postStringIndexmapkey - } else if fieldNum == 2 { - var stringLenmapvalue uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowGenerated - } - if iNdEx >= l { - return io.ErrUnexpectedEOF + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Config = append(m.Config[:0], dAtA[iNdEx:postIndex]...) + if m.Config == nil { + m.Config = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PodTemplateMetadata) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PodTemplateMetadata: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PodTemplateMetadata: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Labels == nil { + m.Labels = make(map[string]string) + } + var mapkey string + var mapvalue string + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthGenerated + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey < 0 { + return ErrInvalidLengthGenerated + } + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var stringLenmapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF } b := dAtA[iNdEx] iNdEx++ @@ -28007,56 +29069,42 @@ func (m *PrometheusMetric) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipGenerated(dAtA[iNdEx:]) - if err != nil { - return err + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RangeQuery", wireType) } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthGenerated + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF + if msglen < 0 { + return ErrInvalidLengthGenerated } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *RequiredDuringSchedulingIgnoredDuringExecution) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowGenerated + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated } - if iNdEx >= l { + if postIndex > l { return io.ErrUnexpectedEOF } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break + if m.RangeQuery == nil { + m.RangeQuery = &PrometheusRangeQueryArgs{} } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: RequiredDuringSchedulingIgnoredDuringExecution: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: RequiredDuringSchedulingIgnoredDuringExecution: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { + if err := m.RangeQuery.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -28078,7 +29126,7 @@ func (m *RequiredDuringSchedulingIgnoredDuringExecution) Unmarshal(dAtA []byte) } return nil } -func (m *RollbackWindowSpec) Unmarshal(dAtA []byte) error { +func (m *PrometheusRangeQueryArgs) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -28101,17 +29149,17 @@ func (m *RollbackWindowSpec) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: RollbackWindowSpec: wiretype end group for non-group") + return fmt.Errorf("proto: PrometheusRangeQueryArgs: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: RollbackWindowSpec: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: PrometheusRangeQueryArgs: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Revisions", wireType) + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Start", wireType) } - m.Revisions = 0 + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowGenerated @@ -28121,26 +29169,222 @@ func (m *RollbackWindowSpec) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Revisions |= int32(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - default: - iNdEx = preIndex - skippy, err := skipGenerated(dAtA[iNdEx:]) - if err != nil { - return err + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated } - if (skippy < 0) || (iNdEx+skippy) < 0 { + postIndex := iNdEx + intStringLen + if postIndex < 0 { return ErrInvalidLengthGenerated } - if (iNdEx + skippy) > l { + if postIndex > l { return io.ErrUnexpectedEOF } - iNdEx += skippy - } - } + m.Start = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field End", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.End = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Step", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Step = DurationString(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RequiredDuringSchedulingIgnoredDuringExecution) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RequiredDuringSchedulingIgnoredDuringExecution: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RequiredDuringSchedulingIgnoredDuringExecution: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RollbackWindowSpec) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RollbackWindowSpec: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RollbackWindowSpec: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Revisions", wireType) + } + m.Revisions = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Revisions |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } if iNdEx > l { return io.ErrUnexpectedEOF @@ -32530,7 +33774,7 @@ func (m *SecretKeyRef) Unmarshal(dAtA []byte) error { } return nil } -func (m *SetCanaryScale) Unmarshal(dAtA []byte) error { +func (m *SecretRef) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -32553,17 +33797,17 @@ func (m *SetCanaryScale) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: SetCanaryScale: wiretype end group for non-group") + return fmt.Errorf("proto: SecretRef: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: SetCanaryScale: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: SecretRef: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Weight", wireType) + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } - var v int32 + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowGenerated @@ -32573,35 +33817,27 @@ func (m *SetCanaryScale) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int32(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - m.Weight = &v - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Replicas", wireType) + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated } - var v int32 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowGenerated - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= int32(b&0x7F) << shift - if b < 0x80 { - break - } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated } - m.Replicas = &v - case 3: + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field MatchTrafficWeight", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Namespaced", wireType) } var v int for shift := uint(0); ; shift += 7 { @@ -32618,7 +33854,7 @@ func (m *SetCanaryScale) Unmarshal(dAtA []byte) error { break } } - m.MatchTrafficWeight = bool(v != 0) + m.Namespaced = bool(v != 0) default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -32640,7 +33876,7 @@ func (m *SetCanaryScale) Unmarshal(dAtA []byte) error { } return nil } -func (m *SetHeaderRoute) Unmarshal(dAtA []byte) error { +func (m *SetCanaryScale) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -32663,17 +33899,127 @@ func (m *SetHeaderRoute) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: SetHeaderRoute: wiretype end group for non-group") + return fmt.Errorf("proto: SetCanaryScale: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: SetHeaderRoute: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: SetCanaryScale: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Weight", wireType) } - var stringLen uint64 + var v int32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Weight = &v + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Replicas", wireType) + } + var v int32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Replicas = &v + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field MatchTrafficWeight", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.MatchTrafficWeight = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SetHeaderRoute) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SetHeaderRoute: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SetHeaderRoute: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowGenerated @@ -33184,6 +34530,416 @@ func (m *SkyWalkingMetric) Unmarshal(dAtA []byte) error { } return nil } +func (m *StepPluginStatus) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StepPluginStatus: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StepPluginStatus: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) + } + m.Index = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Index |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Operation", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Operation = StepPluginOperation(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Phase", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Phase = StepPluginPhase(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Message", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Message = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field StartedAt", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.StartedAt == nil { + m.StartedAt = &v1.Time{} + } + if err := m.StartedAt.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field UpdatedAt", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.UpdatedAt == nil { + m.UpdatedAt = &v1.Time{} + } + if err := m.UpdatedAt.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FinishedAt", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.FinishedAt == nil { + m.FinishedAt = &v1.Time{} + } + if err := m.FinishedAt.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 9: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Backoff", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Backoff = DurationString(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 10: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Executions", wireType) + } + m.Executions = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Executions |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Disabled", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Disabled = bool(v != 0) + case 12: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Status = append(m.Status[:0], dAtA[iNdEx:postIndex]...) + if m.Status == nil { + m.Status = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *StickinessConfig) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/pkg/apis/rollouts/v1alpha1/generated.proto b/pkg/apis/rollouts/v1alpha1/generated.proto index 0ea0a7bc2c..489548d836 100644 --- a/pkg/apis/rollouts/v1alpha1/generated.proto +++ b/pkg/apis/rollouts/v1alpha1/generated.proto @@ -448,6 +448,9 @@ message CanaryStatus { // StablePingPong For the ping-pong feature holds the current stable service, ping or pong optional string stablePingPong = 5; + + // StepPluginStatuses holds the status of the step plugins executed + repeated StepPluginStatus stepPluginStatuses = 6; } // CanaryStep defines a step of a canary deployment. @@ -477,6 +480,9 @@ message CanaryStep { // SetMirrorRoutes Mirrors traffic that matches rules to a particular destination // +optional optional SetMirrorRoute setMirrorRoute = 8; + + // Plugin defines a plugin to execute for a step + optional PluginStep plugin = 9; } // CanaryStrategy defines parameters for a Replica Based Canary @@ -655,10 +661,13 @@ message DatadogMetric { // +kubebuilder:default=v1 optional string apiVersion = 5; - // +kubebuilder:default="last" // +kubebuilder:validation:Enum=avg;min;max;sum;last;percentile;mean;l2norm;area - // Aggregator is a type of aggregator to use for metrics-based queries (default: last). Used for v2 + // Aggregator is a type of aggregator to use for metrics-based queries (default: ""). Used for v2 optional string aggregator = 6; + + // Secret refers to the name of the secret that should be used for an analysis and should exists in the namespace where the controller is. + // +optional + optional SecretRef secretRef = 7; } // DryRun defines the settings for running the analysis in Dry-Run mode. @@ -1005,6 +1014,7 @@ message Metric { // FailureLimit is the maximum number of times the measurement is allowed to fail, before the // entire metric is considered Failed (default: 0) + // -1 for making it disabled (when opting to use ConsecutiveSuccessLimit solely) optional k8s.io.apimachinery.pkg.util.intstr.IntOrString failureLimit = 7; // InconclusiveLimit is the maximum number of times the measurement is allowed to measure @@ -1017,6 +1027,10 @@ message Metric { // Provider configuration to the external system to use to verify the analysis optional MetricProvider provider = 10; + + // ConsecutiveSuccessLimit is the number of consecutive times the measurement must succeed for the + // entire metric to be considered Successful (default: 0, which means it's disabled) + optional k8s.io.apimachinery.pkg.util.intstr.IntOrString consecutiveSuccessLimit = 11; } // MetricProvider which external system to use to verify the analysis @@ -1104,6 +1118,10 @@ message MetricResult { // the final state which gets used while taking measurements. For example, Prometheus uses this field // to store the final resolved query after substituting the template arguments. map metadata = 12; + + // ConsecutiveSuccess is the number of times a measurement was successful in succession + // Resets to zero when failures, inconclusive measurements, or errors are encountered + optional int32 consecutiveSuccess = 13; } // NewRelicMetric defines the newrelic query to perform canary analysis @@ -1113,6 +1131,10 @@ message NewRelicMetric { // Query is a raw newrelic NRQL query to perform optional string query = 2; + + // Timeout represents the duration limit in seconds that will apply to the NRQL query + // +optional + optional int64 timeout = 3; } // NginxTrafficRouting configuration for Nginx ingress controller to control traffic routing @@ -1130,6 +1152,9 @@ message NginxTrafficRouting { // StableIngresses refers to the names of `Ingress` resources in the same namespace as the `Rollout` in a multi ingress scenario // +optional repeated string stableIngresses = 4; + + // +optional + map canaryIngressAnnotations = 5; } message OAuth2Config { @@ -1178,6 +1203,17 @@ message PingPongSpec { optional string pongService = 2; } +message PluginStep { + // Name of the hashicorp go-plugin step to query + optional string name = 1; + + // +kubebuilder:validation:Schemaless + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Type=object + // Config is the configuration object for the specified plugin + optional bytes config = 2; +} + // PodTemplateMetadata extra labels to add to the template message PodTemplateMetadata { // Labels Additional labels to add to the experiment @@ -1219,6 +1255,22 @@ message PrometheusMetric { // +patchMergeKey=key // +patchStrategy=merge repeated WebMetricHeader headers = 6; + + // Arguments for prometheus + // +optional + optional PrometheusRangeQueryArgs rangeQuery = 7; +} + +// Arguments to perform a prometheus range query +message PrometheusRangeQueryArgs { + // The start time to query in expr format e.g. now(), now() - duration("1h"), now() - duration("{{args.lookback_duration}}") + optional string start = 1; + + // The end time to query in expr format e.g. now(), now() - duration("1h"), now() - duration("{{args.lookback_duration}}") + optional string end = 2; + + // The maximum time between two slices from the start to end (e.g. 30s, 5m, 1h). + optional string step = 3; } // RequiredDuringSchedulingIgnoredDuringExecution defines inter-pod scheduling rule to be RequiredDuringSchedulingIgnoredDuringExecution @@ -1679,6 +1731,14 @@ message SecretKeyRef { optional string key = 2; } +message SecretRef { + // Name refers to the name of the secret that should be used to integrate with Datadog. + optional string name = 1; + + // Namespaced indicates whether the secret is in the namespace where rollouts it installed or in the namespace where the metric was found + optional bool namespaced = 2; +} + // SetCanaryScale defines how to scale the newRS without changing traffic weight message SetCanaryScale { // Weight sets the percentage of replicas the newRS should have @@ -1737,6 +1797,47 @@ message SkyWalkingMetric { optional string interval = 3; } +message StepPluginStatus { + // Index is the matching step index of the executed plugin + optional int32 index = 1; + + // Name is the matching step name of the executed plugin + optional string name = 2; + + // Operation is the name of the operation that produced this status + optional string operation = 3; + + // Phase is the resulting phase of the operation + optional string phase = 4; + + // Message provides details on why the plugin is in its current phase + optional string message = 5; + + // StartedAt indicates when the plugin was first called for the operation + optional k8s.io.apimachinery.pkg.apis.meta.v1.Time startedAt = 6; + + // UpdatedAt indicates when the plugin was last called for the operation + optional k8s.io.apimachinery.pkg.apis.meta.v1.Time updatedAt = 7; + + // FinishedAt indicates when the operation was completed + optional k8s.io.apimachinery.pkg.apis.meta.v1.Time finishedAt = 8; + + // Backoff is a duration to wait before trying to execute the operation again if it was not completed + optional string backoff = 9; + + // Executions is the number of time the operation was executed + optional int32 executions = 10; + + // Disabled indicates if the plugin is globally disabled + optional bool disabled = 11; + + // +kubebuilder:validation:Schemaless + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Type=object + // Status holds the internal status of the plugin for this operation + optional bytes status = 12; +} + message StickinessConfig { optional bool enabled = 1; diff --git a/pkg/apis/rollouts/v1alpha1/openapi_generated.go b/pkg/apis/rollouts/v1alpha1/openapi_generated.go index 2d4d68ff1e..2f60871c4b 100644 --- a/pkg/apis/rollouts/v1alpha1/openapi_generated.go +++ b/pkg/apis/rollouts/v1alpha1/openapi_generated.go @@ -97,9 +97,11 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.ObjectRef": schema_pkg_apis_rollouts_v1alpha1_ObjectRef(ref), "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.PauseCondition": schema_pkg_apis_rollouts_v1alpha1_PauseCondition(ref), "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.PingPongSpec": schema_pkg_apis_rollouts_v1alpha1_PingPongSpec(ref), + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.PluginStep": schema_pkg_apis_rollouts_v1alpha1_PluginStep(ref), "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.PodTemplateMetadata": schema_pkg_apis_rollouts_v1alpha1_PodTemplateMetadata(ref), "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.PreferredDuringSchedulingIgnoredDuringExecution": schema_pkg_apis_rollouts_v1alpha1_PreferredDuringSchedulingIgnoredDuringExecution(ref), "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.PrometheusMetric": schema_pkg_apis_rollouts_v1alpha1_PrometheusMetric(ref), + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.PrometheusRangeQueryArgs": schema_pkg_apis_rollouts_v1alpha1_PrometheusRangeQueryArgs(ref), "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.RequiredDuringSchedulingIgnoredDuringExecution": schema_pkg_apis_rollouts_v1alpha1_RequiredDuringSchedulingIgnoredDuringExecution(ref), "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.RollbackWindowSpec": schema_pkg_apis_rollouts_v1alpha1_RollbackWindowSpec(ref), "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.Rollout": schema_pkg_apis_rollouts_v1alpha1_Rollout(ref), @@ -121,11 +123,13 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.SMITrafficRouting": schema_pkg_apis_rollouts_v1alpha1_SMITrafficRouting(ref), "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.ScopeDetail": schema_pkg_apis_rollouts_v1alpha1_ScopeDetail(ref), "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.SecretKeyRef": schema_pkg_apis_rollouts_v1alpha1_SecretKeyRef(ref), + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.SecretRef": schema_pkg_apis_rollouts_v1alpha1_SecretRef(ref), "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.SetCanaryScale": schema_pkg_apis_rollouts_v1alpha1_SetCanaryScale(ref), "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.SetHeaderRoute": schema_pkg_apis_rollouts_v1alpha1_SetHeaderRoute(ref), "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.SetMirrorRoute": schema_pkg_apis_rollouts_v1alpha1_SetMirrorRoute(ref), "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.Sigv4Config": schema_pkg_apis_rollouts_v1alpha1_Sigv4Config(ref), "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.SkyWalkingMetric": schema_pkg_apis_rollouts_v1alpha1_SkyWalkingMetric(ref), + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.StepPluginStatus": schema_pkg_apis_rollouts_v1alpha1_StepPluginStatus(ref), "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.StickinessConfig": schema_pkg_apis_rollouts_v1alpha1_StickinessConfig(ref), "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.StringMatch": schema_pkg_apis_rollouts_v1alpha1_StringMatch(ref), "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.TCPRoute": schema_pkg_apis_rollouts_v1alpha1_TCPRoute(ref), @@ -1417,11 +1421,25 @@ func schema_pkg_apis_rollouts_v1alpha1_CanaryStatus(ref common.ReferenceCallback Format: "", }, }, + "stepPluginStatuses": { + SchemaProps: spec.SchemaProps{ + Description: "StepPluginStatuses holds the status of the step plugins executed", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.StepPluginStatus"), + }, + }, + }, + }, + }, }, }, }, Dependencies: []string{ - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.RolloutAnalysisRunStatus", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.TrafficWeights"}, + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.RolloutAnalysisRunStatus", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.StepPluginStatus", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.TrafficWeights"}, } } @@ -1475,11 +1493,17 @@ func schema_pkg_apis_rollouts_v1alpha1_CanaryStep(ref common.ReferenceCallback) Ref: ref("github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.SetMirrorRoute"), }, }, + "plugin": { + SchemaProps: spec.SchemaProps{ + Description: "Plugin defines a plugin to execute for a step", + Ref: ref("github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.PluginStep"), + }, + }, }, }, }, Dependencies: []string{ - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.RolloutAnalysis", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.RolloutExperimentStep", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.RolloutPause", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.SetCanaryScale", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.SetHeaderRoute", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.SetMirrorRoute"}, + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.PluginStep", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.RolloutAnalysis", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.RolloutExperimentStep", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.RolloutPause", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.SetCanaryScale", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.SetHeaderRoute", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.SetMirrorRoute"}, } } @@ -1935,14 +1959,23 @@ func schema_pkg_apis_rollouts_v1alpha1_DatadogMetric(ref common.ReferenceCallbac }, "aggregator": { SchemaProps: spec.SchemaProps{ - Description: "Aggregator is a type of aggregator to use for metrics-based queries (default: last). Used for v2", + Description: "Aggregator is a type of aggregator to use for metrics-based queries (default: \"\"). Used for v2", Type: []string{"string"}, Format: "", }, }, + "secretRef": { + SchemaProps: spec.SchemaProps{ + Description: "Secret refers to the name of the secret that should be used for an analysis and should exists in the namespace where the controller is.", + Default: map[string]interface{}{}, + Ref: ref("github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.SecretRef"), + }, + }, }, }, }, + Dependencies: []string{ + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.SecretRef"}, } } @@ -3030,7 +3063,7 @@ func schema_pkg_apis_rollouts_v1alpha1_Metric(ref common.ReferenceCallback) comm }, "failureLimit": { SchemaProps: spec.SchemaProps{ - Description: "FailureLimit is the maximum number of times the measurement is allowed to fail, before the entire metric is considered Failed (default: 0)", + Description: "FailureLimit is the maximum number of times the measurement is allowed to fail, before the entire metric is considered Failed (default: 0) -1 for making it disabled (when opting to use ConsecutiveSuccessLimit solely)", Ref: ref("k8s.io/apimachinery/pkg/util/intstr.IntOrString"), }, }, @@ -3053,6 +3086,12 @@ func schema_pkg_apis_rollouts_v1alpha1_Metric(ref common.ReferenceCallback) comm Ref: ref("github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.MetricProvider"), }, }, + "consecutiveSuccessLimit": { + SchemaProps: spec.SchemaProps{ + Description: "ConsecutiveSuccessLimit is the number of consecutive times the measurement must succeed for the entire metric to be considered Successful (default: 0, which means it's disabled)", + Ref: ref("k8s.io/apimachinery/pkg/util/intstr.IntOrString"), + }, + }, }, Required: []string{"name", "provider"}, }, @@ -3267,6 +3306,13 @@ func schema_pkg_apis_rollouts_v1alpha1_MetricResult(ref common.ReferenceCallback }, }, }, + "consecutiveSuccess": { + SchemaProps: spec.SchemaProps{ + Description: "ConsecutiveSuccess is the number of times a measurement was successful in succession Resets to zero when failures, inconclusive measurements, or errors are encountered", + Type: []string{"integer"}, + Format: "int32", + }, + }, }, Required: []string{"name", "phase"}, }, @@ -3298,6 +3344,13 @@ func schema_pkg_apis_rollouts_v1alpha1_NewRelicMetric(ref common.ReferenceCallba Format: "", }, }, + "timeout": { + SchemaProps: spec.SchemaProps{ + Description: "Timeout represents the duration limit in seconds that will apply to the NRQL query", + Type: []string{"integer"}, + Format: "int64", + }, + }, }, Required: []string{"query"}, }, @@ -3356,6 +3409,21 @@ func schema_pkg_apis_rollouts_v1alpha1_NginxTrafficRouting(ref common.ReferenceC }, }, }, + "canaryIngressAnnotations": { + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, }, }, }, @@ -3509,6 +3577,34 @@ func schema_pkg_apis_rollouts_v1alpha1_PingPongSpec(ref common.ReferenceCallback } } +func schema_pkg_apis_rollouts_v1alpha1_PluginStep(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "name": { + SchemaProps: spec.SchemaProps{ + Description: "Name of the hashicorp go-plugin step to query", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "config": { + SchemaProps: spec.SchemaProps{ + Description: "Config is the configuration object for the specified plugin", + Type: []string{"string"}, + Format: "byte", + }, + }, + }, + Required: []string{"name"}, + }, + }, + } +} + func schema_pkg_apis_rollouts_v1alpha1_PodTemplateMetadata(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -3638,11 +3734,51 @@ func schema_pkg_apis_rollouts_v1alpha1_PrometheusMetric(ref common.ReferenceCall }, }, }, + "rangeQuery": { + SchemaProps: spec.SchemaProps{ + Description: "Arguments for prometheus", + Ref: ref("github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.PrometheusRangeQueryArgs"), + }, + }, }, }, }, Dependencies: []string{ - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.Authentication", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.WebMetricHeader"}, + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.Authentication", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.PrometheusRangeQueryArgs", "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1.WebMetricHeader"}, + } +} + +func schema_pkg_apis_rollouts_v1alpha1_PrometheusRangeQueryArgs(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Arguments to perform a prometheus range query", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "start": { + SchemaProps: spec.SchemaProps{ + Description: "The start time to query in expr format e.g. now(), now() - duration(\"1h\"), now() - duration(\"{{args.lookback_duration}}\")", + Type: []string{"string"}, + Format: "", + }, + }, + "end": { + SchemaProps: spec.SchemaProps{ + Description: "The end time to query in expr format e.g. now(), now() - duration(\"1h\"), now() - duration(\"{{args.lookback_duration}}\")", + Type: []string{"string"}, + Format: "", + }, + }, + "step": { + SchemaProps: spec.SchemaProps{ + Description: "The maximum time between two slices from the start to end (e.g. 30s, 5m, 1h).", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, } } @@ -4950,6 +5086,32 @@ func schema_pkg_apis_rollouts_v1alpha1_SecretKeyRef(ref common.ReferenceCallback } } +func schema_pkg_apis_rollouts_v1alpha1_SecretRef(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "name": { + SchemaProps: spec.SchemaProps{ + Description: "Name refers to the name of the secret that should be used to integrate with Datadog.", + Type: []string{"string"}, + Format: "", + }, + }, + "namespaced": { + SchemaProps: spec.SchemaProps{ + Description: "Namespaced indicates whether the secret is in the namespace where rollouts it installed or in the namespace where the metric was found", + Type: []string{"boolean"}, + Format: "", + }, + }, + }, + }, + }, + } +} + func schema_pkg_apis_rollouts_v1alpha1_SetCanaryScale(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -5126,6 +5288,105 @@ func schema_pkg_apis_rollouts_v1alpha1_SkyWalkingMetric(ref common.ReferenceCall } } +func schema_pkg_apis_rollouts_v1alpha1_StepPluginStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "index": { + SchemaProps: spec.SchemaProps{ + Description: "Index is the matching step index of the executed plugin", + Default: 0, + Type: []string{"integer"}, + Format: "int32", + }, + }, + "name": { + SchemaProps: spec.SchemaProps{ + Description: "Name is the matching step name of the executed plugin", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "operation": { + SchemaProps: spec.SchemaProps{ + Description: "Operation is the name of the operation that produced this status", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "phase": { + SchemaProps: spec.SchemaProps{ + Description: "Phase is the resulting phase of the operation", + Type: []string{"string"}, + Format: "", + }, + }, + "message": { + SchemaProps: spec.SchemaProps{ + Description: "Message provides details on why the plugin is in its current phase", + Type: []string{"string"}, + Format: "", + }, + }, + "startedAt": { + SchemaProps: spec.SchemaProps{ + Description: "StartedAt indicates when the plugin was first called for the operation", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Time"), + }, + }, + "updatedAt": { + SchemaProps: spec.SchemaProps{ + Description: "UpdatedAt indicates when the plugin was last called for the operation", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Time"), + }, + }, + "finishedAt": { + SchemaProps: spec.SchemaProps{ + Description: "FinishedAt indicates when the operation was completed", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Time"), + }, + }, + "backoff": { + SchemaProps: spec.SchemaProps{ + Description: "Backoff is a duration to wait before trying to execute the operation again if it was not completed", + Type: []string{"string"}, + Format: "", + }, + }, + "executions": { + SchemaProps: spec.SchemaProps{ + Description: "Executions is the number of time the operation was executed", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "disabled": { + SchemaProps: spec.SchemaProps{ + Description: "Disabled indicates if the plugin is globally disabled", + Type: []string{"boolean"}, + Format: "", + }, + }, + "status": { + SchemaProps: spec.SchemaProps{ + Description: "Status holds the internal status of the plugin for this operation", + Type: []string{"string"}, + Format: "byte", + }, + }, + }, + Required: []string{"index", "name", "operation"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.Time"}, + } +} + func schema_pkg_apis_rollouts_v1alpha1_StickinessConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ diff --git a/pkg/apis/rollouts/v1alpha1/types.go b/pkg/apis/rollouts/v1alpha1/types.go index 2ab4ea1018..8886e292be 100755 --- a/pkg/apis/rollouts/v1alpha1/types.go +++ b/pkg/apis/rollouts/v1alpha1/types.go @@ -2,6 +2,7 @@ package v1alpha1 import ( "encoding/json" + fmt "fmt" "strconv" "time" @@ -446,6 +447,8 @@ type NginxTrafficRouting struct { // StableIngresses refers to the names of `Ingress` resources in the same namespace as the `Rollout` in a multi ingress scenario // +optional StableIngresses []string `json:"stableIngresses,omitempty" protobuf:"bytes,4,rep,name=stableIngresses"` + // +optional + CanaryIngressAnnotations map[string]string `json:"canaryIngressAnnotations,omitempty" protobuf:"bytes,5,rep,name=canaryIngressAnnotations"` } // IstioTrafficRouting configuration for Istio service mesh to enable fine grain configuration @@ -636,6 +639,19 @@ type CanaryStep struct { // SetMirrorRoutes Mirrors traffic that matches rules to a particular destination // +optional SetMirrorRoute *SetMirrorRoute `json:"setMirrorRoute,omitempty" protobuf:"bytes,8,opt,name=setMirrorRoute"` + // Plugin defines a plugin to execute for a step + Plugin *PluginStep `json:"plugin,omitempty" protobuf:"bytes,9,opt,name=plugin"` +} + +type PluginStep struct { + // Name of the hashicorp go-plugin step to query + Name string `json:"name" protobuf:"bytes,1,opt,name=name"` + + // +kubebuilder:validation:Schemaless + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Type=object + // Config is the configuration object for the specified plugin + Config json.RawMessage `json:"config,omitempty" protobuf:"bytes,2,opt,name=config"` } type SetMirrorRoute struct { @@ -984,6 +1000,8 @@ type CanaryStatus struct { Weights *TrafficWeights `json:"weights,omitempty" protobuf:"bytes,4,opt,name=weights"` // StablePingPong For the ping-pong feature holds the current stable service, ping or pong StablePingPong PingPongType `json:"stablePingPong,omitempty" protobuf:"bytes,5,opt,name=stablePingPong"` + // StepPluginStatuses holds the status of the step plugins executed + StepPluginStatuses []StepPluginStatus `json:"stepPluginStatuses,omitempty" protobuf:"bytes,6,rep,name=stepPluginStatuses"` } type PingPongType string @@ -1021,6 +1039,78 @@ type RolloutAnalysisRunStatus struct { Message string `json:"message,omitempty" protobuf:"bytes,3,opt,name=message"` } +type StepPluginStatus struct { + // Index is the matching step index of the executed plugin + Index int32 `json:"index" protobuf:"bytes,1,name=index"` + // Name is the matching step name of the executed plugin + Name string `json:"name" protobuf:"bytes,2,name=name"` + // Operation is the name of the operation that produced this status + Operation StepPluginOperation `json:"operation" protobuf:"bytes,3,name=operation,casttype=StepPluginOperation"` + // Phase is the resulting phase of the operation + Phase StepPluginPhase `json:"phase,omitempty" protobuf:"bytes,4,opt,name=phase,casttype=StepPluginPhase"` + // Message provides details on why the plugin is in its current phase + Message string `json:"message,omitempty" protobuf:"bytes,5,opt,name=message"` + // StartedAt indicates when the plugin was first called for the operation + StartedAt *metav1.Time `json:"startedAt,omitempty" protobuf:"bytes,6,name=startedAt"` + // UpdatedAt indicates when the plugin was last called for the operation + UpdatedAt *metav1.Time `json:"updatedAt,omitempty" protobuf:"bytes,7,opt,name=updatedAt"` + // FinishedAt indicates when the operation was completed + FinishedAt *metav1.Time `json:"finishedAt,omitempty" protobuf:"bytes,8,opt,name=finishedAt"` + // Backoff is a duration to wait before trying to execute the operation again if it was not completed + Backoff DurationString `json:"backoff,omitempty" protobuf:"bytes,9,opt,name=backoff,casttype=DurationString"` + // Executions is the number of time the operation was executed + Executions int32 `json:"executions,omitempty" protobuf:"varint,10,opt,name=executions"` + // Disabled indicates if the plugin is globally disabled + Disabled bool `json:"disabled,omitempty" protobuf:"bytes,11,opt,name=disabled"` + + // +kubebuilder:validation:Schemaless + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Type=object + // Status holds the internal status of the plugin for this operation + Status json.RawMessage `json:"status,omitempty" protobuf:"bytes,12,opt,name=status"` +} + +// StepPluginPhase is the overall phase of a StepPlugin +type StepPluginPhase string + +// Possible StepPluginPhase values +const ( + // StepPluginPhaseRunning is the phase of a step plugin when it has not completed its execution + StepPluginPhaseRunning StepPluginPhase = "Running" + // StepPluginPhaseSuccessful is the phase of a step plugin when the operation completed successfully + StepPluginPhaseSuccessful StepPluginPhase = "Successful" + // StepPluginPhaseFailed is the phase of a step plugin when the operation completed unsuccessfully + StepPluginPhaseFailed StepPluginPhase = "Failed" + // StepPluginPhaseError is the phase of a step plugin when an unexpected error prevented the completion of the operation + StepPluginPhaseError StepPluginPhase = "Error" +) + +// Validate that the object is a valid phase +func (p StepPluginPhase) Validate() error { + switch p { + case StepPluginPhaseRunning: + case StepPluginPhaseSuccessful: + case StepPluginPhaseFailed: + case StepPluginPhaseError: + default: + return fmt.Errorf("phase '%s' is not valid", p) + } + return nil +} + +// StepPluginOperation is the operation executed by a step plugin +type StepPluginOperation string + +// Possible StepPluginOperation values +const ( + // StepPluginOperationRun is the value for the Run operation + StepPluginOperationRun StepPluginOperation = "Run" + // StepPluginOperationRun is the value for the Terminate operation + StepPluginOperationTerminate StepPluginOperation = "Terminate" + // StepPluginOperationRun is the value for the Abort operation + StepPluginOperationAbort StepPluginOperation = "Abort" +) + type ALBStatus struct { LoadBalancer AwsResourceRef `json:"loadBalancer,omitempty" protobuf:"bytes,1,opt,name=loadBalancer"` CanaryTargetGroup AwsResourceRef `json:"canaryTargetGroup,omitempty" protobuf:"bytes,2,opt,name=canaryTargetGroup"` diff --git a/pkg/apis/rollouts/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/rollouts/v1alpha1/zz_generated.deepcopy.go index 496cedc2a0..44cfd72188 100644 --- a/pkg/apis/rollouts/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/rollouts/v1alpha1/zz_generated.deepcopy.go @@ -794,6 +794,13 @@ func (in *CanaryStatus) DeepCopyInto(out *CanaryStatus) { *out = new(TrafficWeights) (*in).DeepCopyInto(*out) } + if in.StepPluginStatuses != nil { + in, out := &in.StepPluginStatuses, &out.StepPluginStatuses + *out = make([]StepPluginStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } @@ -845,6 +852,11 @@ func (in *CanaryStep) DeepCopyInto(out *CanaryStep) { *out = new(SetMirrorRoute) (*in).DeepCopyInto(*out) } + if in.Plugin != nil { + in, out := &in.Plugin, &out.Plugin + *out = new(PluginStep) + (*in).DeepCopyInto(*out) + } return } @@ -1135,6 +1147,7 @@ func (in *DatadogMetric) DeepCopyInto(out *DatadogMetric) { (*out)[key] = val } } + out.SecretRef = in.SecretRef return } @@ -1688,6 +1701,11 @@ func (in *Metric) DeepCopyInto(out *Metric) { **out = **in } in.Provider.DeepCopyInto(&out.Provider) + if in.ConsecutiveSuccessLimit != nil { + in, out := &in.ConsecutiveSuccessLimit, &out.ConsecutiveSuccessLimit + *out = new(intstr.IntOrString) + **out = **in + } return } @@ -1732,7 +1750,7 @@ func (in *MetricProvider) DeepCopyInto(out *MetricProvider) { if in.NewRelic != nil { in, out := &in.NewRelic, &out.NewRelic *out = new(NewRelicMetric) - **out = **in + (*in).DeepCopyInto(*out) } if in.Job != nil { in, out := &in.Job, &out.Job @@ -1820,6 +1838,11 @@ func (in *MetricResult) DeepCopy() *MetricResult { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NewRelicMetric) DeepCopyInto(out *NewRelicMetric) { *out = *in + if in.Timeout != nil { + in, out := &in.Timeout, &out.Timeout + *out = new(int64) + **out = **in + } return } @@ -1848,6 +1871,13 @@ func (in *NginxTrafficRouting) DeepCopyInto(out *NginxTrafficRouting) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.CanaryIngressAnnotations != nil { + in, out := &in.CanaryIngressAnnotations, &out.CanaryIngressAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } return } @@ -1931,6 +1961,27 @@ func (in *PingPongSpec) DeepCopy() *PingPongSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PluginStep) DeepCopyInto(out *PluginStep) { + *out = *in + if in.Config != nil { + in, out := &in.Config, &out.Config + *out = make(json.RawMessage, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PluginStep. +func (in *PluginStep) DeepCopy() *PluginStep { + if in == nil { + return nil + } + out := new(PluginStep) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodTemplateMetadata) DeepCopyInto(out *PodTemplateMetadata) { *out = *in @@ -1991,6 +2042,11 @@ func (in *PrometheusMetric) DeepCopyInto(out *PrometheusMetric) { *out = make([]WebMetricHeader, len(*in)) copy(*out, *in) } + if in.RangeQuery != nil { + in, out := &in.RangeQuery, &out.RangeQuery + *out = new(PrometheusRangeQueryArgs) + **out = **in + } return } @@ -2004,6 +2060,22 @@ func (in *PrometheusMetric) DeepCopy() *PrometheusMetric { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PrometheusRangeQueryArgs) DeepCopyInto(out *PrometheusRangeQueryArgs) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrometheusRangeQueryArgs. +func (in *PrometheusRangeQueryArgs) DeepCopy() *PrometheusRangeQueryArgs { + if in == nil { + return nil + } + out := new(PrometheusRangeQueryArgs) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RequiredDuringSchedulingIgnoredDuringExecution) DeepCopyInto(out *RequiredDuringSchedulingIgnoredDuringExecution) { *out = *in @@ -2630,6 +2702,22 @@ func (in *SecretKeyRef) DeepCopy() *SecretKeyRef { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretRef) DeepCopyInto(out *SecretRef) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretRef. +func (in *SecretRef) DeepCopy() *SecretRef { + if in == nil { + return nil + } + out := new(SecretRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SetCanaryScale) DeepCopyInto(out *SetCanaryScale) { *out = *in @@ -2739,6 +2827,39 @@ func (in *SkyWalkingMetric) DeepCopy() *SkyWalkingMetric { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StepPluginStatus) DeepCopyInto(out *StepPluginStatus) { + *out = *in + if in.StartedAt != nil { + in, out := &in.StartedAt, &out.StartedAt + *out = (*in).DeepCopy() + } + if in.UpdatedAt != nil { + in, out := &in.UpdatedAt, &out.UpdatedAt + *out = (*in).DeepCopy() + } + if in.FinishedAt != nil { + in, out := &in.FinishedAt, &out.FinishedAt + *out = (*in).DeepCopy() + } + if in.Status != nil { + in, out := &in.Status, &out.Status + *out = make(json.RawMessage, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StepPluginStatus. +func (in *StepPluginStatus) DeepCopy() *StepPluginStatus { + if in == nil { + return nil + } + out := new(StepPluginStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StickinessConfig) DeepCopyInto(out *StickinessConfig) { *out = *in diff --git a/pkg/apis/rollouts/validation/validation.go b/pkg/apis/rollouts/validation/validation.go index 8eb7cdac19..d9c31a37f1 100644 --- a/pkg/apis/rollouts/validation/validation.go +++ b/pkg/apis/rollouts/validation/validation.go @@ -48,8 +48,8 @@ const ( InvalidDurationMessage = "Duration needs to be greater than 0" // InvalidMaxSurgeMaxUnavailable indicates both maxSurge and MaxUnavailable can not be set to zero InvalidMaxSurgeMaxUnavailable = "MaxSurge and MaxUnavailable both can not be zero" - // InvalidStepMessage indicates that a step must have either setWeight or pause set - InvalidStepMessage = "Step must have one of the following set: experiment, setWeight, setCanaryScale or pause" + // InvalidStepMessage indicates that a step must have either experiment, setWeight, setCanaryScale, plugin or pause + InvalidStepMessage = "Step must have one of the following set: experiment, setWeight, setCanaryScale, plugin or pause" // InvalidStrategyMessage indicates that multiple strategies can not be listed InvalidStrategyMessage = "Multiple Strategies can not be listed" // DuplicatedServicesBlueGreenMessage the message to indicate that the rollout uses the same service for the active and preview services @@ -308,9 +308,9 @@ func ValidateRolloutStrategyCanary(rollout *v1alpha1.Rollout, fldPath *field.Pat stepFldPath := fldPath.Child("steps").Index(i) allErrs = append(allErrs, hasMultipleStepsType(step, stepFldPath)...) if step.Experiment == nil && step.Pause == nil && step.SetWeight == nil && step.Analysis == nil && step.SetCanaryScale == nil && - step.SetHeaderRoute == nil && step.SetMirrorRoute == nil { - errVal := fmt.Sprintf("step.Experiment: %t step.Pause: %t step.SetWeight: %t step.Analysis: %t step.SetCanaryScale: %t step.SetHeaderRoute: %t step.SetMirrorRoutes: %t", - step.Experiment == nil, step.Pause == nil, step.SetWeight == nil, step.Analysis == nil, step.SetCanaryScale == nil, step.SetHeaderRoute == nil, step.SetMirrorRoute == nil) + step.SetHeaderRoute == nil && step.SetMirrorRoute == nil && step.Plugin == nil { + errVal := fmt.Sprintf("step.Experiment: %t step.Pause: %t step.SetWeight: %t step.Analysis: %t step.SetCanaryScale: %t step.SetHeaderRoute: %t step.SetMirrorRoute: %t step.Plugin: %t", + step.Experiment == nil, step.Pause == nil, step.SetWeight == nil, step.Analysis == nil, step.SetCanaryScale == nil, step.SetHeaderRoute == nil, step.SetMirrorRoute == nil, step.Plugin == nil) allErrs = append(allErrs, field.Invalid(stepFldPath, errVal, InvalidStepMessage)) } @@ -387,8 +387,8 @@ func ValidateRolloutStrategyCanary(rollout *v1alpha1.Rollout, fldPath *field.Pat if template.Weight != nil { if canary.TrafficRouting == nil { allErrs = append(allErrs, field.Invalid(stepFldPath.Child("experiment").Child("templates").Index(tmplIndex).Child("weight"), *canary.Steps[i].Experiment.Templates[tmplIndex].Weight, InvalidCanaryExperimentTemplateWeightWithoutTrafficRouting)) - } else if canary.TrafficRouting.ALB == nil && canary.TrafficRouting.SMI == nil && canary.TrafficRouting.Istio == nil { - allErrs = append(allErrs, field.Invalid(stepFldPath.Child("experiment").Child("templates").Index(tmplIndex).Child("weight"), *canary.Steps[i].Experiment.Templates[tmplIndex].Weight, "Experiment template weight is only available for TrafficRouting with SMI, ALB, and Istio at this time")) + } else if canary.TrafficRouting.ALB == nil && canary.TrafficRouting.SMI == nil && canary.TrafficRouting.Istio == nil && len(canary.TrafficRouting.Plugins) == 0 { + allErrs = append(allErrs, field.Invalid(stepFldPath.Child("experiment").Child("templates").Index(tmplIndex).Child("weight"), *canary.Steps[i].Experiment.Templates[tmplIndex].Weight, "Experiment template weight is only available for TrafficRouting with SMI, ALB, Istio and Plugins at this time")) } } } diff --git a/pkg/apis/rollouts/validation/validation_references.go b/pkg/apis/rollouts/validation/validation_references.go index 9079536c20..f51df2925d 100644 --- a/pkg/apis/rollouts/validation/validation_references.go +++ b/pkg/apis/rollouts/validation/validation_references.go @@ -1,6 +1,7 @@ package validation import ( + "errors" "fmt" appsv1 "k8s.io/api/apps/v1" @@ -342,11 +343,11 @@ func ValidateRolloutVirtualServicesConfig(r *v1alpha1.Rollout) error { if canary.TrafficRouting != nil && canary.TrafficRouting.Istio != nil { if istioutil.MultipleVirtualServiceConfigured(r) { if r.Spec.Strategy.Canary.TrafficRouting.Istio.VirtualService != nil { - return field.InternalError(fldPath, fmt.Errorf(errorString)) + return field.InternalError(fldPath, errors.New(errorString)) } } else { if r.Spec.Strategy.Canary.TrafficRouting.Istio.VirtualService == nil { - return field.InternalError(fldPath, fmt.Errorf(errorString)) + return field.InternalError(fldPath, errors.New(errorString)) } } } diff --git a/pkg/apis/rollouts/validation/validation_test.go b/pkg/apis/rollouts/validation/validation_test.go index c9fc9ad923..84fc93b5dd 100644 --- a/pkg/apis/rollouts/validation/validation_test.go +++ b/pkg/apis/rollouts/validation/validation_test.go @@ -17,7 +17,7 @@ import ( ) const ( - errTrafficRoutingWithExperimentSupport = "Experiment template weight is only available for TrafficRouting with SMI, ALB, and Istio at this time" + errTrafficRoutingWithExperimentSupport = "Experiment template weight is only available for TrafficRouting with SMI, ALB, Istio and Plugins at this time" ) func TestValidateRollout(t *testing.T) { @@ -1094,4 +1094,15 @@ func TestCanaryExperimentStepWithWeight(t *testing.T) { assert.Equal(t, 1, len(allErrs)) assert.Equal(t, errTrafficRoutingWithExperimentSupport, allErrs[0].Detail) }) + + t.Run("success - Plugins", func(t *testing.T) { + invalidRo := ro.DeepCopy() + invalidRo.Spec.Strategy.Canary.TrafficRouting = &v1alpha1.RolloutTrafficRouting{ + Plugins: map[string]json.RawMessage{ + "any/plugin": {}, + }, + } + allErrs := ValidateRolloutStrategyCanary(invalidRo, field.NewPath("")) + assert.Equal(t, 0, len(allErrs)) + }) } diff --git a/pkg/kubectl-argo-rollouts/cmd/cmd.go b/pkg/kubectl-argo-rollouts/cmd/cmd.go index 372f8c434d..24610550cd 100644 --- a/pkg/kubectl-argo-rollouts/cmd/cmd.go +++ b/pkg/kubectl-argo-rollouts/cmd/cmd.go @@ -1,6 +1,8 @@ package cmd import ( + notificationcmd "github.com/argoproj/notifications-engine/pkg/cmd" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/cmd/abort" "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/cmd/completion" @@ -20,7 +22,6 @@ import ( "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/cmd/version" "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/options" "github.com/argoproj/argo-rollouts/utils/record" - notificationcmd "github.com/argoproj/notifications-engine/pkg/cmd" "github.com/spf13/cobra" ) diff --git a/pkg/kubectl-argo-rollouts/cmd/completion/completion.go b/pkg/kubectl-argo-rollouts/cmd/completion/completion.go index bc045f07b8..4f51ee45fe 100644 --- a/pkg/kubectl-argo-rollouts/cmd/completion/completion.go +++ b/pkg/kubectl-argo-rollouts/cmd/completion/completion.go @@ -1,8 +1,9 @@ package completion import ( - "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/options" "github.com/spf13/cobra" + + "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/options" ) func NewCmdCompletion(o *options.ArgoRolloutsOptions) *cobra.Command { diff --git a/pkg/kubectl-argo-rollouts/cmd/completion/completion_test.go b/pkg/kubectl-argo-rollouts/cmd/completion/completion_test.go index 64bd376c40..7e62fd603d 100644 --- a/pkg/kubectl-argo-rollouts/cmd/completion/completion_test.go +++ b/pkg/kubectl-argo-rollouts/cmd/completion/completion_test.go @@ -4,8 +4,9 @@ import ( "bytes" "testing" - options "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/options/fake" "github.com/tj/assert" + + options "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/options/fake" ) func TestShellNotFound(t *testing.T) { diff --git a/pkg/kubectl-argo-rollouts/cmd/create/create_test.go b/pkg/kubectl-argo-rollouts/cmd/create/create_test.go index b4cf5bb16e..e920067a30 100644 --- a/pkg/kubectl-argo-rollouts/cmd/create/create_test.go +++ b/pkg/kubectl-argo-rollouts/cmd/create/create_test.go @@ -11,9 +11,10 @@ import ( "k8s.io/apimachinery/pkg/runtime" core "k8s.io/client-go/testing" + dynamicfake "k8s.io/client-go/dynamic/fake" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" options "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/options/fake" - dynamicfake "k8s.io/client-go/dynamic/fake" ) func TestCreateRollout(t *testing.T) { diff --git a/pkg/kubectl-argo-rollouts/cmd/dashboard/dashboard.go b/pkg/kubectl-argo-rollouts/cmd/dashboard/dashboard.go index f0c720df72..f431a479ba 100644 --- a/pkg/kubectl-argo-rollouts/cmd/dashboard/dashboard.go +++ b/pkg/kubectl-argo-rollouts/cmd/dashboard/dashboard.go @@ -3,17 +3,28 @@ package dashboard import ( "context" + "github.com/spf13/cobra" + "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/options" "github.com/argoproj/argo-rollouts/server" - "github.com/spf13/cobra" +) + +var ( + dashBoardExample = ` + # Start UI dashboard + %[1]s dashboard + + # Start UI dashboard on a specific port + %[1]s dashboard --port 8080` ) func NewCmdDashboard(o *options.ArgoRolloutsOptions) *cobra.Command { var rootPath string var port int var cmd = &cobra.Command{ - Use: "dashboard", - Short: "Start UI dashboard", + Use: "dashboard", + Short: "Start UI dashboard", + Example: o.Example(dashBoardExample), RunE: func(c *cobra.Command, args []string) error { namespace := o.Namespace() kubeclientset := o.KubeClientset() diff --git a/pkg/kubectl-argo-rollouts/cmd/get/get_rollout.go b/pkg/kubectl-argo-rollouts/cmd/get/get_rollout.go index f5b0719509..dd56693a45 100644 --- a/pkg/kubectl-argo-rollouts/cmd/get/get_rollout.go +++ b/pkg/kubectl-argo-rollouts/cmd/get/get_rollout.go @@ -25,7 +25,10 @@ const ( %[1]s get rollout guestbook # Watch progress of a rollout - %[1]s get rollout guestbook -w` + %[1]s get rollout guestbook -w + + # Watch the rollout, fail if it takes more than 60 seconds + %[1]s get rollout guestbook -w --timeout-seconds 60` ) // NewCmdGetRollout returns a new instance of an `rollouts get rollout` command diff --git a/pkg/kubectl-argo-rollouts/cmd/lint/lint.go b/pkg/kubectl-argo-rollouts/cmd/lint/lint.go index 810abf2c8e..8043dd848e 100644 --- a/pkg/kubectl-argo-rollouts/cmd/lint/lint.go +++ b/pkg/kubectl-argo-rollouts/cmd/lint/lint.go @@ -6,11 +6,6 @@ import ( "io" "os" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/validation" - "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/options" - ingressutil "github.com/argoproj/argo-rollouts/utils/ingress" "github.com/spf13/cobra" goyaml "gopkg.in/yaml.v2" v1 "k8s.io/api/core/v1" @@ -19,6 +14,12 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/validation/field" "sigs.k8s.io/yaml" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/validation" + "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/options" + ingressutil "github.com/argoproj/argo-rollouts/utils/ingress" ) type LintOptions struct { @@ -53,12 +54,7 @@ func NewCmdLint(o *options.ArgoRolloutsOptions) *cobra.Command { return o.UsageErr(c) } - err := lintOptions.lintResource(lintOptions.File) - if err != nil { - return err - } - - return nil + return lintOptions.lintResource(lintOptions.File) }, } cmd.Flags().StringVarP(&lintOptions.File, "filename", "f", "", "File to lint") diff --git a/pkg/kubectl-argo-rollouts/cmd/lint/lint_test.go b/pkg/kubectl-argo-rollouts/cmd/lint/lint_test.go index e9b66fd2ed..d5f532d6de 100644 --- a/pkg/kubectl-argo-rollouts/cmd/lint/lint_test.go +++ b/pkg/kubectl-argo-rollouts/cmd/lint/lint_test.go @@ -4,8 +4,9 @@ import ( "bytes" "testing" - options "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/options/fake" "github.com/stretchr/testify/assert" + + options "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/options/fake" ) func TestLintValidRollout(t *testing.T) { @@ -85,7 +86,7 @@ func TestLintInvalidRollout(t *testing.T) { }, { filename: "testdata/invalid-nginx-canary.yml", - errmsg: "Error: spec.strategy.steps[1].experiment.templates[0].weight: Invalid value: 20: Experiment template weight is only available for TrafficRouting with SMI, ALB, and Istio at this time\n", + errmsg: "Error: spec.strategy.steps[1].experiment.templates[0].weight: Invalid value: 20: Experiment template weight is only available for TrafficRouting with SMI, ALB, Istio and Plugins at this time\n", }, } diff --git a/pkg/kubectl-argo-rollouts/cmd/list/list_experiments.go b/pkg/kubectl-argo-rollouts/cmd/list/list_experiments.go index cb83a157fa..4b4ec6b302 100644 --- a/pkg/kubectl-argo-rollouts/cmd/list/list_experiments.go +++ b/pkg/kubectl-argo-rollouts/cmd/list/list_experiments.go @@ -62,7 +62,7 @@ func NewCmdListExperiments(o *options.ArgoRolloutsOptions) *cobra.Command { return nil }, } - cmd.Flags().BoolVar(&listOptions.allNamespaces, "all-namespaces", false, "Include all namespaces") + cmd.Flags().BoolVarP(&listOptions.allNamespaces, "all-namespaces", "A", false, "Include all namespaces") return cmd } @@ -84,7 +84,7 @@ func (o *ListOptions) PrintExperimentTable(expList *v1alpha1.ExperimentList) err headerStr = "NAMESPACE\t" + headerStr fmtStr = "%-9s\t" + fmtStr } - fmt.Fprintf(w, headerStr) + fmt.Fprint(w, headerStr) for _, exp := range expList.Items { age := duration.HumanDuration(timeutil.MetaNow().Sub(exp.CreationTimestamp.Time)) dur := "-" diff --git a/pkg/kubectl-argo-rollouts/cmd/list/list_rollouts.go b/pkg/kubectl-argo-rollouts/cmd/list/list_rollouts.go index 6746876767..16dcc68c75 100644 --- a/pkg/kubectl-argo-rollouts/cmd/list/list_rollouts.go +++ b/pkg/kubectl-argo-rollouts/cmd/list/list_rollouts.go @@ -18,6 +18,9 @@ const ( listRolloutsExample = ` # List rollouts %[1]s list rollouts + + # List rollouts with a specific name + %[1]s list rollouts --name my-rollout # List rollouts from all namespaces %[1]s list rollouts --all-namespaces @@ -90,7 +93,7 @@ func (o *ListOptions) PrintRolloutTable(roList *v1alpha1.RolloutList) error { if o.timestamps { headerStr = "TIMESTAMP\t" + headerStr } - fmt.Fprintf(w, headerStr) + fmt.Fprint(w, headerStr) for _, ro := range roList.Items { roLine := newRolloutInfo(ro) fmt.Fprintln(w, roLine.String(o.timestamps, o.allNamespaces)) diff --git a/pkg/kubectl-argo-rollouts/cmd/set/set_image.go b/pkg/kubectl-argo-rollouts/cmd/set/set_image.go index 1a5b5dce24..942db298bc 100644 --- a/pkg/kubectl-argo-rollouts/cmd/set/set_image.go +++ b/pkg/kubectl-argo-rollouts/cmd/set/set_image.go @@ -19,8 +19,11 @@ import ( const ( setImageExample = ` - # Set rollout image - %[1]s set image my-rollout www=image:v2` + # Set rollout image (containers contains 'initContainer', 'container', 'ephemeralContainer') + %[1]s set image my-rollout containerName=imageName + + # Set rollout image for all containers + %[1]s set image my-rollout *=imageName` ) const ( diff --git a/pkg/kubectl-argo-rollouts/cmd/status/status.go b/pkg/kubectl-argo-rollouts/cmd/status/status.go index 79d7272d88..4d5841394c 100644 --- a/pkg/kubectl-argo-rollouts/cmd/status/status.go +++ b/pkg/kubectl-argo-rollouts/cmd/status/status.go @@ -5,12 +5,13 @@ import ( "fmt" "time" + "github.com/spf13/cobra" + "github.com/argoproj/argo-rollouts/pkg/apiclient/rollout" "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/cmd/signals" "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/options" completionutil "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/util/completion" "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/viewcontroller" - "github.com/spf13/cobra" ) const ( @@ -20,6 +21,9 @@ the rollout is healthy upon completion and an error otherwise.` # Watch the rollout until it succeeds %[1]s status guestbook + # Show the rollout status + %[1]s status guestbook --watch false + # Watch the rollout until it succeeds, fail if it takes more than 60 seconds %[1]s status --timeout 60s guestbook ` diff --git a/pkg/kubectl-argo-rollouts/cmd/status/status_test.go b/pkg/kubectl-argo-rollouts/cmd/status/status_test.go index ddbee9b24e..20e4b91078 100644 --- a/pkg/kubectl-argo-rollouts/cmd/status/status_test.go +++ b/pkg/kubectl-argo-rollouts/cmd/status/status_test.go @@ -4,9 +4,10 @@ import ( "bytes" "testing" + "github.com/stretchr/testify/assert" + "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/info/testdata" options "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/options/fake" - "github.com/stretchr/testify/assert" ) const noWatch = "--watch=false" diff --git a/pkg/kubectl-argo-rollouts/cmd/terminate/terminate.go b/pkg/kubectl-argo-rollouts/cmd/terminate/terminate.go index f0dd7e91b1..c4123f0e78 100644 --- a/pkg/kubectl-argo-rollouts/cmd/terminate/terminate.go +++ b/pkg/kubectl-argo-rollouts/cmd/terminate/terminate.go @@ -18,7 +18,7 @@ const ( const ( terminateExample = ` - # Terminate an analysisRun + # Terminate an AnalysisRun %[1]s terminate analysisrun guestbook-877894d5b-4-success-rate.1 # Terminate a failed experiment @@ -26,7 +26,7 @@ const ( terminateAnalysisRunExample = ` # Terminate an AnalysisRun - %[1]s terminate analysis guestbook-877894d5b-4-success-rate.1` + %[1]s terminate analysisrun guestbook-877894d5b-4-success-rate.1` terminateExperimentExample = ` # Terminate an experiment diff --git a/pkg/kubectl-argo-rollouts/cmd/undo/undo.go b/pkg/kubectl-argo-rollouts/cmd/undo/undo.go index d3a78d15f2..4c74235edd 100644 --- a/pkg/kubectl-argo-rollouts/cmd/undo/undo.go +++ b/pkg/kubectl-argo-rollouts/cmd/undo/undo.go @@ -15,14 +15,15 @@ import ( "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" - "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/options" - completionutil "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/util/completion" - routils "github.com/argoproj/argo-rollouts/utils/unstructured" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" appsclient "k8s.io/client-go/kubernetes/typed/apps/v1" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/options" + completionutil "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/util/completion" + routils "github.com/argoproj/argo-rollouts/utils/unstructured" ) const ( @@ -32,7 +33,7 @@ const ( # Undo a rollout %[1]s undo guestbook - # Undo a rollout revision 3 + # Undo a rollout to revision 3 %[1]s undo guestbook --to-revision=3` ) @@ -58,7 +59,7 @@ func NewCmdUndo(o *options.ArgoRolloutsOptions) *cobra.Command { if err != nil { return err } - fmt.Fprintf(o.Out, result) + fmt.Fprint(o.Out, result) return nil }, ValidArgsFunction: completionutil.RolloutNameCompletionFunc(o), diff --git a/pkg/kubectl-argo-rollouts/cmd/undo/undo_test.go b/pkg/kubectl-argo-rollouts/cmd/undo/undo_test.go index da9099d5de..1eeff66014 100644 --- a/pkg/kubectl-argo-rollouts/cmd/undo/undo_test.go +++ b/pkg/kubectl-argo-rollouts/cmd/undo/undo_test.go @@ -7,7 +7,6 @@ import ( "fmt" "testing" - "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/info/testdata" "github.com/stretchr/testify/assert" v1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -16,6 +15,8 @@ import ( dynamicfake "k8s.io/client-go/dynamic/fake" kubetesting "k8s.io/client-go/testing" + "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/info/testdata" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" options "github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/options/fake" ) diff --git a/pkg/kubectl-argo-rollouts/info/pod_info.go b/pkg/kubectl-argo-rollouts/info/pod_info.go index e53f350848..88ce420de0 100644 --- a/pkg/kubectl-argo-rollouts/info/pod_info.go +++ b/pkg/kubectl-argo-rollouts/info/pod_info.go @@ -5,11 +5,12 @@ import ( "sort" "strings" - "github.com/argoproj/argo-rollouts/pkg/apiclient/rollout" corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" k8snode "k8s.io/kubernetes/pkg/util/node" + + "github.com/argoproj/argo-rollouts/pkg/apiclient/rollout" ) func addPodInfos(rsInfos []*rollout.ReplicaSetInfo, allPods []*corev1.Pod) []*rollout.ReplicaSetInfo { @@ -53,6 +54,12 @@ func newPodInfo(pod *corev1.Pod) rollout.PodInfo { }, } restarts := 0 + rs := make(map[string]bool, len(pod.Spec.InitContainers)) + for _, c := range pod.Spec.InitContainers { + p := c.RestartPolicy + rs[c.Name] = p != nil && *p == corev1.ContainerRestartPolicyAlways + } + totalContainers := len(pod.Spec.Containers) readyContainers := 0 @@ -69,7 +76,7 @@ func newPodInfo(pod *corev1.Pod) rollout.PodInfo { continue case container.State.Terminated != nil: // initialization is failed - if len(container.State.Terminated.Reason) == 0 { + if container.State.Terminated.Reason == "" { if container.State.Terminated.Signal != 0 { reason = fmt.Sprintf("Init:Signal:%d", container.State.Terminated.Signal) } else { @@ -79,6 +86,10 @@ func newPodInfo(pod *corev1.Pod) rollout.PodInfo { reason = "Init:" + container.State.Terminated.Reason } initializing = true + case rs[container.Name] && container.Started != nil && *container.Started: + if container.Ready { + continue + } case container.State.Waiting != nil && len(container.State.Waiting.Reason) > 0 && container.State.Waiting.Reason != "PodInitializing": reason = "Init:" + container.State.Waiting.Reason initializing = true diff --git a/pkg/kubectl-argo-rollouts/viewcontroller/viewcontroller_test.go b/pkg/kubectl-argo-rollouts/viewcontroller/viewcontroller_test.go index 7e1dd9c30e..21976b5581 100644 --- a/pkg/kubectl-argo-rollouts/viewcontroller/viewcontroller_test.go +++ b/pkg/kubectl-argo-rollouts/viewcontroller/viewcontroller_test.go @@ -6,13 +6,14 @@ import ( "testing" "time" - "github.com/argoproj/argo-rollouts/pkg/apiclient/rollout" - rolloutsfake "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/fake" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" k8sfake "k8s.io/client-go/kubernetes/fake" + "github.com/argoproj/argo-rollouts/pkg/apiclient/rollout" + rolloutsfake "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/fake" + v1alpha1 "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" ) diff --git a/rollout/analysis.go b/rollout/analysis.go index 7bb0d47f1f..f23f680ce0 100644 --- a/rollout/analysis.go +++ b/rollout/analysis.go @@ -74,8 +74,9 @@ func (c *rolloutContext) reconcileAnalysisRuns() error { isAborted := c.pauseContext.IsAborted() rollbackToScaleDownDelay := replicasetutil.HasScaleDownDeadline(c.newRS) initialDeploy := c.rollout.Status.StableRS == "" - if isAborted || c.rollout.Status.PromoteFull || rollbackToScaleDownDelay || initialDeploy { - c.log.Infof("Skipping analysis: isAborted: %v, promoteFull: %v, rollbackToScaleDownDelay: %v, initialDeploy: %v", isAborted, c.rollout.Status.PromoteFull, rollbackToScaleDownDelay, initialDeploy) + isRollbackWithinWindow := c.isRollbackWithinWindow() + if isAborted || c.rollout.Status.PromoteFull || rollbackToScaleDownDelay || initialDeploy || isRollbackWithinWindow { + c.log.Infof("Skipping analysis: isAborted: %v, promoteFull: %v, rollbackToScaleDownDelay: %v, initialDeploy: %v, isRollbackWithinWindow: %v", isAborted, c.rollout.Status.PromoteFull, rollbackToScaleDownDelay, initialDeploy, isRollbackWithinWindow) allArs := append(c.currentArs.ToArray(), c.otherArs...) c.SetCurrentAnalysisRuns(c.currentArs) return c.cancelAnalysisRuns(allArs) diff --git a/rollout/analysis_test.go b/rollout/analysis_test.go index 46a741bd63..e9526cb75f 100644 --- a/rollout/analysis_test.go +++ b/rollout/analysis_test.go @@ -406,7 +406,7 @@ func TestInvalidSpecMissingClusterTemplatesBackgroundAnalysis(t *testing.T) { f.objects = append(f.objects, r) patchIndex := f.expectPatchRolloutAction(r) - f.run(getKey(r, t)) + f.runExpectError(getKey(r, t), true) expectedPatchWithoutSub := `{ "status": { @@ -961,7 +961,7 @@ func TestFailCreateStepAnalysisRunIfInvalidTemplateRef(t *testing.T) { f.objects = append(f.objects, r, at) patchIndex := f.expectPatchRolloutAction(r) - f.run(getKey(r, t)) + f.runExpectError(getKey(r, t), true) expectedPatchWithoutSub := `{ "status": { @@ -1006,7 +1006,7 @@ func TestFailCreateBackgroundAnalysisRunIfInvalidTemplateRef(t *testing.T) { f.objects = append(f.objects, r, at) patchIndex := f.expectPatchRolloutAction(r) - f.run(getKey(r, t)) + f.runExpectError(getKey(r, t), true) expectedPatchWithoutSub := `{ "status": { @@ -1055,7 +1055,7 @@ func TestFailCreateBackgroundAnalysisRunIfMetricRepeated(t *testing.T) { f.objects = append(f.objects, r, at, at2) patchIndex := f.expectPatchRolloutAction(r) - f.run(getKey(r, t)) + f.runExpectError(getKey(r, t), true) expectedPatchWithoutSub := `{ "status": { @@ -1918,6 +1918,47 @@ func TestDoNotCreateBackgroundAnalysisRunOnNewCanaryRolloutStableRSEmpty(t *test f.run(getKey(r1, t)) } +func TestDoNotCreateBackgroundAnalysisRunWhenWithinRollbackWindow(t *testing.T) { + f := newFixture(t) + defer f.Close() + + at := analysisTemplate("bar") + + r1 := newCanaryRollout("foo", 1, nil, nil, pointer.Int32Ptr(0), intstr.FromInt(0), intstr.FromInt(1)) + r1.Spec.Strategy.Canary.Analysis = &v1alpha1.RolloutAnalysisBackground{ + RolloutAnalysis: v1alpha1.RolloutAnalysis{ + Templates: []v1alpha1.AnalysisTemplateRef{ + { + TemplateName: at.Name, + }, + }, + }, + } + r1.Spec.RollbackWindow = &v1alpha1.RollbackWindowSpec{Revisions: 1} + + r2 := bumpVersion(r1) + rs1 := newReplicaSetWithStatus(r1, 1, 1) + rs2 := newReplicaSetWithStatus(r2, 0, 0) + + rs2.CreationTimestamp = timeutil.MetaTime(time.Now().Add(-1 * time.Hour)) + rs1.CreationTimestamp = timeutil.MetaNow() + + f.kubeobjects = append(f.kubeobjects, rs1, rs2) + f.replicaSetLister = append(f.replicaSetLister, rs1, rs2) + + rs1PodHash := rs1.Labels[v1alpha1.DefaultRolloutUniqueLabelKey] + + r2 = updateCanaryRolloutStatus(r2, rs1PodHash, 1, 0, 1, false) + + f.rolloutLister = append(f.rolloutLister, r2) + f.analysisTemplateLister = append(f.analysisTemplateLister, at) + f.objects = append(f.objects, r2, at) + + f.expectUpdateReplicaSetAction(rs2) + f.expectPatchRolloutAction(r2) + f.run(getKey(r2, t)) +} + func TestCreatePrePromotionAnalysisRun(t *testing.T) { f := newFixture(t) defer f.Close() @@ -2491,7 +2532,7 @@ func TestRolloutPostPromotionAnalysisSuccess(t *testing.T) { Status: v1alpha1.AnalysisPhaseRunning, } - rs1 := newReplicaSetWithStatus(r1, 0, 0) + rs1 := newReplicaSetWithStatus(r1, 1, 1) rs2 := newReplicaSetWithStatus(r2, 1, 1) rs1PodHash := rs1.Labels[v1alpha1.DefaultRolloutUniqueLabelKey] rs2PodHash := rs2.Labels[v1alpha1.DefaultRolloutUniqueLabelKey] @@ -2517,6 +2558,7 @@ func TestRolloutPostPromotionAnalysisSuccess(t *testing.T) { patch := f.getPatchedRollout(patchIndex) expectedPatch := fmt.Sprintf(`{ "status": { + "replicas":2, "stableRS": "%s", "blueGreen": { "postPromotionAnalysisRunStatus":{"status":"Successful"} diff --git a/rollout/bluegreen.go b/rollout/bluegreen.go index 9904de9dab..cf6cbc6ac7 100644 --- a/rollout/bluegreen.go +++ b/rollout/bluegreen.go @@ -21,7 +21,7 @@ func (c *rolloutContext) rolloutBlueGreen() error { if err != nil { return err } - c.newRS, err = c.getAllReplicaSetsAndSyncRevision(true) + c.newRS, err = c.getAllReplicaSetsAndSyncRevision() if err != nil { return fmt.Errorf("failed to getAllReplicaSetsAndSyncRevision in rolloutBlueGreen create true: %w", err) } @@ -71,18 +71,14 @@ func (c *rolloutContext) rolloutBlueGreen() error { return c.syncRolloutStatusBlueGreen(previewSvc, activeSvc) } -func (c *rolloutContext) reconcileBlueGreenStableReplicaSet(activeSvc *corev1.Service) error { - if _, ok := activeSvc.Spec.Selector[v1alpha1.DefaultRolloutUniqueLabelKey]; !ok { - return nil - } - activeRS, _ := replicasetutil.GetReplicaSetByTemplateHash(c.allRSs, activeSvc.Spec.Selector[v1alpha1.DefaultRolloutUniqueLabelKey]) - if activeRS == nil { - c.log.Warn("There shouldn't be a nil active replicaset if the active Service selector is set") +func (c *rolloutContext) reconcileBlueGreenStableReplicaSet() error { + if c.stableRS == nil { + c.log.Info("Stable ReplicaSet doesn't exist and hence no reconciliation is required.") return nil } - c.log.Infof("Reconciling stable ReplicaSet '%s'", activeRS.Name) - _, _, err := c.scaleReplicaSetAndRecordEvent(activeRS, defaults.GetReplicasOrDefault(c.rollout.Spec.Replicas)) + c.log.Infof("Reconciling stable ReplicaSet '%s'", c.stableRS.Name) + _, _, err := c.scaleReplicaSetAndRecordEvent(c.stableRS, defaults.GetReplicasOrDefault(c.rollout.Spec.Replicas)) if err != nil { return fmt.Errorf("failed to scaleReplicaSetAndRecordEvent in reconcileBlueGreenStableReplicaSet: %w", err) } @@ -94,7 +90,7 @@ func (c *rolloutContext) reconcileBlueGreenReplicaSets(activeSvc *corev1.Service if err != nil { return err } - err = c.reconcileBlueGreenStableReplicaSet(activeSvc) + err = c.reconcileBlueGreenStableReplicaSet() if err != nil { return err } diff --git a/rollout/bluegreen_test.go b/rollout/bluegreen_test.go index 2ff9515fe3..5a9bd7cf77 100644 --- a/rollout/bluegreen_test.go +++ b/rollout/bluegreen_test.go @@ -1,12 +1,15 @@ package rollout import ( + "bytes" "encoding/json" "fmt" "strconv" + "strings" "testing" "time" + log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -947,8 +950,10 @@ func TestBlueGreenRolloutStatusHPAStatusFieldsNoActiveSelector(t *testing.T) { f := newFixture(t) defer f.Close() f.objects = append(f.objects, ro) + f.kubeobjects = append(f.kubeobjects, activeSvc) f.rolloutLister = append(f.rolloutLister, ro) f.replicaSetLister = append(f.replicaSetLister, rs) + f.serviceLister = append(f.serviceLister, activeSvc) ctrl, _, _ := f.newController(noResyncPeriodFunc) roCtx, err := ctrl.newRolloutContext(ro) @@ -1007,6 +1012,92 @@ func TestBlueGreenRolloutScaleUpdateActiveRS(t *testing.T) { f.run(getKey(r2, t)) } +func TestBlueGreenRolloutScaleUpdateStableRS(t *testing.T) { + f := newFixture(t) + defer f.Close() + + r1 := newBlueGreenRollout("foo", 1, nil, "active", "") + rs1 := newReplicaSetWithStatus(r1, 1, 1) + r2 := bumpVersion(r1) + + rs2 := newReplicaSetWithStatus(r2, 1, 1) + f.kubeobjects = append(f.kubeobjects, rs1, rs2) + f.replicaSetLister = append(f.replicaSetLister, rs1, rs2) + + rs1PodHash := rs1.Labels[v1alpha1.DefaultRolloutUniqueLabelKey] + rs2PodHash := rs2.Labels[v1alpha1.DefaultRolloutUniqueLabelKey] + + // Make the new RS the active and the old one stable to simulate post-promotion analysis step. + r2 = updateBlueGreenRolloutStatus(r2, rs2PodHash, rs2PodHash, rs1PodHash, 1, 1, 2, 1, false, true, false) + + f.rolloutLister = append(f.rolloutLister, r2) + + f.objects = append(f.objects, r2) + activeSvc := newService("active", 80, map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs2PodHash}, r2) + f.kubeobjects = append(f.kubeobjects, activeSvc) + f.serviceLister = append(f.serviceLister, activeSvc) + + f.expectPatchRolloutAction(r1) + + // Patch the rollout to get it in the state we want (old RS is stable and new is active) + f.run(getKey(r2, t)) + // Actually update the replicas now that we are in the desired state (old RS is stable and new is active) + r2.Spec.Replicas = pointer.Int32Ptr(2) + + f.expectUpdateReplicaSetAction(rs1) + f.expectUpdateReplicaSetAction(rs2) + f.run(getKey(r2, t)) +} + +func TestBlueGreenStableRSReconciliationShouldNotScaleOnFirstTimeRollout(t *testing.T) { + f := newFixture(t) + prevOutput := log.StandardLogger().Out + defer func() { + log.SetOutput(prevOutput) + }() + defer f.Close() + + // Setup Logging capture + buf := bytes.NewBufferString("") + log.SetOutput(buf) + + r := newBlueGreenRollout("foo", 1, nil, "active", "preview") + r.Status.Conditions = []v1alpha1.RolloutCondition{} + f.rolloutLister = append(f.rolloutLister, r) + f.objects = append(f.objects, r) + previewSvc := newService("preview", 80, nil, r) + activeSvc := newService("active", 80, nil, r) + f.kubeobjects = append(f.kubeobjects, previewSvc, activeSvc) + f.serviceLister = append(f.serviceLister, activeSvc, previewSvc) + + rs := newReplicaSet(r, 1) + rsPodHash := rs.Labels[v1alpha1.DefaultRolloutUniqueLabelKey] + generatedConditions := generateConditionsPatchWithCompleted(false, conditions.ReplicaSetUpdatedReason, rs, false, "", true) + + f.expectCreateReplicaSetAction(rs) + f.expectPatchServiceAction(previewSvc, rsPodHash) + f.expectUpdateReplicaSetAction(rs) // scale up RS + f.expectUpdateRolloutStatusAction(r) + expectedPatchWithoutSubs := `{ + "status":{ + "blueGreen" : { + "previewSelector": "%s" + }, + "conditions": %s, + "selector": "foo=bar", + "stableRS": "%s", + "phase": "Progressing", + "message": "more replicas need to be updated" + } + }` + expectedPatch := calculatePatch(r, fmt.Sprintf(expectedPatchWithoutSubs, rsPodHash, generatedConditions, rsPodHash)) + f.expectPatchRolloutActionWithPatch(r, expectedPatch) + f.run(getKey(r, t)) + + logMessage := buf.String() + assert.True(t, strings.Contains(logMessage, "msg=\"Stable ReplicaSet doesn't exist and hence no reconciliation is required.\""), logMessage) +} + func TestPreviewReplicaCountHandleScaleUpPreviewCheckPoint(t *testing.T) { t.Run("TrueAfterMeetingMinAvailable", func(t *testing.T) { f := newFixture(t) @@ -1315,6 +1406,7 @@ func TestBlueGreenReadyToScaleDownOldReplica(t *testing.T) { f.objects = append(f.objects, r2) f.serviceLister = append(f.serviceLister, s) + f.expectPatchReplicaSetAction(rs2) updatedRSIndex := f.expectUpdateReplicaSetAction(rs2) patchIndex := f.expectPatchRolloutAction(r2) f.run(getKey(r2, t)) diff --git a/rollout/canary.go b/rollout/canary.go index ed27bd0a79..0d3e74fcf7 100644 --- a/rollout/canary.go +++ b/rollout/canary.go @@ -20,14 +20,14 @@ import ( func (c *rolloutContext) rolloutCanary() error { var err error if replicasetutil.PodTemplateOrStepsChanged(c.rollout, c.newRS) { - c.newRS, err = c.getAllReplicaSetsAndSyncRevision(false) + c.newRS, err = c.getAllReplicaSetsAndSyncRevision() if err != nil { return fmt.Errorf("failed to getAllReplicaSetsAndSyncRevision in rolloutCanary with PodTemplateOrStepsChanged: %w", err) } return c.syncRolloutStatusCanary() } - c.newRS, err = c.getAllReplicaSetsAndSyncRevision(true) + c.newRS, err = c.getAllReplicaSetsAndSyncRevision() if err != nil { return fmt.Errorf("failed to getAllReplicaSetsAndSyncRevision in rolloutCanary create true: %w", err) } @@ -87,6 +87,11 @@ func (c *rolloutContext) rolloutCanary() error { return c.syncRolloutStatusCanary() } + err = c.stepPluginContext.reconcile(c) + if err != nil { + return err + } + return c.syncRolloutStatusCanary() } @@ -112,7 +117,7 @@ func (c *rolloutContext) reconcileCanaryStableReplicaSet() (bool, error) { } scaled, _, err := c.scaleReplicaSetAndRecordEvent(c.stableRS, desiredStableRSReplicaCount) if err != nil { - return scaled, fmt.Errorf("failed to scaleReplicaSetAndRecordEvent in reconcileCanaryStableReplicaSet:L %w", err) + return scaled, fmt.Errorf("failed to scaleReplicaSetAndRecordEvent in reconcileCanaryStableReplicaSet: %w", err) } return scaled, err } @@ -296,7 +301,7 @@ func (c *rolloutContext) canProceedWithScaleDownAnnotation(oldRSs []*appsv1.Repl // AWS API calls. return true, nil } - stableSvcName, _ := trafficrouting.GetStableAndCanaryServices(c.rollout) + stableSvcName, _ := trafficrouting.GetStableAndCanaryServices(c.rollout, true) stableSvc, err := c.servicesLister.Services(c.rollout.Namespace).Get(stableSvcName) if err != nil { return false, err @@ -315,7 +320,7 @@ func (c *rolloutContext) completedCurrentCanaryStep() bool { if c.rollout.Spec.Paused { return false } - currentStep, _ := replicasetutil.GetCurrentCanaryStep(c.rollout) + currentStep, currentStepIndex := replicasetutil.GetCurrentCanaryStep(c.rollout) if currentStep == nil { return false } @@ -344,6 +349,8 @@ func (c *rolloutContext) completedCurrentCanaryStep() bool { return true case currentStep.SetMirrorRoute != nil: return true + case currentStep.Plugin != nil: + return c.stepPluginContext.isStepPluginCompleted(*currentStepIndex, currentStep.Plugin) } return false } @@ -354,6 +361,8 @@ func (c *rolloutContext) syncRolloutStatusCanary() error { newStatus.HPAReplicas = replicasetutil.GetActualReplicaCountForReplicaSets(c.allRSs) newStatus.Selector = metav1.FormatLabelSelector(c.rollout.Spec.Selector) newStatus.Canary.StablePingPong = c.rollout.Status.Canary.StablePingPong + newStatus.Canary.StepPluginStatuses = c.rollout.Status.Canary.StepPluginStatuses + c.stepPluginContext.updateStatus(&newStatus) currentStep, currentStepIndex := replicasetutil.GetCurrentCanaryStep(c.rollout) newStatus.StableRS = c.rollout.Status.StableRS @@ -439,13 +448,6 @@ func (c *rolloutContext) reconcileCanaryReplicaSets() (bool, error) { return true, nil } - // If we have updated both the replica count and the pod template hash c.newRS will be nil we want to reconcile the newRS so we look at the - // rollout status to get the newRS to reconcile it. - if c.newRS == nil && c.rollout.Status.CurrentPodHash != c.rollout.Status.StableRS { - rs, _ := replicasetutil.GetReplicaSetByTemplateHash(c.allRSs, c.rollout.Status.CurrentPodHash) - c.newRS = rs - } - scaledNewRS, err := c.reconcileNewReplicaSet() if err != nil { return false, err diff --git a/rollout/canary_test.go b/rollout/canary_test.go index 56b05c7177..ddcfe1ec1e 100644 --- a/rollout/canary_test.go +++ b/rollout/canary_test.go @@ -1,7 +1,6 @@ package rollout import ( - "context" "encoding/json" "fmt" "strconv" @@ -415,8 +414,11 @@ func TestResetCurrentStepIndexOnStepChange(t *testing.T) { f.rolloutLister = append(f.rolloutLister, r2) f.objects = append(f.objects, r2) + f.expectUpdateRolloutStatusAction(r2) patchIndex := f.expectPatchRolloutAction(r2) + createRSIndex := f.expectCreateReplicaSetAction(rs1) f.run(getKey(r2, t)) + createdRS := f.getCreatedReplicaSet(createRSIndex) patch := f.getPatchedRollout(patchIndex) expectedPatchWithoutPodHash := `{ @@ -427,7 +429,7 @@ func TestResetCurrentStepIndexOnStepChange(t *testing.T) { "conditions": %s } }` - newConditions := generateConditionsPatch(true, conditions.ReplicaSetUpdatedReason, r2, false, "", false) + newConditions := generateConditionsPatch(true, conditions.ReplicaSetUpdatedReason, createdRS, false, "", false) expectedPatch := fmt.Sprintf(expectedPatchWithoutPodHash, expectedCurrentPodHash, expectedCurrentStepHash, newConditions) assert.JSONEq(t, calculatePatch(r2, expectedPatch), patch) } @@ -456,10 +458,15 @@ func TestResetCurrentStepIndexOnPodSpecChange(t *testing.T) { f.rolloutLister = append(f.rolloutLister, r2) f.objects = append(f.objects, r2) + f.expectUpdateRolloutStatusAction(r2) patchIndex := f.expectPatchRolloutAction(r2) + createdRSIndex := f.expectCreateReplicaSetAction(rs1) + f.run(getKey(r2, t)) patch := f.getPatchedRollout(patchIndex) + updatedRS := f.getUpdatedReplicaSet(createdRSIndex) + expectedPatchWithoutPodHash := `{ "status": { "currentStepIndex":0, @@ -467,7 +474,7 @@ func TestResetCurrentStepIndexOnPodSpecChange(t *testing.T) { "conditions": %s } }` - newConditions := generateConditionsPatch(true, conditions.ReplicaSetUpdatedReason, r2, false, "", false) + newConditions := generateConditionsPatch(true, conditions.ReplicaSetUpdatedReason, updatedRS, false, "", false) expectedPatch := fmt.Sprintf(expectedPatchWithoutPodHash, expectedCurrentPodHash, newConditions) assert.JSONEq(t, calculatePatch(r2, expectedPatch), patch) @@ -1558,7 +1565,7 @@ func TestCanaryRolloutWithInvalidCanaryServiceName(t *testing.T) { f.kubeobjects = append(f.kubeobjects, rs) patchIndex := f.expectPatchRolloutAction(rollout) - f.run(getKey(rollout, t)) + f.runExpectError(getKey(rollout, t), true) patch := make(map[string]any) patchData := f.getPatchedRollout(patchIndex) @@ -1610,7 +1617,7 @@ func TestCanaryRolloutWithInvalidStableServiceName(t *testing.T) { f.kubeobjects = append(f.kubeobjects, rs) patchIndex := f.expectPatchRolloutAction(rollout) - f.run(getKey(rollout, t)) + f.runExpectError(getKey(rollout, t), true) patch := make(map[string]any) patchData := f.getPatchedRollout(patchIndex) @@ -1659,7 +1666,7 @@ func TestCanaryRolloutWithInvalidPingServiceName(t *testing.T) { f.objects = append(f.objects, r) patchIndex := f.expectPatchRolloutAction(r) - f.run(getKey(r, t)) + f.runExpectError(getKey(r, t), true) patch := make(map[string]any) patchData := f.getPatchedRollout(patchIndex) @@ -1691,7 +1698,7 @@ func TestCanaryRolloutWithInvalidPongServiceName(t *testing.T) { f.serviceLister = append(f.serviceLister, pingSvc) patchIndex := f.expectPatchRolloutAction(r) - f.run(getKey(r, t)) + f.runExpectError(getKey(r, t), true) patch := make(map[string]any) patchData := f.getPatchedRollout(patchIndex) @@ -1890,8 +1897,14 @@ func TestHandleNilNewRSOnScaleAndImageChange(t *testing.T) { f.rolloutLister = append(f.rolloutLister, r2) f.objects = append(f.objects, r2) - f.expectUpdateReplicaSetAction(rs1) + f.expectUpdateRolloutStatusAction(r2) + f.expectPatchRolloutAction(r2) patchIndex := f.expectPatchRolloutAction(r2) + + f.expectCreateReplicaSetAction(rs1) + f.expectUpdateReplicaSetAction(rs1) + f.expectUpdateReplicaSetAction(rs1) + f.run(getKey(r2, t)) patch := f.getPatchedRollout(patchIndex) assert.JSONEq(t, calculatePatch(r2, OnlyObservedGenerationPatch), patch) @@ -2098,46 +2111,3 @@ func TestIsDynamicallyRollingBackToStable(t *testing.T) { }) } } - -func TestCanaryReplicaAndSpecChangedTogether(t *testing.T) { - f := newFixture(t) - defer f.Close() - - originReplicas := 3 - r1 := newCanaryRollout("foo", originReplicas, nil, nil, nil, intstr.FromInt(1), intstr.FromInt(0)) - canarySVCName := "canary" - stableSVCName := "stable" - r1.Spec.Strategy.Canary.CanaryService = canarySVCName - r1.Spec.Strategy.Canary.StableService = stableSVCName - - stableRS := newReplicaSetWithStatus(r1, originReplicas, originReplicas) - stableSVC := newService(stableSVCName, 80, - map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: stableRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]}, r1) - - r2 := bumpVersion(r1) - canaryRS := newReplicaSetWithStatus(r2, originReplicas, originReplicas) - canarySVC := newService(canarySVCName, 80, - map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: canaryRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]}, r2) - - f.replicaSetLister = append(f.replicaSetLister, canaryRS, stableRS) - f.serviceLister = append(f.serviceLister, canarySVC, stableSVC) - - r3 := bumpVersion(r2) - r3.Spec.Replicas = pointer.Int32(int32(originReplicas) + 5) - r3.Status.StableRS = stableRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey] - r3.Status.CurrentPodHash = canaryRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey] - - f.rolloutLister = append(f.rolloutLister, r3) - f.kubeobjects = append(f.kubeobjects, canaryRS, stableRS, canarySVC, stableSVC) - f.objects = append(f.objects, r3) - - ctrl, _, _ := f.newController(noResyncPeriodFunc) - roCtx, err := ctrl.newRolloutContext(r3) - assert.NoError(t, err) - err = roCtx.reconcile() - assert.NoError(t, err) - updated, err := f.kubeclient.AppsV1().ReplicaSets(r3.Namespace).Get(context.Background(), canaryRS.Name, metav1.GetOptions{}) - assert.NoError(t, err) - // check the canary one is updated - assert.NotEqual(t, originReplicas, int(*updated.Spec.Replicas)) -} diff --git a/rollout/context.go b/rollout/context.go index f8fa5b5f03..25f5f99fcc 100644 --- a/rollout/context.go +++ b/rollout/context.go @@ -1,11 +1,8 @@ package rollout import ( - "time" - log "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/util/validation/field" "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" analysisutil "github.com/argoproj/argo-rollouts/utils/analysis" @@ -39,8 +36,9 @@ type rolloutContext struct { currentEx *v1alpha1.Experiment otherExs []*v1alpha1.Experiment - newStatus v1alpha1.RolloutStatus - pauseContext *pauseContext + newStatus v1alpha1.RolloutStatus + pauseContext *pauseContext + stepPluginContext *stepPluginContext // targetsVerified indicates if the pods targets have been verified with underlying LoadBalancer. // This is used in pod-aware flat networks where LoadBalancers target Pods and not Nodes. @@ -52,19 +50,7 @@ type rolloutContext struct { } func (c *rolloutContext) reconcile() error { - // Get Rollout Validation errors - err := c.getRolloutValidationErrors() - if err != nil { - if vErr, ok := err.(*field.Error); ok { - // We want to frequently requeue rollouts with InvalidSpec errors, because the error - // condition might be timing related (e.g. the Rollout was applied before the Service). - c.enqueueRolloutAfter(c.rollout, 20*time.Second) - return c.createInvalidRolloutCondition(vErr, c.rollout) - } - return err - } - - err = c.checkPausedConditions() + err := c.checkPausedConditions() if err != nil { return err } diff --git a/rollout/controller.go b/rollout/controller.go index 972cea882c..6b85f43acb 100644 --- a/rollout/controller.go +++ b/rollout/controller.go @@ -11,7 +11,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts" smiclientset "github.com/servicemeshinterface/smi-sdk-go/pkg/gen/client/split/clientset/versioned" log "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" @@ -35,6 +34,8 @@ import ( "k8s.io/kubernetes/pkg/controller" "k8s.io/utils/pointer" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts" + "github.com/argoproj/argo-rollouts/controller/metrics" register "github.com/argoproj/argo-rollouts/pkg/apis/rollouts" "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" @@ -42,6 +43,7 @@ import ( clientset "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned" informers "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions/rollouts/v1alpha1" listers "github.com/argoproj/argo-rollouts/pkg/client/listers/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/rollout/steps/plugin" "github.com/argoproj/argo-rollouts/rollout/trafficrouting" "github.com/argoproj/argo-rollouts/rollout/trafficrouting/ambassador" "github.com/argoproj/argo-rollouts/rollout/trafficrouting/appmesh" @@ -405,14 +407,30 @@ func (c *Controller) syncHandler(ctx context.Context, key string) error { }() resolveErr := c.refResolver.Resolve(r) + // We could maybe lose setting the error condition from the below if resolveErr != nil {}, and just log the error to clean up the logic roCtx, err := c.newRolloutContext(r) + if roCtx == nil { + logCtx.Error("newRolloutContext returned nil") + return err + } if err != nil { + if _, ok := err.(*field.Error); ok { + // We want to frequently requeue rollouts with InvalidSpec errors, because the error + // condition might be timing related (e.g. the Rollout was applied before the Service). + c.enqueueRolloutAfter(roCtx.rollout, 20*time.Second) + return nil // do not requeue from error because we already re-queued above + } logCtx.Errorf("newRolloutContext err %v", err) return err } + // We should probably delete this if block and just log the error to clean up the logic, a bigger change would be to add a new + // field to the status maybe (reconcileErrMsg) and store the errors there from the processNextWorkItem function in controller/controller.go if resolveErr != nil { - roCtx.createInvalidRolloutCondition(resolveErr, r) - return resolveErr + err := roCtx.createInvalidRolloutCondition(resolveErr, r) + if err != nil { + return fmt.Errorf("failed to create invalid rollout condition during resolving the rollout: %w", err) + } + return fmt.Errorf("failed to resolve rollout: %w", resolveErr) } // In order to work with HPA, the rollout.Spec.Replica field cannot be nil. As a result, the controller will update @@ -520,8 +538,37 @@ func (c *Controller) newRolloutContext(rollout *v1alpha1.Rollout) (*rolloutConte rollout: rollout, log: logCtx, }, + stepPluginContext: &stepPluginContext{ + resolver: plugin.NewResolver(), + log: logCtx, + }, reconcilerBase: c.reconcilerBase, } + + // Get Rollout Validation errors + err = roCtx.getRolloutValidationErrors() + if err != nil { + if vErr, ok := err.(*field.Error); ok { + err := roCtx.createInvalidRolloutCondition(vErr, roCtx.rollout) + if err != nil { + return nil, err + } + return nil, vErr + } + return nil, err + } + + if roCtx.newRS == nil { + roCtx.newRS, err = roCtx.createDesiredReplicaSet() + if err != nil { + return nil, err + } + roCtx.olderRSs = replicasetutil.FindOldReplicaSets(roCtx.rollout, rsList, roCtx.newRS) + roCtx.stableRS = replicasetutil.GetStableRS(roCtx.rollout, roCtx.newRS, roCtx.olderRSs) + roCtx.otherRSs = replicasetutil.GetOtherRSs(roCtx.rollout, roCtx.newRS, roCtx.stableRS, rsList) + roCtx.allRSs = append(rsList, roCtx.newRS) + } + if rolloututil.IsFullyPromoted(rollout) && roCtx.pauseContext.IsAborted() { logCtx.Warnf("Removing abort condition from fully promoted rollout") roCtx.pauseContext.RemoveAbort() @@ -937,3 +984,15 @@ func remarshalRollout(r *v1alpha1.Rollout) *v1alpha1.Rollout { } return &remarshalled } + +// updateReplicaSet updates the replicaset using kubeclient update. It returns the updated replicaset and copies the updated replicaset +// into the passed in pointer as well. +func (c *rolloutContext) updateReplicaSet(ctx context.Context, rs *appsv1.ReplicaSet) (*appsv1.ReplicaSet, error) { + updatedRS, err := c.kubeclientset.AppsV1().ReplicaSets(rs.Namespace).Update(ctx, rs, metav1.UpdateOptions{}) + if err != nil { + return nil, fmt.Errorf("error updating replicaset in updateReplicaSet %s: %w", rs.Name, err) + } + updatedRS.DeepCopyInto(rs) + + return rs, err +} diff --git a/rollout/controller_test.go b/rollout/controller_test.go index 892a2be64f..383a539bd0 100644 --- a/rollout/controller_test.go +++ b/rollout/controller_test.go @@ -95,6 +95,7 @@ type fixture struct { replicaSetLister []*appsv1.ReplicaSet serviceLister []*corev1.Service ingressLister []*ingressutil.Ingress + virtualServiceLister []*unstructured.Unstructured // Actions expected to happen on the client. kubeactions []core.Action actions []core.Action @@ -662,6 +663,9 @@ func (f *fixture) newController(resync resyncFunc) (*Controller, informers.Share for _, ar := range f.analysisRunLister { i.Argoproj().V1alpha1().AnalysisRuns().Informer().GetIndexer().Add(ar) } + for _, vs := range f.virtualServiceLister { + c.IstioController.VirtualServiceInformer.GetIndexer().Add(vs) + } return c, i, k8sI } @@ -792,6 +796,12 @@ func (f *fixture) expectPatchServiceAction(s *corev1.Service, newLabel string) i return len } +func (f *fixture) expectGetReplicaSetAction(r *appsv1.ReplicaSet) int { //nolint:unused + len := len(f.kubeactions) + f.kubeactions = append(f.kubeactions, core.NewGetAction(schema.GroupVersionResource{Resource: "replicasets"}, r.Namespace, r.Name)) + return len +} + func (f *fixture) expectCreateReplicaSetAction(r *appsv1.ReplicaSet) int { len := len(f.kubeactions) f.kubeactions = append(f.kubeactions, core.NewCreateAction(schema.GroupVersionResource{Resource: "replicasets"}, r.Namespace, r)) @@ -950,6 +960,21 @@ func (f *fixture) getUpdatedReplicaSet(index int) *appsv1.ReplicaSet { return rs } +func (f *fixture) getPatchedReplicaSet(index int) *appsv1.ReplicaSet { //nolint + action := filterInformerActions(f.kubeclient.Actions())[index] + patchAction, ok := action.(core.PatchAction) + if !ok { + f.t.Fatalf("Expected Patch action, not %s", action.GetVerb()) + } + + rs := appsv1.ReplicaSet{} + err := json.Unmarshal(patchAction.GetPatch(), &rs) + if err != nil { + panic(err) + } + return &rs +} + func (f *fixture) verifyPatchedReplicaSet(index int, scaleDownDelaySeconds int32) { action := filterInformerActions(f.kubeclient.Actions())[index] patchAction, ok := action.(core.PatchAction) @@ -1182,11 +1207,17 @@ func TestDontSyncRolloutsWithEmptyPodSelector(t *testing.T) { f := newFixture(t) defer f.Close() - r := newBlueGreenRollout("foo", 1, nil, "", "") + r := newBlueGreenRollout("foo", 1, nil, "active", "") + activeSvc := newService("active", 80, nil, r) f.rolloutLister = append(f.rolloutLister, r) + f.serviceLister = append(f.serviceLister, activeSvc) f.objects = append(f.objects, r) + f.kubeobjects = append(f.kubeobjects, activeSvc) + f.expectUpdateRolloutStatusAction(r) f.expectPatchRolloutAction(r) + f.expectCreateReplicaSetAction(&appsv1.ReplicaSet{}) + f.expectUpdateReplicaSetAction(&appsv1.ReplicaSet{}) f.run(getKey(r, t)) } @@ -1219,13 +1250,12 @@ func TestAdoptReplicaSet(t *testing.T) { } func TestRequeueStuckRollout(t *testing.T) { + //t.Skip("broken in the refactor") rollout := func(progressingConditionReason string, rolloutCompleted bool, rolloutPaused bool, progressDeadlineSeconds *int32) *v1alpha1.Rollout { - r := &v1alpha1.Rollout{ - Spec: v1alpha1.RolloutSpec{ - Replicas: pointer.Int32Ptr(0), - ProgressDeadlineSeconds: progressDeadlineSeconds, - }, - } + r := newCanaryRollout("foo", 0, nil, nil, nil, intstr.FromInt(0), intstr.FromInt(1)) + r.Status.Conditions = nil + r.Spec.ProgressDeadlineSeconds = progressDeadlineSeconds + //r.Spec.Strategy.BlueGreen = &v1alpha1.BlueGreenStrategy{} r.Generation = 123 if rolloutPaused { r.Status.PauseConditions = []v1alpha1.PauseCondition{{ @@ -1234,6 +1264,8 @@ func TestRequeueStuckRollout(t *testing.T) { } if rolloutCompleted { r.Status.ObservedGeneration = strconv.Itoa(int(r.Generation)) + r.Status.StableRS = "fakesha" + r.Status.CurrentPodHash = "fakesha" } if progressingConditionReason != "" { @@ -1289,9 +1321,14 @@ func TestRequeueStuckRollout(t *testing.T) { for i := range tests { test := tests[i] t.Run(test.name, func(t *testing.T) { + savedRollout := test.rollout.DeepCopy() f := newFixture(t) defer f.Close() c, _, _ := f.newController(noResyncPeriodFunc) + f.client.PrependReactor("*", "rollouts", func(action core.Action) (bool, runtime.Object, error) { + savedRollout.DeepCopyInto(test.rollout) + return true, savedRollout, nil + }) roCtx, err := c.newRolloutContext(test.rollout) assert.NoError(t, err) duration := roCtx.requeueStuckRollout(test.rollout.Status) @@ -1315,7 +1352,10 @@ func TestSetReplicaToDefault(t *testing.T) { f.rolloutLister = append(f.rolloutLister, r) f.objects = append(f.objects, r) + //updateIndex := f.expectUpdateRolloutAction(r) + f.expectUpdateRolloutStatusAction(r) updateIndex := f.expectUpdateRolloutAction(r) + f.expectCreateReplicaSetAction(&appsv1.ReplicaSet{}) f.run(getKey(r, t)) updatedRollout := f.getUpdatedRollout(updateIndex) assert.Equal(t, defaults.DefaultReplicas, *updatedRollout.Spec.Replicas) @@ -1326,7 +1366,8 @@ func TestSwitchInvalidSpecMessage(t *testing.T) { f := newFixture(t) defer f.Close() - r := newBlueGreenRollout("foo", 1, nil, "", "") + r := newBlueGreenRollout("foo", 1, nil, "active", "") + activeSvc := newService("active", 80, nil, r) r.Spec.Selector = &metav1.LabelSelector{} cond := conditions.NewRolloutCondition(v1alpha1.InvalidSpec, corev1.ConditionTrue, conditions.InvalidSpecReason, conditions.RolloutSelectAllMessage) conditions.SetRolloutCondition(&r.Status, *cond) @@ -1335,9 +1376,12 @@ func TestSwitchInvalidSpecMessage(t *testing.T) { r.Spec.Selector = nil f.rolloutLister = append(f.rolloutLister, r) f.objects = append(f.objects, r) + f.kubeobjects = append(f.kubeobjects, activeSvc) + f.serviceLister = append(f.serviceLister, activeSvc) patchIndex := f.expectPatchRolloutAction(r) - f.run(getKey(r, t)) + //f.run(getKey(r, t)) + f.runExpectError(getKey(r, t), true) expectedPatchWithoutSub := `{ "status": { @@ -1581,8 +1625,7 @@ func newInvalidSpecCondition(reason string, resourceObj runtime.Object, optional } func TestGetReferencedAnalyses(t *testing.T) { - f := newFixture(t) - defer f.Close() + //f := newFixture(t) rolloutAnalysisFail := v1alpha1.RolloutAnalysis{ Templates: []v1alpha1.AnalysisTemplateRef{{ @@ -1592,53 +1635,81 @@ func TestGetReferencedAnalyses(t *testing.T) { } t.Run("blueGreen pre-promotion analysis - fail", func(t *testing.T) { + f := newFixture(t) + defer f.Close() r := newBlueGreenRollout("rollout", 1, nil, "active-service", "preview-service") + activeSvc := newService("active-service", 80, nil, r) + previewSvc := newService("preview-service", 80, nil, r) + f.objects = append(f.objects, r) + f.kubeobjects = append(f.kubeobjects, activeSvc, previewSvc) + f.rolloutLister = append(f.rolloutLister, r) + f.serviceLister = append(f.serviceLister, activeSvc, previewSvc) r.Spec.Strategy.BlueGreen.PrePromotionAnalysis = &rolloutAnalysisFail c, _, _ := f.newController(noResyncPeriodFunc) roCtx, err := c.newRolloutContext(r) - assert.NoError(t, err) - _, err = roCtx.getReferencedRolloutAnalyses() assert.NotNil(t, err) + assert.Nil(t, roCtx) msg := "spec.strategy.blueGreen.prePromotionAnalysis.templates: Invalid value: \"does-not-exist\": AnalysisTemplate 'does-not-exist' not found" assert.Equal(t, msg, err.Error()) }) t.Run("blueGreen post-promotion analysis - fail", func(t *testing.T) { + f := newFixture(t) + defer f.Close() r := newBlueGreenRollout("rollout", 1, nil, "active-service", "preview-service") + activeSvc := newService("active-service", 80, nil, r) + previewSvc := newService("preview-service", 80, nil, r) + f.objects = append(f.objects, r) + f.kubeobjects = append(f.kubeobjects, activeSvc, previewSvc) + f.rolloutLister = append(f.rolloutLister, r) + f.serviceLister = append(f.serviceLister, activeSvc, previewSvc) r.Spec.Strategy.BlueGreen.PostPromotionAnalysis = &rolloutAnalysisFail c, _, _ := f.newController(noResyncPeriodFunc) roCtx, err := c.newRolloutContext(r) - assert.NoError(t, err) - _, err = roCtx.getReferencedRolloutAnalyses() assert.NotNil(t, err) + assert.Nil(t, roCtx) msg := "spec.strategy.blueGreen.postPromotionAnalysis.templates: Invalid value: \"does-not-exist\": AnalysisTemplate 'does-not-exist' not found" assert.Equal(t, msg, err.Error()) }) t.Run("canary analysis - fail", func(t *testing.T) { + f := newFixture(t) + defer f.Close() r := newCanaryRollout("rollout-canary", 1, nil, nil, int32Ptr(0), intstr.FromInt(0), intstr.FromInt(1)) + activeSvc := newService("active-service", 80, nil, r) + previewSvc := newService("preview-service", 80, nil, r) + f.objects = append(f.objects, r) + f.kubeobjects = append(f.kubeobjects, activeSvc, previewSvc) + f.rolloutLister = append(f.rolloutLister, r) + f.serviceLister = append(f.serviceLister, activeSvc, previewSvc) r.Spec.Strategy.Canary.Analysis = &v1alpha1.RolloutAnalysisBackground{ RolloutAnalysis: rolloutAnalysisFail, } c, _, _ := f.newController(noResyncPeriodFunc) roCtx, err := c.newRolloutContext(r) - assert.NoError(t, err) - _, err = roCtx.getReferencedRolloutAnalyses() assert.NotNil(t, err) + assert.Nil(t, roCtx) msg := "spec.strategy.canary.analysis.templates: Invalid value: \"does-not-exist\": AnalysisTemplate 'does-not-exist' not found" assert.Equal(t, msg, err.Error()) }) t.Run("canary step analysis - fail", func(t *testing.T) { + f := newFixture(t) + defer f.Close() canarySteps := []v1alpha1.CanaryStep{{ Analysis: &rolloutAnalysisFail, }} r := newCanaryRollout("rollout-canary", 1, nil, canarySteps, int32Ptr(0), intstr.FromInt(0), intstr.FromInt(1)) + activeSvc := newService("active-service", 80, nil, r) + previewSvc := newService("preview-service", 80, nil, r) + f.objects = append(f.objects, r) + f.kubeobjects = append(f.kubeobjects, activeSvc, previewSvc) + f.rolloutLister = append(f.rolloutLister, r) + f.serviceLister = append(f.serviceLister, activeSvc, previewSvc) c, _, _ := f.newController(noResyncPeriodFunc) roCtx, err := c.newRolloutContext(r) - assert.NoError(t, err) - _, err = roCtx.getReferencedRolloutAnalyses() assert.NotNil(t, err) + assert.Nil(t, roCtx) msg := "spec.strategy.canary.steps[0].analysis.templates: Invalid value: \"does-not-exist\": AnalysisTemplate 'does-not-exist' not found" assert.Equal(t, msg, err.Error()) }) @@ -1654,6 +1725,12 @@ func TestGetReferencedClusterAnalysisTemplate(t *testing.T) { ClusterScope: true, }}, } + activeSvc := newService("active-service", 80, nil, r) + previewSvc := newService("preview-service", 80, nil, r) + f.rolloutLister = append(f.rolloutLister, r) + f.objects = append(f.objects, r) + f.kubeobjects = append(f.kubeobjects, activeSvc, previewSvc) + f.serviceLister = append(f.serviceLister, activeSvc, previewSvc) t.Run("get referenced analysisTemplate - fail", func(t *testing.T) { c, _, _ := f.newController(noResyncPeriodFunc) @@ -1685,6 +1762,12 @@ func TestGetInnerReferencedAnalysisTemplate(t *testing.T) { }}, } f.clusterAnalysisTemplateLister = append(f.clusterAnalysisTemplateLister, clusterAnalysisTemplateWithAnalysisRefs("first-cluster-analysis-template-name", "second-cluster-analysis-template-name", "third-cluster-analysis-template-name")) + activeSvc := newService("active-service", 80, nil, r) + previewSvc := newService("preview-service", 80, nil, r) + f.kubeobjects = append(f.kubeobjects, activeSvc, previewSvc) + f.serviceLister = append(f.serviceLister, activeSvc, previewSvc) + f.objects = append(f.objects, r) + f.rolloutLister = append(f.rolloutLister, r) t.Run("get inner referenced analysisTemplate - fail", func(t *testing.T) { c, _, _ := f.newController(noResyncPeriodFunc) @@ -1730,14 +1813,19 @@ func TestGetReferencedIngressesALB(t *testing.T) { }, } r.Namespace = metav1.NamespaceDefault + stableSvc := newService("stable", 80, nil, r) + canarySvc := newService("canary", 80, nil, r) + r.Spec.Strategy.Canary.StableService = stableSvc.Name + r.Spec.Strategy.Canary.CanaryService = canarySvc.Name + f.kubeobjects = append(f.kubeobjects, stableSvc, canarySvc) + f.serviceLister = append(f.serviceLister, stableSvc, canarySvc) + f.objects = append(f.objects, r) + f.rolloutLister = append(f.rolloutLister, r) t.Run("get referenced ALB ingress - fail", func(t *testing.T) { c, _, _ := f.newController(noResyncPeriodFunc) - roCtx, err := c.newRolloutContext(r) - assert.NoError(t, err) - _, err = roCtx.getReferencedIngresses() - expectedErr := field.Invalid(field.NewPath("spec", "strategy", "canary", "trafficRouting", "alb", "ingress"), "alb-ingress-name", "ingress.extensions \"alb-ingress-name\" not found") - assert.Equal(t, expectedErr.Error(), err.Error()) + _, err := c.newRolloutContext(r) + assert.Error(t, err) }) t.Run("get referenced ALB ingress - success", func(t *testing.T) { @@ -1746,6 +1834,31 @@ func TestGetReferencedIngressesALB(t *testing.T) { Name: "alb-ingress-name", Namespace: metav1.NamespaceDefault, }, + Spec: extensionsv1beta1.IngressSpec{ + IngressClassName: pointer.StringPtr("alb"), + Backend: &extensionsv1beta1.IngressBackend{ + ServiceName: "active-service", + ServicePort: intstr.IntOrString{IntVal: 80}, + }, + Rules: []extensionsv1beta1.IngressRule{ + { + Host: "example.com", + IngressRuleValue: extensionsv1beta1.IngressRuleValue{ + HTTP: &extensionsv1beta1.HTTPIngressRuleValue{ + Paths: []extensionsv1beta1.HTTPIngressPath{{ + Path: "", + PathType: nil, + Backend: extensionsv1beta1.IngressBackend{ + ServiceName: "stable", + ServicePort: intstr.IntOrString{IntVal: 80}, + }, + }, + }, + }, + }, + }, + }, + }, } f.ingressLister = append(f.ingressLister, ingressutil.NewLegacyIngress(ingress)) c, _, _ := f.newController(noResyncPeriodFunc) @@ -1808,6 +1921,7 @@ func TestGetReferencedIngressesALBMultiIngress(t *testing.T) { }, } + //TODO: cleanup expectedErr for _, test := range tests { t.Run(test.name, func(t *testing.T) { // clear fixture @@ -1816,10 +1930,10 @@ func TestGetReferencedIngressesALBMultiIngress(t *testing.T) { f.ingressLister = append(f.ingressLister, ing) } c, _, _ := f.newController(noResyncPeriodFunc) - roCtx, err := c.newRolloutContext(r) - assert.NoError(t, err) - _, err = roCtx.getReferencedIngresses() - assert.Equal(t, test.expectedErr.Error(), err.Error()) + _, err := c.newRolloutContext(r) + assert.Error(t, err) + //_, err = roCtx.getReferencedIngresses() + //assert.Equal(t, test.expectedErr.Error(), err.Error()) }) } @@ -1831,15 +1945,78 @@ func TestGetReferencedIngressesALBMultiIngress(t *testing.T) { Name: primaryIngress, Namespace: metav1.NamespaceDefault, }, + Spec: extensionsv1beta1.IngressSpec{ + IngressClassName: pointer.StringPtr("alb"), + Backend: &extensionsv1beta1.IngressBackend{ + ServiceName: "active-service", + ServicePort: intstr.IntOrString{IntVal: 80}, + }, + Rules: []extensionsv1beta1.IngressRule{ + { + Host: "example.com", + IngressRuleValue: extensionsv1beta1.IngressRuleValue{ + HTTP: &extensionsv1beta1.HTTPIngressRuleValue{ + Paths: []extensionsv1beta1.HTTPIngressPath{{ + Path: "", + PathType: nil, + Backend: extensionsv1beta1.IngressBackend{ + ServiceName: "active-service", + ServicePort: intstr.IntOrString{IntVal: 80}, + }, + }, + }, + }, + }, + }, + }, + }, } ingressAdditional := &extensionsv1beta1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: addIngress, Namespace: metav1.NamespaceDefault, }, + Spec: extensionsv1beta1.IngressSpec{ + IngressClassName: pointer.StringPtr("alb"), + Backend: &extensionsv1beta1.IngressBackend{ + ServiceName: "active-service", + ServicePort: intstr.IntOrString{IntVal: 80}, + }, + Rules: []extensionsv1beta1.IngressRule{ + { + Host: "example.com", + IngressRuleValue: extensionsv1beta1.IngressRuleValue{ + HTTP: &extensionsv1beta1.HTTPIngressRuleValue{ + Paths: []extensionsv1beta1.HTTPIngressPath{{ + Path: "", + PathType: nil, + Backend: extensionsv1beta1.IngressBackend{ + ServiceName: "active-service", + ServicePort: intstr.IntOrString{IntVal: 80}, + }, + }, + }, + }, + }, + }, + }, + }, } f.ingressLister = append(f.ingressLister, ingressutil.NewLegacyIngress(ingress)) f.ingressLister = append(f.ingressLister, ingressutil.NewLegacyIngress(ingressAdditional)) + f.kubeobjects = append(f.objects, ingress, ingressAdditional) + + activeSvc := newService("active-service", 80, nil, r) + previewSvc := newService("preview-service", 80, nil, r) + f.kubeobjects = append(f.kubeobjects, activeSvc, previewSvc) + f.serviceLister = append(f.serviceLister, activeSvc, previewSvc) + + r.Spec.Strategy.Canary.CanaryService = previewSvc.Name + r.Spec.Strategy.Canary.StableService = activeSvc.Name + + f.rolloutLister = append(f.rolloutLister, r) + f.objects = append(f.objects, r) + c, _, _ := f.newController(noResyncPeriodFunc) roCtx, err := c.newRolloutContext(r) assert.NoError(t, err) @@ -1866,11 +2043,8 @@ func TestGetReferencedIngressesNginx(t *testing.T) { // clear fixture f.ingressLister = []*ingressutil.Ingress{} c, _, _ := f.newController(noResyncPeriodFunc) - roCtx, err := c.newRolloutContext(r) - assert.NoError(t, err) - _, err = roCtx.getReferencedIngresses() - expectedErr := field.Invalid(field.NewPath("spec", "strategy", "canary", "trafficRouting", "nginx", "stableIngress"), primaryIngress, fmt.Sprintf("ingress.extensions \"%s\" not found", primaryIngress)) - assert.Equal(t, expectedErr.Error(), err.Error()) + _, err := c.newRolloutContext(r) + assert.Error(t, err) }) t.Run("get referenced Nginx ingress - success", func(t *testing.T) { @@ -1881,8 +2055,46 @@ func TestGetReferencedIngressesNginx(t *testing.T) { Name: primaryIngress, Namespace: metav1.NamespaceDefault, }, + Spec: extensionsv1beta1.IngressSpec{ + IngressClassName: pointer.StringPtr("alb"), + Backend: &extensionsv1beta1.IngressBackend{ + ServiceName: "active-service", + ServicePort: intstr.IntOrString{IntVal: 80}, + }, + Rules: []extensionsv1beta1.IngressRule{ + { + Host: "example.com", + IngressRuleValue: extensionsv1beta1.IngressRuleValue{ + HTTP: &extensionsv1beta1.HTTPIngressRuleValue{ + Paths: []extensionsv1beta1.HTTPIngressPath{{ + Path: "", + PathType: nil, + Backend: extensionsv1beta1.IngressBackend{ + ServiceName: "active-service", + ServicePort: intstr.IntOrString{IntVal: 80}, + }, + }, + }, + }, + }, + }, + }, + }, } f.ingressLister = append(f.ingressLister, ingressutil.NewLegacyIngress(ingress)) + f.kubeobjects = append(f.kubeobjects, ingress) + + activeSvc := newService("active-service", 80, nil, r) + previewSvc := newService("preview-service", 80, nil, r) + f.kubeobjects = append(f.kubeobjects, activeSvc, previewSvc) + f.serviceLister = append(f.serviceLister, activeSvc, previewSvc) + + r.Spec.Strategy.Canary.CanaryService = previewSvc.Name + r.Spec.Strategy.Canary.StableService = activeSvc.Name + + f.rolloutLister = append(f.rolloutLister, r) + f.objects = append(f.objects, r) + c, _, _ := f.newController(noResyncPeriodFunc) roCtx, err := c.newRolloutContext(r) assert.NoError(t, err) @@ -1949,10 +2161,8 @@ func TestGetReferencedIngressesNginxMultiIngress(t *testing.T) { f.ingressLister = append(f.ingressLister, ing) } c, _, _ := f.newController(noResyncPeriodFunc) - roCtx, err := c.newRolloutContext(r) - assert.NoError(t, err) - _, err = roCtx.getReferencedIngresses() - assert.Equal(t, test.expectedErr.Error(), err.Error()) + _, err := c.newRolloutContext(r) + assert.Error(t, err) }) } @@ -1964,15 +2174,77 @@ func TestGetReferencedIngressesNginxMultiIngress(t *testing.T) { Name: primaryIngress, Namespace: metav1.NamespaceDefault, }, + Spec: extensionsv1beta1.IngressSpec{ + IngressClassName: pointer.StringPtr("alb"), + Backend: &extensionsv1beta1.IngressBackend{ + ServiceName: "active-service", + ServicePort: intstr.IntOrString{IntVal: 80}, + }, + Rules: []extensionsv1beta1.IngressRule{ + { + Host: "example.com", + IngressRuleValue: extensionsv1beta1.IngressRuleValue{ + HTTP: &extensionsv1beta1.HTTPIngressRuleValue{ + Paths: []extensionsv1beta1.HTTPIngressPath{{ + Path: "", + PathType: nil, + Backend: extensionsv1beta1.IngressBackend{ + ServiceName: "active-service", + ServicePort: intstr.IntOrString{IntVal: 80}, + }, + }, + }, + }, + }, + }, + }, + }, } ingressAdditional := &extensionsv1beta1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: addIngress, Namespace: metav1.NamespaceDefault, }, + Spec: extensionsv1beta1.IngressSpec{ + IngressClassName: pointer.StringPtr("alb"), + Backend: &extensionsv1beta1.IngressBackend{ + ServiceName: "active-service", + ServicePort: intstr.IntOrString{IntVal: 80}, + }, + Rules: []extensionsv1beta1.IngressRule{ + { + Host: "example.com", + IngressRuleValue: extensionsv1beta1.IngressRuleValue{ + HTTP: &extensionsv1beta1.HTTPIngressRuleValue{ + Paths: []extensionsv1beta1.HTTPIngressPath{{ + Path: "", + PathType: nil, + Backend: extensionsv1beta1.IngressBackend{ + ServiceName: "active-service", + ServicePort: intstr.IntOrString{IntVal: 80}, + }, + }, + }, + }, + }, + }, + }, + }, } f.ingressLister = append(f.ingressLister, ingressutil.NewLegacyIngress(ingress)) f.ingressLister = append(f.ingressLister, ingressutil.NewLegacyIngress(ingressAdditional)) + f.kubeobjects = append(f.kubeobjects, ingress, ingressAdditional) + + activeSvc := newService("active-service", 80, nil, r) + previewSvc := newService("preview-service", 80, nil, r) + f.kubeobjects = append(f.kubeobjects, activeSvc, previewSvc) + f.serviceLister = append(f.serviceLister, activeSvc, previewSvc) + + r.Spec.Strategy.Canary.CanaryService = previewSvc.Name + r.Spec.Strategy.Canary.StableService = activeSvc.Name + + f.objects = append(f.objects, r) + c, _, _ := f.newController(noResyncPeriodFunc) roCtx, err := c.newRolloutContext(r) assert.NoError(t, err) @@ -2009,11 +2281,8 @@ func TestGetReferencedAppMeshResources(t *testing.T) { c, _, _ := f.newController(noResyncPeriodFunc) rCopy := r.DeepCopy() rCopy.Spec.Strategy.Canary.TrafficRouting.AppMesh.VirtualService = nil - roCtx, err := c.newRolloutContext(rCopy) - assert.NoError(t, err) - _, err = roCtx.getRolloutReferencedResources() - expectedErr := field.Invalid(field.NewPath("spec", "strategy", "canary", "trafficRouting", "appmesh", "virtualService"), "null", "must provide virtual-service") - assert.Equal(t, expectedErr.Error(), err.Error()) + _, err := c.newRolloutContext(rCopy) + assert.Error(t, err) }) t.Run("should return error when virtual-service is not-found", func(t *testing.T) { @@ -2021,11 +2290,8 @@ func TestGetReferencedAppMeshResources(t *testing.T) { defer f.Close() c, _, _ := f.newController(noResyncPeriodFunc) - roCtx, err := c.newRolloutContext(r) - assert.NoError(t, err) - _, err = roCtx.getRolloutReferencedResources() - expectedErr := field.Invalid(field.NewPath("spec", "strategy", "canary", "trafficRouting", "appmesh", "virtualService"), "mysvc.default", "virtualservices.appmesh.k8s.aws \"mysvc\" not found") - assert.Equal(t, expectedErr.Error(), err.Error()) + _, err := c.newRolloutContext(r) + assert.Error(t, err) }) t.Run("should return error when virtual-router is not-found", func(t *testing.T) { @@ -2047,11 +2313,8 @@ spec: uVsvc := unstructuredutil.StrToUnstructuredUnsafe(vsvc) f.objects = append(f.objects, uVsvc) c, _, _ := f.newController(noResyncPeriodFunc) - roCtx, err := c.newRolloutContext(r) - assert.NoError(t, err) - _, err = roCtx.getRolloutReferencedResources() - expectedErr := field.Invalid(field.NewPath("spec", "strategy", "canary", "trafficRouting", "appmesh", "virtualService"), "mysvc.default", "virtualrouters.appmesh.k8s.aws \"mysvc-vrouter\" not found") - assert.Equal(t, expectedErr.Error(), err.Error()) + _, err := c.newRolloutContext(r) + assert.Error(t, err) }) t.Run("get referenced App Mesh - success", func(t *testing.T) { @@ -2096,10 +2359,20 @@ spec: name: mysvc-stable-vn weight: 100 ` + r := r.DeepCopy() + activeSvc := newService("active-service", 80, nil, r) + previewSvc := newService("preview-service", 80, nil, r) + f.kubeobjects = append(f.kubeobjects, activeSvc, previewSvc) + f.serviceLister = append(f.serviceLister, activeSvc, previewSvc) + + r.Spec.Strategy.Canary.CanaryService = previewSvc.Name + r.Spec.Strategy.Canary.StableService = activeSvc.Name uVsvc := unstructuredutil.StrToUnstructuredUnsafe(vsvc) uVrouter := unstructuredutil.StrToUnstructuredUnsafe(vrouter) f.objects = append(f.objects, uVsvc, uVrouter) + f.rolloutLister = append(f.rolloutLister, r) + f.objects = append(f.objects, r) c, _, _ := f.newController(noResyncPeriodFunc) roCtx, err := c.newRolloutContext(r) assert.NoError(t, err) @@ -2128,14 +2401,14 @@ func TestGetAmbassadorMappings(t *testing.T) { }, } r.Namespace = metav1.NamespaceDefault - roCtx, err := c.newRolloutContext(r) - assert.NoError(t, err) - - // when - _, err = roCtx.getAmbassadorMappings() - - // then + _, err := c.newRolloutContext(r) assert.Error(t, err) + + //// when + //_, err = roCtx.getAmbassadorMappings() + // + //// then + //assert.Error(t, err) }) } @@ -2157,7 +2430,7 @@ func TestRolloutStrategyNotSet(t *testing.T) { f.serviceLister = append(f.serviceLister, previewSvc, activeSvc) patchIndex := f.expectPatchRolloutAction(r) - f.run(getKey(r, t)) + f.runExpectError(getKey(r, t), true) patchedRollout := f.getPatchedRollout(patchIndex) assert.Contains(t, patchedRollout, `Rollout has missing field '.spec.strategy.canary or .spec.strategy.blueGreen'`) } diff --git a/rollout/ephemeralmetadata.go b/rollout/ephemeralmetadata.go index 60b745ed29..49bacb345a 100644 --- a/rollout/ephemeralmetadata.go +++ b/rollout/ephemeralmetadata.go @@ -63,12 +63,27 @@ func (c *rolloutContext) syncEphemeralMetadata(ctx context.Context, rs *appsv1.R if !modified { return nil } - // 1. Sync ephemeral metadata to pods + + // Used to access old ephemeral data when updating pods below + originalRSCopy := rs.DeepCopy() + + // Order of the following two steps is important for race condition + // First update replicasets, then pods owned by it. + // So that any replicas created in the interim between the two steps are using the new updated version. + // 1. Update ReplicaSet so that any new pods it creates will have the metadata + rs, err := c.updateReplicaSet(ctx, modifiedRS) + if err != nil { + c.log.Infof("failed to sync ephemeral metadata %v to ReplicaSet %s: %v", podMetadata, originalRSCopy.Name, err) + return fmt.Errorf("failed to sync ephemeral metadata: %w", err) + } + c.log.Infof("synced ephemeral metadata %v to ReplicaSet %s", podMetadata, rs.Name) + + // 2. Sync ephemeral metadata to pods pods, err := replicasetutil.GetPodsOwnedByReplicaSet(ctx, c.kubeclientset, rs) if err != nil { return err } - existingPodMetadata := replicasetutil.ParseExistingPodMetadata(rs) + existingPodMetadata := replicasetutil.ParseExistingPodMetadata(originalRSCopy) for _, pod := range pods { newPodObjectMeta, podModified := replicasetutil.SyncEphemeralPodMetadata(&pod.ObjectMeta, existingPodMetadata, podMetadata) if podModified { @@ -81,15 +96,5 @@ func (c *rolloutContext) syncEphemeralMetadata(ctx context.Context, rs *appsv1.R } } - // 2. Update ReplicaSet so that any new pods it creates will have the metadata - rs, err = c.kubeclientset.AppsV1().ReplicaSets(modifiedRS.Namespace).Update(ctx, modifiedRS, metav1.UpdateOptions{}) - if err != nil { - return fmt.Errorf("error updating replicaset in syncEphemeralMetadata: %w", err) - } - err = c.replicaSetInformer.GetIndexer().Update(rs) - if err != nil { - return fmt.Errorf("error updating replicaset informer in syncEphemeralMetadata: %w", err) - } - c.log.Infof("synced ephemeral metadata %v to ReplicaSet %s", podMetadata, rs.Name) return nil } diff --git a/rollout/ephemeralmetadata_test.go b/rollout/ephemeralmetadata_test.go index 59daf1be0e..7c1af82da2 100644 --- a/rollout/ephemeralmetadata_test.go +++ b/rollout/ephemeralmetadata_test.go @@ -133,9 +133,9 @@ func TestSyncCanaryEphemeralMetadataSecondRevision(t *testing.T) { f.expectUpdateRolloutStatusAction(r2) // Update Rollout conditions rs2idx := f.expectCreateReplicaSetAction(rs2) // Create revision 2 ReplicaSet + rs1idx := f.expectUpdateReplicaSetAction(rs1) // update stable replicaset with stable metadata f.expectListPodAction(r1.Namespace) // list pods to patch ephemeral data on revision 1 ReplicaSets pods podIdx := f.expectUpdatePodAction(&pod) // Update pod with ephemeral data - rs1idx := f.expectUpdateReplicaSetAction(rs1) // update stable replicaset with stable metadata f.expectUpdateReplicaSetAction(rs1) // scale revision 1 ReplicaSet down f.expectPatchRolloutAction(r2) // Patch Rollout status @@ -169,7 +169,7 @@ func TestSyncBlueGreenEphemeralMetadataSecondRevision(t *testing.T) { f := newFixture(t) defer f.Close() - r1 := newBlueGreenRollout("foo", 1, nil, "active", "preview") + r1 := newBlueGreenRollout("foo", 3, nil, "active", "preview") r1.Spec.Strategy.BlueGreen.AutoPromotionEnabled = pointer.BoolPtr(false) r1.Annotations[annotations.RevisionAnnotation] = "1" r1.Spec.Strategy.BlueGreen.PreviewMetadata = &v1alpha1.PodTemplateMetadata{ @@ -213,9 +213,9 @@ func TestSyncBlueGreenEphemeralMetadataSecondRevision(t *testing.T) { rs2idx := f.expectCreateReplicaSetAction(rs2) // Create revision 2 ReplicaSet f.expectPatchServiceAction(previewSvc, rs2PodHash) // Update preview service to point at revision 2 replicaset f.expectUpdateReplicaSetAction(rs2) // scale revision 2 ReplicaSet up + rs1idx := f.expectUpdateReplicaSetAction(rs1) // update stable replicaset with stable metadata f.expectListPodAction(r1.Namespace) // list pods to patch ephemeral data on revision 1 ReplicaSets pods` podIdx := f.expectUpdatePodAction(&pod) // Update pod with ephemeral data - rs1idx := f.expectUpdateReplicaSetAction(rs1) // update stable replicaset with stable metadata f.expectPatchRolloutAction(r2) // Patch Rollout status f.run(getKey(r2, t)) diff --git a/rollout/mocks/TrafficRoutingReconciler.go b/rollout/mocks/TrafficRoutingReconciler.go index c8c3a7cde8..ceabc90a3d 100644 --- a/rollout/mocks/TrafficRoutingReconciler.go +++ b/rollout/mocks/TrafficRoutingReconciler.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.42.2. DO NOT EDIT. package mocks @@ -17,6 +17,10 @@ type TrafficRoutingReconciler struct { func (_m *TrafficRoutingReconciler) RemoveManagedRoutes() error { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for RemoveManagedRoutes") + } + var r0 error if rf, ok := ret.Get(0).(func() error); ok { r0 = rf() @@ -31,6 +35,10 @@ func (_m *TrafficRoutingReconciler) RemoveManagedRoutes() error { func (_m *TrafficRoutingReconciler) SetHeaderRoute(setHeaderRoute *v1alpha1.SetHeaderRoute) error { ret := _m.Called(setHeaderRoute) + if len(ret) == 0 { + panic("no return value specified for SetHeaderRoute") + } + var r0 error if rf, ok := ret.Get(0).(func(*v1alpha1.SetHeaderRoute) error); ok { r0 = rf(setHeaderRoute) @@ -45,6 +53,10 @@ func (_m *TrafficRoutingReconciler) SetHeaderRoute(setHeaderRoute *v1alpha1.SetH func (_m *TrafficRoutingReconciler) SetMirrorRoute(setMirrorRoute *v1alpha1.SetMirrorRoute) error { ret := _m.Called(setMirrorRoute) + if len(ret) == 0 { + panic("no return value specified for SetMirrorRoute") + } + var r0 error if rf, ok := ret.Get(0).(func(*v1alpha1.SetMirrorRoute) error); ok { r0 = rf(setMirrorRoute) @@ -66,6 +78,10 @@ func (_m *TrafficRoutingReconciler) SetWeight(desiredWeight int32, additionalDes _ca = append(_ca, _va...) ret := _m.Called(_ca...) + if len(ret) == 0 { + panic("no return value specified for SetWeight") + } + var r0 error if rf, ok := ret.Get(0).(func(int32, ...v1alpha1.WeightDestination) error); ok { r0 = rf(desiredWeight, additionalDestinations...) @@ -80,6 +96,10 @@ func (_m *TrafficRoutingReconciler) SetWeight(desiredWeight int32, additionalDes func (_m *TrafficRoutingReconciler) Type() string { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for Type") + } + var r0 string if rf, ok := ret.Get(0).(func() string); ok { r0 = rf() @@ -101,6 +121,10 @@ func (_m *TrafficRoutingReconciler) UpdateHash(canaryHash string, stableHash str _ca = append(_ca, _va...) ret := _m.Called(_ca...) + if len(ret) == 0 { + panic("no return value specified for UpdateHash") + } + var r0 error if rf, ok := ret.Get(0).(func(string, string, ...v1alpha1.WeightDestination) error); ok { r0 = rf(canaryHash, stableHash, additionalDestinations...) @@ -122,6 +146,10 @@ func (_m *TrafficRoutingReconciler) VerifyWeight(desiredWeight int32, additional _ca = append(_ca, _va...) ret := _m.Called(_ca...) + if len(ret) == 0 { + panic("no return value specified for VerifyWeight") + } + var r0 *bool var r1 error if rf, ok := ret.Get(0).(func(int32, ...v1alpha1.WeightDestination) (*bool, error)); ok { diff --git a/rollout/replicaset.go b/rollout/replicaset.go index c16ce6f037..8412420be9 100644 --- a/rollout/replicaset.go +++ b/rollout/replicaset.go @@ -39,10 +39,6 @@ func (c *rolloutContext) removeScaleDownDelay(rs *appsv1.ReplicaSet) error { return fmt.Errorf("error removing scale-down-deadline annotation from RS '%s': %w", rs.Name, err) } c.log.Infof("Removed '%s' annotation from RS '%s'", v1alpha1.DefaultReplicaSetScaleDownDeadlineAnnotationKey, rs.Name) - err = c.replicaSetInformer.GetIndexer().Update(rs) - if err != nil { - return fmt.Errorf("error updating replicaset informer in removeScaleDownDelay: %w", err) - } return err } @@ -68,10 +64,6 @@ func (c *rolloutContext) addScaleDownDelay(rs *appsv1.ReplicaSet, scaleDownDelay return fmt.Errorf("error adding scale-down-deadline annotation to RS '%s': %w", rs.Name, err) } c.log.Infof("Set '%s' annotation on '%s' to %s (%s)", v1alpha1.DefaultReplicaSetScaleDownDeadlineAnnotationKey, rs.Name, deadline, scaleDownDelaySeconds) - err = c.replicaSetInformer.GetIndexer().Update(rs) - if err != nil { - return fmt.Errorf("error updating replicaset informer in addScaleDownDelay: %w", err) - } return err } @@ -115,6 +107,16 @@ func (c *rolloutContext) removeScaleDownDeadlines() error { toRemove = append(toRemove, c.stableRS) } } + for _, rs := range c.otherRSs { + remainScaleDownDeadlines, err := replicasetutil.GetTimeRemainingBeforeScaleDownDeadline(rs) + if err != nil { + c.log.Warnf("%v", err) + continue + } + if replicasetutil.HasScaleDownDeadline(rs) && remainScaleDownDeadlines == nil { + toRemove = append(toRemove, rs) + } + } for _, rs := range toRemove { err := c.removeScaleDownDelay(rs) if err != nil { diff --git a/rollout/replicaset_test.go b/rollout/replicaset_test.go index 2e61606c37..bf185a1c2c 100644 --- a/rollout/replicaset_test.go +++ b/rollout/replicaset_test.go @@ -365,14 +365,21 @@ func TestReconcileOldReplicaSet(t *testing.T) { oldRS := rs("foo-old", test.oldReplicas, oldSelector, noTimestamp, nil) oldRS.Annotations = map[string]string{annotations.DesiredReplicasAnnotation: strconv.Itoa(test.oldReplicas)} oldRS.Status.AvailableReplicas = int32(test.readyPodsFromOldRS) - rollout := newBlueGreenRollout("foo", test.rolloutReplicas, nil, "", "") + + rollout := newBlueGreenRollout("foo", test.rolloutReplicas, nil, "active-service", "preview-service") rollout.Spec.Strategy.BlueGreen.ScaleDownDelayRevisionLimit = pointer.Int32Ptr(0) rollout.Spec.Selector = &metav1.LabelSelector{MatchLabels: newSelector} + + activeService := newService("active-service", 80, nil, nil) + previewService := newService("preview-service", 80, nil, nil) + rollout.Spec.Template.Labels["foo"] = "new" + f := newFixture(t) defer f.Close() f.objects = append(f.objects, rollout) f.replicaSetLister = append(f.replicaSetLister, oldRS, newRS) - f.kubeobjects = append(f.kubeobjects, oldRS, newRS) + f.serviceLister = append(f.serviceLister, activeService, previewService) + f.kubeobjects = append(f.kubeobjects, oldRS, newRS, activeService, previewService) c, informers, _ := f.newController(noResyncPeriodFunc) stopCh := make(chan struct{}) informers.Start(stopCh) diff --git a/rollout/restart_test.go b/rollout/restart_test.go index a6a20b4188..1ef873f36d 100644 --- a/rollout/restart_test.go +++ b/rollout/restart_test.go @@ -2,7 +2,7 @@ package rollout import ( "bytes" - "fmt" + "errors" "testing" "time" @@ -199,7 +199,7 @@ func TestRestartReconcile(t *testing.T) { } client.PrependReactor("create", "pods", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { // this is the pod eviction - return true, nil, fmt.Errorf(expectedErrMsg) + return true, nil, errors.New(expectedErrMsg) }) r := RolloutPodRestarter{ client: client, @@ -309,7 +309,7 @@ func TestRestartReplicaSetPod(t *testing.T) { client := fake.NewSimpleClientset() expectedErrMsg := "big bad error" client.PrependReactor("list", "pods", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, fmt.Errorf(expectedErrMsg) + return true, nil, errors.New(expectedErrMsg) }) r := RolloutPodRestarter{client: client} roCtx := &rolloutContext{ diff --git a/rollout/scale_utils_test.go b/rollout/scale_utils_test.go index 0127cfab69..6749761821 100644 --- a/rollout/scale_utils_test.go +++ b/rollout/scale_utils_test.go @@ -5,16 +5,17 @@ import ( "fmt" "testing" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" - "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/fake" - logutil "github.com/argoproj/argo-rollouts/utils/log" - "github.com/argoproj/argo-rollouts/utils/record" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" k8sfake "k8s.io/client-go/kubernetes/fake" k8stesting "k8s.io/client-go/testing" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/fake" + logutil "github.com/argoproj/argo-rollouts/utils/log" + "github.com/argoproj/argo-rollouts/utils/record" ) type DeploymentActions interface { diff --git a/rollout/service.go b/rollout/service.go index 69739b9315..91bbd39566 100644 --- a/rollout/service.go +++ b/rollout/service.go @@ -4,6 +4,12 @@ import ( "context" "fmt" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + patchtypes "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" "github.com/argoproj/argo-rollouts/rollout/trafficrouting" "github.com/argoproj/argo-rollouts/utils/annotations" @@ -15,11 +21,6 @@ import ( replicasetutil "github.com/argoproj/argo-rollouts/utils/replicaset" rolloututils "github.com/argoproj/argo-rollouts/utils/rollout" serviceutil "github.com/argoproj/argo-rollouts/utils/service" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - patchtypes "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/pointer" ) const ( @@ -130,7 +131,7 @@ func (c *rolloutContext) areTargetsVerified() bool { // by an ALB Ingress, which can be determined if there exists a TargetGroupBinding object in the // namespace that references the given service func (c *rolloutContext) awsVerifyTargetGroups(svc *corev1.Service) error { - if !c.shouldVerifyTargetGroup(svc) { + if !shouldVerifyTargetGroup(c.rollout, c.newRS, svc) { return nil } logCtx := c.log.WithField(logutil.ServiceKey, svc.Name) @@ -182,14 +183,14 @@ func (c *rolloutContext) awsVerifyTargetGroups(svc *corev1.Service) error { } // shouldVerifyTargetGroup returns whether or not we should verify the target group -func (c *rolloutContext) shouldVerifyTargetGroup(svc *corev1.Service) bool { +func shouldVerifyTargetGroup(rollout *v1alpha1.Rollout, newRS *appsv1.ReplicaSet, svc *corev1.Service) bool { if !defaults.VerifyTargetGroup() { // feature is disabled return false } - desiredPodHash := c.newRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey] - if c.rollout.Spec.Strategy.BlueGreen != nil { - if c.rollout.Status.StableRS == desiredPodHash { + desiredPodHash := newRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey] + if rollout.Spec.Strategy.BlueGreen != nil { + if rollout.Status.StableRS == desiredPodHash { // for blue-green, we only verify targets right after switching active service. So if // we are fully promoted, then there is no need to verify targets. // NOTE: this is the opposite of canary, where we only verify targets if stable == desired @@ -200,17 +201,17 @@ func (c *rolloutContext) shouldVerifyTargetGroup(svc *corev1.Service) bool { // we have not yet switched service selector return false } - if c.rollout.Status.BlueGreen.PostPromotionAnalysisRunStatus != nil { + if rollout.Status.BlueGreen.PostPromotionAnalysisRunStatus != nil { // we already started post-promotion analysis, so verification already occurred return false } return true - } else if c.rollout.Spec.Strategy.Canary != nil { - if c.rollout.Spec.Strategy.Canary.TrafficRouting == nil || c.rollout.Spec.Strategy.Canary.TrafficRouting.ALB == nil { + } else if rollout.Spec.Strategy.Canary != nil { + if rollout.Spec.Strategy.Canary.TrafficRouting == nil || rollout.Spec.Strategy.Canary.TrafficRouting.ALB == nil { // not ALB canary, so no need to verify targets return false } - if c.rollout.Status.StableRS != desiredPodHash { + if rollout.Status.StableRS != desiredPodHash { // for canary, we only verify targets right after switching stable service, which happens // after the update. So if stable != desired, we are still in the middle of an update // and there is no need to verify targets. @@ -243,7 +244,7 @@ func (c *rolloutContext) getPreviewAndActiveServices() (*corev1.Service, *corev1 func (c *rolloutContext) reconcilePingAndPongService() error { if trafficrouting.IsPingPongEnabled(c.rollout) && !rolloututils.IsFullyPromoted(c.rollout) { - _, canaryService := trafficrouting.GetStableAndCanaryServices(c.rollout) + _, canaryService := trafficrouting.GetStableAndCanaryServices(c.rollout, true) return c.ensureSVCTargets(canaryService, c.newRS, false) } return nil diff --git a/rollout/service_test.go b/rollout/service_test.go index 0466634bde..f31cd1bf3a 100644 --- a/rollout/service_test.go +++ b/rollout/service_test.go @@ -13,7 +13,6 @@ import ( "github.com/stretchr/testify/mock" corev1 "k8s.io/api/core/v1" extensionsv1beta1 "k8s.io/api/extensions/v1beta1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/intstr" @@ -53,7 +52,6 @@ func newService(name string, port int, selector map[string]string, ro *v1alpha1. } func TestGetPreviewAndActiveServices(t *testing.T) { - f := newFixture(t) defer f.Close() expActive := newService("active", 80, nil, nil) @@ -66,19 +64,16 @@ func TestGetPreviewAndActiveServices(t *testing.T) { otherRoSvc := newService("other-svc", 80, nil, otherRo) f.kubeobjects = append(f.kubeobjects, expActive, expPreview, otherRoSvc) f.serviceLister = append(f.serviceLister, expActive, expPreview, otherRoSvc) - rollout := &v1alpha1.Rollout{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: metav1.NamespaceDefault, - }, - Spec: v1alpha1.RolloutSpec{ - Strategy: v1alpha1.RolloutStrategy{ - BlueGreen: &v1alpha1.BlueGreenStrategy{ - PreviewService: "preview", - ActiveService: "active", - }, - }, + rollout := newRollout("foo", 1, nil, map[string]string{"foo": "bar"}) + rollout.Spec.Strategy = v1alpha1.RolloutStrategy{ + BlueGreen: &v1alpha1.BlueGreenStrategy{ + PreviewService: "preview", + ActiveService: "active", }, } + f.rolloutLister = append(f.rolloutLister, rollout) + f.objects = append(f.objects, rollout) + c, _, _ := f.newController(noResyncPeriodFunc) t.Run("Get Both", func(t *testing.T) { roCtx, err := c.newRolloutContext(rollout) @@ -91,30 +86,22 @@ func TestGetPreviewAndActiveServices(t *testing.T) { t.Run("Preview not found", func(t *testing.T) { noPreviewSvcRollout := rollout.DeepCopy() noPreviewSvcRollout.Spec.Strategy.BlueGreen.PreviewService = "not-preview" - roCtx, err := c.newRolloutContext(noPreviewSvcRollout) - assert.NoError(t, err) - _, _, err = roCtx.getPreviewAndActiveServices() - assert.NotNil(t, err) - assert.True(t, errors.IsNotFound(err)) + _, err := c.newRolloutContext(noPreviewSvcRollout) + assert.Error(t, err) }) t.Run("Active not found", func(t *testing.T) { noActiveSvcRollout := rollout.DeepCopy() noActiveSvcRollout.Spec.Strategy.BlueGreen.ActiveService = "not-active" - roCtx, err := c.newRolloutContext(noActiveSvcRollout) - assert.NoError(t, err) - _, _, err = roCtx.getPreviewAndActiveServices() - assert.NotNil(t, err) - assert.True(t, errors.IsNotFound(err)) + _, err := c.newRolloutContext(noActiveSvcRollout) + assert.Error(t, err) }) t.Run("Invalid Spec: No Active Svc", func(t *testing.T) { noActiveSvcRollout := rollout.DeepCopy() noActiveSvcRollout.Spec.Strategy.BlueGreen.ActiveService = "" roCtx, err := c.newRolloutContext(noActiveSvcRollout) - assert.NoError(t, err) - _, _, err = roCtx.getPreviewAndActiveServices() - assert.NotNil(t, err) - assert.EqualError(t, err, "service \"\" not found") + assert.Error(t, err) + assert.Nil(t, roCtx) }) } @@ -132,7 +119,7 @@ func TestActiveServiceNotFound(t *testing.T) { f.serviceLister = append(f.serviceLister, previewSvc) patchIndex := f.expectPatchRolloutAction(r) - f.run(getKey(r, t)) + f.runExpectError(getKey(r, t), true) patch := f.getPatchedRollout(patchIndex) errmsg := "The Rollout \"foo\" is invalid: spec.strategy.blueGreen.activeService: Invalid value: \"active-svc\": service \"active-svc\" not found" @@ -160,7 +147,7 @@ func TestPreviewServiceNotFound(t *testing.T) { f.kubeobjects = append(f.kubeobjects, activeSvc) patchIndex := f.expectPatchRolloutAction(r) - f.run(getKey(r, t)) + f.runExpectError(getKey(r, t), true) patch := f.getPatchedRollout(patchIndex) errmsg := "The Rollout \"foo\" is invalid: spec.strategy.blueGreen.previewService: Invalid value: \"preview-svc\": service \"preview-svc\" not found" @@ -679,17 +666,12 @@ func TestShouldVerifyTargetGroups(t *testing.T) { defaults.SetVerifyTargetGroup(true) defer defaults.SetVerifyTargetGroup(false) - f := newFixture(t) - defer f.Close() - ctrl, _, _ := f.newController(noResyncPeriodFunc) - t.Run("CanaryNotUsingTrafficRouting", func(t *testing.T) { ro := newCanaryRollout("foo", 3, nil, nil, nil, intstr.FromString("25%"), intstr.FromString("25%")) - roCtx, err := ctrl.newRolloutContext(ro) - roCtx.newRS = newReplicaSetWithStatus(ro, 3, 3) - stableSvc := newService("stable", 80, map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: roCtx.newRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]}, ro) - assert.NoError(t, err) - assert.False(t, roCtx.shouldVerifyTargetGroup(stableSvc)) + + newRS := newReplicaSetWithStatus(ro, 3, 3) + stableSvc := newService("stable", 80, map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: newRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]}, ro) + assert.False(t, shouldVerifyTargetGroup(ro, newRS, stableSvc)) }) t.Run("CanaryNotFullyPromoted", func(t *testing.T) { ro := newCanaryRollout("foo", 3, nil, nil, nil, intstr.FromString("25%"), intstr.FromString("25%")) @@ -698,12 +680,10 @@ func TestShouldVerifyTargetGroups(t *testing.T) { Ingress: "ingress", }, } - roCtx, err := ctrl.newRolloutContext(ro) - roCtx.newRS = newReplicaSetWithStatus(ro, 3, 3) - stableSvc := newService("stable", 80, map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: roCtx.newRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]}, ro) + newRS := newReplicaSetWithStatus(ro, 3, 3) + stableSvc := newService("stable", 80, map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: newRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]}, ro) ro.Status.StableRS = "somethingelse" - assert.NoError(t, err) - assert.False(t, roCtx.shouldVerifyTargetGroup(stableSvc)) + assert.False(t, shouldVerifyTargetGroup(ro, newRS, stableSvc)) }) t.Run("CanaryFullyPromoted", func(t *testing.T) { ro := newCanaryRollout("foo", 3, nil, nil, nil, intstr.FromString("25%"), intstr.FromString("25%")) @@ -712,49 +692,40 @@ func TestShouldVerifyTargetGroups(t *testing.T) { Ingress: "ingress", }, } - roCtx, err := ctrl.newRolloutContext(ro) - roCtx.newRS = newReplicaSetWithStatus(ro, 3, 3) - stableSvc := newService("stable", 80, map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: roCtx.newRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]}, ro) - ro.Status.StableRS = roCtx.newRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey] - assert.NoError(t, err) - assert.True(t, roCtx.shouldVerifyTargetGroup(stableSvc)) + newRS := newReplicaSetWithStatus(ro, 3, 3) + stableSvc := newService("stable", 80, map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: newRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]}, ro) + ro.Status.StableRS = newRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey] + assert.True(t, shouldVerifyTargetGroup(ro, newRS, stableSvc)) }) t.Run("BlueGreenFullyPromoted", func(t *testing.T) { ro := newBlueGreenRollout("foo", 3, nil, "active-svc", "") - roCtx, err := ctrl.newRolloutContext(ro) - roCtx.newRS = newReplicaSetWithStatus(ro, 3, 3) - activeSvc := newService("active-svc", 80, map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: roCtx.newRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]}, ro) - ro.Status.StableRS = roCtx.newRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey] - assert.NoError(t, err) - assert.False(t, roCtx.shouldVerifyTargetGroup(activeSvc)) + + newRS := newReplicaSetWithStatus(ro, 3, 3) + activeSvc := newService("active-svc", 80, map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: newRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]}, ro) + ro.Status.StableRS = newRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey] + assert.False(t, shouldVerifyTargetGroup(ro, newRS, activeSvc)) }) t.Run("BlueGreenBeforePromotion", func(t *testing.T) { ro := newBlueGreenRollout("foo", 3, nil, "active-svc", "") - roCtx, err := ctrl.newRolloutContext(ro) - roCtx.newRS = newReplicaSetWithStatus(ro, 3, 3) + newRS := newReplicaSetWithStatus(ro, 3, 3) activeSvc := newService("active-svc", 80, map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: "oldrshash"}, ro) ro.Status.StableRS = "oldrshash" - assert.NoError(t, err) - assert.False(t, roCtx.shouldVerifyTargetGroup(activeSvc)) + assert.False(t, shouldVerifyTargetGroup(ro, newRS, activeSvc)) }) t.Run("BlueGreenAfterPromotionAfterPromotionAnalysisStarted", func(t *testing.T) { ro := newBlueGreenRollout("foo", 3, nil, "active-svc", "") - roCtx, err := ctrl.newRolloutContext(ro) - roCtx.newRS = newReplicaSetWithStatus(ro, 3, 3) - activeSvc := newService("active-svc", 80, map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: roCtx.newRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]}, ro) + newRS := newReplicaSetWithStatus(ro, 3, 3) + activeSvc := newService("active-svc", 80, map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: newRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]}, ro) ro.Status.StableRS = "oldrshash" ro.Status.BlueGreen.PostPromotionAnalysisRunStatus = &v1alpha1.RolloutAnalysisRunStatus{} - assert.NoError(t, err) - assert.False(t, roCtx.shouldVerifyTargetGroup(activeSvc)) + assert.False(t, shouldVerifyTargetGroup(ro, newRS, activeSvc)) }) t.Run("BlueGreenAfterPromotion", func(t *testing.T) { ro := newBlueGreenRollout("foo", 3, nil, "active-svc", "") - roCtx, err := ctrl.newRolloutContext(ro) - roCtx.newRS = newReplicaSetWithStatus(ro, 3, 3) - activeSvc := newService("active-svc", 80, map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: roCtx.newRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]}, ro) + newRS := newReplicaSetWithStatus(ro, 3, 3) + activeSvc := newService("active-svc", 80, map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: newRS.Labels[v1alpha1.DefaultRolloutUniqueLabelKey]}, ro) ro.Status.StableRS = "oldrshash" - assert.NoError(t, err) - assert.True(t, roCtx.shouldVerifyTargetGroup(activeSvc)) + assert.True(t, shouldVerifyTargetGroup(ro, newRS, activeSvc)) }) } @@ -773,6 +744,9 @@ func TestDelayCanaryStableServiceLabelInjection(t *testing.T) { f.kubeobjects = append(f.kubeobjects, canarySvc, stableSvc) f.serviceLister = append(f.serviceLister, canarySvc, stableSvc) + f.objects = append(f.objects, ro2) + f.rolloutLister = append(f.rolloutLister, ro2) + { // first ensure we don't update service because new/stable are both not available ctrl, _, _ := f.newController(noResyncPeriodFunc) @@ -840,6 +814,9 @@ func TestDelayCanaryStableServiceDelayOnAdoptedService(t *testing.T) { f.kubeobjects = append(f.kubeobjects, canarySvc, stableSvc) f.serviceLister = append(f.serviceLister, canarySvc, stableSvc) + f.objects = append(f.objects, ro2) + f.rolloutLister = append(f.rolloutLister, ro2) + t.Run("AdoptedService No Availability", func(t *testing.T) { // first ensure we don't update service because new/stable are both not available ctrl, _, _ := f.newController(noResyncPeriodFunc) diff --git a/rollout/stepplugin.go b/rollout/stepplugin.go new file mode 100644 index 0000000000..ef4e1da17a --- /dev/null +++ b/rollout/stepplugin.go @@ -0,0 +1,281 @@ +package rollout + +import ( + "fmt" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/rollout/steps/plugin" + "github.com/argoproj/argo-rollouts/utils/conditions" + "github.com/argoproj/argo-rollouts/utils/record" + replicasetutil "github.com/argoproj/argo-rollouts/utils/replicaset" + rolloututil "github.com/argoproj/argo-rollouts/utils/rollout" +) + +var ( + defaultControllerErrorBackoff = time.Second * 30 + defaultBackoffDelay = time.Second * 5 +) + +type stepPluginContext struct { + resolver plugin.Resolver + log *log.Entry + + stepPluginStatuses []v1alpha1.StepPluginStatus + hasError bool +} + +func (spc *stepPluginContext) reconcile(c *rolloutContext) error { + rollout := c.rollout.DeepCopy() + spc.stepPluginStatuses = rollout.Status.Canary.StepPluginStatuses + + //On abort, we need to abort all successful previous steps + if c.pauseContext.IsAborted() { + for i := len(spc.stepPluginStatuses) - 1; i >= 0; i-- { + pluginStatus := spc.stepPluginStatuses[i] + if pluginStatus.Operation != v1alpha1.StepPluginOperationRun { + // Only call abort for Run operation. + continue + } + pluginStep := rollout.Spec.Strategy.Canary.Steps[pluginStatus.Index] + if pluginStep.Plugin == nil { + continue + } + + stepPlugin, err := spc.resolver.Resolve(pluginStatus.Index, *pluginStep.Plugin, c.log) + if err != nil { + return spc.handleError(c, fmt.Errorf("could not create step plugin at index %d : %w", pluginStatus.Index, err)) + } + status, err := stepPlugin.Abort(rollout) + if err != nil { + return spc.handleError(c, fmt.Errorf("failed to abort plugin: %w", err)) + } + phaseTransition := spc.updateStepPluginStatus(status) + if phaseTransition { + spc.recordPhase(c, status) + } + } + return nil + } + + // On full promotion, we want to Terminate only the last step still in Running, if any + if rollout.Status.PromoteFull || rolloututil.IsFullyPromoted(rollout) { + stepIndex := spc.getStepToTerminate(rollout) + if stepIndex == nil { + return nil + } + + pluginStep := rollout.Spec.Strategy.Canary.Steps[*stepIndex] + if pluginStep.Plugin == nil { + return nil + } + + stepPlugin, err := spc.resolver.Resolve(*stepIndex, *pluginStep.Plugin, c.log) + if err != nil { + return spc.handleError(c, fmt.Errorf("could not create step plugin at index %d : %w", *stepIndex, err)) + } + + status, err := stepPlugin.Terminate(rollout) + if err != nil { + return spc.handleError(c, fmt.Errorf("failed to terminate plugin: %w", err)) + } + + phaseTransition := spc.updateStepPluginStatus(status) + if phaseTransition { + spc.recordPhase(c, status) + } + return nil + } + + // If we retry an aborted rollout, we need to have a clean status + spc.cleanStatusForRetry(rollout) + + // Normal execution flow of a step plugin + currentStep, currentStepIndex := replicasetutil.GetCurrentCanaryStep(rollout) + if currentStep == nil || currentStep.Plugin == nil { + return nil + } + + if rollout.Status.Phase != v1alpha1.RolloutPhaseProgressing { + spc.log.Debug("Not reconciling step plugin because it is not progressing") + return nil + } + + stepPlugin, err := spc.resolver.Resolve(*currentStepIndex, *currentStep.Plugin, c.log) + if err != nil { + return spc.handleError(c, fmt.Errorf("could not create step plugin at index %d : %w", *currentStepIndex, err)) + } + status, err := stepPlugin.Run(rollout) + if err != nil { + return spc.handleError(c, fmt.Errorf("failed to run plugin: %w", err)) + } + + phaseTransition := spc.updateStepPluginStatus(status) + if phaseTransition { + spc.recordPhase(c, status) + } + + if status == nil || status.Disabled { + return nil + } + + if status.Phase == v1alpha1.StepPluginPhaseRunning || status.Phase == v1alpha1.StepPluginPhaseError { + backoff, err := status.Backoff.Duration() + if err != nil { + return spc.handleError(c, fmt.Errorf("failed to parse backoff duration: %w", err)) + } + + // Get the remaining time until the backoff + a little buffer + remaining := time.Until(status.UpdatedAt.Add(backoff)) + defaultBackoffDelay + c.log.Debugf("queueing up rollout in %s because step plugin phase is %s", remaining, status.Phase) + c.enqueueRolloutAfter(rollout, remaining) + return nil + } + + if status.Phase == v1alpha1.StepPluginPhaseFailed { + c.pauseContext.AddAbort(fmt.Sprintf("Step Plugin %d (%s) failed: %s", status.Index+1, status.Name, status.Message)) + } + + return nil +} + +// handleError handles any error that should not cause the rollout reconciliation to fail +func (spc *stepPluginContext) handleError(c *rolloutContext, e error) error { + spc.hasError = true + + msg := fmt.Sprintf(conditions.RolloutReconciliationErrorMessage, e.Error()) + c.recorder.Warnf(c.rollout, record.EventOptions{EventReason: conditions.RolloutReconciliationErrorReason}, msg) + + c.log.Debugf("queueing up rollout in %s because of transient error", defaultControllerErrorBackoff) + c.enqueueRolloutAfter(c.rollout, defaultControllerErrorBackoff) + + return nil +} + +func (spc *stepPluginContext) recordPhase(c *rolloutContext, status *v1alpha1.StepPluginStatus) { + if status.Disabled || status.Operation == v1alpha1.StepPluginOperationRun && status.Phase == v1alpha1.StepPluginPhaseSuccessful { + // If the run status is successful, do not record event because the controller will record the RolloutStepCompleted + return + } + + msg := fmt.Sprintf(conditions.StepPluginTransitionRunMessage, status.Name, status.Index+1, status.Phase) + if status.Operation == v1alpha1.StepPluginOperationAbort { + msg = fmt.Sprintf(conditions.StepPluginTransitionAbortMessage, status.Name, status.Index+1, status.Phase) + } else if status.Operation == v1alpha1.StepPluginOperationTerminate { + msg = fmt.Sprintf(conditions.StepPluginTransitionTerminateMessage, status.Name, status.Index+1, status.Phase) + } + + if status.Phase == v1alpha1.StepPluginPhaseError || status.Phase == v1alpha1.StepPluginPhaseFailed { + c.recorder.Warnf(c.rollout, record.EventOptions{EventReason: conditions.StepPluginTransitionReason}, msg) + } else { + c.recorder.Eventf(c.rollout, record.EventOptions{EventReason: conditions.StepPluginTransitionReason}, msg) + } +} + +func (spc *stepPluginContext) updateStatus(status *v1alpha1.RolloutStatus) { + if spc.stepPluginStatuses != nil { + status.Canary.StepPluginStatuses = spc.stepPluginStatuses + } +} + +func (spc *stepPluginContext) isStepPluginCompleted(stepIndex int32, _ *v1alpha1.PluginStep) bool { + if spc.hasError { + // If there was a transient error during the reconcile, we should retry + return false + } + + runStatus := spc.findCurrentStepStatus(stepIndex, v1alpha1.StepPluginOperationRun) + if runStatus != nil && runStatus.Disabled { + return true + } + + isRunning := runStatus != nil && runStatus.Phase == v1alpha1.StepPluginPhaseRunning + if isRunning { + terminateStatus := spc.findCurrentStepStatus(stepIndex, v1alpha1.StepPluginOperationTerminate) + abortStatus := spc.findCurrentStepStatus(stepIndex, v1alpha1.StepPluginOperationAbort) + isRunning = terminateStatus == nil && abortStatus == nil + } + return runStatus != nil && + ((!isRunning && runStatus.Phase == v1alpha1.StepPluginPhaseRunning) || + runStatus.Phase == v1alpha1.StepPluginPhaseFailed || + runStatus.Phase == v1alpha1.StepPluginPhaseSuccessful) +} + +func (spc *stepPluginContext) findCurrentStepStatus(stepIndex int32, operation v1alpha1.StepPluginOperation) *v1alpha1.StepPluginStatus { + for _, s := range spc.stepPluginStatuses { + if s.Index == stepIndex && s.Operation == operation { + return &s + } + } + return nil +} + +func (spc *stepPluginContext) updateStepPluginStatus(status *v1alpha1.StepPluginStatus) bool { + phaseChanged := false + if status == nil { + return phaseChanged + } + + // Update new status and preserve order + statusUpdated := false + for i, s := range spc.stepPluginStatuses { + if !statusUpdated && s.Index == status.Index && s.Operation == status.Operation { + spc.stepPluginStatuses[i] = *status + statusUpdated = true + phaseChanged = s.Phase != status.Phase + break + } + } + if !statusUpdated { + spc.stepPluginStatuses = append(spc.stepPluginStatuses, *status) + phaseChanged = true + } + + return phaseChanged +} + +func (spc *stepPluginContext) getStepToTerminate(rollout *v1alpha1.Rollout) *int32 { + for i := len(rollout.Status.Canary.StepPluginStatuses) - 1; i >= 0; i-- { + pluginStep := rollout.Status.Canary.StepPluginStatuses[i] + + if pluginStep.Operation == v1alpha1.StepPluginOperationTerminate && pluginStep.Phase == v1alpha1.StepPluginPhaseSuccessful { + // last running step is already terminated + return nil + } + + if pluginStep.Operation == v1alpha1.StepPluginOperationRun && pluginStep.Phase == v1alpha1.StepPluginPhaseRunning { + // found the last running step + return &pluginStep.Index + } + } + return nil +} + +// cleanStatusForRetry is expected to be called on a non-aborted rollout. +// It validates that stepPluginStatuses does not contain outdated results, and remove them +// if it does. +func (spc *stepPluginContext) cleanStatusForRetry(rollout *v1alpha1.Rollout) { + if len(spc.stepPluginStatuses) == 0 || rollout.Status.Abort { + return + } + + currentStep, currentStepIndex := replicasetutil.GetCurrentCanaryStep(rollout) + if currentStep == nil || int(*currentStepIndex) > 0 { + // Nothing to clean if rollout steps are completed or in progress + return + } + + // if we are at step 0, it could be either that we haven't started, or we are progressing. + // if step 0 is a plugin, check if it is completed. In that case, we know that we should be at step > 0 + // abd we are retrying + shouldCleanCurrentStatus := true + if currentStep.Plugin != nil { + shouldCleanCurrentStatus = spc.isStepPluginCompleted(*currentStepIndex, currentStep.Plugin) + } + + if shouldCleanCurrentStatus { + spc.stepPluginStatuses = []v1alpha1.StepPluginStatus{} + } +} diff --git a/rollout/stepplugin_test.go b/rollout/stepplugin_test.go new file mode 100644 index 0000000000..da59296c5a --- /dev/null +++ b/rollout/stepplugin_test.go @@ -0,0 +1,897 @@ +package rollout + +import ( + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/rollout/steps/plugin/mocks" + logutil "github.com/argoproj/argo-rollouts/utils/log" + "github.com/argoproj/argo-rollouts/utils/record" +) + +func newStepPluginRollout() *v1alpha1.Rollout { + steps := []v1alpha1.CanaryStep{ + { + Plugin: &v1alpha1.PluginStep{ + Name: "test-plugin", + }, + }, + } + return newCanaryRollout("foo", 3, nil, steps, ptr.To(int32(0)), intstr.FromInt(1), intstr.FromInt(0)) +} + +func newStepPluginStatus(operation v1alpha1.StepPluginOperation, phase v1alpha1.StepPluginPhase) *v1alpha1.StepPluginStatus { + now := metav1.Now() + status := &v1alpha1.StepPluginStatus{ + Index: 0, + Name: "test-plugin", + Message: "this is a default message", + Status: json.RawMessage("value"), + Operation: operation, + Phase: phase, + StartedAt: &now, + UpdatedAt: &now, + Executions: 1, + Backoff: "0s", + } + return status +} + +func Test_stepPluginContext_reconcile_ReconciliationError(t *testing.T) { + stepPluginResolver := mocks.NewResolver(t) + r := newStepPluginRollout() + logCtx := logutil.WithRollout(r) + roCtx := &rolloutContext{ + rollout: r, + log: logutil.WithRollout(r), + reconcilerBase: reconcilerBase{ + enqueueRollout: func(obj any) { t.Error("enqueueRollout should not be called") }, + enqueueRolloutAfter: func(obj any, duration time.Duration) { t.Error("enqueueRolloutAfter should not be called") }, + recorder: record.NewFakeEventRecorder(), + }, + pauseContext: &pauseContext{ + rollout: r, + log: logCtx, + }, + stepPluginContext: &stepPluginContext{ + resolver: stepPluginResolver, + log: logCtx, + }, + } + + stepPluginResolver.On("Resolve", mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("test error")) + + var requeuedAfter time.Duration + roCtx.enqueueRolloutAfter = func(obj any, duration time.Duration) { + requeuedAfter = duration + } + + err := roCtx.stepPluginContext.reconcile(roCtx) + + require.NoError(t, err) + assert.Equal(t, roCtx.rollout.Status.Canary.StepPluginStatuses, roCtx.stepPluginContext.stepPluginStatuses) + assert.Equal(t, defaultControllerErrorBackoff, requeuedAfter) + assert.True(t, roCtx.stepPluginContext.hasError) + +} + +func Test_stepPluginContext_reconcile_SuccessfulReconciliation(t *testing.T) { + setup := func(t *testing.T) (*rolloutContext, *v1alpha1.StepPluginStatus) { + stepPluginResolver := mocks.NewResolver(t) + stepPluginMock := mocks.NewStepPlugin(t) + stepPluginResolver.On("Resolve", int32(0), mock.Anything, mock.Anything).Return(stepPluginMock, nil) + + r := newStepPluginRollout() + logCtx := logutil.WithRollout(r) + roCtx := &rolloutContext{ + rollout: r, + log: logutil.WithRollout(r), + reconcilerBase: reconcilerBase{ + enqueueRollout: func(obj any) { t.Error("enqueueRollout should not be called") }, + enqueueRolloutAfter: func(obj any, duration time.Duration) { t.Error("enqueueRolloutAfter should not be called") }, + recorder: record.NewFakeEventRecorder(), + }, + pauseContext: &pauseContext{ + rollout: r, + log: logCtx, + }, + stepPluginContext: &stepPluginContext{ + resolver: stepPluginResolver, + log: logCtx, + }, + } + + runStatus := newStepPluginStatus(v1alpha1.StepPluginOperationRun, v1alpha1.StepPluginPhaseSuccessful) + stepPluginMock.On("Run", r).Return(runStatus, nil) + return roCtx, runStatus + } + + t.Run("Status is added when not present", func(t *testing.T) { + roCtx, runStatus := setup(t) + + err := roCtx.stepPluginContext.reconcile(roCtx) + + require.NoError(t, err) + require.Len(t, roCtx.stepPluginContext.stepPluginStatuses, 1) + assert.EqualExportedValues(t, roCtx.stepPluginContext.stepPluginStatuses[0], *runStatus) + }) + t.Run("Status is updated when existing", func(t *testing.T) { + roCtx, runStatus := setup(t) + + roCtx.rollout.Status.Canary.StepPluginStatuses = []v1alpha1.StepPluginStatus{ + { + Index: runStatus.Index, + Name: runStatus.Name, + Message: "this is the existing status", + Operation: v1alpha1.StepPluginOperationRun, + Phase: v1alpha1.StepPluginPhaseRunning, + }, + } + + err := roCtx.stepPluginContext.reconcile(roCtx) + + require.NoError(t, err) + require.Len(t, roCtx.stepPluginContext.stepPluginStatuses, 1) + assert.EqualExportedValues(t, roCtx.stepPluginContext.stepPluginStatuses[0], *runStatus) + }) + t.Run("Status order is preserved when updating", func(t *testing.T) { + roCtx, runStatus := setup(t) + + roCtx.rollout.Status.Canary.StepPluginStatuses = []v1alpha1.StepPluginStatus{ + { + Index: 123, + Name: runStatus.Name, + Operation: v1alpha1.StepPluginOperationRun, + }, + { + Index: runStatus.Index, + Name: runStatus.Name, + Operation: v1alpha1.StepPluginOperationRun, + }, + { + Index: 456, + Name: "other", + Operation: v1alpha1.StepPluginOperationRun, + }, + } + + err := roCtx.stepPluginContext.reconcile(roCtx) + + require.NoError(t, err) + require.Len(t, roCtx.stepPluginContext.stepPluginStatuses, 3) + assert.Equal(t, int32(123), roCtx.stepPluginContext.stepPluginStatuses[0].Index) + assert.Equal(t, runStatus.Index, roCtx.stepPluginContext.stepPluginStatuses[1].Index) + assert.Equal(t, int32(456), roCtx.stepPluginContext.stepPluginStatuses[2].Index) + }) +} + +func Test_stepPluginContext_reconcile_RunningReconciliation(t *testing.T) { + setup := func(t *testing.T, phase v1alpha1.StepPluginPhase, backoff *time.Duration) (*rolloutContext, *v1alpha1.StepPluginStatus) { + stepPluginResolver := mocks.NewResolver(t) + stepPluginMock := mocks.NewStepPlugin(t) + stepPluginResolver.On("Resolve", int32(0), mock.Anything, mock.Anything).Return(stepPluginMock, nil) + + r := newStepPluginRollout() + logCtx := logutil.WithRollout(r) + roCtx := &rolloutContext{ + rollout: r, + log: logutil.WithRollout(r), + reconcilerBase: reconcilerBase{ + enqueueRollout: func(obj any) { t.Error("enqueueRollout should not be called") }, + enqueueRolloutAfter: func(obj any, duration time.Duration) {}, + recorder: record.NewFakeEventRecorder(), + }, + pauseContext: &pauseContext{ + rollout: r, + log: logCtx, + }, + stepPluginContext: &stepPluginContext{ + resolver: stepPluginResolver, + log: logCtx, + }, + } + + runStatus := newStepPluginStatus(v1alpha1.StepPluginOperationRun, phase) + if backoff != nil { + runStatus.Backoff = v1alpha1.DurationString(backoff.String()) + } + stepPluginMock.On("Run", r).Return(runStatus, nil) + return roCtx, runStatus + } + + t.Run("Status is added when not present", func(t *testing.T) { + roCtx, runStatus := setup(t, v1alpha1.StepPluginPhaseRunning, nil) + + err := roCtx.stepPluginContext.reconcile(roCtx) + + require.NoError(t, err) + require.Len(t, roCtx.stepPluginContext.stepPluginStatuses, 1) + assert.EqualExportedValues(t, roCtx.stepPluginContext.stepPluginStatuses[0], *runStatus) + }) + t.Run("Status is updated when existing", func(t *testing.T) { + roCtx, runStatus := setup(t, v1alpha1.StepPluginPhaseRunning, nil) + + roCtx.rollout.Status.Canary.StepPluginStatuses = []v1alpha1.StepPluginStatus{ + { + Index: runStatus.Index, + Name: runStatus.Name, + Message: "this is the existing status", + Operation: v1alpha1.StepPluginOperationRun, + Phase: v1alpha1.StepPluginPhaseRunning, + }, + } + + err := roCtx.stepPluginContext.reconcile(roCtx) + + require.NoError(t, err) + require.Len(t, roCtx.stepPluginContext.stepPluginStatuses, 1) + assert.EqualExportedValues(t, roCtx.stepPluginContext.stepPluginStatuses[0], *runStatus) + }) + t.Run("Rollout is added to the queue", func(t *testing.T) { + expectedRequeueAfter := 123 * time.Second + roCtx, _ := setup(t, v1alpha1.StepPluginPhaseRunning, &expectedRequeueAfter) + + var requeuedAfter time.Duration + roCtx.enqueueRolloutAfter = func(obj any, duration time.Duration) { + requeuedAfter = duration + } + + err := roCtx.stepPluginContext.reconcile(roCtx) + + require.NoError(t, err) + assert.GreaterOrEqual(t, requeuedAfter, expectedRequeueAfter) + assert.LessOrEqual(t, requeuedAfter, expectedRequeueAfter+defaultBackoffDelay) + }) +} + +func Test_stepPluginContext_reconcile_FailedReconciliation(t *testing.T) { + setup := func(t *testing.T, phase v1alpha1.StepPluginPhase) (*rolloutContext, *v1alpha1.StepPluginStatus) { + stepPluginResolver := mocks.NewResolver(t) + stepPluginMock := mocks.NewStepPlugin(t) + stepPluginResolver.On("Resolve", int32(0), mock.Anything, mock.Anything).Return(stepPluginMock, nil) + + r := newStepPluginRollout() + logCtx := logutil.WithRollout(r) + roCtx := &rolloutContext{ + rollout: r, + log: logCtx, + reconcilerBase: reconcilerBase{ + enqueueRollout: func(obj any) { t.Error("enqueueRollout should not be called") }, + enqueueRolloutAfter: func(obj any, duration time.Duration) { t.Error("enqueueRolloutAfter should not be called") }, + recorder: record.NewFakeEventRecorder(), + }, + pauseContext: &pauseContext{ + rollout: r, + log: logCtx, + }, + stepPluginContext: &stepPluginContext{ + resolver: stepPluginResolver, + log: logCtx, + }, + } + + runStatus := newStepPluginStatus(v1alpha1.StepPluginOperationRun, phase) + stepPluginMock.On("Run", r).Return(runStatus, nil) + return roCtx, runStatus + } + + t.Run("Status is added when not present", func(t *testing.T) { + roCtx, runStatus := setup(t, v1alpha1.StepPluginPhaseFailed) + + err := roCtx.stepPluginContext.reconcile(roCtx) + + require.NoError(t, err) + require.Len(t, roCtx.stepPluginContext.stepPluginStatuses, 1) + assert.EqualExportedValues(t, roCtx.stepPluginContext.stepPluginStatuses[0], *runStatus) + }) + t.Run("Status is updated when existing", func(t *testing.T) { + roCtx, runStatus := setup(t, v1alpha1.StepPluginPhaseFailed) + + roCtx.rollout.Status.Canary.StepPluginStatuses = []v1alpha1.StepPluginStatus{ + { + Index: runStatus.Index, + Name: runStatus.Name, + Message: "this is the existing status", + Operation: v1alpha1.StepPluginOperationRun, + Phase: v1alpha1.StepPluginPhaseRunning, + }, + } + + err := roCtx.stepPluginContext.reconcile(roCtx) + + require.NoError(t, err) + require.Len(t, roCtx.stepPluginContext.stepPluginStatuses, 1) + assert.EqualExportedValues(t, roCtx.stepPluginContext.stepPluginStatuses[0], *runStatus) + }) + t.Run("Rollout is aborted", func(t *testing.T) { + roCtx, failedStatus := setup(t, v1alpha1.StepPluginPhaseFailed) + + err := roCtx.stepPluginContext.reconcile(roCtx) + + require.NoError(t, err) + assert.True(t, roCtx.pauseContext.IsAborted()) + assert.Contains(t, roCtx.pauseContext.abortMessage, failedStatus.Message) + }) +} + +func Test_stepPluginContext_reconcile_FullyPromoted(t *testing.T) { + newRolloutContext := func(t *testing.T) *rolloutContext { + r := newStepPluginRollout() + + logCtx := logutil.WithRollout(r) + roCtx := &rolloutContext{ + rollout: r, + log: logCtx, + reconcilerBase: reconcilerBase{ + enqueueRollout: func(obj any) { t.Error("enqueueRollout should not be called") }, + enqueueRolloutAfter: func(obj any, duration time.Duration) { t.Error("enqueueRolloutAfter should not be called") }, + recorder: record.NewFakeEventRecorder(), + }, + pauseContext: &pauseContext{ + rollout: r, + log: logCtx, + }, + stepPluginContext: &stepPluginContext{ + log: logCtx, + }, + } + return roCtx + } + setup := func(t *testing.T) (*rolloutContext, *mocks.StepPlugin) { + stepPluginResolver := mocks.NewResolver(t) + stepPluginMock := mocks.NewStepPlugin(t) + stepPluginResolver.On("Resolve", int32(0), mock.Anything, mock.Anything).Return(stepPluginMock, nil) + + roCtx := newRolloutContext(t) + roCtx.stepPluginContext.resolver = stepPluginResolver + + return roCtx, stepPluginMock + } + + t.Run("Rollout is Terminated on full promotion", func(t *testing.T) { + roCtx, stepPluginMock := setup(t) + roCtx.rollout.Status.PromoteFull = true + roCtx.rollout.Status.StableRS = "stable-value" + roCtx.rollout.Status.CurrentPodHash = "current-value" + + runStatus := newStepPluginStatus(v1alpha1.StepPluginOperationTerminate, v1alpha1.StepPluginPhaseSuccessful) + roCtx.rollout.Status.Canary.StepPluginStatuses = []v1alpha1.StepPluginStatus{ + { + Index: runStatus.Index, + Name: runStatus.Name, + Phase: v1alpha1.StepPluginPhaseRunning, + Operation: v1alpha1.StepPluginOperationRun, + }, + } + + stepPluginMock.On("Terminate", mock.Anything).Return(runStatus, nil) + + err := roCtx.stepPluginContext.reconcile(roCtx) + + require.NoError(t, err) + require.Len(t, roCtx.stepPluginContext.stepPluginStatuses, 2) + assert.EqualExportedValues(t, roCtx.stepPluginContext.stepPluginStatuses[1], *runStatus) + }) + + t.Run("Rollout is Terminated on fully promoted", func(t *testing.T) { + roCtx, stepPluginMock := setup(t) + roCtx.rollout.Status.PromoteFull = false + roCtx.rollout.Status.StableRS = "stable-value" + roCtx.rollout.Status.CurrentPodHash = "stable-value" + + runStatus := newStepPluginStatus(v1alpha1.StepPluginOperationTerminate, v1alpha1.StepPluginPhaseSuccessful) + roCtx.rollout.Status.Canary.StepPluginStatuses = []v1alpha1.StepPluginStatus{ + { + Index: runStatus.Index, + Name: runStatus.Name, + Phase: v1alpha1.StepPluginPhaseRunning, + Operation: v1alpha1.StepPluginOperationRun, + }, + } + + stepPluginMock.On("Terminate", mock.Anything).Return(runStatus, nil) + + err := roCtx.stepPluginContext.reconcile(roCtx) + + require.NoError(t, err) + require.Len(t, roCtx.stepPluginContext.stepPluginStatuses, 2) + assert.EqualExportedValues(t, roCtx.stepPluginContext.stepPluginStatuses[1], *runStatus) + }) + + t.Run("Rollout is Reconciled when already terminated", func(t *testing.T) { + roCtx := newRolloutContext(t) + roCtx.rollout.Status.PromoteFull = false + roCtx.rollout.Status.StableRS = "stable-value" + roCtx.rollout.Status.CurrentPodHash = "stable-value" + roCtx.rollout.Status.CurrentStepIndex = ptr.To(int32(1)) + + runStatus := newStepPluginStatus(v1alpha1.StepPluginOperationTerminate, v1alpha1.StepPluginPhaseSuccessful) + roCtx.rollout.Status.Canary.StepPluginStatuses = []v1alpha1.StepPluginStatus{ + { + Index: runStatus.Index, + Name: runStatus.Name, + Phase: v1alpha1.StepPluginPhaseRunning, + Operation: v1alpha1.StepPluginOperationRun, + }, + { + Index: runStatus.Index, + Name: runStatus.Name, + Phase: v1alpha1.StepPluginPhaseSuccessful, + Operation: v1alpha1.StepPluginOperationTerminate, + }, + } + + err := roCtx.stepPluginContext.reconcile(roCtx) + + require.NoError(t, err) + require.Len(t, roCtx.stepPluginContext.stepPluginStatuses, 2) + assert.EqualExportedValues(t, roCtx.rollout.Status.Canary.StepPluginStatuses[1], roCtx.stepPluginContext.stepPluginStatuses[1]) + }) + + t.Run("Reconciliation error", func(t *testing.T) { + roCtx, stepPluginMock := setup(t) + roCtx.rollout.Status.PromoteFull = true + + runStatus := newStepPluginStatus(v1alpha1.StepPluginOperationTerminate, v1alpha1.StepPluginPhaseSuccessful) + roCtx.rollout.Status.Canary.StepPluginStatuses = []v1alpha1.StepPluginStatus{ + { + Index: runStatus.Index, + Name: runStatus.Name, + Phase: v1alpha1.StepPluginPhaseRunning, + Operation: v1alpha1.StepPluginOperationRun, + }, + } + + var requeuedAfter time.Duration + roCtx.enqueueRolloutAfter = func(obj any, duration time.Duration) { + requeuedAfter = duration + } + + stepPluginMock.On("Terminate", mock.Anything).Return(nil, fmt.Errorf("error")) + + err := roCtx.stepPluginContext.reconcile(roCtx) + + require.NoError(t, err) + require.Len(t, roCtx.stepPluginContext.stepPluginStatuses, 1) + assert.Equal(t, roCtx.rollout.Status.Canary.StepPluginStatuses, roCtx.stepPluginContext.stepPluginStatuses) + assert.Equal(t, defaultControllerErrorBackoff, requeuedAfter) + assert.True(t, roCtx.stepPluginContext.hasError) + }) +} + +func Test_stepPluginContext_reconcile_Aborted(t *testing.T) { + setup := func(t *testing.T) (*rolloutContext, *mocks.Resolver) { + stepPluginResolver := mocks.NewResolver(t) + + r := newStepPluginRollout() + r.Status.Abort = true + + logCtx := logutil.WithRollout(r) + roCtx := &rolloutContext{ + rollout: r, + log: logCtx, + reconcilerBase: reconcilerBase{ + enqueueRollout: func(obj any) { t.Error("enqueueRollout should not be called") }, + enqueueRolloutAfter: func(obj any, duration time.Duration) { t.Error("enqueueRolloutAfter should not be called") }, + recorder: record.NewFakeEventRecorder(), + }, + pauseContext: &pauseContext{ + rollout: r, + log: logCtx, + }, + stepPluginContext: &stepPluginContext{ + resolver: stepPluginResolver, + log: logCtx, + }, + } + + return roCtx, stepPluginResolver + } + t.Run("Abort called on each plugin step", func(t *testing.T) { + roCtx, stepPluginResolver := setup(t) + roCtx.rollout.Status.Canary.StepPluginStatuses = []v1alpha1.StepPluginStatus{ + { + Index: 0, + Name: "test-plugin", + Operation: v1alpha1.StepPluginOperationRun, + Phase: v1alpha1.StepPluginPhaseSuccessful, + }, + { + Index: 2, + Name: "test-plugin", + Operation: v1alpha1.StepPluginOperationRun, + Phase: v1alpha1.StepPluginPhaseRunning, + }, + } + roCtx.rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ + { + Plugin: &v1alpha1.PluginStep{ + Name: "test-plugin", + }, + }, + { + Pause: &v1alpha1.RolloutPause{}, // Not a step plugin + }, + { + Plugin: &v1alpha1.PluginStep{ + Name: "test-plugin", + }, + }, + } + roCtx.rollout.Status.CurrentStepIndex = int32Ptr(0) + + expectedAbortStatus := []*v1alpha1.StepPluginStatus{} + for _, stepIndex := range []int32{0, 2} { + abortStatus := &v1alpha1.StepPluginStatus{ + Index: stepIndex, + Name: "test-plugin", + Operation: v1alpha1.StepPluginOperationAbort, + Phase: v1alpha1.StepPluginPhaseSuccessful, + } + stepPluginMock := mocks.NewStepPlugin(t) + stepPluginResolver.On("Resolve", stepIndex, mock.Anything, mock.Anything).Return(stepPluginMock, nil) + stepPluginMock.On("Abort", mock.Anything).Return(abortStatus, nil) + expectedAbortStatus = append(expectedAbortStatus, abortStatus) + } + + err := roCtx.stepPluginContext.reconcile(roCtx) + + require.NoError(t, err) + require.Len(t, roCtx.stepPluginContext.stepPluginStatuses, 4) + assert.EqualExportedValues(t, roCtx.rollout.Status.Canary.StepPluginStatuses[0], roCtx.stepPluginContext.stepPluginStatuses[0]) + assert.EqualExportedValues(t, roCtx.rollout.Status.Canary.StepPluginStatuses[1], roCtx.stepPluginContext.stepPluginStatuses[1]) + assert.EqualExportedValues(t, *expectedAbortStatus[1], roCtx.stepPluginContext.stepPluginStatuses[2]) + assert.EqualExportedValues(t, *expectedAbortStatus[0], roCtx.stepPluginContext.stepPluginStatuses[3]) + }) + t.Run("Rollout is reconciled when already aborted", func(t *testing.T) { + roCtx, stepPluginResolver := setup(t) + roCtx.rollout.Status.Canary.StepPluginStatuses = []v1alpha1.StepPluginStatus{ + { + Index: 0, + Name: "test-plugin", + Operation: v1alpha1.StepPluginOperationRun, + Phase: v1alpha1.StepPluginPhaseSuccessful, + }, + { + Index: 2, + Name: "test-plugin", + Operation: v1alpha1.StepPluginOperationRun, + Phase: v1alpha1.StepPluginPhaseRunning, + }, + { + Index: 2, + Name: "test-plugin", + Operation: v1alpha1.StepPluginOperationAbort, + Phase: v1alpha1.StepPluginPhaseSuccessful, + }, + { + Index: 0, + Name: "test-plugin", + Operation: v1alpha1.StepPluginOperationAbort, + Phase: v1alpha1.StepPluginPhaseSuccessful, + }, + } + roCtx.rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ + { + Plugin: &v1alpha1.PluginStep{ + Name: "test-plugin", + }, + }, + { + Pause: &v1alpha1.RolloutPause{}, // Not a step plugin + }, + { + Plugin: &v1alpha1.PluginStep{ + Name: "test-plugin", + }, + }, + } + roCtx.rollout.Status.CurrentStepIndex = int32Ptr(0) + + for _, stepIndex := range []int32{0, 2} { + stepPluginMock := mocks.NewStepPlugin(t) + stepPluginResolver.On("Resolve", stepIndex, mock.Anything, mock.Anything).Return(stepPluginMock, nil) + stepPluginMock.On("Abort", mock.Anything).Return(nil, nil) + } + + err := roCtx.stepPluginContext.reconcile(roCtx) + + require.NoError(t, err) + require.Len(t, roCtx.stepPluginContext.stepPluginStatuses, 4) + assert.Equal(t, roCtx.rollout.Status.Canary.StepPluginStatuses, roCtx.stepPluginContext.stepPluginStatuses) + }) + t.Run("Reconciliation error", func(t *testing.T) { + roCtx, stepPluginResolver := setup(t) + roCtx.rollout.Status.Canary.StepPluginStatuses = []v1alpha1.StepPluginStatus{ + { + Index: 0, + Name: "test-plugin", + Operation: v1alpha1.StepPluginOperationRun, + Phase: v1alpha1.StepPluginPhaseSuccessful, + }, + } + roCtx.rollout.Status.CurrentStepIndex = int32Ptr(0) + + runStatus := newStepPluginStatus(v1alpha1.StepPluginOperationTerminate, v1alpha1.StepPluginPhaseSuccessful) + roCtx.rollout.Status.Canary.StepPluginStatuses = []v1alpha1.StepPluginStatus{ + { + Index: runStatus.Index, + Name: runStatus.Name, + Phase: v1alpha1.StepPluginPhaseRunning, + Operation: v1alpha1.StepPluginOperationRun, + }, + } + + var requeuedAfter time.Duration + roCtx.enqueueRolloutAfter = func(obj any, duration time.Duration) { + requeuedAfter = duration + } + + stepPluginMock := mocks.NewStepPlugin(t) + stepPluginResolver.On("Resolve", int32(0), mock.Anything, mock.Anything).Return(stepPluginMock, nil) + stepPluginMock.On("Abort", mock.Anything).Return(nil, fmt.Errorf("error")) + + err := roCtx.stepPluginContext.reconcile(roCtx) + + require.NoError(t, err) + require.Len(t, roCtx.stepPluginContext.stepPluginStatuses, 1) + assert.Equal(t, roCtx.rollout.Status.Canary.StepPluginStatuses, roCtx.stepPluginContext.stepPluginStatuses) + assert.Equal(t, defaultControllerErrorBackoff, requeuedAfter) + assert.True(t, roCtx.stepPluginContext.hasError) + }) +} + +func Test_stepPluginContext_reconcile_Retry_After_Abort(t *testing.T) { + setup := func(t *testing.T) (*rolloutContext, *mocks.Resolver) { + stepPluginResolver := mocks.NewResolver(t) + + r := newStepPluginRollout() + r.Status.Abort = true + + logCtx := logutil.WithRollout(r) + roCtx := &rolloutContext{ + rollout: r, + log: logCtx, + reconcilerBase: reconcilerBase{ + enqueueRollout: func(obj any) { t.Error("enqueueRollout should not be called") }, + enqueueRolloutAfter: func(obj any, duration time.Duration) { t.Error("enqueueRolloutAfter should not be called") }, + recorder: record.NewFakeEventRecorder(), + }, + pauseContext: &pauseContext{ + rollout: r, + log: logCtx, + }, + stepPluginContext: &stepPluginContext{ + resolver: stepPluginResolver, + log: logCtx, + }, + } + + return roCtx, stepPluginResolver + } + + t.Run("Rollout is retried when already aborted", func(t *testing.T) { + roCtx, stepPluginResolver := setup(t) + roCtx.rollout.Status.Canary.StepPluginStatuses = []v1alpha1.StepPluginStatus{ + { + Index: 0, + Name: "test-plugin", + Operation: v1alpha1.StepPluginOperationRun, + Phase: v1alpha1.StepPluginPhaseSuccessful, + }, + { + Index: 2, + Name: "test-plugin", + Operation: v1alpha1.StepPluginOperationRun, + Phase: v1alpha1.StepPluginPhaseRunning, + }, + { + Index: 2, + Name: "test-plugin", + Operation: v1alpha1.StepPluginOperationAbort, + Phase: v1alpha1.StepPluginPhaseSuccessful, + }, + { + Index: 0, + Name: "test-plugin", + Operation: v1alpha1.StepPluginOperationAbort, + Phase: v1alpha1.StepPluginPhaseSuccessful, + }, + } + roCtx.rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ + { + Plugin: &v1alpha1.PluginStep{ + Name: "test-plugin", + }, + }, + { + Pause: &v1alpha1.RolloutPause{}, + }, + { + Plugin: &v1alpha1.PluginStep{ + Name: "test-plugin", + }, + }, + } + roCtx.rollout.Status.CurrentStepIndex = int32Ptr(0) + roCtx.rollout.Status.Abort = false + roCtx.rollout.Status.AbortedAt = nil + + stepPluginMock := mocks.NewStepPlugin(t) + stepPluginResolver.On("Resolve", int32(0), mock.Anything, mock.Anything).Return(stepPluginMock, nil) + runStatus := newStepPluginStatus(v1alpha1.StepPluginOperationRun, v1alpha1.StepPluginPhaseSuccessful) + stepPluginMock.On("Run", roCtx.rollout).Return(runStatus, nil) + + err := roCtx.stepPluginContext.reconcile(roCtx) + + require.NoError(t, err) + require.Len(t, roCtx.stepPluginContext.stepPluginStatuses, 1) + assert.Equal(t, *runStatus, roCtx.stepPluginContext.stepPluginStatuses[0]) + }) +} + +func Test_stepPluginContext_isStepPluginCompleted(t *testing.T) { + newRolloutContext := func(statuses []*v1alpha1.StepPluginStatus, hasError bool) *rolloutContext { + r := newStepPluginRollout() + logCtx := logutil.WithRollout(r) + roCtx := &rolloutContext{ + rollout: r, + log: logCtx, + stepPluginContext: &stepPluginContext{ + log: logCtx, + }, + } + + for _, s := range statuses { + roCtx.stepPluginContext.stepPluginStatuses = append(roCtx.stepPluginContext.stepPluginStatuses, *s) + } + roCtx.stepPluginContext.hasError = hasError + return roCtx + } + + tests := []struct { + name string + statuses []*v1alpha1.StepPluginStatus + index int32 + hasError bool + want bool + }{ + { + name: "Status is not set", + statuses: nil, + index: 0, + want: false, + }, + { + name: "Phase is successful", + statuses: []*v1alpha1.StepPluginStatus{ + {Index: 0, Operation: v1alpha1.StepPluginOperationRun, Phase: v1alpha1.StepPluginPhaseSuccessful}, + }, + index: 0, + want: true, + }, + { + name: "With transient error", + statuses: []*v1alpha1.StepPluginStatus{ + {Index: 0, Operation: v1alpha1.StepPluginOperationRun, Phase: v1alpha1.StepPluginPhaseSuccessful}, + }, + index: 0, + hasError: true, + want: false, + }, + { + name: "Phase is failed", + statuses: []*v1alpha1.StepPluginStatus{ + {Index: 0, Operation: v1alpha1.StepPluginOperationRun, Phase: v1alpha1.StepPluginPhaseFailed}, + }, + index: 0, + want: true, + }, + { + name: "Phase is error", + statuses: []*v1alpha1.StepPluginStatus{ + {Index: 0, Operation: v1alpha1.StepPluginOperationRun, Phase: v1alpha1.StepPluginPhaseError}, + }, + index: 0, + want: false, + }, + { + name: "Phase is running", + statuses: []*v1alpha1.StepPluginStatus{ + {Index: 0, Operation: v1alpha1.StepPluginOperationRun, Phase: v1alpha1.StepPluginPhaseRunning}, + }, + index: 0, + want: false, + }, + { + name: "Phase is running, but terminated", + statuses: []*v1alpha1.StepPluginStatus{ + {Index: 0, Operation: v1alpha1.StepPluginOperationRun, Phase: v1alpha1.StepPluginPhaseRunning}, + {Index: 0, Operation: v1alpha1.StepPluginOperationTerminate}, + }, + index: 0, + want: true, + }, + { + name: "Phase is running, but aborted", + statuses: []*v1alpha1.StepPluginStatus{ + {Index: 0, Operation: v1alpha1.StepPluginOperationRun, Phase: v1alpha1.StepPluginPhaseRunning}, + {Index: 0, Operation: v1alpha1.StepPluginOperationTerminate}, + }, + index: 0, + want: true, + }, + { + name: "status for index is missing", + statuses: []*v1alpha1.StepPluginStatus{ + {Index: 1, Operation: v1alpha1.StepPluginOperationRun, Phase: v1alpha1.StepPluginPhaseSuccessful}, + }, + index: 0, + want: false, + }, + { + name: "status is disabled", + statuses: []*v1alpha1.StepPluginStatus{ + {Index: 0, Operation: v1alpha1.StepPluginOperationRun, Disabled: true}, + }, + index: 0, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := newRolloutContext(tt.statuses, tt.hasError) + if got := c.stepPluginContext.isStepPluginCompleted(tt.index, c.rollout.Spec.Strategy.Canary.Steps[tt.index].Plugin); got != tt.want { + t.Errorf("rolloutContext.isStepPluginCompleted() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_stepPluginContext_updateStatus(t *testing.T) { + + t.Run("status is not", func(t *testing.T) { + stepPluginContext := stepPluginContext{ + stepPluginStatuses: nil, + } + + expected := []v1alpha1.StepPluginStatus{} + status := &v1alpha1.RolloutStatus{ + Canary: v1alpha1.CanaryStatus{ + StepPluginStatuses: expected, + }, + } + + stepPluginContext.updateStatus(status) + + assert.Equal(t, expected, status.Canary.StepPluginStatuses) + }) + t.Run("status is set", func(t *testing.T) { + + expected := []v1alpha1.StepPluginStatus{{}} + + stepPluginContext := stepPluginContext{ + stepPluginStatuses: expected, + } + + status := &v1alpha1.RolloutStatus{ + Canary: v1alpha1.CanaryStatus{ + StepPluginStatuses: []v1alpha1.StepPluginStatus{}, + }, + } + + stepPluginContext.updateStatus(status) + + assert.Equal(t, expected, status.Canary.StepPluginStatuses) + }) +} diff --git a/rollout/steps/plugin/client/client.go b/rollout/steps/plugin/client/client.go new file mode 100644 index 0000000000..51de61d227 --- /dev/null +++ b/rollout/steps/plugin/client/client.go @@ -0,0 +1,106 @@ +package client + +import ( + "fmt" + "os/exec" + "sync" + + goPlugin "github.com/hashicorp/go-plugin" + + "github.com/argoproj/argo-rollouts/rollout/steps/plugin/rpc" + "github.com/argoproj/argo-rollouts/utils/plugin" + "github.com/argoproj/argo-rollouts/utils/plugin/types" +) + +type stepPlugin struct { + client map[string]*goPlugin.Client + plugin map[string]rpc.StepPlugin +} + +var pluginClients *stepPlugin +var once sync.Once +var mutex sync.Mutex + +var handshakeConfig = goPlugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "ARGO_ROLLOUTS_RPC_PLUGIN", + MagicCookieValue: "step", +} + +// pluginMap is the map of plugins we can dispense. +var pluginMap = map[string]goPlugin.Plugin{ + "RpcStepPlugin": &rpc.RpcStepPlugin{}, +} + +// GetPlugin returns a singleton plugin client for the given plugin. Calling this multiple times +// returns the same plugin client instance for the plugin name defined in the rollout object. +func GetPlugin(pluginName string) (rpc.StepPlugin, error) { + once.Do(func() { + pluginClients = &stepPlugin{ + client: make(map[string]*goPlugin.Client), + plugin: make(map[string]rpc.StepPlugin), + } + }) + plugin, err := pluginClients.startPlugin(pluginName) + if err != nil { + return nil, fmt.Errorf("unable to start plugin system: %w", err) + } + return plugin, nil +} + +func (t *stepPlugin) startPlugin(pluginName string) (rpc.StepPlugin, error) { + mutex.Lock() + defer mutex.Unlock() + + if t.client[pluginName] == nil || t.client[pluginName].Exited() { + + pluginPath, args, err := plugin.GetPluginInfo(pluginName, types.PluginTypeStep) + if err != nil { + return nil, fmt.Errorf("unable to find plugin (%s): %w", pluginName, err) + } + + t.client[pluginName] = goPlugin.NewClient(&goPlugin.ClientConfig{ + HandshakeConfig: handshakeConfig, + Plugins: pluginMap, + Cmd: exec.Command(pluginPath, args...), + Managed: true, + }) + + rpcClient, err := t.client[pluginName].Client() + if err != nil { + return nil, fmt.Errorf("unable to get plugin client (%s): %w", pluginName, err) + } + + // Request the plugin + plugin, err := rpcClient.Dispense("RpcStepPlugin") + if err != nil { + return nil, fmt.Errorf("unable to dispense plugin (%s): %w", pluginName, err) + } + + pluginType, ok := plugin.(rpc.StepPlugin) + if !ok { + return nil, fmt.Errorf("unexpected type from plugin") + } + t.plugin[pluginName] = pluginType + + resp := t.plugin[pluginName].InitPlugin() + if resp.HasError() { + return nil, fmt.Errorf("unable to initialize plugin via rpc (%s): %w", pluginName, resp) + } + } + + client, err := t.client[pluginName].Client() + if err != nil { + // If we are not able to create the client, something is utterly wrong + // we should try to re-download the plugin and restart because the file + // can be corrupted + return nil, fmt.Errorf("unable to get plugin client (%s) for ping: %w", pluginName, err) + } + if err := client.Ping(); err != nil { + t.client[pluginName].Kill() + t.client[pluginName] = nil + return nil, fmt.Errorf("could not ping plugin will cleanup process so we can restart it next reconcile (%w)", err) + } + + return t.plugin[pluginName], nil +} diff --git a/rollout/steps/plugin/mocks/Resolver.go b/rollout/steps/plugin/mocks/Resolver.go new file mode 100644 index 0000000000..356b69e0b5 --- /dev/null +++ b/rollout/steps/plugin/mocks/Resolver.go @@ -0,0 +1,61 @@ +// Code generated by mockery v2.42.2. DO NOT EDIT. + +package mocks + +import ( + logrus "github.com/sirupsen/logrus" + mock "github.com/stretchr/testify/mock" + + plugin "github.com/argoproj/argo-rollouts/rollout/steps/plugin" + + v1alpha1 "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" +) + +// Resolver is an autogenerated mock type for the Resolver type +type Resolver struct { + mock.Mock +} + +// Resolve provides a mock function with given fields: index, _a1, log +func (_m *Resolver) Resolve(index int32, _a1 v1alpha1.PluginStep, log *logrus.Entry) (plugin.StepPlugin, error) { + ret := _m.Called(index, _a1, log) + + if len(ret) == 0 { + panic("no return value specified for Resolve") + } + + var r0 plugin.StepPlugin + var r1 error + if rf, ok := ret.Get(0).(func(int32, v1alpha1.PluginStep, *logrus.Entry) (plugin.StepPlugin, error)); ok { + return rf(index, _a1, log) + } + if rf, ok := ret.Get(0).(func(int32, v1alpha1.PluginStep, *logrus.Entry) plugin.StepPlugin); ok { + r0 = rf(index, _a1, log) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(plugin.StepPlugin) + } + } + + if rf, ok := ret.Get(1).(func(int32, v1alpha1.PluginStep, *logrus.Entry) error); ok { + r1 = rf(index, _a1, log) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewResolver creates a new instance of Resolver. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewResolver(t interface { + mock.TestingT + Cleanup(func()) +}) *Resolver { + mock := &Resolver{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/rollout/steps/plugin/mocks/StepPlugin.go b/rollout/steps/plugin/mocks/StepPlugin.go new file mode 100644 index 0000000000..2a5b7cb679 --- /dev/null +++ b/rollout/steps/plugin/mocks/StepPlugin.go @@ -0,0 +1,118 @@ +// Code generated by mockery v2.42.2. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" + + v1alpha1 "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" +) + +// StepPlugin is an autogenerated mock type for the StepPlugin type +type StepPlugin struct { + mock.Mock +} + +// Abort provides a mock function with given fields: _a0 +func (_m *StepPlugin) Abort(_a0 *v1alpha1.Rollout) (*v1alpha1.StepPluginStatus, error) { + ret := _m.Called(_a0) + + if len(ret) == 0 { + panic("no return value specified for Abort") + } + + var r0 *v1alpha1.StepPluginStatus + var r1 error + if rf, ok := ret.Get(0).(func(*v1alpha1.Rollout) (*v1alpha1.StepPluginStatus, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(*v1alpha1.Rollout) *v1alpha1.StepPluginStatus); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1alpha1.StepPluginStatus) + } + } + + if rf, ok := ret.Get(1).(func(*v1alpha1.Rollout) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Run provides a mock function with given fields: _a0 +func (_m *StepPlugin) Run(_a0 *v1alpha1.Rollout) (*v1alpha1.StepPluginStatus, error) { + ret := _m.Called(_a0) + + if len(ret) == 0 { + panic("no return value specified for Run") + } + + var r0 *v1alpha1.StepPluginStatus + var r1 error + if rf, ok := ret.Get(0).(func(*v1alpha1.Rollout) (*v1alpha1.StepPluginStatus, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(*v1alpha1.Rollout) *v1alpha1.StepPluginStatus); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1alpha1.StepPluginStatus) + } + } + + if rf, ok := ret.Get(1).(func(*v1alpha1.Rollout) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Terminate provides a mock function with given fields: _a0 +func (_m *StepPlugin) Terminate(_a0 *v1alpha1.Rollout) (*v1alpha1.StepPluginStatus, error) { + ret := _m.Called(_a0) + + if len(ret) == 0 { + panic("no return value specified for Terminate") + } + + var r0 *v1alpha1.StepPluginStatus + var r1 error + if rf, ok := ret.Get(0).(func(*v1alpha1.Rollout) (*v1alpha1.StepPluginStatus, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(*v1alpha1.Rollout) *v1alpha1.StepPluginStatus); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v1alpha1.StepPluginStatus) + } + } + + if rf, ok := ret.Get(1).(func(*v1alpha1.Rollout) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewStepPlugin creates a new instance of StepPlugin. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewStepPlugin(t interface { + mock.TestingT + Cleanup(func()) +}) *StepPlugin { + mock := &StepPlugin{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/rollout/steps/plugin/plugin.go b/rollout/steps/plugin/plugin.go new file mode 100644 index 0000000000..083d398318 --- /dev/null +++ b/rollout/steps/plugin/plugin.go @@ -0,0 +1,256 @@ +package plugin + +import ( + "encoding/json" + "fmt" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/rollout/steps/plugin/rpc" + "github.com/argoproj/argo-rollouts/utils/plugin/types" + metatime "github.com/argoproj/argo-rollouts/utils/time" +) + +type stepPlugin struct { + rpc rpc.StepPlugin + index int32 + name string + config json.RawMessage + log *log.Entry +} + +type disabledStepPlugin struct { + index int32 + name string +} + +// StepPlugin allows to execute different operation for a step plugin specifc to a rollout +type StepPlugin interface { + // Run executes the run operation for of a step plugin and returns a Run operation status + Run(*v1alpha1.Rollout) (*v1alpha1.StepPluginStatus, error) + // Terminate cancels an ongoing running Run operation and returns a Terminate status if it did + Terminate(*v1alpha1.Rollout) (*v1alpha1.StepPluginStatus, error) + // Abort reverts a completed Run operation and returns a Terminate status if it did + Abort(*v1alpha1.Rollout) (*v1alpha1.StepPluginStatus, error) +} + +var ( + minRequeueDuration = time.Second * 10 + defaultRequeuDuration = time.Second * 30 + defaultErrorBackoff = time.Second * 30 +) + +func (p *stepPlugin) Run(rollout *v1alpha1.Rollout) (*v1alpha1.StepPluginStatus, error) { + stepStatus := p.getStepStatus(rollout, v1alpha1.StepPluginOperationRun) + if stepStatus == nil || stepStatus.Disabled { + now := metatime.MetaNow() + stepStatus = &v1alpha1.StepPluginStatus{ + Index: p.index, + Name: p.name, + StartedAt: &now, + Operation: v1alpha1.StepPluginOperationRun, + Phase: v1alpha1.StepPluginPhaseRunning, + } + } + + if stepStatus.Phase == v1alpha1.StepPluginPhaseSuccessful || stepStatus.Phase == v1alpha1.StepPluginPhaseFailed { + // Already completed + return nil, nil + } + + if stepStatus.Executions > 0 { + // If status existed, check the backoff to know if we are ready to retry. + // If we are not, return the status without modifying it. + backoff, err := stepStatus.Backoff.Duration() + if err != nil { + return nil, fmt.Errorf("could not parse backoff duration: %w", err) + } + if stepStatus.UpdatedAt.Add(backoff).After(metatime.Now()) { + p.log.Debug("skipping plugin Run due to backoff") + return stepStatus, nil + } + } + + p.log.Debug("calling RPC Run") + resp, err := p.rpc.Run(rollout.DeepCopy(), p.getStepContext(stepStatus)) + finishedAt := metatime.MetaNow() + stepStatus.Backoff = "" + stepStatus.UpdatedAt = &finishedAt + stepStatus.Executions++ + if err.HasError() { + p.log.Errorf("error during plugin execution") + stepStatus.Phase = v1alpha1.StepPluginPhaseError + stepStatus.Message = err.Error() + stepStatus.Backoff = v1alpha1.DurationString(defaultErrorBackoff.String()) + return stepStatus, nil + } + + stepStatus.Message = resp.Message + if resp.Phase != "" { + stepStatus.Phase = v1alpha1.StepPluginPhase(resp.Phase) + if err := stepStatus.Phase.Validate(); err != nil { + return nil, fmt.Errorf("could not validate rpc phase: %w", err) + } + } + + if stepStatus.Phase == v1alpha1.StepPluginPhaseSuccessful || stepStatus.Phase == v1alpha1.StepPluginPhaseFailed { + stepStatus.FinishedAt = &finishedAt + } + + if stepStatus.Phase != v1alpha1.StepPluginPhaseError { + // do not update status on error because it can be invalid and we want to retry later on current status + stepStatus.Status = resp.Status + } + + if stepStatus.Phase == v1alpha1.StepPluginPhaseRunning { + backoff := defaultRequeuDuration + if resp.RequeueAfter > minRequeueDuration { + backoff = resp.RequeueAfter + } + stepStatus.Backoff = v1alpha1.DurationString(backoff.String()) + } + + return stepStatus, nil +} + +func (p *stepPlugin) Terminate(rollout *v1alpha1.Rollout) (*v1alpha1.StepPluginStatus, error) { + terminateStatus := p.getStepStatus(rollout, v1alpha1.StepPluginOperationTerminate) + if terminateStatus != nil { + // Already terminated + return nil, nil + } + + stepStatus := p.getStepStatus(rollout, v1alpha1.StepPluginOperationRun) + if stepStatus == nil || stepStatus.Disabled || stepStatus.Phase != v1alpha1.StepPluginPhaseRunning { + // Step is not running, no need to call terminate + return nil, nil + } + + now := metatime.MetaNow() + terminateStatus = &v1alpha1.StepPluginStatus{ + Index: stepStatus.Index, + Name: stepStatus.Name, + StartedAt: &now, + Operation: v1alpha1.StepPluginOperationTerminate, + Phase: v1alpha1.StepPluginPhaseSuccessful, + } + + p.log.Debug("calling RPC Terminate") + resp, err := p.rpc.Terminate(rollout.DeepCopy(), p.getStepContext(stepStatus)) + finishedAt := metatime.MetaNow() + stepStatus.UpdatedAt = &finishedAt + if err.HasError() { + terminateStatus.Phase = v1alpha1.StepPluginPhaseError + terminateStatus.Message = err.Error() + terminateStatus.FinishedAt = &finishedAt + return terminateStatus, nil + } + + if resp.Phase != "" { + terminateStatus.Phase = v1alpha1.StepPluginPhase(resp.Phase) + if err := terminateStatus.Phase.Validate(); err != nil { + return nil, fmt.Errorf("could not validate rpc phase: %w", err) + } + } + + if terminateStatus.Phase == v1alpha1.StepPluginPhaseRunning { + p.log.Warnf("terminate cannot run asynchronously. Overriding status phase to %s.", v1alpha1.StepPluginPhaseFailed) + terminateStatus.Phase = v1alpha1.StepPluginPhaseFailed + } + + terminateStatus.Message = resp.Message + terminateStatus.FinishedAt = &finishedAt + return terminateStatus, nil +} + +func (p *stepPlugin) Abort(rollout *v1alpha1.Rollout) (*v1alpha1.StepPluginStatus, error) { + abortStatus := p.getStepStatus(rollout, v1alpha1.StepPluginOperationAbort) + if abortStatus != nil { + // Already aborted + return nil, nil + } + + stepStatus := p.getStepStatus(rollout, v1alpha1.StepPluginOperationRun) + if stepStatus == nil || stepStatus.Disabled || (stepStatus.Phase != v1alpha1.StepPluginPhaseRunning && stepStatus.Phase != v1alpha1.StepPluginPhaseSuccessful) { + // Step plugin isn't in a phase where it needs to be aborted + return nil, nil + } + + now := metatime.MetaNow() + abortStatus = &v1alpha1.StepPluginStatus{ + Index: stepStatus.Index, + Name: stepStatus.Name, + StartedAt: &now, + Operation: v1alpha1.StepPluginOperationAbort, + Phase: v1alpha1.StepPluginPhaseSuccessful, + } + + p.log.Debug("calling RPC Abort") + resp, err := p.rpc.Abort(rollout.DeepCopy(), p.getStepContext(stepStatus)) + finishedAt := metatime.MetaNow() + stepStatus.UpdatedAt = &finishedAt + if err.HasError() { + abortStatus.Phase = v1alpha1.StepPluginPhaseError + abortStatus.Message = err.Error() + abortStatus.FinishedAt = &finishedAt + return abortStatus, nil + } + + if resp.Phase != "" { + abortStatus.Phase = v1alpha1.StepPluginPhase(resp.Phase) + if err := abortStatus.Phase.Validate(); err != nil { + return nil, fmt.Errorf("could not validate rpc phase: %w", err) + } + } + + if abortStatus.Phase == v1alpha1.StepPluginPhaseRunning { + p.log.Warnf("abort cannot run asynchronously. Overriding status phase to %s.", v1alpha1.StepPluginPhaseFailed) + abortStatus.Phase = v1alpha1.StepPluginPhaseFailed + } + + abortStatus.Message = resp.Message + abortStatus.FinishedAt = &finishedAt + return abortStatus, nil +} + +func (p *disabledStepPlugin) Run(_ *v1alpha1.Rollout) (*v1alpha1.StepPluginStatus, error) { + return &v1alpha1.StepPluginStatus{ + Index: p.index, + Name: p.name, + Operation: v1alpha1.StepPluginOperationRun, + Disabled: true, + }, nil +} + +func (p *disabledStepPlugin) Terminate(_ *v1alpha1.Rollout) (*v1alpha1.StepPluginStatus, error) { + return nil, nil +} + +func (p *disabledStepPlugin) Abort(_ *v1alpha1.Rollout) (*v1alpha1.StepPluginStatus, error) { + return nil, nil +} + +// getStepStatus returns the existing status for the current operation +func (p *stepPlugin) getStepStatus(rollout *v1alpha1.Rollout, operation v1alpha1.StepPluginOperation) *v1alpha1.StepPluginStatus { + for _, s := range rollout.Status.Canary.StepPluginStatuses { + if s.Index == p.index && s.Name == p.name && s.Operation == operation { + return s.DeepCopy() + } + } + return nil +} + +// getStepContext returns the current step configuration with the from a previous operation, if any +func (p *stepPlugin) getStepContext(stepStatus *v1alpha1.StepPluginStatus) *types.RpcStepContext { + var status json.RawMessage = nil + if stepStatus != nil { + status = stepStatus.Status + } + return &types.RpcStepContext{ + PluginName: p.name, + Config: p.config, + Status: status, + } +} diff --git a/rollout/steps/plugin/plugin_test.go b/rollout/steps/plugin/plugin_test.go new file mode 100644 index 0000000000..68ca508882 --- /dev/null +++ b/rollout/steps/plugin/plugin_test.go @@ -0,0 +1,770 @@ +package plugin + +import ( + "encoding/json" + "testing" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/rollout/steps/plugin/rpc/mocks" + "github.com/argoproj/argo-rollouts/utils/plugin/types" +) + +func Test_stepPlugin_Run(t *testing.T) { + setup := func(t *testing.T) (*stepPlugin, *mocks.StepPlugin) { + plugin := &stepPlugin{ + name: "test-plugin", + index: 0, + config: json.RawMessage("value"), + log: log.WithFields(log.Fields{}), + } + rpcPluginMock := mocks.NewStepPlugin(t) + plugin.rpc = rpcPluginMock + return plugin, rpcPluginMock + } + + t.Run("Argument set without status", func(t *testing.T) { + p, rpcMock := setup(t) + r := &v1alpha1.Rollout{ + Status: v1alpha1.RolloutStatus{ + Canary: v1alpha1.CanaryStatus{ + StepPluginStatuses: []v1alpha1.StepPluginStatus{ + { + Index: 0, + Name: p.name, + Status: json.RawMessage("step status value"), + Operation: v1alpha1.StepPluginOperationRun, + }, + }, + }, + }, + } + validateArguments := func(args mock.Arguments) { + rollout, ok0 := args.Get(0).(*v1alpha1.Rollout) + context, ok1 := args.Get(1).(*types.RpcStepContext) + require.Truef(t, ok0, "Argument 0 is of the wrong type") + require.Truef(t, ok1, "Argument 1 is of the wrong type") + + assert.NotNil(t, rollout) + assert.NotSame(t, r, rollout) + assert.NotNil(t, context) + assert.Equal(t, p.name, context.PluginName) + assert.Equal(t, p.config, context.Config) + assert.Nil(t, context.Status) + } + + rpcMock.On("Run", mock.Anything, mock.Anything).Run(validateArguments).Return(types.RpcStepResult{}, types.RpcError{}).Once() + + p.index = 1 + _, err := p.Run(r) + + require.NoError(t, err) + rpcMock.AssertExpectations(t) + + }) + t.Run("Argument set with status", func(t *testing.T) { + p, rpcMock := setup(t) + r := &v1alpha1.Rollout{ + Status: v1alpha1.RolloutStatus{ + Canary: v1alpha1.CanaryStatus{ + StepPluginStatuses: []v1alpha1.StepPluginStatus{ + { + Index: 0, + Name: p.name, + Status: json.RawMessage("step status value"), + Operation: v1alpha1.StepPluginOperationRun, + }, + }, + }, + }, + } + validateArguments := func(args mock.Arguments) { + rollout, ok0 := args.Get(0).(*v1alpha1.Rollout) + context, ok1 := args.Get(1).(*types.RpcStepContext) + require.Truef(t, ok0, "Argument 0 is of the wrong type") + require.Truef(t, ok1, "Argument 1 is of the wrong type") + + assert.NotNil(t, rollout) + assert.NotSame(t, r, rollout) + assert.NotNil(t, context) + assert.Equal(t, p.name, context.PluginName) + assert.Equal(t, p.config, context.Config) + assert.Equal(t, r.Status.Canary.StepPluginStatuses[0].Status, context.Status) + } + + rpcMock.On("Run", mock.Anything, mock.Anything).Run(validateArguments).Return(types.RpcStepResult{}, types.RpcError{}).Once() + + p.index = 0 + _, err := p.Run(r) + + require.NoError(t, err) + rpcMock.AssertExpectations(t) + + }) + t.Run("use existing state", func(t *testing.T) { + p, rpcMock := setup(t) + currentStatus := &v1alpha1.StepPluginStatus{ + Index: 0, + Name: p.name, + Status: json.RawMessage("step status value"), + StartedAt: &v1.Time{Time: time.Now().Add(30 * time.Minute * -1)}, + Phase: v1alpha1.StepPluginPhaseRunning, + Operation: v1alpha1.StepPluginOperationRun, + } + r := &v1alpha1.Rollout{ + Status: v1alpha1.RolloutStatus{ + Canary: v1alpha1.CanaryStatus{ + StepPluginStatuses: []v1alpha1.StepPluginStatus{ + *currentStatus, + }, + }, + }, + } + + rpcResult := types.RpcStepResult{ + Phase: types.PhaseSuccessful, + Message: "Good message", + RequeueAfter: time.Hour, + Status: json.RawMessage("status"), + } + rpcMock.On("Run", mock.Anything, mock.Anything).Return(rpcResult, types.RpcError{}).Once() + + status, err := p.Run(r) + + require.NoError(t, err) + rpcMock.AssertExpectations(t) + + assert.Equal(t, p.name, status.Name) + assert.Equal(t, p.index, status.Index) + assert.Equal(t, currentStatus.StartedAt, status.StartedAt) + assert.NotEqual(t, currentStatus.UpdatedAt, status.UpdatedAt) + assert.Equal(t, v1alpha1.StepPluginPhase(rpcResult.Phase), status.Phase) + assert.Equal(t, v1alpha1.StepPluginOperationRun, status.Operation) + assert.Equal(t, rpcResult.Message, status.Message) + assert.Equal(t, rpcResult.Status, status.Status) + assert.NotNil(t, status.FinishedAt) + assert.Empty(t, status.Backoff) + assert.False(t, status.Disabled) + }) + t.Run("Successful status", func(t *testing.T) { + p, rpcMock := setup(t) + r := &v1alpha1.Rollout{ + Status: v1alpha1.RolloutStatus{ + Canary: v1alpha1.CanaryStatus{ + StepPluginStatuses: []v1alpha1.StepPluginStatus{}, + }, + }, + } + + rpcResult := types.RpcStepResult{ + Phase: types.PhaseSuccessful, + Message: "Good message", + RequeueAfter: time.Hour, + Status: json.RawMessage("status"), + } + rpcMock.On("Run", mock.Anything, mock.Anything).Return(rpcResult, types.RpcError{}).Once() + + status, err := p.Run(r) + + require.NoError(t, err) + rpcMock.AssertExpectations(t) + + assert.Equal(t, p.name, status.Name) + assert.Equal(t, p.index, status.Index) + assert.NotNil(t, status.StartedAt) + assert.NotNil(t, status.FinishedAt) + assert.Greater(t, status.FinishedAt.Time, status.StartedAt.Time) + assert.Equal(t, v1alpha1.StepPluginPhase(rpcResult.Phase), status.Phase) + assert.Equal(t, v1alpha1.StepPluginOperationRun, status.Operation) + assert.Equal(t, rpcResult.Message, status.Message) + assert.Equal(t, rpcResult.Status, status.Status) + assert.EqualValues(t, "", status.Backoff) + assert.False(t, status.Disabled) + }) + t.Run("Running status", func(t *testing.T) { + p, rpcMock := setup(t) + currentStatus := &v1alpha1.StepPluginStatus{ + Index: 0, + Name: p.name, + Status: json.RawMessage("step status value"), + StartedAt: &v1.Time{Time: time.Now().Add(30 * time.Minute * -1)}, + Operation: v1alpha1.StepPluginOperationRun, + } + r := &v1alpha1.Rollout{ + Status: v1alpha1.RolloutStatus{ + Canary: v1alpha1.CanaryStatus{ + StepPluginStatuses: []v1alpha1.StepPluginStatus{ + *currentStatus, + }, + }, + }, + } + + rpcResult := types.RpcStepResult{ + Phase: types.PhaseRunning, + Message: "Good message", + RequeueAfter: time.Hour, + Status: json.RawMessage("status"), + } + rpcMock.On("Run", mock.Anything, mock.Anything).Return(rpcResult, types.RpcError{}).Once() + + status, err := p.Run(r) + + require.NoError(t, err) + rpcMock.AssertExpectations(t) + + assert.Equal(t, p.name, status.Name) + assert.Equal(t, p.index, status.Index) + assert.NotNil(t, status.StartedAt) + assert.Nil(t, status.FinishedAt) + assert.Equal(t, v1alpha1.StepPluginPhase(rpcResult.Phase), status.Phase) + assert.Equal(t, v1alpha1.StepPluginOperationRun, status.Operation) + assert.Equal(t, rpcResult.Message, status.Message) + assert.Equal(t, rpcResult.Status, status.Status) + assert.EqualValues(t, rpcResult.RequeueAfter.String(), status.Backoff) + assert.False(t, status.Disabled) + }) + t.Run("Running status without requeue", func(t *testing.T) { + p, rpcMock := setup(t) + r := &v1alpha1.Rollout{ + Status: v1alpha1.RolloutStatus{ + Canary: v1alpha1.CanaryStatus{ + StepPluginStatuses: []v1alpha1.StepPluginStatus{}, + }, + }, + } + + rpcResult := types.RpcStepResult{ + Phase: types.PhaseRunning, + Message: "Good message", + Status: json.RawMessage("status"), + } + rpcMock.On("Run", mock.Anything, mock.Anything).Return(rpcResult, types.RpcError{}).Once() + + status, err := p.Run(r) + + require.NoError(t, err) + rpcMock.AssertExpectations(t) + + assert.Equal(t, p.name, status.Name) + assert.Equal(t, p.index, status.Index) + assert.NotNil(t, status.StartedAt) + assert.Nil(t, status.FinishedAt) + assert.Equal(t, v1alpha1.StepPluginPhase(rpcResult.Phase), status.Phase) + assert.Equal(t, v1alpha1.StepPluginOperationRun, status.Operation) + assert.Equal(t, rpcResult.Message, status.Message) + assert.Equal(t, rpcResult.Status, status.Status) + assert.EqualValues(t, defaultRequeuDuration.String(), status.Backoff) + assert.False(t, status.Disabled) + }) + t.Run("Running status with requeue too fast", func(t *testing.T) { + p, rpcMock := setup(t) + r := &v1alpha1.Rollout{ + Status: v1alpha1.RolloutStatus{ + Canary: v1alpha1.CanaryStatus{ + StepPluginStatuses: []v1alpha1.StepPluginStatus{}, + }, + }, + } + + rpcResult := types.RpcStepResult{ + Phase: types.PhaseRunning, + Message: "Good message", + RequeueAfter: 1 * time.Second, + Status: json.RawMessage("status"), + } + rpcMock.On("Run", mock.Anything, mock.Anything).Return(rpcResult, types.RpcError{}).Once() + + status, err := p.Run(r) + + require.NoError(t, err) + rpcMock.AssertExpectations(t) + + assert.Equal(t, p.name, status.Name) + assert.Equal(t, p.index, status.Index) + assert.NotNil(t, status.StartedAt) + assert.Nil(t, status.FinishedAt) + assert.Equal(t, v1alpha1.StepPluginPhase(rpcResult.Phase), status.Phase) + assert.Equal(t, v1alpha1.StepPluginOperationRun, status.Operation) + assert.Equal(t, rpcResult.Message, status.Message) + assert.Equal(t, rpcResult.Status, status.Status) + assert.EqualValues(t, defaultRequeuDuration.String(), status.Backoff) + assert.False(t, status.Disabled) + }) + t.Run("Failed status", func(t *testing.T) { + p, rpcMock := setup(t) + r := &v1alpha1.Rollout{ + Status: v1alpha1.RolloutStatus{ + Canary: v1alpha1.CanaryStatus{ + StepPluginStatuses: []v1alpha1.StepPluginStatus{}, + }, + }, + } + + rpcResult := types.RpcStepResult{ + Phase: types.PhaseSuccessful, + Message: "Good message", + RequeueAfter: time.Hour, + Status: json.RawMessage("status"), + } + rpcMock.On("Run", mock.Anything, mock.Anything).Return(rpcResult, types.RpcError{}).Once() + + status, err := p.Run(r) + + require.NoError(t, err) + rpcMock.AssertExpectations(t) + + assert.Equal(t, p.name, status.Name) + assert.Equal(t, p.index, status.Index) + assert.NotNil(t, status.StartedAt) + assert.NotNil(t, status.FinishedAt) + assert.Greater(t, status.FinishedAt.Time, status.StartedAt.Time) + assert.Equal(t, v1alpha1.StepPluginPhase(rpcResult.Phase), status.Phase) + assert.Equal(t, v1alpha1.StepPluginOperationRun, status.Operation) + assert.Equal(t, rpcResult.Message, status.Message) + assert.Equal(t, rpcResult.Status, status.Status) + assert.Empty(t, status.Backoff) + assert.False(t, status.Disabled) + }) + t.Run("Error status", func(t *testing.T) { + p, rpcMock := setup(t) + currentStatus := &v1alpha1.StepPluginStatus{ + Index: 0, + Name: p.name, + Status: json.RawMessage("step status value"), + StartedAt: &v1.Time{Time: time.Now().Add(30 * time.Minute * -1)}, + Operation: v1alpha1.StepPluginOperationRun, + } + r := &v1alpha1.Rollout{ + Status: v1alpha1.RolloutStatus{ + Canary: v1alpha1.CanaryStatus{ + StepPluginStatuses: []v1alpha1.StepPluginStatus{ + *currentStatus, + }, + }, + }, + } + + invalidResult := types.RpcStepResult{ + Phase: types.PhaseSuccessful, + Message: "This message should not be used", + RequeueAfter: time.Hour, + Status: json.RawMessage("invalid status"), + } + expectedError := types.RpcError{ + ErrorString: "This is an error", + } + rpcMock.On("Run", mock.Anything, mock.Anything).Return(invalidResult, expectedError).Once() + + status, err := p.Run(r) + + require.NoError(t, err) + rpcMock.AssertExpectations(t) + + assert.Equal(t, p.name, status.Name) + assert.Equal(t, p.index, status.Index) + assert.Equal(t, currentStatus.StartedAt, status.StartedAt) + assert.NotEqual(t, currentStatus.UpdatedAt, status.UpdatedAt) + assert.Equal(t, v1alpha1.StepPluginPhaseError, status.Phase) + assert.Equal(t, v1alpha1.StepPluginOperationRun, status.Operation) + assert.Equal(t, expectedError.Error(), status.Message) + assert.Equal(t, currentStatus.Status, status.Status) + assert.EqualValues(t, "30s", status.Backoff) + assert.False(t, status.Disabled) + }) +} + +func Test_stepPlugin_Terminate(t *testing.T) { + setup := func(t *testing.T) (*stepPlugin, *mocks.StepPlugin) { + plugin := &stepPlugin{ + name: "test-plugin", + index: 0, + config: json.RawMessage("value"), + log: log.WithFields(log.Fields{}), + } + rpcPluginMock := mocks.NewStepPlugin(t) + plugin.rpc = rpcPluginMock + return plugin, rpcPluginMock + } + newRunningStatus := func() *v1alpha1.StepPluginStatus { + return &v1alpha1.StepPluginStatus{ + Index: 0, + Name: "test-plugin", + Status: json.RawMessage("step status value"), + StartedAt: &v1.Time{Time: time.Now().Add(30 * time.Minute * -1)}, + Phase: v1alpha1.StepPluginPhaseRunning, + Operation: v1alpha1.StepPluginOperationRun, + } + } + newRollout := func(s *v1alpha1.StepPluginStatus) *v1alpha1.Rollout { + return &v1alpha1.Rollout{ + Status: v1alpha1.RolloutStatus{ + Canary: v1alpha1.CanaryStatus{ + StepPluginStatuses: []v1alpha1.StepPluginStatus{ + *s, + }, + }, + }, + } + } + + t.Run("Return nil status if not running", func(t *testing.T) { + p, rpcMock := setup(t) + currentStatus := newRunningStatus() + currentStatus.Phase = v1alpha1.StepPluginPhaseSuccessful + currentStatus.Operation = v1alpha1.StepPluginOperationRun + r := newRollout(currentStatus) + + rpcMock.On("Terminate", mock.Anything, mock.Anything).Maybe().Panic("Terminate should not be called when plugin is not running") + + status, err := p.Terminate(r) + + require.NoError(t, err) + assert.Nil(t, status) + }) + + t.Run("Return nil status if already terminated", func(t *testing.T) { + p, rpcMock := setup(t) + currentStatus := newRunningStatus() + currentStatus.Phase = v1alpha1.StepPluginPhaseSuccessful + currentStatus.Operation = v1alpha1.StepPluginOperationTerminate + r := newRollout(currentStatus) + + rpcMock.On("Terminate", mock.Anything, mock.Anything).Maybe().Panic("Terminate should not be called when plugin is not running") + + status, err := p.Terminate(r) + + require.NoError(t, err) + assert.Nil(t, status) + }) + t.Run("Running phase overridden to failed if running", func(t *testing.T) { + p, rpcMock := setup(t) + currentStatus := newRunningStatus() + r := newRollout(currentStatus) + + rpcResult := types.RpcStepResult{ + Phase: types.PhaseRunning, + Message: "Good message", + RequeueAfter: time.Hour, + Status: json.RawMessage("status"), + } + rpcMock.On("Terminate", mock.Anything, mock.Anything).Return(rpcResult, types.RpcError{}) + + status, err := p.Terminate(r) + + require.NoError(t, err) + assert.Equal(t, p.name, status.Name) + assert.Equal(t, p.index, status.Index) + assert.NotNil(t, status.FinishedAt) + assert.Greater(t, status.FinishedAt.Time, status.StartedAt.Time) + assert.Equal(t, v1alpha1.StepPluginOperationTerminate, status.Operation) + assert.Equal(t, v1alpha1.StepPluginPhaseFailed, status.Phase) + assert.Equal(t, rpcResult.Message, status.Message) + assert.Nil(t, status.Status) + assert.False(t, status.Disabled) + }) + t.Run("Completes successfully", func(t *testing.T) { + p, rpcMock := setup(t) + currentStatus := newRunningStatus() + r := newRollout(currentStatus) + + rpcResult := types.RpcStepResult{ + Phase: types.PhaseSuccessful, + Message: "Good message", + RequeueAfter: time.Hour, + Status: json.RawMessage("status"), + } + rpcMock.On("Terminate", mock.Anything, mock.Anything).Return(rpcResult, types.RpcError{}) + + status, err := p.Terminate(r) + + require.NoError(t, err) + + assert.Equal(t, p.name, status.Name) + assert.Equal(t, p.index, status.Index) + assert.NotNil(t, status.FinishedAt) + assert.Greater(t, status.FinishedAt.Time, status.StartedAt.Time) + assert.Equal(t, v1alpha1.StepPluginPhase(rpcResult.Phase), status.Phase) + assert.Equal(t, v1alpha1.StepPluginOperationTerminate, status.Operation) + assert.Equal(t, rpcResult.Message, status.Message) + assert.Nil(t, status.Status) + assert.False(t, status.Disabled) + }) + + t.Run("Error status", func(t *testing.T) { + p, rpcMock := setup(t) + currentStatus := newRunningStatus() + r := newRollout(currentStatus) + + invalidResult := types.RpcStepResult{ + Phase: types.PhaseSuccessful, + Message: "This message should not be used", + RequeueAfter: time.Hour, + Status: json.RawMessage("invalid status"), + } + expectedError := types.RpcError{ + ErrorString: "This is an error", + } + rpcMock.On("Terminate", mock.Anything, mock.Anything).Return(invalidResult, expectedError) + + status, err := p.Terminate(r) + + require.NoError(t, err) + + assert.Equal(t, p.name, status.Name) + assert.Equal(t, p.index, status.Index) + assert.NotNil(t, status.FinishedAt) + assert.Greater(t, status.FinishedAt.Time, status.StartedAt.Time) + assert.Equal(t, v1alpha1.StepPluginOperationTerminate, status.Operation) + assert.Equal(t, v1alpha1.StepPluginPhaseError, status.Phase) + assert.Contains(t, status.Message, expectedError.Error()) + assert.Nil(t, status.Status) + assert.False(t, status.Disabled) + }) +} + +func Test_stepPlugin_Abort(t *testing.T) { + setup := func(t *testing.T) (*stepPlugin, *mocks.StepPlugin) { + plugin := &stepPlugin{ + name: "test-plugin", + index: 0, + config: json.RawMessage("value"), + log: log.WithFields(log.Fields{}), + } + rpcPluginMock := mocks.NewStepPlugin(t) + plugin.rpc = rpcPluginMock + return plugin, rpcPluginMock + } + newRunningStatus := func() *v1alpha1.StepPluginStatus { + return &v1alpha1.StepPluginStatus{ + Index: 0, + Name: "test-plugin", + Status: json.RawMessage("step status value"), + StartedAt: &v1.Time{Time: time.Now().Add(30 * time.Minute * -1)}, + Phase: v1alpha1.StepPluginPhaseRunning, + Operation: v1alpha1.StepPluginOperationRun, + } + } + newRollout := func(s *v1alpha1.StepPluginStatus) *v1alpha1.Rollout { + return &v1alpha1.Rollout{ + Status: v1alpha1.RolloutStatus{ + Canary: v1alpha1.CanaryStatus{ + StepPluginStatuses: []v1alpha1.StepPluginStatus{ + *s, + }, + }, + }, + } + } + + t.Run("Return nil status if run is in error", func(t *testing.T) { + p, rpcMock := setup(t) + currentStatus := newRunningStatus() + currentStatus.Phase = v1alpha1.StepPluginPhaseError + currentStatus.Operation = v1alpha1.StepPluginOperationRun + r := newRollout(currentStatus) + + rpcMock.On("Abort", mock.Anything, mock.Anything).Maybe().Panic("Abort should not be called when plugin is not running or copmpleted") + + status, err := p.Abort(r) + + require.NoError(t, err) + assert.Nil(t, status) + }) + + t.Run("Return nil status if run has failed", func(t *testing.T) { + p, rpcMock := setup(t) + currentStatus := newRunningStatus() + currentStatus.Phase = v1alpha1.StepPluginPhaseFailed + currentStatus.Operation = v1alpha1.StepPluginOperationRun + r := newRollout(currentStatus) + + rpcMock.On("Abort", mock.Anything, mock.Anything).Maybe().Panic("Abort should not be called when plugin is not running or completed") + + status, err := p.Abort(r) + + require.NoError(t, err) + assert.Nil(t, status) + }) + + t.Run("Return nil status if already aborted", func(t *testing.T) { + p, rpcMock := setup(t) + currentStatus := newRunningStatus() + currentStatus.Phase = v1alpha1.StepPluginPhaseSuccessful + currentStatus.Operation = v1alpha1.StepPluginOperationAbort + r := newRollout(currentStatus) + + rpcMock.On("Abort", mock.Anything, mock.Anything).Maybe().Panic("Abort should not be called when plugin is not running") + + status, err := p.Abort(r) + + require.NoError(t, err) + assert.Nil(t, status) + }) + t.Run("Running phase overridden to failed if running", func(t *testing.T) { + p, rpcMock := setup(t) + currentStatus := newRunningStatus() + r := newRollout(currentStatus) + + rpcResult := types.RpcStepResult{ + Phase: types.PhaseRunning, + Message: "Good message", + RequeueAfter: time.Hour, + Status: json.RawMessage("status"), + } + rpcMock.On("Abort", mock.Anything, mock.Anything).Return(rpcResult, types.RpcError{}) + + status, err := p.Abort(r) + + require.NoError(t, err) + assert.Equal(t, p.name, status.Name) + assert.Equal(t, p.index, status.Index) + assert.NotNil(t, status.FinishedAt) + assert.Greater(t, status.FinishedAt.Time, status.StartedAt.Time) + assert.Equal(t, v1alpha1.StepPluginOperationAbort, status.Operation) + assert.Equal(t, v1alpha1.StepPluginPhaseFailed, status.Phase) + assert.Equal(t, rpcResult.Message, status.Message) + assert.Nil(t, status.Status) + assert.False(t, status.Disabled) + }) + t.Run("Completes successfully", func(t *testing.T) { + p, rpcMock := setup(t) + currentStatus := newRunningStatus() + r := newRollout(currentStatus) + + rpcResult := types.RpcStepResult{ + Phase: types.PhaseSuccessful, + Message: "Good message", + RequeueAfter: time.Hour, + Status: json.RawMessage("status"), + } + rpcMock.On("Abort", mock.Anything, mock.Anything).Return(rpcResult, types.RpcError{}) + + status, err := p.Abort(r) + + require.NoError(t, err) + + assert.Equal(t, p.name, status.Name) + assert.Equal(t, p.index, status.Index) + assert.NotNil(t, status.FinishedAt) + assert.Greater(t, status.FinishedAt.Time, status.StartedAt.Time) + assert.Equal(t, v1alpha1.StepPluginPhase(rpcResult.Phase), status.Phase) + assert.Equal(t, v1alpha1.StepPluginOperationAbort, status.Operation) + assert.Equal(t, rpcResult.Message, status.Message) + assert.Nil(t, status.Status) + assert.False(t, status.Disabled) + }) + + t.Run("Error status", func(t *testing.T) { + p, rpcMock := setup(t) + currentStatus := newRunningStatus() + r := newRollout(currentStatus) + + invalidResult := types.RpcStepResult{ + Phase: types.PhaseSuccessful, + Message: "This message should not be used", + RequeueAfter: time.Hour, + Status: json.RawMessage("invalid status"), + } + expectedError := types.RpcError{ + ErrorString: "This is an error", + } + rpcMock.On("Abort", mock.Anything, mock.Anything).Return(invalidResult, expectedError) + + status, err := p.Abort(r) + + require.NoError(t, err) + + assert.Equal(t, p.name, status.Name) + assert.Equal(t, p.index, status.Index) + assert.NotNil(t, status.FinishedAt) + assert.Greater(t, status.FinishedAt.Time, status.StartedAt.Time) + assert.Equal(t, v1alpha1.StepPluginOperationAbort, status.Operation) + assert.Equal(t, v1alpha1.StepPluginPhaseError, status.Phase) + assert.Contains(t, status.Message, expectedError.Error()) + assert.Nil(t, status.Status) + assert.False(t, status.Disabled) + }) +} + +func Test_disabledStepPlugin(t *testing.T) { + t.Run("Run always return disabled status", func(t *testing.T) { + p := disabledStepPlugin{ + index: 0, + name: "test", + } + r := &v1alpha1.Rollout{ + Status: v1alpha1.RolloutStatus{ + Canary: v1alpha1.CanaryStatus{ + StepPluginStatuses: []v1alpha1.StepPluginStatus{}, + }, + }, + } + + status, err := p.Run(r) + + require.NoError(t, err) + + assert.Equal(t, p.name, status.Name) + assert.Equal(t, p.index, status.Index) + assert.Equal(t, v1alpha1.StepPluginOperationRun, status.Operation) + assert.True(t, status.Disabled) + }) + t.Run("Abort has no effect", func(t *testing.T) { + p := disabledStepPlugin{ + index: 0, + name: "test", + } + r := &v1alpha1.Rollout{ + Status: v1alpha1.RolloutStatus{ + Canary: v1alpha1.CanaryStatus{ + StepPluginStatuses: []v1alpha1.StepPluginStatus{ + { + Index: 0, + Name: "test", + Operation: v1alpha1.StepPluginOperationRun, + Phase: v1alpha1.StepPluginPhaseSuccessful, + }, + }, + }, + }, + } + + status, err := p.Abort(r) + + require.NoError(t, err) + assert.Nil(t, status) + }) + t.Run("Terminate has no effect", func(t *testing.T) { + p := disabledStepPlugin{ + index: 0, + name: "test", + } + r := &v1alpha1.Rollout{ + Status: v1alpha1.RolloutStatus{ + Canary: v1alpha1.CanaryStatus{ + StepPluginStatuses: []v1alpha1.StepPluginStatus{ + { + Index: 0, + Name: "test", + Operation: v1alpha1.StepPluginOperationRun, + Phase: v1alpha1.StepPluginPhaseRunning, + }, + }, + }, + }, + } + + status, err := p.Terminate(r) + + require.NoError(t, err) + assert.Nil(t, status) + }) +} diff --git a/rollout/steps/plugin/resolver.go b/rollout/steps/plugin/resolver.go new file mode 100644 index 0000000000..8a1393ab57 --- /dev/null +++ b/rollout/steps/plugin/resolver.go @@ -0,0 +1,54 @@ +package plugin + +import ( + "fmt" + + "github.com/sirupsen/logrus" + log "github.com/sirupsen/logrus" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/rollout/steps/plugin/client" + "github.com/argoproj/argo-rollouts/utils/config" + "github.com/argoproj/argo-rollouts/utils/plugin/types" +) + +type resolver struct { +} + +// Resolver allows to resolve a StepPlugin object +type Resolver interface { + // Resolve is a factory to create the correct StepPlugin based on the current step and global configurations + Resolve(index int32, plugin v1alpha1.PluginStep, log *log.Entry) (StepPlugin, error) +} + +// NewResolver creaates a new Resolver +func NewResolver() Resolver { + return &resolver{} +} + +func (r *resolver) Resolve(index int32, plugin v1alpha1.PluginStep, log *log.Entry) (StepPlugin, error) { + if config, err := config.GetConfig(); err != nil { + return nil, fmt.Errorf("could not get config: %w", err) + } else { + plugin := config.GetPlugin(plugin.Name, types.PluginTypeStep) + if plugin != nil && plugin.Disabled { + return &disabledStepPlugin{ + index: index, + name: plugin.Name, + }, nil + } + } + + pluginClient, err := client.GetPlugin(plugin.Name) + if err != nil { + return nil, fmt.Errorf("failed to get step plugin %s: %w", plugin.Name, err) + } + + return &stepPlugin{ + rpc: pluginClient, + index: index, + name: plugin.Name, + config: plugin.Config, + log: log.WithFields(logrus.Fields{"stepplugin": plugin.Name, "stepindex": index}), + }, nil +} diff --git a/rollout/steps/plugin/rpc/mocks/StepPlugin.go b/rollout/steps/plugin/rpc/mocks/StepPlugin.go new file mode 100644 index 0000000000..c858f4fa58 --- /dev/null +++ b/rollout/steps/plugin/rpc/mocks/StepPlugin.go @@ -0,0 +1,150 @@ +// Code generated by mockery v2.42.2. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" + + types "github.com/argoproj/argo-rollouts/utils/plugin/types" + + v1alpha1 "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" +) + +// StepPlugin is an autogenerated mock type for the StepPlugin type +type StepPlugin struct { + mock.Mock +} + +// Abort provides a mock function with given fields: _a0, _a1 +func (_m *StepPlugin) Abort(_a0 *v1alpha1.Rollout, _a1 *types.RpcStepContext) (types.RpcStepResult, types.RpcError) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for Abort") + } + + var r0 types.RpcStepResult + var r1 types.RpcError + if rf, ok := ret.Get(0).(func(*v1alpha1.Rollout, *types.RpcStepContext) (types.RpcStepResult, types.RpcError)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(*v1alpha1.Rollout, *types.RpcStepContext) types.RpcStepResult); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Get(0).(types.RpcStepResult) + } + + if rf, ok := ret.Get(1).(func(*v1alpha1.Rollout, *types.RpcStepContext) types.RpcError); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Get(1).(types.RpcError) + } + + return r0, r1 +} + +// InitPlugin provides a mock function with given fields: +func (_m *StepPlugin) InitPlugin() types.RpcError { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for InitPlugin") + } + + var r0 types.RpcError + if rf, ok := ret.Get(0).(func() types.RpcError); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(types.RpcError) + } + + return r0 +} + +// Run provides a mock function with given fields: _a0, _a1 +func (_m *StepPlugin) Run(_a0 *v1alpha1.Rollout, _a1 *types.RpcStepContext) (types.RpcStepResult, types.RpcError) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for Run") + } + + var r0 types.RpcStepResult + var r1 types.RpcError + if rf, ok := ret.Get(0).(func(*v1alpha1.Rollout, *types.RpcStepContext) (types.RpcStepResult, types.RpcError)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(*v1alpha1.Rollout, *types.RpcStepContext) types.RpcStepResult); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Get(0).(types.RpcStepResult) + } + + if rf, ok := ret.Get(1).(func(*v1alpha1.Rollout, *types.RpcStepContext) types.RpcError); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Get(1).(types.RpcError) + } + + return r0, r1 +} + +// Terminate provides a mock function with given fields: _a0, _a1 +func (_m *StepPlugin) Terminate(_a0 *v1alpha1.Rollout, _a1 *types.RpcStepContext) (types.RpcStepResult, types.RpcError) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for Terminate") + } + + var r0 types.RpcStepResult + var r1 types.RpcError + if rf, ok := ret.Get(0).(func(*v1alpha1.Rollout, *types.RpcStepContext) (types.RpcStepResult, types.RpcError)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(*v1alpha1.Rollout, *types.RpcStepContext) types.RpcStepResult); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Get(0).(types.RpcStepResult) + } + + if rf, ok := ret.Get(1).(func(*v1alpha1.Rollout, *types.RpcStepContext) types.RpcError); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Get(1).(types.RpcError) + } + + return r0, r1 +} + +// Type provides a mock function with given fields: +func (_m *StepPlugin) Type() string { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Type") + } + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// NewStepPlugin creates a new instance of StepPlugin. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewStepPlugin(t interface { + mock.TestingT + Cleanup(func()) +}) *StepPlugin { + mock := &StepPlugin{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/rollout/steps/plugin/rpc/rpc.go b/rollout/steps/plugin/rpc/rpc.go new file mode 100644 index 0000000000..86483b7f64 --- /dev/null +++ b/rollout/steps/plugin/rpc/rpc.go @@ -0,0 +1,198 @@ +package rpc + +import ( + "encoding/gob" + "fmt" + "net/rpc" + + "github.com/argoproj/argo-rollouts/utils/plugin/types" + + "github.com/hashicorp/go-plugin" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" +) + +type RunArgs struct { + Rollout *v1alpha1.Rollout + Context *types.RpcStepContext +} + +type TerminateArgs struct { + Rollout *v1alpha1.Rollout + Context *types.RpcStepContext +} + +type AbortArgs struct { + Rollout *v1alpha1.Rollout + Context *types.RpcStepContext +} + +type Response struct { + Result types.RpcStepResult + Error types.RpcError +} + +func init() { + gob.RegisterName("step.RunArgs", new(RunArgs)) + gob.RegisterName("step.TerminateArgs", new(TerminateArgs)) + gob.RegisterName("step.AbortArgs", new(AbortArgs)) +} + +// StepPlugin is the interface that we're exposing as a plugin. It needs to match metricproviders.Providers but we can +// not import that package because it would create a circular dependency. +type StepPlugin interface { + InitPlugin() types.RpcError + types.RpcStep +} + +// StepPluginRPC Here is an implementation that talks over RPC +type StepPluginRPC struct{ client *rpc.Client } + +// InitPlugin is the client aka the controller side function that calls the server side rpc (plugin) +// this gets called once during startup of the plugin and can be used to set up informers, k8s clients, etc. +func (g *StepPluginRPC) InitPlugin() types.RpcError { + var resp types.RpcError + err := g.client.Call("Plugin.InitPlugin", new(any), &resp) + if err != nil { + return types.RpcError{ErrorString: fmt.Sprintf("InitPlugin rpc call error: %s", err)} + } + return resp +} + +// Run executes the step +func (g *StepPluginRPC) Run(rollout *v1alpha1.Rollout, context *types.RpcStepContext) (types.RpcStepResult, types.RpcError) { + var resp Response + var args any = RunArgs{ + Rollout: rollout, + Context: context, + } + err := g.client.Call("Plugin.Run", &args, &resp) + if err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Sprintf("Run rpc call error: %s", err)} + } + return resp.Result, resp.Error +} + +// Terminate stops the execution of a running step and exits early +func (g *StepPluginRPC) Terminate(rollout *v1alpha1.Rollout, context *types.RpcStepContext) (types.RpcStepResult, types.RpcError) { + var resp Response + var args any = TerminateArgs{ + Rollout: rollout, + Context: context, + } + err := g.client.Call("Plugin.Terminate", &args, &resp) + if err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Sprintf("Terminate rpc call error: %s", err)} + } + return resp.Result, resp.Error +} + +// Abort reverts previous operation executed by the step if necessary +func (g *StepPluginRPC) Abort(rollout *v1alpha1.Rollout, context *types.RpcStepContext) (types.RpcStepResult, types.RpcError) { + var resp Response + var args any = AbortArgs{ + Rollout: rollout, + Context: context, + } + err := g.client.Call("Plugin.Abort", &args, &resp) + if err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Sprintf("Abort rpc call error: %s", err)} + } + return resp.Result, resp.Error +} + +// Type returns the type of the traffic routing reconciler +func (g *StepPluginRPC) Type() string { + var resp string + err := g.client.Call("Plugin.Type", new(any), &resp) + if err != nil { + return fmt.Sprintf("Type rpc call error: %s", err) + } + + return resp +} + +// StepRPCServer Here is the RPC server that MetricsPluginRPC talks to, conforming to +// the requirements of net/rpc +type StepRPCServer struct { + // This is the real implementation + Impl StepPlugin +} + +// InitPlugin this is the server aka the controller side function that receives calls from the client side rpc (controller) +// this gets called once during startup of the plugin and can be used to set up informers or k8s clients etc. +func (s *StepRPCServer) InitPlugin(args any, resp *types.RpcError) error { + *resp = s.Impl.InitPlugin() + return nil +} + +// Run executes the step +func (s *StepRPCServer) Run(args any, resp *Response) error { + runArgs, ok := args.(*RunArgs) + if !ok { + return fmt.Errorf("invalid args %s", args) + } + result, err := s.Impl.Run(runArgs.Rollout, runArgs.Context) + *resp = Response{ + Result: result, + Error: err, + } + return nil +} + +// Terminate stops the execution of a running step and exits early +func (s *StepRPCServer) Terminate(args any, resp *Response) error { + runArgs, ok := args.(*TerminateArgs) + if !ok { + return fmt.Errorf("invalid args %s", args) + } + result, err := s.Impl.Terminate(runArgs.Rollout, runArgs.Context) + *resp = Response{ + Result: result, + Error: err, + } + return nil +} + +// Abort reverts previous operation executed by the step if necessary +func (s *StepRPCServer) Abort(args any, resp *Response) error { + runArgs, ok := args.(*AbortArgs) + if !ok { + return fmt.Errorf("invalid args %s", args) + } + result, err := s.Impl.Abort(runArgs.Rollout, runArgs.Context) + *resp = Response{ + Result: result, + Error: err, + } + return nil +} + +// Type returns the type of the traffic routing reconciler +func (s *StepRPCServer) Type(args any, resp *string) error { + *resp = s.Impl.Type() + return nil +} + +// RpcStepPlugin This is the implementation of plugin.Plugin so we can serve/consume +// +// This has two methods: Server must return an RPC server for this plugin +// type. We construct a StepRPCServer for this. +// +// Client must return an implementation of our interface that communicates +// over an RPC client. We return StepPluginRPC for this. +// +// Ignore MuxBroker. That is used to create more multiplexed streams on our +// plugin connection and is a more advanced use case. +type RpcStepPlugin struct { + // Impl Injection + Impl StepPlugin +} + +func (p *RpcStepPlugin) Server(*plugin.MuxBroker) (any, error) { + return &StepRPCServer{Impl: p.Impl}, nil +} + +func (RpcStepPlugin) Client(b *plugin.MuxBroker, c *rpc.Client) (any, error) { + return &StepPluginRPC{client: c}, nil +} diff --git a/rollout/steps/plugin/rpc/rpc_test.go b/rollout/steps/plugin/rpc/rpc_test.go new file mode 100644 index 0000000000..211aefd04d --- /dev/null +++ b/rollout/steps/plugin/rpc/rpc_test.go @@ -0,0 +1,150 @@ +package rpc + +import ( + "context" + "testing" + "time" + + "github.com/argoproj/argo-rollouts/utils/plugin/types" + + goPlugin "github.com/hashicorp/go-plugin" + "github.com/tj/assert" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" +) + +var testHandshake = goPlugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "ARGO_ROLLOUTS_RPC_PLUGIN", + MagicCookieValue: "step", +} + +func pluginClient(t *testing.T) (StepPlugin, goPlugin.ClientProtocol, func(), chan struct{}) { + ctx, cancel := context.WithCancel(context.Background()) + + pluginImpl := &testRpcPlugin{} + + // pluginMap is the map of plugins we can dispense. + var pluginMap = map[string]goPlugin.Plugin{ + "RpcStepPlugin": &RpcStepPlugin{Impl: pluginImpl}, + } + + ch := make(chan *goPlugin.ReattachConfig, 1) + closeCh := make(chan struct{}) + go goPlugin.Serve(&goPlugin.ServeConfig{ + HandshakeConfig: testHandshake, + Plugins: pluginMap, + Test: &goPlugin.ServeTestConfig{ + Context: ctx, + ReattachConfigCh: ch, + CloseCh: closeCh, + }, + }) + + // We should get a config + var config *goPlugin.ReattachConfig + select { + case config = <-ch: + case <-time.After(2000 * time.Millisecond): + t.Fatal("should've received reattach") + } + if config == nil { + t.Fatal("config should not be nil") + } + + // Connect! + c := goPlugin.NewClient(&goPlugin.ClientConfig{ + Cmd: nil, + HandshakeConfig: testHandshake, + Plugins: pluginMap, + Reattach: config, + }) + client, err := c.Client() + if err != nil { + t.Fatalf("err: %s", err) + } + + // Request the plugin + raw, err := client.Dispense("RpcStepPlugin") + if err != nil { + t.Fail() + } + + plugin, ok := raw.(StepPlugin) + if !ok { + t.Fail() + } + + return plugin, client, cancel, closeCh +} + +func TestPlugin(t *testing.T) { + plugin, _, cancel, closeCh := pluginClient(t) + defer cancel() + + err := plugin.InitPlugin() + if err.Error() != "" { + t.Fail() + } + + ro := v1alpha1.Rollout{} + + _, err = plugin.Run(&ro, &types.RpcStepContext{}) + assert.Equal(t, "", err.Error()) + + _, err = plugin.Terminate(&ro, &types.RpcStepContext{}) + assert.Equal(t, "", err.Error()) + + _, err = plugin.Abort(&ro, &types.RpcStepContext{}) + assert.Equal(t, "", err.Error()) + + typeString := plugin.Type() + assert.Equal(t, "StepPlugin Test", typeString) + + // Canceling should cause an exit + cancel() + <-closeCh +} + +func TestPluginClosedConnection(t *testing.T) { + plugin, client, cancel, closeCh := pluginClient(t) + defer cancel() + + client.Close() + time.Sleep(100 * time.Millisecond) + + const expectedError = "connection is shut down" + + err := plugin.InitPlugin() + assert.Contains(t, err.Error(), expectedError) + + _, err = plugin.Run(&v1alpha1.Rollout{}, &types.RpcStepContext{}) + assert.Contains(t, err.Error(), expectedError) + + _, err = plugin.Terminate(&v1alpha1.Rollout{}, &types.RpcStepContext{}) + assert.Contains(t, err.Error(), expectedError) + + _, err = plugin.Abort(&v1alpha1.Rollout{}, &types.RpcStepContext{}) + assert.Contains(t, err.Error(), expectedError) + + cancel() + <-closeCh +} + +func TestInvalidArgs(t *testing.T) { + server := StepRPCServer{} + badtype := struct { + Args string + }{} + + var resp Response + err := server.Run(badtype, &resp) + assert.Error(t, err) + + err = server.Terminate(badtype, &resp) + assert.Error(t, err) + + err = server.Abort(badtype, &resp) + assert.Error(t, err) + +} diff --git a/rollout/steps/plugin/rpc/rpc_test_implementation.go b/rollout/steps/plugin/rpc/rpc_test_implementation.go new file mode 100644 index 0000000000..db577edf94 --- /dev/null +++ b/rollout/steps/plugin/rpc/rpc_test_implementation.go @@ -0,0 +1,29 @@ +package rpc + +import ( + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/utils/plugin/types" +) + +type testRpcPlugin struct{} + +func (p *testRpcPlugin) InitPlugin() types.RpcError { + return types.RpcError{} +} + +func (p *testRpcPlugin) Run(_ *v1alpha1.Rollout, _ *types.RpcStepContext) (types.RpcStepResult, types.RpcError) { + return types.RpcStepResult{}, types.RpcError{} +} + +func (p *testRpcPlugin) Terminate(_ *v1alpha1.Rollout, _ *types.RpcStepContext) (types.RpcStepResult, types.RpcError) { + return types.RpcStepResult{}, types.RpcError{} +} + +func (p *testRpcPlugin) Abort(_ *v1alpha1.Rollout, _ *types.RpcStepContext) (types.RpcStepResult, types.RpcError) { + return types.RpcStepResult{}, types.RpcError{} +} + +// Type returns the type of the step plugin +func (p *testRpcPlugin) Type() string { + return "StepPlugin Test" +} diff --git a/rollout/sync.go b/rollout/sync.go index 875949ee55..f4c6ca9715 100644 --- a/rollout/sync.go +++ b/rollout/sync.go @@ -42,18 +42,13 @@ import ( // // Note that currently the rollout controller is using caches to avoid querying the server for reads. // This may lead to stale reads of replica sets, thus incorrect v status. -func (c *rolloutContext) getAllReplicaSetsAndSyncRevision(createIfNotExisted bool) (*appsv1.ReplicaSet, error) { +func (c *rolloutContext) getAllReplicaSetsAndSyncRevision() (*appsv1.ReplicaSet, error) { // Get new replica set with the updated revision number newRS, err := c.syncReplicaSetRevision() if err != nil { return nil, err } - if newRS == nil && createIfNotExisted { - newRS, err = c.createDesiredReplicaSet() - if err != nil { - return nil, err - } - } + return newRS, nil } @@ -83,17 +78,13 @@ func (c *rolloutContext) syncReplicaSetRevision() (*appsv1.ReplicaSet, error) { affinityNeedsUpdate := replicasetutil.IfInjectedAntiAffinityRuleNeedsUpdate(rsCopy.Spec.Template.Spec.Affinity, *c.rollout) if annotationsUpdated || minReadySecondsNeedsUpdate || affinityNeedsUpdate { + rsCopy.Spec.MinReadySeconds = c.rollout.Spec.MinReadySeconds rsCopy.Spec.Template.Spec.Affinity = replicasetutil.GenerateReplicaSetAffinity(*c.rollout) - rs, err := c.kubeclientset.AppsV1().ReplicaSets(rsCopy.ObjectMeta.Namespace).Update(ctx, rsCopy, metav1.UpdateOptions{}) - if err != nil { - c.log.WithError(err).Error("Error: updating replicaset revision") - return nil, fmt.Errorf("error updating replicaset revision: %v", err) - } - c.log.Infof("Synced revision on ReplicaSet '%s' to '%s'", rs.Name, newRevision) - err = c.replicaSetInformer.GetIndexer().Update(rs) + + rs, err := c.updateReplicaSet(ctx, rsCopy) if err != nil { - return nil, fmt.Errorf("error updating replicaset informer in syncReplicaSetRevision: %w", err) + return nil, fmt.Errorf("failed to update replicaset revision on %s: %w", rsCopy.Name, err) } return rs, nil } @@ -113,7 +104,7 @@ func (c *rolloutContext) syncReplicaSetRevision() (*appsv1.ReplicaSet, error) { conditions.SetRolloutCondition(&c.rollout.Status, *condition) updatedRollout, err := c.argoprojclientset.ArgoprojV1alpha1().Rollouts(c.rollout.Namespace).UpdateStatus(ctx, c.rollout, metav1.UpdateOptions{}) if err != nil { - c.log.WithError(err).Error("Error: updating rollout revision") + c.log.WithError(err).Error("Error: updating rollout status in syncReplicaSetRevision") return nil, err } c.rollout = updatedRollout @@ -245,7 +236,7 @@ func (c *rolloutContext) createDesiredReplicaSet() (*appsv1.ReplicaSet, error) { cond := conditions.NewRolloutCondition(v1alpha1.RolloutProgressing, corev1.ConditionFalse, conditions.FailedRSCreateReason, msg) patchErr := c.patchCondition(c.rollout, newStatus, cond) if patchErr != nil { - c.log.Warnf("Error Patching Rollout: %s", patchErr.Error()) + c.log.Warnf("Error Patching Rollout Conditions: %s", patchErr.Error()) } return nil, err default: @@ -281,7 +272,7 @@ func (c *rolloutContext) createDesiredReplicaSet() (*appsv1.ReplicaSet, error) { func (c *rolloutContext) syncReplicasOnly() error { c.log.Infof("Syncing replicas only due to scaling event") var err error - c.newRS, err = c.getAllReplicaSetsAndSyncRevision(false) + c.newRS, err = c.getAllReplicaSetsAndSyncRevision() if err != nil { return fmt.Errorf("failed to getAllReplicaSetsAndSyncRevision in syncReplicasOnly: %w", err) } @@ -328,12 +319,15 @@ func (c *rolloutContext) syncReplicasOnly() error { // rsList should come from getReplicaSetsForRollout(r). func (c *rolloutContext) isScalingEvent() (bool, error) { var err error - c.newRS, err = c.getAllReplicaSetsAndSyncRevision(false) + c.newRS, err = c.getAllReplicaSetsAndSyncRevision() if err != nil { return false, fmt.Errorf("failed to getAllReplicaSetsAndSyncRevision in isScalingEvent: %w", err) } - for _, rs := range controller.FilterActiveReplicaSets(c.allRSs) { + // We only care about scaling events on the newRS and stableRS because these are the only replicasets that we ever + // adjust the replicas counts on as well as the desired annotation. When we have stacked rollouts going the middle + // replicasets will never have the desired annotation updated this can cause a tight loop of isScalingEvent -> syncReplicasOnly -> isScalingEvent + for _, rs := range controller.FilterActiveReplicaSets([]*appsv1.ReplicaSet{c.newRS, c.stableRS}) { desired, ok := annotations.GetDesiredReplicasAnnotation(rs) if !ok { continue @@ -370,25 +364,21 @@ func (c *rolloutContext) scaleReplicaSet(rs *appsv1.ReplicaSet, newScale int32, rolloutReplicas := defaults.GetReplicasOrDefault(rollout.Spec.Replicas) annotationsNeedUpdate := annotations.ReplicasAnnotationsNeedUpdate(rs, rolloutReplicas) - scaled := false var err error + scaled := false if sizeNeedsUpdate || annotationsNeedUpdate { rsCopy := rs.DeepCopy() oldScale := defaults.GetReplicasOrDefault(rs.Spec.Replicas) *(rsCopy.Spec.Replicas) = newScale annotations.SetReplicasAnnotations(rsCopy, rolloutReplicas) if fullScaleDown && !c.shouldDelayScaleDownOnAbort() { + // This bypasses the normal call to removeScaleDownDelay and then depends on the removal via an update in updateReplicaSet delete(rsCopy.Annotations, v1alpha1.DefaultReplicaSetScaleDownDeadlineAnnotationKey) } - rs, err = c.kubeclientset.AppsV1().ReplicaSets(rsCopy.Namespace).Update(ctx, rsCopy, metav1.UpdateOptions{}) + rs, err = c.updateReplicaSet(ctx, rsCopy) if err != nil { - return scaled, rs, fmt.Errorf("error updating replicaset %s: %w", rsCopy.Name, err) - } - err = c.replicaSetInformer.GetIndexer().Update(rs) - if err != nil { - err = fmt.Errorf("error updating replicaset informer in scaleReplicaSet: %w", err) - return scaled, rs, err + return scaled, rs, fmt.Errorf("failed to updateReplicaSet in scaleReplicaSet: %w", err) } if sizeNeedsUpdate { @@ -502,7 +492,7 @@ func (c *rolloutContext) checkPausedConditions() error { var updatedConditions []*v1alpha1.RolloutCondition - if (isPaused != progCondPaused) && !abortCondExists { + if (isPaused != progCondPaused) && !abortCondExists && !conditions.RolloutCompleted(&c.rollout.Status) { if isPaused { updatedConditions = append(updatedConditions, conditions.NewRolloutCondition(v1alpha1.RolloutProgressing, corev1.ConditionUnknown, conditions.RolloutPausedReason, conditions.RolloutPausedMessage)) } else { @@ -575,7 +565,10 @@ func isIndefiniteStep(r *v1alpha1.Rollout) bool { if currentStep != nil && (currentStep.Experiment != nil || currentStep.Analysis != nil || currentStep.Pause != nil) { return true } - return false + // also check the pause condition to cover blueGreen + pauseCond := conditions.GetRolloutCondition(r.Status, v1alpha1.RolloutPaused) + pausedCondTrue := pauseCond != nil && pauseCond.Status == corev1.ConditionTrue + return pausedCondTrue } // isWaitingForReplicaSetScaleDown returns whether or not the rollout still has other replica sets with a scale down deadline annotation @@ -667,7 +660,7 @@ func (c *rolloutContext) calculateRolloutConditions(newStatus v1alpha1.RolloutSt // everything but lastTransitionTime. SetRolloutCondition already does that but // it also is not updating conditions when the reason of the new condition is the // same as the old. The Progressing condition is a special case because we want to - // update with the same reason and change just lastUpdateTime iff we notice any + // update with the same reason and change just lastUpdateTime if we notice any // progress. That's why we handle it here. if currentCond != nil { if currentCond.Status == corev1.ConditionTrue { @@ -726,7 +719,7 @@ func (c *rolloutContext) calculateRolloutConditions(newStatus v1alpha1.RolloutSt conditions.RemoveRolloutCondition(&newStatus, v1alpha1.RolloutReplicaFailure) } - if conditions.RolloutCompleted(c.rollout, &newStatus) { + if conditions.RolloutCompleted(&newStatus) { // The event gets triggered in function promoteStable updateCompletedCond := conditions.NewRolloutCondition(v1alpha1.RolloutCompleted, corev1.ConditionTrue, conditions.RolloutCompletedReason, conditions.RolloutCompletedReason) @@ -737,7 +730,7 @@ func (c *rolloutContext) calculateRolloutConditions(newStatus v1alpha1.RolloutSt if conditions.SetRolloutCondition(&newStatus, *updateCompletedCond) { revision, _ := replicasetutil.Revision(c.rollout) c.recorder.Eventf(c.rollout, record.EventOptions{EventReason: conditions.RolloutNotCompletedReason}, - conditions.RolloutNotCompletedMessage, revision+1, newStatus.CurrentPodHash) + conditions.RolloutNotCompletedMessage, revision, newStatus.CurrentPodHash) } } @@ -903,6 +896,7 @@ func (c *rolloutContext) resetRolloutStatus(newStatus *v1alpha1.RolloutStatus) { newStatus.BlueGreen.ScaleUpPreviewCheckPoint = false newStatus.Canary.CurrentStepAnalysisRunStatus = nil newStatus.Canary.CurrentBackgroundAnalysisRunStatus = nil + newStatus.Canary.StepPluginStatuses = nil newStatus.CurrentStepIndex = replicasetutil.ResetCurrentStepIndex(c.rollout) } diff --git a/rollout/sync_test.go b/rollout/sync_test.go index 7b2552e6c9..d71bb9e496 100644 --- a/rollout/sync_test.go +++ b/rollout/sync_test.go @@ -1,6 +1,7 @@ package rollout import ( + "encoding/json" "strconv" "testing" "time" @@ -639,3 +640,66 @@ func TestScaleDownDeploymentOnSuccess(t *testing.T) { assert.NotNil(t, err) } + +// This tests validates that when there are old replicasets that have miss matched desired-count annotations aka they do +// not match the spec.replicas field. When this happens we get stuck in a reconcile loop of only trying to scale replicasets +// only because of a supposed scaling event. This test validates that we do not get stuck in this loop by checking that status.replicas +// and status.readyreplicas are updated because those will not be updated in that loop. +func TestIsScalingEventMissMatchedDesiredOldReplicas(t *testing.T) { + f := newFixture(t) + defer f.Close() + + // these steps should be ignored + steps := []v1alpha1.CanaryStep{ + { + SetWeight: int32Ptr(10), + }, + { + Pause: &v1alpha1.RolloutPause{ + Duration: v1alpha1.DurationFromInt(60), + }, + }, + } + + r0 := newCanaryRollout("foo", 10, int32Ptr(4), steps, int32Ptr(0), intstr.FromInt(10), intstr.FromInt(0)) + r0.Annotations[annotations.RevisionAnnotation] = "1" + oldRs := newReplicaSetWithStatus(r0, 3, 3) + oldRs.Annotations[annotations.DesiredReplicasAnnotation] = "2" + + r1 := bumpVersion(r0) + + // Desired rs will be created during reconcile + stableRs := newReplicaSetWithStatus(r1, 10, 10) + r1.Status.StableRS = stableRs.Labels[v1alpha1.DefaultRolloutUniqueLabelKey] + r2 := bumpVersion(r1) + r2.Annotations[annotations.RevisionAnnotation] = "2" + + f.rolloutLister = append(f.rolloutLister, r2) + f.objects = append(f.objects, r2) + f.kubeobjects = append(f.kubeobjects, oldRs, stableRs) + f.replicaSetLister = append(f.replicaSetLister, oldRs, stableRs) + + f.expectUpdateRolloutAction(r2) // update rollout revision + f.expectUpdateRolloutStatusAction(r2) + updatedROIndex := f.expectPatchRolloutAction(r2) + createdRS2Index := f.expectCreateReplicaSetAction(stableRs) + updatedRS2Index := f.expectUpdateReplicaSetAction(stableRs) + f.run(getKey(r2, t)) + + createdRS2 := f.getCreatedReplicaSet(createdRS2Index) + assert.Equal(t, int32(0), *createdRS2.Spec.Replicas) + updatedRS2 := f.getUpdatedReplicaSet(updatedRS2Index) + assert.Equal(t, int32(1), *updatedRS2.Spec.Replicas) + + updateRO := f.getPatchedRollout(updatedROIndex) + + //Will only contain status + roStatus := v1alpha1.Rollout{} + err := json.Unmarshal([]byte(updateRO), &roStatus) + assert.Nil(t, err) + + // We have two ReplicaSets, one with 3 pods and one with 10 + assert.Equal(t, int32(13), roStatus.Status.Replicas) + assert.Equal(t, int32(13), roStatus.Status.ReadyReplicas) + assert.Equal(t, int32(0), roStatus.Status.UpdatedReplicas) +} diff --git a/rollout/trafficrouting.go b/rollout/trafficrouting.go index f8b01951a4..df00d70c77 100644 --- a/rollout/trafficrouting.go +++ b/rollout/trafficrouting.go @@ -10,6 +10,8 @@ import ( "github.com/argoproj/argo-rollouts/rollout/trafficrouting/plugin" + appsv1 "k8s.io/api/apps/v1" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" "github.com/argoproj/argo-rollouts/rollout/trafficrouting" "github.com/argoproj/argo-rollouts/rollout/trafficrouting/alb" @@ -41,9 +43,9 @@ func (c *Controller) NewTrafficRoutingReconciler(roCtx *rolloutContext) ([]traff } if rollout.Spec.Strategy.Canary.TrafficRouting.Istio != nil { if c.IstioController.VirtualServiceInformer.HasSynced() { - trafficReconcilers = append(trafficReconcilers, istio.NewReconciler(rollout, c.IstioController.DynamicClientSet, c.recorder, c.IstioController.VirtualServiceLister, c.IstioController.DestinationRuleLister)) + trafficReconcilers = append(trafficReconcilers, istio.NewReconciler(rollout, c.IstioController.DynamicClientSet, c.recorder, c.IstioController.VirtualServiceLister, c.IstioController.DestinationRuleLister, roCtx.allRSs)) } else { - trafficReconcilers = append(trafficReconcilers, istio.NewReconciler(rollout, c.IstioController.DynamicClientSet, c.recorder, nil, nil)) + trafficReconcilers = append(trafficReconcilers, istio.NewReconciler(rollout, c.IstioController.DynamicClientSet, c.recorder, nil, nil, roCtx.allRSs)) } } if rollout.Spec.Strategy.Canary.TrafficRouting.Nginx != nil { @@ -133,6 +135,23 @@ func (c *Controller) NewTrafficRoutingReconciler(roCtx *rolloutContext) ([]traff return nil, nil } +func (c *rolloutContext) checkReplicasAvailable(rs *appsv1.ReplicaSet, desiredWeight int32) bool { + if rs == nil { + return false + } + availableReplicas := rs.Status.AvailableReplicas + totalReplicas := *c.rollout.Spec.Replicas + + desiredReplicas := (desiredWeight * totalReplicas) / 100 + if availableReplicas < desiredReplicas { + c.log.Infof("ReplicaSet '%s' has %d available replicas, waiting for %d", rs.Name, availableReplicas, desiredReplicas) + return false + } + + return true + +} + // this currently only be used in the canary strategy func (c *rolloutContext) reconcileTrafficRouting() error { reconcilers, err := c.newTrafficRoutingReconciler(c) @@ -198,6 +217,15 @@ func (c *rolloutContext) reconcileTrafficRouting() error { } else if c.newRS == nil || c.newRS.Status.AvailableReplicas == 0 { // when newRS is not available or replicas num is 0. never weight to canary weightDestinations = append(weightDestinations, c.calculateWeightDestinationsFromExperiment()...) + // If a user changes their mind in the middle of an V1 -> V2 update, and then applies a V3 + // there might have been a V2 ReplicaSet that was scaled up, but is now defunct. + // During the V2 rollout, managed routes could have been setup and would continue + // to direct traffic to the canary service which is now in front of 0 available replicas. + // We want to remove these managed routes alongside the safety here of never weighting to the canary. + err := reconciler.RemoveManagedRoutes() + if err != nil { + return err + } } else if c.rollout.Status.PromoteFull { // on a promote full, desired stable weight should be 0 (100% to canary), // But we can only increase canary weight according to available replica counts of the canary. @@ -234,6 +262,10 @@ func (c *rolloutContext) reconcileTrafficRouting() error { desiredWeight = weightutil.MaxTrafficWeight(c.rollout) } } + + if !c.checkReplicasAvailable(c.stableRS, weightutil.MaxTrafficWeight(c.rollout)-desiredWeight) { + return nil + } // We need to check for revision > 1 because when we first install the rollout we run step 0 this prevents that. // There is a bigger fix needed for the reasons on why we run step 0 on rollout install, that needs to be explored. revision, revisionFound := annotations.GetRevisionAnnotation(c.rollout) @@ -288,6 +320,12 @@ func (c *rolloutContext) reconcileTrafficRouting() error { } else { c.log.Infof("Desired weight (stepIdx: %s) %d not yet verified", indexString, desiredWeight) c.enqueueRolloutAfter(c.rollout, defaults.GetRolloutVerifyRetryInterval()) + // At the end of the rollout we need to verify the weight is correct, and return an error if not because we don't want the rest of the + // reconcile process to continue. We don't need to do this if we are in the middle of the rollout because the rest of the reconcile + // process won't scale down the old replicasets yet due to being in the middle of some steps. + if desiredWeight == weightutil.MaxTrafficWeight(c.rollout) && len(c.rollout.Spec.Strategy.Canary.Steps) >= int(*c.rollout.Status.CurrentStepIndex) { + return fmt.Errorf("end of rollout, desired weight %d not yet verified", desiredWeight) + } } } } diff --git a/rollout/trafficrouting/alb/alb.go b/rollout/trafficrouting/alb/alb.go index 8e81b113c7..f06c7ea452 100644 --- a/rollout/trafficrouting/alb/alb.go +++ b/rollout/trafficrouting/alb/alb.go @@ -15,6 +15,8 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/utils/pointer" + elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" "github.com/argoproj/argo-rollouts/rollout/trafficrouting" "github.com/argoproj/argo-rollouts/utils/aws" @@ -24,7 +26,6 @@ import ( jsonutil "github.com/argoproj/argo-rollouts/utils/json" logutil "github.com/argoproj/argo-rollouts/utils/log" "github.com/argoproj/argo-rollouts/utils/record" - elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" ) const ( @@ -202,7 +203,7 @@ func (r *Reconciler) VerifyWeight(desiredWeight int32, additionalDestinations .. return nil, nil } - if !rolloututil.ShouldVerifyWeight(r.cfg.Rollout) { + if !rolloututil.ShouldVerifyWeight(r.cfg.Rollout, desiredWeight) { // If we should not verify weight but the ALB status has not been set yet due to a Rollout resource just being // installed in the cluster we want to actually run the rest of the function, so we do not return if // r.cfg.Rollout.Status.ALB is nil. However, if we should not verify, and we have already updated the status once @@ -242,7 +243,7 @@ func (r *Reconciler) VerifyWeightPerIngress(desiredWeight int32, ingresses []str } resourceIDToDest := map[string]v1alpha1.WeightDestination{} - stableService, canaryService := trafficrouting.GetStableAndCanaryServices(rollout) + stableService, canaryService := trafficrouting.GetStableAndCanaryServices(rollout, true) canaryResourceID := aws.BuildTargetGroupResourceID(rollout.Namespace, ingress.GetName(), canaryService, rollout.Spec.Strategy.Canary.TrafficRouting.ALB.ServicePort) stableResourceID := aws.BuildTargetGroupResourceID(rollout.Namespace, ingress.GetName(), stableService, rollout.Spec.Strategy.Canary.TrafficRouting.ALB.ServicePort) @@ -347,7 +348,7 @@ func updateTargetGroupStatus(status *v1alpha1.ALBStatus, tg *aws.TargetGroupMeta } func getForwardActionString(r *v1alpha1.Rollout, port int32, desiredWeight int32, additionalDestinations ...v1alpha1.WeightDestination) (string, error) { - stableService, canaryService := trafficrouting.GetStableAndCanaryServices(r) + stableService, canaryService := trafficrouting.GetStableAndCanaryServices(r, true) portStr := strconv.Itoa(int(port)) stableWeight := int32(100) targetGroups := make([]ingressutil.ALBTargetGroup, 0) @@ -409,6 +410,9 @@ func getDesiredAnnotations(current *ingressutil.Ingress, r *v1alpha1.Rollout, po if err != nil { return nil, err } + if desired == nil { + desired = make(map[string]string) + } desired[key] = value return modifyManagedAnnotation(desired, r.Name, true, key) } @@ -479,7 +483,7 @@ func removeValue(array []string, key string) []string { } func getTrafficForwardActionString(r *v1alpha1.Rollout, port int32) (string, error) { - _, canaryService := trafficrouting.GetStableAndCanaryServices(r) + _, canaryService := trafficrouting.GetStableAndCanaryServices(r, true) portStr := strconv.Itoa(int(port)) weight := int64(100) targetGroups := make([]ingressutil.ALBTargetGroup, 0) diff --git a/rollout/trafficrouting/alb/alb_test.go b/rollout/trafficrouting/alb/alb_test.go index fb131849d4..562c3b599a 100644 --- a/rollout/trafficrouting/alb/alb_test.go +++ b/rollout/trafficrouting/alb/alb_test.go @@ -3,6 +3,7 @@ package alb import ( "context" "encoding/json" + "errors" "fmt" "strings" "testing" @@ -668,7 +669,7 @@ func TestErrorPatching(t *testing.T) { errMessage := "some error occurred" r.cfg.Client.(*fake.Clientset).Fake.AddReactor("patch", "ingresses", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, fmt.Errorf(errMessage) + return true, nil, errors.New(errMessage) }) err = r.SetWeight(10) @@ -700,7 +701,7 @@ func TestErrorPatchingMultiIngress(t *testing.T) { errMessage := "some error occurred" r.cfg.Client.(*fake.Clientset).Fake.AddReactor("patch", "ingresses", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, fmt.Errorf(errMessage) + return true, nil, errors.New(errMessage) }) err = r.SetWeight(10) @@ -918,6 +919,7 @@ func TestVerifyWeight(t *testing.T) { { var status v1alpha1.RolloutStatus r, fakeClient := newFakeReconciler(&status) + fakeClient.loadBalancers = []*elbv2types.LoadBalancer{ { LoadBalancerName: pointer.StringPtr("lb-abc123-name"), @@ -955,6 +957,48 @@ func TestVerifyWeight(t *testing.T) { assert.Equal(t, *status.ALB, *fakeClient.getAlbStatus("ingress")) } + // LoadBalancer found, at max weight, end of rollout + { + var status v1alpha1.RolloutStatus + status.CurrentStepIndex = pointer.Int32Ptr(2) + r, fakeClient := newFakeReconciler(&status) + fakeClient.loadBalancers = []*elbv2types.LoadBalancer{ + { + LoadBalancerName: pointer.StringPtr("lb-abc123-name"), + LoadBalancerArn: pointer.StringPtr("arn:aws:elasticloadbalancing:us-east-2:123456789012:loadbalancer/app/lb-abc123-name/1234567890123456"), + DNSName: pointer.StringPtr("verify-weight-test-abc-123.us-west-2.elb.amazonaws.com"), + }, + } + fakeClient.targetGroups = []aws.TargetGroupMeta{ + { + TargetGroup: elbv2types.TargetGroup{ + TargetGroupName: pointer.StringPtr("canary-tg-abc123-name"), + TargetGroupArn: pointer.StringPtr("arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/canary-tg-abc123-name/1234567890123456"), + }, + Weight: pointer.Int32Ptr(100), + Tags: map[string]string{ + aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-canary-svc:443", + }, + }, + { + TargetGroup: elbv2types.TargetGroup{ + TargetGroupName: pointer.StringPtr("stable-tg-abc123-name"), + TargetGroupArn: pointer.StringPtr("arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/stable-tg-abc123-name/1234567890123456"), + }, + Weight: pointer.Int32Ptr(0), + Tags: map[string]string{ + aws.AWSLoadBalancerV2TagKeyResourceID: "default/ingress-stable-svc:443", + }, + }, + } + + weightVerified, err := r.VerifyWeight(100) + assert.NoError(t, err) + assert.True(t, *weightVerified) + assert.Equal(t, status.ALBs[0], *status.ALB) + assert.Equal(t, *status.ALB, *fakeClient.getAlbStatus("ingress")) + } + // LoadBalancer found, but ARNs are unparsable { var status v1alpha1.RolloutStatus diff --git a/rollout/trafficrouting/ambassador/ambassador.go b/rollout/trafficrouting/ambassador/ambassador.go index 5c50f38f26..06466535f6 100644 --- a/rollout/trafficrouting/ambassador/ambassador.go +++ b/rollout/trafficrouting/ambassador/ambassador.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "strconv" "strings" "sync" "time" @@ -214,11 +213,12 @@ func (r *Reconciler) createCanaryMapping(ctx context.Context, if weight != 0 { msg := fmt.Sprintf("Ambassador mapping %q can not define weight", baseMappingName) r.sendWarningEvent(AmbassadorMappingConfigError, msg) - return fmt.Errorf(msg) + return errors.New(msg) } canarySvc := r.Rollout.Spec.Strategy.Canary.CanaryService - canaryMapping := buildCanaryMapping(baseMapping, canarySvc, desiredWeight) + stableService := r.Rollout.Spec.Strategy.Canary.StableService + canaryMapping := buildCanaryMapping(baseMapping, canarySvc, stableService, desiredWeight) _, err = client.Create(ctx, canaryMapping, metav1.CreateOptions{}) if err != nil { msg := fmt.Sprintf("Error creating canary mapping: %s", err) @@ -227,9 +227,9 @@ func (r *Reconciler) createCanaryMapping(ctx context.Context, return err } -func buildCanaryMapping(baseMapping *unstructured.Unstructured, canarySvc string, desiredWeight int32) *unstructured.Unstructured { +func buildCanaryMapping(baseMapping *unstructured.Unstructured, canarySvc string, stableService string, desiredWeight int32) *unstructured.Unstructured { canaryMapping := baseMapping.DeepCopy() - svc := buildCanaryService(baseMapping, canarySvc) + svc := buildCanaryService(baseMapping, canarySvc, stableService) unstructured.RemoveNestedField(canaryMapping.Object, "metadata") cMappingName := buildCanaryMappingName(baseMapping.GetName()) canaryMapping.SetName(cMappingName) @@ -239,19 +239,9 @@ func buildCanaryMapping(baseMapping *unstructured.Unstructured, canarySvc string return canaryMapping } -func buildCanaryService(baseMapping *unstructured.Unstructured, canarySvc string) string { +func buildCanaryService(baseMapping *unstructured.Unstructured, canarySvc string, stableService string) string { curSvc := GetMappingService(baseMapping) - parts := strings.Split(curSvc, ":") - if len(parts) < 2 { - return canarySvc - } - // Check if the last part is a valid int that can be used as the port - port := parts[len(parts)-1] - if _, err := strconv.Atoi(port); err != nil { - return canarySvc - - } - return fmt.Sprintf("%s:%s", canarySvc, port) + return strings.Replace(curSvc, stableService, canarySvc, 1) } func (r *Reconciler) VerifyWeight(desiredWeight int32, additionalDestinations ...v1alpha1.WeightDestination) (*bool, error) { diff --git a/rollout/trafficrouting/ambassador/ambassador_test.go b/rollout/trafficrouting/ambassador/ambassador_test.go index 66d193b83a..ee1ac347e3 100644 --- a/rollout/trafficrouting/ambassador/ambassador_test.go +++ b/rollout/trafficrouting/ambassador/ambassador_test.go @@ -31,7 +31,7 @@ metadata: spec: prefix: /myapp/ rewrite: /myapp/ - service: myapp:8080` + service: main-service:8080` baseMappingNoPort = ` apiVersion: getambassador.io/v2 @@ -42,7 +42,7 @@ metadata: spec: prefix: /myapp/ rewrite: /myapp/ - service: myapp` + service: main-service` baseMappingWithWeight = ` apiVersion: getambassador.io/v2 @@ -53,7 +53,7 @@ metadata: spec: prefix: /myapp/ rewrite: /myapp/ - service: myapp:8080 + service: main-service:8080 weight: 20` baseV3Mapping = ` @@ -66,7 +66,7 @@ spec: hostname: 'example.com' prefix: /myapp/ rewrite: /myapp/ - service: myapp:8080` + service: main-service:8080` canaryMapping = ` apiVersion: getambassador.io/v2 @@ -77,7 +77,7 @@ metadata: spec: prefix: /myapp/ rewrite: /myapp/ - service: myapp:8080 + service: main-service:8080 weight: 20` canaryMappingWithZeroWeight = ` @@ -89,7 +89,7 @@ metadata: spec: prefix: /myapp/ rewrite: /myapp/ - service: myapp:8080 + service: main-service:8080 weight: 0` ) diff --git a/rollout/trafficrouting/apisix/apisix.go b/rollout/trafficrouting/apisix/apisix.go index 6269cc698e..fe121140b8 100644 --- a/rollout/trafficrouting/apisix/apisix.go +++ b/rollout/trafficrouting/apisix/apisix.go @@ -5,13 +5,14 @@ import ( "fmt" "time" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" - "github.com/argoproj/argo-rollouts/utils/record" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/utils/record" ) var controllerKind = v1alpha1.SchemeGroupVersion.WithKind("Rollout") diff --git a/rollout/trafficrouting/apisix/apisix_test.go b/rollout/trafficrouting/apisix/apisix_test.go index 3749e64d6d..0abde5e9db 100644 --- a/rollout/trafficrouting/apisix/apisix_test.go +++ b/rollout/trafficrouting/apisix/apisix_test.go @@ -5,12 +5,13 @@ import ( "fmt" "testing" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" - "github.com/argoproj/argo-rollouts/rollout/trafficrouting/apisix/mocks" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/rollout/trafficrouting/apisix/mocks" ) const SetHeaderRouteName = "set-header" diff --git a/rollout/trafficrouting/apisix/mocks/apisix.go b/rollout/trafficrouting/apisix/mocks/apisix.go index 2bba83c499..6c7a69e104 100644 --- a/rollout/trafficrouting/apisix/mocks/apisix.go +++ b/rollout/trafficrouting/apisix/mocks/apisix.go @@ -7,7 +7,6 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" - argoRecord "github.com/argoproj/argo-rollouts/utils/record" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -15,6 +14,8 @@ import ( "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/dynamic" "k8s.io/client-go/tools/record" + + argoRecord "github.com/argoproj/argo-rollouts/utils/record" ) type FakeDynamicClient struct { diff --git a/rollout/trafficrouting/appmesh/resource_client.go b/rollout/trafficrouting/appmesh/resource_client.go index 4dbc4bd8cc..653dd4c76f 100644 --- a/rollout/trafficrouting/appmesh/resource_client.go +++ b/rollout/trafficrouting/appmesh/resource_client.go @@ -4,10 +4,11 @@ import ( "context" "errors" - appmeshutil "github.com/argoproj/argo-rollouts/utils/appmesh" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/client-go/dynamic" + + appmeshutil "github.com/argoproj/argo-rollouts/utils/appmesh" ) type ResourceClient struct { diff --git a/rollout/trafficrouting/istio/istio.go b/rollout/trafficrouting/istio/istio.go index 5308d59a42..9fd9402c09 100644 --- a/rollout/trafficrouting/istio/istio.go +++ b/rollout/trafficrouting/istio/istio.go @@ -6,10 +6,14 @@ import ( "fmt" "strings" + "github.com/mitchellh/mapstructure" + appsv1 "k8s.io/api/apps/v1" + + replicasetutil "github.com/argoproj/argo-rollouts/utils/replicaset" + "github.com/argoproj/argo-rollouts/rollout/trafficrouting" jsonpatch "github.com/evanphx/json-patch/v5" - "github.com/mitchellh/mapstructure" log "github.com/sirupsen/logrus" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,7 +36,7 @@ const Type = "Istio" const SpecHttpNotFound = "spec.http not found" // NewReconciler returns a reconciler struct that brings the Virtual Service into the desired state -func NewReconciler(r *v1alpha1.Rollout, client dynamic.Interface, recorder record.EventRecorder, virtualServiceLister, destinationRuleLister dynamiclister.Lister) *Reconciler { +func NewReconciler(r *v1alpha1.Rollout, client dynamic.Interface, recorder record.EventRecorder, virtualServiceLister, destinationRuleLister dynamiclister.Lister, replicaSets []*appsv1.ReplicaSet) *Reconciler { return &Reconciler{ rollout: r, log: logutil.WithRollout(r), @@ -41,6 +45,7 @@ func NewReconciler(r *v1alpha1.Rollout, client dynamic.Interface, recorder recor recorder: recorder, virtualServiceLister: virtualServiceLister, destinationRuleLister: destinationRuleLister, + replicaSets: replicaSets, } } @@ -52,6 +57,7 @@ type Reconciler struct { recorder record.EventRecorder virtualServiceLister dynamiclister.Lister destinationRuleLister dynamiclister.Lister + replicaSets []*appsv1.ReplicaSet } type virtualServicePatch struct { @@ -125,7 +131,7 @@ func (patches virtualServicePatches) patchVirtualService(httpRoutes []any, tlsRo } func (r *Reconciler) generateVirtualServicePatches(rolloutVsvcRouteNames []string, httpRoutes []VirtualServiceHTTPRoute, rolloutVsvcTLSRoutes []v1alpha1.TLSRoute, tlsRoutes []VirtualServiceTLSRoute, rolloutVsvcTCPRoutes []v1alpha1.TCPRoute, tcpRoutes []VirtualServiceTCPRoute, desiredWeight int64, additionalDestinations ...v1alpha1.WeightDestination) virtualServicePatches { - stableSvc, canarySvc := trafficrouting.GetStableAndCanaryServices(r.rollout) + stableSvc, canarySvc := trafficrouting.GetStableAndCanaryServices(r.rollout, false) canarySubset := "" stableSubset := "" if r.rollout.Spec.Strategy.Canary.TrafficRouting.Istio.DestinationRule != nil { @@ -315,6 +321,17 @@ func (r *Reconciler) reconcileVirtualService(obj *unstructured.Unstructured, vsv } func (r *Reconciler) UpdateHash(canaryHash, stableHash string, additionalDestinations ...v1alpha1.WeightDestination) error { + // We need to check if the replicasets are ready here as well if we didn't define any services in the rollout + // See: https://github.com/argoproj/argo-rollouts/issues/2507 + if r.rollout.Spec.Strategy.Canary.CanaryService == "" && r.rollout.Spec.Strategy.Canary.StableService == "" { + + for _, rs := range r.replicaSets { + if *rs.Spec.Replicas > 0 && !replicasetutil.IsReplicaSetAvailable(rs) { + return fmt.Errorf("delaying destination rule switch: ReplicaSet %s not fully available", rs.Name) + } + } + } + dRuleSpec := r.rollout.Spec.Strategy.Canary.TrafficRouting.Istio.DestinationRule if dRuleSpec == nil { return nil @@ -718,7 +735,7 @@ func (r *Reconciler) reconcileVirtualServiceHeaderRoutes(virtualService v1alpha1 return err } - _, canarySvc := trafficrouting.GetStableAndCanaryServices(r.rollout) + _, canarySvc := trafficrouting.GetStableAndCanaryServices(r.rollout, false) if destRuleHost != "" { canarySvc = destRuleHost } @@ -1023,7 +1040,7 @@ func searchTcpRoute(tcpRoute v1alpha1.TCPRoute, istioTcpRoutes []VirtualServiceT // ValidateHTTPRoutes ensures that all the routes in the rollout exist func ValidateHTTPRoutes(r *v1alpha1.Rollout, routeNames []string, httpRoutes []VirtualServiceHTTPRoute) error { - stableSvc, canarySvc := trafficrouting.GetStableAndCanaryServices(r) + stableSvc, canarySvc := trafficrouting.GetStableAndCanaryServices(r, false) routeIndexesToPatch, err := getHttpRouteIndexesToPatch(routeNames, httpRoutes) if err != nil { @@ -1060,7 +1077,7 @@ func ValidateHTTPRoutes(r *v1alpha1.Rollout, routeNames []string, httpRoutes []V // ValidateTlsRoutes ensures that all the routes in the rollout exist and they only have two destinations func ValidateTlsRoutes(r *v1alpha1.Rollout, vsvcTLSRoutes []v1alpha1.TLSRoute, tlsRoutes []VirtualServiceTLSRoute) error { - stableSvc, canarySvc := trafficrouting.GetStableAndCanaryServices(r) + stableSvc, canarySvc := trafficrouting.GetStableAndCanaryServices(r, false) routeIndexesToPatch, err := getTlsRouteIndexesToPatch(vsvcTLSRoutes, tlsRoutes) if err != nil { @@ -1081,7 +1098,7 @@ func ValidateTlsRoutes(r *v1alpha1.Rollout, vsvcTLSRoutes []v1alpha1.TLSRoute, t // ValidateTcpRoutes ensures that all the routes in the rollout exist and they only have two destinations func ValidateTcpRoutes(r *v1alpha1.Rollout, vsvcTCPRoutes []v1alpha1.TCPRoute, tcpRoutes []VirtualServiceTCPRoute) error { - stableSvc, canarySvc := trafficrouting.GetStableAndCanaryServices(r) + stableSvc, canarySvc := trafficrouting.GetStableAndCanaryServices(r, false) routeIndexesToPatch, err := getTcpRouteIndexesToPatch(vsvcTCPRoutes, tcpRoutes) if err != nil { @@ -1189,7 +1206,7 @@ func (r *Reconciler) reconcileVirtualServiceMirrorRoutes(virtualService v1alpha1 if err != nil { return fmt.Errorf("[reconcileVirtualServiceMirrorRoutes] failed to get destination rule host: %w", err) } - _, canarySvc := trafficrouting.GetStableAndCanaryServices(r.rollout) + _, canarySvc := trafficrouting.GetStableAndCanaryServices(r.rollout, false) if destRuleHost != "" { canarySvc = destRuleHost } diff --git a/rollout/trafficrouting/istio/istio_test.go b/rollout/trafficrouting/istio/istio_test.go index c6c0f8d9a1..32050fe2b4 100644 --- a/rollout/trafficrouting/istio/istio_test.go +++ b/rollout/trafficrouting/istio/istio_test.go @@ -4,12 +4,15 @@ import ( "context" "encoding/json" "fmt" + "testing" "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/client-go/dynamic" "k8s.io/client-go/dynamic/dynamicinformer" "k8s.io/client-go/dynamic/dynamiclister" @@ -817,7 +820,7 @@ func TestHttpReconcileHeaderRouteHostBased(t *testing.T) { obj := unstructuredutil.StrToUnstructuredUnsafe(regularVsvc) client := testutil.NewFakeDynamicClient(obj) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() const headerName = "test-header-route" @@ -892,7 +895,7 @@ spec: obj := unstructuredutil.StrToUnstructuredUnsafe(singleRouteSubsetVsvc) client := testutil.NewFakeDynamicClient(obj, dRule) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() const headerName = "test-header-route" @@ -933,7 +936,7 @@ func TestHttpReconcileHeaderRouteWithExtra(t *testing.T) { obj := unstructuredutil.StrToUnstructuredUnsafe(regularVsvcWithExtra) client := testutil.NewFakeDynamicClient(obj) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() const headerName = "test-header-route" @@ -1035,7 +1038,7 @@ func AssertReconcileUpdateHeader(t *testing.T, vsvc string, ro *v1alpha1.Rollout obj := unstructuredutil.StrToUnstructuredUnsafe(vsvc) client := testutil.NewFakeDynamicClient(obj) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() var setHeader = &v1alpha1.SetHeaderRoute{ @@ -1303,7 +1306,7 @@ func AssertReconcileUpdateVirtualService(t *testing.T, vsvc string, ro *v1alpha1 obj := unstructuredutil.StrToUnstructuredUnsafe(vsvc) client := testutil.NewFakeDynamicClient(obj) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() err := r.SetWeight(10) assert.Nil(t, err) @@ -1317,7 +1320,7 @@ func TestReconcileNoChanges(t *testing.T) { obj := unstructuredutil.StrToUnstructuredUnsafe(regularVsvc) client := testutil.NewFakeDynamicClient(obj) ro := rolloutWithHttpRoutes("stable", "canary", "vsvc", []string{"primary"}) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), nil, nil) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), nil, nil, nil) err := r.SetWeight(0) assert.Nil(t, err) assert.Len(t, client.Actions(), 1) @@ -1328,7 +1331,7 @@ func TestReconcileVirtualServiceExperimentStep(t *testing.T) { obj := unstructuredutil.StrToUnstructuredUnsafe(regularVsvc) client := testutil.NewFakeDynamicClient(obj) ro := rolloutWithHttpRoutes("stable", "canary", "vsvc", []string{"primary"}) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), nil, nil) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), nil, nil, nil) additionalDestinations := []v1alpha1.WeightDestination{ { ServiceName: "exp-svc", @@ -1361,7 +1364,7 @@ func TestHostSplitExperimentStep(t *testing.T) { obj := unstructuredutil.StrToUnstructuredUnsafe(regularVsvc) client := testutil.NewFakeDynamicClient(obj) ro := rolloutWithHttpRoutes("stable", "canary", "vsvc", []string{"primary"}) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), nil, nil) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), nil, nil, nil) additionalDestinations := []v1alpha1.WeightDestination{ { ServiceName: "exp-svc", @@ -1395,7 +1398,7 @@ func TestTlsReconcileNoChanges(t *testing.T) { }, }, ) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), nil, nil) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), nil, nil, nil) err := r.SetWeight(0) assert.Nil(t, err) assert.Len(t, client.Actions(), 1) @@ -1412,7 +1415,7 @@ func TestTcpReconcileNoChanges(t *testing.T) { }, }, ) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), nil, nil) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), nil, nil, nil) err := r.SetWeight(0) assert.Nil(t, err) assert.Len(t, client.Actions(), 1) @@ -1475,7 +1478,7 @@ func TestReconcileInvalidValidation(t *testing.T) { client := testutil.NewFakeDynamicClient(obj) ro := rolloutWithHttpRoutes("stable", "canary", "vsvc", []string{"route-not-found"}) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() err := r.SetWeight(0) assert.Equal(t, "HTTP Route 'route-not-found' is not found in the defined Virtual Service.", err.Error()) @@ -1492,7 +1495,7 @@ func TestTlsReconcileInvalidValidation(t *testing.T) { }, ) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() err := r.SetWeight(0) assert.Equal(t, NoTlsRouteFoundError, err.Error()) @@ -1509,7 +1512,7 @@ func TestTcpReconcileInvalidValidation(t *testing.T) { }, ) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() err := r.SetWeight(0) assert.Equal(t, NoTcpRouteFoundError, err.Error()) @@ -1519,7 +1522,7 @@ func TestReconcileVirtualServiceNotFound(t *testing.T) { client := testutil.NewFakeDynamicClient() ro := rolloutWithHttpRoutes("stable", "canary", "vsvc", []string{"primary"}) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() err := r.SetWeight(10) assert.NotNil(t, err) @@ -1532,7 +1535,7 @@ func TestReconcileAmbiguousRoutes(t *testing.T) { client := testutil.NewFakeDynamicClient(obj) ro := rolloutWithHttpRoutes("stable", "canary", "vsvc", nil) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() err := r.SetWeight(0) assert.Equal(t, "spec.http[] should be set in VirtualService and it must have exactly one route when omitting spec.strategy.canary.trafficRouting.istio.virtualService.routes", err.Error()) @@ -1543,7 +1546,7 @@ func TestTlsReconcileAmbiguousRoutes(t *testing.T) { client := testutil.NewFakeDynamicClient(obj) ro := rolloutWithHttpRoutes("stable", "canary", "vsvc", nil) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() err := r.SetWeight(0) assert.Equal(t, "spec.tls[] should be set in VirtualService and it must have exactly one route when omitting spec.strategy.canary.trafficRouting.istio.virtualService.tlsRoutes", err.Error()) @@ -1554,7 +1557,7 @@ func TestTcpReconcileAmbiguousRoutes(t *testing.T) { client := testutil.NewFakeDynamicClient(obj) ro := rolloutWithTcpRoutes("stable", "canary", "vsvc", nil) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() err := r.SetWeight(0) assert.Equal(t, "spec.tcp[] should be set in VirtualService and it must have exactly one route when omitting spec.strategy.canary.trafficRouting.istio.virtualService.tcpRoutes", err.Error()) @@ -1654,7 +1657,7 @@ func TestReconcileInferredSingleRoute(t *testing.T) { func TestType(t *testing.T) { client := testutil.NewFakeDynamicClient() ro := rolloutWithHttpRoutes("stable", "canary", "vsvc", []string{"primary"}) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), nil, nil) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), nil, nil, nil) assert.Equal(t, Type, r.Type()) } @@ -2023,7 +2026,7 @@ spec: `) client := testutil.NewFakeDynamicClient(obj) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() err := r.UpdateHash("abc123", "def456") @@ -2070,7 +2073,7 @@ spec: `) client := testutil.NewFakeDynamicClient(obj) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() err := r.UpdateHash("abc123", "def456") @@ -2111,7 +2114,7 @@ spec: `) client := testutil.NewFakeDynamicClient(obj) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() err := r.UpdateHash("abc123", "def456") @@ -2135,7 +2138,7 @@ spec: - name: canary `) client := testutil.NewFakeDynamicClient(obj) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), nil, nil) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), nil, nil, nil) client.ClearActions() err := r.UpdateHash("abc123", "def456") @@ -2158,7 +2161,7 @@ func TestUpdateHashDestinationRuleNotFound(t *testing.T) { ro := rolloutWithDestinationRule() client := testutil.NewFakeDynamicClient() vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() err := r.UpdateHash("abc123", "def456") @@ -2182,7 +2185,7 @@ spec: `) client := testutil.NewFakeDynamicClient(obj) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() // UpdateHash for 1 additional destination @@ -2212,7 +2215,7 @@ spec: // Add another additionalDestination client = testutil.NewFakeDynamicClient(dRuleUn) vsvcLister, druleLister = getIstioListers(client) - r = NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r = NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() additionalDestinations = append(additionalDestinations, v1alpha1.WeightDestination{ ServiceName: "exp-svc2", @@ -2236,7 +2239,7 @@ spec: // Remove 1 of additionalDestinations client = testutil.NewFakeDynamicClient(dRuleUn) vsvcLister, druleLister = getIstioListers(client) - r = NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r = NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() err = r.UpdateHash("abc123", "def456", additionalDestinations[1]) assert.NoError(t, err) @@ -2471,7 +2474,7 @@ func TestMultipleVirtualServiceReconcileNoChanges(t *testing.T) { client := testutil.NewFakeDynamicClient(obj1, obj2) multipleVirtualService := []v1alpha1.IstioVirtualService{{Name: "vsvc1", Routes: []string{"primary", "secondary"}}, {Name: "vsvc2", Routes: []string{"blue-green"}}} ro := multiVsRollout("stable", "canary", multipleVirtualService) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), nil, nil) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), nil, nil, nil) err := r.SetWeight(0) assert.Nil(t, err) assert.Len(t, client.Actions(), 2) @@ -2486,7 +2489,7 @@ func TestMultipleVirtualServiceReconcileUpdateVirtualServices(t *testing.T) { multipleVirtualService := []v1alpha1.IstioVirtualService{{Name: "vsvc1", Routes: []string{"primary", "secondary"}}, {Name: "vsvc2", Routes: []string{"blue-green"}}} ro := multiVsRollout("stable", "canary", multipleVirtualService) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() err := r.SetWeight(10) assert.Nil(t, err) @@ -2503,7 +2506,7 @@ func TestMultipleVirtualServiceReconcileInvalidValidation(t *testing.T) { multipleVirtualService := []v1alpha1.IstioVirtualService{{Name: "vsvc1", Routes: []string{"route-not-found"}}, {Name: "vsvc2", Routes: []string{"route-not-found"}}} ro := multiVsRollout("stable", "canary", multipleVirtualService) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() err := r.SetWeight(0) assert.Equal(t, "HTTP Route 'route-not-found' is not found in the defined Virtual Service.", err.Error()) @@ -2515,7 +2518,7 @@ func TestMultipleVirtualServiceReconcileVirtualServiceNotFound(t *testing.T) { multipleVirtualService := []v1alpha1.IstioVirtualService{{Name: "vsvc1", Routes: []string{"primary", "secondary"}}, {Name: "vsvc2", Routes: []string{"blue-green"}}} ro := multiVsRollout("stable", "canary", multipleVirtualService) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() err := r.SetWeight(10) assert.NotNil(t, err) @@ -2530,7 +2533,7 @@ func TestMultipleVirtualServiceReconcileAmbiguousRoutes(t *testing.T) { multipleVirtualService := []v1alpha1.IstioVirtualService{{Name: "vsvc1", Routes: nil}, {Name: "vsvc2", Routes: nil}} ro := multiVsRollout("stable", "canary", multipleVirtualService) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() err := r.SetWeight(0) assert.Equal(t, "spec.http[] should be set in VirtualService and it must have exactly one route when omitting spec.strategy.canary.trafficRouting.istio.virtualService.routes", err.Error()) @@ -2544,7 +2547,7 @@ func TestMultipleVirtualServiceReconcileInferredSingleRoute(t *testing.T) { multipleVirtualService := []v1alpha1.IstioVirtualService{{Name: "vsvc1", Routes: nil}, {Name: "vsvc2", Routes: nil}} ro := multiVsRollout("stable", "canary", multipleVirtualService) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() err := r.SetWeight(10) assert.NoError(t, err) @@ -2569,7 +2572,7 @@ func TestHttpReconcileMirrorRoute(t *testing.T) { obj := unstructuredutil.StrToUnstructuredUnsafe(regularVsvc) client := testutil.NewFakeDynamicClient(obj) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() // Test for both the HTTP VS & Mixed VS @@ -2661,7 +2664,7 @@ func TestSingleTlsRouteReconcile(t *testing.T) { obj := unstructuredutil.StrToUnstructuredUnsafe(singleRouteTlsVsvc) client := testutil.NewFakeDynamicClient(obj) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() err := r.SetWeight(30, v1alpha1.WeightDestination{}) @@ -2685,7 +2688,7 @@ func TestHttpReconcileMirrorRouteWithExtraFields(t *testing.T) { obj := unstructuredutil.StrToUnstructuredUnsafe(regularVsvcWithExtra) client := testutil.NewFakeDynamicClient(obj) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() // Test for both the HTTP VS & Mixed VS @@ -2737,7 +2740,7 @@ func TestHttpReconcileMirrorRouteOrder(t *testing.T) { obj := unstructuredutil.StrToUnstructuredUnsafe(regularVsvc) client := testutil.NewFakeDynamicClient(obj) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() setMirror1 := &v1alpha1.SetMirrorRoute{ @@ -2825,7 +2828,7 @@ func TestHttpReconcileMirrorRouteOrderSingleRouteNoName(t *testing.T) { obj := unstructuredutil.StrToUnstructuredUnsafe(singleRouteVsvc) client := testutil.NewFakeDynamicClient(obj) _, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), nil, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), nil, druleLister, nil) client.ClearActions() setMirror1 := &v1alpha1.SetMirrorRoute{ @@ -2945,7 +2948,7 @@ spec: obj := unstructuredutil.StrToUnstructuredUnsafe(singleRouteSubsetVsvc) client := testutil.NewFakeDynamicClient(obj, dRule) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() // Test for both the HTTP VS & Mixed VS @@ -3006,7 +3009,7 @@ func AssertReconcileUpdateMirror(t *testing.T, vsvc string, ro *v1alpha1.Rollout obj := unstructuredutil.StrToUnstructuredUnsafe(vsvc) client := testutil.NewFakeDynamicClient(obj) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() setMirror := &v1alpha1.SetMirrorRoute{ @@ -3031,7 +3034,7 @@ func TestReconcileHeaderRouteAvoidDuplicates(t *testing.T) { obj := unstructuredutil.StrToUnstructuredUnsafe(regularVsvc) client := testutil.NewFakeDynamicClient(obj) vsvcLister, druleLister := getIstioListers(client) - r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister) + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, nil) client.ClearActions() const headerName = "test-header-route" @@ -3068,3 +3071,88 @@ func TestReconcileHeaderRouteAvoidDuplicates(t *testing.T) { assert.Equal(t, httpRoutes[1].Name, "primary") assert.Equal(t, httpRoutes[2].Name, "secondary") } + +// TestUpdateHashNoReadyReplicaSets verifies we don't change rules when the destination ReplicaSets are not fully Ready yet +func TestUpdateHashNoReadyReplicaSets(t *testing.T) { + ro := rolloutWithDestinationRule() + + client := testutil.NewFakeDynamicClient() + vsvcLister, druleLister := getIstioListers(client) + + replicaSets := []*appsv1.ReplicaSet{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "ReplicaSetForTesting", + UID: uuid.NewUUID(), + Namespace: metav1.NamespaceDefault, + }, + Spec: appsv1.ReplicaSetSpec{ + Replicas: func() *int32 { i := int32(1); return &i }(), + Template: ro.Spec.Template, + }, + Status: appsv1.ReplicaSetStatus{ + Replicas: 1, + AvailableReplicas: 0, + }, + }, + } + + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, replicaSets) + client.ClearActions() + + err := r.UpdateHash("abc123", "def456") + assert.Error(t, err, "delaying destination rule switch: ReplicaSet ReplicaSetForTesting not fully available") + actions := client.Actions() + assert.Len(t, actions, 0) +} + +// TestUpdateHashReadyReplicaSets verifies we do change rules when the destination ReplicaSets are fully Ready +func TestUpdateHashReadyReplicaSets(t *testing.T) { + ro := rolloutWithDestinationRule() + + obj := unstructuredutil.StrToUnstructuredUnsafe(` +apiVersion: networking.istio.io/v1alpha3 +kind: DestinationRule +metadata: + name: istio-destrule + namespace: default + annotations: + argo-rollouts.argoproj.io/managed-by-rollouts: rollout +spec: + host: ratings.prod.svc.cluster.local + subsets: + - name: stable + labels: + rollouts-pod-template-hash: def456 + - name: canary + labels: + rollouts-pod-template-hash: abc123 +`) + client := testutil.NewFakeDynamicClient(obj) + vsvcLister, druleLister := getIstioListers(client) + replicaSets := []*appsv1.ReplicaSet{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "ReplicaSetForTesting", + UID: uuid.NewUUID(), + Namespace: metav1.NamespaceDefault, + }, + Spec: appsv1.ReplicaSetSpec{ + Replicas: func() *int32 { i := int32(1); return &i }(), + Template: ro.Spec.Template, + }, + Status: appsv1.ReplicaSetStatus{ + Replicas: 1, + AvailableReplicas: 1, + }, + }, + } + + r := NewReconciler(ro, client, record.NewFakeEventRecorder(), vsvcLister, druleLister, replicaSets) + client.ClearActions() + + err := r.UpdateHash("abc123", "def456") + assert.NoError(t, err) + actions := client.Actions() + assert.Len(t, actions, 0) +} diff --git a/rollout/trafficrouting/istio/istio_types.go b/rollout/trafficrouting/istio/istio_types.go index 9544efc49e..aa58aff6ec 100644 --- a/rollout/trafficrouting/istio/istio_types.go +++ b/rollout/trafficrouting/istio/istio_types.go @@ -1,8 +1,9 @@ package istio import ( - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" ) // VirtualService is an Istio VirtualService containing only the fields which we care about diff --git a/rollout/trafficrouting/nginx/nginx.go b/rollout/trafficrouting/nginx/nginx.go index 649931c258..73227fa75c 100644 --- a/rollout/trafficrouting/nginx/nginx.go +++ b/rollout/trafficrouting/nginx/nginx.go @@ -129,6 +129,12 @@ func (r *Reconciler) buildCanaryIngress(stableIngress *networkingv1.Ingress, nam } desiredCanaryIngress.Annotations[k] = v } + + // Process additional full annotations, overwriting any colliding values from the above + for k, v := range r.cfg.Rollout.Spec.Strategy.Canary.TrafficRouting.Nginx.CanaryIngressAnnotations { + desiredCanaryIngress.Annotations[k] = v + } + // Always set `canary` and `canary-weight` - `canary-by-header` and `canary-by-cookie`, if set, will always take precedence desiredCanaryIngress.Annotations[fmt.Sprintf("%s/canary", annotationPrefix)] = "true" desiredCanaryIngress.Annotations[fmt.Sprintf("%s/canary-weight", annotationPrefix)] = fmt.Sprintf("%d", desiredWeight) @@ -209,6 +215,12 @@ func (r *Reconciler) buildLegacyCanaryIngress(stableIngress *extensionsv1beta1.I } desiredCanaryIngress.Annotations[k] = v } + + // Process additional full annotations, overwriting any colliding values from the above + for k, v := range r.cfg.Rollout.Spec.Strategy.Canary.TrafficRouting.Nginx.CanaryIngressAnnotations { + desiredCanaryIngress.Annotations[k] = v + } + // Always set `canary` and `canary-weight` - `canary-by-header` and `canary-by-cookie`, if set, will always take precedence desiredCanaryIngress.Annotations[fmt.Sprintf("%s/canary", annotationPrefix)] = "true" desiredCanaryIngress.Annotations[fmt.Sprintf("%s/canary-weight", annotationPrefix)] = fmt.Sprintf("%d", desiredWeight) @@ -313,6 +325,7 @@ func (r *Reconciler) SetWeightPerIngress(desiredWeight int32, ingresses []string } // Make patches + desiredCanaryIngress.SetAnnotations(getDesiredAnnotations(canaryIngress, desiredCanaryIngress)) patch, modified, err := ingressutil.BuildIngressPatch(canaryIngress.Mode(), canaryIngress, desiredCanaryIngress, ingressutil.WithAnnotations(), ingressutil.WithLabels(), ingressutil.WithSpec()) @@ -359,3 +372,16 @@ func (r *Reconciler) SetMirrorRoute(setMirrorRoute *v1alpha1.SetMirrorRoute) err func (r *Reconciler) RemoveManagedRoutes() error { return nil } + +func getDesiredAnnotations(current, desired *ingressutil.Ingress) map[string]string { + // Merge existing annotations into the desired Ingress (giving precedence to the desired values) + // This is necessary because the desired Ingress may not have all annotations previously added + // by other controllers (e.g. Rancher) + desiredAnnotations := desired.GetAnnotations() + for k, v := range current.GetAnnotations() { + if _, ok := desiredAnnotations[k]; !ok { + desiredAnnotations[k] = v + } + } + return desiredAnnotations +} diff --git a/rollout/trafficrouting/nginx/nginx_test.go b/rollout/trafficrouting/nginx/nginx_test.go index c3b4ce357a..5d258a74d0 100644 --- a/rollout/trafficrouting/nginx/nginx_test.go +++ b/rollout/trafficrouting/nginx/nginx_test.go @@ -556,6 +556,9 @@ func TestCanaryIngressAdditionalAnnotations(t *testing.T) { "canary-by-header": "X-Foo", "canary-by-header-value": "DoCanary", } + r.cfg.Rollout.Spec.Strategy.Canary.TrafficRouting.Nginx.CanaryIngressAnnotations = map[string]string{ + "annotation-prefix.some.group/some-key": "some-value", + } stable := extensionsIngress(ing, 80, stableService) stableIngress := ingressutil.NewLegacyIngress(stable) @@ -569,6 +572,74 @@ func TestCanaryIngressAdditionalAnnotations(t *testing.T) { assert.Equal(t, "15", annotations["nginx.ingress.kubernetes.io/canary-weight"], "canary-weight annotation set to expected value") assert.Equal(t, "X-Foo", annotations["nginx.ingress.kubernetes.io/canary-by-header"], "canary-by-header annotation set") assert.Equal(t, "DoCanary", annotations["nginx.ingress.kubernetes.io/canary-by-header-value"], "canary-by-header-value annotation set") + assert.Equal(t, "some-value", annotations["annotation-prefix.some.group/some-key"], "custom full annotation set without default prefix") + } + }) + } +} + +func TestCanaryIngressAdditionalAnnotationsNewIngress(t *testing.T) { + tests := generateMultiIngressTestData() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + for _, ing := range test.ingresses { + r := Reconciler{ + cfg: ReconcilerConfig{ + Rollout: fakeRollout(stableService, canaryService, test.singleIngress, test.multiIngress), + }, + } + r.cfg.Rollout.Spec.Strategy.Canary.TrafficRouting.Nginx.AdditionalIngressAnnotations = map[string]string{ + "canary-by-header": "X-Foo", + "canary-by-header-value": "DoCanary", + } + r.cfg.Rollout.Spec.Strategy.Canary.TrafficRouting.Nginx.CanaryIngressAnnotations = map[string]string{ + "annotation-prefix.some.group/some-key": "some-value", + } + stable := networkingIngress(StableIngress, 80, stableService) + stableIngress := ingressutil.NewIngress(stable) + + desiredCanaryIngress, err := r.canaryIngress(stableIngress, ingressutil.GetCanaryIngressName(r.cfg.Rollout.GetName(), ing), 15) + assert.Nil(t, err, "No error returned when calling canaryIngress") + + checkBackendService(t, desiredCanaryIngress, canaryService) + + annotations := desiredCanaryIngress.GetAnnotations() + assert.Equal(t, "true", annotations["nginx.ingress.kubernetes.io/canary"], "canary annotation set to true") + assert.Equal(t, "15", annotations["nginx.ingress.kubernetes.io/canary-weight"], "canary-weight annotation set to expected value") + assert.Equal(t, "X-Foo", annotations["nginx.ingress.kubernetes.io/canary-by-header"], "canary-by-header annotation set") + assert.Equal(t, "DoCanary", annotations["nginx.ingress.kubernetes.io/canary-by-header-value"], "canary-by-header-value annotation set") + assert.Equal(t, "some-value", annotations["annotation-prefix.some.group/some-key"], "custom full annotation set without default prefix") + } + }) + } +} + +func TestCanaryIngressRetainCurrentAnnotations(t *testing.T) { + tests := generateMultiIngressTestData() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + for _, ing := range test.ingresses { + r := Reconciler{ + cfg: ReconcilerConfig{ + Rollout: fakeRollout(stableService, canaryService, test.singleIngress, test.multiIngress), + }, + } + stable := networkingIngress(StableIngress, 80, stableService) + stableIngress := ingressutil.NewIngress(stable) + canary := networkingIngress(ingressutil.GetCanaryIngressName(r.cfg.Rollout.GetName(), ing), 80, canaryService) + canary.SetAnnotations(map[string]string{ + "nginx.ingress.kubernetes.io/canary": "false", + "annotation-prefix.some.group/some-key": "some-value", + }) + canaryIngress := ingressutil.NewIngress(canary) + + desiredCanaryIngress, err := r.canaryIngress(stableIngress, ingressutil.GetCanaryIngressName(r.cfg.Rollout.GetName(), ing), 15) + assert.Nil(t, err, "No error returned when calling canaryIngress") + + annotations := getDesiredAnnotations(canaryIngress, desiredCanaryIngress) + assert.Equal(t, "true", annotations["nginx.ingress.kubernetes.io/canary"], "canary annotation set to true") + assert.Equal(t, "15", annotations["nginx.ingress.kubernetes.io/canary-weight"], "canary-weight annotation set to expected value") + assert.Equal(t, "some-value", annotations["annotation-prefix.some.group/some-key"], "existing canary annotation is retained") } }) } diff --git a/rollout/trafficrouting/plugin/client/client.go b/rollout/trafficrouting/plugin/client/client.go index a9a8b0b713..d1e6fc3731 100644 --- a/rollout/trafficrouting/plugin/client/client.go +++ b/rollout/trafficrouting/plugin/client/client.go @@ -5,9 +5,11 @@ import ( "os/exec" "sync" + goPlugin "github.com/hashicorp/go-plugin" + "github.com/argoproj/argo-rollouts/rollout/trafficrouting/plugin/rpc" "github.com/argoproj/argo-rollouts/utils/plugin" - goPlugin "github.com/hashicorp/go-plugin" + "github.com/argoproj/argo-rollouts/utils/plugin/types" ) type trafficPlugin struct { @@ -52,7 +54,7 @@ func (t *trafficPlugin) startPlugin(pluginName string) (rpc.TrafficRouterPlugin, if t.pluginClient[pluginName] == nil || t.pluginClient[pluginName].Exited() { - pluginPath, args, err := plugin.GetPluginInfo(pluginName) + pluginPath, args, err := plugin.GetPluginInfo(pluginName, types.PluginTypeTrafficRouter) if err != nil { return nil, fmt.Errorf("unable to find plugin (%s): %w", pluginName, err) } diff --git a/rollout/trafficrouting/plugin/plugin.go b/rollout/trafficrouting/plugin/plugin.go index 7ded9ad2f1..e9ca2bc09f 100644 --- a/rollout/trafficrouting/plugin/plugin.go +++ b/rollout/trafficrouting/plugin/plugin.go @@ -3,11 +3,12 @@ package plugin import ( "fmt" + "k8s.io/client-go/kubernetes" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" "github.com/argoproj/argo-rollouts/rollout/trafficrouting/plugin/client" "github.com/argoproj/argo-rollouts/rollout/trafficrouting/plugin/rpc" "github.com/argoproj/argo-rollouts/utils/record" - "k8s.io/client-go/kubernetes" ) type ReconcilerConfig struct { diff --git a/rollout/trafficrouting/plugin/rpc/rpc.go b/rollout/trafficrouting/plugin/rpc/rpc.go index 297ff9d21b..ba61cc9e46 100644 --- a/rollout/trafficrouting/plugin/rpc/rpc.go +++ b/rollout/trafficrouting/plugin/rpc/rpc.go @@ -7,8 +7,9 @@ import ( "github.com/argoproj/argo-rollouts/utils/plugin/types" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" "github.com/hashicorp/go-plugin" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" ) type UpdateHashArgs struct { diff --git a/rollout/trafficrouting/plugin/rpc/rpc_test.go b/rollout/trafficrouting/plugin/rpc/rpc_test.go index 5ca189e186..3de031a689 100644 --- a/rollout/trafficrouting/plugin/rpc/rpc_test.go +++ b/rollout/trafficrouting/plugin/rpc/rpc_test.go @@ -7,9 +7,10 @@ import ( "github.com/argoproj/argo-rollouts/utils/plugin/types" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" goPlugin "github.com/hashicorp/go-plugin" "github.com/tj/assert" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" ) var testHandshake = goPlugin.HandshakeConfig{ diff --git a/rollout/trafficrouting/service_helper.go b/rollout/trafficrouting/service_helper.go index 15d03cad53..8bae069fe7 100644 --- a/rollout/trafficrouting/service_helper.go +++ b/rollout/trafficrouting/service_helper.go @@ -7,17 +7,26 @@ import ( // GetStableAndCanaryServices return a service names for current stable and canary services. // If ping-pong feature enabled then the current ping or pong service will be returned. Which is a stable is defined // based on a rollout status field Status.Canary.StablePingPong -func GetStableAndCanaryServices(ro *v1alpha1.Rollout) (string, string) { - if IsPingPongEnabled(ro) { + +// isPingpongPreferred is needed when Rollout uses both pingpong service and stable/canary service +// for ALB trafficRouting, isPingpongPreferred is true. It uses pingpong service as priority +// for other trafficRouting, isPingpongPrefrered is false. It uses stable/canary service +// This is to ensure it is compatible with previous release. + +func GetStableAndCanaryServices(ro *v1alpha1.Rollout, isPingpongPreferred bool) (string, string) { + pingPongNotPreferredOtherServiceNotDefined := !isPingpongPreferred && ro.Spec.Strategy.Canary.StableService == "" && ro.Spec.Strategy.Canary.CanaryService == "" + if IsPingPongEnabled(ro) && + (isPingpongPreferred || pingPongNotPreferredOtherServiceNotDefined) { canary := ro.Spec.Strategy.Canary if IsStablePing(ro) { return canary.PingPong.PingService, canary.PingPong.PongService } else { return canary.PingPong.PongService, canary.PingPong.PingService } - } else { - return ro.Spec.Strategy.Canary.StableService, ro.Spec.Strategy.Canary.CanaryService } + + return ro.Spec.Strategy.Canary.StableService, ro.Spec.Strategy.Canary.CanaryService + } // IsStablePing return true if the 'ping' service is pointing to the stable replica set. diff --git a/rollout/trafficrouting/service_helper_test.go b/rollout/trafficrouting/service_helper_test.go new file mode 100644 index 0000000000..acb63673a4 --- /dev/null +++ b/rollout/trafficrouting/service_helper_test.go @@ -0,0 +1,83 @@ +package trafficrouting + +import ( + "testing" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" +) + +const PING_SVC = "ping-service" +const PONG_SVC = "pong-service" + +func fakeRollout(stableSvc, canarySvc string, pingPong *v1alpha1.PingPongSpec, stableIng string, port int32) *v1alpha1.Rollout { + return &v1alpha1.Rollout{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rollout", + Namespace: metav1.NamespaceDefault, + }, + Spec: v1alpha1.RolloutSpec{ + Strategy: v1alpha1.RolloutStrategy{ + Canary: &v1alpha1.CanaryStrategy{ + StableService: stableSvc, + CanaryService: canarySvc, + PingPong: pingPong, + TrafficRouting: &v1alpha1.RolloutTrafficRouting{ + ALB: &v1alpha1.ALBTrafficRouting{ + Ingress: stableIng, + ServicePort: port, + }, + Istio: &v1alpha1.IstioTrafficRouting{ + VirtualService: &v1alpha1.IstioVirtualService{ + Name: "istio-vsvc", + }, + DestinationRule: &v1alpha1.IstioDestinationRule{ + Name: "istio-destrule", + CanarySubsetName: "canary", + StableSubsetName: "stable", + }, + }, + }, + }, + }, + }, + } +} + +func TestGetStableAndCanaryServices(t *testing.T) { + // Rollout has no pingPong + rollout := fakeRollout("stable-service", "canary-service", nil, "stable-ingress", 443) + + stableService, canaryService := GetStableAndCanaryServices(rollout, true) + assert.Equal(t, "stable-service", stableService) + assert.Equal(t, "canary-service", canaryService) + + stableService, canaryService = GetStableAndCanaryServices(rollout, false) + assert.Equal(t, "stable-service", stableService) + assert.Equal(t, "canary-service", canaryService) + + // Rollout has pingPong and stable/canary + pp := &v1alpha1.PingPongSpec{PingService: PING_SVC, PongService: PONG_SVC} + rollout = fakeRollout("stable-service", "canary-service", pp, "stable-ingress", 443) + + stableService, canaryService = GetStableAndCanaryServices(rollout, true) + assert.Equal(t, PONG_SVC, stableService) + assert.Equal(t, PING_SVC, canaryService) + + stableService, canaryService = GetStableAndCanaryServices(rollout, false) + assert.Equal(t, "stable-service", stableService) + assert.Equal(t, "canary-service", canaryService) + + // Rollout has pingPong, no stable/canary + rollout = fakeRollout("", "", pp, "stable-ingress", 443) + + stableService, canaryService = GetStableAndCanaryServices(rollout, true) + assert.Equal(t, PONG_SVC, stableService) + assert.Equal(t, PING_SVC, canaryService) + + stableService, canaryService = GetStableAndCanaryServices(rollout, false) + assert.Equal(t, PONG_SVC, stableService) + assert.Equal(t, PING_SVC, canaryService) +} diff --git a/rollout/trafficrouting/traefik/mocks/traefik.go b/rollout/trafficrouting/traefik/mocks/traefik.go index b0cd361809..e53295c1b2 100644 --- a/rollout/trafficrouting/traefik/mocks/traefik.go +++ b/rollout/trafficrouting/traefik/mocks/traefik.go @@ -6,7 +6,6 @@ import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime" - argoRecord "github.com/argoproj/argo-rollouts/utils/record" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -14,6 +13,8 @@ import ( "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/dynamic" "k8s.io/client-go/tools/record" + + argoRecord "github.com/argoproj/argo-rollouts/utils/record" ) type FakeDynamicClient struct{} diff --git a/rollout/trafficrouting/traefik/traefik.go b/rollout/trafficrouting/traefik/traefik.go index 8b5cfb7312..3782ee824e 100644 --- a/rollout/trafficrouting/traefik/traefik.go +++ b/rollout/trafficrouting/traefik/traefik.go @@ -23,12 +23,6 @@ const Type = "Traefik" const traefikServices = "traefikservices" const TraefikServiceUpdateError = "TraefikServiceUpdateError" -var ( - apiGroupToResource = map[string]string{ - defaults.GetTraefikAPIGroup(): traefikServices, - } -) - type ReconcilerConfig struct { Rollout *v1alpha1.Rollout Client ClientInterface @@ -41,6 +35,13 @@ type Reconciler struct { Recorder record.EventRecorder } +func apiGroupToResource(group string) string { + apiGroupToResource := map[string]string{ + defaults.GetTraefikAPIGroup(): traefikServices, + } + return apiGroupToResource[group] +} + func (r *Reconciler) sendWarningEvent(id, msg string) { r.sendEvent(corev1.EventTypeWarning, id, msg) } @@ -71,7 +72,8 @@ func GetMappingGVR() schema.GroupVersionResource { group := defaults.GetTraefikAPIGroup() parts := strings.Split(defaults.GetTraefikVersion(), "/") version := parts[len(parts)-1] - resourceName := apiGroupToResource[group] + resourceName := apiGroupToResource(group) + return schema.GroupVersionResource{ Group: group, Version: version, diff --git a/rollout/trafficrouting/traefik/traefik_test.go b/rollout/trafficrouting/traefik/traefik_test.go index 0b5b0c9701..2fccaf9a9f 100644 --- a/rollout/trafficrouting/traefik/traefik_test.go +++ b/rollout/trafficrouting/traefik/traefik_test.go @@ -3,12 +3,13 @@ package traefik import ( "testing" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" - "github.com/argoproj/argo-rollouts/rollout/trafficrouting/traefik/mocks" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/rollout/trafficrouting/traefik/mocks" ) const traefikService = ` diff --git a/rollout/trafficrouting_test.go b/rollout/trafficrouting_test.go index f7dd459964..2b72fc95e3 100644 --- a/rollout/trafficrouting_test.go +++ b/rollout/trafficrouting_test.go @@ -7,6 +7,10 @@ import ( "testing" "time" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "github.com/argoproj/argo-rollouts/rollout/trafficrouting/apisix" "github.com/stretchr/testify/assert" @@ -29,8 +33,10 @@ import ( traefikMocks "github.com/argoproj/argo-rollouts/rollout/trafficrouting/traefik/mocks" testutil "github.com/argoproj/argo-rollouts/test/util" "github.com/argoproj/argo-rollouts/utils/conditions" + ingressutil "github.com/argoproj/argo-rollouts/utils/ingress" istioutil "github.com/argoproj/argo-rollouts/utils/istio" logutil "github.com/argoproj/argo-rollouts/utils/log" + replicasetutil "github.com/argoproj/argo-rollouts/utils/replicaset" timeutil "github.com/argoproj/argo-rollouts/utils/time" ) @@ -130,6 +136,53 @@ func TestReconcileTrafficRoutingVerifyWeightFalse(t *testing.T) { assert.True(t, enqueued) } +func TestReconcileTrafficRoutingVerifyWeightEndOfRollout(t *testing.T) { + f := newFixture(t) + defer f.Close() + + steps := []v1alpha1.CanaryStep{ + { + SetWeight: pointer.Int32Ptr(10), + }, + { + Pause: &v1alpha1.RolloutPause{}, + }, + } + r1 := newCanaryRollout("foo", 10, nil, steps, pointer.Int32Ptr(2), intstr.FromInt(1), intstr.FromInt(0)) + r2 := bumpVersion(r1) + r2.Spec.Strategy.Canary.TrafficRouting = &v1alpha1.RolloutTrafficRouting{} + r2.Spec.Strategy.Canary.CanaryService = "canary" + r2.Spec.Strategy.Canary.StableService = "stable" + + rs1 := newReplicaSetWithStatus(r1, 10, 10) + rs2 := newReplicaSetWithStatus(r2, 10, 10) + + rs1PodHash := rs1.Labels[v1alpha1.DefaultRolloutUniqueLabelKey] + rs2PodHash := rs2.Labels[v1alpha1.DefaultRolloutUniqueLabelKey] + canarySelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs2PodHash} + stableSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs1PodHash} + canarySvc := newService("canary", 80, canarySelector, r2) + stableSvc := newService("stable", 80, stableSelector, r2) + + f.kubeobjects = append(f.kubeobjects, rs1, rs2, canarySvc, stableSvc) + f.replicaSetLister = append(f.replicaSetLister, rs1, rs2) + + r2 = updateCanaryRolloutStatus(r2, rs1PodHash, 10, 0, 10, false) + f.rolloutLister = append(f.rolloutLister, r2) + f.objects = append(f.objects, r2) + + f.fakeTrafficRouting = newUnmockedFakeTrafficRoutingReconciler() + f.fakeTrafficRouting.On("UpdateHash", mock.Anything, mock.Anything, mock.Anything).Return(nil) + f.fakeTrafficRouting.On("SetWeight", mock.Anything, mock.Anything).Return(func(desiredWeight int32, additionalDestinations ...v1alpha1.WeightDestination) error { + // make sure SetWeight was called with correct value + assert.Equal(t, int32(100), desiredWeight) + return nil + }) + f.fakeTrafficRouting.On("SetHeaderRoute", mock.Anything, mock.Anything).Return(nil) + f.fakeTrafficRouting.On("VerifyWeight", mock.Anything).Return(pointer.BoolPtr(false), nil) + f.runExpectError(getKey(r2, t), true) +} + func TestRolloutUseDesiredWeight(t *testing.T) { f := newFixture(t) defer f.Close() @@ -1085,6 +1138,34 @@ func TestRolloutReplicaIsAvailableAndGenerationNotBeModifiedShouldModifyVirtualS }, } r1 := newCanaryRollout("foo", 1, nil, steps, pointer.Int32(1), intstr.FromInt(1), intstr.FromInt(1)) + vs := istio.VirtualService{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: r1.Namespace, + }, + Spec: istio.VirtualServiceSpec{ + HTTP: []istio.VirtualServiceHTTPRoute{{ + Name: "primary", + Route: []istio.VirtualServiceRouteDestination{{ + Destination: istio.VirtualServiceDestination{ + Host: "stable", + //Port: &istio.Port{ + // Number: 80, + //}, + }, + Weight: 100, + }, { + Destination: istio.VirtualServiceDestination{ + Host: "canary", + //Port: &istio.Port{ + // Number: 80, + //}, + }, + Weight: 0, + }}, + }}, + }, + } r1.Spec.Strategy.Canary.TrafficRouting = &v1alpha1.RolloutTrafficRouting{ Istio: &v1alpha1.IstioTrafficRouting{ VirtualService: &v1alpha1.IstioVirtualService{ @@ -1093,11 +1174,6 @@ func TestRolloutReplicaIsAvailableAndGenerationNotBeModifiedShouldModifyVirtualS "primary", }, }, - DestinationRule: &v1alpha1.IstioDestinationRule{ - Name: "test", - StableSubsetName: "stable", - CanarySubsetName: "canary", - }, }, ManagedRoutes: []v1alpha1.MangedRoutes{ { @@ -1110,8 +1186,14 @@ func TestRolloutReplicaIsAvailableAndGenerationNotBeModifiedShouldModifyVirtualS APIVersion: "apps/v1", Kind: "Deployment", } - r1.Spec.SelectorResolvedFromRef = true + r1.Spec.SelectorResolvedFromRef = false r1.Spec.TemplateResolvedFromRef = true + r1.Spec.Strategy.Canary.CanaryService = "canary" + r1.Spec.Strategy.Canary.StableService = "stable" + r1.Spec.Selector = &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + } + r1.Labels = map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: "testsha"} r2 := bumpVersion(r1) // if set WorkloadRef it does not change the generation @@ -1138,13 +1220,26 @@ func TestRolloutReplicaIsAvailableAndGenerationNotBeModifiedShouldModifyVirtualS PodTemplateHash: rs1PodHash, }, } + + mapObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&vs) + assert.Nil(t, err) + + unstructuredObj := &unstructured.Unstructured{Object: mapObj} + unstructuredObj.SetGroupVersionKind(schema.GroupVersionKind{ + Group: istioutil.GetIstioVirtualServiceGVR().Group, + Version: istioutil.GetIstioVirtualServiceGVR().Version, + Kind: "VirtualService", + }) + f.kubeobjects = append(f.kubeobjects, rs1, rs2, canarySvc, stableSvc) + f.serviceLister = append(f.serviceLister, canarySvc, stableSvc) f.replicaSetLister = append(f.replicaSetLister, rs1, rs2) f.rolloutLister = append(f.rolloutLister, r2) - f.objects = append(f.objects, r2) + f.objects = append(f.objects, r2, unstructuredObj) + f.expectUpdateRolloutAction(r2) + f.expectUpdateRolloutStatusAction(r2) f.expectPatchRolloutAction(r2) - f.expectPatchReplicaSetAction(rs1) - f.expectPatchReplicaSetAction(rs2) + f.expectCreateReplicaSetAction(rs2) f.fakeTrafficRouting = newUnmockedFakeTrafficRoutingReconciler() f.fakeTrafficRouting.On("SetHeaderRoute", &v1alpha1.SetHeaderRoute{ Name: "test-header", @@ -1250,3 +1345,156 @@ func TestDontWeightToZeroWhenDynamicallyRollingBackToStable(t *testing.T) { rs1Updated := f.getUpdatedReplicaSet(scaleUpIndex) assert.Equal(t, int32(10), *rs1Updated.Spec.Replicas) } + +// TestDontWeightOrHaveManagedRoutesDuringInterruptedUpdate builds off of TestCanaryDontScaleDownOldRsDuringInterruptedUpdate +// in canary_test when we scale down an intermediate V2 ReplicaSet when applying a V3 spec in the middle of updating. +// We want to make sure that traffic routing is cleared in both weight AND managed routes when the V2 rs is +// nil or has 0 available replicas. +func TestDontWeightOrHaveManagedRoutesDuringInterruptedUpdate(t *testing.T) { + f := newFixture(t) + defer f.Close() + + steps := []v1alpha1.CanaryStep{ + { + SetHeaderRoute: &v1alpha1.SetHeaderRoute{ + Name: "test-header", + Match: []v1alpha1.HeaderRoutingMatch{ + { + HeaderName: "test", + HeaderValue: &v1alpha1.StringMatch{ + Exact: "test", + }, + }, + }, + }, + }, + { + SetWeight: pointer.Int32(90), + }, + { + Pause: &v1alpha1.RolloutPause{}, + }, + } + r1 := newCanaryRollout("foo", 5, nil, steps, pointer.Int32Ptr(1), intstr.FromInt(1), intstr.FromInt(0)) + r1.Spec.Strategy.Canary.TrafficRouting = &v1alpha1.RolloutTrafficRouting{ + ALB: &v1alpha1.ALBTrafficRouting{ + Ingress: "test-ingress", + }, + ManagedRoutes: []v1alpha1.MangedRoutes{ + {Name: "test-header"}, + }, + } + + r1.Spec.Strategy.Canary.StableService = "stable-svc" + r1.Spec.Strategy.Canary.CanaryService = "canary-svc" + r2 := bumpVersion(r1) + r3 := bumpVersion(r2) + + stableSvc := newService("stable-svc", 80, map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: r1.Status.CurrentPodHash}, r1) + canarySvc := newService("canary-svc", 80, map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: r3.Status.CurrentPodHash}, r3) + r3.Status.StableRS = r1.Status.CurrentPodHash + + ingress := newIngress("test-ingress", canarySvc, stableSvc) + ingress.Spec.Rules[0].HTTP.Paths[0].Backend.ServiceName = stableSvc.Name + + rs1 := newReplicaSetWithStatus(r1, 5, 5) + rs2 := newReplicaSetWithStatus(r2, 5, 5) + rs3 := newReplicaSetWithStatus(r3, 5, 0) + r3.Status.Canary.Weights = &v1alpha1.TrafficWeights{ + Canary: v1alpha1.WeightDestination{ + PodTemplateHash: replicasetutil.GetPodTemplateHash(rs2), + }, + } + + f.objects = append(f.objects, r3) + f.kubeobjects = append(f.kubeobjects, rs1, rs2, rs3, canarySvc, stableSvc, ingress) + f.replicaSetLister = append(f.replicaSetLister, rs1, rs2, rs3) + f.serviceLister = append(f.serviceLister, canarySvc, stableSvc) + f.ingressLister = append(f.ingressLister, ingressutil.NewLegacyIngress(ingress)) + + f.expectPatchRolloutAction(r3) + f.run(getKey(r3, t)) + + r3.Status.Canary.Weights = &v1alpha1.TrafficWeights{ + Canary: v1alpha1.WeightDestination{ + PodTemplateHash: replicasetutil.GetPodTemplateHash(rs3), + }, + } + + f.expectUpdateReplicaSetAction(rs3) + f.run(getKey(r3, t)) + + // Make sure that our weight is zero + assert.Equal(t, int32(0), r3.Status.Canary.Weights.Canary.Weight) + assert.Equal(t, replicasetutil.GetPodTemplateHash(rs3), r3.Status.Canary.Weights.Canary.PodTemplateHash) + // Make sure that RemoveManagedRoutes was called + f.fakeTrafficRouting.AssertCalled(t, "RemoveManagedRoutes", mock.Anything, mock.Anything) + +} + +// This test verifies that if we are shifting traffic to stable replicaset without the stable replicaset being available proportional to the weight, the traffic shouldn't be switched immediately to the stable replicaset. +func TestCheckReplicaSetAvailable(t *testing.T) { + fix := newFixture(t) + defer fix.Close() + + steps := []v1alpha1.CanaryStep{ + { + SetWeight: pointer.Int32(60), + }, + { + Pause: &v1alpha1.RolloutPause{}, + }, + } + + rollout1 := newCanaryRollout("test-rollout", 10, nil, steps, pointer.Int32(1), intstr.FromInt(1), intstr.FromInt(1)) + rollout1.Spec.Strategy.Canary.DynamicStableScale = true + rollout1.Spec.Strategy.Canary.TrafficRouting = &v1alpha1.RolloutTrafficRouting{ + SMI: &v1alpha1.SMITrafficRouting{}, + } + rollout1.Spec.Strategy.Canary.CanaryService = "canary-service" + rollout1.Spec.Strategy.Canary.StableService = "stable-service" + rollout1.Status.ReadyReplicas = 10 + rollout1.Status.AvailableReplicas = 10 + + rollout2 := bumpVersion(rollout1) + + replicaSet1 := newReplicaSetWithStatus(rollout1, 1, 1) + replicaSet2 := newReplicaSetWithStatus(rollout2, 9, 9) + + replicaSet1Hash := replicaSet1.Labels[v1alpha1.DefaultRolloutUniqueLabelKey] + replicaSet2Hash := replicaSet2.Labels[v1alpha1.DefaultRolloutUniqueLabelKey] + canarySelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: replicaSet2Hash} + stableSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: replicaSet1Hash} + canarySvc := newService("canary-service", 80, canarySelector, rollout1) + stableSvc := newService("stable-service", 80, stableSelector, rollout1) + + rollout2.Spec = rollout1.Spec + rollout2.Status.StableRS = replicaSet1Hash + rollout2.Status.CurrentPodHash = replicaSet1Hash + rollout2.Status.Canary.Weights = &v1alpha1.TrafficWeights{ + Canary: v1alpha1.WeightDestination{ + Weight: 10, + ServiceName: "canary-service", + PodTemplateHash: replicaSet2Hash, + }, + Stable: v1alpha1.WeightDestination{ + Weight: 90, + ServiceName: "stable-service", + PodTemplateHash: replicaSet1Hash, + }, + } + + fix.kubeobjects = append(fix.kubeobjects, replicaSet1, replicaSet2, canarySvc, stableSvc) + fix.replicaSetLister = append(fix.replicaSetLister, replicaSet1, replicaSet2) + + fix.rolloutLister = append(fix.rolloutLister, rollout2) + fix.objects = append(fix.objects, rollout2) + + fix.expectUpdateReplicaSetAction(replicaSet1) + fix.expectUpdateRolloutAction(rollout2) + fix.expectUpdateReplicaSetAction(replicaSet1) + fix.expectPatchRolloutAction(rollout2) + fix.fakeTrafficRouting = newUnmockedFakeTrafficRoutingReconciler() + fix.fakeTrafficRouting.On("RemoveManagedRoutes", mock.Anything, mock.Anything, mock.Anything).Return(nil) + fix.run(getKey(rollout1, t)) +} diff --git a/server/server.go b/server/server.go index 3482634491..9a3e65e328 100644 --- a/server/server.go +++ b/server/server.go @@ -179,7 +179,10 @@ func (s *ArgoRolloutsServer) initRolloutViewController(namespace string, name st } func (s *ArgoRolloutsServer) getRolloutInfo(namespace string, name string) (*rollout.RolloutInfo, error) { - controller := s.initRolloutViewController(namespace, name, context.Background()) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + controller := s.initRolloutViewController(namespace, name, ctx) ri, err := controller.GetRolloutInfo() if err != nil { return nil, err diff --git a/server/server_static.go b/server/server_static.go index 44f2fb78f6..1a7de6b150 100644 --- a/server/server_static.go +++ b/server/server_static.go @@ -30,7 +30,7 @@ func (s *ArgoRolloutsServer) staticFileHttpHandler(w http.ResponseWriter, r *htt requestedURI := path.Clean(r.RequestURI) rootPath := path.Clean("/" + s.Options.RootPath) - if requestedURI == "/" { + if requestedURI == "/" && rootPath != "/" { http.Redirect(w, r, rootPath+"/", http.StatusFound) return } diff --git a/server/server_static_test.go b/server/server_static_test.go index 82b5b19ac9..f749ea842d 100644 --- a/server/server_static_test.go +++ b/server/server_static_test.go @@ -16,31 +16,39 @@ const TestRootPath = "/test-root" var ( //go:embed static_test/* - staticTestData embed.FS //nolint - mockServer ArgoRolloutsServer + staticTestData embed.FS //nolint + mockServerPrefix ArgoRolloutsServer + mockServerRoot ArgoRolloutsServer ) func init() { static = staticTestData staticBasePath = "static_test" indexHtmlFile = staticBasePath + "/index.html" - mockServer = mockArgoRolloutServer() + mockServerPrefix = mockArgoRolloutServer(TestRootPath) + mockServerRoot = mockArgoRolloutServer("/") } func TestIndexHtmlIsServed(t *testing.T) { tests := []struct { + server ArgoRolloutsServer requestPath string }{ - {TestRootPath + "/"}, - {TestRootPath + "/index.html"}, - {TestRootPath + "/nonsense/../index.html"}, - {TestRootPath + "/test-dir/test.css"}, + {mockServerPrefix, TestRootPath + "/"}, + {mockServerPrefix, TestRootPath + "/index.html"}, + {mockServerPrefix, TestRootPath + "/nonsense/../index.html"}, + {mockServerPrefix, TestRootPath + "/test-dir/test.css"}, + + {mockServerRoot, "/"}, + {mockServerRoot, "/index.html"}, + {mockServerRoot, "/nonsense/../index.html"}, + {mockServerRoot, "/test-dir/test.css"}, } for _, test := range tests { t.Run(test.requestPath, func(t *testing.T) { req := httptest.NewRequest(http.MethodGet, test.requestPath, nil) w := httptest.NewRecorder() - mockServer.staticFileHttpHandler(w, req) + test.server.staticFileHttpHandler(w, req) res := w.Result() defer res.Body.Close() data, err := io.ReadAll(res.Body) @@ -60,7 +68,7 @@ func TestIndexHtmlIsServed(t *testing.T) { func TestWhenFileNotFoundSendIndexPageForUiReactRouter(t *testing.T) { req := httptest.NewRequest(http.MethodGet, TestRootPath+"/namespace-default", nil) w := httptest.NewRecorder() - mockServer.staticFileHttpHandler(w, req) + mockServerPrefix.staticFileHttpHandler(w, req) res := w.Result() defer res.Body.Close() data, err := io.ReadAll(res.Body) @@ -72,7 +80,7 @@ func TestWhenFileNotFoundSendIndexPageForUiReactRouter(t *testing.T) { func TestSlashWillBeRedirectedToRootPath(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/", nil) w := httptest.NewRecorder() - mockServer.staticFileHttpHandler(w, req) + mockServerPrefix.staticFileHttpHandler(w, req) res := w.Result() defer res.Body.Close() _, err := io.ReadAll(res.Body) @@ -95,7 +103,7 @@ func TestInvalidFilesOrHackingAttemptReturn404(t *testing.T) { t.Run(test.requestPath, func(t *testing.T) { req := httptest.NewRequest(http.MethodGet, test.requestPath, nil) w := httptest.NewRecorder() - mockServer.staticFileHttpHandler(w, req) + mockServerPrefix.staticFileHttpHandler(w, req) res := w.Result() defer res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotFound) @@ -103,10 +111,10 @@ func TestInvalidFilesOrHackingAttemptReturn404(t *testing.T) { } } -func mockArgoRolloutServer() ArgoRolloutsServer { +func mockArgoRolloutServer(rootPath string) ArgoRolloutsServer { s := ArgoRolloutsServer{ Options: ServerOptions{ - RootPath: TestRootPath, + RootPath: rootPath, }, } return s diff --git a/service/service_test.go b/service/service_test.go index 35722443ac..d8233a567e 100644 --- a/service/service_test.go +++ b/service/service_test.go @@ -16,11 +16,12 @@ import ( k8stesting "k8s.io/client-go/testing" "k8s.io/client-go/util/workqueue" + "k8s.io/client-go/tools/cache" + "github.com/argoproj/argo-rollouts/controller/metrics" "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/fake" informers "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions" - "k8s.io/client-go/tools/cache" ) func newService(name string, port int, selector map[string]string) *corev1.Service { diff --git a/test/cmd/metrics-plugin-sample/internal/plugin/plugin.go b/test/cmd/metrics-plugin-sample/internal/plugin/plugin.go index 69d3763469..9cf3db3381 100644 --- a/test/cmd/metrics-plugin-sample/internal/plugin/plugin.go +++ b/test/cmd/metrics-plugin-sample/internal/plugin/plugin.go @@ -13,15 +13,16 @@ import ( "github.com/argoproj/argo-rollouts/utils/plugin/types" + "github.com/prometheus/client_golang/api" + v1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" + log "github.com/sirupsen/logrus" + "github.com/argoproj/argo-rollouts/metricproviders/plugin" "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" "github.com/argoproj/argo-rollouts/utils/evaluate" metricutil "github.com/argoproj/argo-rollouts/utils/metric" timeutil "github.com/argoproj/argo-rollouts/utils/time" - "github.com/prometheus/client_golang/api" - v1 "github.com/prometheus/client_golang/api/prometheus/v1" - "github.com/prometheus/common/model" - log "github.com/sirupsen/logrus" ) const EnvVarArgoRolloutsPrometheusAddress string = "ARGO_ROLLOUTS_PROMETHEUS_ADDRESS" diff --git a/test/cmd/metrics-plugin-sample/main.go b/test/cmd/metrics-plugin-sample/main.go index fdd01e6ba3..e84b40f36d 100644 --- a/test/cmd/metrics-plugin-sample/main.go +++ b/test/cmd/metrics-plugin-sample/main.go @@ -1,10 +1,11 @@ package main import ( - "github.com/argoproj/argo-rollouts/metricproviders/plugin/rpc" - "github.com/argoproj/argo-rollouts/test/cmd/metrics-plugin-sample/internal/plugin" goPlugin "github.com/hashicorp/go-plugin" log "github.com/sirupsen/logrus" + + "github.com/argoproj/argo-rollouts/metricproviders/plugin/rpc" + "github.com/argoproj/argo-rollouts/test/cmd/metrics-plugin-sample/internal/plugin" ) // handshakeConfigs are used to just do a basic handshake between diff --git a/test/cmd/step-plugin-e2e/internal/plugin/plugin.go b/test/cmd/step-plugin-e2e/internal/plugin/plugin.go new file mode 100644 index 0000000000..8bb2aa4a7b --- /dev/null +++ b/test/cmd/step-plugin-e2e/internal/plugin/plugin.go @@ -0,0 +1,173 @@ +package plugin + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/google/uuid" + log "github.com/sirupsen/logrus" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/rollout/steps/plugin/rpc" + "github.com/argoproj/argo-rollouts/utils/plugin/types" +) + +type Config struct { + Return string + Requeue string + + Abort Abort + Terminate Terminate +} + +type Abort struct { + Return string +} + +type Terminate struct { + Return string +} + +type State struct { + Data string + Count int +} + +type rpcPlugin struct { + LogCtx *log.Entry +} + +func New(logCtx *log.Entry) rpc.StepPlugin { + return &rpcPlugin{ + LogCtx: logCtx, + } +} + +func (p *rpcPlugin) InitPlugin() types.RpcError { + p.LogCtx.Infof("InitPlugin") + return types.RpcError{} +} + +func (p *rpcPlugin) Run(rollout *v1alpha1.Rollout, context *types.RpcStepContext) (types.RpcStepResult, types.RpcError) { + p.LogCtx.Infof("Run plugin for rollout %s/%s", rollout.Namespace, rollout.Name) + + // Get configs + var config Config + var state State + if context != nil { + if context.Config != nil { + if err := json.Unmarshal(context.Config, &config); err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Errorf("could not unmarshal config: %w", err).Error()} + } + p.LogCtx.Infof("Using config: %+v", config) + } + if context.Status != nil { + if err := json.Unmarshal(context.Status, &state); err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Errorf("could not unmarshal status: %w", err).Error()} + } + p.LogCtx.Infof("Using status: %+v", state) + } + } + + if state.Data == "" { + state.Data = uuid.New().String() + } + state.Count = state.Count + 1 + + var requeue time.Duration + if config.Requeue != "" { + v, err := time.ParseDuration(config.Requeue) + if err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Errorf("could not parse requeue duration: %w", err).Error()} + } + requeue = v + } + + phase := types.PhaseSuccessful + if config.Return != "" { + phase = types.StepPhase(config.Return) + if err := phase.Validate(); err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Errorf("could not parse phase: %w", err).Error()} + } + } + + return Result(state, phase, requeue) +} + +func (p *rpcPlugin) Terminate(rollout *v1alpha1.Rollout, context *types.RpcStepContext) (types.RpcStepResult, types.RpcError) { + p.LogCtx.Infof("Terminate plugin for rollout %s/%s", rollout.Namespace, rollout.Name) + + // Get configs + var config Config + var state State + if context != nil { + if context.Config != nil { + if err := json.Unmarshal(context.Config, &config); err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Errorf("could not unmarshal config: %w", err).Error()} + } + } + if context.Status != nil { + if err := json.Unmarshal(context.Status, &state); err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Errorf("could not unmarshal status: %w", err).Error()} + } + } + } + + phase := types.PhaseSuccessful + if config.Terminate.Return != "" { + phase = types.StepPhase(config.Terminate.Return) + if err := phase.Validate(); err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Errorf("could not parse phase: %w", err).Error()} + } + } + + return Result(state, phase, 0) +} + +func (p *rpcPlugin) Abort(rollout *v1alpha1.Rollout, context *types.RpcStepContext) (types.RpcStepResult, types.RpcError) { + p.LogCtx.Infof("Abort plugin for rollout %s/%s", rollout.Namespace, rollout.Name) + + // Get configs + var config Config + var state State + if context != nil { + if context.Config != nil { + if err := json.Unmarshal(context.Config, &config); err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Errorf("could not unmarshal config: %w", err).Error()} + } + } + if context.Status != nil { + if err := json.Unmarshal(context.Status, &state); err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Errorf("could not unmarshal status: %w", err).Error()} + } + } + } + + phase := types.PhaseSuccessful + if config.Abort.Return != "" { + phase = types.StepPhase(config.Abort.Return) + if err := phase.Validate(); err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Errorf("could not parse phase: %w", err).Error()} + } + } + + return Result(state, phase, 0) +} + +func (p *rpcPlugin) Type() string { + return "e2e-plugin" +} + +func Result(state State, phase types.StepPhase, requeue time.Duration) (types.RpcStepResult, types.RpcError) { + stateRaw, err := json.Marshal(state) + if err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Sprintf("Could not marshal state: %v", err)} + } + + return types.RpcStepResult{ + Phase: phase, + RequeueAfter: requeue, + Status: stateRaw, + }, types.RpcError{} +} diff --git a/test/cmd/step-plugin-e2e/main.go b/test/cmd/step-plugin-e2e/main.go new file mode 100644 index 0000000000..54f97bd862 --- /dev/null +++ b/test/cmd/step-plugin-e2e/main.go @@ -0,0 +1,61 @@ +package main + +import ( + "strings" + + goPlugin "github.com/hashicorp/go-plugin" + log "github.com/sirupsen/logrus" + + rolloutsPlugin "github.com/argoproj/argo-rollouts/rollout/steps/plugin/rpc" + "github.com/argoproj/argo-rollouts/test/cmd/step-plugin-e2e/internal/plugin" +) + +var handshakeConfig = goPlugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "ARGO_ROLLOUTS_RPC_PLUGIN", + MagicCookieValue: "step", +} + +func main() { + logCtx := log.WithFields(log.Fields{"plugin": "e2e"}) + + setLogLevel("debug") + log.SetFormatter(createFormatter("text")) + + // pluginMap is the map of plugins we can dispense. + var pluginMap = map[string]goPlugin.Plugin{ + "RpcStepPlugin": &rolloutsPlugin.RpcStepPlugin{Impl: plugin.New(logCtx)}, + } + + goPlugin.Serve(&goPlugin.ServeConfig{ + HandshakeConfig: handshakeConfig, + Plugins: pluginMap, + }) +} + +func createFormatter(logFormat string) log.Formatter { + var formatType log.Formatter + switch strings.ToLower(logFormat) { + case "json": + formatType = &log.JSONFormatter{} + case "text": + formatType = &log.TextFormatter{ + FullTimestamp: true, + } + default: + log.Infof("Unknown format: %s. Using text logformat", logFormat) + formatType = &log.TextFormatter{ + FullTimestamp: true, + } + } + + return formatType +} + +func setLogLevel(logLevel string) { + level, err := log.ParseLevel(logLevel) + if err != nil { + log.Fatal(err) + } + log.SetLevel(level) +} diff --git a/test/cmd/step-plugin-sample/internal/plugin/plugin.go b/test/cmd/step-plugin-sample/internal/plugin/plugin.go new file mode 100644 index 0000000000..27ab9bcc5e --- /dev/null +++ b/test/cmd/step-plugin-sample/internal/plugin/plugin.go @@ -0,0 +1,264 @@ +package plugin + +import ( + "encoding/json" + "fmt" + "math/rand" + "strconv" + "sync" + "time" + + "github.com/google/uuid" + log "github.com/sirupsen/logrus" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/rollout/steps/plugin/rpc" + "github.com/argoproj/argo-rollouts/utils/plugin/types" +) + +type Config struct { + Async bool + Aggregate bool +} + +type State struct { + Id string + SharedId string + + Value string +} + +type Result struct { + Value int + Id string +} + +type rpcPlugin struct { + LogCtx *log.Entry + Seed int64 + + lock *sync.RWMutex + generator *rand.Rand + randomMap map[string]*Result +} + +func New(logCtx *log.Entry, seed int64) rpc.StepPlugin { + return &rpcPlugin{ + LogCtx: logCtx, + Seed: seed, + lock: &sync.RWMutex{}, + } +} + +func (p *rpcPlugin) InitPlugin() types.RpcError { + p.lock.Lock() + defer p.lock.Unlock() + + p.LogCtx.Infof("InitPlugin with seed %d", p.Seed) + p.generator = rand.New(rand.NewSource(p.Seed)) + p.randomMap = map[string]*Result{} + return types.RpcError{} +} + +func (p *rpcPlugin) Run(rollout *v1alpha1.Rollout, context *types.RpcStepContext) (types.RpcStepResult, types.RpcError) { + // if !p.validate(rollout) { + + // } + + // Get configs + var config Config + var state State + if context != nil { + if context.Config != nil { + if err := json.Unmarshal(context.Config, &config); err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Errorf("could not unmarshal config: %w", err).Error()} + } + } + if context.Status != nil { + if err := json.Unmarshal(context.Status, &state); err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Errorf("could not unmarshal status: %w", err).Error()} + } + } + } + + // Already completed + if state.Value != "" { + return CompletedResult(state) + } + + // If Aggregate, look for previous steps. We want to re-use the value in memory for that step id + if config.Aggregate && state.SharedId == "" { + lastStep := getLastStep(rollout, context.PluginName) + if lastStep != nil { + var lastState State + if err := json.Unmarshal(lastStep.Status, &lastState); err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: "could not unmarshal last step status"} + } + if lastState.SharedId != "" { + // consecutive aggregate steps all use the same id + state.SharedId = lastState.SharedId + } else { + // Most likely the last step was not an aggregate, so we restart a sequence with this id + state.SharedId = lastState.Id + } + } + } + + // Already started, look if it is completed + if state.Id != "" { + p.lock.RLock() + v, ok := p.randomMap[state.getId()] + p.lock.RUnlock() + if ok { + if v.Id == state.Id { + // Make sure the current step is the one that updated the value + state.Value = strconv.Itoa(v.Value) + return CompletedResult(state) + } + } + return RunningResult(state) + } + + state.Id = uuid.New().String() + if config.Async { + go func(state State) { + time.Sleep(60 * time.Second) + p.generate(state) + }(state) + return RunningResult(state) + } else { + state.Value = strconv.Itoa(p.generate(state).Value) + } + + return CompletedResult(state) + +} + +func (p *rpcPlugin) Terminate(rollout *v1alpha1.Rollout, context *types.RpcStepContext) (types.RpcStepResult, types.RpcError) { + // Get configs + var config Config + var state State + if context != nil { + if context.Config != nil { + if err := json.Unmarshal(context.Config, &config); err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Errorf("could not unmarshal config: %w", err).Error()} + } + } + if context.Status != nil { + if err := json.Unmarshal(context.Status, &state); err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Errorf("could not unmarshal status: %w", err).Error()} + } + } + } + + log.Infof("Ignoring future value for '%s'", state.Id) + state.Value = "0" + return CompletedResult(state) + +} + +func (p *rpcPlugin) Abort(_ *v1alpha1.Rollout, context *types.RpcStepContext) (types.RpcStepResult, types.RpcError) { + + // Get configs + var config Config + var state State + if context != nil { + if context.Config != nil { + if err := json.Unmarshal(context.Config, &config); err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Errorf("could not unmarshal config: %w", err).Error()} + } + } + if context.Status != nil { + if err := json.Unmarshal(context.Status, &state); err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Errorf("could not unmarshal status: %w", err).Error()} + } + } + } + + p.lock.Lock() + defer p.lock.Unlock() + + log.Infof("deleting entry for id '%s'", state.getId()) + delete(p.randomMap, state.getId()) + + return CompletedResult(state) +} + +func (p *rpcPlugin) Type() string { + return "plugin-example" +} + +// func (p *RpcPlugin) validate(rollout *v1alpha1.Rollout) bool { +// if rollout == nil || rollout.Status == nil || rollout.Status.Canary == nil {} +// } + +func (p *rpcPlugin) generate(state State) *Result { + p.lock.Lock() + defer p.lock.Unlock() + + base := 0 + if state.SharedId != "" { + v, ok := p.randomMap[state.SharedId] + if ok { + log.Infof("Using base '%d' for aggregate %s", v.Value, state.SharedId) + base = v.Value + } + } + + result := &Result{ + Value: p.generator.Intn(100) + base, + Id: state.Id, + } + p.randomMap[state.getId()] = result + log.Infof("Set '%d' for id %s", result.Value, state.getId()) + + return result +} + +func (s State) getId() string { + if s.SharedId != "" { + return s.SharedId + } + return s.Id +} + +func getLastStep(rollout *v1alpha1.Rollout, name string) *v1alpha1.StepPluginStatus { + var last *v1alpha1.StepPluginStatus + currentStepIndex := *rollout.Status.CurrentStepIndex + for _, status := range rollout.Status.Canary.StepPluginStatuses { + if status.Name != name || status.Index == currentStepIndex { + continue + } + + if last == nil || status.Index > last.Index { + last = &status + } + } + return last +} + +func CompletedResult(state State) (types.RpcStepResult, types.RpcError) { + stateRaw, err := json.Marshal(state) + if err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Sprintf("Could not marshal state: %v", err)} + } + + return types.RpcStepResult{ + Phase: types.PhaseSuccessful, + Message: "Operation completed", + Status: stateRaw, + }, types.RpcError{} +} + +func RunningResult(state State) (types.RpcStepResult, types.RpcError) { + stateRaw, err := json.Marshal(state) + if err != nil { + return types.RpcStepResult{}, types.RpcError{ErrorString: fmt.Sprintf("Could not marshal state: %v", err)} + } + + return types.RpcStepResult{ + Phase: types.PhaseRunning, + RequeueAfter: 15 * time.Second, + Status: stateRaw, + }, types.RpcError{} +} diff --git a/test/cmd/step-plugin-sample/internal/plugin/plugin_test.go b/test/cmd/step-plugin-sample/internal/plugin/plugin_test.go new file mode 100644 index 0000000000..40f3fe0252 --- /dev/null +++ b/test/cmd/step-plugin-sample/internal/plugin/plugin_test.go @@ -0,0 +1,369 @@ +package plugin + +import ( + "encoding/json" + "sync" + "testing" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/utils/ptr" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/utils/plugin/types" +) + +func new(logCtx *log.Entry, seed int64) *rpcPlugin { + return &rpcPlugin{ + LogCtx: logCtx, + Seed: seed, + lock: &sync.RWMutex{}, + } +} + +func newRollout() *v1alpha1.Rollout { + return &v1alpha1.Rollout{ + Spec: v1alpha1.RolloutSpec{ + Strategy: v1alpha1.RolloutStrategy{ + Canary: &v1alpha1.CanaryStrategy{}, + }, + }, + Status: v1alpha1.RolloutStatus{ + Canary: v1alpha1.CanaryStatus{}, + }, + } +} + +func getStateFromResult(t *testing.T, result types.RpcStepResult) *State { + require.NotNil(t, result) + + var state *State + if err := json.Unmarshal(result.Status, &state); err != nil { + require.NotNil(t, err) + } + return state +} + +func toRaw(obj any) json.RawMessage { + raw, err := json.Marshal(obj) + if err != nil { + panic(err) + } + return raw +} + +func Test_rpcPlugin_Run(t *testing.T) { + t.Run("Return value with nil context", func(t *testing.T) { + p := New(log.WithFields(log.Fields{}), 0) + err := p.InitPlugin() + if err.HasError() { + t.Fatalf("Got error during init: %s", err.Error()) + } + + rollout := newRollout() + + got, err := p.Run(rollout, nil) + + require.False(t, err.HasError(), "Error: %s", err.Error()) + assert.Equal(t, types.PhaseSuccessful, got.Phase) + state := getStateFromResult(t, got) + assert.Equal(t, "74", state.Value) + + }) + t.Run("Return value with empty context", func(t *testing.T) { + p := New(log.WithFields(log.Fields{}), 0) + err := p.InitPlugin() + if err.HasError() { + t.Fatalf("Got error during init: %s", err.Error()) + } + + rollout := newRollout() + context := &types.RpcStepContext{ + PluginName: "test", + Config: nil, + Status: nil, + } + + got, err := p.Run(rollout, context) + + require.False(t, err.HasError(), "Error: %s", err.Error()) + assert.Equal(t, types.PhaseSuccessful, got.Phase) + state := getStateFromResult(t, got) + assert.Equal(t, "74", state.Value) + + }) + + t.Run("Return value with existing state", func(t *testing.T) { + p := New(log.WithFields(log.Fields{}), 0) + err := p.InitPlugin() + if err.HasError() { + t.Fatalf("Got error during init: %s", err.Error()) + } + + rollout := newRollout() + context := &types.RpcStepContext{ + PluginName: "test", + Config: nil, + Status: toRaw(&State{ + Id: "fake-id", + Value: "123", + }), + } + + got, err := p.Run(rollout, context) + + require.False(t, err.HasError(), "Error: %s", err.Error()) + assert.Equal(t, types.PhaseSuccessful, got.Phase) + state := getStateFromResult(t, got) + assert.Equal(t, "123", state.Value) + }) + + t.Run("Running status with async config", func(t *testing.T) { + p := New(log.WithFields(log.Fields{}), 0) + err := p.InitPlugin() + if err.HasError() { + t.Fatalf("Got error during init: %s", err.Error()) + } + + rollout := newRollout() + context := &types.RpcStepContext{ + PluginName: "test", + Config: toRaw(&Config{ + Async: true, + }), + Status: nil, + } + + got, err := p.Run(rollout, context) + + require.False(t, err.HasError(), "Error: %s", err.Error()) + assert.Equal(t, types.PhaseRunning, got.Phase) + state := getStateFromResult(t, got) + assert.Equal(t, "", state.Value) + }) + + t.Run("Return running with async config and existing state not completed", func(t *testing.T) { + p := new(log.WithFields(log.Fields{}), 0) + err := p.InitPlugin() + if err.HasError() { + t.Fatalf("Got error during init: %s", err.Error()) + } + + rollout := newRollout() + context := &types.RpcStepContext{ + PluginName: "test", + Config: toRaw(&Config{ + Async: true, + }), + Status: toRaw(&State{ + Id: "fake-id", + }), + } + + got, err := p.Run(rollout, context) + + require.False(t, err.HasError(), "Error: %s", err.Error()) + assert.Equal(t, types.PhaseRunning, got.Phase) + state := getStateFromResult(t, got) + assert.Equal(t, "", state.Value) + }) + + t.Run("Return value with async config and existing state", func(t *testing.T) { + p := new(log.WithFields(log.Fields{}), 0) + err := p.InitPlugin() + if err.HasError() { + t.Fatalf("Got error during init: %s", err.Error()) + } + p.randomMap["fake-id"] = &Result{ + Value: 44, + Id: "fake-id", + } + + rollout := newRollout() + context := &types.RpcStepContext{ + PluginName: "test", + Config: toRaw(&Config{ + Async: true, + }), + Status: toRaw(&State{ + Id: "fake-id", + }), + } + + got, err := p.Run(rollout, context) + + require.False(t, err.HasError(), "Error: %s", err.Error()) + assert.Equal(t, types.PhaseSuccessful, got.Phase) + state := getStateFromResult(t, got) + assert.Equal(t, "44", state.Value) + }) + + t.Run("Return value with aggregate and no previous step", func(t *testing.T) { + p := new(log.WithFields(log.Fields{}), 0) + err := p.InitPlugin() + if err.HasError() { + t.Fatalf("Got error during init: %s", err.Error()) + } + p.randomMap["foo"] = &Result{ + Value: 6, + Id: "foo", + } + + rollout := newRollout() + rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ + { + Plugin: &v1alpha1.PluginStep{ + Name: "test", + }, + }, + { + Plugin: &v1alpha1.PluginStep{ + Name: "test", + }, + }, + } + rollout.Status.CurrentStepIndex = ptr.To(int32(0)) + rollout.Status.Canary.StepPluginStatuses = []v1alpha1.StepPluginStatus{ + { + Index: 0, + Name: "test", + Status: toRaw(&State{ + Id: "foo", + Value: "99", + }), + }, + } + context := &types.RpcStepContext{ + PluginName: "test", + Config: toRaw(&Config{ + Aggregate: true, + }), + Status: nil, + } + + got, err := p.Run(rollout, context) + + require.False(t, err.HasError(), "Error: %s", err.Error()) + assert.Equal(t, types.PhaseSuccessful, got.Phase) + state := getStateFromResult(t, got) + assert.Equal(t, "74", state.Value) + }) + + t.Run("Return value with aggregate", func(t *testing.T) { + p := new(log.WithFields(log.Fields{}), 0) + err := p.InitPlugin() + if err.HasError() { + t.Fatalf("Got error during init: %s", err.Error()) + } + p.randomMap["foo"] = &Result{ + Value: 6, + Id: "foo", + } + + rollout := newRollout() + rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ + { + Plugin: &v1alpha1.PluginStep{ + Name: "test", + }, + }, + { + Plugin: &v1alpha1.PluginStep{ + Name: "test", + }, + }, + } + rollout.Status.CurrentStepIndex = ptr.To(int32(1)) + rollout.Status.Canary.StepPluginStatuses = []v1alpha1.StepPluginStatus{ + { + Index: 0, + Name: "test", + Status: toRaw(&State{ + Id: "foo", + Value: "6", + }), + }, + } + context := &types.RpcStepContext{ + PluginName: "test", + Config: toRaw(&Config{ + Aggregate: true, + }), + Status: nil, + } + + got, err := p.Run(rollout, context) + + require.False(t, err.HasError(), "Error: %s", err.Error()) + assert.Equal(t, types.PhaseSuccessful, got.Phase) + state := getStateFromResult(t, got) + assert.Equal(t, "80", state.Value) + }) + + t.Run("Return value with aggregate of an aggregate", func(t *testing.T) { + p := new(log.WithFields(log.Fields{}), 0) + err := p.InitPlugin() + if err.HasError() { + t.Fatalf("Got error during init: %s", err.Error()) + } + p.randomMap["foo"] = &Result{ + Value: 80, + Id: "bar", + } + + rollout := newRollout() + rollout.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{ + { + Plugin: &v1alpha1.PluginStep{ + Name: "test", + }, + }, + { + Plugin: &v1alpha1.PluginStep{ + Name: "test", + }, + }, + { + Plugin: &v1alpha1.PluginStep{ + Name: "test", + }, + }, + } + rollout.Status.CurrentStepIndex = ptr.To(int32(2)) + rollout.Status.Canary.StepPluginStatuses = []v1alpha1.StepPluginStatus{ + { + Index: 0, + Name: "test", + Status: toRaw(&State{ + Id: "foo", + Value: "6", + }), + }, + { + Index: 1, + Name: "test", + Status: toRaw(&State{ + Id: "bar", + SharedId: "foo", + Value: "80", + }), + }, + } + context := &types.RpcStepContext{ + PluginName: "test", + Config: toRaw(&Config{ + Aggregate: true, + }), + Status: nil, + } + + got, err := p.Run(rollout, context) + + require.False(t, err.HasError(), "Error: %s", err.Error()) + assert.Equal(t, types.PhaseSuccessful, got.Phase) + state := getStateFromResult(t, got) + assert.Equal(t, "154", state.Value) + }) + +} diff --git a/test/cmd/step-plugin-sample/main.go b/test/cmd/step-plugin-sample/main.go new file mode 100644 index 0000000000..74d8a4550d --- /dev/null +++ b/test/cmd/step-plugin-sample/main.go @@ -0,0 +1,87 @@ +package main + +import ( + "os" + "strconv" + "strings" + "time" + + goPlugin "github.com/hashicorp/go-plugin" + log "github.com/sirupsen/logrus" + + rolloutsPlugin "github.com/argoproj/argo-rollouts/rollout/steps/plugin/rpc" + "github.com/argoproj/argo-rollouts/test/cmd/step-plugin-sample/internal/plugin" +) + +// handshakeConfigs are used to just do a basic handshake between +// a plugin and host. If the handshake fails, a user friendly error is shown. +// This prevents users from executing bad plugins or executing a plugin +// directory. It is a UX feature, not a security feature. +var handshakeConfig = goPlugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "ARGO_ROLLOUTS_RPC_PLUGIN", + MagicCookieValue: "step", +} + +func main() { + logCtx := log.WithFields(log.Fields{"plugin": "step"}) + + setLogLevel("debug") + log.SetFormatter(createFormatter("text")) + + seed := time.Now().UnixNano() + + args := os.Args[1:] + if len(args) > 0 { + if args[0] == "--seed" { + if len(args) >= 2 { + n, err := strconv.ParseInt(args[1], 10, 64) + if err != nil { + log.Fatal(err) + } + seed = n + } else { + log.Fatal("No value for --seed argument") + } + } + } + + // pluginMap is the map of plugins we can dispense. + var pluginMap = map[string]goPlugin.Plugin{ + "RpcStepPlugin": &rolloutsPlugin.RpcStepPlugin{Impl: plugin.New(logCtx, seed)}, + } + + logCtx.Debug("message from plugin", "foo", "bar") + + goPlugin.Serve(&goPlugin.ServeConfig{ + HandshakeConfig: handshakeConfig, + Plugins: pluginMap, + }) +} + +func createFormatter(logFormat string) log.Formatter { + var formatType log.Formatter + switch strings.ToLower(logFormat) { + case "json": + formatType = &log.JSONFormatter{} + case "text": + formatType = &log.TextFormatter{ + FullTimestamp: true, + } + default: + log.Infof("Unknown format: %s. Using text logformat", logFormat) + formatType = &log.TextFormatter{ + FullTimestamp: true, + } + } + + return formatType +} + +func setLogLevel(logLevel string) { + level, err := log.ParseLevel(logLevel) + if err != nil { + log.Fatal(err) + } + log.SetLevel(level) +} diff --git a/test/cmd/step-plugin-sample/main_test.go b/test/cmd/step-plugin-sample/main_test.go new file mode 100644 index 0000000000..f7703b2042 --- /dev/null +++ b/test/cmd/step-plugin-sample/main_test.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "testing" + "time" + + goPlugin "github.com/hashicorp/go-plugin" + log "github.com/sirupsen/logrus" + + "github.com/argoproj/argo-rollouts/rollout/steps/plugin/rpc" + rolloutsPlugin "github.com/argoproj/argo-rollouts/rollout/steps/plugin/rpc" + "github.com/argoproj/argo-rollouts/test/cmd/step-plugin-sample/internal/plugin" + "github.com/argoproj/argo-rollouts/utils/plugin/types" + + "github.com/tj/assert" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" +) + +var testHandshake = goPlugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "ARGO_ROLLOUTS_RPC_PLUGIN", + MagicCookieValue: "step", +} + +func pluginClient(t *testing.T) (rpc.StepPlugin, goPlugin.ClientProtocol, func(), chan struct{}) { + ctx, cancel := context.WithCancel(context.Background()) + + pluginImpl := plugin.New(log.WithFields(log.Fields{}), 0) + + // pluginMap is the map of plugins we can dispense. + var pluginMap = map[string]goPlugin.Plugin{ + "RpcStepPlugin": &rolloutsPlugin.RpcStepPlugin{Impl: pluginImpl}, + } + + ch := make(chan *goPlugin.ReattachConfig, 1) + closeCh := make(chan struct{}) + go goPlugin.Serve(&goPlugin.ServeConfig{ + HandshakeConfig: testHandshake, + Plugins: pluginMap, + Test: &goPlugin.ServeTestConfig{ + Context: ctx, + ReattachConfigCh: ch, + CloseCh: closeCh, + }, + }) + + // We should get a config + var config *goPlugin.ReattachConfig + select { + case config = <-ch: + case <-time.After(2000 * time.Millisecond): + t.Fatal("should've received reattach") + } + if config == nil { + t.Fatal("config should not be nil") + } + + // Connect! + c := goPlugin.NewClient(&goPlugin.ClientConfig{ + Cmd: nil, + HandshakeConfig: testHandshake, + Plugins: pluginMap, + Reattach: config, + }) + client, err := c.Client() + if err != nil { + t.Fatalf("err: %s", err) + } + + // Request the plugin + raw, err := client.Dispense("RpcStepPlugin") + if err != nil { + t.Fail() + } + + plugin, ok := raw.(rpc.StepPlugin) + if !ok { + t.Fail() + } + + return plugin, client, cancel, closeCh +} + +func TestPlugin(t *testing.T) { + plugin, _, cancel, closeCh := pluginClient(t) + defer cancel() + + err := plugin.InitPlugin() + if err.Error() != "" { + t.Fail() + } + + ro := v1alpha1.Rollout{} + rpcCtx := &types.RpcStepContext{ + PluginName: "test-1", + Config: nil, + Status: nil, + } + + result, err := plugin.Run(&ro, rpcCtx) + assert.Equal(t, "", err.Error()) + assert.NotNil(t, result) + + // Canceling should cause an exit + cancel() + <-closeCh +} diff --git a/test/cmd/trafficrouter-plugin-sample/internal/plugin/plugin.go b/test/cmd/trafficrouter-plugin-sample/internal/plugin/plugin.go index 0d18c5d069..c4fd9c8c07 100644 --- a/test/cmd/trafficrouter-plugin-sample/internal/plugin/plugin.go +++ b/test/cmd/trafficrouter-plugin-sample/internal/plugin/plugin.go @@ -8,11 +8,6 @@ import ( "strings" "time" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" - "github.com/argoproj/argo-rollouts/rollout/trafficrouting/plugin/rpc" - "github.com/argoproj/argo-rollouts/utils/defaults" - ingressutil "github.com/argoproj/argo-rollouts/utils/ingress" - pluginTypes "github.com/argoproj/argo-rollouts/utils/plugin/types" "github.com/sirupsen/logrus" extensionsv1beta1 "k8s.io/api/extensions/v1beta1" networkingv1 "k8s.io/api/networking/v1" @@ -23,6 +18,12 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/rollout/trafficrouting/plugin/rpc" + "github.com/argoproj/argo-rollouts/utils/defaults" + ingressutil "github.com/argoproj/argo-rollouts/utils/ingress" + pluginTypes "github.com/argoproj/argo-rollouts/utils/plugin/types" ) var _ rpc.TrafficRouterPlugin = &RpcPlugin{} diff --git a/test/cmd/trafficrouter-plugin-sample/main.go b/test/cmd/trafficrouter-plugin-sample/main.go index 05122fbe0f..79743e6b3e 100644 --- a/test/cmd/trafficrouter-plugin-sample/main.go +++ b/test/cmd/trafficrouter-plugin-sample/main.go @@ -3,10 +3,11 @@ package main import ( "strings" - rolloutsPlugin "github.com/argoproj/argo-rollouts/rollout/trafficrouting/plugin/rpc" - "github.com/argoproj/argo-rollouts/test/cmd/trafficrouter-plugin-sample/internal/plugin" goPlugin "github.com/hashicorp/go-plugin" log "github.com/sirupsen/logrus" + + rolloutsPlugin "github.com/argoproj/argo-rollouts/rollout/trafficrouting/plugin/rpc" + "github.com/argoproj/argo-rollouts/test/cmd/trafficrouter-plugin-sample/internal/plugin" ) // handshakeConfigs are used to just do a basic handshake between diff --git a/test/e2e/aws_test.go b/test/e2e/aws_test.go index f8b0553fe2..fcbb1a2ef5 100644 --- a/test/e2e/aws_test.go +++ b/test/e2e/aws_test.go @@ -94,6 +94,71 @@ func (s *AWSSuite) TestALBPingPongUpdate() { Assert(assertWeights(s, "ping-service", "pong-service", 100, 0)) } +// Rollout uses both alb and mesh for trafficRouting. +// also uses both pingpong service and stable/canary services +// Expecting: * alb is using pingpong +// - mesh is using stable/canary +func (s *AWSSuite) TestALBMesh_PingPong_StableCanary_Update() { + s.Given(). + RolloutObjects("@functional/albmesh-pingpong-stablecanary-rollout.yaml"). + When().ApplyManifests().WaitForRolloutStatus("Healthy"). + Then(). + Assert(assertWeights(s, "ping-service", "pong-service", 100, 0)). + Assert(func(t *fixtures.Then) { + vsvc := t.GetVirtualService() + assert.Equal(s.T(), int64(100), vsvc.Spec.HTTP[0].Route[0].Weight) + assert.Equal(s.T(), int64(0), vsvc.Spec.HTTP[0].Route[1].Weight) + assert.Equal(s.T(), "stable-service", vsvc.Spec.HTTP[0].Route[0].Destination.Host) + assert.Equal(s.T(), "canary-service", vsvc.Spec.HTTP[0].Route[1].Destination.Host) + }). + // Update 1. Test the weight switch from ping => pong + When().UpdateSpec(). + WaitForRolloutCanaryStepIndex(1).Sleep(1 * time.Second).Then(). + Assert(assertWeights(s, "ping-service", "pong-service", 75, 25)). + Assert(func(t *fixtures.Then) { + vsvc := t.GetVirtualService() + assert.Equal(s.T(), int64(75), vsvc.Spec.HTTP[0].Route[0].Weight) + assert.Equal(s.T(), int64(25), vsvc.Spec.HTTP[0].Route[1].Weight) + assert.Equal(s.T(), "stable-service", vsvc.Spec.HTTP[0].Route[0].Destination.Host) + assert.Equal(s.T(), "canary-service", vsvc.Spec.HTTP[0].Route[1].Destination.Host) + }). + When().PromoteRollout(). + WaitForRolloutStatus("Healthy"). + Sleep(1 * time.Second). + Then(). + Assert(assertWeights(s, "ping-service", "pong-service", 0, 100)). + Assert(func(t *fixtures.Then) { + vsvc := t.GetVirtualService() + assert.Equal(s.T(), int64(100), vsvc.Spec.HTTP[0].Route[0].Weight) + assert.Equal(s.T(), int64(0), vsvc.Spec.HTTP[0].Route[1].Weight) + assert.Equal(s.T(), "stable-service", vsvc.Spec.HTTP[0].Route[0].Destination.Host) + assert.Equal(s.T(), "canary-service", vsvc.Spec.HTTP[0].Route[1].Destination.Host) + }). + // Update 2. Test the weight switch from pong => ping + When().UpdateSpec(). + WaitForRolloutCanaryStepIndex(1).Sleep(1 * time.Second).Then(). + Assert(assertWeights(s, "ping-service", "pong-service", 25, 75)). + Assert(func(t *fixtures.Then) { + vsvc := t.GetVirtualService() + assert.Equal(s.T(), int64(75), vsvc.Spec.HTTP[0].Route[0].Weight) + assert.Equal(s.T(), int64(25), vsvc.Spec.HTTP[0].Route[1].Weight) + assert.Equal(s.T(), "stable-service", vsvc.Spec.HTTP[0].Route[0].Destination.Host) + assert.Equal(s.T(), "canary-service", vsvc.Spec.HTTP[0].Route[1].Destination.Host) + }). + When().PromoteRollout(). + WaitForRolloutStatus("Healthy"). + Sleep(1 * time.Second). + Then(). + Assert(assertWeights(s, "ping-service", "pong-service", 100, 0)). + Assert(func(t *fixtures.Then) { + vsvc := t.GetVirtualService() + assert.Equal(s.T(), int64(100), vsvc.Spec.HTTP[0].Route[0].Weight) + assert.Equal(s.T(), int64(0), vsvc.Spec.HTTP[0].Route[1].Weight) + assert.Equal(s.T(), "stable-service", vsvc.Spec.HTTP[0].Route[0].Destination.Host) + assert.Equal(s.T(), "canary-service", vsvc.Spec.HTTP[0].Route[1].Destination.Host) + }) +} + func (s *AWSSuite) TestALBPingPongUpdateMultiIngress() { s.Given(). RolloutObjects("@functional/alb-pingpong-multi-ingress-rollout.yaml"). @@ -239,6 +304,7 @@ func (s *AWSSuite) TestALBExperimentStepMultiIngress() { } func (s *AWSSuite) TestALBExperimentStepNoSetWeight() { + //TODO: this test is flaky s.Given(). RolloutObjects("@alb/rollout-alb-experiment-no-setweight.yaml"). When(). @@ -272,6 +338,7 @@ func (s *AWSSuite) TestALBExperimentStepNoSetWeight() { } func (s *AWSSuite) TestALBExperimentStepNoSetWeightMultiIngress() { + //TODO: this test is flaky s.Given(). RolloutObjects("@alb/rollout-alb-multi-ingress-experiment-no-setweight.yaml"). When(). diff --git a/test/e2e/canary_test.go b/test/e2e/canary_test.go index 2c68ec8bd7..195e08f9db 100644 --- a/test/e2e/canary_test.go +++ b/test/e2e/canary_test.go @@ -4,6 +4,7 @@ package e2e import ( + "github.com/argoproj/argo-rollouts/utils/conditions" "log" "testing" "time" @@ -47,7 +48,7 @@ func (s *CanarySuite) TestCanarySetCanaryScale() { - pause: {duration: 5s} ` s.Given(). - RolloutTemplate("@functional/nginx-template.yaml", "set-canary-scale"). + RolloutTemplate("@functional/nginx-template.yaml", map[string]string{"REPLACEME": "set-canary-scale"}). SetSteps(canarySteps). When(). ApplyManifests(). @@ -149,7 +150,10 @@ spec: spec: containers: - name: updatescaling - command: [/bad-command]`). + resources: + requests: + memory: 16Mi + cpu: 2m`). WaitForRolloutReplicas(7). Then(). ExpectCanaryStablePodCount(4, 3). @@ -601,6 +605,7 @@ func (s *CanarySuite) TestCanaryWithPausedRollout() { WaitForRolloutStatus("Paused"). UpdateSpec(). // update to revision 3 WaitForRolloutStatus("Paused"). + Sleep(1*time.Second). Then(). ExpectRevisionPodCount("1", 3). ExpectRevisionPodCount("2", 0). @@ -628,7 +633,7 @@ func (s *CanarySuite) TestCanaryDynamicStableScale() { ApplyManifests(). MarkPodsReady("1", 4). // mark all 4 pods ready WaitForRolloutStatus("Healthy"). - UpdateSpec(). // update to revision 2 + UpdateSpec(). // update to revision 2 MarkPodsReady("2", 1). // mark 1 of 1 canary pods ready WaitForRolloutStatus("Paused"). Sleep(2*time.Second). @@ -658,6 +663,7 @@ func (s *CanarySuite) TestCanaryDynamicStableScale() { When(). MarkPodsReady("1", 1). // mark last remaining stable pod as ready (4/4 stable are ready) WaitForRevisionPodCount("2", 0). + Sleep(2*time.Second). //WaitForRevisionPodCount does not wait for terminating pods and so ExpectServiceSelector fails sleep a bit for the terminating pods to be deleted Then(). // Expect that the canary service selector is now set to stable because of dynamic stable scale is over and we have all pods up on stable rs ExpectServiceSelector("dynamic-stable-scale-canary", map[string]string{"app": "dynamic-stable-scale", "rollouts-pod-template-hash": "868d98995b"}, false). @@ -703,7 +709,7 @@ func (s *CanarySuite) TestCanaryDynamicStableScaleRollbackToStable() { assert.Equal(s.T(), int32(75), ro.Status.Canary.Weights.Stable.Weight) }). When(). - MarkPodsReady("3", 1). // marks the 4th pod of stableRS/newRS (revision 3) ready + MarkPodsReady("3", 1). // marks the 4th pod of stableRS/newRS (revision 3) ready WaitForRevisionPodCount("2", 0). // make sure we scale down the previous desired (revision 2) Then(). Assert(func(t *fixtures.Then) { @@ -722,3 +728,56 @@ func (s *CanarySuite) TestCanaryDynamicStableScaleRollbackToStable() { }) } + +// TestCanaryWithSpecPause This tests that we do not go into a update loop where we flap back updating condition patches. +func (s *CanarySuite) TestCanaryWithSpecPause() { + s.Given(). + RolloutObjects(` +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: rollout-canary-with-pause +spec: + replicas: 3 + revisionHistoryLimit: 2 + progressDeadlineSeconds: 5 + selector: + matchLabels: + app: rollout-canary-with-pause + template: + metadata: + labels: + app: rollout-canary-with-pause + spec: + containers: + - name: rollouts-demo + image: nginx:1.19-alpine + ports: + - containerPort: 80 + readinessProbe: + initialDelaySeconds: 10 + httpGet: + path: / + port: 80 + periodSeconds: 30 + strategy: + canary: + steps: + - setWeight: 20 +`). + When(). + ApplyManifests(). + WaitForRolloutStatus("Degraded"). + WaitForRolloutStatus("Healthy"). + WaitForRolloutReplicas(3). + UpdateSpec(`{"spec":{"paused": true}}`). + WaitForRolloutConditionToNotExist(func(ro *v1alpha1.Rollout) bool { + for _, c := range ro.Status.Conditions { + if c.Type == v1alpha1.RolloutProgressing && c.Reason == conditions.RolloutPausedReason && c.Status == corev1.ConditionUnknown { + return true + } + } + return false + }, "ProgressingPausedUnknown", 5*time.Second). + WaitForRolloutStatus("Paused") +} diff --git a/test/e2e/functional/albmesh-pingpong-stablecanary-rollout.yaml b/test/e2e/functional/albmesh-pingpong-stablecanary-rollout.yaml new file mode 100644 index 0000000000..0476b1076e --- /dev/null +++ b/test/e2e/functional/albmesh-pingpong-stablecanary-rollout.yaml @@ -0,0 +1,139 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: ping-service +spec: + type: NodePort + ports: + - port: 80 + targetPort: http + protocol: TCP + name: http + selector: + app: alb-canary +--- +apiVersion: v1 +kind: Service +metadata: + name: pong-service +spec: + type: NodePort + ports: + - port: 80 + targetPort: http + protocol: TCP + name: http + selector: + app: alb-canary +--- +--- +apiVersion: v1 +kind: Service +metadata: + name: stable-service +spec: + type: NodePort + ports: + - port: 80 + targetPort: http + protocol: TCP + name: http + selector: + app: alb-canary +--- +apiVersion: v1 +kind: Service +metadata: + name: canary-service +spec: + type: NodePort + ports: + - port: 80 + targetPort: http + protocol: TCP + name: http + selector: + app: alb-canary +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: alb-canary-ingress + annotations: + kubernetes.io/ingress.class: alb +spec: + rules: + - http: + paths: + - path: /* + backend: + service: + name: alb-rollout-root + port: + name: use-annotation + pathType: ImplementationSpecific +--- +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: alb-canary +spec: + replicas: 2 + selector: + matchLabels: + app: alb-canary + template: + metadata: + labels: + app: alb-canary + spec: + containers: + - name: alb-canary + image: "argoproj/rollouts-demo:red" + ports: + - name: http + containerPort: 80 + protocol: TCP + resources: + requests: + memory: 16Mi + cpu: 5m + strategy: + canary: + scaleDownDelaySeconds: 2 + canaryService: canary-service + stableService: stable-service + pingPong: + pingService: ping-service + pongService: pong-service + trafficRouting: + alb: + ingress: alb-canary-ingress + rootService: alb-rollout-root + servicePort: 80 + istio: + virtualService: + name: istio-host-split-vsvc + routes: + - primary + steps: + - setWeight: 25 + - pause: {} +--- +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: istio-host-split-vsvc +spec: + hosts: + - istio-host-split + http: + - name: primary + route: + - destination: + host: stable-service + weight: 100 + - destination: + host: canary-service + weight: 0 diff --git a/test/e2e/functional/analysistemplate-sleep-job.yaml b/test/e2e/functional/analysistemplate-sleep-job.yaml index 86faa2c877..4fdd369dee 100644 --- a/test/e2e/functional/analysistemplate-sleep-job.yaml +++ b/test/e2e/functional/analysistemplate-sleep-job.yaml @@ -6,7 +6,7 @@ metadata: spec: args: - name: duration - value: 0s + value: "0" - name: exit-code value: "0" - name: count diff --git a/test/e2e/functional_test.go b/test/e2e/functional_test.go index 930c7d0c61..8cc95f9914 100644 --- a/test/e2e/functional_test.go +++ b/test/e2e/functional_test.go @@ -91,26 +91,26 @@ spec: ExpectRevisionPodCount("2", 1). ExpectRolloutEvents([]string{ "RolloutAddedToInformer", // Rollout added to informer cache - "RolloutNotCompleted", // Rollout not completed, started update to revision 0 (7fd9b5545c) "RolloutUpdated", // Rollout updated to revision 1 "NewReplicaSetCreated", // Created ReplicaSet abort-retry-promote-698fbfb9dc (revision 1) + "RolloutNotCompleted", // Rollout not completed, started update to revision 2 (7fd9b5545c) "ScalingReplicaSet", // Scaled up ReplicaSet abort-retry-promote-698fbfb9dc (revision 1) from 0 to 1 "RolloutCompleted", // Rollout completed update to revision 1 (698fbfb9dc): Initial deploy - "RolloutNotCompleted", - "RolloutUpdated", // Rollout updated to revision 2 - "NewReplicaSetCreated", // Created ReplicaSet abort-retry-promote-75dcb5ddd6 (revision 2) - "ScalingReplicaSet", // Scaled up ReplicaSet abort-retry-promote-75dcb5ddd6 (revision 2) from 0 to 1 - "RolloutStepCompleted", // Rollout step 1/2 completed (setWeight: 50) - "RolloutPaused", // Rollout is paused (CanaryPauseStep) - "ScalingReplicaSet", // Scaled down ReplicaSet abort-retry-promote-75dcb5ddd6 (revision 2) from 1 to 0 - "RolloutAborted", // Rollout aborted update to revision 2 - "ScalingReplicaSet", // Scaled up ReplicaSet abort-retry-promote-75dcb5ddd6 (revision 2) from 0 to 1 - "RolloutStepCompleted", // Rollout step 1/2 completed (setWeight: 50) - "RolloutPaused", // Rollout is paused (CanaryPauseStep) - "RolloutStepCompleted", // Rollout step 2/2 completed (pause: 3s) - "RolloutResumed", // Rollout is resumed - "ScalingReplicaSet", // Scaled down ReplicaSet abort-retry-promote-698fbfb9dc (revision 1) from 1 to 0 - "RolloutCompleted", // Rollout completed update to revision 2 (75dcb5ddd6): Completed all 2 canary steps + "RolloutUpdated", // Rollout updated to revision 2 + "NewReplicaSetCreated", // Created ReplicaSet abort-retry-promote-75dcb5ddd6 (revision 2) + "RolloutNotCompleted", // Rollout not completed, started update to revision 3 (5bb7978cd) + "ScalingReplicaSet", // Scaled up ReplicaSet abort-retry-promote-75dcb5ddd6 (revision 2) from 0 to 1 + "RolloutStepCompleted", // Rollout step 1/2 completed (setWeight: 50) + "RolloutPaused", // Rollout is paused (CanaryPauseStep) + "ScalingReplicaSet", // Scaled down ReplicaSet abort-retry-promote-75dcb5ddd6 (revision 2) from 1 to 0 + "RolloutAborted", // Rollout aborted update to revision 2 + "ScalingReplicaSet", // Scaled up ReplicaSet abort-retry-promote-75dcb5ddd6 (revision 2) from 0 to 1 + "RolloutStepCompleted", // Rollout step 1/2 completed (setWeight: 50) + "RolloutPaused", // Rollout is paused (CanaryPauseStep) + "RolloutStepCompleted", // Rollout step 2/2 completed (pause: 3s) + "RolloutResumed", // Rollout is resumed + "ScalingReplicaSet", // Scaled down ReplicaSet abort-retry-promote-698fbfb9dc (revision 1) from 1 to 0 + "RolloutCompleted", // Rollout completed update to revision 2 (75dcb5ddd6): Completed all 2 canary steps }) } @@ -166,13 +166,14 @@ spec: UpdateSpec(). WaitForRolloutStatus("Paused"). // At step 1 (pause: {duration: 24h}) PromoteRollout(). - Sleep(2*time.Second). + Sleep(3*time.Second). + WaitForInlineAnalysisRunPhase("Running"). Then(). + ExpectRolloutStatus("Progressing"). // At step 2 (analysis: sleep-job - 24h) + ExpectAnalysisRunCount(1). ExpectRollout("status.currentStepIndex == 1", func(r *v1alpha1.Rollout) bool { return *r.Status.CurrentStepIndex == 1 }). - ExpectRolloutStatus("Progressing"). // At step 2 (analysis: sleep-job - 24h) - ExpectAnalysisRunCount(1). When(). PromoteRollout(). Sleep(2 * time.Second). @@ -205,6 +206,9 @@ spec: prePromotionAnalysis: templates: - templateName: sleep-job + args: + - name: duration + value: "10" postPromotionAnalysis: templates: - templateName: sleep-job @@ -228,7 +232,8 @@ spec: ApplyManifests(). WaitForRolloutStatus("Healthy"). UpdateSpec(). - Sleep(3 * time.Second). + Sleep(5 * time.Second). + WaitForPrePromotionAnalysisRunPhase("Running"). PromoteRolloutFull(). WaitForRolloutStatus("Healthy"). Then(). @@ -1610,3 +1615,62 @@ spec: Then(). ExpectDeploymentReplicasCount("The deployment has not been scaled", "rollout-ref-deployment", 2) } + +func (s *FunctionalSuite) TestSpecAndReplicaChangeSameTime() { + s.Given(). + HealthyRollout(` +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: canary-change-same-time +spec: + replicas: 2 + strategy: + canary: + steps: + - setWeight: 10 + - pause: {duration: 10s} + - setWeight: 20 + - pause: {duration: 5s} + selector: + matchLabels: + app: canary-change-same-time + template: + metadata: + labels: + app: canary-change-same-time + spec: + containers: + - name: canary-change-same-time + image: nginx:1.19-alpine + resources: + requests: + memory: 16Mi + cpu: 1m +`). + When(). + WaitForRolloutStatus("Healthy"). + PatchSpec(` +spec: + replicas: 3 + template: + spec: + containers: + - name: canary-change-same-time + env: + - name: TEST + value: test`). + WaitForRolloutStatus("Paused"). + PatchSpec(` +spec: + replicas: 4 + template: + spec: + containers: + - name: canary-change-same-time + env: + - name: TEST + value: test-new`). + WaitForRolloutStatus("Healthy").Then(). + ExpectReplicaCounts(4, 4, 4, 4, 4) +} diff --git a/test/e2e/istio_test.go b/test/e2e/istio_test.go index 9797b3e503..7370dc5238 100644 --- a/test/e2e/istio_test.go +++ b/test/e2e/istio_test.go @@ -335,6 +335,7 @@ func (s *IstioSuite) TestIstioUpdateInMiddleZeroCanaryReplicas() { ExpectRevisionPodCount("2", 1). When(). UpdateSpec(). + WaitForRevisionPodCount("3", 1). WaitForRolloutStatus("Paused"). Then(). ExpectRevisionPodCount("3", 1) diff --git a/test/e2e/step-plugin/argo-rollouts-config.yaml b/test/e2e/step-plugin/argo-rollouts-config.yaml new file mode 100644 index 0000000000..281d851d7a --- /dev/null +++ b/test/e2e/step-plugin/argo-rollouts-config.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: rollouts-controller + app.kubernetes.io/name: argo-rollouts + app.kubernetes.io/part-of: argo-rollouts + name: argo-rollouts-config + namespace: argo-rollouts +data: + stepPlugins: |- + - name: "step/e2e-test" # name of the plugin, it must match the name required by the plugin so it can find it's configuration + location: "file://plugin-bin/e2e-step-plugin" # supports http(s):// urls and file:// + disabled: false + - name: "step/e2e-test-disabled" # name of the plugin, it must match the name required by the plugin so it can find it's configuration + location: "file://plugin-bin/e2e-step-plugin" # supports http(s):// urls and file:// + disabled: true diff --git a/test/e2e/step-plugin/disabled-rollout.yaml b/test/e2e/step-plugin/disabled-rollout.yaml new file mode 100644 index 0000000000..3ebc15c679 --- /dev/null +++ b/test/e2e/step-plugin/disabled-rollout.yaml @@ -0,0 +1,31 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: disabled-step-plugin +spec: + selector: + matchLabels: + app: disabled-step-plugin + strategy: + canary: + steps: + - plugin: + name: $$disabled_plugin$$ + config: + return: Successful + - plugin: + name: step/e2e-test + config: + return: Successful + template: + metadata: + labels: + app: disabled-step-plugin + spec: + containers: + - name: basic + image: nginx:1.19-alpine + resources: + requests: + memory: 16Mi + cpu: 1m diff --git a/test/e2e/step-plugin/invalid-rollout.yaml b/test/e2e/step-plugin/invalid-rollout.yaml new file mode 100644 index 0000000000..df75248db1 --- /dev/null +++ b/test/e2e/step-plugin/invalid-rollout.yaml @@ -0,0 +1,26 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: invalid-step-plugin +spec: + progressDeadlineSeconds: 15 + selector: + matchLabels: + app: invalid-step-plugin + strategy: + canary: + steps: + - plugin: + name: step/e2e-test-invalid-name + template: + metadata: + labels: + app: invalid-step-plugin + spec: + containers: + - name: basic + image: nginx:1.19-alpine + resources: + requests: + memory: 16Mi + cpu: 1m diff --git a/test/e2e/step-plugin/template-rollout.yaml b/test/e2e/step-plugin/template-rollout.yaml new file mode 100644 index 0000000000..1d4d064662 --- /dev/null +++ b/test/e2e/step-plugin/template-rollout.yaml @@ -0,0 +1,32 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: step-plugin-e2e-$$name$$ +spec: + selector: + matchLabels: + app: step-plugin-e2e-$$name$$ + strategy: + canary: + steps: + - plugin: + name: step/e2e-test + config: + return: Successful + - plugin: + name: step/e2e-test + config: + return: $$phase$$ + requeue: 17s + template: + metadata: + labels: + app: step-plugin-e2e-$$name$$ + spec: + containers: + - name: basic + image: nginx:1.19-alpine + resources: + requests: + memory: 16Mi + cpu: 1m diff --git a/test/e2e/step_plugin_test.go b/test/e2e/step_plugin_test.go new file mode 100644 index 0000000000..5a938f3f51 --- /dev/null +++ b/test/e2e/step_plugin_test.go @@ -0,0 +1,338 @@ +//go:build e2e +// +build e2e + +package e2e + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/suite" + "github.com/tj/assert" + "gopkg.in/yaml.v2" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/test/fixtures" + "github.com/argoproj/argo-rollouts/utils/plugin/types" + corev1 "k8s.io/api/core/v1" +) + +const E2EStepPluginName = "step/e2e-test" +const E2EStepPluginNameDisabled = "step/e2e-test-disabled" + +type StepPluginSuite struct { + fixtures.E2ESuite +} + +func TestStepPluginSuite(t *testing.T) { + suite.Run(t, new(StepPluginSuite)) +} + +func (s *StepPluginSuite) SetupSuite() { + s.E2ESuite.SetupSuite() + if !IsStepPluginConfigured(&s.Common, s.GetControllerConfig()) { + s.T().SkipNow() + } +} + +// getControllerConfiguredPlugin look at the controller default config map to find the list of plugins +// This is a best effort because it does not mean the controller have these plugins configured in-memory +func IsStepPluginConfigured(c *fixtures.Common, config *corev1.ConfigMap) bool { + if config == nil { + return false + } + + var stepPlugins []types.PluginItem + if err := yaml.Unmarshal([]byte(config.Data["stepPlugins"]), &stepPlugins); err != nil { + c.CheckError(err) + } + + hasPlugin := false + hasPluginDisabled := false + for _, p := range stepPlugins { + if p.Name == E2EStepPluginName { + hasPlugin = true + } + if p.Name == E2EStepPluginNameDisabled { + hasPluginDisabled = true + } + } + return hasPlugin && hasPluginDisabled +} + +// test step plugin ignored when disabled +// test rollout error when step plugin not configured + +func (s *StepPluginSuite) TestRolloutCompletesWhenStepSuccessful() { + s.Given(). + RolloutTemplate("@step-plugin/template-rollout.yaml", map[string]string{"$$name$$": "successful", "$$phase$$": string(types.PhaseSuccessful)}). + When(). + ApplyManifests().WaitForRolloutStatus("Healthy"). + UpdateSpec().WaitForRolloutStatus("Healthy"). + Then(). + ExpectStableRevision("2"). + Assert(func(t *fixtures.Then) { + rollout := t.GetRollout() + assert.EqualValues(s.T(), 2, len(rollout.Status.Canary.StepPluginStatuses)) + + stepStatus := rollout.Status.Canary.StepPluginStatuses[1] + assert.EqualValues(s.T(), E2EStepPluginName, stepStatus.Name) + assert.EqualValues(s.T(), 1, stepStatus.Index) + assert.EqualValues(s.T(), v1alpha1.StepPluginPhaseSuccessful, stepStatus.Phase) + assert.EqualValues(s.T(), v1alpha1.StepPluginOperationRun, stepStatus.Operation) + }) +} + +func (s *StepPluginSuite) TestRolloutAbortWhenStepFails() { + s.Given(). + RolloutTemplate("@step-plugin/template-rollout.yaml", map[string]string{"$$name$$": "failed", "$$phase$$": string(types.PhaseFailed)}). + When(). + ApplyManifests().WaitForRolloutStatus("Healthy"). + UpdateSpec().WaitForRolloutStatus("Degraded"). + Then(). + ExpectStableRevision("1"). + Assert(func(t *fixtures.Then) { + rollout := t.GetRollout() + assert.True(s.T(), rollout.Status.Abort) + assert.EqualValues(s.T(), 3, len(rollout.Status.Canary.StepPluginStatuses)) + + stepStatus := rollout.Status.Canary.StepPluginStatuses[1] + assert.EqualValues(s.T(), E2EStepPluginName, stepStatus.Name) + assert.EqualValues(s.T(), 1, stepStatus.Index) + assert.EqualValues(s.T(), v1alpha1.StepPluginPhaseFailed, stepStatus.Phase) + assert.EqualValues(s.T(), v1alpha1.StepPluginOperationRun, stepStatus.Operation) + + stepStatus = rollout.Status.Canary.StepPluginStatuses[2] + assert.EqualValues(s.T(), E2EStepPluginName, stepStatus.Name) + assert.EqualValues(s.T(), 0, stepStatus.Index) + assert.EqualValues(s.T(), v1alpha1.StepPluginPhaseSuccessful, stepStatus.Phase) + assert.EqualValues(s.T(), v1alpha1.StepPluginOperationAbort, stepStatus.Operation) + }) +} + +func (s *StepPluginSuite) TestRolloutAbortStepsWhenAborted() { + s.Given(). + RolloutTemplate("@step-plugin/template-rollout.yaml", map[string]string{"$$name$$": "aborted", "$$phase$$": string(types.PhaseRunning)}). + When(). + ApplyManifests().WaitForRolloutStatus("Healthy"). + UpdateSpec().WaitForRolloutStatus("Progressing"). + WaitForRolloutCanaryStepIndex(1). + WaitForRolloutStepPluginRunning(). + AbortRollout().WaitForRolloutStatus("Degraded"). + Then(). + Assert(func(t *fixtures.Then) { + rollout := t.GetRollout() + assert.True(s.T(), rollout.Status.Abort) + assert.EqualValues(s.T(), 4, len(rollout.Status.Canary.StepPluginStatuses)) + + stepStatus := rollout.Status.Canary.StepPluginStatuses[1] + assert.EqualValues(s.T(), E2EStepPluginName, stepStatus.Name) + assert.EqualValues(s.T(), 1, stepStatus.Index) + assert.EqualValues(s.T(), v1alpha1.StepPluginPhaseRunning, stepStatus.Phase) + assert.EqualValues(s.T(), v1alpha1.StepPluginOperationRun, stepStatus.Operation) + + stepStatus = rollout.Status.Canary.StepPluginStatuses[2] + assert.EqualValues(s.T(), E2EStepPluginName, stepStatus.Name) + assert.EqualValues(s.T(), 1, stepStatus.Index) + assert.EqualValues(s.T(), v1alpha1.StepPluginPhaseSuccessful, stepStatus.Phase) + assert.EqualValues(s.T(), v1alpha1.StepPluginOperationAbort, stepStatus.Operation) + + stepStatus = rollout.Status.Canary.StepPluginStatuses[3] + assert.EqualValues(s.T(), E2EStepPluginName, stepStatus.Name) + assert.EqualValues(s.T(), 0, stepStatus.Index) + assert.EqualValues(s.T(), v1alpha1.StepPluginPhaseSuccessful, stepStatus.Phase) + assert.EqualValues(s.T(), v1alpha1.StepPluginOperationAbort, stepStatus.Operation) + }) +} + +func (s *StepPluginSuite) TestRolloutRetryAbortedRollout() { + s.Given(). + RolloutTemplate("@step-plugin/template-rollout.yaml", map[string]string{"$$name$$": "aborted", "$$phase$$": string(types.PhaseRunning)}). + When(). + ApplyManifests().WaitForRolloutStatus("Healthy"). + UpdateSpec().WaitForRolloutStatus("Progressing"). + WaitForRolloutCanaryStepIndex(1). + WaitForRolloutStepPluginRunning(). + AbortRollout().WaitForRolloutStatus("Degraded"). + RetryRollout().WaitForRolloutStepPluginRunning(). + Then(). + Assert(func(t *fixtures.Then) { + rollout := t.GetRollout() + assert.False(s.T(), rollout.Status.Abort) + assert.EqualValues(s.T(), 2, len(rollout.Status.Canary.StepPluginStatuses)) + + stepStatus := rollout.Status.Canary.StepPluginStatuses[1] + assert.EqualValues(s.T(), E2EStepPluginName, stepStatus.Name) + assert.EqualValues(s.T(), 1, stepStatus.Index) + assert.EqualValues(s.T(), v1alpha1.StepPluginPhaseRunning, stepStatus.Phase) + assert.EqualValues(s.T(), v1alpha1.StepPluginOperationRun, stepStatus.Operation) + }) +} + +func (s *StepPluginSuite) TestRolloutCompletesWhenPromotedAndStepRunning() { + s.Given(). + RolloutTemplate("@step-plugin/template-rollout.yaml", map[string]string{"$$name$$": "full-promotion", "$$phase$$": string(types.PhaseRunning)}). + When(). + ApplyManifests().WaitForRolloutStatus("Healthy"). + UpdateSpec().WaitForRolloutStatus("Progressing"). + WaitForRolloutCanaryStepIndex(1). + WaitForRolloutStepPluginRunning(). + PromoteRolloutFull().WaitForRolloutStatus("Healthy"). + Then(). + Assert(func(t *fixtures.Then) { + rollout := t.GetRollout() + assert.EqualValues(s.T(), 3, len(rollout.Status.Canary.StepPluginStatuses)) + + stepStatus := rollout.Status.Canary.StepPluginStatuses[1] + assert.EqualValues(s.T(), E2EStepPluginName, stepStatus.Name) + assert.EqualValues(s.T(), 1, stepStatus.Index) + assert.EqualValues(s.T(), v1alpha1.StepPluginPhaseRunning, stepStatus.Phase) + assert.EqualValues(s.T(), v1alpha1.StepPluginOperationRun, stepStatus.Operation) + + stepStatus = rollout.Status.Canary.StepPluginStatuses[2] + assert.EqualValues(s.T(), E2EStepPluginName, stepStatus.Name) + assert.EqualValues(s.T(), 1, stepStatus.Index) + assert.EqualValues(s.T(), v1alpha1.StepPluginPhaseSuccessful, stepStatus.Phase) + assert.EqualValues(s.T(), v1alpha1.StepPluginOperationTerminate, stepStatus.Operation) + }) +} + +func (s *StepPluginSuite) TestRolloutRetriesUntilDeadlineWhenRunning() { + s.Given(). + RolloutTemplate("@step-plugin/template-rollout.yaml", map[string]string{"$$name$$": "running", "$$phase$$": string(types.PhaseRunning)}). + When().ApplyManifests().WaitForRolloutStatus("Healthy"). + UpdateSpec(` +spec: + progressDeadlineSeconds: 45`). + UpdateSpec().WaitForRolloutStatus("Progressing"). + WaitForRolloutCanaryStepIndex(1). + WaitForRolloutStepPluginRunning(). + Wait(50 * time.Second). + WaitForRolloutStatus("Degraded"). + Then(). + Assert(func(t *fixtures.Then) { + rollout := t.GetRollout() + assert.EqualValues(s.T(), 1, *rollout.Status.CurrentStepIndex) + assert.EqualValues(s.T(), 2, len(rollout.Status.Canary.StepPluginStatuses)) + + stepStatus := rollout.Status.Canary.StepPluginStatuses[1] + assert.EqualValues(s.T(), E2EStepPluginName, stepStatus.Name) + assert.EqualValues(s.T(), 1, stepStatus.Index) + assert.EqualValues(s.T(), v1alpha1.StepPluginPhaseRunning, stepStatus.Phase) + assert.EqualValues(s.T(), v1alpha1.StepPluginOperationRun, stepStatus.Operation) + assert.NotEmpty(s.T(), stepStatus.Status) + + // Performance test validating that the plugin is not constantly executed + assert.EqualValues(s.T(), "17s", stepStatus.Backoff) + assert.EqualValues(s.T(), 3, stepStatus.Executions) + + }) +} + +func (s *StepPluginSuite) TestRolloutRetriesUntilDeadlineWhenError() { + s.Given(). + RolloutTemplate("@step-plugin/template-rollout.yaml", map[string]string{"$$name$$": "error", "$$phase$$": "invalidPhaseCausingPluginError"}). + When().ApplyManifests().WaitForRolloutStatus("Healthy"). + UpdateSpec(` +spec: + progressDeadlineSeconds: 45`). + UpdateSpec().WaitForRolloutStatus("Progressing"). + Wait(50 * time.Second). + WaitForRolloutStatus("Degraded"). + Then(). + Assert(func(t *fixtures.Then) { + rollout := t.GetRollout() + assert.EqualValues(s.T(), 1, *rollout.Status.CurrentStepIndex) + assert.EqualValues(s.T(), 2, len(rollout.Status.Canary.StepPluginStatuses)) + + stepStatus := rollout.Status.Canary.StepPluginStatuses[1] + assert.EqualValues(s.T(), E2EStepPluginName, stepStatus.Name) + assert.EqualValues(s.T(), 1, stepStatus.Index) + assert.EqualValues(s.T(), v1alpha1.StepPluginPhaseError, stepStatus.Phase) + assert.EqualValues(s.T(), v1alpha1.StepPluginOperationRun, stepStatus.Operation) + assert.Contains(s.T(), stepStatus.Message, "phase 'invalidPhaseCausingPluginError' is not valid") + assert.Empty(s.T(), stepStatus.Status) + + // Performance test validating that the plugin is not constantly executed + assert.EqualValues(s.T(), "30s", stepStatus.Backoff) + assert.EqualValues(s.T(), 2, stepStatus.Executions) + + }) +} + +func (s *StepPluginSuite) TestRolloutStatusIsNotUsedOnNewRollout() { + s.Given(). + RolloutTemplate("@step-plugin/template-rollout.yaml", map[string]string{"$$name$$": "run-again", "$$phase$$": string(types.PhaseSuccessful)}). + When(). + ApplyManifests().WaitForRolloutStatus("Healthy"). + UpdateSpec().WaitForRolloutStatus("Healthy").Then().ExpectStableRevision("2").When(). + UpdateSpec(` +spec: + strategy: + canary: + steps: + - plugin: + name: step/e2e-test + config: + return: Successful`). + UpdateSpec().WaitForRolloutStatus("Healthy"). + Then(). + ExpectStableRevision("3"). + Assert(func(t *fixtures.Then) { + rollout := t.GetRollout() + assert.EqualValues(s.T(), 1, len(rollout.Status.Canary.StepPluginStatuses)) + + stepStatus := rollout.Status.Canary.StepPluginStatuses[0] + assert.EqualValues(s.T(), E2EStepPluginName, stepStatus.Name) + assert.EqualValues(s.T(), 0, stepStatus.Index) + assert.EqualValues(s.T(), v1alpha1.StepPluginPhaseSuccessful, stepStatus.Phase) + assert.EqualValues(s.T(), v1alpha1.StepPluginOperationRun, stepStatus.Operation) + }) +} + +func (s *StepPluginSuite) TestRolloutErrorWhenStepPluginNotConfigured() { + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + s.Given(). + StartEventWatch(ctx). + RolloutObjects("@step-plugin/invalid-rollout.yaml"). + When(). + ApplyManifests().WaitForRolloutStatus("Healthy"). + UpdateSpec().WaitForRolloutStatus("Degraded"). + Then(). + ExpectStableRevision("1"). + ExpectRolloutEventsContains([]string{"ReconciliationError"}). + Assert(func(t *fixtures.Then) { + rollout := t.GetRollout() + assert.EqualValues(s.T(), 0, *rollout.Status.CurrentStepIndex) + assert.EqualValues(s.T(), 0, len(rollout.Status.Canary.StepPluginStatuses)) + }) +} + +func (s *StepPluginSuite) TestRolloutSkipPluginWhenDisabled() { + s.Given(). + RolloutTemplate("@step-plugin/disabled-rollout.yaml", map[string]string{"$$disabled_plugin$$": E2EStepPluginNameDisabled}). + When(). + ApplyManifests().WaitForRolloutStatus("Healthy"). + UpdateSpec().WaitForRolloutStatus("Healthy"). + Then(). + ExpectStableRevision("2"). + Assert(func(t *fixtures.Then) { + rollout := t.GetRollout() + assert.EqualValues(s.T(), 2, len(rollout.Status.Canary.StepPluginStatuses)) + + stepStatus := rollout.Status.Canary.StepPluginStatuses[0] + assert.EqualValues(s.T(), E2EStepPluginNameDisabled, stepStatus.Name) + assert.EqualValues(s.T(), 0, stepStatus.Index) + assert.EqualValues(s.T(), v1alpha1.StepPluginOperationRun, stepStatus.Operation) + assert.EqualValues(s.T(), true, stepStatus.Disabled) + + stepStatus = rollout.Status.Canary.StepPluginStatuses[1] + assert.EqualValues(s.T(), E2EStepPluginName, stepStatus.Name) + assert.EqualValues(s.T(), 1, stepStatus.Index) + assert.EqualValues(s.T(), v1alpha1.StepPluginPhaseSuccessful, stepStatus.Phase) + assert.EqualValues(s.T(), v1alpha1.StepPluginOperationRun, stepStatus.Operation) + assert.EqualValues(s.T(), false, stepStatus.Disabled) + }) +} diff --git a/test/fixtures/common.go b/test/fixtures/common.go index 670d2dd2f4..6b9cdf9733 100644 --- a/test/fixtures/common.go +++ b/test/fixtures/common.go @@ -12,6 +12,8 @@ import ( "text/tabwriter" "time" + k8errors "k8s.io/apimachinery/pkg/api/errors" + a6util "github.com/argoproj/argo-rollouts/utils/apisix" smiv1alpha1 "github.com/servicemeshinterface/smi-sdk-go/pkg/apis/split/v1alpha1" @@ -681,6 +683,17 @@ func (c *Common) GetRolloutEventReasons() []string { return reasons } +func (c *Common) GetControllerConfig() *corev1.ConfigMap { + configMap, err := c.kubeClient.CoreV1().ConfigMaps(c.namespace).Get(c.Context, "argo-rollouts-config", metav1.GetOptions{}) + if err != nil { + if k8errors.IsNotFound(err) { + return nil + } + c.CheckError(err) + } + return configMap +} + // PrintRolloutEvents prints all Kubernetes events associated with the given rollout. func (c *Common) PrintRolloutEvents(ro *v1alpha1.Rollout) { opts := metav1.ListOptions{FieldSelector: fields.ParseSelectorOrDie(fmt.Sprintf("involvedObject.uid=%s", ro.UID)).String()} diff --git a/test/fixtures/e2e_suite.go b/test/fixtures/e2e_suite.go index a774afcf94..e78ae1aa68 100644 --- a/test/fixtures/e2e_suite.go +++ b/test/fixtures/e2e_suite.go @@ -54,7 +54,7 @@ const ( ) var ( - E2EWaitTimeout time.Duration = time.Second * 120 + E2EWaitTimeout time.Duration = time.Second * 90 E2EPodDelay = 0 E2EALBIngressAnnotations map[string]string @@ -143,8 +143,8 @@ func (s *E2ESuite) SetupSuite() { restConfig, err := config.ClientConfig() s.CheckError(err) s.Common.kubernetesHost = restConfig.Host - restConfig.Burst = defaults.DefaultBurst * 2 - restConfig.QPS = defaults.DefaultQPS * 2 + restConfig.Burst = defaults.DefaultBurst * 10 + restConfig.QPS = defaults.DefaultQPS * 10 s.namespace, _, err = config.Namespace() s.CheckError(err) s.kubeClient, err = kubernetes.NewForConfig(restConfig) diff --git a/test/fixtures/given.go b/test/fixtures/given.go index 57e17251c6..a6e332d69a 100644 --- a/test/fixtures/given.go +++ b/test/fixtures/given.go @@ -34,9 +34,12 @@ func (g *Given) RolloutObjects(text string) *Given { return g } -func (g *Given) RolloutTemplate(text, name string) *Given { +func (g *Given) RolloutTemplate(text string, values map[string]string) *Given { yamlBytes := g.yamlBytes(text) - newText := strings.ReplaceAll(string(yamlBytes), "REPLACEME", name) + newText := string(yamlBytes) + for k, v := range values { + newText = strings.ReplaceAll(newText, k, v) + } return g.RolloutObjects(newText) } diff --git a/test/fixtures/then.go b/test/fixtures/then.go index 8183cc2adc..68a9d822c8 100644 --- a/test/fixtures/then.go +++ b/test/fixtures/then.go @@ -13,12 +13,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + "github.com/stretchr/testify/assert" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" rov1 "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" "github.com/argoproj/argo-rollouts/utils/defaults" replicasetutil "github.com/argoproj/argo-rollouts/utils/replicaset" rolloututil "github.com/argoproj/argo-rollouts/utils/rollout" - "github.com/stretchr/testify/assert" ) type Then struct { @@ -494,6 +495,15 @@ func (t *Then) ExpectRolloutEvents(reasons []string) *Then { return t } +func (t *Then) ExpectRolloutEventsContains(reasons []string) *Then { + t.t.Helper() + eventReasons := t.GetRolloutEventReasons() + for _, r := range reasons { + assert.Contains(t.Common.t, eventReasons, r) + } + return t +} + func (t *Then) When() *When { return &When{ Common: t.Common, diff --git a/test/fixtures/when.go b/test/fixtures/when.go index d9696a7761..4f8159e5d1 100644 --- a/test/fixtures/when.go +++ b/test/fixtures/when.go @@ -415,6 +415,18 @@ func (w *When) WaitForActiveRevision(revision string, timeout ...time.Duration) return w.WaitForRolloutCondition(checkStatus, fmt.Sprintf("active revision=%s", revision), timeout...) } +func (w *When) WaitForRolloutStepPluginRunning(timeout ...time.Duration) *When { + checkStatus := func(ro *rov1.Rollout) bool { + for _, s := range ro.Status.Canary.StepPluginStatuses { + if s.Index == *ro.Status.CurrentStepIndex && s.Operation == rov1.StepPluginOperationRun && s.Phase == v1alpha1.StepPluginPhaseRunning { + return true + } + } + return false + } + return w.WaitForRolloutCondition(checkStatus, fmt.Sprintf("stepPluginStatus[currentIndex].phase=Running"), timeout...) +} + func (w *When) WaitForRolloutCondition(test func(ro *rov1.Rollout) bool, condition string, timeouts ...time.Duration) *When { start := time.Now() w.log.Infof("Waiting for condition: %s", condition) @@ -457,6 +469,45 @@ func (w *When) WaitForRolloutCondition(test func(ro *rov1.Rollout) bool, conditi } } +// WaitForRolloutConditionToNotExist this function will check for the condition to exist for the given duration, if it is found +// the test fails. +func (w *When) WaitForRolloutConditionToNotExist(test func(ro *rov1.Rollout) bool, condition string, timeout time.Duration) *When { + start := time.Now() + w.log.Infof("Waiting for condition to not exist: %s", condition) + rolloutIf := w.dynamicClient.Resource(rov1.RolloutGVR).Namespace(w.namespace) + ro, err := rolloutIf.Get(w.Context, w.rollout.GetName(), metav1.GetOptions{}) + w.CheckError(err) + retryWatcher, err := watchutil.NewRetryWatcher(ro.GetResourceVersion(), &cache.ListWatch{ + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + opts := metav1.ListOptions{FieldSelector: fields.ParseSelectorOrDie(fmt.Sprintf("metadata.name=%s", w.rollout.GetName())).String()} + return w.rolloutClient.ArgoprojV1alpha1().Rollouts(w.namespace).Watch(w.Context, opts) + }, + }) + w.CheckError(err) + defer retryWatcher.Stop() + timeoutCh := make(chan bool, 1) + go func() { + time.Sleep(timeout) + timeoutCh <- true + }() + for { + select { + case event := <-retryWatcher.ResultChan(): + ro, ok := event.Object.(*rov1.Rollout) + if ok { + if test(ro) { + //w.PrintRollout(ro) + w.log.Infof("Condition '%s' met after %v", condition, time.Since(start).Truncate(time.Second)) + w.t.Fatal("not ok") + } + } + case <-timeoutCh: + w.t.Logf("Condition %s not found after %v", condition, timeout) + return w + } + } +} + func (w *When) DeleteRollout() *When { w.log.Info("Deleting") err := w.rolloutClient.ArgoprojV1alpha1().Rollouts(w.namespace).Delete(w.Context, w.rollout.GetName(), metav1.DeleteOptions{}) diff --git a/ui/src/app/components/analysis-modal/constants.ts b/ui/src/app/components/analysis-modal/constants.ts index c1a35ce0d5..70771ba985 100644 --- a/ui/src/app/components/analysis-modal/constants.ts +++ b/ui/src/app/components/analysis-modal/constants.ts @@ -3,6 +3,7 @@ import {AnalysisStatus, FunctionalStatus} from './types'; export const METRIC_FAILURE_LIMIT_DEFAULT = 0; export const METRIC_INCONCLUSIVE_LIMIT_DEFAULT = 0; export const METRIC_CONSECUTIVE_ERROR_LIMIT_DEFAULT = 4; +export const METRIC_CONSECUTIVE_SUCCESS_LIMIT_DEFAULT = 0; export const ANALYSIS_STATUS_THEME_MAP: {[key in AnalysisStatus]: string} = { Successful: FunctionalStatus.SUCCESS, diff --git a/ui/src/app/components/analysis-modal/criteria-list/criteria-list.tsx b/ui/src/app/components/analysis-modal/criteria-list/criteria-list.tsx index 51247b6d22..04ab3fb051 100644 --- a/ui/src/app/components/analysis-modal/criteria-list/criteria-list.tsx +++ b/ui/src/app/components/analysis-modal/criteria-list/criteria-list.tsx @@ -63,11 +63,13 @@ interface CriteriaListProps { analysisStatus: AnalysisStatus; className?: string[] | string; consecutiveErrors: number; + consecutiveSuccesses: number; failures: number; inconclusives: number; maxConsecutiveErrors: number; maxFailures: number; maxInconclusives: number; + consecutiveSuccessLimit: number; showIcons: boolean; } @@ -75,25 +77,34 @@ const CriteriaList = ({ analysisStatus, className, consecutiveErrors, + consecutiveSuccesses, failures, inconclusives, maxConsecutiveErrors, maxFailures, maxInconclusives, + consecutiveSuccessLimit, showIcons, }: CriteriaListProps) => { let failureStatus = defaultCriterionStatus(analysisStatus); let errorStatus = defaultCriterionStatus(analysisStatus); let inconclusiveStatus = defaultCriterionStatus(analysisStatus); + let successStatus = defaultCriterionStatus(analysisStatus); if (analysisStatus !== AnalysisStatus.Pending && analysisStatus !== AnalysisStatus.Running) { failureStatus = failures <= maxFailures ? CriterionStatus.Pass : CriterionStatus.Fail; errorStatus = consecutiveErrors <= maxConsecutiveErrors ? CriterionStatus.Pass : CriterionStatus.Fail; inconclusiveStatus = inconclusives <= maxInconclusives ? CriterionStatus.Pass : CriterionStatus.Fail; + successStatus = consecutiveSuccesses >= consecutiveSuccessLimit ? CriterionStatus.Pass : CriterionStatus.Fail; } return (
    + {consecutiveSuccessLimit > 0 && ( + + {`Exactly ${consecutiveSuccessLimit} consecutive successes.`} + + )} {maxFailures > -1 && ( {criterionLabel('measurement failures', maxFailures)} diff --git a/ui/src/app/components/analysis-modal/panels/metric-panel.tsx b/ui/src/app/components/analysis-modal/panels/metric-panel.tsx index b1deba1c6f..4aed851308 100644 --- a/ui/src/app/components/analysis-modal/panels/metric-panel.tsx +++ b/ui/src/app/components/analysis-modal/panels/metric-panel.tsx @@ -14,7 +14,7 @@ import MetricTable from '../metric-table/metric-table'; import QueryBox from '../query-box/query-box'; import {AnalysisStatus, FunctionalStatus, TransformedMetricSpec, TransformedMetricStatus} from '../types'; import {isFiniteNumber} from '../transforms'; -import {METRIC_CONSECUTIVE_ERROR_LIMIT_DEFAULT, METRIC_FAILURE_LIMIT_DEFAULT, METRIC_INCONCLUSIVE_LIMIT_DEFAULT} from '../constants'; +import {METRIC_CONSECUTIVE_ERROR_LIMIT_DEFAULT, METRIC_FAILURE_LIMIT_DEFAULT, METRIC_INCONCLUSIVE_LIMIT_DEFAULT, METRIC_CONSECUTIVE_SUCCESS_LIMIT_DEFAULT} from '../constants'; import classNames from 'classnames'; import './styles.scss'; @@ -36,6 +36,7 @@ const MetricPanel = ({className, metricName, metricSpec, metricResults, status, const consecutiveErrorLimit = isFiniteNumber(metricSpec.consecutiveErrorLimit ?? null) ? metricSpec.consecutiveErrorLimit : METRIC_CONSECUTIVE_ERROR_LIMIT_DEFAULT; const failureLimit = isFiniteNumber(metricSpec.failureLimit ?? null) ? metricSpec.failureLimit : METRIC_FAILURE_LIMIT_DEFAULT; const inconclusiveLimit = isFiniteNumber(metricSpec.inconclusiveLimit ?? null) ? metricSpec.inconclusiveLimit : METRIC_INCONCLUSIVE_LIMIT_DEFAULT; + const consecutiveSuccessLimit = isFiniteNumber(metricSpec.consecutiveSuccessLimit ?? null) ? metricSpec.consecutiveSuccessLimit : METRIC_CONSECUTIVE_SUCCESS_LIMIT_DEFAULT; const canChartMetric = metricResults.chartable && metricResults.chartMax !== null; @@ -112,6 +113,9 @@ const MetricPanel = ({className, metricName, metricSpec, metricResults, status, maxFailures={failureLimit} // @ts-ignore maxInconclusives={inconclusiveLimit} + // @ts-ignore + consecutiveSuccessLimit={consecutiveSuccessLimit} + consecutiveSuccesses={metricResults.consecutiveSuccess ?? 0} consecutiveErrors={metricResults.consecutiveError ?? 0} failures={metricResults.failed ?? 0} inconclusives={metricResults.inconclusive ?? 0} diff --git a/ui/src/app/components/analysis-modal/transforms.test.ts b/ui/src/app/components/analysis-modal/transforms.test.ts index 5c2c2a667e..60f7b37a7e 100644 --- a/ui/src/app/components/analysis-modal/transforms.test.ts +++ b/ui/src/app/components/analysis-modal/transforms.test.ts @@ -5,6 +5,7 @@ import { GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1CloudWatchMetric, GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1MetricProvider, GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1MetricResult, + GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1Measurement, } from '../../../models/rollout/generated'; import { analysisEndTime, @@ -25,6 +26,7 @@ import { metricSubstatus, printableCloudWatchQuery, printableDatadogQuery, + transformMeasurements, } from './transforms'; import {AnalysisStatus, FunctionalStatus} from './types'; @@ -376,12 +378,12 @@ describe('analysis modal transforms', () => { }); test('interpolateQuery() for prometheus query and args', () => { expect(interpolateQuery(MOCK_QUERY_PROMETHEUS, MOCK_ARGS_PROMETHEUS)).toBe( - 'sum(irate(istio_requests_total{reporter="source",destination_service=~"istio-host-split-canary",response_code!~"5.*"}[5m])) / sum(irate(istio_requests_total{reporter="source",destination_service=~"istio-host-split-canary"}[5m]))' + 'sum(irate(istio_requests_total{reporter="source",destination_service=~"istio-host-split-canary",response_code!~"5.*"}[5m])) / sum(irate(istio_requests_total{reporter="source",destination_service=~"istio-host-split-canary"}[5m]))', ); }); test('interpolateQuery() for newrelic query and args', () => { expect(interpolateQuery(MOCK_QUERY_NEWRELIC, MOCK_ARGS_NEWRELIC)).toBe( - "FROM Transaction SELECT percentage(count(*), WHERE httpResponseCode != 500) as successRate where appName = 'myApp'" + "FROM Transaction SELECT percentage(count(*), WHERE httpResponseCode != 500) as successRate where appName = 'myApp'", ); }); test('interpolateQuery() for simple datadog query and args', () => { @@ -389,22 +391,22 @@ describe('analysis modal transforms', () => { }); test('interpolateQuery() for wavefront query and args', () => { expect(interpolateQuery(MOCK_QUERY_WAVEFRONT, MOCK_ARGS_WAVEFRONT)).toBe( - 'sum(rate(5m, ts("istio.requestcount.count", response_code!=500 and destination_service="istio-host-split-canary"))) / sum(rate(5m, ts("istio.requestcount.count", reporter=client and destination_service="istio-host-split-canary")))' + 'sum(rate(5m, ts("istio.requestcount.count", response_code!=500 and destination_service="istio-host-split-canary"))) / sum(rate(5m, ts("istio.requestcount.count", reporter=client and destination_service="istio-host-split-canary")))', ); }); test('interpolateQuery() for graphite query and args', () => { expect(interpolateQuery(MOCK_QUERY_GRAPHITE, MOCK_ARGS_GRAPHITE)).toBe( - "target=summarize(asPercent(sumSeries(stats.timers.httpServerRequests.app.istio-host-split-canary.exception.*.method.*.outcome.{CLIENT_ERROR,INFORMATIONAL,REDIRECTION,SUCCESS}.status.*.uri.*.count), sumSeries(stats.timers.httpServerRequests.app.istio-host-split-canary.exception.*.method.*.outcome.*.status.*.uri.*.count)),'5min','avg')" + "target=summarize(asPercent(sumSeries(stats.timers.httpServerRequests.app.istio-host-split-canary.exception.*.method.*.outcome.{CLIENT_ERROR,INFORMATIONAL,REDIRECTION,SUCCESS}.status.*.uri.*.count), sumSeries(stats.timers.httpServerRequests.app.istio-host-split-canary.exception.*.method.*.outcome.*.status.*.uri.*.count)),'5min','avg')", ); }); test('interpolateQuery() for influxdb query and args', () => { expect(interpolateQuery(MOCK_QUERY_INFLUXDB, MOCK_ARGS_INFLUXDB)).toBe( - 'from(bucket: "app_istio") range(start: -15m) filter(fn: (r) => r["destination_workload"] == "myApp")|> filter(fn: (r) => r["_measurement"] == "istio:istio_requests_errors_percentage:rate1m:5xx")' + 'from(bucket: "app_istio") range(start: -15m) filter(fn: (r) => r["destination_workload"] == "myApp")|> filter(fn: (r) => r["_measurement"] == "istio:istio_requests_errors_percentage:rate1m:5xx")', ); }); test('interpolateQuery() for skywalking query and args', () => { expect(interpolateQuery(MOCK_QUERY_SKYWALKING, MOCK_ARGS_SKYWALKING)).toBe( - 'query queryData($duration: Duration!) { service_apdex: readMetricsValues(condition: { name: "service_apdex", entity: { scope: Service, serviceName: "istio-host-split-canary", normal: true } }, duration: $duration) { label values { values { value } } } }' + 'query queryData($duration: Duration!) { service_apdex: readMetricsValues(condition: { name: "service_apdex", entity: { scope: Service, serviceName: "istio-host-split-canary", normal: true } }, duration: $duration) { label values { values { value } } } }', ); }); @@ -545,4 +547,62 @@ describe('analysis modal transforms', () => { tableValue: {latency: null, cpuUsage: null}, }); }); + const MOCK_MEASUREMENTS: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1Measurement[] = [{value: '[5]'}, {value: '[10]'}, {value: '[15]'}]; + const MOCK_MEASUREMENTS_WITH_NAN: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1Measurement[] = [{value: '[NaN]'}, {value: '[10]'}, {value: '[15]'}]; + const MOCK_MEASUREMENTS_WITH_STRING: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1Measurement[] = [{value: 'NaN'}, {value: '10'}, {value: '15'}]; + + it('transforms measurements correctly with valid numbers', () => { + expect(transformMeasurements(['0'], MOCK_MEASUREMENTS)).toEqual({ + chartable: true, + min: 0, + max: null, + measurements: [ + {value: '[5]', chartValue: {'0': 5}, tableValue: {'0': 5}}, + {value: '[10]', chartValue: {'0': 10}, tableValue: {'0': 10}}, + {value: '[15]', chartValue: {'0': 15}, tableValue: {'0': 15}}, + ], + }); + }); + it('transforms measurements correctly with NaN numbers in list', () => { + expect(transformMeasurements(['0'], MOCK_MEASUREMENTS_WITH_NAN)).toEqual({ + chartable: true, + min: 0, + max: null, + measurements: [ + {value: '[NaN]', chartValue: null, tableValue: null}, + {value: '[10]', chartValue: {'0': 10}, tableValue: {'0': 10}}, + {value: '[15]', chartValue: {'0': 15}, tableValue: {'0': 15}}, + ], + }); + }); + it('returns default values when measurements are undefined', () => { + expect(transformMeasurements(['0'])).toEqual({ + chartable: false, + min: 0, + max: null, + measurements: [], + }); + }); + + it('returns default values when measurements are empty', () => { + expect(transformMeasurements(['0'], [])).toEqual({ + chartable: false, + min: 0, + max: null, + measurements: [], + }); + }); + + it('handles measurements with string values', () => { + expect(transformMeasurements(['0'], MOCK_MEASUREMENTS_WITH_STRING)).toEqual({ + chartable: true, + min: 0, + max: 15, + measurements: [ + {value: 'NaN', chartValue: null, tableValue: null}, + {value: '10', chartValue: 10, tableValue: 10}, + {value: '15', chartValue: 15, tableValue: 15}, + ], + }); + }); }); diff --git a/ui/src/app/components/analysis-modal/transforms.ts b/ui/src/app/components/analysis-modal/transforms.ts index a655dda1c4..63459e4e5c 100644 --- a/ui/src/app/components/analysis-modal/transforms.ts +++ b/ui/src/app/components/analysis-modal/transforms.ts @@ -131,7 +131,7 @@ const PROVIDER_CONDITION_SUPPORT: { export const conditionDetails = ( condition?: string, args: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1Argument[] = [], - provider?: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1MetricProvider + provider?: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1MetricProvider, ): { label: string | null; thresholds: number[]; @@ -208,7 +208,7 @@ export const chartMax = (valueMax: number, failThresholds: number[] | null, succ * @param phase analysis phase * @returns analysis phase adjusted to render the UI status with a more accurate functional status */ -export const getAdjustedMetricPhase = (phase?: AnalysisStatus): AnalysisStatus => (phase === AnalysisStatus.Error ? AnalysisStatus.Failed : phase ?? AnalysisStatus.Unknown); +export const getAdjustedMetricPhase = (phase?: AnalysisStatus): AnalysisStatus => (phase === AnalysisStatus.Error ? AnalysisStatus.Failed : (phase ?? AnalysisStatus.Unknown)); /** * @@ -378,7 +378,7 @@ export const interpolateQuery = (query?: string, args?: GithubComArgoprojArgoRol */ export const printableDatadogQuery = ( datadog: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1DatadogMetric, - args: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1Argument[] + args: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1Argument[], ): string[] | undefined => { if ((datadog.apiVersion ?? '').toLowerCase() === 'v1' && 'query' in datadog) { return [interpolateQuery(datadog.query, args)]; @@ -417,7 +417,7 @@ export const printableCloudWatchQuery = (cloudWatch: GithubComArgoprojArgoRollou */ export const metricQueries = ( provider?: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1MetricProvider | null, - args: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1Argument[] = [] + args: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1Argument[] = [], ): string[] | undefined => { if (provider === undefined || provider === null) { return undefined; @@ -467,7 +467,7 @@ export const transformMeasurements = (conditionKeys: string[], measurements?: Gi return measurements.reduce( ( acc: {chartable: boolean; min: number; max: number | null; measurements: TransformedMeasurement[]}, - currMeasurement: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1Measurement + currMeasurement: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1Measurement, ) => { const transformedValue = transformMeasurementValue(conditionKeys, currMeasurement.value); const {canChart, tableValue} = transformedValue; @@ -487,7 +487,7 @@ export const transformMeasurements = (conditionKeys: string[], measurements?: Gi ], }; }, - {chartable: true, min: 0, max: null, measurements: [] as TransformedMeasurement[]} + {chartable: true, min: 0, max: null, measurements: [] as TransformedMeasurement[]}, ); }; @@ -507,7 +507,7 @@ type FormattedMeasurementValue = number | string | null; */ export const formattedValue = (value: any): FormattedMeasurementValue => { const isNum = isFiniteNumber(value); - return isNum ? roundNumber(Number(value)) : value?.toString() ?? null; + return isNum ? roundNumber(Number(value)) : (value?.toString() ?? null); }; /** @@ -533,6 +533,7 @@ const formatNumberMeasurement = (value: number): MeasurementValueInfo => { export const formatSingleItemArrayMeasurement = (value: FormattedMeasurementValue[], accessor: number): MeasurementValueInfo => { if (isFiniteNumber(accessor)) { const measurementValue = value?.[accessor] ?? null; + // if it's a number or null, chart it if (isFiniteNumber(measurementValue) || measurementValue === null) { const displayValue = formattedValue(measurementValue); @@ -617,7 +618,16 @@ const transformMeasurementValue = (conditionKeys: string[], value?: string): Mea }; } - const parsedValue = JSON.parse(value); + let parsedValue; + try { + parsedValue = JSON.parse(value); + } catch { + return { + canChart: true, + chartValue: null, + tableValue: null, + }; + } // single number measurement value if (isFiniteNumber(parsedValue)) { diff --git a/ui/src/app/components/pods/pods.tsx b/ui/src/app/components/pods/pods.tsx index 5e7005fcbd..6b44aa3d41 100644 --- a/ui/src/app/components/pods/pods.tsx +++ b/ui/src/app/components/pods/pods.tsx @@ -21,6 +21,9 @@ export enum PodStatus { } const isPodReady = (ready: string) => { + if (!ready) { + return false; + } // Ready is a string in the format "0/1", "1/1", etc. const [current, total] = ready.split('/'); return current === total; diff --git a/ui/src/app/components/rollout/rollout.scss b/ui/src/app/components/rollout/rollout.scss index ad6247c478..b93f9ed8ff 100644 --- a/ui/src/app/components/rollout/rollout.scss +++ b/ui/src/app/components/rollout/rollout.scss @@ -7,7 +7,7 @@ background: $argo-color-gray-1; border-radius: 5px; - + &__header { display: flex; align-items: center; @@ -53,11 +53,11 @@ } &--current { - border: 2px solid $sherbert; + border: 2px solid $argo-running-color; } &--current:hover { - border: 2px solid $sherbert; + border: 2px solid $argo-running-color; } &-title { @@ -97,6 +97,10 @@ padding-bottom: 4px; font-size: 14px; } + + &-value { + white-space: pre-wrap; + } } } diff --git a/ui/src/app/components/rollout/rollout.tsx b/ui/src/app/components/rollout/rollout.tsx index dfecde7e84..f0cf0037bf 100644 --- a/ui/src/app/components/rollout/rollout.tsx +++ b/ui/src/app/components/rollout/rollout.tsx @@ -1,4 +1,5 @@ import * as React from 'react'; +import * as YAML from 'yaml'; import {Helmet} from 'react-helmet'; import {useParams} from 'react-router-dom'; import { @@ -6,6 +7,7 @@ import { GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1HeaderRoutingMatch, GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1RolloutExperimentTemplate, GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1SetMirrorRoute, + GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1PluginStep, RolloutReplicaSetInfo, RolloutRolloutInfo, RolloutServiceApi, @@ -316,6 +318,7 @@ const Step = (props: {step: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1 const [openAnalysis, setOpenAnalysis] = React.useState(false); const [openHeader, setOpenHeader] = React.useState(false); const [openMirror, setOpenMirror] = React.useState(false); + const [openPlugin, setOpenPlugin] = React.useState(false); let icon: string; let content = ''; @@ -359,6 +362,11 @@ const Step = (props: {step: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1 } } + if (props.step.plugin) { + content = `${props.step.plugin.name}`; + icon = 'fa-puzzle-piece'; + } + return (
    @@ -368,7 +376,8 @@ const Step = (props: {step: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1 (props.step.setCanaryScale && openCanary) || (props.step.analysis && openAnalysis) || (props.step.setHeaderRoute && openHeader) || - (props.step.setMirrorRoute && openMirror) + (props.step.setMirrorRoute && openMirror) || + (props.step.plugin && openPlugin) ? 'steps__step-title--experiment' : '' }`} @@ -395,6 +404,11 @@ const Step = (props: {step: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1
    )} + {props.step.plugin && props.step.plugin.config && ( +
    setOpenPlugin(!openPlugin)}> + +
    + )} {props.step.experiment?.templates && (
    @@ -419,6 +433,7 @@ const Step = (props: {step: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1
    )} {props.step?.setCanaryScale && openCanary && } + {props.step?.plugin && openPlugin && } {props.step?.setHeaderRoute && openHeader && } {props.step?.setMirrorRoute && openMirror && } @@ -563,3 +578,13 @@ const WidgetItemSetHeader = ({values}: {values: GithubComArgoprojArgoRolloutsPkg ); }; + +const WidgetItemPlugin = ({values}: {values: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1PluginStep}) => { + if (!values.config) return null; + return ( +
    +
    CONFIG
    +
    {YAML.stringify(values.config)}
    +
    + ); +}; diff --git a/ui/src/models/rollout/generated/.swagger-codegen/VERSION b/ui/src/models/rollout/generated/.swagger-codegen/VERSION index 9b77657dc7..248908e600 100644 --- a/ui/src/models/rollout/generated/.swagger-codegen/VERSION +++ b/ui/src/models/rollout/generated/.swagger-codegen/VERSION @@ -1 +1 @@ -3.0.25 \ No newline at end of file +3.0.54 \ No newline at end of file diff --git a/ui/src/models/rollout/generated/api.ts b/ui/src/models/rollout/generated/api.ts index 114f705aee..642bd611ed 100755 --- a/ui/src/models/rollout/generated/api.ts +++ b/ui/src/models/rollout/generated/api.ts @@ -13,7 +13,7 @@ */ import * as url from "url"; -import * as portableFetch from "portable-fetch"; +import * as isomorphicFetch from "isomorphic-fetch"; import { Configuration } from "./configuration"; const BASE_PATH = "/".replace(/\/+$/, ""); @@ -56,13 +56,13 @@ export interface FetchArgs { export class BaseAPI { protected configuration: Configuration; - constructor(configuration?: Configuration, protected basePath: string = BASE_PATH, protected fetch: FetchAPI = portableFetch) { + constructor(configuration?: Configuration, protected basePath: string = BASE_PATH, protected fetch: FetchAPI = isomorphicFetch) { if (configuration) { this.configuration = configuration; this.basePath = configuration.basePath || this.basePath; } } -}; +} /** * @@ -71,7 +71,7 @@ export class BaseAPI { * @extends {Error} */ export class RequiredError extends Error { - name: "RequiredError" + name = "RequiredError" constructor(public field: string, msg?: string) { super(msg); } @@ -711,6 +711,12 @@ export interface GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1CanaryStatu * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1CanaryStatus */ stablePingPong?: string; + /** + * + * @type {Array} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1CanaryStatus + */ + stepPluginStatuses?: Array; } /** * CanaryStep defines a step of a canary deployment. @@ -760,6 +766,12 @@ export interface GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1CanaryStep * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1CanaryStep */ setMirrorRoute?: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1SetMirrorRoute; + /** + * + * @type {GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1PluginStep} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1CanaryStep + */ + plugin?: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1PluginStep; } /** * @@ -1043,6 +1055,12 @@ export interface GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1DatadogMetr * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1DatadogMetric */ aggregator?: string; + /** + * + * @type {GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1SecretRef} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1DatadogMetric + */ + secretRef?: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1SecretRef; } /** * DryRun defines the settings for running the analysis in Dry-Run mode. @@ -1473,6 +1491,12 @@ export interface GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1Metric { * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1Metric */ provider?: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1MetricProvider; + /** + * + * @type {K8sIoApimachineryPkgUtilIntstrIntOrString} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1Metric + */ + consecutiveSuccessLimit?: K8sIoApimachineryPkgUtilIntstrIntOrString; } /** * @@ -1631,6 +1655,12 @@ export interface GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1MetricResul * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1MetricResult */ metadata?: { [key: string]: string; }; + /** + * + * @type {number} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1MetricResult + */ + consecutiveSuccess?: number; } /** * @@ -1650,6 +1680,12 @@ export interface GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1NewRelicMet * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1NewRelicMetric */ query?: string; + /** + * + * @type {string} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1NewRelicMetric + */ + timeout?: string; } /** * @@ -1681,6 +1717,12 @@ export interface GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1NginxTraffi * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1NginxTrafficRouting */ stableIngresses?: Array; + /** + * + * @type {{ [key: string]: string; }} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1NginxTrafficRouting + */ + canaryIngressAnnotations?: { [key: string]: string; }; } /** * @@ -1782,6 +1824,25 @@ export interface GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1PingPongSpe */ pongService?: string; } +/** + * + * @export + * @interface GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1PluginStep + */ +export interface GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1PluginStep { + /** + * + * @type {string} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1PluginStep + */ + name?: string; + /** + * + * @type {string} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1PluginStep + */ + config?: string; +} /** * * @export @@ -1856,6 +1917,37 @@ export interface GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1PrometheusM * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1PrometheusMetric */ headers?: Array; + /** + * + * @type {GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1PrometheusRangeQueryArgs} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1PrometheusMetric + */ + rangeQuery?: GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1PrometheusRangeQueryArgs; +} +/** + * + * @export + * @interface GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1PrometheusRangeQueryArgs + */ +export interface GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1PrometheusRangeQueryArgs { + /** + * + * @type {string} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1PrometheusRangeQueryArgs + */ + start?: string; + /** + * + * @type {string} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1PrometheusRangeQueryArgs + */ + end?: string; + /** + * The maximum time between two slices from the start to end (e.g. 30s, 5m, 1h). + * @type {string} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1PrometheusRangeQueryArgs + */ + step?: string; } /** * @@ -2639,6 +2731,25 @@ export interface GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1SecretKeyRe */ key?: string; } +/** + * + * @export + * @interface GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1SecretRef + */ +export interface GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1SecretRef { + /** + * Name refers to the name of the secret that should be used to integrate with Datadog. + * @type {string} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1SecretRef + */ + name?: string; + /** + * + * @type {boolean} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1SecretRef + */ + namespaced?: boolean; +} /** * * @export @@ -2758,6 +2869,85 @@ export interface GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1SkyWalkingM */ interval?: string; } +/** + * + * @export + * @interface GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1StepPluginStatus + */ +export interface GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1StepPluginStatus { + /** + * + * @type {number} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1StepPluginStatus + */ + index?: number; + /** + * + * @type {string} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1StepPluginStatus + */ + name?: string; + /** + * + * @type {string} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1StepPluginStatus + */ + operation?: string; + /** + * + * @type {string} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1StepPluginStatus + */ + phase?: string; + /** + * + * @type {string} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1StepPluginStatus + */ + message?: string; + /** + * + * @type {K8sIoApimachineryPkgApisMetaV1Time} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1StepPluginStatus + */ + startedAt?: K8sIoApimachineryPkgApisMetaV1Time; + /** + * + * @type {K8sIoApimachineryPkgApisMetaV1Time} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1StepPluginStatus + */ + updatedAt?: K8sIoApimachineryPkgApisMetaV1Time; + /** + * + * @type {K8sIoApimachineryPkgApisMetaV1Time} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1StepPluginStatus + */ + finishedAt?: K8sIoApimachineryPkgApisMetaV1Time; + /** + * + * @type {string} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1StepPluginStatus + */ + backoff?: string; + /** + * + * @type {number} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1StepPluginStatus + */ + executions?: number; + /** + * + * @type {boolean} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1StepPluginStatus + */ + disabled?: boolean; + /** + * + * @type {string} + * @memberof GithubComArgoprojArgoRolloutsPkgApisRolloutsV1alpha1StepPluginStatus + */ + status?: string; +} /** * * @export @@ -7883,7 +8073,7 @@ export const RolloutServiceApiFetchParamCreator = function (configuration?: Conf localVarUrlObj.query = Object.assign({}, localVarUrlObj.query, localVarQueryParameter, options.query); // fix override query string Detail: https://stackoverflow.com/a/7517673/1077943 - delete localVarUrlObj.search; + localVarUrlObj.search = null; localVarRequestOptions.headers = Object.assign({}, localVarHeaderParameter, options.headers); const needsSerialization = ("RolloutAbortRolloutRequest" !== "string") || localVarRequestOptions.headers['Content-Type'] === 'application/json'; localVarRequestOptions.body = needsSerialization ? JSON.stringify(body || {}) : (body || ""); @@ -7907,7 +8097,7 @@ export const RolloutServiceApiFetchParamCreator = function (configuration?: Conf localVarUrlObj.query = Object.assign({}, localVarUrlObj.query, localVarQueryParameter, options.query); // fix override query string Detail: https://stackoverflow.com/a/7517673/1077943 - delete localVarUrlObj.search; + localVarUrlObj.search = null; localVarRequestOptions.headers = Object.assign({}, localVarHeaderParameter, options.headers); return { @@ -7941,7 +8131,7 @@ export const RolloutServiceApiFetchParamCreator = function (configuration?: Conf localVarUrlObj.query = Object.assign({}, localVarUrlObj.query, localVarQueryParameter, options.query); // fix override query string Detail: https://stackoverflow.com/a/7517673/1077943 - delete localVarUrlObj.search; + localVarUrlObj.search = null; localVarRequestOptions.headers = Object.assign({}, localVarHeaderParameter, options.headers); return { @@ -7969,7 +8159,7 @@ export const RolloutServiceApiFetchParamCreator = function (configuration?: Conf localVarUrlObj.query = Object.assign({}, localVarUrlObj.query, localVarQueryParameter, options.query); // fix override query string Detail: https://stackoverflow.com/a/7517673/1077943 - delete localVarUrlObj.search; + localVarUrlObj.search = null; localVarRequestOptions.headers = Object.assign({}, localVarHeaderParameter, options.headers); return { @@ -8010,7 +8200,7 @@ export const RolloutServiceApiFetchParamCreator = function (configuration?: Conf localVarUrlObj.query = Object.assign({}, localVarUrlObj.query, localVarQueryParameter, options.query); // fix override query string Detail: https://stackoverflow.com/a/7517673/1077943 - delete localVarUrlObj.search; + localVarUrlObj.search = null; localVarRequestOptions.headers = Object.assign({}, localVarHeaderParameter, options.headers); const needsSerialization = ("RolloutPromoteRolloutRequest" !== "string") || localVarRequestOptions.headers['Content-Type'] === 'application/json'; localVarRequestOptions.body = needsSerialization ? JSON.stringify(body || {}) : (body || ""); @@ -8053,7 +8243,7 @@ export const RolloutServiceApiFetchParamCreator = function (configuration?: Conf localVarUrlObj.query = Object.assign({}, localVarUrlObj.query, localVarQueryParameter, options.query); // fix override query string Detail: https://stackoverflow.com/a/7517673/1077943 - delete localVarUrlObj.search; + localVarUrlObj.search = null; localVarRequestOptions.headers = Object.assign({}, localVarHeaderParameter, options.headers); const needsSerialization = ("RolloutRestartRolloutRequest" !== "string") || localVarRequestOptions.headers['Content-Type'] === 'application/json'; localVarRequestOptions.body = needsSerialization ? JSON.stringify(body || {}) : (body || ""); @@ -8096,7 +8286,7 @@ export const RolloutServiceApiFetchParamCreator = function (configuration?: Conf localVarUrlObj.query = Object.assign({}, localVarUrlObj.query, localVarQueryParameter, options.query); // fix override query string Detail: https://stackoverflow.com/a/7517673/1077943 - delete localVarUrlObj.search; + localVarUrlObj.search = null; localVarRequestOptions.headers = Object.assign({}, localVarHeaderParameter, options.headers); const needsSerialization = ("RolloutRetryRolloutRequest" !== "string") || localVarRequestOptions.headers['Content-Type'] === 'application/json'; localVarRequestOptions.body = needsSerialization ? JSON.stringify(body || {}) : (body || ""); @@ -8157,7 +8347,7 @@ export const RolloutServiceApiFetchParamCreator = function (configuration?: Conf localVarUrlObj.query = Object.assign({}, localVarUrlObj.query, localVarQueryParameter, options.query); // fix override query string Detail: https://stackoverflow.com/a/7517673/1077943 - delete localVarUrlObj.search; + localVarUrlObj.search = null; localVarRequestOptions.headers = Object.assign({}, localVarHeaderParameter, options.headers); const needsSerialization = ("RolloutSetImageRequest" !== "string") || localVarRequestOptions.headers['Content-Type'] === 'application/json'; localVarRequestOptions.body = needsSerialization ? JSON.stringify(body || {}) : (body || ""); @@ -8206,7 +8396,7 @@ export const RolloutServiceApiFetchParamCreator = function (configuration?: Conf localVarUrlObj.query = Object.assign({}, localVarUrlObj.query, localVarQueryParameter, options.query); // fix override query string Detail: https://stackoverflow.com/a/7517673/1077943 - delete localVarUrlObj.search; + localVarUrlObj.search = null; localVarRequestOptions.headers = Object.assign({}, localVarHeaderParameter, options.headers); const needsSerialization = ("RolloutUndoRolloutRequest" !== "string") || localVarRequestOptions.headers['Content-Type'] === 'application/json'; localVarRequestOptions.body = needsSerialization ? JSON.stringify(body || {}) : (body || ""); @@ -8230,7 +8420,7 @@ export const RolloutServiceApiFetchParamCreator = function (configuration?: Conf localVarUrlObj.query = Object.assign({}, localVarUrlObj.query, localVarQueryParameter, options.query); // fix override query string Detail: https://stackoverflow.com/a/7517673/1077943 - delete localVarUrlObj.search; + localVarUrlObj.search = null; localVarRequestOptions.headers = Object.assign({}, localVarHeaderParameter, options.headers); return { @@ -8264,7 +8454,7 @@ export const RolloutServiceApiFetchParamCreator = function (configuration?: Conf localVarUrlObj.query = Object.assign({}, localVarUrlObj.query, localVarQueryParameter, options.query); // fix override query string Detail: https://stackoverflow.com/a/7517673/1077943 - delete localVarUrlObj.search; + localVarUrlObj.search = null; localVarRequestOptions.headers = Object.assign({}, localVarHeaderParameter, options.headers); return { @@ -8292,7 +8482,7 @@ export const RolloutServiceApiFetchParamCreator = function (configuration?: Conf localVarUrlObj.query = Object.assign({}, localVarUrlObj.query, localVarQueryParameter, options.query); // fix override query string Detail: https://stackoverflow.com/a/7517673/1077943 - delete localVarUrlObj.search; + localVarUrlObj.search = null; localVarRequestOptions.headers = Object.assign({}, localVarHeaderParameter, options.headers); return { @@ -8319,7 +8509,7 @@ export const RolloutServiceApiFp = function(configuration?: Configuration) { */ rolloutServiceAbortRollout(body: RolloutAbortRolloutRequest, namespace: string, name: string, options?: any): (fetch?: FetchAPI, basePath?: string) => Promise { const localVarFetchArgs = RolloutServiceApiFetchParamCreator(configuration).rolloutServiceAbortRollout(body, namespace, name, options); - return (fetch: FetchAPI = portableFetch, basePath: string = BASE_PATH) => { + return (fetch: FetchAPI = isomorphicFetch, basePath: string = BASE_PATH) => { return fetch(basePath + localVarFetchArgs.url, localVarFetchArgs.options).then((response) => { if (response.status >= 200 && response.status < 300) { return response.json(); @@ -8336,7 +8526,7 @@ export const RolloutServiceApiFp = function(configuration?: Configuration) { */ rolloutServiceGetNamespace(options?: any): (fetch?: FetchAPI, basePath?: string) => Promise { const localVarFetchArgs = RolloutServiceApiFetchParamCreator(configuration).rolloutServiceGetNamespace(options); - return (fetch: FetchAPI = portableFetch, basePath: string = BASE_PATH) => { + return (fetch: FetchAPI = isomorphicFetch, basePath: string = BASE_PATH) => { return fetch(basePath + localVarFetchArgs.url, localVarFetchArgs.options).then((response) => { if (response.status >= 200 && response.status < 300) { return response.json(); @@ -8355,7 +8545,7 @@ export const RolloutServiceApiFp = function(configuration?: Configuration) { */ rolloutServiceGetRolloutInfo(namespace: string, name: string, options?: any): (fetch?: FetchAPI, basePath?: string) => Promise { const localVarFetchArgs = RolloutServiceApiFetchParamCreator(configuration).rolloutServiceGetRolloutInfo(namespace, name, options); - return (fetch: FetchAPI = portableFetch, basePath: string = BASE_PATH) => { + return (fetch: FetchAPI = isomorphicFetch, basePath: string = BASE_PATH) => { return fetch(basePath + localVarFetchArgs.url, localVarFetchArgs.options).then((response) => { if (response.status >= 200 && response.status < 300) { return response.json(); @@ -8373,7 +8563,7 @@ export const RolloutServiceApiFp = function(configuration?: Configuration) { */ rolloutServiceListRolloutInfos(namespace: string, options?: any): (fetch?: FetchAPI, basePath?: string) => Promise { const localVarFetchArgs = RolloutServiceApiFetchParamCreator(configuration).rolloutServiceListRolloutInfos(namespace, options); - return (fetch: FetchAPI = portableFetch, basePath: string = BASE_PATH) => { + return (fetch: FetchAPI = isomorphicFetch, basePath: string = BASE_PATH) => { return fetch(basePath + localVarFetchArgs.url, localVarFetchArgs.options).then((response) => { if (response.status >= 200 && response.status < 300) { return response.json(); @@ -8393,7 +8583,7 @@ export const RolloutServiceApiFp = function(configuration?: Configuration) { */ rolloutServicePromoteRollout(body: RolloutPromoteRolloutRequest, namespace: string, name: string, options?: any): (fetch?: FetchAPI, basePath?: string) => Promise { const localVarFetchArgs = RolloutServiceApiFetchParamCreator(configuration).rolloutServicePromoteRollout(body, namespace, name, options); - return (fetch: FetchAPI = portableFetch, basePath: string = BASE_PATH) => { + return (fetch: FetchAPI = isomorphicFetch, basePath: string = BASE_PATH) => { return fetch(basePath + localVarFetchArgs.url, localVarFetchArgs.options).then((response) => { if (response.status >= 200 && response.status < 300) { return response.json(); @@ -8413,7 +8603,7 @@ export const RolloutServiceApiFp = function(configuration?: Configuration) { */ rolloutServiceRestartRollout(body: RolloutRestartRolloutRequest, namespace: string, name: string, options?: any): (fetch?: FetchAPI, basePath?: string) => Promise { const localVarFetchArgs = RolloutServiceApiFetchParamCreator(configuration).rolloutServiceRestartRollout(body, namespace, name, options); - return (fetch: FetchAPI = portableFetch, basePath: string = BASE_PATH) => { + return (fetch: FetchAPI = isomorphicFetch, basePath: string = BASE_PATH) => { return fetch(basePath + localVarFetchArgs.url, localVarFetchArgs.options).then((response) => { if (response.status >= 200 && response.status < 300) { return response.json(); @@ -8433,7 +8623,7 @@ export const RolloutServiceApiFp = function(configuration?: Configuration) { */ rolloutServiceRetryRollout(body: RolloutRetryRolloutRequest, namespace: string, name: string, options?: any): (fetch?: FetchAPI, basePath?: string) => Promise { const localVarFetchArgs = RolloutServiceApiFetchParamCreator(configuration).rolloutServiceRetryRollout(body, namespace, name, options); - return (fetch: FetchAPI = portableFetch, basePath: string = BASE_PATH) => { + return (fetch: FetchAPI = isomorphicFetch, basePath: string = BASE_PATH) => { return fetch(basePath + localVarFetchArgs.url, localVarFetchArgs.options).then((response) => { if (response.status >= 200 && response.status < 300) { return response.json(); @@ -8456,7 +8646,7 @@ export const RolloutServiceApiFp = function(configuration?: Configuration) { */ rolloutServiceSetRolloutImage(body: RolloutSetImageRequest, namespace: string, rollout: string, container: string, image: string, tag: string, options?: any): (fetch?: FetchAPI, basePath?: string) => Promise { const localVarFetchArgs = RolloutServiceApiFetchParamCreator(configuration).rolloutServiceSetRolloutImage(body, namespace, rollout, container, image, tag, options); - return (fetch: FetchAPI = portableFetch, basePath: string = BASE_PATH) => { + return (fetch: FetchAPI = isomorphicFetch, basePath: string = BASE_PATH) => { return fetch(basePath + localVarFetchArgs.url, localVarFetchArgs.options).then((response) => { if (response.status >= 200 && response.status < 300) { return response.json(); @@ -8477,7 +8667,7 @@ export const RolloutServiceApiFp = function(configuration?: Configuration) { */ rolloutServiceUndoRollout(body: RolloutUndoRolloutRequest, namespace: string, rollout: string, revision: string, options?: any): (fetch?: FetchAPI, basePath?: string) => Promise { const localVarFetchArgs = RolloutServiceApiFetchParamCreator(configuration).rolloutServiceUndoRollout(body, namespace, rollout, revision, options); - return (fetch: FetchAPI = portableFetch, basePath: string = BASE_PATH) => { + return (fetch: FetchAPI = isomorphicFetch, basePath: string = BASE_PATH) => { return fetch(basePath + localVarFetchArgs.url, localVarFetchArgs.options).then((response) => { if (response.status >= 200 && response.status < 300) { return response.json(); @@ -8494,7 +8684,7 @@ export const RolloutServiceApiFp = function(configuration?: Configuration) { */ rolloutServiceVersion(options?: any): (fetch?: FetchAPI, basePath?: string) => Promise { const localVarFetchArgs = RolloutServiceApiFetchParamCreator(configuration).rolloutServiceVersion(options); - return (fetch: FetchAPI = portableFetch, basePath: string = BASE_PATH) => { + return (fetch: FetchAPI = isomorphicFetch, basePath: string = BASE_PATH) => { return fetch(basePath + localVarFetchArgs.url, localVarFetchArgs.options).then((response) => { if (response.status >= 200 && response.status < 300) { return response.json(); @@ -8513,7 +8703,7 @@ export const RolloutServiceApiFp = function(configuration?: Configuration) { */ rolloutServiceWatchRolloutInfo(namespace: string, name: string, options?: any): (fetch?: FetchAPI, basePath?: string) => Promise { const localVarFetchArgs = RolloutServiceApiFetchParamCreator(configuration).rolloutServiceWatchRolloutInfo(namespace, name, options); - return (fetch: FetchAPI = portableFetch, basePath: string = BASE_PATH) => { + return (fetch: FetchAPI = isomorphicFetch, basePath: string = BASE_PATH) => { return fetch(basePath + localVarFetchArgs.url, localVarFetchArgs.options).then((response) => { if (response.status >= 200 && response.status < 300) { return response.json(); @@ -8531,7 +8721,7 @@ export const RolloutServiceApiFp = function(configuration?: Configuration) { */ rolloutServiceWatchRolloutInfos(namespace: string, options?: any): (fetch?: FetchAPI, basePath?: string) => Promise { const localVarFetchArgs = RolloutServiceApiFetchParamCreator(configuration).rolloutServiceWatchRolloutInfos(namespace, options); - return (fetch: FetchAPI = portableFetch, basePath: string = BASE_PATH) => { + return (fetch: FetchAPI = isomorphicFetch, basePath: string = BASE_PATH) => { return fetch(basePath + localVarFetchArgs.url, localVarFetchArgs.options).then((response) => { if (response.status >= 200 && response.status < 300) { return response.json(); diff --git a/ui/src/models/rollout/generated/custom.d.ts b/ui/src/models/rollout/generated/custom.d.ts index 9a5ceb3588..5d917c1d30 100644 --- a/ui/src/models/rollout/generated/custom.d.ts +++ b/ui/src/models/rollout/generated/custom.d.ts @@ -1,2 +1,2 @@ -declare module 'portable-fetch'; +declare module 'isomorphic-fetch'; declare module 'url'; \ No newline at end of file diff --git a/ui/yarn.lock b/ui/yarn.lock index c8fbf6b82b..68dcaf2cf7 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -8891,33 +8891,10 @@ load-json-file@^2.0.0: pify "^2.0.0" strip-bom "^3.0.0" -loader-runner@^2.4.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-2.4.0.tgz#ed47066bfe534d7e84c4c7b9998c2a75607d9357" - integrity sha512-Jsmr89RcXGIwivFY21FcRrisYZfvLMTWx5kOLc+JTxtpBOG6xML0vzbc6SEQG2FO9/4Fc3wW4LVcB5DmGflaRw== - -loader-utils@1.2.3: - version "1.2.3" - resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.2.3.tgz#1ff5dc6911c9f0a062531a4c04b609406108c2c7" - integrity sha512-fkpz8ejdnEMG3s37wGL07iSBDg99O9D5yflE9RGNH3hRdx9SOwYfnGYdZOUIZitN8E+E2vkq3MUMYMvPYl5ZZA== - dependencies: - big.js "^5.2.2" - emojis-list "^2.0.0" - json5 "^1.0.1" - -loader-utils@2.0.0, loader-utils@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-2.0.0.tgz#e4cace5b816d425a166b5f097e10cd12b36064b0" - integrity sha512-rP4F0h2RaWSvPEkD7BLDFQnvSf+nK+wr3ESUjNTyAGobqrijmW92zc+SO6d4p4B1wh7+B/Jg1mkQe5NYUEHtHQ== - dependencies: - big.js "^5.2.2" - emojis-list "^3.0.0" - json5 "^2.1.2" - -loader-utils@^1.1.0, loader-utils@^1.2.3, loader-utils@^1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.4.0.tgz#c579b5e34cb34b1a74edc6c1fb36bfa371d5a613" - integrity sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA== +loader-utils@^2.0.0, loader-utils@^2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-2.0.4.tgz#8b5cb38b5c34a9a018ee1fc0e6a066d1dfcc528c" + integrity sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw== dependencies: big.js "^5.2.2" emojis-list "^3.0.0" diff --git a/utils/analysis/factory.go b/utils/analysis/factory.go index 0e17e587a7..c2fefd3068 100644 --- a/utils/analysis/factory.go +++ b/utils/analysis/factory.go @@ -157,15 +157,48 @@ func ValidateMetric(metric v1alpha1.Metric) error { failureLimit = metric.FailureLimit.IntValue() } + consecutiveSuccessLimit := 0 + if metric.ConsecutiveSuccessLimit != nil { + consecutiveSuccessLimit = metric.ConsecutiveSuccessLimit.IntValue() + } + inconclusiveLimit := 0 if metric.InconclusiveLimit != nil { inconclusiveLimit = metric.InconclusiveLimit.IntValue() } + // Negative value checks + if consecutiveSuccessLimit < 0 { + return fmt.Errorf("consecutiveSuccessLimit must be >= 0") + } + if failureLimit < -1 { + return fmt.Errorf("failureLimit must be >= 0, or -1 to be disabled") + } + // if consecutiveSuccessLimit is disabled (set to 0), AND + // failureLimit is disabled (set to -1), then this is invalid + if consecutiveSuccessLimit == 0 && failureLimit == -1 { + return fmt.Errorf("failureLimit and consecutiveSuccessLimit cannot both be disabled") + } + + if inconclusiveLimit < 0 { + return fmt.Errorf("inconclusiveLimit must be >= 0") + } + + if metric.ConsecutiveErrorLimit != nil && metric.ConsecutiveErrorLimit.IntValue() < 0 { + return fmt.Errorf("consecutiveErrorLimit must be >= 0") + } + if count > 0 { - if count < failureLimit { - return fmt.Errorf("count must be >= failureLimit") + if consecutiveSuccessLimit >= 0 && failureLimit >= 0 { + if count < failureLimit+consecutiveSuccessLimit { + return fmt.Errorf("count (%d) must be >= failureLimit + consecutiveSuccessLimit (%d + %d) if both >= 0", count, failureLimit, consecutiveSuccessLimit) + } + } else if consecutiveSuccessLimit >= 0 && failureLimit < 0 { + if count < consecutiveSuccessLimit { + return fmt.Errorf("count must be >= consecutiveSuccessLimit") + } } + if count < inconclusiveLimit { return fmt.Errorf("count must be >= inconclusiveLimit") } @@ -184,16 +217,6 @@ func ValidateMetric(metric v1alpha1.Metric) error { } } - if failureLimit < 0 { - return fmt.Errorf("failureLimit must be >= 0") - } - if inconclusiveLimit < 0 { - return fmt.Errorf("inconclusiveLimit must be >= 0") - } - - if metric.ConsecutiveErrorLimit != nil && metric.ConsecutiveErrorLimit.IntValue() < 0 { - return fmt.Errorf("consecutiveErrorLimit must be >= 0") - } numProviders := 0 if metric.Provider.Prometheus != nil { numProviders++ diff --git a/utils/analysis/factory_test.go b/utils/analysis/factory_test.go index d3fe7dfec1..8873ce0f3f 100644 --- a/utils/analysis/factory_test.go +++ b/utils/analysis/factory_test.go @@ -4,8 +4,6 @@ import ( "fmt" "testing" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" - "github.com/argoproj/argo-rollouts/utils/annotations" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -13,6 +11,9 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/utils/pointer" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/utils/annotations" ) func TestBuildArgumentsForRolloutAnalysisRun(t *testing.T) { @@ -178,7 +179,7 @@ func TestBackgroundLabels(t *testing.T) { } func TestValidateMetrics(t *testing.T) { - t.Run("Ensure count >= failureLimit", func(t *testing.T) { + t.Run("Ensure count >= failureLimit + consecutiveSuccessLimit", func(t *testing.T) { failureLimit := intstr.FromInt(2) count := intstr.FromInt(1) spec := v1alpha1.AnalysisTemplateSpec{ @@ -194,7 +195,7 @@ func TestValidateMetrics(t *testing.T) { }, } err := ValidateMetrics(spec.Metrics) - assert.EqualError(t, err, "metrics[0]: count must be >= failureLimit") + assert.EqualError(t, err, "metrics[0]: count (1) must be >= failureLimit + consecutiveSuccessLimit (2 + 0) if both >= 0") count = intstr.FromInt(0) spec.Metrics[0].Count = &count err = ValidateMetrics(spec.Metrics) @@ -323,8 +324,8 @@ func TestValidateMetrics(t *testing.T) { err := ValidateMetrics(spec.Metrics) assert.EqualError(t, err, "metrics[1]: duplicate name 'success-rate'") }) - t.Run("Ensure failureLimit >= 0", func(t *testing.T) { - failureLimit := intstr.FromInt(-1) + t.Run("Ensure failureLimit >= -1", func(t *testing.T) { + failureLimit := intstr.FromInt(-2) spec := v1alpha1.AnalysisTemplateSpec{ Metrics: []v1alpha1.Metric{ { @@ -337,7 +338,7 @@ func TestValidateMetrics(t *testing.T) { }, } err := ValidateMetrics(spec.Metrics) - assert.EqualError(t, err, "metrics[0]: failureLimit must be >= 0") + assert.EqualError(t, err, "metrics[0]: failureLimit must be >= 0, or -1 to be disabled") }) t.Run("Ensure inconclusiveLimit >= 0", func(t *testing.T) { inconclusiveLimit := intstr.FromInt(-1) @@ -407,6 +408,40 @@ func TestValidateMetrics(t *testing.T) { err := ValidateMetrics(spec.Metrics) assert.EqualError(t, err, "metrics[0]: multiple providers specified") }) + t.Run("Ensure consecutiveSuccessLimit >= 0", func(t *testing.T) { + consecutiveSuccessLimit := intstr.FromInt(-1) + spec := v1alpha1.AnalysisTemplateSpec{ + Metrics: []v1alpha1.Metric{ + { + Name: "success-rate", + ConsecutiveSuccessLimit: &consecutiveSuccessLimit, + Provider: v1alpha1.MetricProvider{ + Prometheus: &v1alpha1.PrometheusMetric{}, + }, + }, + }, + } + err := ValidateMetrics(spec.Metrics) + assert.EqualError(t, err, "metrics[0]: consecutiveSuccessLimit must be >= 0") + }) + t.Run("Ensure consecutiveSuccessLimit and failureLimit are not both disabled", func(t *testing.T) { + consecutiveSuccessLimit := intstr.FromInt(0) + failureLimit := intstr.FromInt(-1) + spec := v1alpha1.AnalysisTemplateSpec{ + Metrics: []v1alpha1.Metric{ + { + Name: "success-rate", + ConsecutiveSuccessLimit: &consecutiveSuccessLimit, + FailureLimit: &failureLimit, + Provider: v1alpha1.MetricProvider{ + Prometheus: &v1alpha1.PrometheusMetric{}, + }, + }, + }, + } + err := ValidateMetrics(spec.Metrics) + assert.EqualError(t, err, "metrics[0]: failureLimit and consecutiveSuccessLimit cannot both be disabled") + }) } // TestResolveMetricArgs verifies that metric arguments are resolved @@ -444,7 +479,7 @@ func TestResolveMetricArgsWithQuotes(t *testing.T) { } newMetric, err := ResolveMetricArgs(metric, arguments) assert.NoError(t, err) - assert.Equal(t, fmt.Sprintf(arg), newMetric.SuccessCondition) + assert.Equal(t, arg, newMetric.SuccessCondition) } func Test_extractValueFromRollout(t *testing.T) { diff --git a/utils/analysis/helpers.go b/utils/analysis/helpers.go index 42ded67315..142b6daeb1 100644 --- a/utils/analysis/helpers.go +++ b/utils/analysis/helpers.go @@ -7,7 +7,6 @@ import ( "fmt" "regexp" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" log "github.com/sirupsen/logrus" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -15,6 +14,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" patchtypes "k8s.io/apimachinery/pkg/types" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + argoprojclient "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/typed/rollouts/v1alpha1" ) diff --git a/utils/annotations/annotations.go b/utils/annotations/annotations.go index 121c6c0803..067492bc2b 100644 --- a/utils/annotations/annotations.go +++ b/utils/annotations/annotations.go @@ -28,6 +28,8 @@ const ( DesiredReplicasAnnotation = RolloutLabel + "/desired-replicas" // WorkloadGenerationAnnotation is the generation of the referenced workload WorkloadGenerationAnnotation = RolloutLabel + "/workload-generation" + // NotificationEngineAnnotation the annotation notification engine uses to determine if it should notify + NotificationEngineAnnotation = "notified.notifications.argoproj.io" ) // GetDesiredReplicasAnnotation returns the number of desired replicas @@ -219,6 +221,7 @@ var annotationsToSkip = map[string]bool{ RevisionAnnotation: true, RevisionHistoryAnnotation: true, DesiredReplicasAnnotation: true, + NotificationEngineAnnotation: true, } // skipCopyAnnotation returns true if we should skip copying the annotation with the given annotation key diff --git a/utils/apisix/apisix.go b/utils/apisix/apisix.go index 3f5984e022..010aaf60ba 100644 --- a/utils/apisix/apisix.go +++ b/utils/apisix/apisix.go @@ -4,10 +4,11 @@ import ( "context" "strings" - "github.com/argoproj/argo-rollouts/utils/defaults" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" + + "github.com/argoproj/argo-rollouts/utils/defaults" ) const apisixRoutes = "apisixroutes" diff --git a/utils/appmesh/appmesh.go b/utils/appmesh/appmesh.go index 83e77572ff..19c9d2133b 100644 --- a/utils/appmesh/appmesh.go +++ b/utils/appmesh/appmesh.go @@ -3,10 +3,11 @@ package appmesh import ( "context" - "github.com/argoproj/argo-rollouts/utils/defaults" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" + + "github.com/argoproj/argo-rollouts/utils/defaults" ) const AppMeshCRDGroup = "appmesh.k8s.aws" diff --git a/utils/aws/aws.go b/utils/aws/aws.go index 42b4907836..42685302fd 100644 --- a/utils/aws/aws.go +++ b/utils/aws/aws.go @@ -8,7 +8,6 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" - "github.com/argoproj/argo-rollouts/utils/defaults" "github.com/aws/aws-sdk-go-v2/config" elbv2 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" elbv2types "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types" @@ -18,6 +17,8 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/dynamic" + + "github.com/argoproj/argo-rollouts/utils/defaults" ) // AWSLoadBalancerV2TagKeyResourceID is the tag applied to an AWS resource by the AWS Load Balancer diff --git a/utils/aws/mocks/ELBv2APIClient.go b/utils/aws/mocks/ELBv2APIClient.go index 5e9f9dc380..f004df4702 100644 --- a/utils/aws/mocks/ELBv2APIClient.go +++ b/utils/aws/mocks/ELBv2APIClient.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.32.0. DO NOT EDIT. +// Code generated by mockery v2.42.2. DO NOT EDIT. package mocks @@ -25,6 +25,10 @@ func (_m *ELBv2APIClient) DescribeListeners(_a0 context.Context, _a1 *elasticloa _ca = append(_ca, _va...) ret := _m.Called(_ca...) + if len(ret) == 0 { + panic("no return value specified for DescribeListeners") + } + var r0 *elasticloadbalancingv2.DescribeListenersOutput var r1 error if rf, ok := ret.Get(0).(func(context.Context, *elasticloadbalancingv2.DescribeListenersInput, ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.DescribeListenersOutput, error)); ok { @@ -58,6 +62,10 @@ func (_m *ELBv2APIClient) DescribeLoadBalancers(_a0 context.Context, _a1 *elasti _ca = append(_ca, _va...) ret := _m.Called(_ca...) + if len(ret) == 0 { + panic("no return value specified for DescribeLoadBalancers") + } + var r0 *elasticloadbalancingv2.DescribeLoadBalancersOutput var r1 error if rf, ok := ret.Get(0).(func(context.Context, *elasticloadbalancingv2.DescribeLoadBalancersInput, ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.DescribeLoadBalancersOutput, error)); ok { @@ -91,6 +99,10 @@ func (_m *ELBv2APIClient) DescribeRules(ctx context.Context, params *elasticload _ca = append(_ca, _va...) ret := _m.Called(_ca...) + if len(ret) == 0 { + panic("no return value specified for DescribeRules") + } + var r0 *elasticloadbalancingv2.DescribeRulesOutput var r1 error if rf, ok := ret.Get(0).(func(context.Context, *elasticloadbalancingv2.DescribeRulesInput, ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.DescribeRulesOutput, error)); ok { @@ -124,6 +136,10 @@ func (_m *ELBv2APIClient) DescribeTags(ctx context.Context, params *elasticloadb _ca = append(_ca, _va...) ret := _m.Called(_ca...) + if len(ret) == 0 { + panic("no return value specified for DescribeTags") + } + var r0 *elasticloadbalancingv2.DescribeTagsOutput var r1 error if rf, ok := ret.Get(0).(func(context.Context, *elasticloadbalancingv2.DescribeTagsInput, ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.DescribeTagsOutput, error)); ok { @@ -157,6 +173,10 @@ func (_m *ELBv2APIClient) DescribeTargetGroups(_a0 context.Context, _a1 *elastic _ca = append(_ca, _va...) ret := _m.Called(_ca...) + if len(ret) == 0 { + panic("no return value specified for DescribeTargetGroups") + } + var r0 *elasticloadbalancingv2.DescribeTargetGroupsOutput var r1 error if rf, ok := ret.Get(0).(func(context.Context, *elasticloadbalancingv2.DescribeTargetGroupsInput, ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.DescribeTargetGroupsOutput, error)); ok { @@ -190,6 +210,10 @@ func (_m *ELBv2APIClient) DescribeTargetHealth(ctx context.Context, params *elas _ca = append(_ca, _va...) ret := _m.Called(_ca...) + if len(ret) == 0 { + panic("no return value specified for DescribeTargetHealth") + } + var r0 *elasticloadbalancingv2.DescribeTargetHealthOutput var r1 error if rf, ok := ret.Get(0).(func(context.Context, *elasticloadbalancingv2.DescribeTargetHealthInput, ...func(*elasticloadbalancingv2.Options)) (*elasticloadbalancingv2.DescribeTargetHealthOutput, error)); ok { diff --git a/utils/conditions/conditions.go b/utils/conditions/conditions.go index 77cd401c63..57d8c0f02f 100644 --- a/utils/conditions/conditions.go +++ b/utils/conditions/conditions.go @@ -133,6 +133,11 @@ const ( // RolloutExperimentFailedMessage is added in a rollout when the experiment owned by a rollout fails to show any progress RolloutExperimentFailedMessage = "Experiment '%s' owned by the Rollout '%q' has timed out." + // RolloutReconciliationErrorReason is added in a rollout when the reconciliation returns an error preventing progress + RolloutReconciliationErrorReason = "ReconciliationError" + // RolloutReconciliationErrorMessage is added in a rollout when the reconciliation returns an error preventing progress + RolloutReconciliationErrorMessage = "Reconciliation failed with error: %v" + // TimedOutReason is added in a rollout when its newest replica set fails to show any progress // within the given deadline (progressDeadlineSeconds). TimedOutReason = "ProgressDeadlineExceeded" @@ -157,6 +162,12 @@ const ( // ServiceReferencingManagedService is added in a rollout when the multiple rollouts reference a Rollout ServiceReferencingManagedService = "Service %q is managed by another Rollout" + // StepPluginTransitionReason is added to a Rollout when a step plugin transition to an unsuccessful phase + StepPluginTransitionReason = "StepPluginTransition" + StepPluginTransitionRunMessage = "Step plugin %s (step %d) transitioned to %s" + StepPluginTransitionAbortMessage = "Step plugin %s (step %d) aborted (%s)" + StepPluginTransitionTerminateMessage = "Step plugin %s (step %d) terminated (%s)" + // TargetGroupHealthyReason is emitted when target group has been verified TargetGroupVerifiedReason = "TargetGroupVerified" TargetGroupVerifiedRegistrationMessage = "Service %s (TargetGroup %s) verified: %d endpoints registered" @@ -302,7 +313,7 @@ func RolloutHealthy(rollout *v1alpha1.Rollout, newStatus *v1alpha1.RolloutStatus } // RolloutCompleted considers a rollout to be complete once StableRS == CurrentPodHash -func RolloutCompleted(rollout *v1alpha1.Rollout, newStatus *v1alpha1.RolloutStatus) bool { +func RolloutCompleted(newStatus *v1alpha1.RolloutStatus) bool { return newStatus.StableRS != "" && newStatus.StableRS == newStatus.CurrentPodHash } @@ -335,7 +346,7 @@ func RolloutTimedOut(rollout *v1alpha1.Rollout, newStatus *v1alpha1.RolloutStatu // When a rollout is retried, the controller should not evaluate for a timeout based on the // aborted condition because the abort could have happened a while back and the rollout should // not enter degraded as a result of that - if condition == nil || condition.Reason == RolloutAbortedReason { + if condition == nil || condition.Reason == RolloutAbortedReason || condition.Reason == RolloutPausedReason { return false } diff --git a/utils/conditions/rollouts_test.go b/utils/conditions/rollouts_test.go index 842f01f594..af6a69d40c 100644 --- a/utils/conditions/rollouts_test.go +++ b/utils/conditions/rollouts_test.go @@ -499,10 +499,10 @@ func TestRolloutComplete(t *testing.T) { return r } r := rollout(5, 5, 5, 5) - assert.Equal(t, true, RolloutCompleted(r, &r.Status)) + assert.Equal(t, true, RolloutCompleted(&r.Status)) r.Status.StableRS = "not-current-pod-hash" - assert.Equal(t, false, RolloutCompleted(r, &r.Status)) + assert.Equal(t, false, RolloutCompleted(&r.Status)) } func TestRolloutTimedOut(t *testing.T) { @@ -555,6 +555,22 @@ func TestRolloutTimedOut(t *testing.T) { }, expected: true, }, + { + name: "Rollout has not time out when paused", + progressDeadlineSeconds: 5, + newStatus: v1alpha1.RolloutStatus{ + Conditions: conditions(RolloutPausedReason, before), + }, + expected: false, + }, + { + name: "Rollout has not time out when aborted", + progressDeadlineSeconds: 5, + newStatus: v1alpha1.RolloutStatus{ + Conditions: conditions(RolloutAbortedReason, before), + }, + expected: false, + }, } for i := range tests { test := tests[i] diff --git a/utils/config/config.go b/utils/config/config.go index 7989374e8e..7b20520341 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -4,26 +4,29 @@ import ( "context" "fmt" "regexp" + "slices" "sync" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "github.com/argoproj/argo-rollouts/utils/defaults" - "github.com/argoproj/argo-rollouts/utils/plugin/types" v1 "k8s.io/api/core/v1" k8errors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/yaml" + + "github.com/argoproj/argo-rollouts/utils/defaults" + "github.com/argoproj/argo-rollouts/utils/plugin/types" ) // Config is the in memory representation of the configmap with some additional fields/functions for ease of use. type Config struct { configMap *v1.ConfigMap plugins []types.PluginItem + lock *sync.RWMutex } var configMemoryCache *Config -var mutex sync.RWMutex +var mutex = &sync.RWMutex{} // Regex to match plugin names, this matches github username and repo limits var re = regexp.MustCompile(`^([a-zA-Z0-9\-]+)\/{1}([a-zA-Z0-9_\-.]+)$`) @@ -34,7 +37,9 @@ func InitializeConfig(k8sClientset kubernetes.Interface, configMapName string) ( configMapCluster, err := k8sClientset.CoreV1().ConfigMaps(defaults.Namespace()).Get(context.Background(), configMapName, metav1.GetOptions{}) if err != nil { if k8errors.IsNotFound(err) { - configMemoryCache = &Config{} // We create an empty config so that we don't try to initialize again + configMemoryCache = &Config{ + lock: &sync.RWMutex{}, + } // We create an empty config so that we don't try to initialize again // If the configmap is not found, we return return configMemoryCache, nil } @@ -45,16 +50,31 @@ func InitializeConfig(k8sClientset kubernetes.Interface, configMapName string) ( if err = yaml.Unmarshal([]byte(configMapCluster.Data["trafficRouterPlugins"]), &trafficRouterPlugins); err != nil { return nil, fmt.Errorf("failed to unmarshal traffic router plugins while initializing: %w", err) } + for i := range trafficRouterPlugins { + trafficRouterPlugins[i].Type = types.PluginTypeTrafficRouter + } var metricProviderPlugins []types.PluginItem if err = yaml.Unmarshal([]byte(configMapCluster.Data["metricProviderPlugins"]), &metricProviderPlugins); err != nil { return nil, fmt.Errorf("failed to unmarshal metric provider plugins while initializing: %w", err) } + for i := range metricProviderPlugins { + metricProviderPlugins[i].Type = types.PluginTypeMetricProvider + } + + var stepPlugins []types.PluginItem + if err = yaml.Unmarshal([]byte(configMapCluster.Data["stepPlugins"]), &stepPlugins); err != nil { + return nil, fmt.Errorf("failed to unmarshal step plugins while initializing: %w", err) + } + for i := range stepPlugins { + stepPlugins[i].Type = types.PluginTypeStep + } mutex.Lock() configMemoryCache = &Config{ configMap: configMapCluster, - plugins: append(trafficRouterPlugins, metricProviderPlugins...), + plugins: slices.Concat(trafficRouterPlugins, metricProviderPlugins, stepPlugins), + lock: &sync.RWMutex{}, } mutex.Unlock() @@ -70,6 +90,7 @@ func InitializeConfig(k8sClientset kubernetes.Interface, configMapName string) ( func GetConfig() (*Config, error) { mutex.RLock() defer mutex.RUnlock() + if configMemoryCache == nil { return nil, fmt.Errorf("config not initialized, please initialize before use") } @@ -85,12 +106,30 @@ func UnInitializeConfig() { // GetAllPlugins returns a flattened list of plugin items. This is useful for iterating over all plugins. func (c *Config) GetAllPlugins() []types.PluginItem { - mutex.RLock() - defer mutex.RUnlock() + c.lock.RLock() + defer c.lock.RUnlock() + // Return a copy of the slice + return append([]types.PluginItem{}, c.plugins...) +} + +// GetPlugin returns the plugin item by name and type if it exists +func (c *Config) GetPlugin(name string, pluginType types.PluginType) *types.PluginItem { + for _, plugin := range c.GetAllPlugins() { + if plugin.Name == name && plugin.Type == pluginType { + return &plugin + } + } + return nil +} - var copiedPlugins []types.PluginItem - copiedPlugins = append(copiedPlugins, configMemoryCache.plugins...) - return copiedPlugins +func (c *Config) ValidateConfig() error { + for _, pluginItem := range c.GetAllPlugins() { + matches := re.FindAllStringSubmatch(pluginItem.Name, -1) + if len(matches) != 1 || len(matches[0]) != 3 { + return fmt.Errorf("plugin repository (%s) must be in the format of /", pluginItem.Name) + } + } + return nil } // GetPluginDirectoryAndFilename this functions return the directory and file name from a given pluginName such as @@ -105,16 +144,3 @@ func GetPluginDirectoryAndFilename(pluginName string) (directory string, filenam return namespace, plugin, nil } - -func (c *Config) ValidateConfig() error { - mutex.RLock() - defer mutex.RUnlock() - - for _, pluginItem := range c.GetAllPlugins() { - matches := re.FindAllStringSubmatch(pluginItem.Name, -1) - if len(matches) != 1 || len(matches[0]) != 3 { - return fmt.Errorf("plugin repository (%s) must be in the format of /", pluginItem.Name) - } - } - return nil -} diff --git a/utils/controller/controller.go b/utils/controller/controller.go index 2530e1d5fa..b5c1fc875b 100644 --- a/utils/controller/controller.go +++ b/utils/controller/controller.go @@ -6,6 +6,8 @@ import ( "runtime/debug" "time" + "k8s.io/apimachinery/pkg/api/errors" + log "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -157,6 +159,11 @@ func processNextWorkItem(ctx context.Context, workqueue workqueue.RateLimitingIn if err := runSyncHandler(); err != nil { logCtx.Errorf("%s syncHandler error: %v", objType, err) metricsServer.IncError(namespace, name, objType) + + if errors.IsNotFound(err) { + workqueue.Forget(obj) + return nil + } // Put the item back on // the workqueue to handle any transient errors. workqueue.AddRateLimited(key) diff --git a/utils/controller/controller_test.go b/utils/controller/controller_test.go index 3761d05ba5..39f3c134b5 100644 --- a/utils/controller/controller_test.go +++ b/utils/controller/controller_test.go @@ -23,13 +23,14 @@ import ( kubetesting "k8s.io/client-go/testing" "k8s.io/client-go/util/workqueue" + "k8s.io/client-go/tools/cache" + "github.com/argoproj/argo-rollouts/controller/metrics" register "github.com/argoproj/argo-rollouts/pkg/apis/rollouts" "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/fake" informers "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions" "github.com/argoproj/argo-rollouts/utils/log" - "k8s.io/client-go/tools/cache" ) func TestProcessNextWorkItemHandlePanic(t *testing.T) { diff --git a/utils/evaluate/evaluate.go b/utils/evaluate/evaluate.go index a2c9a607cd..435622303e 100644 --- a/utils/evaluate/evaluate.go +++ b/utils/evaluate/evaluate.go @@ -6,6 +6,7 @@ import ( "math" "reflect" "strconv" + "time" "github.com/antonmedv/expr" "github.com/antonmedv/expr/file" @@ -56,6 +57,39 @@ func EvaluateResult(result any, metric v1alpha1.Metric, logCtx logrus.Entry) (v1 return v1alpha1.AnalysisPhaseSuccessful, nil } +func EvalTime(expression string) (time.Time, error) { + var err error + + env := map[string]any{ + "isNaN": math.IsNaN, + "isInf": isInf, + } + + unwrapFileErr := func(e error) error { + if fileErr, ok := err.(*file.Error); ok { + e = errors.New(fileErr.Message) + } + return e + } + + program, err := expr.Compile(expression, expr.Env(env)) + if err != nil { + return time.Time{}, unwrapFileErr(err) + } + + output, err := expr.Run(program, env) + if err != nil { + return time.Time{}, unwrapFileErr(err) + } + + switch val := output.(type) { + case time.Time: + return val, nil + default: + return time.Time{}, fmt.Errorf("expected time.Time, but got %T", val) + } +} + // EvalCondition evaluates the condition with the resultValue as an input func EvalCondition(resultValue any, condition string) (bool, error) { var err error diff --git a/utils/evaluate/evaluate_test.go b/utils/evaluate/evaluate_test.go index 7839e47256..61dfc702b0 100644 --- a/utils/evaluate/evaluate_test.go +++ b/utils/evaluate/evaluate_test.go @@ -5,6 +5,7 @@ import ( "math" "reflect" "testing" + "time" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -329,3 +330,21 @@ func TestValueFromPointer(t *testing.T) { assert.True(t, valueFromPointer(1) == 1) assert.True(t, valueFromPointer(false) == false) } + +func TestEvalTimeWithSuccessExpr(t *testing.T) { + status, err := EvalTime(`date("2023-08-14 00:00:00", "2006-01-02 15:04:05", "UTC") - duration("1h")`) + assert.Equal(t, time.Date(2023, time.August, 13, 23, 0, 0, 0, time.UTC), status) + assert.NoError(t, err) +} + +func TestEvalTimeWithNotTimeResult(t *testing.T) { + status, err := EvalTime(`hello`) + assert.Equal(t, time.Time{}, status) + assert.Error(t, err) +} + +func TestEvalTimeWithInvalidExpression(t *testing.T) { + status, err := EvalTime(`now() -- ?`) + assert.Equal(t, time.Time{}, status) + assert.Error(t, err) +} diff --git a/utils/istio/istio_test.go b/utils/istio/istio_test.go index a4e5f5fe46..8e0d20d6d7 100644 --- a/utils/istio/istio_test.go +++ b/utils/istio/istio_test.go @@ -6,11 +6,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" - "github.com/argoproj/argo-rollouts/utils/defaults" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/runtime" dynamicfake "k8s.io/client-go/dynamic/fake" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/utils/defaults" ) func NewFakeDynamicClient(objects ...runtime.Object) *dynamicfake.FakeDynamicClient { diff --git a/utils/plugin/downloader.go b/utils/plugin/downloader.go index b8b1ad6263..17c7ba1bf7 100644 --- a/utils/plugin/downloader.go +++ b/utils/plugin/downloader.go @@ -1,6 +1,7 @@ package plugin import ( + "context" "crypto/sha256" "fmt" "io" @@ -10,6 +11,9 @@ import ( "path/filepath" "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + argoConfig "github.com/argoproj/argo-rollouts/utils/config" "github.com/argoproj/argo-rollouts/utils/defaults" @@ -19,15 +23,20 @@ import ( // FileDownloader is an interface that allows us to mock the http.Get function type FileDownloader interface { - Get(url string) (resp *http.Response, err error) + Get(url string, header http.Header) (resp *http.Response, err error) } // FileDownloaderImpl is the default/real implementation of the FileDownloader interface type FileDownloaderImpl struct { } -func (fd FileDownloaderImpl) Get(url string) (resp *http.Response, err error) { - return http.Get(url) +func (fd FileDownloaderImpl) Get(url string, header http.Header) (resp *http.Response, err error) { + request, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + request.Header = header + return http.DefaultClient.Do(request) } // checkPluginExists this function checks if the plugin exists in the configured path on the filesystem @@ -43,12 +52,17 @@ func checkPluginExists(pluginLocation string) error { } func checkShaOfPlugin(pluginLocation string, expectedSha256 string) (bool, error) { - hasher := sha256.New() fileBytes, err := os.ReadFile(pluginLocation) if err != nil { return false, fmt.Errorf("failed to read file %s: %w", pluginLocation, err) } - fileSha256 := fmt.Sprintf("%x", hasher.Sum(fileBytes)) + var fileSha256 string + if len(expectedSha256) == 64 { + fileSha256 = fmt.Sprintf("%x", sha256.Sum256(fileBytes)) + } else { + hasher := sha256.New() + fileSha256 = fmt.Sprintf("%x", hasher.Sum(fileBytes)) + } match := fileSha256 == expectedSha256 if !match { log.Printf("expected sha256: %s, actual sha256: %s, of downloaded metric plugin (%s)", expectedSha256, fileSha256, pluginLocation) @@ -56,12 +70,17 @@ func checkShaOfPlugin(pluginLocation string, expectedSha256 string) (bool, error return match, nil } -func downloadFile(filepath string, url string, downloader FileDownloader) error { - // Get the data - resp, err := downloader.Get(url) +func downloadFile(filepath string, url string, downloader FileDownloader, header http.Header) error { + // Get the data with credentials + resp, err := downloader.Get(url, header) if err != nil { return fmt.Errorf("failed to download file from %s: %w", url, err) } + + if isFailure(resp.StatusCode) { + return fmt.Errorf("failed to download file from %s: response code %s", url, http.StatusText(resp.StatusCode)) + } + defer resp.Body.Close() // Create the file @@ -87,7 +106,7 @@ func downloadFile(filepath string, url string, downloader FileDownloader) error } // DownloadPlugins this function downloads and/or checks that a plugin executable exits on the filesystem -func DownloadPlugins(fd FileDownloader) error { +func DownloadPlugins(fd FileDownloader, kubeClient kubernetes.Interface) error { config, err := argoConfig.GetConfig() if err != nil { return fmt.Errorf("failed to get config: %w", err) @@ -121,7 +140,18 @@ func DownloadPlugins(fd FileDownloader) error { case "http", "https": log.Infof("Downloading plugin %s from: %s", plugin.Name, plugin.Location) startTime := time.Now() - err = downloadFile(finalFileLocation, urlObj.String(), fd) + requestHeader := http.Header{} + for _, header := range plugin.HeadersFrom { + secret, err := kubeClient.CoreV1().Secrets(defaults.Namespace()).Get(context.Background(), header.SecretRef.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get secret in secretRef: %w", err) + } + for k, v := range secret.Data { + requestHeader.Add(k, string(v)) + } + } + + err = downloadFile(finalFileLocation, urlObj.String(), fd, requestHeader) if err != nil { return fmt.Errorf("failed to download plugin from %s: %w", plugin.Location, err) } @@ -196,3 +226,8 @@ func copyFile(src, dst string) error { } return nil } + +// isFailure determines if the response has a 2xx response +func isFailure(statusCode int) bool { + return statusCode < http.StatusOK || statusCode >= http.StatusBadRequest +} diff --git a/utils/plugin/downloader_test.go b/utils/plugin/downloader_test.go index 75dc4cae71..601e20dda4 100644 --- a/utils/plugin/downloader_test.go +++ b/utils/plugin/downloader_test.go @@ -10,18 +10,30 @@ import ( "github.com/argoproj/argo-rollouts/utils/config" - "github.com/argoproj/argo-rollouts/utils/defaults" "github.com/tj/assert" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" + + "github.com/argoproj/argo-rollouts/utils/defaults" ) type MockFileDownloader struct { FileDownloader } -func (m MockFileDownloader) Get(url string) (*http.Response, error) { +func (m MockFileDownloader) Get(url string, header http.Header) (*http.Response, error) { + if url == "https://test/plugin/fail" { + return &http.Response{ + Status: "404", + StatusCode: 404, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: nil, + ContentLength: 4, + }, nil + } responseBody := io.NopCloser(bytes.NewReader([]byte(`test`))) return &http.Response{ Status: "200", @@ -47,16 +59,23 @@ func TestPlugin(t *testing.T) { Name: "argo-rollouts-config", Namespace: "argo-rollouts", }, - Data: map[string]string{"metricProviderPlugins": "\n - name: argoproj-labs/http\n location: https://test/plugin\n - name: argoproj-labs/http-sha\n location: https://test/plugin\n sha256: 74657374e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + Data: map[string]string{"metricProviderPlugins": "\n - name: argoproj-labs/http\n location: https://test/plugin\n - name: argoproj-labs/http-sha\n location: https://test/plugin\n sha256: 74657374e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n - name: argoproj-labs/http-sha-correct\n location: https://test/plugin\n sha256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08\n - name: argoproj-labs/http-headers-correct\n location: https://test/plugin\n headersFrom:\n - secretRef:\n name: secret-name"}, } - client := fake.NewSimpleClientset(cm) + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-name", + Namespace: "argo-rollouts", + }, + Data: map[string][]byte{"Authorization": []byte("Basic VE9LRU4=")}, + } + client := fake.NewSimpleClientset(cm, secret) config.UnInitializeConfig() _, err := config.InitializeConfig(client, "argo-rollouts-config") assert.NoError(t, err) - err = DownloadPlugins(MockFileDownloader{}) + err = DownloadPlugins(MockFileDownloader{}, client) assert.NoError(t, err) dir, filename, err := config.GetPluginDirectoryAndFilename("argoproj-labs/http") @@ -68,12 +87,60 @@ func TestPlugin(t *testing.T) { dir, filename, err = config.GetPluginDirectoryAndFilename("argoproj-labs/http-sha") assert.NoError(t, err) + err = os.Remove(filepath.Join(defaults.DefaultRolloutPluginFolder, dir, filename)) + assert.NoError(t, err) + + dir, filename, err = config.GetPluginDirectoryAndFilename("argoproj-labs/http-headers-correct") + assert.NoError(t, err) + err = os.Remove(filepath.Join(defaults.DefaultRolloutPluginFolder, dir, filename)) assert.NoError(t, err) err = os.RemoveAll(defaults.DefaultRolloutPluginFolder) assert.NoError(t, err) }) + t.Run("test failed download", func(t *testing.T) { + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "argo-rollouts-config", + Namespace: "argo-rollouts", + }, + Data: map[string]string{"metricProviderPlugins": "\n - name: argoproj-labs/http-fail\n location: https://test/plugin/fail\n"}, + } + client := fake.NewSimpleClientset(cm) + + config.UnInitializeConfig() + + _, err := config.InitializeConfig(client, "argo-rollouts-config") + assert.NoError(t, err) + + err = DownloadPlugins(MockFileDownloader{}, client) + assert.Error(t, err) + err = os.RemoveAll(defaults.DefaultRolloutPluginFolder) + assert.NoError(t, err) + }) + + t.Run("test failed finding secret for header", func(t *testing.T) { + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "argo-rollouts-config", + Namespace: "argo-rollouts", + }, + Data: map[string]string{"metricProviderPlugins": " - name: argoproj-labs/http-headers-correct\n location: https://test/plugin\n headersFrom:\n - secretRef:\n name: secret-name"}, + } + client := fake.NewSimpleClientset(cm) + + config.UnInitializeConfig() + + _, err := config.InitializeConfig(client, "argo-rollouts-config") + assert.NoError(t, err) + + err = DownloadPlugins(MockFileDownloader{}, client) + assert.Error(t, err) + err = os.RemoveAll(defaults.DefaultRolloutPluginFolder) + assert.NoError(t, err) + }) + t.Run("test bad sha", func(t *testing.T) { cm := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -89,7 +156,7 @@ func TestPlugin(t *testing.T) { _, err := config.InitializeConfig(client, defaults.DefaultRolloutsConfigMapName) assert.NoError(t, err) - err = DownloadPlugins(MockFileDownloader{}) + err = DownloadPlugins(MockFileDownloader{}, client) assert.Error(t, err) dir, filename, err := config.GetPluginDirectoryAndFilename("argoproj-labs/http-badsha") @@ -107,9 +174,9 @@ func TestPlugin(t *testing.T) { cm, err := config.InitializeConfig(client, defaults.DefaultRolloutsConfigMapName) assert.NoError(t, err) - assert.Equal(t, cm, &config.Config{}) + assert.Equal(t, 0, len(cm.GetAllPlugins())) - err = DownloadPlugins(MockFileDownloader{}) + err = DownloadPlugins(MockFileDownloader{}, client) assert.NoError(t, err) err = os.RemoveAll(defaults.DefaultRolloutPluginFolder) assert.NoError(t, err) @@ -130,7 +197,7 @@ func TestPlugin(t *testing.T) { _, err := config.InitializeConfig(client, defaults.DefaultRolloutsConfigMapName) assert.NoError(t, err) - err = DownloadPlugins(MockFileDownloader{}) + err = DownloadPlugins(MockFileDownloader{}, client) assert.NoError(t, err) dir, filename, err := config.GetPluginDirectoryAndFilename("argoproj-labs/file-plugin") @@ -159,7 +226,7 @@ func TestPlugin(t *testing.T) { _, err = config.InitializeConfig(client, defaults.DefaultRolloutsConfigMapName) assert.NoError(t, err) - err = DownloadPlugins(MockFileDownloader{}) + err = DownloadPlugins(MockFileDownloader{}, client) assert.NoError(t, err) dir, filename, err := config.GetPluginDirectoryAndFilename("namespace/file-plugin") @@ -186,7 +253,7 @@ func TestPlugin(t *testing.T) { _, err := config.InitializeConfig(client, "argo-rollouts-config") assert.Error(t, err) - err = DownloadPlugins(MockFileDownloader{}) + err = DownloadPlugins(MockFileDownloader{}, client) assert.Error(t, err) }) @@ -205,7 +272,7 @@ func TestPlugin(t *testing.T) { _, err := config.InitializeConfig(client, "argo-rollouts-config") assert.NoError(t, err) - err = DownloadPlugins(MockFileDownloader{}) + err = DownloadPlugins(MockFileDownloader{}, client) assert.Error(t, err) err = os.RemoveAll(defaults.DefaultRolloutPluginFolder) @@ -251,9 +318,18 @@ func TestCheckShaOfPlugin(t *testing.T) { } func TestDownloadFile(t *testing.T) { - err := downloadFile("error", "", FileDownloaderImpl{}) - assert.Error(t, err) - assert.Contains(t, err.Error(), "failed to download file from") + t.Run("test sha of real file", func(t *testing.T) { + err := downloadFile("error", " ", FileDownloaderImpl{}, nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to download file from") + }) + + t.Run("test download fail with invalid url", func(t *testing.T) { + url := "://example.com" + err := downloadFile("error", url, FileDownloaderImpl{}, nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to download file from") + }) } func Test_copyFile(t *testing.T) { diff --git a/utils/plugin/plugin.go b/utils/plugin/plugin.go index 3fd7a013a0..a76ed5e942 100644 --- a/utils/plugin/plugin.go +++ b/utils/plugin/plugin.go @@ -5,30 +5,32 @@ import ( "path/filepath" "github.com/argoproj/argo-rollouts/utils/defaults" + "github.com/argoproj/argo-rollouts/utils/plugin/types" "github.com/argoproj/argo-rollouts/utils/config" ) // GetPluginInfo returns the location & command arguments of the plugin on the filesystem via plugin name. If the plugin is not // configured in the configmap, an error is returned. -func GetPluginInfo(pluginName string) (string, []string, error) { +func GetPluginInfo(pluginName string, pluginType types.PluginType) (string, []string, error) { configMap, err := config.GetConfig() if err != nil { return "", nil, fmt.Errorf("failed to get config: %w", err) } - for _, item := range configMap.GetAllPlugins() { - if pluginName == item.Name { - dir, filename, err := config.GetPluginDirectoryAndFilename(item.Name) - if err != nil { - return "", nil, err - } - absFilePath, err := filepath.Abs(filepath.Join(defaults.DefaultRolloutPluginFolder, dir, filename)) - if err != nil { - return "", nil, fmt.Errorf("failed to get absolute path of plugin folder: %w", err) - } - return absFilePath, item.Args, nil - } + plugin := configMap.GetPlugin(pluginName, pluginType) + if plugin == nil { + return "", nil, fmt.Errorf("plugin %s not configured in configmap", pluginName) } - return "", nil, fmt.Errorf("plugin %s not configured in configmap", pluginName) + + dir, filename, err := config.GetPluginDirectoryAndFilename(plugin.Name) + if err != nil { + return "", nil, err + } + absFilePath, err := filepath.Abs(filepath.Join(defaults.DefaultRolloutPluginFolder, dir, filename)) + if err != nil { + return "", nil, fmt.Errorf("failed to get absolute path of plugin folder: %w", err) + } + return absFilePath, plugin.Args, nil + } diff --git a/utils/plugin/plugin_test.go b/utils/plugin/plugin_test.go index 7226d2d312..f380105a5b 100644 --- a/utils/plugin/plugin_test.go +++ b/utils/plugin/plugin_test.go @@ -5,12 +5,14 @@ import ( "testing" "github.com/argoproj/argo-rollouts/utils/defaults" + "github.com/argoproj/argo-rollouts/utils/plugin/types" - "github.com/argoproj/argo-rollouts/utils/config" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" + + "github.com/argoproj/argo-rollouts/utils/config" ) func TestGetPluginInfo(t *testing.T) { @@ -29,14 +31,14 @@ func TestGetPluginInfo(t *testing.T) { _, err := config.InitializeConfig(client, "argo-rollouts-config") assert.NoError(t, err) - location, args, err := GetPluginInfo("argoproj-labs/http") + location, args, err := GetPluginInfo("argoproj-labs/http", types.PluginTypeMetricProvider) assert.NoError(t, err) fp, err := filepath.Abs(filepath.Join(defaults.DefaultRolloutPluginFolder, "argoproj-labs/http")) assert.NoError(t, err) assert.Equal(t, fp, location) assert.Equal(t, args, cmdArgs) - _, args, _ = GetPluginInfo("argoproj-labs/http-sha") + _, args, _ = GetPluginInfo("argoproj-labs/http-sha", types.PluginTypeMetricProvider) assert.Equal(t, len(args), 0) }) @@ -54,14 +56,39 @@ func TestGetPluginInfo(t *testing.T) { _, err := config.InitializeConfig(client, "argo-rollouts-config") assert.NoError(t, err) - location, args, err := GetPluginInfo("argoproj-labs/router") + location, args, err := GetPluginInfo("argoproj-labs/router", types.PluginTypeTrafficRouter) assert.NoError(t, err) fp, err := filepath.Abs(filepath.Join(defaults.DefaultRolloutPluginFolder, "argoproj-labs/router")) assert.NoError(t, err) assert.Equal(t, fp, location) assert.Equal(t, args, cmdArgs) - _, args, _ = GetPluginInfo("argoproj-labs/router-sha") + _, args, _ = GetPluginInfo("argoproj-labs/router-sha", types.PluginTypeTrafficRouter) + assert.Equal(t, len(args), 0) + }) + + t.Run("tests getting plugin location of step plugins", func(t *testing.T) { + + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "argo-rollouts-config", + Namespace: "argo-rollouts", + }, + Data: map[string]string{"stepPlugins": "\n - name: argoproj-labs/steps\n location: https://test/plugin\n args: [\"-l 2\"]\n - name: argoproj-labs/steps-sha\n location: https://test/plugin\n sha256: 74657374e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + } + client := fake.NewSimpleClientset(cm) + + _, err := config.InitializeConfig(client, "argo-rollouts-config") + assert.NoError(t, err) + + location, args, err := GetPluginInfo("argoproj-labs/steps", types.PluginTypeStep) + assert.NoError(t, err) + fp, err := filepath.Abs(filepath.Join(defaults.DefaultRolloutPluginFolder, "argoproj-labs/steps")) + assert.NoError(t, err) + assert.Equal(t, fp, location) + assert.Equal(t, args, cmdArgs) + + _, args, _ = GetPluginInfo("argoproj-labs/step-sha", types.PluginTypeStep) assert.Equal(t, len(args), 0) }) @@ -78,10 +105,30 @@ func TestGetPluginInfo(t *testing.T) { _, err := config.InitializeConfig(client, "argo-rollouts-config") assert.NoError(t, err) - location, args, err := GetPluginInfo("does-not-exist") + location, args, err := GetPluginInfo("does-not-exist", types.PluginTypeMetricProvider) assert.Error(t, err) assert.Equal(t, "plugin does-not-exist not configured in configmap", err.Error()) assert.Equal(t, "", location) assert.Equal(t, len(args), 0) }) + + t.Run("test getting plugin location from a plugin of a different type", func(t *testing.T) { + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "argo-rollouts-config", + Namespace: "argo-rollouts", + }, + Data: map[string]string{"metricProviderPlugins": "\n - name: argoproj-labs/http\n location: https://test/plugin\n - name: argoproj-labs/http-sha\n location: https://test/plugin\n sha256: 74657374e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + } + client := fake.NewSimpleClientset(cm) + + _, err := config.InitializeConfig(client, "argo-rollouts-config") + assert.NoError(t, err) + + location, args, err := GetPluginInfo("argoproj-labs/http", types.PluginTypeStep) + assert.Error(t, err) + assert.Equal(t, "plugin argoproj-labs/http not configured in configmap", err.Error()) + assert.Equal(t, "", location) + assert.Equal(t, len(args), 0) + }) } diff --git a/utils/plugin/types/steps.go b/utils/plugin/types/steps.go new file mode 100644 index 0000000000..f54d7ec00b --- /dev/null +++ b/utils/plugin/types/steps.go @@ -0,0 +1,55 @@ +package types + +import ( + "encoding/json" + "fmt" + "time" +) + +// StepPhase is the type of phase of step plugin result +type StepPhase string + +const ( + // PhaseRunning is the Running phase of a step plugin + PhaseRunning StepPhase = "Running" + // PhaseRunning is the Successful phase of a step plugin + PhaseSuccessful StepPhase = "Successful" + // PhaseRunning is the Failed phase of a step plugin + PhaseFailed StepPhase = "Failed" + // PhaseRunning is the Error phase of a step plugin + PhaseError StepPhase = "Error" +) + +// RpcStepContext is the context of the step plugin operation +type RpcStepContext struct { + // PluginName is the name of the plugin as defined by the user + PluginName string + // Config holds the user specified configuration in the Rollout object for this plugin step + Config json.RawMessage + // Status holds a previous execution status related to the operation + Status json.RawMessage +} + +type RpcStepResult struct { + // Phase of the operation to idicate if it has completed or not + Phase StepPhase + // Message contains information about the execution + Message string + // RequeueAfter is the duration to wait before executing the operation again when it does not return a completed phase + RequeueAfter time.Duration + // Status hold the execution status of this plugin step. It can be used to persist a state between executions + Status json.RawMessage +} + +// Validate the phase of a step plugin +func (p StepPhase) Validate() error { + switch p { + case PhaseRunning: + case PhaseSuccessful: + case PhaseFailed: + case PhaseError: + default: + return fmt.Errorf("phase '%s' is not valid", p) + } + return nil +} diff --git a/utils/plugin/types/types.go b/utils/plugin/types/types.go index e89ba40f2a..e3555a9ae5 100644 --- a/utils/plugin/types/types.go +++ b/utils/plugin/types/types.go @@ -85,24 +85,65 @@ type RpcTrafficRoutingReconciler interface { Type() string } -//type Plugin struct { -// MetricProviders []PluginItem `json:"metricProviders" yaml:"metricProviders"` -// TrafficRouters []PluginItem `json:"trafficRouters" yaml:"trafficRouters"` -//} +type RpcStep interface { + // Run executes a step plugin for the RpcStepContext and returns the result to the controller or an RpcError for unexpeted failures + Run(*v1alpha1.Rollout, *RpcStepContext) (RpcStepResult, RpcError) + // Terminate stops an uncompleted operation started by the Run operation + Terminate(*v1alpha1.Rollout, *RpcStepContext) (RpcStepResult, RpcError) + // Abort reverts the actions performed during the Run operation if necessary + Abort(*v1alpha1.Rollout, *RpcStepContext) (RpcStepResult, RpcError) + // Type returns the type of the step plugin + Type() string +} type TrafficRouterPlugins struct { + // TrafficRouters is the list of plugin that implements a RpcTrafficRoutingReconciler TrafficRouters []PluginItem `json:"trafficRouterPlugins" yaml:"trafficRouterPlugins"` } type MetricProviderPlugins struct { + // MetricProviders is the list of plugin that implements a RpcMetricProvider MetricProviders []PluginItem `json:"metricProviderPlugins" yaml:"metricProviderPlugins"` } +type StepPlugins struct { + // Steps is the list of plugin that implements a RpcStep + Steps []PluginItem `json:"stepPlugins" yaml:"stepPlugins"` +} + +// PluginType is a type of plugin +type PluginType string + +const ( + // PluginTypeMetricProvider is the type for a MetricProvider plugin + PluginTypeMetricProvider PluginType = "MetricProvider" + // PluginTypeTrafficRouter is the type for a TrafficRouter plugin + PluginTypeTrafficRouter PluginType = "TrafficRouter" + // PluginTypeStep is the type for a Step plugin + PluginTypeStep PluginType = "Step" +) + type PluginItem struct { - Name string `json:"name" yaml:"name"` + // Name of the plugin to use in the Rollout custom resources + Name string `json:"name" yaml:"name"` + // Location of the plugin. Supports http(s):// urls and file:// prefix Location string `json:"location" yaml:"location"` - Sha256 string `json:"sha256" yaml:"sha256"` - - // Args holds command line arguments + // Sha256 is the checksum of the file specified at the provided Location + Sha256 string `json:"sha256" yaml:"sha256"` + // Type of the plugin + Type PluginType + // Disabled indicates if the plugin should be ignored when referenced in Rollout custom resources. Only valid for a plugin of type Step. + Disabled bool `json:"disabled" yaml:"disabled"` + // Args holds command line arguments to initialize the plugin Args []string `json:"args" yaml:"args"` + // HeadersFrom holds the names of secrets where the headers should be pulled from + HeadersFrom []HeadersFrom `json:"headersFrom" yaml:"headersFrom"` +} + +type HeadersFrom struct { + SecretRef SecretRef `json:"secretRef" yaml:"secretRef"` +} + +type SecretRef struct { + Name string `json:"name" yaml:"name"` } diff --git a/utils/record/record.go b/utils/record/record.go index 96be7c8582..8180981213 100644 --- a/utils/record/record.go +++ b/utils/record/record.go @@ -12,8 +12,6 @@ import ( "sync" "time" - argoinformers "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions/rollouts/v1alpha1" - timeutil "github.com/argoproj/argo-rollouts/utils/time" "github.com/argoproj/notifications-engine/pkg/api" "github.com/argoproj/notifications-engine/pkg/services" "github.com/argoproj/notifications-engine/pkg/subscriptions" @@ -33,6 +31,9 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/kubectl/pkg/scheme" + argoinformers "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions/rollouts/v1alpha1" + timeutil "github.com/argoproj/argo-rollouts/utils/time" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" rolloutscheme "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/scheme" "github.com/argoproj/argo-rollouts/utils/annotations" diff --git a/utils/record/record_test.go b/utils/record/record_test.go index dfea598eae..213fc456b1 100644 --- a/utils/record/record_test.go +++ b/utils/record/record_test.go @@ -9,12 +9,6 @@ import ( "testing" "time" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" - argofake "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/fake" - argoinformersfactory "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions" - argoinformers "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions/rollouts/v1alpha1" - "github.com/argoproj/argo-rollouts/utils/defaults" - timeutil "github.com/argoproj/argo-rollouts/utils/time" "github.com/argoproj/notifications-engine/pkg/api" notificationapi "github.com/argoproj/notifications-engine/pkg/api" "github.com/argoproj/notifications-engine/pkg/mocks" @@ -33,6 +27,13 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + argofake "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/fake" + argoinformersfactory "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions" + argoinformers "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/utils/defaults" + timeutil "github.com/argoproj/argo-rollouts/utils/time" ) var ( diff --git a/utils/replicaset/canary.go b/utils/replicaset/canary.go old mode 100755 new mode 100644 index cf41e6baa5..49c5ed4582 --- a/utils/replicaset/canary.go +++ b/utils/replicaset/canary.go @@ -4,6 +4,8 @@ import ( "encoding/json" "math" + "github.com/argoproj/argo-rollouts/utils/annotations" + log "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -15,7 +17,7 @@ import ( const ( // EphemeralMetadataAnnotation denotes pod metadata which is ephemerally injected to canary/stable pods - EphemeralMetadataAnnotation = "rollout.argoproj.io/ephemeral-metadata" + EphemeralMetadataAnnotation = annotations.RolloutLabel + "/ephemeral-metadata" ) func allDesiredAreAvailable(rs *appsv1.ReplicaSet, desired int32) bool { @@ -638,18 +640,26 @@ func SyncEphemeralPodMetadata(metadata *metav1.ObjectMeta, existingPodMetadata, if existingPodMetadata != nil { for k := range existingPodMetadata.Annotations { if desiredPodMetadata == nil || !isMetadataStillDesired(k, desiredPodMetadata.Annotations) { - if metadata.Annotations != nil { - delete(metadata.Annotations, k) - modified = true + if metadata.Annotations == nil { + continue + } + if _, ok := metadata.Annotations[k]; !ok { + continue } + delete(metadata.Annotations, k) + modified = true } } for k := range existingPodMetadata.Labels { if desiredPodMetadata == nil || !isMetadataStillDesired(k, desiredPodMetadata.Labels) { - if metadata.Labels != nil { - delete(metadata.Labels, k) - modified = true + if metadata.Labels == nil { + continue } + if _, ok := metadata.Labels[k]; !ok { + continue + } + delete(metadata.Labels, k) + modified = true } } } diff --git a/utils/replicaset/canary_test.go b/utils/replicaset/canary_test.go index dbe4ce5a03..a0d5900a4f 100755 --- a/utils/replicaset/canary_test.go +++ b/utils/replicaset/canary_test.go @@ -1339,14 +1339,19 @@ func TestSyncEphemeralPodMetadata(t *testing.T) { "ddd": "444", }, } - { - // verify modified is false if there are no changes + t.Run("verify modified is false if there are no changes", func(t *testing.T) { newMetadata, modified := SyncEphemeralPodMetadata(&meta, &existing, &existing) assert.False(t, modified) assert.Equal(t, meta, *newMetadata) - } - { - // verify we don't touch metadata that we did not inject ourselves + }) + t.Run("verify modified is false if there are no actual deletions", func(t *testing.T) { + existingWithExtraLabel := existing.DeepCopy() + existingWithExtraLabel.Labels["foo"] = "bar" + newMetadata, modified := SyncEphemeralPodMetadata(&meta, existingWithExtraLabel, &existing) + assert.False(t, modified) + assert.Equal(t, meta, *newMetadata) + }) + t.Run("verify we don't touch metadata that we did not inject ourselves", func(t *testing.T) { desired := v1alpha1.PodTemplateMetadata{ Labels: map[string]string{ "aaa": "222", @@ -1356,7 +1361,6 @@ func TestSyncEphemeralPodMetadata(t *testing.T) { }, } newMetadata, modified := SyncEphemeralPodMetadata(&meta, &existing, &desired) - assert.True(t, modified) expected := metav1.ObjectMeta{ Labels: map[string]string{ "aaa": "222", @@ -1369,8 +1373,7 @@ func TestSyncEphemeralPodMetadata(t *testing.T) { } assert.True(t, modified) assert.Equal(t, expected, *newMetadata) - } - + }) } func TestGetReplicasForScaleDown(t *testing.T) { diff --git a/utils/replicaset/replicaset.go b/utils/replicaset/replicaset.go index b2664afd53..272449e778 100644 --- a/utils/replicaset/replicaset.go +++ b/utils/replicaset/replicaset.go @@ -72,6 +72,7 @@ func FindNewReplicaSet(rollout *v1alpha1.Rollout, rsList []*appsv1.ReplicaSet) * } } // new ReplicaSet does not exist. + return nil } diff --git a/utils/rollout/rolloututil.go b/utils/rollout/rolloututil.go index 0b7df7ff38..35a3bca9c3 100644 --- a/utils/rollout/rolloututil.go +++ b/utils/rollout/rolloututil.go @@ -4,6 +4,8 @@ import ( "fmt" "strconv" + "github.com/argoproj/argo-rollouts/utils/weightutil" + replicasetutil "github.com/argoproj/argo-rollouts/utils/replicaset" "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" @@ -179,18 +181,21 @@ func CanaryStepString(c v1alpha1.CanaryStep) string { return fmt.Sprintf("setCanaryScale{replicas: %d}", *c.SetCanaryScale.Replicas) } } + if c.Plugin != nil { + return fmt.Sprintf("plugin: %s", c.Plugin.Name) + } return "invalid" } // ShouldVerifyWeight We use this to test if we should verify weights because weight verification could involve // API calls to the cloud provider which could incur rate limiting -func ShouldVerifyWeight(ro *v1alpha1.Rollout) bool { +func ShouldVerifyWeight(ro *v1alpha1.Rollout, desiredWeight int32) bool { currentStep, _ := replicasetutil.GetCurrentCanaryStep(ro) // If we are in the middle of an update at a setWeight step, also perform weight verification. // Note that we don't do this every reconciliation because weight verification typically involves // API calls to the cloud provider which could incur rate limitingq - shouldVerifyWeight := ro.Status.StableRS != "" && - !IsFullyPromoted(ro) && - currentStep != nil && currentStep.SetWeight != nil + shouldVerifyWeight := (ro.Status.StableRS != "" && !IsFullyPromoted(ro) && currentStep != nil && currentStep.SetWeight != nil) || + (ro.Status.StableRS != "" && !IsFullyPromoted(ro) && currentStep == nil && desiredWeight == weightutil.MaxTrafficWeight(ro)) // We are at end of rollout + return shouldVerifyWeight } diff --git a/utils/rollout/rolloututil_test.go b/utils/rollout/rolloututil_test.go index 37c1810f00..d5a09423b5 100644 --- a/utils/rollout/rolloututil_test.go +++ b/utils/rollout/rolloututil_test.go @@ -5,12 +5,13 @@ import ( "testing" "time" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" - "github.com/argoproj/argo-rollouts/utils/annotations" "github.com/tj/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" + + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/argoproj/argo-rollouts/utils/annotations" ) func newCanaryRollout() *v1alpha1.Rollout { @@ -389,6 +390,10 @@ func TestCanaryStepString(t *testing.T) { step: v1alpha1.CanaryStep{SetCanaryScale: &v1alpha1.SetCanaryScale{Replicas: pointer.Int32Ptr(5)}}, expectedString: "setCanaryScale{replicas: 5}", }, + { + step: v1alpha1.CanaryStep{Plugin: &v1alpha1.PluginStep{Name: "foo"}}, + expectedString: "plugin: foo", + }, } for _, test := range tests { assert.Equal(t, test.expectedString, CanaryStepString(test.step)) @@ -422,15 +427,21 @@ func TestShouldVerifyWeight(t *testing.T) { ro.Spec.Strategy.Canary.Steps = []v1alpha1.CanaryStep{{ SetWeight: pointer.Int32Ptr(20), }} - assert.Equal(t, true, ShouldVerifyWeight(ro)) + assert.Equal(t, true, ShouldVerifyWeight(ro, 20)) ro.Status.StableRS = "" - assert.Equal(t, false, ShouldVerifyWeight(ro)) + assert.Equal(t, false, ShouldVerifyWeight(ro, 20)) + + ro.Status.StableRS = "34feab23f" + ro.Status.CurrentStepIndex = nil + ro.Spec.Strategy.Canary.Steps = nil + assert.Equal(t, false, ShouldVerifyWeight(ro, 20)) + // Test when the weight is 100, because we are at end of rollout ro.Status.StableRS = "34feab23f" ro.Status.CurrentStepIndex = nil ro.Spec.Strategy.Canary.Steps = nil - assert.Equal(t, false, ShouldVerifyWeight(ro)) + assert.Equal(t, true, ShouldVerifyWeight(ro, 100)) } func Test_isGenerationObserved(t *testing.T) { diff --git a/utils/template/template_test.go b/utils/template/template_test.go index 8f493bcd58..84580c9c8a 100644 --- a/utils/template/template_test.go +++ b/utils/template/template_test.go @@ -5,11 +5,12 @@ import ( "testing" "time" - "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" + "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" + "github.com/stretchr/testify/assert" )