diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..7e78e55 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,34 @@ +name: Lint + +on: + push: + pull_request: + +jobs: + golangci-lint: + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Run linter + uses: golangci/golangci-lint-action@v7 + with: + version: v2.3.0 + + buf-lint: + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@v4 + + - name: Install the `buf` CLI + uses: bufbuild/buf-setup-action@v1 + + - name: Run buf lint + uses: bufbuild/buf-lint-action@v1 diff --git a/.github/workflows/test-chart.yml b/.github/workflows/test-chart.yml new file mode 100644 index 0000000..4674510 --- /dev/null +++ b/.github/workflows/test-chart.yml @@ -0,0 +1,87 @@ +name: Test Chart + +on: + push: + pull_request: + +jobs: + test-chart: + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Install the latest version of kind + run: | + curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 + chmod +x ./kind + sudo mv ./kind /usr/local/bin/kind + + - name: Verify kind installation + run: kind version + + - name: Create kind cluster + run: kind create cluster + + - name: Prepare nodedrain + run: | + go mod tidy + make docker-build IMG_NAME_CONTROLLER=nodedrain IMG_TAG=v0.1.0 + kind load docker-image IMG_NAME_CONTROLLER=nodedrain IMG_TAG=v0.1.0 + + - name: Install Helm + run: | + curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + + - name: Verify Helm installation + run: helm version + + - name: Lint Helm Chart + run: | + helm lint ./dist/chart + + - name: Install cert-manager via Helm + run: | + helm repo add jetstack https://charts.jetstack.io + helm repo update + helm install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --set installCRDs=true + + - name: Wait for cert-manager to be ready + run: | + kubectl wait --namespace cert-manager --for=condition=available --timeout=300s deployment/cert-manager + kubectl wait --namespace cert-manager --for=condition=available --timeout=300s deployment/cert-manager-cainjector + kubectl wait --namespace cert-manager --for=condition=available --timeout=300s deployment/cert-manager-webhook +# TODO: Uncomment if Prometheus is enabled +# - name: Install Prometheus Operator CRDs +# run: | +# helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +# helm repo update +# helm install prometheus-crds prometheus-community/prometheus-operator-crds +# +# - name: Install Prometheus via Helm +# run: | +# helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +# helm repo update +# helm install prometheus prometheus-community/prometheus --namespace monitoring --create-namespace +# +# - name: Wait for Prometheus to be ready +# run: | +# kubectl wait --namespace monitoring --for=condition=available --timeout=300s deployment/prometheus-server + + - name: Install Helm chart for project + run: | + helm install my-release ./dist/chart --create-namespace --namespace nodedrain-system + + - name: Check Helm release status + run: | + helm status my-release --namespace nodedrain-system + +# TODO: Uncomment if prometheus.enabled is set to true to confirm that the ServiceMonitor gets created +# - name: Check Presence of ServiceMonitor +# run: | +# kubectl wait --namespace nodedrain-system --for=jsonpath='{.kind}'=ServiceMonitor servicemonitor/nodedrain-controller-manager-metrics-monitor diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml new file mode 100644 index 0000000..68fd1ed --- /dev/null +++ b/.github/workflows/test-e2e.yml @@ -0,0 +1,32 @@ +name: E2E Tests + +on: + push: + pull_request: + +jobs: + test-e2e: + name: Run on Ubuntu + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Install the latest version of kind + run: | + curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64 + chmod +x ./kind + sudo mv ./kind /usr/local/bin/kind + + - name: Verify kind installation + run: kind version + + - name: Running Test e2e + run: | + go mod tidy + make test-e2e diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..67dcfed --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,22 @@ +name: Tests + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Running Tests + run: | + go mod tidy + make test diff --git a/.gitignore b/.gitignore index 7f02333..ada68ff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ - # Binaries for programs and plugins *.exe *.exe~ @@ -8,14 +7,16 @@ bin/* Dockerfile.cross -# Test binary, build with `go test -c` +# Test binary, built with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out -# Kubernetes Generated files - skip generated files, except for vendored files +# Go workspace file +go.work +# Kubernetes Generated files - skip generated files, except for vendored files !vendor/**/zz_generated.* # editor and IDE paraphernalia diff --git a/.golangci.yml b/.golangci.yml index aac8a13..a7246fb 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,33 +1,15 @@ +version: "2" run: - timeout: 5m allow-parallel-runners: true - -issues: - # don't skip warning about doc comments - # don't exclude the default set of lint - exclude-use-default: false - # restore some of the defaults - # (fill in the rest as needed) - exclude-rules: - - path: "api/*" - linters: - - lll - - path: "internal/*" - linters: - - dupl - - lll linters: - disable-all: true + default: none enable: + - copyloopvar - dupl - errcheck - - exportloopref - ginkgolinter - goconst - gocyclo - - gofmt - - goimports - - gosimple - govet - ineffassign - lll @@ -36,12 +18,34 @@ linters: - prealloc - revive - staticcheck - - typecheck - unconvert - unparam - unused - -linters-settings: - revive: + settings: + revive: + rules: + - name: comment-spacings + exclusions: + generated: lax rules: - - name: comment-spacings + - linters: + - lll + path: api/* + - linters: + - dupl + - lll + path: internal/* + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + enable: + - gofmt + - goimports + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/Dockerfile b/Dockerfile index b1e137c..a2fdcb5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.23.2 AS builder +FROM golang:1.24.5 AS builder ARG TARGETOS ARG TARGETARCH @@ -21,11 +21,24 @@ COPY internal/ internal/ # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go +RUN --mount=type=cache,target=/root/.cache \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -o manager cmd/main.go + + +FROM builder AS builder-plugin +COPY examples/ examples/ +RUN --mount=type=cache,target=/root/.cache \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -o example-plugin.so examples/plugin/example-plugin.go + +FROM alpine:3.22.1 AS example-plugin +WORKDIR / +COPY --from=builder-plugin /workspace/example-plugin.so . +USER 65532:65532 +CMD ["cp", "-v", "/example-plugin.so", "/plugins/example-plugin.so"] # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/static:nonroot +FROM gcr.io/distroless/static:debug-nonroot AS controller WORKDIR / COPY --from=builder /workspace/manager . USER 65532:65532 diff --git a/Makefile b/Makefile index 3bca36a..4bb87ad 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,11 @@ # Image URL to use all building/pushing image targets -IMG ?= controller:latest -# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.31.0 +IMG_REGISTRY ?= +IMG_NAME_CONTROLLER ?= controller +IMG_NAME_EXAM_PLUGIN ?= example-plugin +IMG_TAG ?= latest +KUBE_CONTEXT ?= kind-nodedrain-test-e2e +KUSTOMIZE_CONFIG ?= default +IMG_REGISTRY_FULL := $(if ${IMG_REGISTRY},$(patsubst %/,%,${IMG_REGISTRY})/) # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -48,39 +52,71 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases .PHONY: generate -generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. +generate: controller-gen buf ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + $(BUF) generate $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." .PHONY: fmt -fmt: ## Run go fmt against code. +fmt: buf go fmt ./... + $(BUF) format -w .PHONY: vet vet: ## Run go vet against code. go vet ./... .PHONY: test -test: manifests generate fmt vet envtest ## Run tests. +test: manifests generate fmt vet setup-envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out -# Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. -.PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. -test-e2e: - go test ./test/e2e/ -v -ginkgo.v +# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. +# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. +# CertManager is installed by default; skip with: +# - CERT_MANAGER_INSTALL_SKIP=true +KIND_CLUSTER ?= nodedrain-test-e2e + +.PHONY: setup-test-e2e +setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist + @command -v $(KIND) >/dev/null 2>&1 || { \ + echo "Kind is not installed. Please install Kind manually."; \ + exit 1; \ + } + @case "$$($(KIND) get clusters)" in \ + *"$(KIND_CLUSTER)"*) \ + echo "Kind cluster '$(KIND_CLUSTER)' already exists. Skipping creation." ;; \ + *) \ + echo "Creating Kind cluster '$(KIND_CLUSTER)'..."; \ + $(KIND) create cluster --name $(KIND_CLUSTER) ;; \ + esac + +.PHONY: test-e2e +test-e2e: cleanup-test-e2e setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. + KIND_CLUSTER=$(KIND_CLUSTER) go test ./test/e2e/ -v -ginkgo.v + $(MAKE) cleanup-test-e2e + +.PHONY: cleanup-test-e2e +cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests + @$(KIND) delete cluster --name $(KIND_CLUSTER) .PHONY: lint -lint: golangci-lint ## Run golangci-lint linter +lint: golangci-lint buf ## Run golangci-lint linter $(GOLANGCI_LINT) run + $(BUF) lint .PHONY: lint-fix lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes $(GOLANGCI_LINT) run --fix +.PHONY: lint-config +lint-config: golangci-lint ## Verify golangci-lint linter configuration + $(GOLANGCI_LINT) config verify + ##@ Build .PHONY: build build: manifests generate fmt vet ## Build manager binary. go build -o bin/manager cmd/main.go + go build -o bin/example-plugin.so examples/plugin/example-plugin.go .PHONY: run run: manifests generate fmt vet ## Run a controller from your host. @@ -91,17 +127,19 @@ run: manifests generate fmt vet ## Run a controller from your host. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: docker-build docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build -t ${IMG} . + $(CONTAINER_TOOL) build --target controller -t $(IMG_REGISTRY_FULL)$(IMG_NAME_CONTROLLER):$(IMG_TAG) . + $(CONTAINER_TOOL) build --target example-plugin -t $(IMG_REGISTRY_FULL)$(IMG_NAME_EXAM_PLUGIN):$(IMG_TAG) . .PHONY: docker-push docker-push: ## Push docker image with the manager. - $(CONTAINER_TOOL) push ${IMG} + $(CONTAINER_TOOL) push $(IMG_REGISTRY_FULL)$(IMG_NAME_CONTROLLER):$(IMG_TAG) + $(CONTAINER_TOOL) push $(IMG_REGISTRY_FULL)$(IMG_NAME_EXAM_PLUGIN):$(IMG_TAG) # PLATFORMS defines the target platforms for the manager image be built to provide support to multiple -# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: +# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator IMG_TAG=0.0.1). To use this option you need to: # - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ # - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) +# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) # To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le .PHONY: docker-buildx @@ -110,15 +148,19 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - $(CONTAINER_TOOL) buildx create --name nodedrain-builder $(CONTAINER_TOOL) buildx use nodedrain-builder - - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . + - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag $(IMG_REGISTRY_FULL)$(IMG_NAME_CONTROLLER):$(IMG_TAG) -f Dockerfile.cross . - $(CONTAINER_TOOL) buildx rm nodedrain-builder rm Dockerfile.cross .PHONY: build-installer build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. mkdir -p dist - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default > dist/install.yaml + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG_REGISTRY_FULL)$(IMG_NAME_CONTROLLER):$(IMG_TAG) + $(KUSTOMIZE) build config/$(KUSTOMIZE_CONFIG) > dist/install.yaml + +.PHONY: build-helm-chart +build-helm-chart: manifests + kubebuilder edit --plugins=helm/v1-alpha ##@ Deployment @@ -128,20 +170,20 @@ endif .PHONY: install install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - + $(KUSTOMIZE) build config/crd | $(KUBECTL) --context $(KUBE_CONTEXT) apply -f - .PHONY: uninstall uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - + $(KUSTOMIZE) build config/crd | $(KUBECTL) --context $(KUBE_CONTEXT) delete --ignore-not-found=$(ignore-not-found) -f - .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - + cd config/${KUSTOMIZE_CONFIG} && $(KUSTOMIZE) edit set image controller=$(IMG_REGISTRY_FULL)$(IMG_NAME_CONTROLLER):$(IMG_TAG) + $(KUSTOMIZE) build config/${KUSTOMIZE_CONFIG} | $(KUBECTL) --context ${KUBE_CONTEXT} apply -f - .PHONY: undeploy undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - + $(KUSTOMIZE) build config/${KUSTOMIZE_CONFIG} | $(KUBECTL) --context ${KUBE_CONTEXT} delete --ignore-not-found=$(ignore-not-found) -f - ##@ Dependencies @@ -152,16 +194,22 @@ $(LOCALBIN): ## Tool Binaries KUBECTL ?= kubectl +KIND ?= kind KUSTOMIZE ?= $(LOCALBIN)/kustomize CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest GOLANGCI_LINT = $(LOCALBIN)/golangci-lint +BUF = $(LOCALBIN)/buf ## Tool Versions -KUSTOMIZE_VERSION ?= v5.4.3 -CONTROLLER_TOOLS_VERSION ?= v0.16.1 -ENVTEST_VERSION ?= release-0.19 -GOLANGCI_LINT_VERSION ?= v1.59.1 +KUSTOMIZE_VERSION ?= v5.6.0 +CONTROLLER_TOOLS_VERSION ?= v0.18.0 +#ENVTEST_VERSION is the version of controller-runtime release branch to fetch the envtest setup script (i.e. release-0.20) +ENVTEST_VERSION ?= $(shell go list -m -f "{{ .Version }}" sigs.k8s.io/controller-runtime | awk -F'[v.]' '{printf "release-%d.%d", $$2, $$3}') +#ENVTEST_K8S_VERSION is the version of Kubernetes to use for setting up ENVTEST binaries (i.e. 1.31) +ENVTEST_K8S_VERSION ?= $(shell go list -m -f "{{ .Version }}" k8s.io/api | awk -F'[v.]' '{printf "1.%d", $$3}') +GOLANGCI_LINT_VERSION ?= v2.3.0 +BUF_VERSION ?= v1.56.0 .PHONY: kustomize kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. @@ -173,6 +221,19 @@ controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessar $(CONTROLLER_GEN): $(LOCALBIN) $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) +.PHONY: buf +buf: $(BUF) ## Download buf locally if necessary. +$(BUF): $(LOCALBIN) + $(call go-install-tool,$(BUF),github.com/bufbuild/buf/cmd/buf,$(BUF_VERSION)) + +.PHONY: setup-envtest +setup-envtest: envtest ## Download the binaries required for ENVTEST in the local bin directory. + @echo "Setting up envtest binaries for Kubernetes version $(ENVTEST_K8S_VERSION)..." + @$(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path || { \ + echo "Error: Failed to set up envtest binaries for version $(ENVTEST_K8S_VERSION)."; \ + exit 1; \ + } + .PHONY: envtest envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. $(ENVTEST): $(LOCALBIN) @@ -181,7 +242,7 @@ $(ENVTEST): $(LOCALBIN) .PHONY: golangci-lint golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. $(GOLANGCI_LINT): $(LOCALBIN) - $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) + $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/v2/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist # $1 - target path with name of binary @@ -198,3 +259,33 @@ mv $(1) $(1)-$(3) ;\ } ;\ ln -sf $(1)-$(3) $(1) endef + +.PHONY: minikube-start +minikube-start: + minikube -p nodedrain status || { \ + minikube start -p nodedrain --embed-certs=true --interactive=false --install-addons=false --nodes=2 ;\ + minikube -p nodedrain addons enable registry ;\ + }; + [ ! "$$($(CONTAINER_TOOL) ps -a -q -f name=nodedrain-registry-proxy)" ] && $(CONTAINER_TOOL) run --rm -d --name nodedrain-registry-proxy --network=host alpine ash -c "apk add socat && socat TCP-LISTEN:5000,reuseaddr,fork TCP:$$(minikube -p nodedrain ip):5000" || true + timeout 300 bash -c 'while [[ "$$(curl -s -o /dev/null -w ''%{http_code}'' localhost:5000)" != "200" ]]; do sleep 1; done' || false + + +.PHONY: minikube-cleanup +minikube-cleanup: + docker rm -f nodedrain-registry-proxy + minikube -p nodedrain delete + +.PHONY: minikube-cert-manager +minikube-cert-manager: + helm --kube-context nodedrain status -n cert-manager cert-manager > /dev/null || { \ + helm --kube-context nodedrain repo add jetstack https://charts.jetstack.io ;\ + helm --kube-context nodedrain upgrade -i cert-manager jetstack/cert-manager --wait --namespace cert-manager --create-namespace --set installCRDs=true ;\ + $(KUBECTL) --context nodedrain wait --namespace cert-manager --for=condition=available --timeout=300s deployment/cert-manager ;\ + $(KUBECTL) --context nodedrain wait --namespace cert-manager --for=condition=available --timeout=300s deployment/cert-manager-cainjector ;\ + $(KUBECTL) --context nodedrain wait --namespace cert-manager --for=condition=available --timeout=300s deployment/cert-manager-webhook ;\ + }; + +.PHONY: minikube-deploy +minikube-deploy: manifests generate minikube-start minikube-cert-manager + $(MAKE) -e IMG_REGISTRY=localhost:5000/ -e KUBE_CONTEXT=nodedrain -e KUSTOMIZE_CONFIG=dev docker-build docker-push deploy + $(KUBECTL) --context nodedrain rollout restart deployment nodedrain-controller-manager -n nodedrain-system diff --git a/PROJECT b/PROJECT index 44a3890..bed41df 100644 --- a/PROJECT +++ b/PROJECT @@ -5,6 +5,9 @@ domain: k8s.slyng.dk layout: - go.kubebuilder.io/v4 +- helm.kubebuilder.io/v1-alpha +plugins: + helm.kubebuilder.io/v1-alpha: {} projectName: nodedrain repo: github.com/slyngdk/node-drain resources: @@ -16,4 +19,8 @@ resources: kind: Node path: github.com/slyngdk/node-drain/api/v1 version: v1 + webhooks: + defaulting: true + validation: true + webhookVersion: v1 version: "3" diff --git a/README.md b/README.md index c465745..55ab3e2 100644 --- a/README.md +++ b/README.md @@ -1,96 +1,93 @@ -# nodedrain -// TODO(user): Add simple overview of use/purpose - -## Description -// TODO(user): An in-depth paragraph about your project and overview of use +# Nodedrain +Nodedrain is a kubernetes operator, build for draining, rebooting and upgrading nodes with support for plugins hooking into this process. + +## Roadmap + +- [X] Drain + - [X] Plugin architecture + - [X] [Basic Interface definition](./api/plugins/) + - [X] [Basic example implementation](./examples/plugin/example-plugin.go) +- [ ] Reboot + - [X] Check reboot required (/var/run/reboot-required) + - [X] Request reboot if required (Single node) + - [ ] Reboot Controller (Automatic reboot when required, withing configured schedule) +- [ ] Upgrade + - [ ] Upgrade node using requested image + - [ ] Upgrade Controller (Rollout configuration handling (CRD)) + - [ ] Ensure control planes upgraded first + - [ ] Detect version mismatch before upgrade ## Getting Started -### Prerequisites -- go version v1.22.0+ -- docker version 17.03+. -- kubectl version v1.11.3+. -- Access to a Kubernetes v1.11.3+ cluster. +### Helm Chart (WIP) -### To Deploy on the cluster -**Build and push your image to the location specified by `IMG`:** +Helm can be found in [chart](./dist/chart) -```sh -make docker-build docker-push IMG=/nodedrain:tag -``` +## Local development -**NOTE:** This image ought to be published in the personal registry you specified. -And it is required to have access to pull the image from the working environment. -Make sure you have the proper permission to the registry if the above commands don’t work. - -**Install the CRDs into the cluster:** - -```sh -make install +### Prerequisites +- make +- go version v1.24+ +- docker with buildx +- kubectl +- minikube + +### Run locally +For build and start local development using minikube run: +```bash +make minikube-deploy ``` +That will: +- Build the project +- Start minikube (nodedrain profile) with cert-manager and image registry +- Start registry proxy container on port 5000 +- Deploy manifests for a fully working environment -**Deploy the Manager to the cluster with the image specified by `IMG`:** +Make changes to code, and run `make minikube-deploy` again. -```sh -make deploy IMG=/nodedrain:tag +#### Stop/Cleanup +```bash +make minikube-cleanup ``` -> **NOTE**: If you encounter RBAC errors, you may need to grant yourself cluster-admin -privileges or be logged in as admin. - -**Create instances of your solution** -You can apply the samples (examples) from the config/sample: - -```sh -kubectl apply -k config/samples/ +### Linting +```bash +make lint ``` ->**NOTE**: Ensure that the samples has default values to test it out. - -### To Uninstall -**Delete the instances (CRs) from the cluster:** - -```sh -kubectl delete -k config/samples/ +### Test +```bash +make test ``` -**Delete the APIs(CRDs) from the cluster:** - -```sh -make uninstall +### Test E2E +```bash +make test-e2e ``` -**UnDeploy the controller from the cluster:** - -```sh -make undeploy +### Build and push image +```bash +make docker-build docker-push IMG_REGISTRY=localhost:5000 IMG_NAME_CONTROLLER=controller IMG_TAG=latest ``` -## Project Distribution - -Following are the steps to build the installer and distribute this project to users. - -1. Build the installer for the image built and published in the registry: +### Sample config ```sh -make build-installer IMG=/nodedrain:tag +kubectl apply -k config/samples/ ``` -NOTE: The makefile target mentioned above generates an 'install.yaml' -file in the dist directory. This file contains all the resources built -with Kustomize, which are necessary to install this project without -its dependencies. +## Contributing -2. Using the installer +Contributions are what make the open source community such an amazing place to learn, inspire, and create. Any contributions you make are **greatly appreciated**. -Users can just run kubectl apply -f to install the project, i.e.: +If you have a suggestion that would make this better, please fork the repo and create a pull request. You can also simply open an issue with the tag "enhancement". +Don't forget to give the project a star! Thanks again! -```sh -kubectl apply -f https://raw.githubusercontent.com//nodedrain//dist/install.yaml -``` - -## Contributing -// TODO(user): Add detailed information on how you would like others to contribute to this project +1. Fork the Project +2. Create your Branch (`git checkout -b amazing-feature`) +3. Commit your Changes (`git commit -m 'Add some AmazingFeature'`) +4. Push to the Branch (`git push origin amazing-feature`) +5. Open a Pull Request **NOTE:** Run `make help` for more information on all potential `make` targets @@ -98,7 +95,7 @@ More information can be found via the [Kubebuilder Documentation](https://book.k ## License -Copyright 2024. +Copyright 2025. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/api/plugins/drain-client.go b/api/plugins/drain-client.go new file mode 100644 index 0000000..4f0faf7 --- /dev/null +++ b/api/plugins/drain-client.go @@ -0,0 +1,57 @@ +package plugins + +import ( + "context" + + "github.com/hashicorp/go-hclog" + proto "github.com/slyngdk/node-drain/api/plugins/proto/v1" +) + +var _ DrainPlugin = &DrainClient{} + +type DrainClient struct{ client proto.DrainServiceClient } + +func (c DrainClient) Init(ctx context.Context, logger hclog.Logger, settings DrainPluginSettings) (DrainPluginInfo, error) { + resp, err := c.client.Init(ctx, &proto.InitRequest{}) + if err != nil { + return DrainPluginInfo{}, err + } + + return DrainPluginInfo{ + ID: resp.Id, + }, nil +} + +func (c DrainClient) IsSupported(ctx context.Context) (bool, error) { + resp, err := c.client.IsSupported(ctx, &proto.IsSupportedRequest{}) + if err != nil { + return false, err + } + return resp.Supported, nil +} + +func (c DrainClient) IsHealthy(ctx context.Context) (bool, error) { + resp, err := c.client.IsHealthy(ctx, &proto.IsHealthyRequest{}) + if err != nil { + return false, err + } + return resp.Healthy, nil +} + +func (c DrainClient) IsDrainOk(ctx context.Context, nodeName string) (bool, error) { + resp, err := c.client.IsDrainOk(ctx, &proto.IsDrainOkRequest{NodeName: nodeName}) + if err != nil { + return false, err + } + return resp.Ok, nil +} + +func (c DrainClient) PreDrain(ctx context.Context, nodeName string) error { + _, err := c.client.PreDrain(ctx, &proto.PreDrainRequest{NodeName: nodeName}) + return err +} + +func (c DrainClient) PostDrain(ctx context.Context, nodeName string) error { + _, err := c.client.PostDrain(ctx, &proto.PostDrainRequest{NodeName: nodeName}) + return err +} diff --git a/api/plugins/drain-server.go b/api/plugins/drain-server.go new file mode 100644 index 0000000..dab2850 --- /dev/null +++ b/api/plugins/drain-server.go @@ -0,0 +1,75 @@ +package plugins + +import ( + "context" + "os" + + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-plugin" + proto "github.com/slyngdk/node-drain/api/plugins/proto/v1" +) + +var _ proto.DrainServiceServer = DrainServer{} + +type DrainServer struct { + Logger hclog.Logger + Impl DrainPlugin +} + +func (s DrainServer) Init(ctx context.Context, request *proto.InitRequest) (*proto.InitResponse, error) { + info, err := s.Impl.Init(ctx, s.Logger, DrainPluginSettings{}) + if err != nil { + return nil, err + } + + return &proto.InitResponse{ + Id: info.ID, + }, nil +} + +func (s DrainServer) IsSupported(ctx context.Context, request *proto.IsSupportedRequest) (*proto.IsSupportedResponse, error) { + supported, err := s.Impl.IsSupported(ctx) + return &proto.IsSupportedResponse{Supported: supported}, err +} + +func (s DrainServer) IsHealthy(ctx context.Context, request *proto.IsHealthyRequest) (*proto.IsHealthyResponse, error) { + healthy, err := s.Impl.IsHealthy(ctx) + return &proto.IsHealthyResponse{Healthy: healthy}, err +} + +func (s DrainServer) IsDrainOk(ctx context.Context, request *proto.IsDrainOkRequest) (*proto.IsDrainOkResponse, error) { + ok, err := s.Impl.IsDrainOk(ctx, request.NodeName) + return &proto.IsDrainOkResponse{Ok: ok}, err +} + +func (s DrainServer) PreDrain(ctx context.Context, request *proto.PreDrainRequest) (*proto.PreDrainResponse, error) { + err := s.Impl.PreDrain(ctx, request.NodeName) + return &proto.PreDrainResponse{}, err +} + +func (s DrainServer) PostDrain(ctx context.Context, request *proto.PostDrainRequest) (*proto.PostDrainResponse, error) { + err := s.Impl.PostDrain(ctx, request.NodeName) + return &proto.PostDrainResponse{}, err +} + +func Serve(impl DrainPlugin) { + logger := hclog.New(&hclog.LoggerOptions{ + Level: hclog.Trace, + Output: os.Stderr, + JSONFormat: true, + }) + + plugin.Serve(&plugin.ServeConfig{ + HandshakeConfig: Handshake, + Logger: logger, + VersionedPlugins: map[int]plugin.PluginSet{ + 0: { + "drain": &GRPCDrainPlugin{ + Logger: logger, + Impl: impl, + }, + }, + }, + GRPCServer: plugin.DefaultGRPCServer, + }) +} diff --git a/api/plugins/drain.go b/api/plugins/drain.go new file mode 100644 index 0000000..b5b18de --- /dev/null +++ b/api/plugins/drain.go @@ -0,0 +1,50 @@ +package plugins + +import ( + "context" + + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-plugin" + "google.golang.org/grpc" + + proto "github.com/slyngdk/node-drain/api/plugins/proto/v1" +) + +var Handshake = plugin.HandshakeConfig{ + MagicCookieKey: "NODEDRAIN_PLUGIN", + MagicCookieValue: "4ae46e30-c5de-4ab9-a2c5-4618fdcab7ae", +} + +type DrainPluginSettings struct { +} +type DrainPluginInfo struct { + ID string +} +type DrainPlugin interface { + Init(ctx context.Context, logger hclog.Logger, settings DrainPluginSettings) (DrainPluginInfo, error) + IsSupported(ctx context.Context) (bool, error) + IsHealthy(ctx context.Context) (bool, error) + IsDrainOk(ctx context.Context, nodeName string) (bool, error) + PreDrain(ctx context.Context, nodeName string) error + PostDrain(ctx context.Context, nodeName string) error +} + +var _ plugin.GRPCPlugin = GRPCDrainPlugin{} + +type GRPCDrainPlugin struct { + plugin.NetRPCUnsupportedPlugin + Logger hclog.Logger + Impl DrainPlugin +} + +func (p GRPCDrainPlugin) GRPCServer(broker *plugin.GRPCBroker, s *grpc.Server) error { + proto.RegisterDrainServiceServer(s, &DrainServer{ + Logger: p.Logger, + Impl: p.Impl, + }) + return nil +} + +func (p GRPCDrainPlugin) GRPCClient(ctx context.Context, broker *plugin.GRPCBroker, c *grpc.ClientConn) (interface{}, error) { + return &DrainClient{client: proto.NewDrainServiceClient(c)}, nil +} diff --git a/api/plugins/proto/v1/drain_plugin.pb.go b/api/plugins/proto/v1/drain_plugin.pb.go new file mode 100644 index 0000000..bfdb4af --- /dev/null +++ b/api/plugins/proto/v1/drain_plugin.pb.go @@ -0,0 +1,614 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.8 +// protoc (unknown) +// source: api/plugins/proto/v1/drain_plugin.proto + +package v1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type InitRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InitRequest) Reset() { + *x = InitRequest{} + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InitRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InitRequest) ProtoMessage() {} + +func (x *InitRequest) ProtoReflect() protoreflect.Message { + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InitRequest.ProtoReflect.Descriptor instead. +func (*InitRequest) Descriptor() ([]byte, []int) { + return file_api_plugins_proto_v1_drain_plugin_proto_rawDescGZIP(), []int{0} +} + +type InitResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InitResponse) Reset() { + *x = InitResponse{} + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InitResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InitResponse) ProtoMessage() {} + +func (x *InitResponse) ProtoReflect() protoreflect.Message { + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InitResponse.ProtoReflect.Descriptor instead. +func (*InitResponse) Descriptor() ([]byte, []int) { + return file_api_plugins_proto_v1_drain_plugin_proto_rawDescGZIP(), []int{1} +} + +func (x *InitResponse) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type IsSupportedResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Supported bool `protobuf:"varint,1,opt,name=supported,proto3" json:"supported,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *IsSupportedResponse) Reset() { + *x = IsSupportedResponse{} + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *IsSupportedResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IsSupportedResponse) ProtoMessage() {} + +func (x *IsSupportedResponse) ProtoReflect() protoreflect.Message { + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IsSupportedResponse.ProtoReflect.Descriptor instead. +func (*IsSupportedResponse) Descriptor() ([]byte, []int) { + return file_api_plugins_proto_v1_drain_plugin_proto_rawDescGZIP(), []int{2} +} + +func (x *IsSupportedResponse) GetSupported() bool { + if x != nil { + return x.Supported + } + return false +} + +type IsHealthyRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *IsHealthyRequest) Reset() { + *x = IsHealthyRequest{} + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *IsHealthyRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IsHealthyRequest) ProtoMessage() {} + +func (x *IsHealthyRequest) ProtoReflect() protoreflect.Message { + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IsHealthyRequest.ProtoReflect.Descriptor instead. +func (*IsHealthyRequest) Descriptor() ([]byte, []int) { + return file_api_plugins_proto_v1_drain_plugin_proto_rawDescGZIP(), []int{3} +} + +type IsHealthyResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Healthy bool `protobuf:"varint,1,opt,name=healthy,proto3" json:"healthy,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *IsHealthyResponse) Reset() { + *x = IsHealthyResponse{} + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *IsHealthyResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IsHealthyResponse) ProtoMessage() {} + +func (x *IsHealthyResponse) ProtoReflect() protoreflect.Message { + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IsHealthyResponse.ProtoReflect.Descriptor instead. +func (*IsHealthyResponse) Descriptor() ([]byte, []int) { + return file_api_plugins_proto_v1_drain_plugin_proto_rawDescGZIP(), []int{4} +} + +func (x *IsHealthyResponse) GetHealthy() bool { + if x != nil { + return x.Healthy + } + return false +} + +type IsSupportedRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *IsSupportedRequest) Reset() { + *x = IsSupportedRequest{} + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *IsSupportedRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IsSupportedRequest) ProtoMessage() {} + +func (x *IsSupportedRequest) ProtoReflect() protoreflect.Message { + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IsSupportedRequest.ProtoReflect.Descriptor instead. +func (*IsSupportedRequest) Descriptor() ([]byte, []int) { + return file_api_plugins_proto_v1_drain_plugin_proto_rawDescGZIP(), []int{5} +} + +type IsDrainOkResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Ok bool `protobuf:"varint,1,opt,name=ok,proto3" json:"ok,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *IsDrainOkResponse) Reset() { + *x = IsDrainOkResponse{} + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *IsDrainOkResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IsDrainOkResponse) ProtoMessage() {} + +func (x *IsDrainOkResponse) ProtoReflect() protoreflect.Message { + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IsDrainOkResponse.ProtoReflect.Descriptor instead. +func (*IsDrainOkResponse) Descriptor() ([]byte, []int) { + return file_api_plugins_proto_v1_drain_plugin_proto_rawDescGZIP(), []int{6} +} + +func (x *IsDrainOkResponse) GetOk() bool { + if x != nil { + return x.Ok + } + return false +} + +type IsDrainOkRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + NodeName string `protobuf:"bytes,1,opt,name=node_name,json=nodeName,proto3" json:"node_name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *IsDrainOkRequest) Reset() { + *x = IsDrainOkRequest{} + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *IsDrainOkRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IsDrainOkRequest) ProtoMessage() {} + +func (x *IsDrainOkRequest) ProtoReflect() protoreflect.Message { + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IsDrainOkRequest.ProtoReflect.Descriptor instead. +func (*IsDrainOkRequest) Descriptor() ([]byte, []int) { + return file_api_plugins_proto_v1_drain_plugin_proto_rawDescGZIP(), []int{7} +} + +func (x *IsDrainOkRequest) GetNodeName() string { + if x != nil { + return x.NodeName + } + return "" +} + +type PreDrainRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + NodeName string `protobuf:"bytes,1,opt,name=node_name,json=nodeName,proto3" json:"node_name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PreDrainRequest) Reset() { + *x = PreDrainRequest{} + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PreDrainRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PreDrainRequest) ProtoMessage() {} + +func (x *PreDrainRequest) ProtoReflect() protoreflect.Message { + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PreDrainRequest.ProtoReflect.Descriptor instead. +func (*PreDrainRequest) Descriptor() ([]byte, []int) { + return file_api_plugins_proto_v1_drain_plugin_proto_rawDescGZIP(), []int{8} +} + +func (x *PreDrainRequest) GetNodeName() string { + if x != nil { + return x.NodeName + } + return "" +} + +type PreDrainResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PreDrainResponse) Reset() { + *x = PreDrainResponse{} + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PreDrainResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PreDrainResponse) ProtoMessage() {} + +func (x *PreDrainResponse) ProtoReflect() protoreflect.Message { + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PreDrainResponse.ProtoReflect.Descriptor instead. +func (*PreDrainResponse) Descriptor() ([]byte, []int) { + return file_api_plugins_proto_v1_drain_plugin_proto_rawDescGZIP(), []int{9} +} + +type PostDrainRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + NodeName string `protobuf:"bytes,1,opt,name=node_name,json=nodeName,proto3" json:"node_name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PostDrainRequest) Reset() { + *x = PostDrainRequest{} + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PostDrainRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PostDrainRequest) ProtoMessage() {} + +func (x *PostDrainRequest) ProtoReflect() protoreflect.Message { + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PostDrainRequest.ProtoReflect.Descriptor instead. +func (*PostDrainRequest) Descriptor() ([]byte, []int) { + return file_api_plugins_proto_v1_drain_plugin_proto_rawDescGZIP(), []int{10} +} + +func (x *PostDrainRequest) GetNodeName() string { + if x != nil { + return x.NodeName + } + return "" +} + +type PostDrainResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PostDrainResponse) Reset() { + *x = PostDrainResponse{} + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PostDrainResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PostDrainResponse) ProtoMessage() {} + +func (x *PostDrainResponse) ProtoReflect() protoreflect.Message { + mi := &file_api_plugins_proto_v1_drain_plugin_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PostDrainResponse.ProtoReflect.Descriptor instead. +func (*PostDrainResponse) Descriptor() ([]byte, []int) { + return file_api_plugins_proto_v1_drain_plugin_proto_rawDescGZIP(), []int{11} +} + +var File_api_plugins_proto_v1_drain_plugin_proto protoreflect.FileDescriptor + +const file_api_plugins_proto_v1_drain_plugin_proto_rawDesc = "" + + "\n" + + "'api/plugins/proto/v1/drain_plugin.proto\x12\x14api.plugins.proto.v1\"\r\n" + + "\vInitRequest\"\x1e\n" + + "\fInitResponse\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"3\n" + + "\x13IsSupportedResponse\x12\x1c\n" + + "\tsupported\x18\x01 \x01(\bR\tsupported\"\x12\n" + + "\x10IsHealthyRequest\"-\n" + + "\x11IsHealthyResponse\x12\x18\n" + + "\ahealthy\x18\x01 \x01(\bR\ahealthy\"\x14\n" + + "\x12IsSupportedRequest\"#\n" + + "\x11IsDrainOkResponse\x12\x0e\n" + + "\x02ok\x18\x01 \x01(\bR\x02ok\"/\n" + + "\x10IsDrainOkRequest\x12\x1b\n" + + "\tnode_name\x18\x01 \x01(\tR\bnodeName\".\n" + + "\x0fPreDrainRequest\x12\x1b\n" + + "\tnode_name\x18\x01 \x01(\tR\bnodeName\"\x12\n" + + "\x10PreDrainResponse\"/\n" + + "\x10PostDrainRequest\x12\x1b\n" + + "\tnode_name\x18\x01 \x01(\tR\bnodeName\"\x13\n" + + "\x11PostDrainResponse2\xb6\x04\n" + + "\fDrainService\x12M\n" + + "\x04Init\x12!.api.plugins.proto.v1.InitRequest\x1a\".api.plugins.proto.v1.InitResponse\x12b\n" + + "\vIsSupported\x12(.api.plugins.proto.v1.IsSupportedRequest\x1a).api.plugins.proto.v1.IsSupportedResponse\x12\\\n" + + "\tIsHealthy\x12&.api.plugins.proto.v1.IsHealthyRequest\x1a'.api.plugins.proto.v1.IsHealthyResponse\x12\\\n" + + "\tIsDrainOk\x12&.api.plugins.proto.v1.IsDrainOkRequest\x1a'.api.plugins.proto.v1.IsDrainOkResponse\x12Y\n" + + "\bPreDrain\x12%.api.plugins.proto.v1.PreDrainRequest\x1a&.api.plugins.proto.v1.PreDrainResponse\x12\\\n" + + "\tPostDrain\x12&.api.plugins.proto.v1.PostDrainRequest\x1a'.api.plugins.proto.v1.PostDrainResponseB\xb5\x01\n" + + "\x18com.api.plugins.proto.v1B\x10DrainPluginProtoP\x01Z\x14api/plugins/proto/v1\xa2\x02\x03APP\xaa\x02\x14Api.Plugins.Proto.V1\xca\x02\x14Api\\Plugins\\Proto\\V1\xe2\x02 Api\\Plugins\\Proto\\V1\\GPBMetadata\xea\x02\x17Api::Plugins::Proto::V1b\x06proto3" + +var ( + file_api_plugins_proto_v1_drain_plugin_proto_rawDescOnce sync.Once + file_api_plugins_proto_v1_drain_plugin_proto_rawDescData []byte +) + +func file_api_plugins_proto_v1_drain_plugin_proto_rawDescGZIP() []byte { + file_api_plugins_proto_v1_drain_plugin_proto_rawDescOnce.Do(func() { + file_api_plugins_proto_v1_drain_plugin_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_api_plugins_proto_v1_drain_plugin_proto_rawDesc), len(file_api_plugins_proto_v1_drain_plugin_proto_rawDesc))) + }) + return file_api_plugins_proto_v1_drain_plugin_proto_rawDescData +} + +var file_api_plugins_proto_v1_drain_plugin_proto_msgTypes = make([]protoimpl.MessageInfo, 12) +var file_api_plugins_proto_v1_drain_plugin_proto_goTypes = []any{ + (*InitRequest)(nil), // 0: api.plugins.proto.v1.InitRequest + (*InitResponse)(nil), // 1: api.plugins.proto.v1.InitResponse + (*IsSupportedResponse)(nil), // 2: api.plugins.proto.v1.IsSupportedResponse + (*IsHealthyRequest)(nil), // 3: api.plugins.proto.v1.IsHealthyRequest + (*IsHealthyResponse)(nil), // 4: api.plugins.proto.v1.IsHealthyResponse + (*IsSupportedRequest)(nil), // 5: api.plugins.proto.v1.IsSupportedRequest + (*IsDrainOkResponse)(nil), // 6: api.plugins.proto.v1.IsDrainOkResponse + (*IsDrainOkRequest)(nil), // 7: api.plugins.proto.v1.IsDrainOkRequest + (*PreDrainRequest)(nil), // 8: api.plugins.proto.v1.PreDrainRequest + (*PreDrainResponse)(nil), // 9: api.plugins.proto.v1.PreDrainResponse + (*PostDrainRequest)(nil), // 10: api.plugins.proto.v1.PostDrainRequest + (*PostDrainResponse)(nil), // 11: api.plugins.proto.v1.PostDrainResponse +} +var file_api_plugins_proto_v1_drain_plugin_proto_depIdxs = []int32{ + 0, // 0: api.plugins.proto.v1.DrainService.Init:input_type -> api.plugins.proto.v1.InitRequest + 5, // 1: api.plugins.proto.v1.DrainService.IsSupported:input_type -> api.plugins.proto.v1.IsSupportedRequest + 3, // 2: api.plugins.proto.v1.DrainService.IsHealthy:input_type -> api.plugins.proto.v1.IsHealthyRequest + 7, // 3: api.plugins.proto.v1.DrainService.IsDrainOk:input_type -> api.plugins.proto.v1.IsDrainOkRequest + 8, // 4: api.plugins.proto.v1.DrainService.PreDrain:input_type -> api.plugins.proto.v1.PreDrainRequest + 10, // 5: api.plugins.proto.v1.DrainService.PostDrain:input_type -> api.plugins.proto.v1.PostDrainRequest + 1, // 6: api.plugins.proto.v1.DrainService.Init:output_type -> api.plugins.proto.v1.InitResponse + 2, // 7: api.plugins.proto.v1.DrainService.IsSupported:output_type -> api.plugins.proto.v1.IsSupportedResponse + 4, // 8: api.plugins.proto.v1.DrainService.IsHealthy:output_type -> api.plugins.proto.v1.IsHealthyResponse + 6, // 9: api.plugins.proto.v1.DrainService.IsDrainOk:output_type -> api.plugins.proto.v1.IsDrainOkResponse + 9, // 10: api.plugins.proto.v1.DrainService.PreDrain:output_type -> api.plugins.proto.v1.PreDrainResponse + 11, // 11: api.plugins.proto.v1.DrainService.PostDrain:output_type -> api.plugins.proto.v1.PostDrainResponse + 6, // [6:12] is the sub-list for method output_type + 0, // [0:6] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_api_plugins_proto_v1_drain_plugin_proto_init() } +func file_api_plugins_proto_v1_drain_plugin_proto_init() { + if File_api_plugins_proto_v1_drain_plugin_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_api_plugins_proto_v1_drain_plugin_proto_rawDesc), len(file_api_plugins_proto_v1_drain_plugin_proto_rawDesc)), + NumEnums: 0, + NumMessages: 12, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_api_plugins_proto_v1_drain_plugin_proto_goTypes, + DependencyIndexes: file_api_plugins_proto_v1_drain_plugin_proto_depIdxs, + MessageInfos: file_api_plugins_proto_v1_drain_plugin_proto_msgTypes, + }.Build() + File_api_plugins_proto_v1_drain_plugin_proto = out.File + file_api_plugins_proto_v1_drain_plugin_proto_goTypes = nil + file_api_plugins_proto_v1_drain_plugin_proto_depIdxs = nil +} diff --git a/api/plugins/proto/v1/drain_plugin.proto b/api/plugins/proto/v1/drain_plugin.proto new file mode 100644 index 0000000..61458e8 --- /dev/null +++ b/api/plugins/proto/v1/drain_plugin.proto @@ -0,0 +1,48 @@ +syntax = "proto3"; +package api.plugins.proto.v1; + +option go_package = "api/plugins/proto/v1"; + +message InitRequest {} + +message InitResponse { + string id = 1; +} + +message IsSupportedResponse { + bool supported = 1; +} + +message IsHealthyRequest {} +message IsHealthyResponse { + bool healthy = 1; +} + +message IsSupportedRequest {} + +message IsDrainOkResponse { + bool ok = 1; +} + +message IsDrainOkRequest { + string node_name = 1; +} + +message PreDrainRequest { + string node_name = 1; +} +message PreDrainResponse {} + +message PostDrainRequest { + string node_name = 1; +} +message PostDrainResponse {} + +service DrainService { + rpc Init(InitRequest) returns (InitResponse); + rpc IsSupported(IsSupportedRequest) returns (IsSupportedResponse); + rpc IsHealthy(IsHealthyRequest) returns (IsHealthyResponse); + rpc IsDrainOk(IsDrainOkRequest) returns (IsDrainOkResponse); + rpc PreDrain(PreDrainRequest) returns (PreDrainResponse); + rpc PostDrain(PostDrainRequest) returns (PostDrainResponse); +} diff --git a/api/plugins/proto/v1/drain_plugin_grpc.pb.go b/api/plugins/proto/v1/drain_plugin_grpc.pb.go new file mode 100644 index 0000000..e1a98bc --- /dev/null +++ b/api/plugins/proto/v1/drain_plugin_grpc.pb.go @@ -0,0 +1,309 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc (unknown) +// source: api/plugins/proto/v1/drain_plugin.proto + +package v1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + DrainService_Init_FullMethodName = "/api.plugins.proto.v1.DrainService/Init" + DrainService_IsSupported_FullMethodName = "/api.plugins.proto.v1.DrainService/IsSupported" + DrainService_IsHealthy_FullMethodName = "/api.plugins.proto.v1.DrainService/IsHealthy" + DrainService_IsDrainOk_FullMethodName = "/api.plugins.proto.v1.DrainService/IsDrainOk" + DrainService_PreDrain_FullMethodName = "/api.plugins.proto.v1.DrainService/PreDrain" + DrainService_PostDrain_FullMethodName = "/api.plugins.proto.v1.DrainService/PostDrain" +) + +// DrainServiceClient is the client API for DrainService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type DrainServiceClient interface { + Init(ctx context.Context, in *InitRequest, opts ...grpc.CallOption) (*InitResponse, error) + IsSupported(ctx context.Context, in *IsSupportedRequest, opts ...grpc.CallOption) (*IsSupportedResponse, error) + IsHealthy(ctx context.Context, in *IsHealthyRequest, opts ...grpc.CallOption) (*IsHealthyResponse, error) + IsDrainOk(ctx context.Context, in *IsDrainOkRequest, opts ...grpc.CallOption) (*IsDrainOkResponse, error) + PreDrain(ctx context.Context, in *PreDrainRequest, opts ...grpc.CallOption) (*PreDrainResponse, error) + PostDrain(ctx context.Context, in *PostDrainRequest, opts ...grpc.CallOption) (*PostDrainResponse, error) +} + +type drainServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewDrainServiceClient(cc grpc.ClientConnInterface) DrainServiceClient { + return &drainServiceClient{cc} +} + +func (c *drainServiceClient) Init(ctx context.Context, in *InitRequest, opts ...grpc.CallOption) (*InitResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(InitResponse) + err := c.cc.Invoke(ctx, DrainService_Init_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *drainServiceClient) IsSupported(ctx context.Context, in *IsSupportedRequest, opts ...grpc.CallOption) (*IsSupportedResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(IsSupportedResponse) + err := c.cc.Invoke(ctx, DrainService_IsSupported_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *drainServiceClient) IsHealthy(ctx context.Context, in *IsHealthyRequest, opts ...grpc.CallOption) (*IsHealthyResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(IsHealthyResponse) + err := c.cc.Invoke(ctx, DrainService_IsHealthy_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *drainServiceClient) IsDrainOk(ctx context.Context, in *IsDrainOkRequest, opts ...grpc.CallOption) (*IsDrainOkResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(IsDrainOkResponse) + err := c.cc.Invoke(ctx, DrainService_IsDrainOk_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *drainServiceClient) PreDrain(ctx context.Context, in *PreDrainRequest, opts ...grpc.CallOption) (*PreDrainResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(PreDrainResponse) + err := c.cc.Invoke(ctx, DrainService_PreDrain_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *drainServiceClient) PostDrain(ctx context.Context, in *PostDrainRequest, opts ...grpc.CallOption) (*PostDrainResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(PostDrainResponse) + err := c.cc.Invoke(ctx, DrainService_PostDrain_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// DrainServiceServer is the server API for DrainService service. +// All implementations should embed UnimplementedDrainServiceServer +// for forward compatibility. +type DrainServiceServer interface { + Init(context.Context, *InitRequest) (*InitResponse, error) + IsSupported(context.Context, *IsSupportedRequest) (*IsSupportedResponse, error) + IsHealthy(context.Context, *IsHealthyRequest) (*IsHealthyResponse, error) + IsDrainOk(context.Context, *IsDrainOkRequest) (*IsDrainOkResponse, error) + PreDrain(context.Context, *PreDrainRequest) (*PreDrainResponse, error) + PostDrain(context.Context, *PostDrainRequest) (*PostDrainResponse, error) +} + +// UnimplementedDrainServiceServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedDrainServiceServer struct{} + +func (UnimplementedDrainServiceServer) Init(context.Context, *InitRequest) (*InitResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Init not implemented") +} +func (UnimplementedDrainServiceServer) IsSupported(context.Context, *IsSupportedRequest) (*IsSupportedResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method IsSupported not implemented") +} +func (UnimplementedDrainServiceServer) IsHealthy(context.Context, *IsHealthyRequest) (*IsHealthyResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method IsHealthy not implemented") +} +func (UnimplementedDrainServiceServer) IsDrainOk(context.Context, *IsDrainOkRequest) (*IsDrainOkResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method IsDrainOk not implemented") +} +func (UnimplementedDrainServiceServer) PreDrain(context.Context, *PreDrainRequest) (*PreDrainResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method PreDrain not implemented") +} +func (UnimplementedDrainServiceServer) PostDrain(context.Context, *PostDrainRequest) (*PostDrainResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method PostDrain not implemented") +} +func (UnimplementedDrainServiceServer) testEmbeddedByValue() {} + +// UnsafeDrainServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to DrainServiceServer will +// result in compilation errors. +type UnsafeDrainServiceServer interface { + mustEmbedUnimplementedDrainServiceServer() +} + +func RegisterDrainServiceServer(s grpc.ServiceRegistrar, srv DrainServiceServer) { + // If the following call pancis, it indicates UnimplementedDrainServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&DrainService_ServiceDesc, srv) +} + +func _DrainService_Init_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(InitRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DrainServiceServer).Init(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: DrainService_Init_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DrainServiceServer).Init(ctx, req.(*InitRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _DrainService_IsSupported_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(IsSupportedRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DrainServiceServer).IsSupported(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: DrainService_IsSupported_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DrainServiceServer).IsSupported(ctx, req.(*IsSupportedRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _DrainService_IsHealthy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(IsHealthyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DrainServiceServer).IsHealthy(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: DrainService_IsHealthy_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DrainServiceServer).IsHealthy(ctx, req.(*IsHealthyRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _DrainService_IsDrainOk_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(IsDrainOkRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DrainServiceServer).IsDrainOk(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: DrainService_IsDrainOk_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DrainServiceServer).IsDrainOk(ctx, req.(*IsDrainOkRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _DrainService_PreDrain_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PreDrainRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DrainServiceServer).PreDrain(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: DrainService_PreDrain_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DrainServiceServer).PreDrain(ctx, req.(*PreDrainRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _DrainService_PostDrain_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PostDrainRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DrainServiceServer).PostDrain(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: DrainService_PostDrain_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DrainServiceServer).PostDrain(ctx, req.(*PostDrainRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// DrainService_ServiceDesc is the grpc.ServiceDesc for DrainService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var DrainService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "api.plugins.proto.v1.DrainService", + HandlerType: (*DrainServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Init", + Handler: _DrainService_Init_Handler, + }, + { + MethodName: "IsSupported", + Handler: _DrainService_IsSupported_Handler, + }, + { + MethodName: "IsHealthy", + Handler: _DrainService_IsHealthy_Handler, + }, + { + MethodName: "IsDrainOk", + Handler: _DrainService_IsDrainOk_Handler, + }, + { + MethodName: "PreDrain", + Handler: _DrainService_PreDrain_Handler, + }, + { + MethodName: "PostDrain", + Handler: _DrainService_PostDrain_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "api/plugins/proto/v1/drain_plugin.proto", +} diff --git a/api/v1/node_types.go b/api/v1/node_types.go index 006e1c5..cdbd6e9 100644 --- a/api/v1/node_types.go +++ b/api/v1/node_types.go @@ -23,17 +23,79 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -type NodeDrainStatus string +type NodeState string -const NodeDrainStatusQueued = "Queued" -const NodeDrainStatusNext = "Next" +func (c NodeState) String() string { + return string(c) +} + +const ( + NodeStateActive NodeState = "Active" + NodeStateCordoned NodeState = "Cordoned" + NodeStateDrained NodeState = "Drained" + NodeStateRebootIfRequired NodeState = "RebootIfRequired" + // TODO Upgrade +) + +func (c NodeState) Drain() bool { + switch c { + case NodeStateDrained, NodeStateRebootIfRequired: + return true + default: + return false + } +} // NodeSpec defines the desired state of Node type NodeSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file - // Foo is an example field of Node. Edit node_types.go to remove/update + // +kubebuilder:validation:Required + // +kubebuilder:default=Active + // +kubebuilder:validation:Enum=Active;Cordoned;Drained;RebootIfRequired + State NodeState `json:"state,omitempty"` +} +type NodeCurrentState string + +func (c NodeCurrentState) String() string { + return string(c) +} +func (c NodeCurrentState) WorkState() bool { + switch c { + case NodeCurrentStateOk, NodeCurrentStateCordoned, NodeCurrentStateQueued: + return false + default: + return true + } +} + +const ( + NodeCurrentStateOk NodeCurrentState = "OK" + NodeCurrentStateCordoned NodeCurrentState = "Cordoned" + NodeCurrentStateQueued NodeCurrentState = "Queued" + NodeCurrentStateNext NodeCurrentState = "Next" + NodeCurrentStateDraining NodeCurrentState = "Draining" + NodeCurrentStateDrained NodeCurrentState = "Drained" + NodeCurrentStateUndraining NodeCurrentState = "Undraining" + NodeCurrentStateRebooting NodeCurrentState = "Rebooting" +) + +var ( + nodeCurrentStates = [...]NodeCurrentState{ + NodeCurrentStateOk, + NodeCurrentStateCordoned, + NodeCurrentStateQueued, + NodeCurrentStateNext, + NodeCurrentStateDraining, + NodeCurrentStateDrained, + NodeCurrentStateUndraining, + NodeCurrentStateRebooting, + } +) + +func GetNodeCurrentStates() []NodeCurrentState { + return nodeCurrentStates[:] } // NodeStatus defines the observed state of Node @@ -41,18 +103,44 @@ type NodeStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster // Important: Run "make" to regenerate code after modifying this file - RebootRequiredLastChecked *metav1.Time `json:"rebootRequiredLastChecked,omitempty"` - RebootRequired bool `json:"rebootRequired"` - Status NodeDrainStatus `json:"status,omitempty"` - StatusChanged *metav1.Time `json:"statusChanged,omitempty"` + Conditions []Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,1,rep,name=conditions"` + + // +optional + // +kubebuilder:validation:Type=string + // +kubebuilder:validation:Format=date-time + RebootRequiredLastChecked *metav1.Time `json:"rebootRequiredLastChecked,omitempty"` + // +optional + RebootRequired *bool `json:"rebootRequired"` + + // +kubebuilder:validation:Required + // +kubebuilder:default=false + Drained bool `json:"drained"` + + // +kubebuilder:validation:Required + // +kubebuilder:default=OK + // +kubebuilder:validation:Enum=OK;Cordoned;Queued;Next;Draining;Drained;Undraining;Rebooting + CurrentState NodeCurrentState `json:"currentState,omitempty"` + + BootID string `json:"bootID,omitempty"` +} + +type Condition struct { + metav1.Condition `json:",inline"` + + // lastCheckTime is the last time the condition has been checked. + // +optional + // +kubebuilder:validation:Type=string + // +kubebuilder:validation:Format=date-time + LastCheckTime *metav1.Time `json:"lastCheckTime" protobuf:"bytes,4,opt,name=lastCheckTime"` } // +kubebuilder:object:root=true // +kubebuilder:subresource:status -// +kubebuilder:resource:scope=Cluster -//+kubebuilder:printcolumn:name="Reboot Required",type="boolean",JSONPath=".status.rebootRequired" -//+kubebuilder:printcolumn:name="Reboot Required Last Checked",type="string",JSONPath=".status.rebootRequiredLastChecked" -//+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.status" +// +kubebuilder:resource:scope=Cluster,shortName=nd +// +kubebuilder:printcolumn:name="Requested State",type="string",JSONPath=".spec.state" +// +kubebuilder:printcolumn:name="CurrentState",type="string",JSONPath=".status.currentState" +// +kubebuilder:printcolumn:name="Drained",type="boolean",JSONPath=".status.drained" +// +kubebuilder:printcolumn:name="Reboot Required",type="boolean",JSONPath=".status.rebootRequired" // Node is the Schema for the nodes API type Node struct { diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 59b78d7..654c087 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -1,7 +1,7 @@ //go:build !ignore_autogenerated /* -Copyright 2024. +Copyright 2025. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -24,13 +24,33 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Condition) DeepCopyInto(out *Condition) { + *out = *in + in.Condition.DeepCopyInto(&out.Condition) + if in.LastCheckTime != nil { + in, out := &in.LastCheckTime, &out.LastCheckTime + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Condition. +func (in *Condition) DeepCopy() *Condition { + if in == nil { + return nil + } + out := new(Condition) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Node) DeepCopyInto(out *Node) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) out.Spec = in.Spec - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Node. @@ -101,6 +121,22 @@ func (in *NodeSpec) DeepCopy() *NodeSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeStatus) DeepCopyInto(out *NodeStatus) { *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.RebootRequiredLastChecked != nil { + in, out := &in.RebootRequiredLastChecked, &out.RebootRequiredLastChecked + *out = (*in).DeepCopy() + } + if in.RebootRequired != nil { + in, out := &in.RebootRequired, &out.RebootRequired + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeStatus. diff --git a/buf.gen.yaml b/buf.gen.yaml new file mode 100644 index 0000000..cd42aad --- /dev/null +++ b/buf.gen.yaml @@ -0,0 +1,13 @@ +--- +version: v2 +managed: + enabled: true +plugins: + - remote: buf.build/protocolbuffers/go + out: . + opt: paths=source_relative + - remote: buf.build/grpc/go:v1.5.1 + out: . + opt: + - paths=source_relative + - require_unimplemented_servers=false diff --git a/buf.yaml b/buf.yaml new file mode 100644 index 0000000..9ff3911 --- /dev/null +++ b/buf.yaml @@ -0,0 +1,8 @@ +# For details on buf.yaml configuration, visit https://buf.build/docs/configuration/v2/buf-yaml +version: v2 +lint: + use: + - STANDARD +breaking: + use: + - FILE diff --git a/cmd/main.go b/cmd/main.go index 64720a0..2c08685 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,5 +1,5 @@ /* -Copyright 2024. +Copyright 2025. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,12 +19,21 @@ package main import ( "crypto/tls" "flag" + "fmt" + "log/slog" + "os" + "path/filepath" + + "github.com/go-logr/logr" "github.com/go-logr/zapr" - "github.com/pkg/errors" + config "github.com/slyngdk/node-drain/internal/config" + "github.com/slyngdk/node-drain/internal/utils" "go.uber.org/zap" - "go.uber.org/zap/zapcore" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/rest" "k8s.io/klog/v2" - "os" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) @@ -35,6 +44,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -42,30 +52,35 @@ import ( drainv1 "github.com/slyngdk/node-drain/api/v1" "github.com/slyngdk/node-drain/internal/controller" + webhookv1 "github.com/slyngdk/node-drain/internal/webhook/v1" // +kubebuilder:scaffold:imports ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() ) func init() { + config.LoadDefaultConfig() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(drainv1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } +// nolint:gocyclo func main() { var metricsAddr string + var metricsCertPath, metricsCertName, metricsCertKey string + var webhookCertPath, webhookCertName, webhookCertKey string var enableLeaderElection bool var probeAddr string var secureMetrics bool var enableHTTP2 bool - var logLevel, logFormat string var tlsOpts []func(*tls.Config) var managerNamespace string + var configMapName string flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") @@ -74,27 +89,44 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&webhookCertPath, "webhook-cert-path", "", "The directory that contains the webhook certificate.") + flag.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", "The name of the webhook certificate file.") + flag.StringVar(&webhookCertKey, "webhook-cert-key", "tls.key", "The name of the webhook key file.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", + "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") - flag.StringVar(&logLevel, "log-level", "info", "The log level to output and above") - flag.StringVar(&logFormat, "log-format", "json", "The log format (json, console)") - flag.StringVar(&managerNamespace, "namespace", os.Getenv("POD_NAMESPACE"), "The namespace to use for creating pods, defaults to env 'POD_NAMESPACE' else default") + flag.StringVar(&managerNamespace, "namespace", os.Getenv("POD_NAMESPACE"), + "The namespace to use for creating pods, defaults to env 'POD_NAMESPACE' else default") + flag.StringVar(&configMapName, "config-map-name", "nodedrain-config", "The configMap to load configuration from") flag.Parse() if managerNamespace == "" { managerNamespace = "default" } - l, err := getLogger(logLevel, logFormat) + conf, err := config.LoadConfig() if err != nil { - setupLog.Error(err, "Failed to create logger") + _, _ = fmt.Fprintf(os.Stderr, "Failed to load config: %v\n", err) os.Exit(1) } - zap.ReplaceGlobals(l) - klog.ClearLogger() - klog.SetLogger(zapr.NewLogger(l.Named("kubeclient"))) - log.SetLogger(zapr.NewLogger(l)) - ctrl.SetLogger(zapr.NewLogger(l)) + + l, err := config.GetLogger(conf.Log.Level, conf.Log.Format) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "Failed to create logger: %v\n", err) + os.Exit(1) + } + defer l.Sync() // nolint:errcheck + setGlobalLoggers(l) + setupLog := l.Named("setup") + + nodeName := os.Getenv("POD_NODENAME") + if nodeName == "" { + nodeName = "no-node-name" + setupLog.Sugar().Warnf("POD_NODENAME environment variable not set, using %s as node name", nodeName) + } // if the enable-http2 flag is false (the default), http/2 should be disabled // due to its vulnerabilities. More specifically, disabling http/2 will @@ -111,34 +143,84 @@ func main() { tlsOpts = append(tlsOpts, disableHTTP2) } + // Create watchers for metrics and webhooks certificates + var metricsCertWatcher, webhookCertWatcher *certwatcher.CertWatcher + + // Initial webhook TLS options + webhookTLSOpts := tlsOpts + + if len(webhookCertPath) > 0 { + setupLog.With( + zap.String("webhook-cert-path", webhookCertPath), + zap.String("webhook-cert-name", webhookCertName), + zap.String("webhook-cert-key", webhookCertKey)). + Info("Initializing webhook certificate watcher using provided certificates") + + var err error + webhookCertWatcher, err = certwatcher.New( + filepath.Join(webhookCertPath, webhookCertName), + filepath.Join(webhookCertPath, webhookCertKey), + ) + if err != nil { + setupLog.With(zap.Error(err)).Fatal("Failed to initialize webhook certificate watcher") + } + + webhookTLSOpts = append(webhookTLSOpts, func(config *tls.Config) { + config.GetCertificate = webhookCertWatcher.GetCertificate + }) + } + webhookServer := webhook.NewServer(webhook.Options{ - TLSOpts: tlsOpts, + TLSOpts: webhookTLSOpts, }) // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. // More info: - // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.0/pkg/metrics/server + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.20.2/pkg/metrics/server // - https://book.kubebuilder.io/reference/metrics.html metricsServerOptions := metricsserver.Options{ BindAddress: metricsAddr, SecureServing: secureMetrics, - // TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are - // not provided, self-signed certificates will be generated by default. This option is not recommended for - // production environments as self-signed certificates do not offer the same level of trust and security - // as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing - // unauthorized access to sensitive metrics data. Consider replacing with CertDir, CertName, and KeyName - // to provide certificates, ensuring the server communicates using trusted and secure certificates. - TLSOpts: tlsOpts, + TLSOpts: tlsOpts, } if secureMetrics { // FilterProvider is used to protect the metrics endpoint with authn/authz. // These configurations ensure that only authorized users and service accounts // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: - // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.0/pkg/metrics/filters#WithAuthenticationAndAuthorization + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.20.2/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization } + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. + if len(metricsCertPath) > 0 { + setupLog.With( + zap.String("metrics-cert-path", metricsCertPath), + zap.String("metrics-cert-name", metricsCertName), + zap.String("metrics-cert-key", metricsCertKey)). + Info("Initializing metrics certificate watcher using provided certificates") + + var err error + metricsCertWatcher, err = certwatcher.New( + filepath.Join(metricsCertPath, metricsCertName), + filepath.Join(metricsCertPath, metricsCertKey), + ) + if err != nil { + setupLog.With(zap.Error(err)).Fatal("to initialize metrics certificate watcher") + } + + metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { + config.GetCertificate = metricsCertWatcher.GetCertificate + }) + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, Metrics: metricsServerOptions, @@ -157,87 +239,87 @@ func main() { // if you are doing or is intended to do any operation such as perform cleanups // after the manager stops then its usage might be unsafe. // LeaderElectionReleaseOnCancel: true, + Cache: cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.ConfigMap{}: { + Namespaces: map[string]cache.Config{ + managerNamespace: {}, + }, + }, + }, + }, + NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { + c, err := client.New(config, options) + if err != nil { + return nil, err + } + return client.WithFieldOwner(c, utils.GetFieldOwner(managerNamespace)), nil + }, }) if err != nil { - setupLog.Error(err, "unable to start manager") - os.Exit(1) + setupLog.With(zap.Error(err)).Fatal("unable to create new manager") } - if err = (&controller.NodeReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Node") - os.Exit(1) - } + ctx := ctrl.SetupSignalHandler() - if err = (&controller.KubeNodeReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor("node-drain"), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "KubeNode") - os.Exit(1) + nodeReconciler, err := controller.NewNodeReconciler( + ctx, + mgr.GetClient(), + mgr.GetScheme(), + mgr.GetConfig(), + mgr.GetEventRecorderFor("nodedrain-controller"), + managerNamespace, + nodeName, + ) + if err != nil { + setupLog.With(zap.Error(err), zap.String("controller", "Node")).Fatal("unable to create node reconciler") + } + if err = nodeReconciler.SetupWithManager(mgr); err != nil { + setupLog.With(zap.Error(err), zap.String("controller", "Node")).Fatal("unable to create controller") } + // nolint:goconst + if os.Getenv("ENABLE_WEBHOOKS") != "false" { + if err := webhookv1.SetupNodeWebhookWithManager(mgr); err != nil { + setupLog.With(zap.Error(err), zap.String("webhook", "Node")).Fatal("unable to create webhook") + } + } // +kubebuilder:scaffold:builder - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") - os.Exit(1) - } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") - os.Exit(1) + if metricsCertWatcher != nil { + setupLog.Info("Adding metrics certificate watcher to manager") + if err := mgr.Add(metricsCertWatcher); err != nil { + setupLog.With(zap.Error(err)).Fatal("unable to add metrics certificate watcher to manager") + } } - ctx := ctrl.SetupSignalHandler() + if webhookCertWatcher != nil { + setupLog.Info("Adding webhook certificate watcher to manager") + if err := mgr.Add(webhookCertWatcher); err != nil { + setupLog.With(zap.Error(err)).Fatal("unable to add webhook certificate watcher to manager") + } + } - drainer := &controller.Drainer{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - RestConfig: mgr.GetConfig(), - NameSpace: managerNamespace, + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.With(zap.Error(err)).Fatal("unable to set up health check") + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.With(zap.Error(err)).Fatal("unable to set up ready check") } - go drainer.Start(ctx) setupLog.Info("starting manager") if err := mgr.Start(ctx); err != nil { - setupLog.Error(err, "problem running manager") - os.Exit(1) + setupLog.With(zap.Error(err)).Fatal("problem running manager") } } -func getLogger(logLevel, logFormat string) (*zap.Logger, error) { - level, err := zap.ParseAtomicLevel(logLevel) - if err != nil { - return nil, errors.Wrap(err, "failed to parse log level") - } - - disableStackTrace := true - encoderConfig := zap.NewProductionEncoderConfig() - - if logFormat == "json" { - disableStackTrace = false - } - if logFormat == "console" { - encoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder - } - - encoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder - loggerConfig := zap.Config{ - Level: level, - Encoding: logFormat, - EncoderConfig: encoderConfig, - OutputPaths: []string{"stdout"}, - ErrorOutputPaths: []string{"stderr"}, - DisableStacktrace: disableStackTrace, - } - - logger, err := loggerConfig.Build() - if err != nil { - return nil, errors.Wrap(err, "failed to build logger") - } - return logger, nil +func setGlobalLoggers(l *zap.Logger) { + zap.ReplaceGlobals(l) + klog.ClearLogger() + klog.SetLogger(zapr.NewLogger(l.Named("kubeclient"))) + logger := zapr.NewLogger(l) + log.SetLogger(logger) + ctrl.SetLogger(logger) + slog.SetDefault(slog.New(logr.ToSlogHandler(logger))) } diff --git a/config/certmanager/certificate-metrics.yaml b/config/certmanager/certificate-metrics.yaml new file mode 100644 index 0000000..ed9f2a7 --- /dev/null +++ b/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: nodedrain + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert diff --git a/config/certmanager/certificate-webhook.yaml b/config/certmanager/certificate-webhook.yaml new file mode 100644 index 0000000..1f4baa3 --- /dev/null +++ b/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: nodedrain + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert diff --git a/config/certmanager/issuer.yaml b/config/certmanager/issuer.yaml new file mode 100644 index 0000000..8666284 --- /dev/null +++ b/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: nodedrain + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml new file mode 100644 index 0000000..fcb7498 --- /dev/null +++ b/config/certmanager/kustomization.yaml @@ -0,0 +1,7 @@ +resources: +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml new file mode 100644 index 0000000..cf6f89e --- /dev/null +++ b/config/certmanager/kustomizeconfig.yaml @@ -0,0 +1,8 @@ +# This configuration is for teaching kustomize how to update name ref substitution +nameReference: +- kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name diff --git a/config/crd/bases/drain.k8s.slyng.dk_nodes.yaml b/config/crd/bases/drain.k8s.slyng.dk_nodes.yaml index 416da41..162d78a 100644 --- a/config/crd/bases/drain.k8s.slyng.dk_nodes.yaml +++ b/config/crd/bases/drain.k8s.slyng.dk_nodes.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.1 + controller-gen.kubebuilder.io/version: v0.18.0 name: nodes.drain.k8s.slyng.dk spec: group: drain.k8s.slyng.dk @@ -11,19 +11,24 @@ spec: kind: Node listKind: NodeList plural: nodes + shortNames: + - nd singular: node scope: Cluster versions: - additionalPrinterColumns: + - jsonPath: .spec.state + name: Requested State + type: string + - jsonPath: .status.currentState + name: CurrentState + type: string + - jsonPath: .status.drained + name: Drained + type: boolean - jsonPath: .status.rebootRequired name: Reboot Required type: boolean - - jsonPath: .status.rebootRequiredLastChecked - name: Reboot Required Last Checked - type: string - - jsonPath: .status.status - name: Status - type: string name: v1 schema: openAPIV3Schema: @@ -48,22 +53,105 @@ spec: type: object spec: description: NodeSpec defines the desired state of Node + properties: + state: + default: Active + enum: + - Active + - Cordoned + - Drained + - RebootIfRequired + type: string + required: + - state type: object status: description: NodeStatus defines the observed state of Node properties: + bootID: + type: string + conditions: + items: + properties: + lastCheckTime: + description: lastCheckTime is the last time the condition has + been checked. + format: date-time + type: string + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + currentState: + default: OK + enum: + - OK + - Cordoned + - Queued + - Next + - Draining + - Drained + - Undraining + - Rebooting + type: string + drained: + default: false + type: boolean rebootRequired: type: boolean rebootRequiredLastChecked: format: date-time type: string - status: - type: string - statusChanged: - format: date-time - type: string required: - - rebootRequired + - currentState + - drained type: object type: object served: true diff --git a/config/default/cert_metrics_manager_patch.yaml b/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 0000000..d975015 --- /dev/null +++ b/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,30 @@ +# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs. + +# Add the volumeMount for the metrics-server certs +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true + +# Add the --metrics-cert-path argument for the metrics server +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs + +# Add the metrics-server certs volume configuration +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 8f8bece..0593d1d 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -14,18 +14,18 @@ namePrefix: nodedrain- # pairs: # someName: someValue -resources: -- ../crd -- ../rbac -- ../manager # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml -#- ../webhook # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. -#- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus # [METRICS] Expose the controller manager metrics service. +resources: +- ../crd +- ../rbac +- ../manager +- ../webhook +- ../certmanager - metrics_service.yaml # [NETWORK POLICY] Protect the /metrics endpoint and Webhook Server with NetworkPolicy. # Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics. @@ -33,119 +33,188 @@ resources: # be able to communicate with the Webhook Server. #- ../network-policy -# Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager -patches: +# Uncomment the patches line if you enable Metrics # [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics -- path: manager_metrics_patch.yaml - target: - kind: Deployment + +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml -#- path: manager_webhook_patch.yaml - -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. -# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. -# 'CERTMANAGER' needs to be enabled to use ca injection -#- path: webhookcainjection_patch.yaml +patches: +- path: manager_metrics_patch.yaml + target: + kind: Deployment +- path: manager_webhook_patch.yaml + target: + kind: Deployment # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations -#replacements: -# - source: # Add cert-manager annotation to ValidatingWebhookConfiguration, MutatingWebhookConfiguration and CRDs -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldPath: .metadata.namespace # namespace of the certificate CR -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: CustomResourceDefinition -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - source: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldPath: .metadata.name -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: CustomResourceDefinition -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - source: # Add cert-manager annotation to the webhook Service -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.name # namespace of the service -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 0 -# create: true -# - source: -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.namespace # namespace of the service -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 1 -# create: true +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - select: # Uncomment the following to set the Service name for TLS config in Prometheus ServiceMonitor +# kind: ServiceMonitor +# group: monitoring.coreos.com +# version: v1 +# name: controller-manager-metrics-monitor +# fieldPaths: +# - spec.endpoints.0.tlsConfig.serverName +# options: +# delimiter: '.' +# index: 0 +# create: true +# +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# - select: # Uncomment the following to set the Service namespace for TLS in Prometheus ServiceMonitor +# kind: ServiceMonitor +# group: monitoring.coreos.com +# version: v1 +# name: controller-manager-metrics-monitor +# fieldPaths: +# - spec.endpoints.0.tlsConfig.serverName +# options: +# delimiter: '.' +# index: 1 +# create: true +# +# +# +replacements: +- source: + fieldPath: .metadata.name + kind: Service + name: webhook-service + version: v1 + targets: + - fieldPaths: + - .spec.dnsNames.0 + - .spec.dnsNames.1 + options: + create: true + delimiter: . + select: + group: cert-manager.io + kind: Certificate + name: serving-cert + version: v1 +- source: + fieldPath: .metadata.namespace + kind: Service + name: webhook-service + version: v1 + targets: + - fieldPaths: + - .spec.dnsNames.0 + - .spec.dnsNames.1 + options: + create: true + delimiter: . + index: 1 + select: + group: cert-manager.io + kind: Certificate + name: serving-cert + version: v1 +- source: + fieldPath: .metadata.namespace + group: cert-manager.io + kind: Certificate + name: serving-cert + version: v1 + targets: + - fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + create: true + delimiter: / + select: + kind: ValidatingWebhookConfiguration +- source: + fieldPath: .metadata.name + group: cert-manager.io + kind: Certificate + name: serving-cert + version: v1 + targets: + - fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + create: true + delimiter: / + index: 1 + select: + kind: ValidatingWebhookConfiguration +- source: + fieldPath: .metadata.namespace + group: cert-manager.io + kind: Certificate + name: serving-cert + version: v1 + targets: + - fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + create: true + delimiter: / + select: + kind: MutatingWebhookConfiguration +- source: + fieldPath: .metadata.name + group: cert-manager.io + kind: Certificate + name: serving-cert + version: v1 + targets: + - fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + create: true + delimiter: / + index: 1 + select: + kind: MutatingWebhookConfiguration +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: example.com/nodedrain + newTag: v0.0.1 diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml new file mode 100644 index 0000000..963c8a4 --- /dev/null +++ b/config/default/manager_webhook_patch.yaml @@ -0,0 +1,31 @@ +# This patch ensures the webhook certificates are properly mounted in the manager container. +# It configures the necessary arguments, volumes, volume mounts, and container ports. + +# Add the --webhook-cert-path argument for configuring the webhook certificate path +- op: add + path: /spec/template/spec/containers/0/args/- + value: --webhook-cert-path=/tmp/k8s-webhook-server/serving-certs + +# Add the volumeMount for the webhook certificates +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-webhook-server/serving-certs + name: webhook-certs + readOnly: true + +# Add the port configuration for the webhook server +- op: add + path: /spec/template/spec/containers/0/ports/- + value: + containerPort: 9443 + name: webhook-server + protocol: TCP + +# Add the volume configuration for the webhook certificates +- op: add + path: /spec/template/spec/volumes/- + value: + name: webhook-certs + secret: + secretName: webhook-server-cert diff --git a/config/default/metrics_service.yaml b/config/default/metrics_service.yaml index 7d95f6d..145d20c 100644 --- a/config/default/metrics_service.yaml +++ b/config/default/metrics_service.yaml @@ -15,3 +15,4 @@ spec: targetPort: 8443 selector: control-plane: controller-manager + app.kubernetes.io/name: nodedrain diff --git a/config/dev/kustomization.yaml b/config/dev/kustomization.yaml new file mode 100644 index 0000000..24c1908 --- /dev/null +++ b/config/dev/kustomization.yaml @@ -0,0 +1,34 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../default +images: +- name: controller + newName: localhost:5000/controller + newTag: latest +patches: +- patch: | + - op: replace + path: "/spec/template/spec/containers/0/imagePullPolicy" + value: Always + - op: replace + path: "/spec/template/spec/containers/0/image" + value: localhost:5000/controller + - op: add + path: "/spec/template/spec/initContainers" + value: + - name: example-plugins + image: localhost:5000/example-plugin + imagePullPolicy: Always + volumeMounts: + - name: plugins + readOnly: false + mountPath: /plugins + - op: add + path: "/spec/template/spec/containers/0/env/-" + value: + name: NODEDRAIN_CONTAINERNODE + value: "true" + target: + kind: Deployment + name: controller-manager diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 4a476c3..38c3411 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -20,6 +20,7 @@ spec: selector: matchLabels: control-plane: controller-manager + app.kubernetes.io/name: nodedrain replicas: 1 template: metadata: @@ -27,6 +28,7 @@ spec: kubectl.kubernetes.io/default-container: manager labels: control-plane: controller-manager + app.kubernetes.io/name: nodedrain spec: # TODO(user): Uncomment the following code to configure the nodeAffinity expression # according to the platforms which are supported by your solution. @@ -49,14 +51,12 @@ spec: # values: # - linux securityContext: + # Projects are configured by default to adhere to the "restricted" Pod Security Standards. + # This ensures that deployments meet the highest security requirements for Kubernetes. + # For more details, see: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted runAsNonRoot: true - # TODO(user): For common cases that do not require escalating privileges - # it is recommended to ensure that all your Pods/Containers are restrictive. - # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted - # Please uncomment the following code if your project does NOT have to work on old Kubernetes - # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). - # seccompProfile: - # type: RuntimeDefault + seccompProfile: + type: RuntimeDefault containers: - command: - /manager @@ -66,6 +66,7 @@ spec: image: controller:latest imagePullPolicy: IfNotPresent name: manager + ports: [] securityContext: allowPrivilegeEscalation: false capabilities: @@ -88,6 +89,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: POD_NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName # TODO(user): Configure the resources accordingly based on the project requirements. # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ resources: @@ -97,5 +102,20 @@ spec: requests: cpu: 10m memory: 64Mi + volumeMounts: + - name: config + readOnly: true + mountPath: /config + - name: plugins + readOnly: false + mountPath: /plugins + volumes: + - name: config + secret: + secretName: nodedrain-config + optional: true + - name: plugins + emptyDir: + sizeLimit: 1G serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 diff --git a/config/network-policy/allow-metrics-traffic.yaml b/config/network-policy/allow-metrics-traffic.yaml index a96e069..b4c5760 100644 --- a/config/network-policy/allow-metrics-traffic.yaml +++ b/config/network-policy/allow-metrics-traffic.yaml @@ -1,6 +1,6 @@ # This NetworkPolicy allows ingress traffic # with Pods running on namespaces labeled with 'metrics: enabled'. Only Pods on those -# namespaces are able to gathering data from the metrics endpoint. +# namespaces are able to gather data from the metrics endpoint. apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -13,6 +13,7 @@ spec: podSelector: matchLabels: control-plane: controller-manager + app.kubernetes.io/name: nodedrain policyTypes: - Ingress ingress: diff --git a/config/network-policy/allow-webhook-traffic.yaml b/config/network-policy/allow-webhook-traffic.yaml new file mode 100644 index 0000000..9c451ab --- /dev/null +++ b/config/network-policy/allow-webhook-traffic.yaml @@ -0,0 +1,27 @@ +# This NetworkPolicy allows ingress traffic to your webhook server running +# as part of the controller-manager from specific namespaces and pods. CR(s) which uses webhooks +# will only work when applied in namespaces labeled with 'webhook: enabled' +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + app.kubernetes.io/name: nodedrain + app.kubernetes.io/managed-by: kustomize + name: allow-webhook-traffic + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: nodedrain + policyTypes: + - Ingress + ingress: + # This allows ingress traffic from any namespace with the label webhook: enabled + - from: + - namespaceSelector: + matchLabels: + webhook: enabled # Only from namespaces with this label + ports: + - port: 443 + protocol: TCP diff --git a/config/network-policy/kustomization.yaml b/config/network-policy/kustomization.yaml index ec0fb5e..0872bee 100644 --- a/config/network-policy/kustomization.yaml +++ b/config/network-policy/kustomization.yaml @@ -1,2 +1,3 @@ resources: +- allow-webhook-traffic.yaml - allow-metrics-traffic.yaml diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml index ed13716..fdc5481 100644 --- a/config/prometheus/kustomization.yaml +++ b/config/prometheus/kustomization.yaml @@ -1,2 +1,11 @@ resources: - monitor.yaml + +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml +# to mount the "metrics-server-cert" secret in the Manager Deployment. +#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml index 026f73a..ad7a48a 100644 --- a/config/prometheus/monitor.yaml +++ b/config/prometheus/monitor.yaml @@ -16,15 +16,12 @@ spec: bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables - # certificate verification. This poses a significant security risk by making the system vulnerable to - # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between - # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, - # compromising the integrity and confidentiality of the information. - # Please use the following options for secure configurations: - # caFile: /etc/metrics-certs/ca.crt - # certFile: /etc/metrics-certs/tls.crt - # keyFile: /etc/metrics-certs/tls.key + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-server-cert' secret. insecureSkipVerify: true selector: matchLabels: control-plane: controller-manager + app.kubernetes.io/name: nodedrain diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index f4d7d62..9d4e620 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,6 +4,13 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch - apiGroups: - "" resources: @@ -14,6 +21,32 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - get +- apiGroups: + - "" + resources: + - pods + verbs: + - delete + - get + - list +- apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - get - apiGroups: - drain.k8s.slyng.dk resources: @@ -45,7 +78,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: manager-role - namespace: $(SERVICE_NAMESPACE) + namespace: system rules: - apiGroups: - "" @@ -58,3 +91,9 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - pods/exec + verbs: + - create diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml index 0eef031..67e1beb 100644 --- a/config/rbac/role_binding.yaml +++ b/config/rbac/role_binding.yaml @@ -21,7 +21,7 @@ metadata: app.kubernetes.io/name: nodedrain app.kubernetes.io/managed-by: kustomize name: manager-rolebinding - namespace: $(SERVICE_NAMESPACE) + namespace: system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role diff --git a/config/samples/config.yaml b/config/samples/config.yaml new file mode 100644 index 0000000..3b6491e --- /dev/null +++ b/config/samples/config.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: nodedrain-config + labels: + app.kubernetes.io/name: nodedrain + app.kubernetes.io/managed-by: kustomize +stringData: + config.yaml: | + log: + loggers: + node: + level: debug + node-webhook: + level: debug + drain-plugin-client: + level: debug + reboot: + checkInterval: 5m diff --git a/config/samples/drain_v1_node.yaml b/config/samples/drain_v1_node.yaml deleted file mode 100644 index f7d5b49..0000000 --- a/config/samples/drain_v1_node.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: drain.k8s.slyng.dk/v1 -kind: Node -metadata: - labels: - app.kubernetes.io/name: nodedrain - app.kubernetes.io/managed-by: kustomize - name: node-sample -spec: - # TODO(user): Add fields here diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 1349914..41bcc9c 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -1,4 +1,4 @@ ## Append samples of your project ## resources: -- drain_v1_node.yaml +- config.yaml # +kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 0000000..9cf2613 --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- manifests.yaml +- service.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/webhook/kustomizeconfig.yaml b/config/webhook/kustomizeconfig.yaml new file mode 100644 index 0000000..206316e --- /dev/null +++ b/config/webhook/kustomizeconfig.yaml @@ -0,0 +1,22 @@ +# the following config is for teaching kustomize where to look at when substituting nameReference. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + +namespace: +- kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true +- kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml new file mode 100644 index 0000000..8fc88b5 --- /dev/null +++ b/config/webhook/manifests.yaml @@ -0,0 +1,52 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-drain-k8s-slyng-dk-v1-node + failurePolicy: Fail + name: mnode-v1.kb.io + rules: + - apiGroups: + - drain.k8s.slyng.dk + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - nodes + sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-drain-k8s-slyng-dk-v1-node + failurePolicy: Fail + name: vnode-v1.kb.io + rules: + - apiGroups: + - drain.k8s.slyng.dk + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - nodes + sideEffects: None diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml new file mode 100644 index 0000000..78a2a3d --- /dev/null +++ b/config/webhook/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: nodedrain + app.kubernetes.io/managed-by: kustomize + name: webhook-service + namespace: system +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager + app.kubernetes.io/name: nodedrain diff --git a/dist/chart/.helmignore b/dist/chart/.helmignore new file mode 100644 index 0000000..7d92f7f --- /dev/null +++ b/dist/chart/.helmignore @@ -0,0 +1,25 @@ +# Patterns to ignore when building Helm packages. +# Operating system files +.DS_Store + +# Version control directories +.git/ +.gitignore +.bzr/ +.hg/ +.hgignore +.svn/ + +# Backup and temporary files +*.swp +*.tmp +*.bak +*.orig +*~ + +# IDE and editor-related files +.idea/ +.vscode/ + +# Helm chart artifacts +dist/chart/*.tgz diff --git a/dist/chart/Chart.yaml b/dist/chart/Chart.yaml new file mode 100644 index 0000000..7ae0bbd --- /dev/null +++ b/dist/chart/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: nodedrain +description: A Helm chart to distribute the project nodedrain +type: application +version: 0.1.0 +appVersion: "0.1.0" +icon: "https://example.com/icon.png" diff --git a/dist/chart/templates/_helpers.tpl b/dist/chart/templates/_helpers.tpl new file mode 100644 index 0000000..80382b4 --- /dev/null +++ b/dist/chart/templates/_helpers.tpl @@ -0,0 +1,50 @@ +{{- define "chart.name" -}} +{{- if .Chart }} + {{- if .Chart.Name }} + {{- .Chart.Name | trunc 63 | trimSuffix "-" }} + {{- else if .Values.nameOverride }} + {{ .Values.nameOverride | trunc 63 | trimSuffix "-" }} + {{- else }} + nodedrain + {{- end }} +{{- else }} + nodedrain +{{- end }} +{{- end }} + + +{{- define "chart.labels" -}} +{{- if .Chart.AppVersion -}} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Chart.Version }} +helm.sh/chart: {{ .Chart.Version | quote }} +{{- end }} +app.kubernetes.io/name: {{ include "chart.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + + +{{- define "chart.selectorLabels" -}} +app.kubernetes.io/name: {{ include "chart.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{- define "chart.hasMutatingWebhooks" -}} +{{- $hasMutating := false }} +{{- range . }} + {{- if eq .type "mutating" }} + $hasMutating = true }}{{- end }} +{{- end }} +{{ $hasMutating }}}}{{- end }} + + +{{- define "chart.hasValidatingWebhooks" -}} +{{- $hasValidating := false }} +{{- range . }} + {{- if eq .type "validating" }} + $hasValidating = true }}{{- end }} +{{- end }} +{{ $hasValidating }}}}{{- end }} diff --git a/dist/chart/templates/certmanager/certificate.yaml b/dist/chart/templates/certmanager/certificate.yaml new file mode 100644 index 0000000..b82959e --- /dev/null +++ b/dist/chart/templates/certmanager/certificate.yaml @@ -0,0 +1,60 @@ +{{- if .Values.certmanager.enable }} +# Self-signed Issuer +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + {{- include "chart.labels" . | nindent 4 }} + name: selfsigned-issuer + namespace: {{ .Release.Namespace }} +spec: + selfSigned: {} +{{- if .Values.webhook.enable }} +--- +# Certificate for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + annotations: + {{- if .Values.crd.keep }} + "helm.sh/resource-policy": keep + {{- end }} + name: serving-cert + namespace: {{ .Release.Namespace }} + labels: + {{- include "chart.labels" . | nindent 4 }} +spec: + dnsNames: + - nodedrain.{{ .Release.Namespace }}.svc + - nodedrain.{{ .Release.Namespace }}.svc.cluster.local + - nodedrain-webhook-service.{{ .Release.Namespace }}.svc + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert +{{- end }} +{{- if .Values.metrics.enable }} +--- +# Certificate for the metrics +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + annotations: + {{- if .Values.crd.keep }} + "helm.sh/resource-policy": keep + {{- end }} + labels: + {{- include "chart.labels" . | nindent 4 }} + name: metrics-certs + namespace: {{ .Release.Namespace }} +spec: + dnsNames: + - nodedrain.{{ .Release.Namespace }}.svc + - nodedrain.{{ .Release.Namespace }}.svc.cluster.local + - nodedrain-metrics-service.{{ .Release.Namespace }}.svc + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert +{{- end }} +{{- end }} diff --git a/dist/chart/templates/crd/drain.k8s.slyng.dk_nodes.yaml b/dist/chart/templates/crd/drain.k8s.slyng.dk_nodes.yaml new file mode 100644 index 0000000..e7863e2 --- /dev/null +++ b/dist/chart/templates/crd/drain.k8s.slyng.dk_nodes.yaml @@ -0,0 +1,167 @@ +{{- if .Values.crd.enable }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + labels: + {{- include "chart.labels" . | nindent 4 }} + annotations: + {{- if .Values.crd.keep }} + "helm.sh/resource-policy": keep + {{- end }} + controller-gen.kubebuilder.io/version: v0.18.0 + name: nodes.drain.k8s.slyng.dk +spec: + group: drain.k8s.slyng.dk + names: + kind: Node + listKind: NodeList + plural: nodes + shortNames: + - nd + singular: node + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.state + name: Requested State + type: string + - jsonPath: .status.currentState + name: CurrentState + type: string + - jsonPath: .status.drained + name: Drained + type: boolean + - jsonPath: .status.rebootRequired + name: Reboot Required + type: boolean + name: v1 + schema: + openAPIV3Schema: + description: Node is the Schema for the nodes API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NodeSpec defines the desired state of Node + properties: + state: + default: Active + enum: + - Active + - Cordoned + - Drained + - RebootIfRequired + type: string + required: + - state + type: object + status: + description: NodeStatus defines the observed state of Node + properties: + bootID: + type: string + conditions: + items: + properties: + lastCheckTime: + description: lastCheckTime is the last time the condition has + been checked. + format: date-time + type: string + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + currentState: + default: OK + enum: + - OK + - Cordoned + - Queued + - Next + - Draining + - Drained + - Undraining + - Rebooting + type: string + drained: + default: false + type: boolean + rebootRequired: + type: boolean + rebootRequiredLastChecked: + format: date-time + type: string + required: + - currentState + - drained + type: object + type: object + served: true + storage: true + subresources: + status: {} +{{- end -}} diff --git a/dist/chart/templates/manager/manager.yaml b/dist/chart/templates/manager/manager.yaml new file mode 100644 index 0000000..a04fc01 --- /dev/null +++ b/dist/chart/templates/manager/manager.yaml @@ -0,0 +1,118 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nodedrain-controller-manager + namespace: {{ .Release.Namespace }} + labels: + {{- include "chart.labels" . | nindent 4 }} + control-plane: controller-manager +spec: + replicas: {{ .Values.controllerManager.replicas }} + selector: + matchLabels: + {{- include "chart.selectorLabels" . | nindent 6 }} + control-plane: controller-manager + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + {{- include "chart.labels" . | nindent 8 }} + control-plane: controller-manager + {{- if and .Values.controllerManager.pod .Values.controllerManager.pod.labels }} + {{- range $key, $value := .Values.controllerManager.pod.labels }} + {{ $key }}: {{ $value }} + {{- end }} + {{- end }} + spec: + {{- if gt (len .Values.plugins) 0}} + initContainers: + {{- range .Values.plugins }} + - name: {{ required "Name is required" .name }} + image: {{ required "Image is required" .image }} + imagePullPolicy: {{ .imagePullPolicy | default "IfNotPresent" }} + {{- if .command }} + command: {{ .command }} + {{- end }} + {{- if .args }} + args: {{ .args }} + {{- end }} + volumeMounts: + - name: plugins + readOnly: false + mountPath: /plugins + {{- end }} + {{- end }} + containers: + - name: manager + args: + {{- range .Values.controllerManager.container.args }} + - {{ . }} + {{- end }} + command: + - /manager + image: {{ .Values.controllerManager.container.image.repository }}:{{ .Values.controllerManager.container.image.tag }} + {{- if .Values.controllerManager.container.env }} + env: + {{- range $key, $value := .Values.controllerManager.container.env }} + - name: {{ $key }} + value: {{ $value }} + {{- end }} + {{- end }} + livenessProbe: + {{- toYaml .Values.controllerManager.container.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.controllerManager.container.readinessProbe | nindent 12 }} + {{- if .Values.webhook.enable }} + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + {{- end }} + resources: + {{- toYaml .Values.controllerManager.container.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.controllerManager.container.securityContext | nindent 12 }} + {{- if and .Values.certmanager.enable (or .Values.webhook.enable .Values.metrics.enable) }} + volumeMounts: + - name: config + readOnly: true + mountPath: /config + - name: plugins + readOnly: false + mountPath: /plugins + {{- if and .Values.webhook.enable .Values.certmanager.enable }} + - name: webhook-cert + mountPath: /tmp/k8s-webhook-server/serving-certs + readOnly: true + {{- end }} + {{- if and .Values.metrics.enable .Values.certmanager.enable }} + - name: metrics-certs + mountPath: /tmp/k8s-metrics-server/metrics-certs + readOnly: true + {{- end }} + {{- end }} + securityContext: + {{- toYaml .Values.controllerManager.securityContext | nindent 8 }} + serviceAccountName: {{ .Values.controllerManager.serviceAccountName }} + terminationGracePeriodSeconds: {{ .Values.controllerManager.terminationGracePeriodSeconds }} + {{- if and .Values.certmanager.enable (or .Values.webhook.enable .Values.metrics.enable) }} + volumes: + - name: config + secret: + secretName: nodedrain-config + optional: true + - name: plugins + emptyDir: + sizeLimit: 1G + {{- if and .Values.webhook.enable .Values.certmanager.enable }} + - name: webhook-cert + secret: + secretName: webhook-server-cert + {{- end }} + {{- if and .Values.metrics.enable .Values.certmanager.enable }} + - name: metrics-certs + secret: + secretName: metrics-server-cert + {{- end }} + {{- end }} diff --git a/dist/chart/templates/metrics/metrics-service.yaml b/dist/chart/templates/metrics/metrics-service.yaml new file mode 100644 index 0000000..0bf547d --- /dev/null +++ b/dist/chart/templates/metrics/metrics-service.yaml @@ -0,0 +1,18 @@ +{{- if .Values.metrics.enable }} +apiVersion: v1 +kind: Service +metadata: + name: nodedrain-controller-manager-metrics-service + namespace: {{ .Release.Namespace }} + labels: + {{- include "chart.labels" . | nindent 4 }} + control-plane: controller-manager +spec: + ports: + - port: 8443 + targetPort: 8443 + protocol: TCP + name: https + selector: + control-plane: controller-manager +{{- end }} diff --git a/dist/chart/templates/network-policy/allow-metrics-traffic.yaml b/dist/chart/templates/network-policy/allow-metrics-traffic.yaml new file mode 100644 index 0000000..4c96a19 --- /dev/null +++ b/dist/chart/templates/network-policy/allow-metrics-traffic.yaml @@ -0,0 +1,28 @@ +{{- if .Values.networkPolicy.enable }} +# This NetworkPolicy allows ingress traffic +# with Pods running on namespaces labeled with 'metrics: enabled'. Only Pods on those +# namespaces are able to gather data from the metrics endpoint. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + {{- include "chart.labels" . | nindent 4 }} + name: allow-metrics-traffic + namespace: {{ .Release.Namespace }} +spec: + podSelector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: nodedrain + policyTypes: + - Ingress + ingress: + # This allows ingress traffic from any namespace with the label metrics: enabled + - from: + - namespaceSelector: + matchLabels: + metrics: enabled # Only from namespaces with this label + ports: + - port: 8443 + protocol: TCP +{{- end -}} diff --git a/dist/chart/templates/network-policy/allow-webhook-traffic.yaml b/dist/chart/templates/network-policy/allow-webhook-traffic.yaml new file mode 100644 index 0000000..d715a82 --- /dev/null +++ b/dist/chart/templates/network-policy/allow-webhook-traffic.yaml @@ -0,0 +1,28 @@ +{{- if .Values.networkPolicy.enable }} +# This NetworkPolicy allows ingress traffic to your webhook server running +# as part of the controller-manager from specific namespaces and pods. CR(s) which uses webhooks +# will only work when applied in namespaces labeled with 'webhook: enabled' +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + {{- include "chart.labels" . | nindent 4 }} + name: allow-webhook-traffic + namespace: {{ .Release.Namespace }} +spec: + podSelector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: nodedrain + policyTypes: + - Ingress + ingress: + # This allows ingress traffic from any namespace with the label webhook: enabled + - from: + - namespaceSelector: + matchLabels: + webhook: enabled # Only from namespaces with this label + ports: + - port: 443 + protocol: TCP +{{- end -}} diff --git a/dist/chart/templates/prometheus/monitor.yaml b/dist/chart/templates/prometheus/monitor.yaml new file mode 100644 index 0000000..af70186 --- /dev/null +++ b/dist/chart/templates/prometheus/monitor.yaml @@ -0,0 +1,40 @@ +# To integrate with Prometheus. +{{- if .Values.prometheus.enable }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + {{- include "chart.labels" . | nindent 4 }} + control-plane: controller-manager + name: nodedrain-controller-manager-metrics-monitor + namespace: {{ .Release.Namespace }} +spec: + endpoints: + - path: /metrics + port: https + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + {{- if .Values.certmanager.enable }} + serverName: nodedrain-controller-manager-metrics-service.{{ .Release.Namespace }}.svc + # Apply secure TLS configuration with cert-manager + insecureSkipVerify: false + ca: + secret: + name: metrics-server-cert + key: ca.crt + cert: + secret: + name: metrics-server-cert + key: tls.crt + keySecret: + name: metrics-server-cert + key: tls.key + {{- else }} + # Development/Test mode (insecure configuration) + insecureSkipVerify: true + {{- end }} + selector: + matchLabels: + control-plane: controller-manager +{{- end }} diff --git a/dist/chart/templates/rbac/leader_election_role.yaml b/dist/chart/templates/rbac/leader_election_role.yaml new file mode 100644 index 0000000..c9cfe24 --- /dev/null +++ b/dist/chart/templates/rbac/leader_election_role.yaml @@ -0,0 +1,42 @@ +{{- if .Values.rbac.enable }} +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + {{- include "chart.labels" . | nindent 4 }} + namespace: {{ .Release.Namespace }} + name: nodedrain-leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +{{- end -}} diff --git a/dist/chart/templates/rbac/leader_election_role_binding.yaml b/dist/chart/templates/rbac/leader_election_role_binding.yaml new file mode 100644 index 0000000..7940e2e --- /dev/null +++ b/dist/chart/templates/rbac/leader_election_role_binding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.rbac.enable }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + {{- include "chart.labels" . | nindent 4 }} + namespace: {{ .Release.Namespace }} + name: nodedrain-leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nodedrain-leader-election-role +subjects: +- kind: ServiceAccount + name: {{ .Values.controllerManager.serviceAccountName }} + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/dist/chart/templates/rbac/metrics_auth_role.yaml b/dist/chart/templates/rbac/metrics_auth_role.yaml new file mode 100644 index 0000000..9450fb9 --- /dev/null +++ b/dist/chart/templates/rbac/metrics_auth_role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.rbac.enable .Values.metrics.enable }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "chart.labels" . | nindent 4 }} + name: nodedrain-metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +{{- end -}} diff --git a/dist/chart/templates/rbac/metrics_auth_role_binding.yaml b/dist/chart/templates/rbac/metrics_auth_role_binding.yaml new file mode 100644 index 0000000..0854dc0 --- /dev/null +++ b/dist/chart/templates/rbac/metrics_auth_role_binding.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.rbac.enable .Values.metrics.enable }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "chart.labels" . | nindent 4 }} + name: nodedrain-metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nodedrain-metrics-auth-role +subjects: +- kind: ServiceAccount + name: {{ .Values.controllerManager.serviceAccountName }} + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/dist/chart/templates/rbac/metrics_reader_role.yaml b/dist/chart/templates/rbac/metrics_reader_role.yaml new file mode 100644 index 0000000..aab57d8 --- /dev/null +++ b/dist/chart/templates/rbac/metrics_reader_role.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.rbac.enable .Values.metrics.enable }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "chart.labels" . | nindent 4 }} + name: nodedrain-metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get +{{- end -}} diff --git a/dist/chart/templates/rbac/node_editor_role.yaml b/dist/chart/templates/rbac/node_editor_role.yaml new file mode 100644 index 0000000..282e00c --- /dev/null +++ b/dist/chart/templates/rbac/node_editor_role.yaml @@ -0,0 +1,28 @@ +{{- if .Values.rbac.enable }} +# permissions for end users to edit nodes. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "chart.labels" . | nindent 4 }} + name: node-editor-role +rules: +- apiGroups: + - drain.k8s.slyng.dk + resources: + - nodes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - drain.k8s.slyng.dk + resources: + - nodes/status + verbs: + - get +{{- end -}} diff --git a/dist/chart/templates/rbac/node_viewer_role.yaml b/dist/chart/templates/rbac/node_viewer_role.yaml new file mode 100644 index 0000000..4e31400 --- /dev/null +++ b/dist/chart/templates/rbac/node_viewer_role.yaml @@ -0,0 +1,24 @@ +{{- if .Values.rbac.enable }} +# permissions for end users to view nodes. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "chart.labels" . | nindent 4 }} + name: node-viewer-role +rules: +- apiGroups: + - drain.k8s.slyng.dk + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - drain.k8s.slyng.dk + resources: + - nodes/status + verbs: + - get +{{- end -}} diff --git a/dist/chart/templates/rbac/role.yaml b/dist/chart/templates/rbac/role.yaml new file mode 100644 index 0000000..03d9d5d --- /dev/null +++ b/dist/chart/templates/rbac/role.yaml @@ -0,0 +1,103 @@ +{{- if .Values.rbac.enable }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "chart.labels" . | nindent 4 }} + name: nodedrain-manager-role +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - get +- apiGroups: + - "" + resources: + - pods + verbs: + - delete + - get + - list +- apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - get +- apiGroups: + - drain.k8s.slyng.dk + resources: + - nodes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - drain.k8s.slyng.dk + resources: + - nodes/finalizers + verbs: + - update +- apiGroups: + - drain.k8s.slyng.dk + resources: + - nodes/status + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: nodedrain-manager-role + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - deletecollection + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods/exec + verbs: + - create +{{- end -}} diff --git a/dist/chart/templates/rbac/role_binding.yaml b/dist/chart/templates/rbac/role_binding.yaml new file mode 100644 index 0000000..956c449 --- /dev/null +++ b/dist/chart/templates/rbac/role_binding.yaml @@ -0,0 +1,30 @@ +{{- if .Values.rbac.enable }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "chart.labels" . | nindent 4 }} + name: nodedrain-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nodedrain-manager-role +subjects: +- kind: ServiceAccount + name: {{ .Values.controllerManager.serviceAccountName }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: nodedrain-manager-rolebinding + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: nodedrain-manager-role +subjects: + - kind: ServiceAccount + name: {{ .Values.controllerManager.serviceAccountName }} + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/dist/chart/templates/rbac/service_account.yaml b/dist/chart/templates/rbac/service_account.yaml new file mode 100644 index 0000000..93e0a32 --- /dev/null +++ b/dist/chart/templates/rbac/service_account.yaml @@ -0,0 +1,15 @@ +{{- if .Values.rbac.enable }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "chart.labels" . | nindent 4 }} + {{- if and .Values.controllerManager.serviceAccount .Values.controllerManager.serviceAccount.annotations }} + annotations: + {{- range $key, $value := .Values.controllerManager.serviceAccount.annotations }} + {{ $key }}: {{ $value }} + {{- end }} + {{- end }} + name: {{ .Values.controllerManager.serviceAccountName }} + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/dist/chart/templates/webhook/service.yaml b/dist/chart/templates/webhook/service.yaml new file mode 100644 index 0000000..c801fad --- /dev/null +++ b/dist/chart/templates/webhook/service.yaml @@ -0,0 +1,16 @@ +{{- if .Values.webhook.enable }} +apiVersion: v1 +kind: Service +metadata: + name: nodedrain-webhook-service + namespace: {{ .Release.Namespace }} + labels: + {{- include "chart.labels" . | nindent 4 }} +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager +{{- end }} diff --git a/dist/chart/templates/webhook/webhooks.yaml b/dist/chart/templates/webhook/webhooks.yaml new file mode 100644 index 0000000..7ef7356 --- /dev/null +++ b/dist/chart/templates/webhook/webhooks.yaml @@ -0,0 +1,67 @@ +{{- if .Values.webhook.enable }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: nodedrain-mutating-webhook-configuration + namespace: {{ .Release.Namespace }} + annotations: + {{- if .Values.certmanager.enable }} + cert-manager.io/inject-ca-from: "{{ $.Release.Namespace }}/serving-cert" + {{- end }} + labels: + {{- include "chart.labels" . | nindent 4 }} +webhooks: + - name: mnode-v1.kb.io + clientConfig: + service: + name: nodedrain-webhook-service + namespace: {{ .Release.Namespace }} + path: /mutate-drain-k8s-slyng-dk-v1-node + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: + - v1 + rules: + - operations: + - CREATE + - UPDATE + apiGroups: + - drain.k8s.slyng.dk + apiVersions: + - v1 + resources: + - nodes +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: nodedrain-validating-webhook-configuration + namespace: {{ .Release.Namespace }} + annotations: + {{- if .Values.certmanager.enable }} + cert-manager.io/inject-ca-from: "{{ $.Release.Namespace }}/serving-cert" + {{- end }} + labels: + {{- include "chart.labels" . | nindent 4 }} +webhooks: + - name: vnode-v1.kb.io + clientConfig: + service: + name: nodedrain-webhook-service + namespace: {{ .Release.Namespace }} + path: /validate-drain-k8s-slyng-dk-v1-node + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: + - v1 + rules: + - operations: + - CREATE + - UPDATE + apiGroups: + - drain.k8s.slyng.dk + apiVersions: + - v1 + resources: + - nodes +{{- end }} diff --git a/dist/chart/values.yaml b/dist/chart/values.yaml new file mode 100644 index 0000000..8766e61 --- /dev/null +++ b/dist/chart/values.yaml @@ -0,0 +1,91 @@ +# [MANAGER]: Manager Deployment Configurations +controllerManager: + replicas: 1 + container: + image: + repository: controller + tag: latest + args: + - "--leader-elect" + - "--metrics-bind-address=:8443" + - "--health-probe-bind-address=:8081" + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + livenessProbe: + initialDelaySeconds: 15 + periodSeconds: 20 + httpGet: + path: /healthz + port: 8081 + readinessProbe: + initialDelaySeconds: 5 + periodSeconds: 10 + httpGet: + path: /readyz + port: 8081 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + terminationGracePeriodSeconds: 10 + serviceAccountName: nodedrain-controller-manager + +# Plugin container is started as a initContainer and expecting plugins to be copied to `/plugins/*.so` +#plugins: +# - name: example-plugin +# image: asdf +# command: [ "cp", "/example-plugin.so", "/plugins/" ] +# args: [] +plugins: [] + +# [RBAC]: To enable RBAC (Permissions) configurations +rbac: + enable: true + +# [CRDs]: To enable the CRDs +crd: + # This option determines whether the CRDs are included + # in the installation process. + enable: true + + # Enabling this option adds the "helm.sh/resource-policy": keep + # annotation to the CRD, ensuring it remains installed even when + # the Helm release is uninstalled. + # NOTE: Removing the CRDs will also remove all cert-manager CR(s) + # (Certificates, Issuers, ...) due to garbage collection. + keep: true + +# [METRICS]: Set to true to generate manifests for exporting metrics. +# To disable metrics export set false, and ensure that the +# ControllerManager argument "--metrics-bind-address=:8443" is removed. +metrics: + enable: true + +# [WEBHOOKS]: Webhooks configuration +# The following configuration is automatically generated from the manifests +# generated by controller-gen. To update run 'make manifests' and +# the edit command with the '--force' flag +webhook: + enable: true + +# [PROMETHEUS]: To enable a ServiceMonitor to export metrics to Prometheus set true +prometheus: + enable: false + +# [CERT-MANAGER]: To enable cert-manager injection to webhooks set true +certmanager: + enable: true + +# [NETWORK POLICIES]: To enable NetworkPolicies set true +networkPolicy: + enable: false diff --git a/examples/plugin/example-plugin.go b/examples/plugin/example-plugin.go new file mode 100644 index 0000000..e8cf823 --- /dev/null +++ b/examples/plugin/example-plugin.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + + "github.com/hashicorp/go-hclog" + "github.com/slyngdk/node-drain/api/plugins" +) + +type ExampleDrainPlugin struct { + logger hclog.Logger +} + +func (e *ExampleDrainPlugin) Init( + ctx context.Context, + logger hclog.Logger, + settings plugins.DrainPluginSettings) (plugins.DrainPluginInfo, error) { + e.logger = logger + e.logger.Debug("Init()") + + return plugins.DrainPluginInfo{ + ID: "example-plugin", + }, nil +} + +func (e *ExampleDrainPlugin) IsSupported(ctx context.Context) (bool, error) { + e.logger.Debug("IsSupported()") + return true, nil +} + +func (e *ExampleDrainPlugin) IsHealthy(ctx context.Context) (bool, error) { + e.logger.Debug("IsHealthy()") + return true, nil +} + +func (e *ExampleDrainPlugin) IsDrainOk(ctx context.Context, nodeName string) (bool, error) { + e.logger.Debug("IsDrainOk()") + return true, nil +} + +func (e *ExampleDrainPlugin) PreDrain(ctx context.Context, nodeName string) error { + e.logger.Debug("PreDrain()") + return nil +} + +func (e *ExampleDrainPlugin) PostDrain(ctx context.Context, nodeName string) error { + e.logger.Debug("PostDrain()") + return nil +} + +func main() { + plugins.Serve(&ExampleDrainPlugin{}) +} diff --git a/go.mod b/go.mod index 41ca121..b8e35dc 100644 --- a/go.mod +++ b/go.mod @@ -1,101 +1,137 @@ module github.com/slyngdk/node-drain -go 1.23.2 +go 1.24.5 require ( + github.com/fatih/color v1.15.0 + github.com/go-logr/logr v1.4.3 github.com/go-logr/zapr v1.3.0 - github.com/onsi/ginkgo/v2 v2.19.0 - github.com/onsi/gomega v1.33.1 + github.com/hashicorp/go-hclog v1.6.3 + github.com/hashicorp/go-plugin v1.7.0 + github.com/knadh/koanf/parsers/yaml v1.1.0 + github.com/knadh/koanf/providers/env/v2 v2.0.0 + github.com/knadh/koanf/providers/file v1.2.0 + github.com/knadh/koanf/providers/rawbytes v1.0.0 + github.com/knadh/koanf/v2 v2.2.2 + github.com/onsi/ginkgo/v2 v2.23.4 + github.com/onsi/gomega v1.38.0 github.com/pkg/errors v0.9.1 - go.uber.org/zap v1.26.0 - k8s.io/api v0.31.0 - k8s.io/apimachinery v0.31.0 - k8s.io/client-go v0.31.0 + go.uber.org/zap v1.27.0 + google.golang.org/grpc v1.73.0 + google.golang.org/protobuf v1.36.6 + k8s.io/api v0.33.3 + k8s.io/apimachinery v0.33.3 + k8s.io/client-go v0.33.3 k8s.io/klog/v2 v2.130.1 - sigs.k8s.io/controller-runtime v0.19.0 + k8s.io/kubectl v0.33.3 + sigs.k8s.io/controller-runtime v0.21.0 ) require ( + cel.dev/expr v0.23.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect - github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/chai2010/gettext-go v1.0.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/go-errors/errors v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.4 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/cel-go v0.20.1 // indirect - github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/cel-go v0.23.2 // indirect + github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/gorilla/websocket v1.5.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect - github.com/imdario/mergo v0.3.6 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect + github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect + github.com/hashicorp/yamux v0.1.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/knadh/koanf/maps v0.1.2 // indirect + github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/moby/spdystream v0.4.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/moby/spdystream v0.5.0 // indirect + github.com/moby/term v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect - github.com/prometheus/client_golang v1.19.1 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - github.com/spf13/cobra v1.8.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/stoewer/go-strcase v1.2.0 // indirect + github.com/oklog/run v1.1.0 // indirect + github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.65.0 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/spf13/cobra v1.9.1 // indirect + github.com/spf13/pflag v1.0.7 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect - go.opentelemetry.io/otel v1.28.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect - go.opentelemetry.io/otel/metric v1.28.0 // indirect - go.opentelemetry.io/otel/sdk v1.28.0 // indirect - go.opentelemetry.io/otel/trace v1.28.0 // indirect - go.opentelemetry.io/proto/otlp v1.3.1 // indirect + github.com/xlab/treeprint v1.2.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/sdk v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect + go.opentelemetry.io/proto/otlp v1.7.0 // indirect + go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect - golang.org/x/net v0.26.0 // indirect - golang.org/x/oauth2 v0.21.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/term v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect - golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + go.yaml.in/yaml/v3 v3.0.3 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/term v0.33.0 // indirect + golang.org/x/text v0.27.0 // indirect + golang.org/x/time v0.12.0 // indirect + golang.org/x/tools v0.34.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect - google.golang.org/grpc v1.65.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.31.0 // indirect - k8s.io/apiserver v0.31.0 // indirect - k8s.io/component-base v0.31.0 // indirect - k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect - sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + k8s.io/apiextensions-apiserver v0.33.0 // indirect + k8s.io/apiserver v0.33.0 // indirect + k8s.io/cli-runtime v0.33.3 // indirect + k8s.io/component-base v0.33.3 // indirect + k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/kustomize/api v0.19.0 // indirect + sigs.k8s.io/kustomize/kyaml v0.19.0 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index cc2fdb2..35a89cf 100644 --- a/go.sum +++ b/go.sum @@ -1,19 +1,28 @@ +cel.dev/expr v0.23.0 h1:wUb94w6OYQS4uXraxo9U+wUAs9jT47Xvl4iPgAwM2ss= +cel.dev/expr v0.23.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= +github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= +github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= +github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8= +github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= +github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -22,199 +31,282 @@ github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxER github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= -github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= -github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/cel-go v0.20.1 h1:nDx9r8S3L4pE61eDdt8igGj8rf5kjYR3ILxWIpWNi84= -github.com/google/cel-go v0.20.1/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.23.2 h1:UdEe3CvQh3Nv+E/j9r1Y//WO0K0cSyD7/y0bzyLIMI4= +github.com/google/cel-go v0.23.2/go.mod h1:52Pb6QsDbC5kvgxvZhiL9QX1oZEkcUF/ZqaPx1J5Wwo= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= -github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+uBhcekkmy4IkffJww= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-plugin v1.7.0 h1:YghfQH/0QmPNc/AZMTFE3ac8fipZyZECHdDPshfk+mA= +github.com/hashicorp/go-plugin v1.7.0/go.mod h1:BExt6KEaIYx804z8k4gRzRLEvxKVb+kn0NMcihqOqb8= +github.com/hashicorp/yamux v0.1.2 h1:XtB8kyFOyHXYVFnwT5C3+Bdo8gArse7j2AQ0DA0Uey8= +github.com/hashicorp/yamux v0.1.2/go.mod h1:C+zze2n6e/7wshOZep2A70/aQU6QBRWJO/G6FT1wIns= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jhump/protoreflect v1.17.0 h1:qOEr613fac2lOuTgWN4tPAtLL7fUSbuJL5X5XumQh94= +github.com/jhump/protoreflect v1.17.0/go.mod h1:h9+vUUL38jiBzck8ck+6G/aeMX8Z4QUY/NiJPwPNi+8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo= +github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/parsers/yaml v1.1.0 h1:3ltfm9ljprAHt4jxgeYLlFPmUaunuCgu1yILuTXRdM4= +github.com/knadh/koanf/parsers/yaml v1.1.0/go.mod h1:HHmcHXUrp9cOPcuC+2wrr44GTUB0EC+PyfN3HZD9tFg= +github.com/knadh/koanf/providers/env/v2 v2.0.0 h1:Ad5H3eun722u+FvchiIcEIJZsZ2M6oxCkgZfWN5B5KY= +github.com/knadh/koanf/providers/env/v2 v2.0.0/go.mod h1:1g01PE+Ve1gBfWNNw2wmULRP0tc8RJrjn5p2N/jNCIc= +github.com/knadh/koanf/providers/file v1.2.0 h1:hrUJ6Y9YOA49aNu/RSYzOTFlqzXSCpmYIDXI7OJU6+U= +github.com/knadh/koanf/providers/file v1.2.0/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA= +github.com/knadh/koanf/providers/rawbytes v1.0.0 h1:MrKDh/HksJlKJmaZjgs4r8aVBb/zsJyc/8qaSnzcdNI= +github.com/knadh/koanf/providers/rawbytes v1.0.0/go.mod h1:KxwYJf1uezTKy6PBtfE+m725NGp4GPVA7XoNTJ/PtLo= +github.com/knadh/koanf/v2 v2.2.2 h1:ghbduIkpFui3L587wavneC9e3WIliCgiCgdxYO/wd7A= +github.com/knadh/koanf/v2 v2.2.2/go.mod h1:abWQc0cBXLSF/PSOMCB/SK+T13NXDsPvOksbpi5e/9Q= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8= -github.com/moby/spdystream v0.4.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= +github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= -github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= -github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= -github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= +github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= +github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= +github.com/onsi/ginkgo/v2 v2.23.4/go.mod h1:Bt66ApGPBFzHyR+JO10Zbt0Gsp4uWxu5mIOTusL46e8= +github.com/onsi/gomega v1.38.0 h1:c/WX+w8SLAinvuKKQFh77WEucCnPk4j2OTUr7lt7BeY= +github.com/onsi/gomega v1.38.0/go.mod h1:OcXcwId0b9QsE7Y49u+BTrL4IdKOBOKnD6VQNTJEB6o= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= -github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE= +github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M= +github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= -go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= -go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ= -go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= -go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= -go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= -go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= -go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= -go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= -go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 h1:EtFWSnwW9hGObjkIdmlnWSydO+Qs8OwzfzXLUPg4xOc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0/go.mod h1:QjUEoiGCPkvFZ/MjK6ZZfNOS6mfVEVKYE99dFhuN2LI= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= +go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= +go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo= +go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= +go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v3 v3.0.3 h1:bXOww4E/J3f66rav3pX3m8w6jDE4knZjGOw8b5Y6iNE= +go.yaml.in/yaml/v3 v3.0.3/go.mod h1:tBHosrYAkRZjRAOREWbDnBXUf08JOwYq++0QNwQiWzI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= +golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= +golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= -google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= -google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= -google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 h1:oWVWY3NzT7KJppx2UKhKmzPq4SRe0LdCijVRwvGeikY= +google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822/go.mod h1:h3c4v36UTKzUiuaOKQ6gr3S+0hovBtUrXzTG/i3+XEc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= +google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -222,38 +314,45 @@ gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSP gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo= -k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE= -k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk= -k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= -k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= -k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= -k8s.io/apiserver v0.31.0 h1:p+2dgJjy+bk+B1Csz+mc2wl5gHwvNkC9QJV+w55LVrY= -k8s.io/apiserver v0.31.0/go.mod h1:KI9ox5Yu902iBnnyMmy7ajonhKnkeZYJhTZ/YI+WEMk= -k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= -k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= -k8s.io/component-base v0.31.0 h1:/KIzGM5EvPNQcYgwq5NwoQBaOlVFrghoVGr8lG6vNRs= -k8s.io/component-base v0.31.0/go.mod h1:TYVuzI1QmN4L5ItVdMSXKvH7/DtvIuas5/mm8YT3rTo= +k8s.io/api v0.33.3 h1:SRd5t//hhkI1buzxb288fy2xvjubstenEKL9K51KBI8= +k8s.io/api v0.33.3/go.mod h1:01Y/iLUjNBM3TAvypct7DIj0M0NIZc+PzAHCIo0CYGE= +k8s.io/apiextensions-apiserver v0.33.0 h1:d2qpYL7Mngbsc1taA4IjJPRJ9ilnsXIrndH+r9IimOs= +k8s.io/apiextensions-apiserver v0.33.0/go.mod h1:VeJ8u9dEEN+tbETo+lFkwaaZPg6uFKLGj5vyNEwwSzc= +k8s.io/apimachinery v0.33.3 h1:4ZSrmNa0c/ZpZJhAgRdcsFcZOw1PQU1bALVQ0B3I5LA= +k8s.io/apimachinery v0.33.3/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/apiserver v0.33.0 h1:QqcM6c+qEEjkOODHppFXRiw/cE2zP85704YrQ9YaBbc= +k8s.io/apiserver v0.33.0/go.mod h1:EixYOit0YTxt8zrO2kBU7ixAtxFce9gKGq367nFmqI8= +k8s.io/cli-runtime v0.33.3 h1:Dgy4vPjNIu8LMJBSvs8W0LcdV0PX/8aGG1DA1W8lklA= +k8s.io/cli-runtime v0.33.3/go.mod h1:yklhLklD4vLS8HNGgC9wGiuHWze4g7x6XQZ+8edsKEo= +k8s.io/client-go v0.33.3 h1:M5AfDnKfYmVJif92ngN532gFqakcGi6RvaOF16efrpA= +k8s.io/client-go v0.33.3/go.mod h1:luqKBQggEf3shbxHY4uVENAxrDISLOarxpTKMiUuujg= +k8s.io/component-base v0.33.3 h1:mlAuyJqyPlKZM7FyaoM/LcunZaaY353RXiOd2+B5tGA= +k8s.io/component-base v0.33.3/go.mod h1:ktBVsBzkI3imDuxYXmVxZ2zxJnYTZ4HAsVj9iF09qp4= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsAtVhSeUFseziht227YAWYHLGNM8QPwY= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= -sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q= -sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= +k8s.io/kubectl v0.33.3 h1:r/phHvH1iU7gO/l7tTjQk2K01ER7/OAJi8uFHHyWSac= +k8s.io/kubectl v0.33.3/go.mod h1:euj2bG56L6kUGOE/ckZbCoudPwuj4Kud7BR0GzyNiT0= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= +sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/kustomize/api v0.19.0 h1:F+2HB2mU1MSiR9Hp1NEgoU2q9ItNOaBJl0I4Dlus5SQ= +sigs.k8s.io/kustomize/api v0.19.0/go.mod h1:/BbwnivGVcBh1r+8m3tH1VNxJmHSk1PzP5fkP6lbL1o= +sigs.k8s.io/kustomize/kyaml v0.19.0 h1:RFge5qsO1uHhwJsu3ipV7RNolC7Uozc0jUBC/61XSlA= +sigs.k8s.io/kustomize/kyaml v0.19.0/go.mod h1:FeKD5jEOH+FbZPpqUghBP8mrLjJ3+zD3/rf9NNu1cwY= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt index ff72ff2..4671de8 100644 --- a/hack/boilerplate.go.txt +++ b/hack/boilerplate.go.txt @@ -1,5 +1,5 @@ /* -Copyright 2024. +Copyright 2025. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,4 +12,4 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -*/ \ No newline at end of file +*/ diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..8aeaee9 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,133 @@ +package config + +import ( + _ "embed" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + kyaml "github.com/knadh/koanf/parsers/yaml" + kenv "github.com/knadh/koanf/providers/env/v2" + kfile "github.com/knadh/koanf/providers/file" + "github.com/knadh/koanf/providers/rawbytes" + "github.com/knadh/koanf/v2" +) + +//go:embed default-config.yaml +var defaultConfigYaml []byte + +var k = koanf.New(".") +var _config *Config + +type Logger struct { + Level string `koanf:"level"` +} + +type Config struct { + Log struct { + Level string `koanf:"level"` + Format string `koanf:"format"` + Loggers map[string]Logger `koanf:"loggers"` + } `koanf:"log"` + Reboot struct { + CheckInterval time.Duration `koanf:"checkInterval"` + } + ContainerNode bool `koanf:"containerNode"` +} + +func (c *Config) GetLogger(name string) Logger { + if logger, ok := c.Log.Loggers[name]; ok { + if logger.Level == "" { + logger.Level = c.Log.Level + } + return logger + } + return Logger{ + Level: c.Log.Level, + } +} + +func LoadDefaultConfig() { + err := k.Load(rawbytes.Provider(defaultConfigYaml), kyaml.Parser()) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "failed to load default config: %v\n", err) + os.Exit(1) + } +} + +func LoadConfig() (*Config, error) { + + err := k.Load(kenv.Provider(".", kenv.Opt{ + Prefix: "NODEDRAIN_", + TransformFunc: func(k, v string) (string, any) { + k = strings.ReplaceAll(strings.ToLower(strings.TrimPrefix(k, "NODEDRAIN_")), "_", ".") + if strings.Contains(v, " ") { + return k, strings.Split(v, " ") + } + + return k, v + }, + }), nil) + if err != nil { + return nil, err + } + + loadConfigFiles := func() error { + configDir := "/config" + stat, err := os.Stat(configDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("failed to get stat for /config: %w", err) + } + if !stat.IsDir() { + return fmt.Errorf("%s is not a directory", stat.Name()) + } + + files, err := os.ReadDir(configDir) + if err != nil { + return fmt.Errorf("failed to read config directory: %w", err) + } + + sort.Slice(files, func(i, j int) bool { + return files[i].Name() < files[j].Name() + }) + + for _, f := range files { + ext := filepath.Ext(f.Name()) + switch ext { + case ".yaml", ".yml": + err := k.Load(kfile.Provider(filepath.Join(configDir, f.Name())), kyaml.Parser()) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "failed to load config from file %s: %v\n", f.Name(), err) + } + } + } + return nil + } + + if err = loadConfigFiles(); err != nil { + return nil, err + } + + var conf Config + err = k.Unmarshal("", &conf) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal config: %w", err) + } + + _config = &conf + return &conf, nil +} + +func GetKoanf() *koanf.Koanf { + return k +} + +func GetConfig() *Config { + return _config +} diff --git a/internal/config/default-config.yaml b/internal/config/default-config.yaml new file mode 100644 index 0000000..be730f0 --- /dev/null +++ b/internal/config/default-config.yaml @@ -0,0 +1,6 @@ +--- +log: + level: info + format: json +reboot: + checkInterval: 12h diff --git a/internal/config/logger.go b/internal/config/logger.go new file mode 100644 index 0000000..93b7c86 --- /dev/null +++ b/internal/config/logger.go @@ -0,0 +1,99 @@ +package config + +import ( + "fmt" + "strings" + "sync" + + "github.com/fatih/color" + "github.com/pkg/errors" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +var _loggerConfig *zap.Config +var _loggerSync sync.Mutex + +const TraceLevel = zapcore.DebugLevel - 1 + +func GetLogger(logLevel, logFormat string) (*zap.Logger, error) { + level, err := zap.ParseAtomicLevel(logLevel) + if err != nil { + return nil, errors.Wrap(err, "failed to parse log level") + } + + disableStackTrace := true + encoderConfig := zap.NewProductionEncoderConfig() + encoderConfig.EncodeLevel = func(l zapcore.Level, enc zapcore.PrimitiveArrayEncoder) { + if logFormat == "console" { + if l == TraceLevel { + enc.AppendString(color.CyanString("TRACE")) + return + } + zapcore.CapitalColorLevelEncoder(l, enc) + return + } + if l == TraceLevel { + enc.AppendString("trace") + return + } + zapcore.LowercaseLevelEncoder(l, enc) + } + + if logFormat == "json" { + disableStackTrace = false + } + if logFormat == "console" { + encoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder + } + + encoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + loggerConfig := zap.Config{ + Level: level, + Encoding: logFormat, + EncoderConfig: encoderConfig, + OutputPaths: []string{"stdout"}, + ErrorOutputPaths: []string{"stderr"}, + DisableStacktrace: disableStackTrace, + } + + logger, err := loggerConfig.Build() + + if err != nil { + return nil, errors.Wrap(err, "failed to build logger") + } + setLoggerConfig(&loggerConfig) + return logger, nil +} + +func GetNamedLogger(name string) (*zap.Logger, error) { + _loggerSync.Lock() + defer _loggerSync.Unlock() + logger := GetConfig().GetLogger(name) + + var level zap.AtomicLevel + var err error + + if strings.ToLower(logger.Level) == "trace" { + level = zap.NewAtomicLevelAt(TraceLevel) + } else { + level, err = zap.ParseAtomicLevel(logger.Level) + if err != nil { + zap.S().With(zap.Error(err)).Warn("failed to parse log level for named logger, using info", zap.String("name", name)) + level = zap.NewAtomicLevelAt(zapcore.InfoLevel) + } + } + + _loggerConfig.Level = level + l, err := _loggerConfig.Build() + if err != nil { + return nil, fmt.Errorf("error building named zap logger for %s: %w", name, err) + } + return l.Named(name), nil +} + +func setLoggerConfig(loggerConfig *zap.Config) { + _loggerSync.Lock() + defer _loggerSync.Unlock() + _loggerConfig = loggerConfig +} diff --git a/internal/controller/drainer.go b/internal/controller/drainer.go deleted file mode 100644 index 68f253a..0000000 --- a/internal/controller/drainer.go +++ /dev/null @@ -1,122 +0,0 @@ -package controller - -import ( - "context" - v1 "github.com/slyngdk/node-drain/api/v1" - "github.com/slyngdk/node-drain/internal/utils" - "go.uber.org/zap" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "time" -) - -type Drainer struct { - client.Client - Scheme *runtime.Scheme - RestConfig *rest.Config - NameSpace string -} - -//+kubebuilder:rbac:groups="",namespace=$(SERVICE_NAMESPACE),resources=pods,verbs=list;watch;create;get;delete;deletecollection - -func (d *Drainer) Start(ctx context.Context) { - l := zap.S().Named("drainer") - - rebootManager, err := utils.NewRebootManager(l.Desugar(), d.Client, d.RestConfig, d.NameSpace) - if err != nil { - l.Fatal("Failed to create reboot manager", zap.Error(err)) - } - - drainTicker := time.NewTicker(20 * time.Second) //FIXME - checkTicker := time.NewTicker(20 * time.Second) //FIXME - go func() { - for { - select { - case <-drainTicker.C: - nodes := &v1.NodeList{} - - err := d.List(ctx, nodes) - if err != nil { - l.Error(err, "Failed to get nodes") - continue - } - - if len(nodes.Items) == 0 { - continue - } - - node := getActiveNode(nodes) - if node == nil { - node = getNextNode(nodes) - if node == nil { - continue - } - node.Status.Status = v1.NodeDrainStatusNext - if err := d.Status().Update(ctx, node); err != nil { - l.Error(err, "Failed to update node status") - continue - } - continue - } - - case <-checkTicker.C: - - nodes := &v1.NodeList{} - err := d.List(ctx, nodes) - if err != nil { - l.Error(err, "Failed to get nodes") - continue - } - - for _, n := range nodes.Items { - _ = n - - before := metav1.NewTime(time.Now().Add(-1 * 60 * time.Second)) //FIXME configure check interval - - if n.Status.RebootRequiredLastChecked == nil || - n.Status.RebootRequiredLastChecked.Before(&before) { - l.Info("Check if reboot is required", "node", n.Name) - rebootRequired, err := rebootManager.IsRebootRequired(ctx, n.Name) - if err != nil { - l.Error(err, "Failed to check if reboot is required") - continue - } - now := metav1.Now() - n.Status.RebootRequiredLastChecked = &now - n.Status.RebootRequired = rebootRequired - if err := d.Status().Update(ctx, &n); err != nil { - l.Error(err, "Failed to update node status") - continue - } - } - } - - case <-ctx.Done(): - drainTicker.Stop() - return - } - } - }() -} - -func getActiveNode(nodes *v1.NodeList) *v1.Node { - for _, n := range nodes.Items { - switch n.Status.Status { - case v1.NodeDrainStatusNext: - return &n - } - } - return nil -} - -func getNextNode(nodes *v1.NodeList) *v1.Node { - for _, n := range nodes.Items { - switch n.Status.Status { - case v1.NodeDrainStatusQueued: - return &n - } - } - return nil -} diff --git a/internal/controller/kubenode_contoller.go b/internal/controller/kubenode_contoller.go deleted file mode 100644 index ee3fd98..0000000 --- a/internal/controller/kubenode_contoller.go +++ /dev/null @@ -1,135 +0,0 @@ -package controller - -import ( - "context" - "fmt" - v1 "github.com/slyngdk/node-drain/api/v1" - "go.uber.org/zap" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/tools/record" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "strings" - "time" -) - -const ( - nodeDrainFinalizer = "nodedrain.k8s.slyng.dk" -) - -type KubeNodeReconciler struct { - client.Client - Scheme *runtime.Scheme - Recorder record.EventRecorder -} - -// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch;update;patch - -func (r *KubeNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - l := zap.S().Named("kubenode") - - l.Info("kube node reconcile", "request", req) - - node := &corev1.Node{} - if err := r.Get(ctx, req.NamespacedName, node); err != nil { - err = client.IgnoreNotFound(err) - if err != nil { - l.Error(err, "unable to fetch Node") - } - // we'll ignore not-found errors, since they can't be fixed by an immediate - // requeue (we'll need to wait for a new notification), and we can get them - // on deleted requests. - return ctrl.Result{}, err - } - - nodeCRD := &v1.Node{} - if err := r.Get(ctx, req.NamespacedName, nodeCRD); err != nil { - if apierrors.IsNotFound(err) { - // Create node as it is missing - - nodeCRD = &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: node.Name, - Finalizers: nil, - }, - Spec: v1.NodeSpec{}, - } - - err = controllerutil.SetOwnerReference(node, nodeCRD, r.Scheme) - if err != nil { - return ctrl.Result{}, err - } - - controllerutil.AddFinalizer(nodeCRD, nodeDrainFinalizer) - - err := r.Create(ctx, nodeCRD) - if err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{Requeue: true}, nil - } - err = client.IgnoreNotFound(err) - if err != nil { - l.Error(err, "unable to fetch Node") - } - // we'll ignore not-found errors, since they can't be fixed by an immediate - // requeue (we'll need to wait for a new notification), and we can get them - // on deleted requests. - return ctrl.Result{}, err - } - - labels := node.GetObjectMeta().GetLabels() - for label, value := range labels { - if !strings.HasPrefix(label, "nodedrain.k8s.slyng.dk/") { - continue - } - if label == "nodedrain.k8s.slyng.dk/drain" { - if !node.Spec.Unschedulable { - r.Recorder.Event(node, corev1.EventTypeWarning, "Drain", fmt.Sprintf("Waiting for node '%s' is cordon", node.Name)) - return ctrl.Result{Requeue: true, RequeueAfter: 1 * time.Minute}, nil - } - if value == "start" { - nodeCRD := &v1.Node{} - if err := r.Get(ctx, req.NamespacedName, nodeCRD); err != nil { - err = client.IgnoreNotFound(err) - if err != nil { - l.Error(err, "unable to fetch Drain Node") - } - // we'll ignore not-found errors, since they can't be fixed by an immediate - // requeue (we'll need to wait for a new notification), and we can get them - // on deleted requests. - return ctrl.Result{}, err - } - - if nodeCRD.Status.Status == "" { - nodeCRD.Status.Status = v1.NodeDrainStatusQueued - nodeCRD.Status.StatusChanged = time.Now().Format(time.RFC3339) - if err := r.Status().Update(ctx, nodeCRD); err != nil { - return ctrl.Result{}, err - } - } - - patch := client.MergeFrom(node.DeepCopy()) - delete(node.ObjectMeta.Labels, "nodedrain.k8s.slyng.dk/drain") - if err := r.Patch(ctx, node, patch); err != nil { - return ctrl.Result{}, err - } - r.Recorder.Event(node, corev1.EventTypeNormal, "Drain", fmt.Sprintf("Queued node drain '%s'", node.Name)) - } - } - } - - return ctrl.Result{}, nil -} - -func (r *KubeNodeReconciler) SetupWithManager(mgr ctrl.Manager) error { - c := ctrl.NewControllerManagedBy(mgr). - For(&corev1.Node{}) - c.Named("KubeNode") - return c. - Complete(r) -} diff --git a/internal/controller/node_controller.go b/internal/controller/node_controller.go index f8556c0..0c206f3 100644 --- a/internal/controller/node_controller.go +++ b/internal/controller/node_controller.go @@ -17,44 +17,154 @@ limitations under the License. package controller import ( + "bytes" "context" + "encoding/json" + "errors" + "fmt" + "os" + "time" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "k8s.io/client-go/kubernetes" + "k8s.io/kubectl/pkg/drain" + + "github.com/slyngdk/node-drain/internal/config" + "github.com/slyngdk/node-drain/internal/utils" + "k8s.io/client-go/rest" + drainv1 "github.com/slyngdk/node-drain/api/v1" "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) -// NodeReconciler reconciles a Node object -type NodeReconciler struct { - client.Client - Scheme *runtime.Scheme -} +const ( + nodeDrainFinalizer = utils.LabelPrefix + "/node" + currentStateField = "status.currentState" +) + +// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups="",resources=nodes/status,verbs=get // +kubebuilder:rbac:groups=drain.k8s.slyng.dk,resources=nodes,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=drain.k8s.slyng.dk,resources=nodes/status,verbs=get;update;patch // +kubebuilder:rbac:groups=drain.k8s.slyng.dk,resources=nodes/finalizers,verbs=update +// +kubebuilder:rbac:groups="",namespace=system,resources=pods,verbs=list;watch;create;get;delete;deletecollection +// +kubebuilder:rbac:groups="",namespace=system,resources=pods/exec,verbs=create +// +kubebuilder:rbac:groups="",resources=pods,verbs=list;delete;get +// +kubebuilder:rbac:groups="",resources=pods/eviction,verbs=create +// +kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get; + +// NodeReconciler reconciles a Node object +type nodeReconciler struct { + client.Client + Scheme *runtime.Scheme + restConfig *rest.Config + recorder record.EventRecorder + l *zap.Logger + managerNamespace string + nodeName string + rebootManager *utils.RebootManager + drainManager *utils.DrainManager +} + +func NewNodeReconciler(ctx context.Context, client client.Client, schema *runtime.Scheme, restConfig *rest.Config, recorder record.EventRecorder, managerNamespace string, nameNode string) (*nodeReconciler, error) { + l, err := config.GetNamedLogger("node") + if err != nil { + return nil, err + } + + rebootManager, err := utils.NewRebootManager(l, client, restConfig, managerNamespace) + if err != nil { + return nil, fmt.Errorf("failed to create reboot manager: %w", err) + } + _ = rebootManager.CleanupNode(ctx, "") + + drainManager, err := utils.NewDrainManager(ctx, client, restConfig) + if err != nil { + return nil, fmt.Errorf("failed to create drain manager: %w", err) + } + + return &nodeReconciler{ + Client: client, + Scheme: schema, + restConfig: restConfig, + recorder: recorder, + l: l, + managerNamespace: managerNamespace, + nodeName: nameNode, + rebootManager: rebootManager, + drainManager: drainManager, + }, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *nodeReconciler) SetupWithManager(mgr ctrl.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &drainv1.Node{}, currentStateField, func(rawObj client.Object) []string { + node := rawObj.(*drainv1.Node) + if node.Status.CurrentState == "" { + return nil + } + return []string{node.Status.CurrentState.String()} + }); err != nil { + return err + } + + return ctrl.NewControllerManagedBy(mgr). + For(&drainv1.Node{}). + Watches( + &corev1.Node{}, + handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, object client.Object) []reconcile.Request { + node := object.(*corev1.Node) + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Name: node.Name, + }, + }, + } + }), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). + Complete(r) +} // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the Node object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. // // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.0/pkg/reconcile -func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - l := zap.S().Named("node") +func (r *nodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + l := r.l.With(zap.String("node.name", req.Name)) + l.Debug("node reconcile") - l.Info("node reconcile", "request", req) + kubeNode := &corev1.Node{} + if err := r.Get(ctx, req.NamespacedName, kubeNode); err != nil { + l.With(zap.Error(err)).Error("unable to fetch kube node") + return ctrl.Result{}, err + } node := &drainv1.Node{} if err := r.Get(ctx, req.NamespacedName, node); err != nil { - err = client.IgnoreNotFound(err) + if apierrors.IsNotFound(err) { + // Create node as it is missing + return r.createNewNode(ctx, kubeNode) + } if err != nil { - l.Error(err, "unable to fetch Node") + l.With(zap.Error(err)).Error("unable to fetch node") } // we'll ignore not-found errors, since they can't be fixed by an immediate // requeue (we'll need to wait for a new notification), and we can get them @@ -62,11 +172,11 @@ func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. return ctrl.Result{}, err } - if !node.ObjectMeta.DeletionTimestamp.IsZero() { + if !node.DeletionTimestamp.IsZero() { // The object is being deleted if controllerutil.ContainsFinalizer(node, nodeDrainFinalizer) { // our finalizer is present, so lets handle any external dependency - // TODO + // TODO Handle if node is drained, k8s node removed, etc ... // remove our finalizer from the list and update it. controllerutil.RemoveFinalizer(node, nodeDrainFinalizer) @@ -79,12 +189,511 @@ func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. return ctrl.Result{}, nil } + if ok, result, err := r.undrain(ctx, l, node, kubeNode); !ok { + return result, err + } + + if !kubeNode.Spec.Unschedulable && node.Status.Drained { + l.Debug("node is not unschedulable, but is still drained, updating drain status.") + if err := r.setDrained(ctx, node, false); err != nil { + return ctrl.Result{}, err + } + } + + // Check reboot + if err := r.checkRebootRequired(ctx, node, l); err != nil { + return ctrl.Result{}, err + } + + switch node.Spec.State { + case drainv1.NodeStateActive: + managerOfUnschedulable := r.getManagerOfUnschedulable(kubeNode) + if managerOfUnschedulable != "" && managerOfUnschedulable != utils.GetFieldOwner(r.managerNamespace) { + l.Warn("node is current cordon by another manager", zap.String("manager", managerOfUnschedulable)) + r.recorder.Eventf(node, corev1.EventTypeWarning, "NodeNotSchedulable", "Node is current cordon by another manager: %s", managerOfUnschedulable) + return ctrl.Result{}, nil + } + if err := r.setUnschedulable(ctx, kubeNode, false); err != nil { + return ctrl.Result{}, err + } + if err := r.setCurrentState(ctx, l, node, drainv1.NodeCurrentStateOk); err != nil { + return ctrl.Result{}, err + } + case drainv1.NodeStateCordoned: + if err := r.setUnschedulable(ctx, kubeNode, true); err != nil { + return ctrl.Result{}, err + } + if err := r.setCurrentState(ctx, l, node, drainv1.NodeCurrentStateCordoned); err != nil { + return ctrl.Result{}, err + } + case drainv1.NodeStateDrained, drainv1.NodeStateRebootIfRequired: + if !node.Status.CurrentState.WorkState() && node.Status.CurrentState != drainv1.NodeCurrentStateQueued { + if err := r.setCurrentState(ctx, l, node, drainv1.NodeCurrentStateQueued); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 1 * time.Second}, nil + } + } + + if node.Status.CurrentState.WorkState() { + if node.Spec.State == drainv1.NodeStateRebootIfRequired && !node.Status.Drained { + var err error + required := false + if node.Status.RebootRequiredLastChecked == nil || + node.Status.RebootRequiredLastChecked.Time.IsZero() || + node.Status.RebootRequiredLastChecked.Time.Before(time.Now().Add(-(2 * time.Minute))) { + + required, err = r.isRebootRequired(ctx, node, l) + if err != nil { + return ctrl.Result{}, err + } + } else if node.Status.RebootRequired != nil { + required = *node.Status.RebootRequired + } + + if !required { + l.Debug("Reboot is not required, setting node active") + err = r.setState(ctx, l, node, drainv1.NodeStateActive) + if err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + } + if node.Spec.State.Drain() && !node.Status.Drained { + result, err := r.drain(ctx, l, node, kubeNode) + if err != nil { + return ctrl.Result{}, err + } + if result != nil { + return *result, nil + } + } + + if node.Spec.State == drainv1.NodeStateRebootIfRequired { + if node.Status.CurrentState == drainv1.NodeCurrentStateDrained && node.Status.Drained { + l.Debug("Saving bootID before rebooting node") + err := r.setBootID(ctx, node, kubeNode) + if err != nil { + return ctrl.Result{}, err + } + + l.Info("Rebooting node") + err = r.rebootManager.RebootNode(ctx, node.Name) + if err != nil { + return ctrl.Result{}, err + } + + err = r.setCurrentState(ctx, l, node, drainv1.NodeCurrentStateRebooting) + if err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: time.Minute}, nil + } + if node.Status.CurrentState == drainv1.NodeCurrentStateRebooting { + rebooted, err := r.rebootManager.IsNodeRebooted(ctx, kubeNode, node.Status.BootID) + if err != nil { + return ctrl.Result{}, err + } + + if rebooted { + l.Info("node is rebooted") + + patch := client.MergeFrom(node.DeepCopy()) + node.Status.BootID = "" + node.Status.RebootRequired = utils.PtrTo(false) + node.Status.RebootRequiredLastChecked = &metav1.Time{Time: time.Now()} + if err = r.Status().Patch(ctx, node, patch); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to update node reboot required last checked: %w", err) + } + + err = r.setState(ctx, l, node, drainv1.NodeStateActive) + if err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: time.Second}, nil + } else { + l.Info("Node not ready yet after reboot") + return ctrl.Result{RequeueAfter: time.Minute}, nil + } + } + } + } else if node.Status.CurrentState == drainv1.NodeCurrentStateQueued { + l.Debug("Checking if queued node is next") + next, err := r.isNextNode(ctx, l, node) + if err != nil { + return ctrl.Result{}, err + } + if next { + return ctrl.Result{RequeueAfter: 1 * time.Second}, nil + } else { + return ctrl.Result{RequeueAfter: 1 * time.Minute}, nil + } + } + + err := r.rebootManager.Cleanup(ctx) + if err != nil { + l.Warn("Failed to cleanup reboot pods", zap.Error(err)) + } + return ctrl.Result{}, nil } -// SetupWithManager sets up the controller with the Manager. -func (r *NodeReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&drainv1.Node{}). - Complete(r) +func (r *nodeReconciler) createNewNode(ctx context.Context, kubeNode *corev1.Node) (ctrl.Result, error) { + state := drainv1.NodeStateActive + if kubeNode.Spec.Unschedulable { + state = drainv1.NodeStateCordoned + } + + node := &drainv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: kubeNode.Name, + Finalizers: nil, + }, + Spec: drainv1.NodeSpec{ + State: state, + }, + } + + if err := controllerutil.SetOwnerReference(kubeNode, node, r.Scheme); err != nil { + return ctrl.Result{}, err + } + + controllerutil.AddFinalizer(node, nodeDrainFinalizer) + + if err := r.Create(ctx, node); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil +} + +func (r *nodeReconciler) setDrained(ctx context.Context, node *drainv1.Node, drained bool) error { + patch := client.MergeFrom(node.DeepCopy()) + node.Status.Drained = drained + if err := r.Status().Patch(ctx, node, patch); err != nil { + return fmt.Errorf("failed to update drained status on node: %w", err) + } + return nil +} + +func (r *nodeReconciler) setCurrentState(ctx context.Context, l *zap.Logger, node *drainv1.Node, s drainv1.NodeCurrentState) error { + if node.Status.CurrentState == s { + return nil + } + + l.Info("setting current state on node", zap.String("currentState", s.String())) + patch := client.MergeFrom(node.DeepCopy()) + node.Status.CurrentState = s + if err := r.Status().Patch(ctx, node, patch); err != nil { + return fmt.Errorf("failed to update current state on node: %w", err) + } + return nil +} + +func (r *nodeReconciler) setState(ctx context.Context, l *zap.Logger, node *drainv1.Node, s drainv1.NodeState) error { + if node.Spec.State == s { + return nil + } + + l.Info("setting state on node", zap.String("state", s.String())) + patch := client.MergeFrom(node.DeepCopy()) + node.Spec.State = s + if err := r.Patch(ctx, node, patch); err != nil { + return fmt.Errorf("failed to update state on node: %w", err) + } + return nil +} + +func (r *nodeReconciler) setUnschedulable(ctx context.Context, kubeNode *corev1.Node, unschedulable bool) error { + if kubeNode.Spec.Unschedulable != unschedulable { + patch := client.MergeFrom(kubeNode.DeepCopy()) + kubeNode.Spec.Unschedulable = unschedulable + if err := r.Patch(ctx, kubeNode, patch); err != nil { + return fmt.Errorf("failed to update unschedulable status on node: %w", err) + } + } + return nil +} + +func (r *nodeReconciler) setBootID(ctx context.Context, node *drainv1.Node, kubeNode *corev1.Node) error { + if kubeNode.Status.NodeInfo.BootID != node.Status.BootID { + patch := client.MergeFrom(node.DeepCopy()) + node.Status.BootID = kubeNode.Status.NodeInfo.BootID + if err := r.Status().Patch(ctx, node, patch); err != nil { + return fmt.Errorf("failed to update bootID on node: %w", err) + } + } + return nil +} + +func (r *nodeReconciler) checkRebootRequired(ctx context.Context, node *drainv1.Node, l *zap.Logger) error { + rebootCheckInterval := config.GetKoanf().Duration("reboot.checkInterval") + if rebootCheckInterval < 5*time.Minute { + rebootCheckInterval = 24 * time.Hour + } + if node.Status.RebootRequiredLastChecked == nil || + node.Status.RebootRequiredLastChecked.Time.IsZero() || + node.Status.RebootRequiredLastChecked.Time.Before(time.Now().Add(-rebootCheckInterval)) { + _, err := r.isRebootRequired(ctx, node, l) + if err != nil { + return err + } + } + return nil +} + +func (r *nodeReconciler) isRebootRequired(ctx context.Context, node *drainv1.Node, l *zap.Logger) (bool, error) { + l.Debug("Checking if reboot is required") + required, err := r.rebootManager.IsRebootRequired(ctx, node.Name) + if err != nil { + return false, fmt.Errorf("failed to check if reboot is required: %w", err) + } + // TODO change to use conditions + patch := client.MergeFrom(node.DeepCopy()) + node.Status.RebootRequired = utils.PtrTo(required) + node.Status.RebootRequiredLastChecked = &metav1.Time{Time: time.Now()} + if err = r.Status().Patch(ctx, node, patch); err != nil { + return false, fmt.Errorf("failed to update node reboot required last checked: %w", err) + } + if required { + r.recorder.Eventf(node, corev1.EventTypeNormal, "RebootRequired", "Reboot is required") + } + return required, nil +} + +func (r *nodeReconciler) drain(ctx context.Context, l *zap.Logger, node *drainv1.Node, kubeNode *corev1.Node) (*ctrl.Result, error) { + if !node.Spec.State.Drain() || node.Status.Drained { + return nil, nil + } + + if node.Status.CurrentState == drainv1.NodeCurrentStateNext { + + // Check if cluster is healthy before starting drain + healthy, err := r.drainManager.IsHealthy(ctx) + if err != nil { + return nil, fmt.Errorf("failed to check if cluster is healthy: %w", err) + } + if !healthy { + return nil, fmt.Errorf("cluster is not healthy") + } + + // Check if drain of node is ok + drainOk, err := r.drainManager.IsDrainOk(ctx, node.Name) + if err != nil { + return nil, fmt.Errorf("failed to check if node(%s) is ok to drain: %w", node.Name, err) + } + if !drainOk { + return nil, fmt.Errorf("node(%s) is not ok to drain", node.Name) + } + + err = r.setCurrentState(ctx, l, node, drainv1.NodeCurrentStateDraining) + if err != nil { + return nil, err + } + } + + if !kubeNode.Spec.Unschedulable { + l.Info("Disable scheduling on node") + if err := r.setUnschedulable(ctx, kubeNode, true); err != nil { + return nil, err + } + } + + if r.nodeName == node.Name { + l.Info("Running on the node which is about to be drained") + err := r.rescheduleController(ctx) + if err != nil { + return nil, fmt.Errorf("failed to reschedule controller: %w", err) + } + return &ctrl.Result{RequeueAfter: 5 * time.Minute}, nil + } + + // Run Plugin PreDrain + err := r.drainManager.RunPreDrain(ctx, node.Name) + if err != nil { + return nil, fmt.Errorf("failed to run plugin PreDrain for node(%s): %w", node.Name, err) + } + + // TODO Ensure node is drained + + clientSet, err := kubernetes.NewForConfig(r.restConfig) + if err != nil { + return nil, err + } + + stdout := new(bytes.Buffer) + stderr := new(bytes.Buffer) + + drainHelper := &drain.Helper{ + Ctx: ctx, + Client: clientSet, + GracePeriodSeconds: -1, // Wait for pod's terminationGracePeriodSeconds + IgnoreAllDaemonSets: true, + Timeout: 60 * time.Second, + DeleteEmptyDirData: true, + Out: stdout, + ErrOut: stderr, + } + + // if dryRun { + // drainHelper.DryRunStrategy = cmdutil.DryRunServer + // } + + l.Info("draining node") + + err = r.rebootManager.CleanupNode(ctx, node.Name) + if err != nil { + return nil, fmt.Errorf("failed to cleanup node(%s) for reboot manager pods: %w", node.Name, err) + } + + err = drain.RunNodeDrain(drainHelper, node.Name) + if err != nil { + l.Error("failed to drain node", + zap.String("stdout", stdout.String()), + zap.String("stderr", stderr.String())) + return nil, errors.Join(fmt.Errorf("failed to drain node: %s", node.Name), err) + } + + l.Info("drained node", + zap.String("stdout", stdout.String()), + zap.String("stderr", stderr.String())) + + patch := client.MergeFrom(node.DeepCopy()) + node.Status.Drained = true + node.Status.CurrentState = drainv1.NodeCurrentStateDrained + if err := r.Status().Patch(ctx, node, patch); err != nil { + return nil, fmt.Errorf("failed to update drained status on node: %w", err) + } + + return nil, nil +} + +func (r *nodeReconciler) undrain(ctx context.Context, l *zap.Logger, node *drainv1.Node, kubeNode *corev1.Node) (bool, ctrl.Result, error) { + if node.Status.CurrentState == drainv1.NodeCurrentStateUndraining { + if kubeNode.Spec.Unschedulable { + healthy, err := r.drainManager.IsClusterNodesHealthy(ctx) + if err != nil { + l.Error("error checking if cluster is healthy", zap.Error(err)) + return false, ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + if !healthy { + l.Debug("cluster nodes is not healthy, waiting for nodes to be healthy before enabling scheduling") + return false, ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + + if err = r.setUnschedulable(ctx, kubeNode, false); err != nil { + return false, ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + } + + if err := r.drainManager.RunPostDrain(ctx, node.Name); err != nil { + l.Warn("failed to run PostDrain for node", zap.Error(err)) + return false, ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + + if node.Status.Drained { + if err := r.setDrained(ctx, node, false); err != nil { + l.Warn("failed to set Drained=false for node", zap.Error(err)) + return false, ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + } + + if err := r.setCurrentState(ctx, l, node, drainv1.NodeCurrentStateOk); err != nil { + l.Debug("failed to set current state on node", zap.Error(err)) + return false, ctrl.Result{}, err + } + + return true, ctrl.Result{}, nil + } + + undrain := false + if node.Status.CurrentState == drainv1.NodeCurrentStateDrained || + node.Status.CurrentState == drainv1.NodeCurrentStateDraining || + node.Status.CurrentState == drainv1.NodeCurrentStateRebooting { + // Check need for undrain + switch node.Spec.State { + case drainv1.NodeStateActive: + undrain = true + case drainv1.NodeStateDrained, drainv1.NodeStateRebootIfRequired, drainv1.NodeStateCordoned: + break + default: + return false, ctrl.Result{}, fmt.Errorf("unhandled state for undrain: %s", node.Spec.State) + } + } + + if undrain { + err := r.setCurrentState(ctx, l, node, drainv1.NodeCurrentStateUndraining) + if err != nil { + return false, ctrl.Result{}, err + } + return false, ctrl.Result{RequeueAfter: 1}, nil + } + return true, ctrl.Result{}, nil +} + +func (r *nodeReconciler) rescheduleController(ctx context.Context) error { + clientset, err := kubernetes.NewForConfig(r.restConfig) + if err != nil { + return fmt.Errorf("failed to create clientset: %w", err) + } + podName, err := os.Hostname() + if err != nil { + return fmt.Errorf("failed to get hostname/podName: %w", err) + } + + deletePolicy := metav1.DeletePropagationForeground + return clientset.CoreV1().Pods(r.managerNamespace).Delete(ctx, podName, metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }) +} + +func (r *nodeReconciler) isNextNode(ctx context.Context, l *zap.Logger, node *drainv1.Node) (bool, error) { + nodeList := &drainv1.NodeList{} + err := r.List(ctx, nodeList) + if err != nil { + return false, err + } + + for _, n := range nodeList.Items { + if n.Status.CurrentState.WorkState() { + l.Debug("There is already a node doing work") + return false, nil + } + } + + if node.Status.CurrentState == drainv1.NodeCurrentStateQueued { + err = r.setCurrentState(ctx, l, node, drainv1.NodeCurrentStateNext) + if err != nil { + return false, err + } + return true, nil + } + + return false, nil +} + +func (r *nodeReconciler) getManagerOfUnschedulable(kubeNode *corev1.Node) string { + for _, field := range kubeNode.ManagedFields { + if field.Subresource != "" { + continue + } + if field.FieldsV1 != nil { + var f interface{} + err := json.Unmarshal(field.FieldsV1.Raw, &f) + if err != nil { + continue + } + + if m, ok := f.(map[string]interface{}); ok { + if v, ok := m["f:spec"]; ok { + if s, ok := v.(map[string]interface{}); ok { + if _, ok := s["f:unschedulable"]; ok { + return field.Manager + } + } + } + } + } + } + return "" } diff --git a/internal/controller/node_controller_test.go b/internal/controller/node_controller_test.go index 71920a6..66d34de 100644 --- a/internal/controller/node_controller_test.go +++ b/internal/controller/node_controller_test.go @@ -18,15 +18,20 @@ package controller import ( "context" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/api/errors" + "github.com/slyngdk/node-drain/internal/config" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/reconcile" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - drainv1 "github.com/slyngdk/node-drain/api/v1" ) @@ -37,48 +42,66 @@ var _ = Describe("Node Controller", func() { ctx := context.Background() typeNamespacedName := types.NamespacedName{ - Name: resourceName, - Namespace: "default", // TODO(user):Modify as needed + Name: resourceName, } - node := &drainv1.Node{} + + _, err := config.LoadConfig() + Expect(err).NotTo(HaveOccurred()) + + _, _ = config.GetLogger("info", "json") BeforeEach(func() { - By("creating the custom resource for the Kind Node") - err := k8sClient.Get(ctx, typeNamespacedName, node) - if err != nil && errors.IsNotFound(err) { - resource := &drainv1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - Namespace: "default", - }, - // TODO(user): Specify other spec details if needed. - } - Expect(k8sClient.Create(ctx, resource)).To(Succeed()) - } + }) AfterEach(func() { - // TODO(user): Cleanup logic after each test, like removing the resource instance. resource := &drainv1.Node{} err := k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) + Expect(client.IgnoreNotFound(err)).NotTo(HaveOccurred()) + if !apierrors.IsNotFound(err) { + By("Cleanup the specific resource instance Node") + controllerutil.RemoveFinalizer(resource, nodeDrainFinalizer) + Expect(k8sClient.Update(ctx, resource)).To(Succeed()) + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + } - By("Cleanup the specific resource instance Node") - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + kubeNode := &corev1.Node{} + err = k8sClient.Get(ctx, typeNamespacedName, kubeNode) + Expect(client.IgnoreNotFound(err)).NotTo(HaveOccurred()) + if !apierrors.IsNotFound(err) { + By("Cleanup the specific kubenode") + Expect(k8sClient.Delete(ctx, kubeNode)).To(Succeed()) + } }) It("should successfully reconcile the resource", func() { - By("Reconciling the created resource") - controllerReconciler := &NodeReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), + By("Reconciling kube node which should result in a new created node with same name") + + kubeNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + }, } + Expect(k8sClient.Create(ctx, kubeNode)).To(Succeed()) - _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ + c, err := cluster.New(cfg) + Expect(err).NotTo(HaveOccurred()) + recorderFor := c.GetEventRecorderFor("node-controller-test") + Expect(recorderFor).ShouldNot(BeNil()) + + controllerReconciler, err := NewNodeReconciler(ctx, k8sClient, k8sClient.Scheme(), cfg, recorderFor, managerNamespace, "node-test") + Expect(err).NotTo(HaveOccurred()) + + res, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ NamespacedName: typeNamespacedName, }) Expect(err).NotTo(HaveOccurred()) - // TODO(user): Add more specific assertions depending on your controller's reconciliation logic. - // Example: If you expect a certain status condition after reconciliation, verify it here. + Expect(res).ShouldNot(BeNil()) + Expect(res.RequeueAfter).Should(Equal(5 * time.Second)) + + resource := &drainv1.Node{} + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + Expect(resource.Spec.State).Should(Equal(drainv1.NodeStateActive)) }) }) }) diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index a8a3b73..57e3950 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -19,6 +19,7 @@ package controller import ( "context" "fmt" + "os" "path/filepath" "runtime" "testing" @@ -45,6 +46,7 @@ var k8sClient client.Client var testEnv *envtest.Environment var ctx context.Context var cancel context.CancelFunc +var managerNamespace string = "nodedrain-system" func TestControllers(t *testing.T) { RegisterFailHandler(Fail) @@ -71,6 +73,11 @@ var _ = BeforeSuite(func() { fmt.Sprintf("1.31.0-%s-%s", runtime.GOOS, runtime.GOARCH)), } + // Retrieve the first found binary directory to allow running tests from IDEs + if getFirstFoundEnvTestBinaryDir() != "" { + testEnv.BinaryAssetsDirectory = getFirstFoundEnvTestBinaryDir() + } + var err error // cfg is defined in this file globally. cfg, err = testEnv.Start() @@ -94,3 +101,26 @@ var _ = AfterSuite(func() { err := testEnv.Stop() Expect(err).NotTo(HaveOccurred()) }) + +// getFirstFoundEnvTestBinaryDir locates the first binary in the specified path. +// ENVTEST-based tests depend on specific binaries, usually located in paths set by +// controller-runtime. When running tests directly (e.g., via an IDE) without using +// Makefile targets, the 'BinaryAssetsDirectory' must be explicitly configured. +// +// This function streamlines the process by finding the required binaries, similar to +// setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are +// properly set up, run 'make setup-envtest' beforehand. +func getFirstFoundEnvTestBinaryDir() string { + basePath := filepath.Join("..", "..", "bin", "k8s") + entries, err := os.ReadDir(basePath) + if err != nil { + logf.Log.Error(err, "Failed to read directory", "path", basePath) + return "" + } + for _, entry := range entries { + if entry.IsDir() { + return filepath.Join(basePath, entry.Name()) + } + } + return "" +} diff --git a/internal/modules/utils.go b/internal/modules/utils.go index b77c494..69de039 100644 --- a/internal/modules/utils.go +++ b/internal/modules/utils.go @@ -4,11 +4,12 @@ import ( "context" "fmt" "io" + "regexp" + "strings" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" "k8s.io/client-go/tools/remotecommand" - "regexp" - "strings" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" diff --git a/internal/utils/drain-manager.go b/internal/utils/drain-manager.go new file mode 100644 index 0000000..2f98080 --- /dev/null +++ b/internal/utils/drain-manager.go @@ -0,0 +1,351 @@ +package utils + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/hashicorp/go-plugin" + "github.com/pkg/errors" + "github.com/slyngdk/node-drain/api/plugins" + "github.com/slyngdk/node-drain/internal/config" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + kClient "sigs.k8s.io/controller-runtime/pkg/client" +) + +type drainClientInfo struct { + *plugins.DrainClient + info *plugins.DrainPluginInfo +} + +type DrainManager struct { + l *zap.Logger + client kClient.Client + clientSet *kubernetes.Clientset + pluginClients map[string]*plugin.Client + drainClients map[string]drainClientInfo + pluginFileId map[string]string +} + +func NewDrainManager(ctx context.Context, client kClient.Client, restConfig *rest.Config) (*DrainManager, error) { + l, err := config.GetNamedLogger("drain-plugin-client") + if err != nil { + return nil, err + } + + clientSet, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return nil, err + } + + d := &DrainManager{ + l: l, + client: client, + clientSet: clientSet, + drainClients: make(map[string]drainClientInfo), + pluginFileId: make(map[string]string), + } + + err = d.loadPluginsFromDir(ctx) + if err != nil { + return nil, fmt.Errorf("failed to load plugins: %w", err) + } + + return d, nil +} + +func (d *DrainManager) loadPluginsFromDir(ctx context.Context) error { + path := "/plugins" + + d.l.Debug("Loading plugins from path", zap.String("plugin.path", path)) + + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + + entries, err := os.ReadDir(path) + if err != nil { + return fmt.Errorf("failed to load plugins: %w", err) + } + + foundClients := make(map[string]*plugin.Client) + + for _, e := range entries { + if e.IsDir() { + continue + } + if filepath.Ext(e.Name()) == ".so" { + pluginPath := filepath.Join(path, e.Name()) + pluginBase := filepath.Base(pluginPath) + + pluginLogger := d.l.With(zap.String("plugin.file", pluginBase)) + pluginLogger.Debug("Creating new client for plugin") + client := plugin.NewClient(&plugin.ClientConfig{ + HandshakeConfig: plugins.Handshake, + VersionedPlugins: map[int]plugin.PluginSet{ + 0: { + "drain": plugins.GRPCDrainPlugin{}, + }, + }, + Cmd: exec.Command(pluginPath), + Managed: true, + Stderr: d.pluginOutputMonitor("Stderr", pluginLogger, pluginBase), + SyncStdout: d.pluginOutputMonitor("SyncStdout", pluginLogger, pluginBase), + SyncStderr: d.pluginOutputMonitor("SyncStdout", pluginLogger, pluginBase), + AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC}, + Logger: Wrap(pluginLogger), + }) + foundClients[pluginPath] = client + + info, drainClient, err := d.initPlugin(ctx, pluginLogger, pluginBase, client) + if err != nil { + return fmt.Errorf("failed to initialize plugin %s: %w", pluginBase, err) + } + + if _, ok := d.drainClients[info.ID]; ok { + return fmt.Errorf("drain plugin %s(%s) already initialized", pluginBase, info.ID) + } + d.drainClients[info.ID] = drainClientInfo{drainClient, info} + d.pluginFileId[pluginBase] = info.ID + } + } + d.pluginClients = foundClients + return nil +} + +func (d *DrainManager) initPlugin(ctx context.Context, l *zap.Logger, pluginBase string, client *plugin.Client) (*plugins.DrainPluginInfo, *plugins.DrainClient, error) { + l.Debug("Initializing plugin") + c, err := client.Client() + if err != nil { + return nil, nil, fmt.Errorf("failed to get client for plugin %s: %w", pluginBase, err) + } + + l.Debug("Pinging plugin") + if err = c.Ping(); err != nil { + return nil, nil, fmt.Errorf("failed to ping plugin %s: %w", pluginBase, err) + } + + l.Debug("Getting instance of drain plugin") + v, err := c.Dispense("drain") + if err != nil { + return nil, nil, fmt.Errorf("failed to dispense plugin %s: %w", pluginBase, err) + } + + drainClient, ok := v.(*plugins.DrainClient) + if !ok { + return nil, nil, fmt.Errorf("expected drain plugin %s: %T", pluginBase, v) + } + + l.Debug("Calling drain plugin Init()") + info, err := drainClient.Init(ctx, Wrap(l), plugins.DrainPluginSettings{}) + if err != nil { + return nil, nil, fmt.Errorf("failed to init plugin %s: %w", pluginBase, err) + } + + if info.ID == "" { + return nil, nil, fmt.Errorf("drain plugin %s has no ID", pluginBase) + } + return &info, drainClient, nil +} + +func (d *DrainManager) IsClusterNodesHealthy(ctx context.Context) (bool, error) { + d.l.Debug("Checking if cluster nodes are healthy") + nodes, err := d.clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return false, err + } + + allNodesAreReady := true + + for _, node := range nodes.Items { + // TODO Check all conditions and these is up to date (Cilium is not updating) + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady { + if condition.Status != corev1.ConditionTrue { + d.l.Warn("node is not ready", zap.String("node.name", node.Name), zap.String("node.ready", string(condition.Status))) + allNodesAreReady = false + } + } + } + } + return allNodesAreReady, nil +} + +func (d *DrainManager) IsHealthy(ctx context.Context) (bool, error) { + d.l.Debug("Checking if cluster is healthy, including drain plugins.") + healthy, err := d.IsClusterNodesHealthy(ctx) + if !healthy || err != nil { + return false, err + } + + allModulesHealthy := true + + for id, client := range d.drainClients { + l := d.l.With(zap.String("plugin.id", id)) + l.Debug("Checking if drain plugin is supported") + isSupported, err := client.IsSupported(ctx) + if err != nil { + return false, err + } + if !isSupported { + d.l.Debug("Plugin not supported") + continue + } + + l.Debug("Checking if drain plugin is healthy") + isHealthy, err := client.IsHealthy(ctx) + if err != nil { + return false, err + } + if !isHealthy { + d.l.Warn("Plugin is not healthy") + allModulesHealthy = false + continue + } + } + + return allModulesHealthy, err +} + +func (d *DrainManager) IsDrainOk(ctx context.Context, nodeName string) (bool, error) { + l := d.l.With(zap.String("node.name", nodeName)) + l.Debug("Checking if drain is OK") + allModulesReadyToDrain := true + + for id, client := range d.drainClients { + lp := l.With(zap.String("plugin.id", id)) + lp.Debug("Checking if drain plugin is supported") + isSupported, err := client.IsSupported(ctx) + if err != nil { + return false, err + } + if !isSupported { + d.l.Debug("Plugin not supported") + continue + } + + isDrainOk, err := client.IsDrainOk(ctx, nodeName) + if err != nil { + return false, err + } + if !isDrainOk { + lp.Warn("plugin is not ready to be drained") + allModulesReadyToDrain = false + continue + } + } + + return allModulesReadyToDrain, nil +} + +func (d *DrainManager) RunPreDrain(ctx context.Context, nodeName string) error { + l := d.l.With(zap.String("node.name", nodeName)) + l.Debug("Run PreDrain") + + for id, client := range d.drainClients { + lp := l.With(zap.String("plugin.id", id)) + lp.Debug("Checking if drain plugin is supported") + isSupported, err := client.IsSupported(ctx) + if err != nil { + return err + } + if !isSupported { + d.l.Debug("Plugin not supported") + continue + } + + lp.Debug("Running plugin PreDrain") + err = client.PreDrain(ctx, nodeName) + if err != nil { + lp.Debug("Plugin PreDrain failed", zap.Error(err)) + return errors.Wrapf(err, "failed PreDrain on plugin: %s", id) + } + lp.Debug("Plugin PreDrain succeeded without errors") + } + return nil +} + +func (d *DrainManager) RunPostDrain(ctx context.Context, nodeName string) error { + l := d.l.With(zap.String("node.name", nodeName)) + l.Debug("Run PostDrain") + + for id, client := range d.drainClients { + lp := l.With(zap.String("plugin.id", id)) + lp.Debug("Checking if drain plugin is supported") + isSupported, err := client.IsSupported(ctx) + if err != nil { + return err + } + if !isSupported { + d.l.Debug("Plugin not supported") + continue + } + + lp.Debug("Running plugin PostDrain") + err = client.PostDrain(ctx, nodeName) + if err != nil { + lp.Debug("Plugin PostDrain failed", zap.Error(err)) + return errors.Wrapf(err, "failed PostDrain on plugin: %s", id) + } + lp.Debug("Plugin PostDrain succeeded without errors") + } + return nil +} + +func (d *DrainManager) pluginOutputMonitor(streamName string, l *zap.Logger, pluginFile string) io.Writer { + l = l.WithOptions(zap.WithCaller(false), zap.AddStacktrace(zap.FatalLevel)).Named(streamName) + reader, writer := io.Pipe() + + go func() { + addedPluginId := false + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 1024), 1024*1024) // 1MB max buffer + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if len(line) == 0 { + continue + } + + if !addedPluginId { + if id, ok := d.pluginFileId[pluginFile]; ok { + l = l.With(zap.String("plugin.id", id)) + addedPluginId = true + } + } + + err := logJsonLog(l, line) + if err != nil { + switch line := line; { + case strings.HasPrefix(line, "[TRACE]"): + l.Log(config.TraceLevel, line) + case strings.HasPrefix(line, "[DEBUG]"): + l.Debug(line) + case strings.HasPrefix(line, "[INFO]"): + l.Info(line) + case strings.HasPrefix(line, "[WARN]"): + l.Warn(line) + case strings.HasPrefix(line, "[ERROR]"): + l.Error(line) + case strings.HasPrefix(line, "panic: ") || strings.HasPrefix(line, "fatal error: "): + l.Error(line) + default: + l.Info(line) + } + } + } + }() + + return writer +} diff --git a/internal/utils/hclog.go b/internal/utils/hclog.go new file mode 100644 index 0000000..fbcf3d3 --- /dev/null +++ b/internal/utils/hclog.go @@ -0,0 +1,220 @@ +package utils + +import ( + "encoding/json" + "fmt" + "io" + "log" + "runtime" + "time" + + "github.com/hashicorp/go-hclog" + "github.com/slyngdk/node-drain/internal/config" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func Wrap(z *zap.Logger) hclog.Logger { + z = z.WithOptions(zap.AddCallerSkip(1)) + return wrapper{zap: z} +} + +type Level = hclog.Level + +// Wrapper holds *zap.Logger and adapts its methods to declared by hclog.Logger. +type wrapper struct { + zap *zap.Logger +} + +func (w wrapper) Debug(msg string, args ...interface{}) { + w.zap.Debug(msg, w.convertToZapAny(args...)...) +} +func (w wrapper) Info(msg string, args ...interface{}) { + w.zap.Info(msg, w.convertToZapAny(args...)...) +} +func (w wrapper) Warn(msg string, args ...interface{}) { + w.zap.Warn(msg, w.convertToZapAny(args...)...) +} +func (w wrapper) Error(msg string, args ...interface{}) { + w.zap.Error(msg, w.convertToZapAny(args...)...) +} + +// Log logs messages with four simplified levels - Debug,Warn,Error and Info as a default. +func (w wrapper) Log(lvl Level, msg string, args ...interface{}) { + switch lvl { + case hclog.Trace: + w.Trace(msg, args...) + case hclog.Debug: + w.Debug(msg, args...) + case hclog.Info: + w.Info(msg, args...) + case hclog.Warn: + w.Warn(msg, args...) + case hclog.Error: + w.Error(msg, args...) + case hclog.Off: + default: + w.Info(msg, args...) + } +} + +// Trace will log an info-level message in Zap. +func (w wrapper) Trace(msg string, args ...interface{}) { + w.zap.Log(config.TraceLevel, msg, w.convertToZapAny(args...)...) +} + +// With returns a logger with always-presented key-value pairs. +func (w wrapper) With(args ...interface{}) hclog.Logger { + return &wrapper{zap: w.zap.With(w.convertToZapAny(args...)...)} +} + +// Named returns a logger with the specific name. +// The name string will always be presented in a log messages. +func (w wrapper) Named(name string) hclog.Logger { + return &wrapper{zap: w.zap.Named(name)} +} + +// Name returns a logger's name (if presented). +func (w wrapper) Name() string { return w.zap.Name() } + +// ResetNamed has the same implementation as Named. +func (w wrapper) ResetNamed(name string) hclog.Logger { + return &wrapper{zap: w.zap.Named(name)} +} + +// StandardWriter returns os.Stderr as io.Writer. +func (w wrapper) StandardWriter(opts *hclog.StandardLoggerOptions) io.Writer { + return hclog.DefaultOutput +} + +// StandardLogger returns standard logger with os.Stderr as a writer. +func (w wrapper) StandardLogger(opts *hclog.StandardLoggerOptions) *log.Logger { + return log.New(w.StandardWriter(opts), "", log.LstdFlags) +} + +func (w wrapper) IsTrace() bool { + return w.zap.Level() <= config.TraceLevel +} + +func (w wrapper) IsDebug() bool { + return w.zap.Level() <= zapcore.DebugLevel +} + +func (w wrapper) IsInfo() bool { + return w.zap.Level() <= zapcore.InfoLevel +} + +func (w wrapper) IsWarn() bool { + return w.zap.Level() <= zapcore.WarnLevel +} + +func (w wrapper) IsError() bool { + return w.zap.Level() <= zapcore.ErrorLevel +} + +// ImpliedArgs has no implementation. +func (w wrapper) ImpliedArgs() []interface{} { return nil } + +// SetLevel has no implementation. +func (w wrapper) SetLevel(lvl Level) {} + +func (w wrapper) GetLevel() hclog.Level { + // Disable Plugin Stderr handling + pc, _, _, ok := runtime.Caller(1) + details := runtime.FuncForPC(pc) + if ok && details != nil && details.Name() == "github.com/hashicorp/go-plugin.(*Client).logStderr" { + return hclog.Off + } + + if w.zap.Level() == config.TraceLevel { + return hclog.Trace + } + return hclog.LevelFromString(w.zap.Level().String()) +} + +func (w wrapper) convertToZapAny(args ...interface{}) []zapcore.Field { + fields := []zapcore.Field{} + for i := len(args); i > 0; i -= 2 { + left := i - 2 + if left < 0 { + left = 0 + } + + items := args[left:i] + + switch l := len(items); l { + case 2: + k, ok := items[0].(string) + if ok { + fields = append(fields, zap.Any(k, items[1])) + } else { + fields = append(fields, zap.Any(fmt.Sprintf("arg%d", i-1), items[1])) + fields = append(fields, zap.Any(fmt.Sprintf("arg%d", left), items[0])) + } + case 1: + fields = append(fields, zap.Any(fmt.Sprintf("arg%d", left), items[0])) + } + } + + return fields +} + +func getLevel(s string) zapcore.Level { + switch s { + case "trace", "off": + return config.TraceLevel + case "debug": + return zapcore.DebugLevel + case "warn": + return zapcore.WarnLevel + case "error": + return zapcore.ErrorLevel + default: + return zapcore.InfoLevel + } +} + +func logJsonLog(l *zap.Logger, line string) error { + var raw map[string]interface{} + + err := json.Unmarshal([]byte(line), &raw) + if err != nil { + return err + } + + msg := "" + if v, ok := raw["@message"]; ok { + msg = v.(string) + delete(raw, "@message") + } + + level := zap.InfoLevel + if v, ok := raw["@level"]; ok { + levelS := v.(string) + delete(raw, "@level") + level = getLevel(levelS) + } + + ce := l.Check(level, msg) + if ce == nil { + return nil + } + + if v, ok := raw["@timestamp"]; ok { + t, err := time.Parse("2006-01-02T15:04:05.000000Z07:00", v.(string)) + if err != nil { + return err + } + ce.Time = t + delete(raw, "@timestamp") + } + + fields := make([]zapcore.Field, 0) + + for k, v := range raw { + fields = append(fields, zap.Any(k, v)) + } + + ce.Write(fields...) + return nil +} diff --git a/internal/utils/reboot-manager.go b/internal/utils/reboot-manager.go index dbb1839..a2a1a8b 100644 --- a/internal/utils/reboot-manager.go +++ b/internal/utils/reboot-manager.go @@ -4,21 +4,21 @@ import ( "bytes" "context" "fmt" - mod "github.com/slyngdk/node-drain/internal/modules" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" "time" - "k8s.io/apimachinery/pkg/selection" - "github.com/pkg/errors" + "github.com/slyngdk/node-drain/internal/config" + mod "github.com/slyngdk/node-drain/internal/modules" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" ) type RebootManager struct { @@ -44,13 +44,6 @@ func NewRebootManager(l *zap.Logger, client client.Client, restConfig *rest.Conf } func (r *RebootManager) IsRebootRequired(ctx context.Context, nodeName string) (bool, error) { - defer func(r *RebootManager, ctx context.Context) { - err := r.cleanup(ctx) - if err != nil { - r.l.Error("failed to cleanup", zap.Error(err)) - } - }(r, ctx) - if mod.GetNode(ctx, r.clientSet, nodeName) == nil { return false, fmt.Errorf("node don't exists in cluster: %s", nodeName) } @@ -59,6 +52,12 @@ func (r *RebootManager) IsRebootRequired(ctx context.Context, nodeName string) ( if err != nil { return false, errors.Wrap(err, "failed to create reboot-required pod") } + defer func() { + err := r.clientSet.CoreV1().Pods(r.namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}) + if err != nil { + r.l.Error("failed to delete pod", zap.String("node.name", nodeName), zap.Error(err)) + } + }() err = wait.PollUntilContextTimeout(ctx, time.Second, 30*time.Second, false, mod.IsPodRunning(r.clientSet, pod.GetName(), pod.GetNamespace())) if err != nil { @@ -79,6 +78,7 @@ func (r *RebootManager) IsRebootRequired(ctx context.Context, nodeName string) ( Namespace: pod.Namespace, Name: pod.Name, }, command, nil, stdout, stderr) + r.l.Debug("reboot-required pod output", zap.String("node.name", nodeName), zap.String("stdout", stdout.String()), zap.String("stderr", stderr.String()), zap.Error(err)) if err == nil { rebootRequired = true } @@ -88,25 +88,22 @@ func (r *RebootManager) IsRebootRequired(ctx context.Context, nodeName string) ( func (r *RebootManager) rebootRequiredPod(nodeName string) *corev1.Pod { userId := int64(1000) - t := true - hostPathType := corev1.HostPathDirectory return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "reboot-required-", Namespace: r.namespace, - Labels: map[string]string{"kubenodedrainer.cego.dk/component": "reboot-required"}, + Labels: map[string]string{LabelComponent: "reboot-required"}, }, Spec: corev1.PodSpec{ - Tolerations: []corev1.Toleration{{ - Key: "node-role.kubernetes.io/control-plane", - Operator: corev1.TolerationOpExists, - Effect: corev1.TaintEffectNoSchedule, - }, { - Key: "node.kubernetes.io/unschedulable", - Operator: corev1.TolerationOpExists, - Effect: corev1.TaintEffectNoSchedule, + Volumes: []corev1.Volume{{ + Name: "host-var-run", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/run/", + Type: PtrTo(corev1.HostPathDirectory), + }, + }, }}, - NodeName: nodeName, Containers: []corev1.Container{{ Name: "shell", Image: "alpine", @@ -117,33 +114,28 @@ func (r *RebootManager) rebootRequiredPod(nodeName string) *corev1.Pod { MountPath: "/host/var/run", }}, }}, - RestartPolicy: "Never", + RestartPolicy: "Never", + TerminationGracePeriodSeconds: PtrTo(int64(1)), + NodeName: nodeName, SecurityContext: &corev1.PodSecurityContext{ RunAsUser: &userId, RunAsGroup: &userId, - RunAsNonRoot: &t, + RunAsNonRoot: PtrTo(true), }, - Volumes: []corev1.Volume{{ - Name: "host-var-run", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/run/", - Type: &hostPathType, - }, - }, + Tolerations: []corev1.Toleration{{ + Key: "node-role.kubernetes.io/control-plane", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, { + Key: "node.kubernetes.io/unschedulable", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, }}, }, } } func (r *RebootManager) RebootNode(ctx context.Context, nodeName string) error { - defer func(r *RebootManager, ctx context.Context) { - err := r.cleanup(ctx) - if err != nil { - r.l.Error("failed to cleanup", zap.Error(err)) - } - }(r, ctx) - node := mod.GetNode(ctx, r.clientSet, nodeName) if node == nil { return fmt.Errorf("node don't exists in cluster: %s", nodeName) @@ -153,29 +145,25 @@ func (r *RebootManager) RebootNode(ctx context.Context, nodeName string) error { return fmt.Errorf("node needs to cordoned before reboot: %s", nodeName) } - bootIdOld := node.Status.NodeInfo.BootID + if config.GetConfig().ContainerNode { + r.l.Info("Skipping reboot, because running on containers", zap.String("node.name", nodeName)) + return nil + } _, err := r.clientSet.CoreV1().Pods(r.namespace).Create(ctx, r.rebootNodePod(nodeName), metav1.CreateOptions{}) if err != nil { return errors.Wrap(err, "failed to create reboot pod") } - err = wait.PollUntilContextTimeout(ctx, time.Second, 10*time.Minute, false, mod.IsNodeRebooted(r.l, r.clientSet, nodeName, bootIdOld)) - if err != nil { - return errors.Wrap(err, "node reboot did not complete within 10 minutes") - } - - return err + return nil } func (r *RebootManager) rebootNodePod(nodeName string) *corev1.Pod { - t := true - f := false return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "reboot-", Namespace: r.namespace, - Labels: map[string]string{"kubenodedrainer.cego.dk/component": "reboot"}, + Labels: map[string]string{LabelComponent: "reboot"}, Annotations: map[string]string{"container.apparmor.security.beta.kubernetes.io/shell": "unconfined"}, }, Spec: corev1.PodSpec{ @@ -188,20 +176,21 @@ func (r *RebootManager) rebootNodePod(nodeName string) *corev1.Pod { Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule, }}, - HostPID: true, // Facilitate entering the host mount namespace via init - NodeName: nodeName, + HostPID: true, // Facilitate entering the host mount namespace via init + TerminationGracePeriodSeconds: PtrTo(int64(1)), + NodeName: nodeName, Containers: []corev1.Container{{ Name: "shell", Image: "alpine", - Command: []string{"kill", "-39", "1"}, //kill -SIGRTMIN+5 1 - telling systemd to reboot + Command: []string{"kill", "-39", "1"}, // kill -SIGRTMIN+5 1 - telling systemd to reboot SecurityContext: &corev1.SecurityContext{ Capabilities: &corev1.Capabilities{ - Drop: []corev1.Capability{"*"}, - Add: []corev1.Capability{"CAP_KILL"}, + // Drop: []corev1.Capability{"*"}, + Add: []corev1.Capability{"CAP_KILL"}, }, - AllowPrivilegeEscalation: &f, - Privileged: &f, - ReadOnlyRootFilesystem: &t, + AllowPrivilegeEscalation: PtrTo(false), + Privileged: PtrTo(false), + ReadOnlyRootFilesystem: PtrTo(true), }, }}, RestartPolicy: "Never", @@ -209,25 +198,73 @@ func (r *RebootManager) rebootNodePod(nodeName string) *corev1.Pod { } } -func (r *RebootManager) cleanup(ctx context.Context) error { +func (r *RebootManager) IsNodeRebooted(ctx context.Context, kubeNode *corev1.Node, oldBootId string) (bool, error) { + if config.GetConfig().ContainerNode { + r.l.Info("Node was not rebooted, because running on containers", zap.String("node.name", kubeNode.Name)) + pod := r.rebootRequiredPod(kubeNode.Name) + pod.ObjectMeta.GenerateName = "reboot-required-remove-" + pod.Spec.Containers[0].Command = []string{"rm", "-f", "/host/var/run/reboot-required"} + pod.Spec.Containers[0].VolumeMounts[0].ReadOnly = false + pod.Spec.SecurityContext.RunAsUser = PtrTo(int64(0)) + pod.Spec.SecurityContext.RunAsNonRoot = PtrTo(false) + + pod, err := r.clientSet.CoreV1().Pods(r.namespace).Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + return false, errors.Wrap(err, "failed to create reboot-required-remove pod") + } + defer func() { + err := r.clientSet.CoreV1().Pods(r.namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}) + if err != nil { + r.l.Error("failed to delete pod", zap.String("node.name", kubeNode.Name), zap.Error(err)) + } + }() + + err = wait.PollUntilContextTimeout(ctx, time.Second, 30*time.Second, false, mod.IsPodCompleted(r.clientSet, pod.GetName(), pod.GetNamespace())) + if err != nil { + return false, errors.Wrap(err, "pod did not complete while waiting") + } + + return true, nil + } + + if kubeNode.Status.NodeInfo.BootID != oldBootId { + nodeReady := false + for _, condition := range kubeNode.Status.Conditions { + if condition.Type == corev1.NodeReady { + if condition.Status == corev1.ConditionTrue { + nodeReady = true + } + } + } + return nodeReady, nil + } + + return false, nil +} + +func (r *RebootManager) Cleanup(ctx context.Context) error { + return r.CleanupNode(ctx, "") +} + +func (r *RebootManager) CleanupNode(ctx context.Context, nodeName string) error { var labelSelector labels.Selector = labels.ValidatedSetSelector{} - requirement, err := labels.NewRequirement("kubenodedrainer.cego.dk/component", selection.In, []string{"reboot-required", "reboot"}) + + requirement, err := labels.NewRequirement(LabelComponent, selection.In, []string{"reboot-required", "reboot"}) if err != nil { return err } labelSelector = labelSelector.Add(*requirement) - pods, err := r.clientSet.CoreV1().Pods(r.namespace).List(ctx, metav1.ListOptions{ + options := metav1.ListOptions{ LabelSelector: labelSelector.String(), - }) - if err != nil { - return errors.Wrap(err, "failed to get reboot required pods running on node") } - for _, pod := range pods.Items { - err := r.clientSet.CoreV1().Pods(r.namespace).Delete(ctx, pod.GetName(), metav1.DeleteOptions{}) - if err != nil { - return errors.Wrapf(err, "failed to delete existing pod: %s", pod.GetName()) - } + if nodeName != "" { + options.FieldSelector = fmt.Sprintf("spec.nodeName=%s", nodeName) + } + + err = r.clientSet.CoreV1().Pods(r.namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, options) + if err != nil { + return errors.Wrap(err, "failed to delete pods") } return nil } diff --git a/internal/utils/utils.go b/internal/utils/utils.go new file mode 100644 index 0000000..5270163 --- /dev/null +++ b/internal/utils/utils.go @@ -0,0 +1,16 @@ +package utils + +import "fmt" + +const ( + LabelPrefix = "nodedrain.k8s.slyng.dk" + LabelComponent = LabelPrefix + "/component" +) + +func PtrTo[T any](v T) *T { + return &v +} + +func GetFieldOwner(managerNamespace string) string { + return fmt.Sprintf("%s-manager", managerNamespace) +} diff --git a/internal/webhook/v1/node_webhook.go b/internal/webhook/v1/node_webhook.go new file mode 100644 index 0000000..d336580 --- /dev/null +++ b/internal/webhook/v1/node_webhook.go @@ -0,0 +1,154 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" + "fmt" + + "github.com/slyngdk/node-drain/internal/config" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + drainv1 "github.com/slyngdk/node-drain/api/v1" +) + +// SetupNodeWebhookWithManager registers the webhook for Node in the manager. +func SetupNodeWebhookWithManager(mgr ctrl.Manager) error { + l, err := config.GetNamedLogger("node-webhook") + if err != nil { + return fmt.Errorf("failed to get logger: %w", err) + } + + return ctrl.NewWebhookManagedBy(mgr).For(&drainv1.Node{}). + WithValidator(&NodeCustomValidator{ + l: l, + }). + WithDefaulter(&NodeCustomDefaulter{ + l: l, + client: mgr.GetClient(), + }, admission.DefaulterRemoveUnknownOrOmitableFields). + Complete() + +} + +// +kubebuilder:webhook:path=/mutate-drain-k8s-slyng-dk-v1-node,mutating=true,failurePolicy=fail,sideEffects=None,groups=drain.k8s.slyng.dk,resources=nodes,verbs=create;update,versions=v1,name=mnode-v1.kb.io,admissionReviewVersions=v1 + +// NodeCustomDefaulter struct is responsible for setting default values on the custom resource of the +// Kind Node when those are created or updated. +// +// NOTE: The +kubebuilder:object:generate=false marker prevents controller-gen from generating DeepCopy methods, +// as it is used only for temporary operations and does not need to be deeply copied. +type NodeCustomDefaulter struct { + l *zap.Logger + client client.Client +} + +var _ webhook.CustomDefaulter = &NodeCustomDefaulter{} + +// Default implements webhook.CustomDefaulter so a webhook will be registered for the Kind Node. +func (d *NodeCustomDefaulter) Default(ctx context.Context, obj runtime.Object) error { + newNode, ok := obj.(*drainv1.Node) + if !ok { + return fmt.Errorf("expected an Node object but got %T", obj) + } + l := d.l.With(zap.String("name", newNode.Name)) + l.Debug("Defaulting for Node") + + oldNode := &drainv1.Node{} + if err := d.client.Get(ctx, types.NamespacedName{Name: newNode.Name}, oldNode); err != nil { + l.With(zap.Error(err)).Debug("Error getting Node") + if !errors.IsNotFound(err) { + return fmt.Errorf("could not find node %q: %w", newNode.Name, err) + } + } + + // Set default values + d.applyDefaults(newNode) + return nil +} + +func (d *NodeCustomDefaulter) applyDefaults(node *drainv1.Node) { + if node.Spec.State == "" { + node.Spec.State = drainv1.NodeStateActive + } +} + +// NOTE: The 'path' attribute must follow a specific pattern and should not be modified directly here. +// Modifying the path for an invalid path can cause API server errors; failing to locate the webhook. +// +kubebuilder:webhook:path=/validate-drain-k8s-slyng-dk-v1-node,mutating=false,failurePolicy=fail,sideEffects=None,groups=drain.k8s.slyng.dk,resources=nodes,verbs=create;update,versions=v1,name=vnode-v1.kb.io,admissionReviewVersions=v1 + +// NodeCustomValidator struct is responsible for validating the Node resource +// when it is created, updated, or deleted. +// +// NOTE: The +kubebuilder:object:generate=false marker prevents controller-gen from generating DeepCopy methods, +// as this struct is used only for temporary operations and does not need to be deeply copied. +type NodeCustomValidator struct { + l *zap.Logger +} + +var _ webhook.CustomValidator = &NodeCustomValidator{} + +// ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type Node. +func (v *NodeCustomValidator) ValidateCreate(_ context.Context, obj runtime.Object) (admission.Warnings, error) { + node, ok := obj.(*drainv1.Node) + if !ok { + return nil, fmt.Errorf("expected a Node object but got %T", obj) + } + l := v.l.With(zap.String("name", node.GetName())) + l.Debug("Validation for Node upon creation") + + // TODO(user): fill in your validation logic upon object creation. + + return nil, nil +} + +// ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type Node. +func (v *NodeCustomValidator) ValidateUpdate(_ context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + node, ok := newObj.(*drainv1.Node) + if !ok { + return nil, fmt.Errorf("expected a Node object for the newObj but got %T", newObj) + } + l := v.l.With(zap.String("name", node.GetName())) + l.Debug("Validation for Node upon update") + + // TODO(user): fill in your validation logic upon object update. + + return nil, nil +} + +// ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type Node. +func (v *NodeCustomValidator) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + node, ok := obj.(*drainv1.Node) + if !ok { + return nil, fmt.Errorf("expected a Node object but got %T", obj) + } + l := v.l.With(zap.String("name", node.GetName())) + l.Debug("Validation for Node upon deletion") + + // TODO(user): fill in your validation logic upon object deletion. + + return nil, nil +} diff --git a/internal/webhook/v1/node_webhook_test.go b/internal/webhook/v1/node_webhook_test.go new file mode 100644 index 0000000..56f5439 --- /dev/null +++ b/internal/webhook/v1/node_webhook_test.go @@ -0,0 +1,87 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + drainv1 "github.com/slyngdk/node-drain/api/v1" + // TODO (user): Add any additional imports if needed +) + +var _ = Describe("Node Webhook", func() { + var ( + obj *drainv1.Node + oldObj *drainv1.Node + validator NodeCustomValidator + defaulter NodeCustomDefaulter + ) + + BeforeEach(func() { + obj = &drainv1.Node{} + oldObj = &drainv1.Node{} + validator = NodeCustomValidator{} + Expect(validator).NotTo(BeNil(), "Expected validator to be initialized") + defaulter = NodeCustomDefaulter{} + Expect(defaulter).NotTo(BeNil(), "Expected defaulter to be initialized") + Expect(oldObj).NotTo(BeNil(), "Expected oldObj to be initialized") + Expect(obj).NotTo(BeNil(), "Expected obj to be initialized") + // TODO (user): Add any setup logic common to all tests + }) + + AfterEach(func() { + // TODO (user): Add any teardown logic common to all tests + }) + + Context("When creating Node under Defaulting Webhook", func() { + // TODO (user): Add logic for defaulting webhooks + // Example: + // It("Should apply defaults when a required field is empty", func() { + // By("simulating a scenario where defaults should be applied") + // obj.SomeFieldWithDefault = "" + // By("calling the Default method to apply defaults") + // defaulter.Default(ctx, obj) + // By("checking that the default values are set") + // Expect(obj.SomeFieldWithDefault).To(Equal("default_value")) + // }) + }) + + Context("When creating or updating Node under Validating Webhook", func() { + // TODO (user): Add logic for validating webhooks + // Example: + // It("Should deny creation if a required field is missing", func() { + // By("simulating an invalid creation scenario") + // obj.SomeRequiredField = "" + // Expect(validator.ValidateCreate(ctx, obj)).Error().To(HaveOccurred()) + // }) + // + // It("Should admit creation if all required fields are present", func() { + // By("simulating an invalid creation scenario") + // obj.SomeRequiredField = "valid_value" + // Expect(validator.ValidateCreate(ctx, obj)).To(BeNil()) + // }) + // + // It("Should validate updates correctly", func() { + // By("simulating a valid update scenario") + // oldObj.SomeRequiredField = "updated_value" + // obj.SomeRequiredField = "updated_value" + // Expect(validator.ValidateUpdate(ctx, oldObj, obj)).To(BeNil()) + // }) + }) + +}) diff --git a/internal/webhook/v1/webhook_suite_test.go b/internal/webhook/v1/webhook_suite_test.go new file mode 100644 index 0000000..54cb9cc --- /dev/null +++ b/internal/webhook/v1/webhook_suite_test.go @@ -0,0 +1,164 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "os" + "path/filepath" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + drainv1 "github.com/slyngdk/node-drain/api/v1" + // +kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + ctx context.Context + cancel context.CancelFunc + k8sClient client.Client + cfg *rest.Config + testEnv *envtest.Environment +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Webhook Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + var err error + err = drainv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:scheme + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: false, + + WebhookInstallOptions: envtest.WebhookInstallOptions{ + Paths: []string{filepath.Join("..", "..", "..", "config", "webhook")}, + }, + } + + // Retrieve the first found binary directory to allow running tests from IDEs + if getFirstFoundEnvTestBinaryDir() != "" { + testEnv.BinaryAssetsDirectory = getFirstFoundEnvTestBinaryDir() + } + + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + // start webhook server using Manager. + webhookInstallOptions := &testEnv.WebhookInstallOptions + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: webhookInstallOptions.LocalServingHost, + Port: webhookInstallOptions.LocalServingPort, + CertDir: webhookInstallOptions.LocalServingCertDir, + }), + LeaderElection: false, + Metrics: metricsserver.Options{BindAddress: "0"}, + }) + Expect(err).NotTo(HaveOccurred()) + + err = SetupNodeWebhookWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:webhook + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() + + // wait for the webhook server to get ready. + dialer := &net.Dialer{Timeout: time.Second} + addrPort := fmt.Sprintf("%s:%d", webhookInstallOptions.LocalServingHost, webhookInstallOptions.LocalServingPort) + Eventually(func() error { + conn, err := tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true}) + if err != nil { + return err + } + + return conn.Close() + }).Should(Succeed()) +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +// getFirstFoundEnvTestBinaryDir locates the first binary in the specified path. +// ENVTEST-based tests depend on specific binaries, usually located in paths set by +// controller-runtime. When running tests directly (e.g., via an IDE) without using +// Makefile targets, the 'BinaryAssetsDirectory' must be explicitly configured. +// +// This function streamlines the process by finding the required binaries, similar to +// setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are +// properly set up, run 'make setup-envtest' beforehand. +func getFirstFoundEnvTestBinaryDir() string { + basePath := filepath.Join("..", "..", "..", "bin", "k8s") + entries, err := os.ReadDir(basePath) + if err != nil { + logf.Log.Error(err, "Failed to read directory", "path", basePath) + return "" + } + for _, entry := range entries { + if entry.IsDir() { + return filepath.Join(basePath, entry.Name()) + } + } + return "" +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index ebd8a51..bf191d1 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -1,5 +1,5 @@ /* -Copyright 2024. +Copyright 2025. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,15 +18,81 @@ package e2e import ( "fmt" + "os" + "os/exec" "testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + "github.com/slyngdk/node-drain/test/utils" +) + +var ( + // Optional Environment Variables: + // - CERT_MANAGER_INSTALL_SKIP=true: Skips CertManager installation during test setup. + // These variables are useful if CertManager is already installed, avoiding + // re-installation and conflicts. + skipCertManagerInstall = os.Getenv("CERT_MANAGER_INSTALL_SKIP") == "true" + // isCertManagerAlreadyInstalled will be set true when CertManager CRDs be found on the cluster + isCertManagerAlreadyInstalled = false + + projectRegistry = "example.com/" + projectControllerImage = "nodedrain" + projectImageTag = "v0.0.1" + makeImageVars = func(arg string) []string { + return []string{ + arg, + fmt.Sprintf("IMG_REGISTRY=%s", projectRegistry), + fmt.Sprintf("IMG_NAME_CONTROLLER=%s", projectControllerImage), + fmt.Sprintf("IMG_TAG=%s", projectImageTag), + } + } ) -// Run e2e tests using the Ginkgo runner. +// TestE2E runs the end-to-end (e2e) test suite for the project. These tests execute in an isolated, +// temporary environment to validate project changes with the the purposed to be used in CI jobs. +// The default setup requires Kind, builds/loads the Manager Docker image locally, and installs +// CertManager. func TestE2E(t *testing.T) { RegisterFailHandler(Fail) - _, _ = fmt.Fprintf(GinkgoWriter, "Starting nodedrain suite\n") + _, _ = fmt.Fprintf(GinkgoWriter, "Starting nodedrain integration test suite\n") RunSpecs(t, "e2e suite") } + +var _ = BeforeSuite(func() { + By("building the manager(Operator) image") + cmd := exec.Command("make", makeImageVars("docker-build")...) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager(Operator) image") + + // TODO(user): If you want to change the e2e test vendor from Kind, ensure the image is + // built and available before running the tests. Also, remove the following block. + By("loading the manager(Operator) image on Kind") + err = utils.LoadImageToKindClusterWithName( + fmt.Sprintf("%s%s:%s", projectRegistry, projectControllerImage, projectImageTag)) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager(Operator) image into Kind") + + // The tests-e2e are intended to run on a temporary cluster that is created and destroyed for testing. + // To prevent errors when tests run in environments with CertManager already installed, + // we check for its presence before execution. + // Setup CertManager before the suite if not skipped and if not already installed + if !skipCertManagerInstall { + By("checking if cert manager is installed already") + isCertManagerAlreadyInstalled = utils.IsCertManagerCRDsInstalled() + if !isCertManagerAlreadyInstalled { + _, _ = fmt.Fprintf(GinkgoWriter, "Installing CertManager...\n") + Expect(utils.InstallCertManager()).To(Succeed(), "Failed to install CertManager") + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "WARNING: CertManager is already installed. Skipping installation...\n") + } + } +}) + +var _ = AfterSuite(func() { + // Teardown CertManager after the suite if not skipped and if it was not already installed + if !skipCertManagerInstall && !isCertManagerAlreadyInstalled { + _, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling CertManager...\n") + utils.UninstallCertManager() + } +}) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index d5ff7d1..02e6d50 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -1,5 +1,5 @@ /* -Copyright 2024. +Copyright 2025. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,8 +17,11 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" "time" . "github.com/onsi/ginkgo/v2" @@ -27,65 +30,132 @@ import ( "github.com/slyngdk/node-drain/test/utils" ) +// namespace where the project is deployed in const namespace = "nodedrain-system" -var _ = Describe("controller", Ordered, func() { - BeforeAll(func() { - By("installing prometheus operator") - Expect(utils.InstallPrometheusOperator()).To(Succeed()) +// serviceAccountName created for the project +const serviceAccountName = "nodedrain-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "nodedrain-controller-manager-metrics-service" - By("installing the cert-manager") - Expect(utils.InstallCertManager()).To(Succeed()) +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "nodedrain-metrics-binding" +var _ = Describe("Manager", Ordered, func() { + var controllerPodName string + + // Before running the tests, set up the environment by creating the namespace, + // enforce the restricted security policy to the namespace, installing CRDs, + // and deploying the controller. + BeforeAll(func() { By("creating manager namespace") cmd := exec.Command("kubectl", "create", "ns", namespace) - _, _ = utils.Run(cmd) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create namespace") + + // FIXME + // By("labeling the namespace to enforce the restricted security policy") + // cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, + // "pod-security.kubernetes.io/enforce=restricted") + // _, err = utils.Run(cmd) + // Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") + + By("installing CRDs") + cmd = exec.Command("make", "install") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") + + By("deploying the controller-manager") + cmd = exec.Command("make", makeImageVars("deploy")...) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") }) + // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, + // and deleting the namespace. AfterAll(func() { - By("uninstalling the Prometheus manager bundle") - utils.UninstallPrometheusOperator() + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) + + By("delete nodes.drain.k8s.slyng.dk") + cmd = exec.Command("kubectl", "delete", "--all", "nodes.drain.k8s.slyng.dk") + _, _ = utils.Run(cmd) + + By("wait for delete nodes.drain.k8s.slyng.dk") + cmd = exec.Command("kubectl", "wait", "--for=delete", "--all", "nodes.drain.k8s.slyng.dk") + _, _ = utils.Run(cmd) + + By("undeploying the controller-manager") + cmd = exec.Command("make", "undeploy") + _, _ = utils.Run(cmd) - By("uninstalling the cert-manager bundle") - utils.UninstallCertManager() + By("uninstalling CRDs") + cmd = exec.Command("make", "uninstall") + _, _ = utils.Run(cmd) By("removing manager namespace") - cmd := exec.Command("kubectl", "delete", "ns", namespace) + cmd = exec.Command("kubectl", "delete", "ns", namespace) _, _ = utils.Run(cmd) - }) - Context("Operator", func() { - It("should run successfully", func() { - var controllerPodName string - var err error + By("removing cluster role binding for metrics test") + cmd = exec.Command("kubectl", "delete", "clusterrolebinding", metricsRoleBindingName) + _, _ = utils.Run(cmd) + }) - // projectimage stores the name of the image used in the example - var projectimage = "example.com/nodedrain:v0.0.1" + // After each test, check for failures and collect logs, events, + // and pod descriptions for debugging. + AfterEach(func() { + specReport := CurrentSpecReport() + if specReport.Failed() { + By("Fetching controller manager pod logs") + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + controllerLogs, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err) + } - By("building the manager(Operator) image") - cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + By("Fetching Kubernetes events") + cmd = exec.Command("kubectl", "get", "events", "-n", namespace, "--sort-by=.lastTimestamp") + eventsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events:\n%s", eventsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Kubernetes events: %s", err) + } - By("loading the the manager(Operator) image on Kind") - err = utils.LoadImageToKindClusterWithName(projectimage) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + By("Fetching curl-metrics logs") + cmd = exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Metrics logs:\n %s", metricsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get curl-metrics logs: %s", err) + } - By("installing CRDs") - cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + By("Fetching controller manager pod description") + cmd = exec.Command("kubectl", "describe", "pod", controllerPodName, "-n", namespace) + podDescription, err := utils.Run(cmd) + if err == nil { + fmt.Println("Pod description:\n", podDescription) + } else { + fmt.Println("Failed to describe controller pod") + } + } + }) - By("deploying the controller-manager") - cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + Context("Manager", func() { + It("should run successfully", func() { By("validating that the controller-manager pod is running as expected") - verifyControllerUp := func() error { - // Get pod name - - cmd = exec.Command("kubectl", "get", + verifyControllerUp := func(g Gomega) { + // Get the name of the controller-manager pod + cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", "-o", "go-template={{ range .items }}"+ "{{ if not .metadata.deletionTimestamp }}"+ @@ -95,28 +165,216 @@ var _ = Describe("controller", Ordered, func() { ) podOutput, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred()) - podNames := utils.GetNonEmptyLines(string(podOutput)) - if len(podNames) != 1 { - return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames)) - } + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + podNames := utils.GetNonEmptyLines(podOutput) + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") controllerPodName = podNames[0] - ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager")) + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) - // Validate pod status + // Validate the pod's status cmd = exec.Command("kubectl", "get", "pods", controllerPodName, "-o", "jsonpath={.status.phase}", "-n", namespace, ) - status, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred()) - if string(status) != "Running" { - return fmt.Errorf("controller pod in %s status", status) - } - return nil + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Running"), "Incorrect controller-manager pod status") + } + Eventually(verifyControllerUp).Should(Succeed()) + }) + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=nodedrain-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("\"msg\":\"Serving metrics server\""), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted).Should(Succeed()) + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:latest", + "--overrides", + fmt.Sprintf(`{ + "spec": { + "containers": [{ + "name": "curl", + "image": "curlimages/curl:latest", + "command": ["/bin/sh", "-c"], + "args": ["curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics"], + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + }, + "runAsNonRoot": true, + "runAsUser": 1000, + "seccompProfile": { + "type": "RuntimeDefault" + } + } + }], + "serviceAccount": "%s" + } + }`, token, metricsServiceName, namespace, serviceAccountName)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + Expect(metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + }) + + It("should provisioned cert-manager", func() { + By("validating that cert-manager has the certificate Secret") + verifyCertManager := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "secrets", "webhook-server-cert", "-n", namespace) + _, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) } - EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) + Eventually(verifyCertManager).Should(Succeed()) + }) + + It("should have CA injection for mutating webhooks", func() { + By("checking CA injection for mutating webhooks") + verifyCAInjection := func(g Gomega) { + cmd := exec.Command("kubectl", "get", + "mutatingwebhookconfigurations.admissionregistration.k8s.io", + "nodedrain-mutating-webhook-configuration", + "-o", "go-template={{ range .webhooks }}{{ .clientConfig.caBundle }}{{ end }}") + mwhOutput, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(mwhOutput)).To(BeNumerically(">", 10)) + } + Eventually(verifyCAInjection).Should(Succeed()) + }) + It("should have CA injection for validating webhooks", func() { + By("checking CA injection for validating webhooks") + verifyCAInjection := func(g Gomega) { + cmd := exec.Command("kubectl", "get", + "validatingwebhookconfigurations.admissionregistration.k8s.io", + "nodedrain-validating-webhook-configuration", + "-o", "go-template={{ range .webhooks }}{{ .clientConfig.caBundle }}{{ end }}") + vwhOutput, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(vwhOutput)).To(BeNumerically(">", 10)) + } + Eventually(verifyCAInjection).Should(Succeed()) }) + + // +kubebuilder:scaffold:e2e-webhooks-checks + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput := getMetricsOutput() + // Expect(metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + verifyTokenCreation := func(g Gomega) { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal(output, &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + Expect(metricsOutput).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutput +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/test/utils/utils.go b/test/utils/utils.go index 6b96ab5..440fb5e 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -1,5 +1,5 @@ /* -Copyright 2024. +Copyright 2025. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,53 +17,55 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" "strings" - . "github.com/onsi/ginkgo/v2" //nolint:golint,revive + "github.com/onsi/ginkgo/v2" ) const ( - prometheusOperatorVersion = "v0.72.0" + prometheusOperatorVersion = "v0.77.1" prometheusOperatorURL = "https://github.com/prometheus-operator/prometheus-operator/" + "releases/download/%s/bundle.yaml" - certmanagerVersion = "v1.14.4" - certmanagerURLTmpl = "https://github.com/jetstack/cert-manager/releases/download/%s/cert-manager.yaml" + certmanagerVersion = "v1.16.3" + certmanagerURLTmpl = "https://github.com/cert-manager/cert-manager/releases/download/%s/cert-manager.yaml" ) func warnError(err error) { - _, _ = fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) -} - -// InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. -func InstallPrometheusOperator() error { - url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) - cmd := exec.Command("kubectl", "create", "-f", url) - _, err := Run(cmd) - return err + _, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "warning: %v\n", err) } // Run executes the provided command within this context -func Run(cmd *exec.Cmd) ([]byte, error) { +func Run(cmd *exec.Cmd) (string, error) { dir, _ := GetProjectDir() cmd.Dir = dir if err := os.Chdir(cmd.Dir); err != nil { - _, _ = fmt.Fprintf(GinkgoWriter, "chdir dir: %s\n", err) + _, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "chdir dir: %s\n", err) } cmd.Env = append(os.Environ(), "GO111MODULE=on") command := strings.Join(cmd.Args, " ") - _, _ = fmt.Fprintf(GinkgoWriter, "running: %s\n", command) + _, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "running: %s\n", command) output, err := cmd.CombinedOutput() if err != nil { - return output, fmt.Errorf("%s failed with error: (%v) %s", command, err, string(output)) + return string(output), fmt.Errorf("%s failed with error: (%v) %s", command, err, string(output)) } - return output, nil + return string(output), nil +} + +// InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. +func InstallPrometheusOperator() error { + url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) + cmd := exec.Command("kubectl", "create", "-f", url) + _, err := Run(cmd) + return err } // UninstallPrometheusOperator uninstalls the prometheus @@ -75,6 +77,33 @@ func UninstallPrometheusOperator() { } } +// IsPrometheusCRDsInstalled checks if any Prometheus CRDs are installed +// by verifying the existence of key CRDs related to Prometheus. +func IsPrometheusCRDsInstalled() bool { + // List of common Prometheus CRDs + prometheusCRDs := []string{ + "prometheuses.monitoring.coreos.com", + "prometheusrules.monitoring.coreos.com", + "prometheusagents.monitoring.coreos.com", + } + + cmd := exec.Command("kubectl", "get", "crds", "-o", "custom-columns=NAME:.metadata.name") + output, err := Run(cmd) + if err != nil { + return false + } + crdList := GetNonEmptyLines(output) + for _, crd := range prometheusCRDs { + for _, line := range crdList { + if strings.Contains(line, crd) { + return true + } + } + } + + return false +} + // UninstallCertManager uninstalls the cert manager func UninstallCertManager() { url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) @@ -103,6 +132,39 @@ func InstallCertManager() error { return err } +// IsCertManagerCRDsInstalled checks if any Cert Manager CRDs are installed +// by verifying the existence of key CRDs related to Cert Manager. +func IsCertManagerCRDsInstalled() bool { + // List of common Cert Manager CRDs + certManagerCRDs := []string{ + "certificates.cert-manager.io", + "issuers.cert-manager.io", + "clusterissuers.cert-manager.io", + "certificaterequests.cert-manager.io", + "orders.acme.cert-manager.io", + "challenges.acme.cert-manager.io", + } + + // Execute the kubectl command to get all CRDs + cmd := exec.Command("kubectl", "get", "crds") + output, err := Run(cmd) + if err != nil { + return false + } + + // Check if any of the Cert Manager CRDs are present + crdList := GetNonEmptyLines(output) + for _, crd := range certManagerCRDs { + for _, line := range crdList { + if strings.Contains(line, crd) { + return true + } + } + } + + return false +} + // LoadImageToKindClusterWithName loads a local docker image to the kind cluster func LoadImageToKindClusterWithName(name string) error { cluster := "kind" @@ -135,6 +197,55 @@ func GetProjectDir() (string, error) { if err != nil { return wd, err } - wd = strings.Replace(wd, "/test/e2e", "", -1) + wd = strings.ReplaceAll(wd, "/test/e2e", "") return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return err + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %s to be uncomment", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + _, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)) + if err != nil { + return err + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err := out.WriteString("\n"); err != nil { + return err + } + } + + _, err = out.Write(content[idx+len(target):]) + if err != nil { + return err + } + // false positive + // nolint:gosec + return os.WriteFile(filename, out.Bytes(), 0644) +}