Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions imagevector/containers.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ images:
- name: vpn-server
sourceRepository: github.com/gardener/vpn2
repository: europe-docker.pkg.dev/gardener-project/releases/gardener/vpn-server
tag: "0.51.0"
tag: "0.52.0"
# OpenTelemetry
- name: opentelemetry-operator
sourceRepository: github.com/open-telemetry/opentelemetry-operator
Expand Down Expand Up @@ -480,7 +480,7 @@ images:
- name: vpn-client
sourceRepository: github.com/gardener/vpn2
repository: europe-docker.pkg.dev/gardener-project/releases/gardener/vpn-client
tag: "0.51.0"
tag: "0.52.0"
- name: coredns
sourceRepository: github.com/coredns/coredns
repository: registry.k8s.io/coredns/coredns
Expand Down
1 change: 1 addition & 0 deletions pkg/apis/config/gardenlet/v1alpha1/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -525,6 +525,7 @@ type VictoriaLogs struct {
type GardenVictoriaLogs struct {
// Storage is the disk storage capacity of VictoriaLogs.
// Defaults to 100Gi.
// If pvc-autoscaler is enabled, this value is ignored and the initial size is set to 5Gi.
// +optional
Storage *resource.Quantity `json:"storage,omitempty" yaml:"storage,omitempty"`
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
apiVersion: v1
kind: ConfigMap
metadata:
labels:
{{ .Values.labels | toYaml | indent 4 }}
name: istio-ingressgateway-bootstrap-override
namespace: {{ .Release.Namespace }}
data:
bootstrap.yaml: |
overload_manager:
refresh_interval: 1s
resource_monitors:
- name: envoy.resource_monitors.cpu_utilization
typed_config:
"@type": type.googleapis.com/envoy.extensions.resource_monitors.cpu_utilization.v3.CpuUtilizationConfig
actions:
- name: envoy.overload_actions.disable_http_keepalive
triggers:
- name: envoy.resource_monitors.cpu_utilization
threshold:
value: 0.85
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ spec:
service.istio.io/canonical-name: "istio-ingressgateway"
service.istio.io/canonical-revision: "1.29"
annotations:
checksum/bootstrap-override: {{ include (print $.Template.BasePath "/bootstrap-override.yaml") . | sha256sum }}
sidecar.istio.io/inject: "false"
# Evict stale metrics (e.g. for old kube-apiserver pods) after inactivity to prevent unbounded metric growth.
# Replaces the deprecated METRIC_ROTATION_INTERVAL env var removed in Istio v1.28.
Expand All @@ -42,6 +43,9 @@ spec:
# See also https://github.com/envoyproxy/envoy/security/advisories/GHSA-jhv4-f7mr-xx76
"overload.premature_reset_total_stream_count": "100"
"overload.premature_reset_min_stream_lifetime_seconds": "1"
proxyStatsMatcher:
inclusionRegexps:
- ".*goaway.*"
spec:
serviceAccountName: istio-ingressgateway-service-account
securityContext:
Expand Down Expand Up @@ -90,6 +94,8 @@ spec:
env:
- name: PILOT_CERT_PROVIDER
value: istiod
- name: ISTIO_BOOTSTRAP_OVERRIDE
value: /etc/istio/bootstrap-override/bootstrap.yaml
- name: CA_ADDR
value: istiod.{{ .Values.istiodNamespace }}.svc:15012
- name: NODE_NAME
Expand Down Expand Up @@ -158,6 +164,9 @@ spec:
mountPath: /etc/istio/proxy
- name: config-volume
mountPath: /etc/istio/config
- name: bootstrap-override-volume
mountPath: /etc/istio/bootstrap-override
readOnly: true
- mountPath: /var/run/secrets/istio
name: istiod-ca-cert
- name: istio-token
Expand Down Expand Up @@ -207,6 +216,9 @@ spec:
configMap:
name: istio
optional: true
- name: bootstrap-override-volume
configMap:
name: istio-ingressgateway-bootstrap-override
- name: ingressgateway-certs
secret:
secretName: "istio-ingressgateway-certs"
Expand Down
15 changes: 12 additions & 3 deletions pkg/component/networking/istio/istio_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ var _ = Describe("istiod", func() {
const (
deployNS = "test"
deployNSIngress = "test-ingress"

ingressBootstrapOverrideChecksum = "7da3338166d3dcdbab977651cf2e809b70b20c8a6d8cab00f1521fc4172b6939"
)

var (
Expand Down Expand Up @@ -228,10 +230,16 @@ var _ = Describe("istiod", func() {
return string(data)
}

istioIngressDeployment = func(replicas *int) string {
istioIngressDeployment = func(replicas *int, checksum string) string {
data, _ := os.ReadFile("./test_charts/ingress_deployment.yaml")
str := strings.ReplaceAll(string(data), "<REPLICAS>", strconv.Itoa(ptr.Deref(replicas, 2)))
return strings.ReplaceAll(str, "<CPU_REQUESTS>", expectedCPURequests)
str = strings.ReplaceAll(str, "<CPU_REQUESTS>", expectedCPURequests)
return strings.ReplaceAll(str, "<CHECKSUM>", checksum)
}

istioIngressBootstrapOverride = func() string {
data, _ := os.ReadFile("./test_charts/ingress_bootstrap_override.yaml")
return string(data)
}

istioIngressServiceMonitor = func() string {
Expand Down Expand Up @@ -408,7 +416,8 @@ var _ = Describe("istiod", func() {
istioIngressRoleBinding(),
istioIngressServiceInternal(),
istioIngressServiceAccount(),
istioIngressDeployment(minReplicas),
istioIngressDeployment(minReplicas, ingressBootstrapOverrideChecksum),
istioIngressBootstrapOverride(),
istioIngressEnvoyFilter(),
istioIngressMisdirectedRequestsEnvoyFilter(),
istioIngressServiceMonitor(),
Expand Down
1 change: 1 addition & 0 deletions pkg/component/networking/istio/istiod.go
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,7 @@ func (i *istiod) Deploy(ctx context.Context) error {
"envoy_cluster_upstream_cx_rx_bytes_total",
"envoy_cluster_upstream_cx_total",
"envoy_cluster_upstream_cx_tx_bytes_total",
"envoy_http2_goaway_sent",
"envoy_server_hot_restart_epoch",
"istio_build",
"istio_request_bytes_count",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
apiVersion: v1
kind: ConfigMap
metadata:
labels:
app: istio-ingressgateway
foo: bar
name: istio-ingressgateway-bootstrap-override
namespace: test-ingress
data:
bootstrap.yaml: |
overload_manager:
refresh_interval: 1s
resource_monitors:
- name: envoy.resource_monitors.cpu_utilization
typed_config:
"@type": type.googleapis.com/envoy.extensions.resource_monitors.cpu_utilization.v3.CpuUtilizationConfig
actions:
- name: envoy.overload_actions.disable_http_keepalive
triggers:
- name: envoy.resource_monitors.cpu_utilization
threshold:
value: 0.85
12 changes: 12 additions & 0 deletions pkg/component/networking/istio/test_charts/ingress_deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ spec:
service.istio.io/canonical-name: "istio-ingressgateway"
service.istio.io/canonical-revision: "1.29"
annotations:
checksum/bootstrap-override: <CHECKSUM>
sidecar.istio.io/inject: "false"
# Evict stale metrics (e.g. for old kube-apiserver pods) after inactivity to prevent unbounded metric growth.
# Replaces the deprecated METRIC_ROTATION_INTERVAL env var removed in Istio v1.28.
Expand All @@ -42,6 +43,9 @@ spec:
# See also https://github.com/envoyproxy/envoy/security/advisories/GHSA-jhv4-f7mr-xx76
"overload.premature_reset_total_stream_count": "100"
"overload.premature_reset_min_stream_lifetime_seconds": "1"
proxyStatsMatcher:
inclusionRegexps:
- ".*goaway.*"
spec:
serviceAccountName: istio-ingressgateway-service-account
securityContext:
Expand Down Expand Up @@ -88,6 +92,8 @@ spec:
env:
- name: PILOT_CERT_PROVIDER
value: istiod
- name: ISTIO_BOOTSTRAP_OVERRIDE
value: /etc/istio/bootstrap-override/bootstrap.yaml
- name: CA_ADDR
value: istiod.istio-test-system.svc:15012
- name: NODE_NAME
Expand Down Expand Up @@ -152,6 +158,9 @@ spec:
mountPath: /etc/istio/proxy
- name: config-volume
mountPath: /etc/istio/config
- name: bootstrap-override-volume
mountPath: /etc/istio/bootstrap-override
readOnly: true
- mountPath: /var/run/secrets/istio
name: istiod-ca-cert
- name: istio-token
Expand Down Expand Up @@ -201,6 +210,9 @@ spec:
configMap:
name: istio
optional: true
- name: bootstrap-override-volume
configMap:
name: istio-ingressgateway-bootstrap-override
- name: ingressgateway-certs
secret:
secretName: "istio-ingressgateway-certs"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ spec:
endpoints:
- metricRelabelings:
- action: keep
regex: ^(envoy_cluster_upstream_cx_active|envoy_cluster_upstream_cx_connect_fail|envoy_cluster_upstream_cx_rx_bytes_total|envoy_cluster_upstream_cx_total|envoy_cluster_upstream_cx_tx_bytes_total|envoy_server_hot_restart_epoch|istio_build|istio_request_bytes_count|istio_request_bytes_sum|istio_request_duration_milliseconds_count|istio_request_duration_milliseconds_sum|istio_requests_total|istio_response_bytes_count|istio_response_bytes_sum|istio_tcp_connections_closed_total|istio_tcp_connections_opened_total|istio_tcp_received_bytes_total|istio_tcp_sent_bytes_total)$
regex: ^(envoy_cluster_upstream_cx_active|envoy_cluster_upstream_cx_connect_fail|envoy_cluster_upstream_cx_rx_bytes_total|envoy_cluster_upstream_cx_total|envoy_cluster_upstream_cx_tx_bytes_total|envoy_http2_goaway_sent|envoy_server_hot_restart_epoch|istio_build|istio_request_bytes_count|istio_request_bytes_sum|istio_request_duration_milliseconds_count|istio_request_duration_milliseconds_sum|istio_requests_total|istio_response_bytes_count|istio_response_bytes_sum|istio_tcp_connections_closed_total|istio_tcp_connections_opened_total|istio_tcp_received_bytes_total|istio_tcp_sent_bytes_total)$
sourceLabels:
- __name__
path: /stats/prometheus
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -158,37 +158,13 @@ func (v *victoriaOperator) deployment() *appsv1.Deployment {
"--leader-elect",
fmt.Sprintf("--health-probe-bind-address=:%d", healthProbePort),
fmt.Sprintf("--metrics-bind-address=:%d", metricsPort),
"--controller.disableReconcileFor=VLAgent,VLCluster,VLogs,VMAgent,VMAlert,VMAlertmanager,VMAlertmanagerConfig,VMAnomaly,VMAuth,VMCluster,VMNodeScrape,VMPodScrape,VMProbe,VMRule,VMScrapeConfig,VMServiceScrape,VMSingle,VMStaticScrape,VMUser,VTSingle,VTCluster",
"--controller.disableReconcileFor=VLAgent,VLCluster,VLogs,VMAgent,VMAlert,VMAlertmanager,VMAlertmanagerConfig,VMAnomaly,VMAuth,VMCluster,VMNodeScrape,VMPodScrape,VMProbe,VMRule,VMScrapeConfig,VMServiceScrape,VMSingle,VMStaticScrape,VMUser,VTSingle,VTCluster,PodMonitor,ServiceMonitor,PrometheusRule,Probe,AlertmanagerConfig,ScrapeConfig",
},
Env: []corev1.EnvVar{
{
Name: "WATCH_NAMESPACE",
Value: "",
},
{
Name: "VM_ENABLEDPROMETHEUSCONVERTER_PODMONITOR",
Value: "false",
},
{
Name: "VM_ENABLEDPROMETHEUSCONVERTER_SERVICESCRAPE",
Value: "false",
},
{
Name: "VM_ENABLEDPROMETHEUSCONVERTER_PROMETHEUSRULE",
Value: "false",
},
{
Name: "VM_ENABLEDPROMETHEUSCONVERTER_PROBE",
Value: "false",
},
{
Name: "VM_ENABLEDPROMETHEUSCONVERTER_ALERTMANAGERCONFIG",
Value: "false",
},
{
Name: "VM_ENABLEDPROMETHEUSCONVERTER_SCRAPECONFIG",
Value: "false",
},
{
Name: "VM_DISABLESELFSERVICESCRAPECREATION",
Value: "true",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -158,37 +158,13 @@ var _ = Describe("VictoriaOperator", func() {
"--leader-elect",
"--health-probe-bind-address=:8081",
"--metrics-bind-address=:8080",
"--controller.disableReconcileFor=VLAgent,VLCluster,VLogs,VMAgent,VMAlert,VMAlertmanager,VMAlertmanagerConfig,VMAnomaly,VMAuth,VMCluster,VMNodeScrape,VMPodScrape,VMProbe,VMRule,VMScrapeConfig,VMServiceScrape,VMSingle,VMStaticScrape,VMUser,VTSingle,VTCluster",
"--controller.disableReconcileFor=VLAgent,VLCluster,VLogs,VMAgent,VMAlert,VMAlertmanager,VMAlertmanagerConfig,VMAnomaly,VMAuth,VMCluster,VMNodeScrape,VMPodScrape,VMProbe,VMRule,VMScrapeConfig,VMServiceScrape,VMSingle,VMStaticScrape,VMUser,VTSingle,VTCluster,PodMonitor,ServiceMonitor,PrometheusRule,Probe,AlertmanagerConfig,ScrapeConfig",
},
Env: []corev1.EnvVar{
{
Name: "WATCH_NAMESPACE",
Value: "",
},
{
Name: "VM_ENABLEDPROMETHEUSCONVERTER_PODMONITOR",
Value: "false",
},
{
Name: "VM_ENABLEDPROMETHEUSCONVERTER_SERVICESCRAPE",
Value: "false",
},
{
Name: "VM_ENABLEDPROMETHEUSCONVERTER_PROMETHEUSRULE",
Value: "false",
},
{
Name: "VM_ENABLEDPROMETHEUSCONVERTER_PROBE",
Value: "false",
},
{
Name: "VM_ENABLEDPROMETHEUSCONVERTER_ALERTMANAGERCONFIG",
Value: "false",
},
{
Name: "VM_ENABLEDPROMETHEUSCONVERTER_SCRAPECONFIG",
Value: "false",
},
{
Name: "VM_DISABLESELFSERVICESCRAPECREATION",
Value: "true",
Expand Down
11 changes: 10 additions & 1 deletion pkg/component/observability/logging/victorialogs/victorialogs.go
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,15 @@ func (v *victoriaLogs) vlSingle() *victoriametricsv1.VLSingle {
},
},
},
// This annotation tells the VictoriaMetrics operator not to attempt resizing the PVC itself,
// if the CR's storage field is mutated. PVC resizing is instead owned by the dedicated
// pvc-autoscaler component. If pvc-autoscaler gets disabled, the PVC will not be resized
// to the storage size specified in the VLSingle CR as long as this annotation is still present.
StorageMetadata: victoriametricsv1beta1.EmbeddedObjectMetadata{
Annotations: map[string]string{
"operator.victoriametrics.com/pvc-allow-volume-expansion": "false",
},
},
ServiceSpec: &victoriametricsv1beta1.AdditionalServiceSpec{
EmbeddedObjectMetadata: victoriametricsv1beta1.EmbeddedObjectMetadata{
Name: constants.ServiceName,
Expand Down Expand Up @@ -271,7 +280,7 @@ func (v *victoriaLogs) getPVCA(pvcAutoscaling PVCAutoscalingConfig) *pvcautoscal
ScaleUp: &pvcautoscalerv1alpha1.ScalingRules{
UtilizationThresholdPercent: new(70),
StepPercent: new(10),
MinStepAbsolute: new(resource.MustParse("1Gi")),
MinStepAbsolute: new(resource.MustParse("2Gi")),
},
},
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,11 @@ var _ = Describe("VictoriaLogs", func() {
},
},
},
StorageMetadata: victoriametricsv1beta1.EmbeddedObjectMetadata{
Annotations: map[string]string{
"operator.victoriametrics.com/pvc-allow-volume-expansion": "false",
},
},
ServiceSpec: &victoriametricsv1beta1.AdditionalServiceSpec{
EmbeddedObjectMetadata: victoriametricsv1beta1.EmbeddedObjectMetadata{
Name: "logging-vl",
Expand Down Expand Up @@ -554,7 +559,7 @@ func getPVCA(maxCapacity resource.Quantity) *pvcautoscalerv1alpha1.PersistentVol
ScaleUp: &pvcautoscalerv1alpha1.ScalingRules{
UtilizationThresholdPercent: new(70),
StepPercent: new(10),
MinStepAbsolute: new(resource.MustParse("1Gi")),
MinStepAbsolute: new(resource.MustParse("2Gi")),
},
},
},
Expand Down
11 changes: 8 additions & 3 deletions pkg/gardenlet/controller/seed/seed/components.go
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ func (r *Reconciler) instantiateComponents(
if err != nil {
return
}
c.victoriaLogs, err = r.newVictoriaLogs(seed.GetInfo().Spec.Settings)
c.victoriaLogs, err = r.newVictoriaLogs(seed)
if err != nil {
return
}
Expand Down Expand Up @@ -625,12 +625,17 @@ func (r *Reconciler) newVali(seed *seedpkg.Seed, istioIngressGatewayLabels map[s
return deployer, err
}

func (r *Reconciler) newVictoriaLogs(seedSettings *gardencorev1beta1.SeedSettings) (component.DeployWaiter, error) {
func (r *Reconciler) newVictoriaLogs(seed *seedpkg.Seed) (component.DeployWaiter, error) {
var storage *resource.Quantity
if r.Config.Logging != nil && r.Config.Logging.VictoriaLogs != nil && r.Config.Logging.VictoriaLogs.Garden != nil {
storage = r.Config.Logging.VictoriaLogs.Garden.Storage
}

pvcAutoscalerEnabled := v1beta1helper.SeedSettingPersistentVolumeClaimAutoscalerEnabled(seed.GetInfo().Spec.Settings)
if pvcAutoscalerEnabled {
storage = new(resource.MustParse(seed.GetValidVolumeSize("5Gi")))
}

deployer, err := sharedcomponent.NewVictoriaLogs(
r.SeedClientSet.Client(),
r.GardenNamespace,
Expand All @@ -640,7 +645,7 @@ func (r *Reconciler) newVictoriaLogs(seedSettings *gardencorev1beta1.SeedSetting
storage,
false,
victorialogs.PVCAutoscalingConfig{
Enabled: v1beta1helper.SeedSettingPersistentVolumeClaimAutoscalerEnabled(seedSettings),
Enabled: pvcAutoscalerEnabled,
MaxCapacity: resource.MustParse("200Gi"),
},
)
Expand Down
Loading
Loading