Generate correctly the MinIO scrape configs for Multi Tenants scenarios (#2456)

* Simplify check for existing MinIO scrape configs for current tenant

* Allow all actions to prometheuses and prometheusagents resources through
kustomize

Author: JoelRuizRojas

pkg/apis/minio.min.io/v2/names.go

@@ -207,7 +207,7 @@ func (t *Tenant) PrometheusConfigJobName() string {
 // PrometheusOperatorAddlConfigJobName returns the name of the prometheus job
 // when prometheus operator is enabled
 func (t *Tenant) PrometheusOperatorAddlConfigJobName() string {
-	return fmt.Sprintf("%s-minio-job", t.Name)
+	return fmt.Sprintf("%s-%s-minio-job", t.Name, t.Namespace)
 }
 
 // PrometheusConfigMapName returns name of the config map for Prometheus.

pkg/controller/main-controller.go

@@ -304,6 +304,9 @@ func NewController(
 			}
 			controller.enqueueTenant(newObj)
 		},
+		// Enqueue tenant to perform some delete handling actions
+		// during reconciliation
+		DeleteFunc: controller.enqueueTenant,
 	})
 
 	// Set up an event handler for when StatefulSet resources change. This

pkg/controller/prometheus.go

@@ -140,11 +140,13 @@ func (c *Controller) checkAndCreatePrometheusAddlConfig(ctx context.Context, ten
 	}
 
 	promCfg := configmaps.GetPrometheusConfig(tenant, accessKey, secretKey)
+	minIOScrapeConfigs := make([]configmaps.ScrapeConfig, len(promCfg.ScrapeConfigs))
+	copy(minIOScrapeConfigs, promCfg.ScrapeConfigs)
 
 	// If the secret is not found, create the secret
 	if k8serrors.IsNotFound(err) {
 		klog.Infof("Adding MinIO tenant %s/%s Prometheus scrape config", ns, tenant.Name)
-		scrapeCfgYaml, err := yaml.Marshal(&promCfg.ScrapeConfigs)
+		scrapeCfgYaml, err := yaml.Marshal(&minIOScrapeConfigs)
 		if err != nil {
 			return err
 		}
@@ -168,33 +170,47 @@ func (c *Controller) checkAndCreatePrometheusAddlConfig(ctx context.Context, ten
 		if err != nil {
 			return err
 		}
-		// get other scrape configs
+		// Get all non-MinIO scrape configs, as well as MinIO scrape configs for other tenants
 		for _, sc := range scrapeConfigs {
 			if !strings.HasPrefix(sc.JobName, tenant.PrometheusOperatorAddlConfigJobName()) {
 				expectedScrapeConfigs = append(expectedScrapeConfigs, sc)
 			}
 		}
-		ignoreScrapeConfigsIndex := len(expectedScrapeConfigs)
-		expectedScrapeConfigs = append(expectedScrapeConfigs, promCfg.ScrapeConfigs...)
+
+		// Add the expected MinIO scrape configs for the current tenant
+		expectedScrapeConfigs = append(expectedScrapeConfigs, minIOScrapeConfigs...)
+
+		// Verify that all expected scrape configs are present in the installed configs
 		updateScrapeConfig := false
 		if len(scrapeConfigs) != len(expectedScrapeConfigs) {
 			updateScrapeConfig = true
 		} else {
-			for i := range scrapeConfigs {
-				// can't compare that is generated by operator
-				if i < ignoreScrapeConfigsIndex {
-					continue
-				}
-				if scrapeConfigs[i].JobName != expectedScrapeConfigs[i].JobName ||
-					scrapeConfigs[i].MetricsPath != expectedScrapeConfigs[i].MetricsPath ||
-					scrapeConfigs[i].Scheme != expectedScrapeConfigs[i].Scheme ||
-					!reflect.DeepEqual(scrapeConfigs[i].TLSConfig, expectedScrapeConfigs[i].TLSConfig) ||
-					!reflect.DeepEqual(scrapeConfigs[i].StaticConfigs, expectedScrapeConfigs[i].StaticConfigs) {
-					updateScrapeConfig = true
-					break
+			// Verify that all MinIO scrape configs for the current tenant
+			// are included in the installed configs
+			for _, minIOConfig := range minIOScrapeConfigs {
+				configMatched := false
+				for _, config := range scrapeConfigs {
+					if minIOConfig.JobName == config.JobName {
+						if config.MetricsPath != minIOConfig.MetricsPath ||
+							config.Scheme != minIOConfig.Scheme ||
+							!reflect.DeepEqual(config.TLSConfig, minIOConfig.TLSConfig) ||
+							!reflect.DeepEqual(config.StaticConfigs, minIOConfig.StaticConfigs) {
+							updateScrapeConfig = true
+							break
+						}
+						accKey, err := miniov2.GetAccessKeyFromBearerToken(config.BearerToken, secretKey)
+						if err != nil || accKey != accessKey {
+							updateScrapeConfig = true
+							break
+						}
+
+						configMatched = true
+						break
+					}
 				}
-				accKey, _ := miniov2.GetAccessKeyFromBearerToken(scrapeConfigs[i].BearerToken, secretKey)
-				if accKey != accessKey {
+				// Update the installed scrape configs if any MinIO configs for
+				// the current tenant are missing
+				if !configMatched {
 					updateScrapeConfig = true
 					break
 				}
@@ -255,6 +271,7 @@ func (c *Controller) deletePrometheusAddlConfig(ctx context.Context, tenant *min
 		return err
 	}
 	for _, sc := range scrapeConfigs {
+		// Get all non-MinIO scrape configs, as well as MinIO scrape configs for other tenants
 		if !strings.HasPrefix(sc.JobName, tenant.PrometheusOperatorAddlConfigJobName()) {
 			exceptedScrapeConfigs = append(exceptedScrapeConfigs, sc)
 		}