feat: refactor for simplifying adding a new default scrape config

.pipelines/azure-pipeline-build.yml

@@ -4,6 +4,7 @@ trigger:
     - main
     - bot/otelcollector-upgrade-*
     - dependabot*
+    - grace/single-change-for-new-default-config
 pr:
   autoCancel: true
   branches:
@@ -324,6 +325,47 @@ extends:
               test-cluster-yamls/**
             TargetFolder: '$(Build.ArtifactStagingDirectory)/testkube-files'
 
+
+      - job: Run_Unit_Tests
+        displayName: "Test: Run unit tests"
+        pool:
+          name: Azure-Pipelines-CI-Test-EO
+          image: ci-1es-managed-ubuntu-2204
+          os: linux
+        variables:
+          skipComponentGovernanceDetection: true
+        steps:
+        - checkout: self
+          persistCredentials: true
+        - task: GoTool@0
+          displayName: "Test: specify golang version"
+          inputs:
+            version: $(GOLANG_VERSION)
+        - bash: |
+            go test -v .
+          workingDirectory: $(Build.SourcesDirectory)/otelcollector/shared/configmap/mp
+          displayName: "Test: run configmap processing unit tests"
+
+      - job: Run_CCP_Unit_Tests
+        displayName: "Test: Run CCP unit tests"
+        pool:
+          name: Azure-Pipelines-CI-Test-EO
+          image: ci-1es-managed-ubuntu-2204
+          os: linux
+        variables:
+          skipComponentGovernanceDetection: true
+        steps:
+        - checkout: self
+          persistCredentials: true
+        - task: GoTool@0
+          displayName: "Test: specify golang version"
+          inputs:
+            version: $(GOLANG_VERSION)
+        - bash: |
+            go test -v .
+          workingDirectory: $(Build.SourcesDirectory)/otelcollector/shared/configmap/ccp
+          displayName: "Test: run configmap processing unit tests"
+
       - job: Linux_Golang_Reference_App
         displayName: "Build: linux golang reference app image"
         pool:

otelcollector/build/linux/Dockerfile

@@ -2,10 +2,12 @@ ARG GOLANG_VERSION
 ARG FLUENTBIT_GOLANG_VERSION
 FROM --platform=$BUILDPLATFORM mcr.microsoft.com/oss/go/microsoft/golang:${FLUENTBIT_GOLANG_VERSION} as fluent-bit-builder
 WORKDIR /src
-COPY ./fluent-bit/src/go.mod ./fluent-bit/src/go.sum .
-RUN go version
+COPY ./fluent-bit/src/ ./fluent-bit/src/
+COPY ./shared/ ./shared/
+WORKDIR /src/shared/
+RUN go mod download
+WORKDIR /src/fluent-bit/src/
 RUN go mod download
-COPY ./fluent-bit/src/ .
 RUN apt-get update && apt-get install gcc-aarch64-linux-gnu -y
 ARG TARGETOS TARGETARCH
 RUN if [ "$TARGETARCH" = "arm64" ] ; then CC=aarch64-linux-gnu-gcc CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -buildmode=c-shared -ldflags '-extldflags=-Wl,-z,now' -o out_appinsights.so . ; else  CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -buildmode=c-shared -ldflags '-extldflags=-Wl,-z,now' -o out_appinsights.so . ; fi
@@ -63,17 +65,10 @@ FROM --platform=$BUILDPLATFORM mcr.microsoft.com/oss/go/microsoft/golang:${GOLAN
 WORKDIR /
 # Create directories
 RUN mkdir -p ./shared/configmap/mp/
+RUN mkdir -p ./shared/configmap/common/
 RUN mkdir -p ./main/
-# Copy shared go files
-COPY ../shared/*.go ./main/shared/
-COPY ./shared/go.mod ./main/shared/
-COPY ./shared/go.sum ./main/shared/
-COPY ../shared/configmap/mp/*.go ./main/shared/configmap/mp/
-COPY ../shared/configmap/ccp/*.go ./main/shared/configmap/ccp/
-COPY ./shared/configmap/mp/go.mod ./main/shared/configmap/mp/
-COPY ./shared/configmap/mp/go.sum ./main/shared/configmap/mp/
-COPY ./shared/configmap/ccp/go.mod ./main/shared/configmap/ccp/
-COPY ./shared/configmap/ccp/go.sum ./main/shared/configmap/ccp/
+# Copy shared module (includes mp, ccp, common, and test helpers)
+COPY ./shared/ ./main/shared/
 # Copy main go files
 COPY ./main/*.go ./main/
 COPY ./go.mod ./main/
@@ -160,7 +155,7 @@ COPY --from=main-builder --chmod=777 /main/main.exe $tmpdir/main
 COPY ./scripts/*.sh $tmpdir/
 COPY ./metricextension/me.config ./metricextension/me_internal.config ./metricextension/me_ds.config  ./metricextension/me_ds_setdim.config ./metricextension/me_ds_internal.config ./metricextension/me_ds_internal_setdim.config /usr/sbin/
 COPY ./fluent-bit/fluent-bit.yaml ./fluent-bit/fluent-bit-daemonset.yaml ./fluent-bit/fluent-bit-parsers.conf $tmpdir/fluent-bit/
-COPY --from=fluent-bit-builder /src/out_appinsights.so $tmpdir/fluent-bit/bin/
+COPY --from=fluent-bit-builder /src/fluent-bit/src/out_appinsights.so $tmpdir/fluent-bit/bin/
 COPY --from=fluent-bit-binary-builder /usr/local/bin/fluent-bit /usr/local/bin/fluent-bit
 COPY --from=fluent-bit-binary-builder /usr/local/etc/fluent-bit /usr/local/etc/fluent-bit
 COPY ./LICENSE $tmpdir/microsoft

otelcollector/build/linux/ccp/Dockerfile

@@ -39,17 +39,10 @@ FROM --platform=$BUILDPLATFORM mcr.microsoft.com/oss/go/microsoft/golang:${GOLAN
 WORKDIR /
 # Create directories
 RUN mkdir -p ./shared/configmap/mp/
+RUN mkdir -p ./shared/configmap/common/
 RUN mkdir -p ./main/
-# Copy shared go files
-COPY ../shared/*.go ./main/shared/
-COPY ./shared/go.mod ./main/shared/
-COPY ./shared/go.sum ./main/shared/
-COPY ../shared/configmap/mp/*.go ./main/shared/configmap/mp/
-COPY ../shared/configmap/ccp/*.go ./main/shared/configmap/ccp/
-COPY ./shared/configmap/mp/go.mod ./main/shared/configmap/mp/
-COPY ./shared/configmap/mp/go.sum ./main/shared/configmap/mp/
-COPY ./shared/configmap/ccp/go.mod ./main/shared/configmap/ccp/
-COPY ./shared/configmap/ccp/go.sum ./main/shared/configmap/ccp/
+# Copy shared module (includes mp, ccp, common, and helpers)
+COPY ./shared/ ./main/shared/
 # Copy main go files
 COPY ./main/*.go ./main/
 COPY ./go.mod ./main/

otelcollector/configmapparser/default-prom-configs/podannotationsDefault.yml

@@ -27,3 +27,6 @@
     - source_labels: [__meta_kubernetes_pod_name]
       action: replace
       target_label: kubernetes_pod_name
+    - source_labels: [__meta_kubernetes_namespace]
+      action: keep
+      regex: $$AZMON_PROMETHEUS_POD_ANNOTATION_NAMESPACES_REGEX$$

otelcollector/fluent-bit/src/go.mod

@@ -1,13 +1,15 @@
 module Docker-Provider/source/plugins/go/src
 
+replace github.com/prometheus-collector/shared => ../../shared
+
 go 1.24.0
 
-toolchain go1.24.1
 
 require (
 	github.com/fluent/fluent-bit-go v0.0.0-20220311094233-780004bf5562
 	github.com/microsoft/ApplicationInsights-Go v0.4.4
 	github.com/mitchellh/mapstructure v1.5.0
+	github.com/prometheus-collector/shared v0.0.0-00010101000000-000000000000
 	github.com/prometheus/client_golang v1.23.2
 	github.com/shirou/gopsutil/v4 v4.26.1
 	github.com/ugorji/go/codec v1.3.0

otelcollector/fluent-bit/src/go.sum

@@ -220,6 +220,8 @@ k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8
 k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE=
 sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4=
 sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
 sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
 sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco=

otelcollector/fluent-bit/src/telemetry.go

@@ -20,6 +20,7 @@ import (
 	"github.com/fluent/fluent-bit-go/output"
 	"github.com/microsoft/ApplicationInsights-Go/appinsights"
 	"github.com/microsoft/ApplicationInsights-Go/appinsights/contracts"
+	"github.com/prometheus-collector/shared"
 	yaml "gopkg.in/yaml.v2"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
@@ -904,132 +905,15 @@ func PushMEProcessedAndReceivedCountToAppInsightsMetrics() {
 			}
 
 			if os.Getenv(envControllerType) == "ReplicaSet" {
-				if KubeletKeepListRegex != "" {
-					metric.Properties["KubeletKeepListRegex"] = KubeletKeepListRegex
-				}
-				if CoreDNSKeepListRegex != "" {
-					metric.Properties["CoreDNSKeepListRegex"] = CoreDNSKeepListRegex
-				}
-				if CAdvisorKeepListRegex != "" {
-					metric.Properties["CAdvisorKeepListRegex"] = CAdvisorKeepListRegex
-				}
-				if KubeProxyKeepListRegex != "" {
-					metric.Properties["KubeProxyKeepListRegex"] = KubeProxyKeepListRegex
-				}
-				if ApiServerKeepListRegex != "" {
-					metric.Properties["ApiServerKeepListRegex"] = ApiServerKeepListRegex
-				}
-				if KubeStateKeepListRegex != "" {
-					metric.Properties["KubeStateKeepListRegex"] = KubeStateKeepListRegex
-				}
-				if NodeExporterKeepListRegex != "" {
-					metric.Properties["NodeExporterKeepListRegex"] = NodeExporterKeepListRegex
-				}
-				if WinExporterKeepListRegex != "" {
-					metric.Properties["WinExporterKeepListRegex"] = WinExporterKeepListRegex
-				}
-				if WinKubeProxyKeepListRegex != "" {
-					metric.Properties["WinKubeProxyKeepListRegex"] = WinKubeProxyKeepListRegex
-				}
-				if PodannotationKeepListRegex != "" {
-					metric.Properties["PodannotationKeepListRegex"] = PodannotationKeepListRegex
-				}
-				if KappieBasicKeepListRegex != "" {
-					metric.Properties["KappieBasicKeepListRegex"] = KappieBasicKeepListRegex
-				}
-				if AcstorCapacityProvisionerKeepListRegex != "" {
-					metric.Properties["AcstorCapacityProvisionerRegex"] = AcstorCapacityProvisionerKeepListRegex
-				}
-				if AcstorMetricsExporterKeepListRegex != "" {
-					metric.Properties["AcstorMetricsExporterRegex"] = AcstorMetricsExporterKeepListRegex
-				}
-				if LocalCSIDriverKeepListRegex != "" {
-					metric.Properties["LocalCSIDriverExporterRegex"] = LocalCSIDriverKeepListRegex
-				}
-				if ZtunnelKeepListRegex != "" {
-					metric.Properties["ZtunnelKeepListRegex"] = ZtunnelKeepListRegex
-				}
-				if IstioCniKeepListRegex != "" {
-					metric.Properties["IstioCniKeepListRegex"] = IstioCniKeepListRegex
-				}
-				if WaypointProxyKeepListRegex != "" {
-					metric.Properties["WaypointProxyKeepListRegex"] = WaypointProxyKeepListRegex
-				}
-				if DCGMExporterKeepListRegex != "" {
-					metric.Properties["DCGMExporterRegex"] = DCGMExporterKeepListRegex
-				}
-				if NetworkObservabilityCiliumKeepListRegex != "" {
-					metric.Properties["NetworkObservabilityCiliumRegex"] = NetworkObservabilityCiliumKeepListRegex
-				}
-				if NetworkObservabilityHubbleKeepListRegex != "" {
-					metric.Properties["NetworkObservabilityHubbleRegex"] = NetworkObservabilityHubbleKeepListRegex
-				}
-				if NetworkObservabilityRetinaKeepListRegex != "" {
-					metric.Properties["NetworkObservabilityRetinaRegex"] = NetworkObservabilityRetinaKeepListRegex
-				}
 
-				if KubeletScrapeInterval != "" {
-					metric.Properties["KubeletScrapeInterval"] = KubeletScrapeInterval
-				}
-				if CoreDNSScrapeInterval != "" {
-					metric.Properties["CoreDNSScrapeInterval"] = CoreDNSScrapeInterval
-				}
-				if CAdvisorScrapeInterval != "" {
-					metric.Properties["CAdvisorScrapeInterval"] = CAdvisorScrapeInterval
-				}
-				if KubeProxyScrapeInterval != "" {
-					metric.Properties["KubeProxyScrapeInterval"] = KubeProxyScrapeInterval
-				}
-				if ApiServerScrapeInterval != "" {
-					metric.Properties["ApiServerScrapeInterval"] = ApiServerScrapeInterval
-				}
-				if KubeStateScrapeInterval != "" {
-					metric.Properties["KubeStateScrapeInterval"] = KubeStateScrapeInterval
-				}
-				if NodeExporterScrapeInterval != "" {
-					metric.Properties["NodeExporterScrapeInterval"] = NodeExporterScrapeInterval
-				}
-				if WinExporterScrapeInterval != "" {
-					metric.Properties["WinExporterScrapeInterval"] = WinExporterScrapeInterval
-				}
-				if WinKubeProxyScrapeInterval != "" {
-					metric.Properties["WinKubeProxyScrapeInterval"] = WinKubeProxyScrapeInterval
-				}
-				if PromHealthScrapeInterval != "" {
-					metric.Properties["PromHealthScrapeInterval"] = PromHealthScrapeInterval
-				}
-				if PodAnnotationScrapeInterval != "" {
-					metric.Properties["PodAnnotationScrapeInterval"] = PodAnnotationScrapeInterval
-				}
-				if KappieBasicScrapeInterval != "" {
-					metric.Properties["KappieBasicScrapeInterval"] = KappieBasicScrapeInterval
-				}
-				if AcstorCapacityProvisionerScrapeInterval != "" {
-					metric.Properties["AcstorCapacityProvisionerScrapeInterval"] = AcstorCapacityProvisionerScrapeInterval
-				}
-				if AcstorMetricsExporterScrapeInterval != "" {
-					metric.Properties["AcstorMetricsExporterScrapeInterval"] = AcstorMetricsExporterScrapeInterval
-				}
-				if ZtunnelScrapeInterval != "" {
-					metric.Properties["ZtunnelScrapeInterval"] = ZtunnelScrapeInterval
-				}
-				if IstioCniScrapeInterval != "" {
-					metric.Properties["IstioCniScrapeInterval"] = IstioCniScrapeInterval
-				}
-				if WaypointProxyScrapeInterval != "" {
-					metric.Properties["WaypointProxyScrapeInterval"] = WaypointProxyScrapeInterval
-				}
-				if DCGMExporterScrapeInterval != "" {
-					metric.Properties["DCGMExporterScrapeInterval"] = DCGMExporterScrapeInterval
-				}
-				if NetworkObservabilityCiliumScrapeInterval != "" {
-					metric.Properties["NetworkObservabilityCiliumScrapeInterval"] = NetworkObservabilityCiliumScrapeInterval
-				}
-				if NetworkObservabilityHubbleScrapeInterval != "" {
-					metric.Properties["NetworkObservabilityHubbleScrapeInterval"] = NetworkObservabilityHubbleScrapeInterval
-				}
-				if NetworkObservabilityRetinaScrapeInterval != "" {
-					metric.Properties["NetworkObservabilityRetinaScrapeInterval"] = NetworkObservabilityRetinaScrapeInterval
+				for jobName, job := range shared.DefaultScrapeJobs {
+					metric.Properties[fmt.Sprintf("%sEnabled", jobName)] = strconv.FormatBool(job.Enabled)
+					if job.KeepListRegex != "" {
+						metric.Properties[fmt.Sprintf("%sKeepListRegex", jobName)] = job.KeepListRegex
+					}
+					if job.ScrapeInterval != "" {
+						metric.Properties[fmt.Sprintf("%sScrapeInterval", jobName)] = job.ScrapeInterval
+					}
 				}
 			}
 

otelcollector/go.mod

@@ -10,29 +10,39 @@ replace github.com/prometheus-collector/shared/configmap/mp => ./shared/configma
 
 replace github.com/prometheus-collector/shared/configmap/ccp => ./shared/configmap/ccp
 
+replace github.com/prometheus-collector/shared/configmap/common/testhelpers => ./shared/configmap/common/testhelpers
+
 require (
 	github.com/prometheus-collector/shared v0.0.0-00010101000000-000000000000
 	github.com/prometheus-collector/shared/configmap/ccp v0.0.0-00010101000000-000000000000
 	github.com/prometheus-collector/shared/configmap/mp v0.0.0-00010101000000-000000000000
+	github.com/prometheus-collector/shared/configmap/common/testhelpers v0.0.0-00010101000000-000000000000
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/onsi/ginkgo/v2 v2.21.0 // indirect
+	github.com/onsi/gomega v1.35.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.63.0 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.65.0 // indirect
 	github.com/stretchr/testify v1.10.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
-	golang.org/x/net v0.38.0 // indirect
-	golang.org/x/text v0.23.0 // indirect
-	google.golang.org/protobuf v1.36.5 // indirect
+	golang.org/x/net v0.40.0 // indirect
+	golang.org/x/sys v0.33.0 // indirect
+	golang.org/x/text v0.25.0 // indirect
+	golang.org/x/tools v0.26.0 // indirect
+	google.golang.org/protobuf v1.36.6 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect