fix(e2e): scrape MetricCompressor stats from the proxy pod (#9942)

MetricCompressor* probed <gateway LoadBalancer IP>:19001/stats/prometheus.
MetalLB recycles addresses from its pool as tests create and delete Services,
and ProxyMetrics runs right before these tests with a LoadBalancer Service on
the same port 19001 serving /metrics, so the compressor test can inherit that
address and get a steady stream of 404s from the previous owner. In the run
that prompted this, the proxy's own counters show only 2 connections ever
reached 0.0.0.0_19001, so none of the failing probes made it to Envoy.

Scrape the stats endpoint through a port-forward to the proxy pod instead, and
drop the two pieces of the manifests that only existed to reach 19001 through
the LoadBalancer: the unused <compressor>-gtw-metrics Services, which each took
a pool address for nothing, and the envoyService port patch.

Signed-off-by: Huabing (Robin) Zhao <zhaohuabing@gmail.com>
Signed-off-by: Huabing (Robin) Zhao <huabing@tetrate.io>

Author: zhaohuabing

test/e2e/testdata/metric-compressor-brotli.yaml

@@ -25,43 +25,12 @@ metadata:
   namespace: gateway-conformance-infra
 spec:
   ipFamily: IPv4
-  provider:
-    type: Kubernetes
-    kubernetes:
-      envoyService:
-        patch:
-          value:
-            spec:
-              ports:
-                - name: metrics
-                  protocol: TCP
-                  port: 19001
-                  targetPort: 19001
   telemetry:
     metrics:
       prometheus:
         compression:
           type: Brotli
 ---
-apiVersion: v1
-kind: Service
-metadata:
-  name: brotli-gtw-metrics
-  namespace: envoy-gateway-system
-spec:
-  selector:
-    app.kubernetes.io/component: proxy
-    app.kubernetes.io/managed-by: envoy-gateway
-    app.kubernetes.io/name: envoy
-    gateway.envoyproxy.io/owning-gateway-name: brotli-gtw
-    gateway.envoyproxy.io/owning-gateway-namespace: gateway-conformance-infra
-  ports:
-    - name: metrics
-      protocol: TCP
-      port: 19001
-      targetPort: 19001
-  type: LoadBalancer
----
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
 metadata:

test/e2e/testdata/metric-compressor-gzip.yaml

@@ -25,43 +25,12 @@ metadata:
   namespace: gateway-conformance-infra
 spec:
   ipFamily: IPv4
-  provider:
-    type: Kubernetes
-    kubernetes:
-      envoyService:
-        patch:
-          value:
-            spec:
-              ports:
-                - name: metrics
-                  protocol: TCP
-                  port: 19001
-                  targetPort: 19001
   telemetry:
     metrics:
       prometheus:
         compression:
           type: Gzip
 ---
-apiVersion: v1
-kind: Service
-metadata:
-  name: gzip-gtw-metrics
-  namespace: envoy-gateway-system
-spec:
-  selector:
-    app.kubernetes.io/component: proxy
-    app.kubernetes.io/managed-by: envoy-gateway
-    app.kubernetes.io/name: envoy
-    gateway.envoyproxy.io/owning-gateway-name: gzip-gtw
-    gateway.envoyproxy.io/owning-gateway-namespace: gateway-conformance-infra
-  ports:
-    - name: metrics
-      protocol: TCP
-      port: 19001
-      targetPort: 19001
-  type: LoadBalancer
----
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
 metadata:

test/e2e/testdata/metric-compressor-zstd.yaml

@@ -25,43 +25,12 @@ metadata:
   namespace: gateway-conformance-infra
 spec:
   ipFamily: IPv4
-  provider:
-    type: Kubernetes
-    kubernetes:
-      envoyService:
-        patch:
-          value:
-            spec:
-              ports:
-                - name: metrics
-                  protocol: TCP
-                  port: 19001
-                  targetPort: 19001
   telemetry:
     metrics:
       prometheus:
         compression:
           type: Zstd
 ---
-apiVersion: v1
-kind: Service
-metadata:
-  name: zstd-gtw-metrics
-  namespace: envoy-gateway-system
-spec:
-  selector:
-    app.kubernetes.io/component: proxy
-    app.kubernetes.io/managed-by: envoy-gateway
-    app.kubernetes.io/name: envoy
-    gateway.envoyproxy.io/owning-gateway-name: zstd-gtw
-    gateway.envoyproxy.io/owning-gateway-namespace: gateway-conformance-infra
-  ports:
-    - name: metrics
-      protocol: TCP
-      port: 19001
-      targetPort: 19001
-  type: LoadBalancer
----
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
 metadata:

test/e2e/tests/metric.go

@@ -26,6 +26,7 @@ import (
 	"sigs.k8s.io/gateway-api/conformance/utils/tlog"
 
 	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
+	kube "github.com/envoyproxy/gateway/internal/kubernetes"
 	"github.com/envoyproxy/gateway/test/utils/prometheus"
 )
 
@@ -202,9 +203,13 @@ func runMetricCompressorTest(t *testing.T, suite *suite.ConformanceTestSuite, ns
 	}
 	httputils.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse)
 
-	// stats exposed at port 19001
-	statsHost := strings.Replace(gwAddr, ":80", ":19001", 1)
-	statsAddr := fmt.Sprintf("http://%s/stats/prometheus", statsHost)
+	// Scrape the stats endpoint through a port-forward to the proxy pod rather than through the
+	// Gateway's LoadBalancer address: MetalLB recycles addresses from its pool as tests create and
+	// delete Services, so a probe of <gateway address>:19001 can be answered by whatever owned that
+	// address a moment earlier, which shows up as a confusing HTTP error instead of a connect error.
+	fwd := proxyStatsForwarder(t, suite, gwNN)
+	defer fwd.Stop()
+	statsAddr := fmt.Sprintf("http://%s/stats/prometheus", fwd.Address())
 	tlog.Logf(t, "check stats from %s", statsAddr)
 
 	err := wait.PollUntilContextTimeout(t.Context(), time.Second, time.Minute, true, func(_ context.Context) (done bool, err error) {
@@ -220,6 +225,33 @@ func runMetricCompressorTest(t *testing.T, suite *suite.ConformanceTestSuite, ns
 	}
 }
 
+// proxyStatsForwarder starts a port-forward to the stats port of the Envoy proxy pod backing the
+// given Gateway. The caller is responsible for stopping the returned forwarder.
+func proxyStatsForwarder(t *testing.T, suite *suite.ConformanceTestSuite, gwNN types.NamespacedName) kube.PortForwarder {
+	t.Helper()
+
+	cli, err := kube.NewForRestConfig(suite.RestConfig)
+	require.NoError(t, err)
+
+	pods, err := cli.PodsForSelector(GetGatewayResourceNamespace(),
+		"app.kubernetes.io/name=envoy",
+		fmt.Sprintf("gateway.envoyproxy.io/owning-gateway-name=%s", gwNN.Name),
+		fmt.Sprintf("gateway.envoyproxy.io/owning-gateway-namespace=%s", gwNN.Namespace),
+	)
+	require.NoError(t, err)
+	require.NotEmpty(t, pods.Items, "no Envoy proxy pod found for Gateway %s", gwNN.String())
+
+	// stats are exposed at port 19001
+	fwd, err := kube.NewLocalPortForwarder(cli, types.NamespacedName{
+		Namespace: pods.Items[0].Namespace,
+		Name:      pods.Items[0].Name,
+	}, 0, 19001)
+	require.NoError(t, err)
+	require.NoError(t, fwd.Start())
+
+	return fwd
+}
+
 func checkStatsEncoding(suite *suite.ConformanceTestSuite, statsAddr string, compressorType egv1a1.CompressorType) error {
 	req, err := http.NewRequest("GET", statsAddr, nil)
 	if err != nil {