Add override for XDS Addr + Port

docs/content/reference/values.txt

@@ -395,6 +395,7 @@
 |settings.secretOptions.sources[].vault.aws.leaseIncrement|uint32||The time increment, in seconds, used in renewing the lease of the Vault token. See: https://developer.hashicorp.com/vault/docs/concepts/lease#lease-durations-and-renewal. Defaults to 0, which causes the default TTL to be used.|
 |settings.secretOptions.sources[].directory.directory|string||Directory to read secrets from.|
 |settings.kubeResourceOverride.NAME|interface||override fields in the generated resource by specifying the yaml structure to override under the top-level key.|
+|settings.xdsAddress|string||Override the XDS address that gloo proxies use to monitor for configuration chamges.|
 |gloo.deployment.xdsPort|int|9977|port where gloo serves xDS API to Envoy.|
 |gloo.deployment.restXdsPort|uint32|9976|port where gloo serves REST xDS API to Envoy.|
 |gloo.deployment.validationPort|int|9988|port where gloo serves gRPC Proxy Validation to Gateway.|

install/helm/gloo/crds/gloo.solo.io_v1_Settings.yaml

@@ -676,6 +676,11 @@ spec:
                     type: string
                   xdsBindAddr:
                     type: string
+                  xdsClusterAddr:
+                    type: string
+                  xdsClusterPort:
+                    format: int32
+                    type: integer
                 type: object
               graphqlOptions:
                 properties:

install/helm/gloo/generate/values.go

@@ -235,6 +235,7 @@ type Settings struct {
 	DevMode       *bool         `json:"devMode,omitempty" desc:"Whether or not to enable dev mode. Defaults to false. Setting to true at install time will expose the gloo dev admin endpoint on port 10010. Not recommended for production. Warning: this value is deprecated as of 1.17 and will be removed in a future release."`
 	SecretOptions SecretOptions `json:"secretOptions,omitempty" desc:"Options for how Gloo Edge should handle secrets."`
 	*KubeResourceOverride
+	XdsAddress *string `json:"xdsAddress,omitempty" desc:"Override the XDS address that gloo proxies use to monitor for configuration chamges."`
 }
 
 type AwsSettings struct {

install/helm/gloo/templates/18-settings.yaml

@@ -20,6 +20,9 @@ spec:
 {{- toYaml .Values.settings.secretOptions.sources | nindent 6 }}
 {{- end }}
   gloo:
+{{- if .Values.settings.xdsAddress }}
+    xdsClusterAddr: {{ .Values.settings.xdsAddress }}
+{{- end }}
 {{- if .Values.global.glooMtls.enabled }}
     xdsBindAddr: "127.0.0.1:9999"
     restXdsBindAddr: "127.0.0.1:9998"

pkg/utils/setuputils/main_setup.go

@@ -3,7 +3,10 @@ package setuputils
 import (
 	"context"
 	"flag"
+	"io"
+	"net/http"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
@@ -26,6 +29,9 @@ import (
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	zaputil "sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"crypto/tls"
+	"k8s.io/client-go/tools/clientcmd"
+	"github.com/pkg/errors"
 )
 
 type SetupOpts struct {
@@ -70,6 +76,13 @@ func Main(opts SetupOpts) error {
 	loggingContext := append([]interface{}{"version", opts.Version}, opts.LoggingPrefixVals...)
 	ctx = contextutils.WithLoggerValues(ctx, loggingContext...)
 
+	logger := contextutils.LoggerFrom(ctx)
+	logger.Infof("Waiting for Kubernetes API server to be healthy...")
+	// Wait for Kubernetes API server to be healthy
+	if err := waitForKubeApiServer(ctx); err != nil {
+		return err
+	}
+
 	settingsClient, err := fileOrKubeSettingsClient(ctx, setupNamespace, setupDir)
 	if err != nil {
 		return err
@@ -171,3 +184,70 @@ func SetupLogging(ctx context.Context, loggerName string) {
 	// controller-runtime
 	log.SetLogger(zapr.NewLogger(baseLogger))
 }
+
+// waitForKubeApiServer polls the Kubernetes API server until it's healthy or the context is canceled
+func waitForKubeApiServer(ctx context.Context) error {
+	logger := contextutils.LoggerFrom(ctx)
+	logger.Infof("Waiting for Kubernetes API server to be healthy...")
+
+	config, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG"))
+	if err != nil {
+		return errors.Wrap(err, "building kube config")
+	}
+
+	// Create a client for the /healthz endpoint
+	client := http.Client{
+		Timeout: 5 * time.Second,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: config.TLSClientConfig.Insecure,
+			},
+		},
+	}
+
+	// Construct the API server health check URL
+	healthzURL := config.Host
+	if !strings.HasSuffix(healthzURL, "/") {
+		healthzURL += "/"
+	}
+	healthzURL += "healthz"
+
+	// Poll until healthy
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+			req, err := http.NewRequestWithContext(ctx, "GET", healthzURL, nil)
+			if err != nil {
+				logger.Warnf("Error creating request to check API server health: %v", err)
+				time.Sleep(5 * time.Second)
+				continue
+			}
+
+			// Add auth if needed
+			if config.BearerToken != "" {
+				req.Header.Set("Authorization", "Bearer "+config.BearerToken)
+			}
+
+			resp, err := client.Do(req)
+			if err != nil {
+				logger.Debugf("API server health check failed: %v, retrying in 5 seconds", err)
+				time.Sleep(5 * time.Second)
+				continue
+			}
+			defer resp.Body.Close()
+
+			if resp.StatusCode == http.StatusOK {
+				body, _ := io.ReadAll(resp.Body)
+				if string(body) == "ok" {
+					logger.Infof("Kubernetes API server is healthy")
+					return nil
+				}
+			}
+
+			logger.Debugf("API server returned non-OK status: %d, retrying in 5 seconds", resp.StatusCode)
+			time.Sleep(5 * time.Second)
+		}
+	}
+}

pkg/utils/setuputils/setup_syncer.go

@@ -52,6 +52,9 @@ func NewSetupSyncer(settingsRef *core.ResourceRef, setupFunc SetupFunc, identity
 
 func (s *SetupSyncer) Sync(ctx context.Context, snap *v1.SetupSnapshot) error {
 	settings, err := snap.Settings.Find(s.settingsRef.Strings())
+	if settings == nil {
+		return errors.NewNotExistErr("settings not found", "settings", nil)
+	}
 	if err != nil {
 		return errors.Wrapf(err, "finding bootstrap configuration")
 	}

projects/gateway2/helm/gloo-gateway/templates/gateway/proxy-deployment.yaml

@@ -572,11 +572,11 @@ data:
                         "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
 {{- end }} {{/* if $gateway.stats.enabled */}}
       clusters:
-        - name: xds_cluster
+        - name: {{ $gateway.xds.host }}
           alt_stat_name: xds_cluster
           connect_timeout: 5.000s
           load_assignment:
-            cluster_name: xds_cluster
+            cluster_name: {{ $gateway.xds.host }}
             endpoints:
             - lb_endpoints:
               - endpoint:
@@ -675,7 +675,7 @@ data:
         rate_limit_settings: {}
         grpc_services:
         - envoy_grpc:
-            cluster_name: xds_cluster
+            cluster_name: {{ $gateway.xds.host }}
       cds_config:
         resource_api_version: V3
         ads: {}

projects/gateway2/setup/ggv2setup.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/solo-io/gloo/pkg/utils/envutils"
 	"github.com/solo-io/gloo/pkg/utils/setuputils"
@@ -37,6 +38,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	schema "k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
@@ -62,12 +64,31 @@ func createKubeClient(restConfig *rest.Config) (istiokube.Client, error) {
 func getInitialSettings(ctx context.Context, c istiokube.Client, nns types.NamespacedName) *glookubev1.Settings {
 	// get initial settings
 	logger := contextutils.LoggerFrom(ctx)
-	logger.Infof("getting initial settings. gvr: %v", settingsGVR)
+	logger.Infof("attempting to get initial settings. gvr: %v", settingsGVR)
 
-	i, err := c.Dynamic().Resource(settingsGVR).Namespace(nns.Namespace).Get(ctx, nns.Name, metav1.GetOptions{})
-	if err != nil {
-		logger.Panicf("failed to get initial settings: %v", err)
-		return nil
+	var i *unstructured.Unstructured
+	var err error
+
+	// Wait for settings to appear, checking every 5 seconds
+	for {
+		i, err = c.Dynamic().Resource(settingsGVR).Namespace(nns.Namespace).Get(ctx, nns.Name, metav1.GetOptions{})
+		if err == nil {
+			break
+		}
+
+		logger.Infof("settings %s/%s not found, waiting...", nns.Namespace, nns.Name)
+		// Check if context is done to avoid infinite loop if canceled
+		select {
+		case <-ctx.Done():
+			logger.Errorf("context canceled while waiting for settings: %v", ctx.Err())
+			return nil
+		case <-time.After(5 * time.Second):
+			// Wait 5 seconds before trying again
+		}
+		//
+		//// For other errors, log and return nil
+		//logger.Errorf("failed to get initial settings: %v", err)
+		//return nil
 	}
 	logger.Infof("got initial settings")
 

projects/gloo/api/v1/settings.proto

@@ -830,6 +830,10 @@ message GlooOptions {
     }
 
     IstioOptions istio_options = 18;
+
+    string xds_cluster_addr = 19;
+
+    int32 xds_cluster_port = 20;
 }