Send auth on cluster recommendations and report cluster source (C#, Go, TS)

cs/src/Management/TunnelClusterSelectionEventArgs.cs

@@ -0,0 +1,114 @@
+// <copyright file="TunnelClusterSelectionEventArgs.cs" company="Microsoft">
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license.
+// </copyright>
+
+using System;
+
+namespace Microsoft.DevTunnels.Management
+{
+    /// <summary>
+    /// How the cluster for a tunnel create request was chosen.
+    /// </summary>
+    /// <remarks>
+    /// When a create request does not specify a cluster, the client asks the recommendations
+    /// API which cluster to use. That call can fail, and when it does the client falls back to
+    /// global (Traffic Manager) routing, which still works but picks the nearest cluster by
+    /// latency rather than the recommended one. The fallback is therefore invisible to the
+    /// caller, so this value records which path was actually taken.
+    /// </remarks>
+    public enum TunnelClusterSource
+    {
+        /// <summary>
+        /// The caller specified the cluster, so no recommendation was requested.
+        /// </summary>
+        Explicit,
+
+        /// <summary>
+        /// The recommendations API was called and its cluster was used.
+        /// </summary>
+        Recommended,
+
+        /// <summary>
+        /// The recommendations API rejected the caller's token, and the retry without a token
+        /// succeeded. Routing is correct but the caller was not identified, so it is treated as
+        /// anonymous and cannot be assigned a service tier. This indicates a token problem on
+        /// the caller's side that would otherwise be invisible.
+        /// </summary>
+        RecommendedAfterAuthRejected,
+
+        /// <summary>
+        /// The recommendations API returned unauthorized even without a token, so global
+        /// routing was used instead.
+        /// </summary>
+        FallbackAuthFailed,
+
+        /// <summary>
+        /// The recommendations API returned no cluster, so global routing was used instead.
+        /// </summary>
+        FallbackEmpty,
+
+        /// <summary>
+        /// The recommendations API call failed, so global routing was used instead.
+        /// </summary>
+        FallbackError,
+    }
+
+    /// <summary>
+    /// Event args reporting how the cluster for a tunnel create request was chosen.
+    /// </summary>
+    public class TunnelClusterSelectionEventArgs : EventArgs
+    {
+        /// <summary>
+        /// Creates a new instance of the <see cref="TunnelClusterSelectionEventArgs"/> class.
+        /// </summary>
+        public TunnelClusterSelectionEventArgs(
+            TunnelClusterSource source,
+            string? clusterId = null,
+            Exception? exception = null)
+        {
+            this.Source = source;
+            this.ClusterId = clusterId;
+            this.Exception = exception;
+        }
+
+        /// <summary>
+        /// Gets how the cluster was chosen.
+        /// </summary>
+        public TunnelClusterSource Source { get; }
+
+        /// <summary>
+        /// Gets the cluster that was selected, if one was.
+        /// </summary>
+        public string? ClusterId { get; }
+
+        /// <summary>
+        /// Gets the failure that caused a fallback, if there was one.
+        /// </summary>
+        public Exception? Exception { get; }
+
+        /// <summary>
+        /// Gets a value indicating whether the recommendations API was bypassed or failed, so
+        /// the tunnel was placed by global routing rather than by recommendation.
+        /// </summary>
+        public bool IsFallback =>
+            this.Source == TunnelClusterSource.FallbackAuthFailed ||
+            this.Source == TunnelClusterSource.FallbackEmpty ||
+            this.Source == TunnelClusterSource.FallbackError;
+
+        /// <summary>
+        /// Converts a <see cref="TunnelClusterSource"/> to the stable wire value sent to the
+        /// service, which is what makes the client-side path visible in service telemetry.
+        /// </summary>
+        public static string ToHeaderValue(TunnelClusterSource source) => source switch
+        {
+            TunnelClusterSource.Explicit => "explicit",
+            TunnelClusterSource.Recommended => "recommended",
+            TunnelClusterSource.RecommendedAfterAuthRejected => "recommended-after-auth-rejected",
+            TunnelClusterSource.FallbackAuthFailed => "fallback-auth-failed",
+            TunnelClusterSource.FallbackEmpty => "fallback-empty",
+            TunnelClusterSource.FallbackError => "fallback-error",
+            _ => "unknown",
+        };
+    }
+}

cs/src/Management/TunnelManagementClient.cs

@@ -47,6 +47,7 @@ public class TunnelManagementClient : ITunnelManagementClient
         private const string RequestIdHeaderName = "VsSaaS-Request-Id";
         private const string CheckAvailableSubPath = ":checkNameAvailability";
         private const string EnterprisePolicyFailureHeaderName = "X-Enterprise-Policy-Failure";
+        private const string ClusterSourceHeaderName = "X-Tunnel-Cluster-Source";
         private const int CreateNameRetries = 3;
 
         private static readonly string[] ManageAccessTokenScope =
@@ -80,6 +81,18 @@ public class TunnelManagementClient : ITunnelManagementClient
         /// </summary>
         public event EventHandler<TunnelReportProgressEventArgs>? ReportProgress;
 
+        /// <summary>
+        /// Event raised when a tunnel create request selected a cluster via the recommendations
+        /// API, reporting which path was taken.
+        /// </summary>
+        /// <remarks>
+        /// Subscribe to surface recommendation failures. When the recommendations call fails the
+        /// create still succeeds via global routing, so without this event a caller has no way to
+        /// tell that recommendation-based placement stopped working. Raised only when the caller
+        /// did not specify a cluster.
+        /// </remarks>
+        public event EventHandler<TunnelClusterSelectionEventArgs>? ClusterSelected;
+
         /// <summary>
         /// ApiVersion that will be used if one is not specified
         /// </summary>
@@ -1088,29 +1101,59 @@ public async Task<Tunnel> CreateTunnelAsync(
 
             // If the caller didn't specify a cluster, auto-select one via the
             // recommendations API. Failures fall back to global routing.
+            var clusterSource = TunnelClusterSource.Explicit;
             if (string.IsNullOrEmpty(tunnel.ClusterId))
             {
+                Exception? recommendationError = null;
                 try
                 {
-                    var recommendations = await GetClusterRecommendationsAsync(
-                        preferredClusterId: null,
-                        requiredGeo: options.RequiredGeo,
-                        cancellation);
+                    var (recommendations, authRejected) =
+                        await GetClusterRecommendationsInternalAsync(
+                            preferredClusterId: null,
+                            requiredGeo: options.RequiredGeo,
+                            cancellation);
                     if (!string.IsNullOrEmpty(recommendations?.RecommendedClusterId))
                     {
                         tunnel.ClusterId = recommendations!.RecommendedClusterId;
+                        clusterSource = authRejected
+                            ? TunnelClusterSource.RecommendedAfterAuthRejected
+                            : TunnelClusterSource.Recommended;
+                    }
+                    else
+                    {
+                        clusterSource = TunnelClusterSource.FallbackEmpty;
                     }
                 }
-                catch (Exception) when (!cancellation.IsCancellationRequested)
+                catch (Exception ex) when (!cancellation.IsCancellationRequested)
                 {
                     // Fall through to global (Traffic Manager) routing if the
-                    // recommendations request fails for any reason.
+                    // recommendations request fails for any reason. The failure is reported
+                    // rather than swallowed, because global routing still succeeds and the
+                    // caller would otherwise have no indication that the cluster it asked for
+                    // was not the cluster it got.
+                    recommendationError = ex;
+                    clusterSource = ex is UnauthorizedAccessException
+                        ? TunnelClusterSource.FallbackAuthFailed
+                        : TunnelClusterSource.FallbackError;
                 }
+
+                this.ClusterSelected?.Invoke(
+                    this,
+                    new TunnelClusterSelectionEventArgs(
+                        clusterSource, tunnel.ClusterId, recommendationError));
             }
 
             options.AdditionalHeaders ??= new List<KeyValuePair<string, string>>();
             options.AdditionalHeaders = options.AdditionalHeaders.Append(
                 new KeyValuePair<string, string>("If-None-Match", "*"));
+
+            // Report the client-side selection path to the service. Recommendation fallbacks
+            // are otherwise invisible in service telemetry: a create that fell back looks
+            // identical to one that was never recommended at all.
+            options.AdditionalHeaders = options.AdditionalHeaders.Append(
+                new KeyValuePair<string, string>(
+                    ClusterSourceHeaderName,
+                    TunnelClusterSelectionEventArgs.ToHeaderValue(clusterSource)));
             var tunnelId = tunnel.TunnelId;
             var idGenerated = string.IsNullOrEmpty(tunnelId);
             if (idGenerated)
@@ -1624,6 +1667,57 @@ public async Task<ClusterRecommendationResponse> GetClusterRecommendationsAsync(
             string? preferredClusterId = null,
             string? requiredGeo = null,
             CancellationToken cancellation = default)
+        {
+            var (response, _) = await GetClusterRecommendationsInternalAsync(
+                preferredClusterId, requiredGeo, cancellation);
+            return response!;
+        }
+
+        /// <summary>
+        /// Requests cluster recommendations, reporting whether the caller's token was rejected.
+        /// </summary>
+        /// <remarks>
+        /// The token is sent so the service can identify the caller and apply its service tier.
+        /// If the token is rejected the request is retried without it, because the service
+        /// rejects a bad token before the controller runs and does not fall back to treating the
+        /// caller as anonymous. Without the retry, one expired token would silently disable
+        /// recommendation-based routing for that caller.
+        /// </remarks>
+        private async Task<(ClusterRecommendationResponse? Response, bool AuthRejected)>
+            GetClusterRecommendationsInternalAsync(
+                string? preferredClusterId,
+                string? requiredGeo,
+                CancellationToken cancellation)
+        {
+            var uri = BuildClusterRecommendationsUri(preferredClusterId, requiredGeo);
+
+            var authHeader = await this.userTokenCallback();
+            if (authHeader == null)
+            {
+                // No token to offer, so this is an ordinary anonymous request rather than a
+                // rejected one.
+                var anonymous = await SendRequestAsync<object, ClusterRecommendationResponse>(
+                    HttpMethod.Get, uri, options: null, authHeader: null, body: null, cancellation);
+                return (anonymous, false);
+            }
+
+            try
+            {
+                var response = await SendRequestAsync<object, ClusterRecommendationResponse>(
+                    HttpMethod.Get, uri, options: null, authHeader, body: null, cancellation);
+                return (response, false);
+            }
+            catch (UnauthorizedAccessException) when (!cancellation.IsCancellationRequested)
+            {
+                var response = await SendRequestAsync<object, ClusterRecommendationResponse>(
+                    HttpMethod.Get, uri, options: null, authHeader: null, body: null, cancellation);
+                return (response, true);
+            }
+        }
+
+        private Uri BuildClusterRecommendationsUri(
+            string? preferredClusterId,
+            string? requiredGeo)
         {
             var baseAddress = this.httpClient.BaseAddress!;
             var builder = new UriBuilder(baseAddress);
@@ -1648,15 +1742,7 @@ public async Task<ClusterRecommendationResponse> GetClusterRecommendationsAsync(
             }
 
             builder.Query = string.Join("&", queryParts);
-
-            var response = await SendRequestAsync<object, ClusterRecommendationResponse>(
-                HttpMethod.Get,
-                builder.Uri,
-                options: null,
-                authHeader: null,
-                body: null,
-                cancellation);
-            return response!;
+            return builder.Uri;
         }
 
         /// <inheritdoc/>