Expose gRPC client retry options

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## (Unreleased)
 
+- Expose gRPC retry options in Azure Managed extensions ([#447](https://github.com/microsoft/durabletask-dotnet/pull/447))
+
 ## v1.12.0
 
 - Activity tag support ([#426](https://github.com/microsoft/durabletask-dotnet/pull/426))

src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs

@@ -6,8 +6,8 @@
 using Azure.Identity;
 using Grpc.Core;
 using Grpc.Net.Client;
-using Microsoft.DurableTask;
 using Microsoft.DurableTask.Client;
+using GrpcConfig = Grpc.Net.Client.Configuration;
 
 namespace Microsoft.DurableTask;
 
@@ -46,6 +46,11 @@ public class DurableTaskSchedulerClientOptions
     /// </summary>
     public bool AllowInsecureCredentials { get; set; }
 
+    /// <summary>
+    /// Gets or sets the options that determine how and when calls made to the scheduler will be retried.
+    /// </summary>
+    public ClientRetryOptions? RetryOptions { get; set; }
+
     /// <summary>
     /// Creates a new instance of <see cref="DurableTaskSchedulerClientOptions"/> from a connection string.
     /// </summary>
@@ -109,6 +114,45 @@ this.Credential is not null
                 metadata.Add("Authorization", $"Bearer {token.Token}");
             });
 
+        GrpcConfig.ServiceConfig? serviceConfig = GrpcRetryPolicyDefaults.DefaultServiceConfig;
+        if (this.RetryOptions != null)
+        {
+            GrpcConfig.RetryPolicy retryPolicy = new GrpcConfig.RetryPolicy
+            {
+                MaxAttempts = this.RetryOptions.MaxRetries ?? GrpcRetryPolicyDefaults.DefaultMaxAttempts,
+                InitialBackoff = TimeSpan.FromMilliseconds(this.RetryOptions.InitialBackoffMs ?? GrpcRetryPolicyDefaults.DefaultInitialBackoffMs),
+                MaxBackoff = TimeSpan.FromMilliseconds(this.RetryOptions.MaxBackoffMs ?? GrpcRetryPolicyDefaults.DefaultMaxBackoffMs),
+                BackoffMultiplier = this.RetryOptions.BackoffMultiplier ?? GrpcRetryPolicyDefaults.DefaultBackoffMultiplier,
+                RetryableStatusCodes = { StatusCode.Unavailable }, // Always retry on Unavailable.
+            };
+
+            if (this.RetryOptions.RetryableStatusCodes != null)
+            {
+                foreach (StatusCode statusCode in this.RetryOptions.RetryableStatusCodes)
+                {
+                    // Added by default, don't need to have it added twice.
+                    if (statusCode == StatusCode.Unavailable)
+                    {
+                        continue;
+                    }
+
+                    retryPolicy.RetryableStatusCodes.Add(statusCode);
+                }
+            }
+
+            GrpcConfig.MethodConfig methodConfig = new GrpcConfig.MethodConfig
+            {
+                // MethodName.Default applies this retry policy configuration to all gRPC methods on the channel.
+                Names = { GrpcConfig.MethodName.Default },
+                RetryPolicy = retryPolicy,
+            };
+
+            serviceConfig = new GrpcConfig.ServiceConfig
+            {
+                MethodConfigs = { methodConfig },
+            };
+        }
+
         // Production will use HTTPS, but local testing will use HTTP
         ChannelCredentials channelCreds = endpoint.StartsWith("https://", StringComparison.OrdinalIgnoreCase) ?
             ChannelCredentials.SecureSsl :
@@ -117,7 +161,7 @@ this.Credential is not null
         {
             Credentials = ChannelCredentials.Create(channelCreds, managedBackendCreds),
             UnsafeUseInsecureChannelCallCredentials = this.AllowInsecureCredentials,
-            ServiceConfig = GrpcRetryPolicyDefaults.DefaultServiceConfig,
+            ServiceConfig = serviceConfig,
         });
     }
 
@@ -173,4 +217,35 @@ this.Credential is not null
                     nameof(connectionString));
         }
     }
+
+    /// <summary>
+    /// Options used to configure retries used when making calls to the Scheduler.
+    /// </summary>
+    public class ClientRetryOptions
+    {
+        /// <summary>
+        /// Gets or sets the maximum number of times a call should be retried.
+        /// </summary>
+        public int? MaxRetries { get; set; }
+
+        /// <summary>
+        /// Gets or sets the initial backoff in milliseconds.
+        /// </summary>
+        public int? InitialBackoffMs { get; set; }
+
+        /// <summary>
+        /// Gets or sets the maximum backoff in milliseconds.
+        /// </summary>
+        public int? MaxBackoffMs { get; set; }
+
+        /// <summary>
+        /// Gets or sets the backoff multiplier for exponential backoff.
+        /// </summary>
+        public double? BackoffMultiplier { get; set; }
+
+        /// <summary>
+        /// Gets or sets the list of status codes that can be retried.
+        /// </summary>
+        public IList<StatusCode>? RetryableStatusCodes { get; set; }
+    }
 }

test/Client/AzureManaged.Tests/Client.AzureManaged.Tests.csproj

@@ -10,6 +10,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" />
+    <PackageReference Include="Grpc.Core" />
   </ItemGroup>
 
   <ItemGroup>

test/Client/AzureManaged.Tests/DurableTaskSchedulerClientExtensionsTests.cs

@@ -4,6 +4,7 @@
 using Azure.Core;
 using Azure.Identity;
 using FluentAssertions;
+using Grpc.Core;
 using Microsoft.DurableTask.Client.Grpc;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
@@ -109,7 +110,7 @@ public void UseDurableTaskScheduler_WithNullParameters_ShouldThrowOptionsValidat
         // Assert
         var action = () => provider.GetRequiredService<IOptions<DurableTaskSchedulerClientOptions>>().Value;
         action.Should().Throw<OptionsValidationException>()
-            .WithMessage(endpoint == null 
+            .WithMessage(endpoint == null
                 ? "DataAnnotation validation failed for 'DurableTaskSchedulerClientOptions' members: 'EndpointAddress' with the error: 'Endpoint address is required'."
                 : "DataAnnotation validation failed for 'DurableTaskSchedulerClientOptions' members: 'TaskHubName' with the error: 'Task hub name is required'.");
     }
@@ -193,4 +194,90 @@ public void UseDurableTaskScheduler_WithNamedOptions_ShouldConfigureCorrectly()
         options.ResourceId.Should().Be("https://durabletask.io");
         options.AllowInsecureCredentials.Should().BeFalse();
     }
+
+    [Fact]
+    public void UseDurableTaskScheduler_WithEndpointAndCredentialAndRetryOptions_ShouldConfigureCorrectly()
+    {
+        // Arrange
+        ServiceCollection services = new ServiceCollection();
+        Mock<IDurableTaskClientBuilder> mockBuilder = new Mock<IDurableTaskClientBuilder>();
+        mockBuilder.Setup(b => b.Services).Returns(services);
+        DefaultAzureCredential credential = new DefaultAzureCredential();
+
+        // Act
+        mockBuilder.Object.UseDurableTaskScheduler(ValidEndpoint, ValidTaskHub, credential, options =>
+                options.RetryOptions = new DurableTaskSchedulerClientOptions.ClientRetryOptions
+                {
+                    MaxRetries = 5,
+                    InitialBackoffMs = 100,
+                    MaxBackoffMs = 1000,
+                    BackoffMultiplier = 2.0,
+                    RetryableStatusCodes = new List<StatusCode> { StatusCode.Unknown }
+                }
+            );
+
+        // Assert
+        ServiceProvider provider = services.BuildServiceProvider();
+        IOptions<GrpcDurableTaskClientOptions>? options = provider.GetService<IOptions<GrpcDurableTaskClientOptions>>();
+        options.Should().NotBeNull();
+
+        // Validate the configured options
+        DurableTaskSchedulerClientOptions clientOptions = provider.GetRequiredService<IOptions<DurableTaskSchedulerClientOptions>>().Value;
+        clientOptions.EndpointAddress.Should().Be(ValidEndpoint);
+        clientOptions.TaskHubName.Should().Be(ValidTaskHub);
+        clientOptions.Credential.Should().BeOfType<DefaultAzureCredential>();
+        clientOptions.RetryOptions.Should().NotBeNull();
+        // The assert not null doesn't clear the syntax warning about null checks.
+        if (clientOptions.RetryOptions != null)
+        {
+            clientOptions.RetryOptions.MaxRetries.Should().Be(5);
+            clientOptions.RetryOptions.InitialBackoffMs.Should().Be(100);
+            clientOptions.RetryOptions.MaxBackoffMs.Should().Be(1000);
+            clientOptions.RetryOptions.BackoffMultiplier.Should().Be(2.0);
+            clientOptions.RetryOptions.RetryableStatusCodes.Should().Contain(StatusCode.Unknown);
+        }
+    }
+
+    [Fact]
+    public void UseDurableTaskScheduler_WithConnectionStringAndRetryOptions_ShouldConfigureCorrectly()
+    {
+        // Arrange
+        ServiceCollection services = new ServiceCollection();
+        Mock<IDurableTaskClientBuilder> mockBuilder = new Mock<IDurableTaskClientBuilder>();
+        mockBuilder.Setup(b => b.Services).Returns(services);
+        string connectionString = $"Endpoint={ValidEndpoint};Authentication=DefaultAzure;TaskHub={ValidTaskHub}";
+
+        // Act
+        mockBuilder.Object.UseDurableTaskScheduler(connectionString, options =>
+                options.RetryOptions = new DurableTaskSchedulerClientOptions.ClientRetryOptions
+                {
+                    MaxRetries = 5,
+                    InitialBackoffMs = 100,
+                    MaxBackoffMs = 1000,
+                    BackoffMultiplier = 2.0,
+                    RetryableStatusCodes = new List<StatusCode> { StatusCode.Unknown }
+                }
+            );
+
+        // Assert
+        ServiceProvider provider = services.BuildServiceProvider();
+        IOptions<GrpcDurableTaskClientOptions>? options = provider.GetService<IOptions<GrpcDurableTaskClientOptions>>();
+        options.Should().NotBeNull();
+
+        // Validate the configured options
+        DurableTaskSchedulerClientOptions clientOptions = provider.GetRequiredService<IOptions<DurableTaskSchedulerClientOptions>>().Value;
+        clientOptions.EndpointAddress.Should().Be(ValidEndpoint);
+        clientOptions.TaskHubName.Should().Be(ValidTaskHub);
+        clientOptions.Credential.Should().BeOfType<DefaultAzureCredential>();
+        clientOptions.RetryOptions.Should().NotBeNull();
+        // The assert not null doesn't clear the syntax warning about null checks.
+        if (clientOptions.RetryOptions != null)
+        {
+            clientOptions.RetryOptions.MaxRetries.Should().Be(5);
+            clientOptions.RetryOptions.InitialBackoffMs.Should().Be(100);
+            clientOptions.RetryOptions.MaxBackoffMs.Should().Be(1000);
+            clientOptions.RetryOptions.BackoffMultiplier.Should().Be(2.0);
+            clientOptions.RetryOptions.RetryableStatusCodes.Should().Contain(StatusCode.Unknown);
+        }
+    }
 }