From c013457e24bf6bbfd4047e6486ae39d2eaa1cf04 Mon Sep 17 00:00:00 2001 From: "naiyuantian@microsoft.com" Date: Thu, 26 Jun 2025 12:52:21 -0700 Subject: [PATCH 1/7] initial commit --- ...eClientEncodingStrategyIntegrationTests.cs | 269 ++++++++++++++++++ ...zureStorageOrchestrationServiceSettings.cs | 6 + .../MessageManager.cs | 69 ++++- .../QueueClientEncodingStrategy.cs | 36 +++ .../Storage/AzureStorageClient.cs | 60 +++- src/DurableTask.AzureStorage/Storage/Queue.cs | 1 + 6 files changed, 433 insertions(+), 8 deletions(-) create mode 100644 Test/DurableTask.AzureStorage.Tests/QueueClientEncodingStrategyIntegrationTests.cs create mode 100644 src/DurableTask.AzureStorage/QueueClientEncodingStrategy.cs diff --git a/Test/DurableTask.AzureStorage.Tests/QueueClientEncodingStrategyIntegrationTests.cs b/Test/DurableTask.AzureStorage.Tests/QueueClientEncodingStrategyIntegrationTests.cs new file mode 100644 index 000000000..23b0fb63c --- /dev/null +++ b/Test/DurableTask.AzureStorage.Tests/QueueClientEncodingStrategyIntegrationTests.cs @@ -0,0 +1,269 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +namespace DurableTask.AzureStorage.Tests +{ + using DurableTask.AzureStorage; + using DurableTask.Core; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using System; + using System.Runtime.Serialization; + using System.Threading.Tasks; + + [TestClass] + public class QueueClientEncodingStrategyIntegrationTests + { + private const string TestConnectionString = "UseDevelopmentStorage=true"; + + [TestMethod] + // Verifies that messages sent with Base64 encoding can be processed by a worker with UTF8 encoding + public async Task CrossEncodingCompatibility_Base64ToUtf8() + { + string testName = "Base64ToUtf8"; + string input = "世界! 🌍 Test with émojis and spéciål chàracters: ñáéíóúü"; + + // Create service with Base64 encoding to send messages + var base64Settings = new AzureStorageOrchestrationServiceSettings + { + TaskHubName = testName, + StorageAccountClientProvider = new StorageAccountClientProvider(TestConnectionString), + QueueClientEncodingStrategy = QueueClientEncodingStrategy.Base64, + }; + + var base64Service = new AzureStorageOrchestrationService(base64Settings); + await base64Service.CreateIfNotExistsAsync(); + // DON'T start the service - this prevents the dequeue loop from running + + try + { + // Create orchestration instance with Base64 encoding + var base64Client = new TaskHubClient(base64Service); + var instance = await base64Client.CreateOrchestrationInstanceAsync(typeof(HelloOrchestrator), input); + + // Create worker with UTF8 encoding to process the message + var utf8Settings = new AzureStorageOrchestrationServiceSettings + { + TaskHubName = testName, + StorageAccountClientProvider = new StorageAccountClientProvider(TestConnectionString), + QueueClientEncodingStrategy = QueueClientEncodingStrategy.None, + }; + + var utf8Service = new AzureStorageOrchestrationService(utf8Settings); + var utf8Client = new TaskHubClient(utf8Service); + var worker = new TaskHubWorker(utf8Service); + worker.AddTaskOrchestrations(typeof(HelloOrchestrator)); + worker.AddTaskActivities(typeof(Hello)); + + await worker.StartAsync(); + + try + { + // Wait for the orchestration to complete using UTF8 worker + var state = await utf8Client.WaitForOrchestrationAsync(instance, TimeSpan.FromSeconds(60)); + + // Verify UTF8 worker successfully processed the Base64 encoded message + Assert.IsNotNull(state); + Assert.AreEqual(OrchestrationStatus.Completed, state.OrchestrationStatus); + Assert.AreEqual($"\"Hello, {input}!\"", state.Output); + } + finally + { + await worker.StopAsync(); + } + } + finally + { + await base64Service.DeleteAsync(); + } + } + + [TestMethod] + // Verifies that messages sent with UTF8 encoding can be processed by a worker with Base64 encoding + public async Task CrossEncodingCompatibility_Utf8ToBase64() + { + string testName = "Utf8ToBase64test0"; + string input = "世界! 🌍 Test with émojis and spéciål chàracters: ñáéíóúü"; + + // Create service with UTF8 encoding to send messages + var utf8Settings = new AzureStorageOrchestrationServiceSettings + { + TaskHubName = testName, + StorageAccountClientProvider = new StorageAccountClientProvider(TestConnectionString), + QueueClientEncodingStrategy = QueueClientEncodingStrategy.None, + }; + + var utf8Service = new AzureStorageOrchestrationService(utf8Settings); + await utf8Service.CreateIfNotExistsAsync(); + // DON'T start the service - this prevents the dequeue loop from running + + try + { + // Create orchestration instance with UTF8 encoding + var utf8Client = new TaskHubClient(utf8Service); + var instance = await utf8Client.CreateOrchestrationInstanceAsync(typeof(HelloOrchestrator), input); + + // Create worker with Base64 encoding to process the message + var base64Settings = new AzureStorageOrchestrationServiceSettings + { + TaskHubName = testName, + StorageAccountClientProvider = new StorageAccountClientProvider(TestConnectionString), + QueueClientEncodingStrategy = QueueClientEncodingStrategy.Base64, + }; + + var base64Service = new AzureStorageOrchestrationService(base64Settings); + var base64Client = new TaskHubClient(base64Service); + var worker = new TaskHubWorker(base64Service); + worker.AddTaskOrchestrations(typeof(HelloOrchestrator)); + worker.AddTaskActivities(typeof(Hello)); + + await worker.StartAsync(); + + try + { + // Wait for the orchestration to complete using Base64 worker + var state = await base64Client.WaitForOrchestrationAsync(instance, TimeSpan.FromSeconds(60)); + + // Verify Base64 worker successfully processed the UTF8 encoded message + Assert.IsNotNull(state); + Assert.AreEqual(OrchestrationStatus.Completed, state.OrchestrationStatus); + Assert.AreEqual($"\"Hello, {input}!\"", state.Output); + } + finally + { + await worker.StopAsync(); + } + } + finally + { + await utf8Service.DeleteAsync(); + } + } + + [TestMethod] + // Verifies that messages sent with Base64 encoding can be processed by a worker with Base64 encoding + public async Task SameEncodingStrategy_Base64ToBase64() + { + string testName = "Base64ToBase64"; + string input = "世界! 🌍 Test with émojis and spéciål chàracters: ñáéíóúü"; + + var base64Settings = new AzureStorageOrchestrationServiceSettings + { + TaskHubName = testName, + StorageAccountClientProvider = new StorageAccountClientProvider(TestConnectionString), + QueueClientEncodingStrategy = QueueClientEncodingStrategy.Base64, + }; + + var base64Service = new AzureStorageOrchestrationService(base64Settings); + + try + { + var base64Client = new TaskHubClient(base64Service); + var worker = new TaskHubWorker(base64Service); + worker.AddTaskOrchestrations(typeof(HelloOrchestrator)); + worker.AddTaskActivities(typeof(Hello)); + + await worker.StartAsync(); + + try + { + var instance = await base64Client.CreateOrchestrationInstanceAsync(typeof(HelloOrchestrator), input); + var state = await base64Client.WaitForOrchestrationAsync(instance, TimeSpan.FromSeconds(60)); + + // Verify Base64 worker successfully processed Base64 encoded message + Assert.IsNotNull(state); + Assert.AreEqual(OrchestrationStatus.Completed, state.OrchestrationStatus); + Assert.AreEqual($"\"Hello, {input}!\"", state.Output); + } + finally + { + await worker.StopAsync(); + } + } + finally + { + await base64Service.DeleteAsync(); + } + } + + [TestMethod] + // Verifies that messages sent with UTF8 encoding can be processed by a worker with UTF8 encoding + public async Task SameEncodingStrategy_Utf8ToUtf8() + { + string testName = "Utf8ToUtf8"; + string input = "世界! 🌍 Test with émojis and spéciål chàracters: ñáéíóúü"; + + var utf8Settings = new AzureStorageOrchestrationServiceSettings + { + TaskHubName = testName, + StorageAccountClientProvider = new StorageAccountClientProvider(TestConnectionString), + QueueClientEncodingStrategy = QueueClientEncodingStrategy.None, + }; + + var utf8Service = new AzureStorageOrchestrationService(utf8Settings); + + try + { + var utf8Client = new TaskHubClient(utf8Service); + var worker = new TaskHubWorker(utf8Service); + worker.AddTaskOrchestrations(typeof(HelloOrchestrator)); + worker.AddTaskActivities(typeof(Hello)); + + await worker.StartAsync(); + + try + { + var instance = await utf8Client.CreateOrchestrationInstanceAsync(typeof(HelloOrchestrator), input); + var state = await utf8Client.WaitForOrchestrationAsync(instance, TimeSpan.FromSeconds(60)); + + // Verify UTF8 worker successfully processed UTF8 encoded message + Assert.IsNotNull(state); + Assert.AreEqual(OrchestrationStatus.Completed, state.OrchestrationStatus); + Assert.AreEqual($"\"Hello, {input}!\"", state.Output); + } + finally + { + await worker.StopAsync(); + } + } + finally + { + await utf8Service.DeleteAsync(); + } + } + + // Test orchestrator and activity + [KnownType(typeof(Hello))] + internal class HelloOrchestrator : TaskOrchestration + { + public override async Task RunTask(OrchestrationContext context, string input) + { + // Wait for 5 seconds before executing the activity (shorter for faster tests) + // await context.CreateTimer(context.CurrentUtcDateTime.AddSeconds(5), null); + return await context.ScheduleTask(typeof(Hello), input); + } + } + + internal class Hello : TaskActivity + { + protected override string Execute(TaskContext context, string input) + { + if (string.IsNullOrEmpty(input)) + { + throw new ArgumentNullException(nameof(input)); + } + + return $"Hello, {input}!"; + } + } + } +} \ No newline at end of file diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs index a5dd568b2..e57bb96a4 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs @@ -288,5 +288,11 @@ internal LogHelper Logger /// Consumers that require separate dispatch (such as the new out-of-proc v2 SDKs) must set this to true. /// public bool UseSeparateQueueForEntityWorkItems { get; set; } = false; + + /// + /// Gets or sets the encoding strategy used for Azure Storage Queue messages. + /// The default is . + /// + public QueueClientEncodingStrategy QueueClientEncodingStrategy { get; set; } = QueueClientEncodingStrategy.None; } } diff --git a/src/DurableTask.AzureStorage/MessageManager.cs b/src/DurableTask.AzureStorage/MessageManager.cs index a1fc796cf..7c77d2b1f 100644 --- a/src/DurableTask.AzureStorage/MessageManager.cs +++ b/src/DurableTask.AzureStorage/MessageManager.cs @@ -156,16 +156,71 @@ public async Task DeserializeQueueMessageAsync(QueueMessage queueMe { // TODO: Deserialize with Stream? byte[] body = queueMessage.Body.ToArray(); - MessageData envelope; - try + MessageData envelope = null; + + if (this.azureStorageClient.Settings.QueueClientEncodingStrategy == QueueClientEncodingStrategy.Base64) { - envelope = this.DeserializeMessageData(Encoding.UTF8.GetString(body)); + try + { + // For a queue client using the Base64 encoding strategy, error handling is configured via the options. + // Therefore, we should always receive messages encoded in Base64 here. + string messageText = queueMessage.Body.ToString(); + envelope = this.DeserializeMessageData(messageText); + } + catch (Exception ex) + { + this.azureStorageClient.Settings.Logger.GeneralWarning( + this.azureStorageClient.QueueAccountName, + this.azureStorageClient.Settings.TaskHubName, + $"Failed to process message with Base64 client fallback. MessageId: {queueMessage.MessageId}, Error: {ex.Message}"); + + // Re-throw the original exception to maintain the original behavior + throw; + } } - catch(JsonReaderException) + else // queue client use Utf8-encoding. + { + try + { + // For UTF8 encoding strategy, try to decode as UTF8 first + envelope = this.DeserializeMessageData(Encoding.UTF8.GetString(body)); + } + catch (JsonReaderException) + { + // If UTF8 decode fails, the message might be Base64 encoded + // This can happen when MessageDecodingFailed event was triggered + // In this case, queueMessage.Body contains raw Base64 bytes + try + { + // Convert raw bytes to Base64 string, then decode Base64 to get original message + string base64String = Encoding.UTF8.GetString(body); + byte[] decodedBytes = Convert.FromBase64String(base64String); + string decodedMessage = Encoding.UTF8.GetString(decodedBytes); + envelope = this.DeserializeMessageData(decodedMessage); + + // Log successful cross-encoding processing + this.azureStorageClient.Settings.Logger.GeneralWarning( + this.azureStorageClient.QueueAccountName, + this.azureStorageClient.Settings.TaskHubName, + $"Successfully processed Base64 message with UTF8 client. MessageId: {queueMessage.MessageId}"); + } + catch (Exception fallbackException) + { + // Log the fallback failure + this.azureStorageClient.Settings.Logger.GeneralWarning( + this.azureStorageClient.QueueAccountName, + this.azureStorageClient.Settings.TaskHubName, + $"Failed to process message with UTF8 client fallback. MessageId: {queueMessage.MessageId}, Error: {fallbackException.Message}"); + + // Re-throw the original exception to maintain the original behavior + throw; + } + } + } + + if (envelope == null) { - // This catch block is a hotfix and better implementation might be needed in future. - // DTFx.AzureStorage 1.x and 2.x use different encoding methods. Adding this line to enable forward compatibility. - envelope = this.DeserializeMessageData(Encoding.UTF8.GetString(Convert.FromBase64String(Encoding.UTF8.GetString(body)))); + throw new InvalidOperationException($"Failed to deserialize message {queueMessage.MessageId} with encoding strategy {this.azureStorageClient.Settings.QueueClientEncodingStrategy}"); } if (!string.IsNullOrEmpty(envelope.CompressedBlobName)) diff --git a/src/DurableTask.AzureStorage/QueueClientEncodingStrategy.cs b/src/DurableTask.AzureStorage/QueueClientEncodingStrategy.cs new file mode 100644 index 000000000..e854d10b6 --- /dev/null +++ b/src/DurableTask.AzureStorage/QueueClientEncodingStrategy.cs @@ -0,0 +1,36 @@ +// ---------------------------------------------------------------------------------- +// Copyright Microsoft Corporation +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ---------------------------------------------------------------------------------- + +using Azure.Storage.Queues; + +namespace DurableTask.AzureStorage +{ + /// + /// Specifies the encoding strategy used for Azure Storage Queue messages. + /// This enum maps to the Azure Storage SDK's values. + /// + public enum QueueClientEncodingStrategy + { + /// + /// Use UTF8 encoding for queue messages. Maps to . + /// + None = 0, + + /// + /// Use Base64 encoding for queue messages. Maps to . + /// This provides compatibility with older versions and may be required in some scenarios + /// where UTF8 encoding is not supported. + /// + Base64 = 1, + } +} \ No newline at end of file diff --git a/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs b/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs index 9e459fcc7..fd5938f06 100644 --- a/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs +++ b/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs @@ -14,6 +14,7 @@ namespace DurableTask.AzureStorage.Storage { using System; + using System.Text; using Azure.Core; using Azure.Data.Tables; using Azure.Storage.Blobs; @@ -46,7 +47,7 @@ public AzureStorageClient(AzureStorageOrchestrationServiceSettings settings) var timeoutPolicy = new LeaseTimeoutHttpPipelinePolicy(this.Settings.LeaseRenewInterval); var monitoringPolicy = new MonitoringHttpPipelinePolicy(this.Stats); - this.queueClient = CreateClient(settings.StorageAccountClientProvider.Queue, ConfigureClientPolicies); + this.queueClient = CreateClient(settings.StorageAccountClientProvider.Queue, ConfigureQueueClientPolicies); if (settings.HasTrackingStoreStorageAccount) { this.blobClient = CreateClient(settings.TrackingServiceClientProvider!.Blob, ConfigureClientPolicies); @@ -64,6 +65,63 @@ void ConfigureClientPolicies(TClientOptions options) where TClie options.AddPolicy(timeoutPolicy!, HttpPipelinePosition.PerCall); options.AddPolicy(monitoringPolicy!, HttpPipelinePosition.PerRetry); } + + void ConfigureQueueClientPolicies(QueueClientOptions options) + { + // Configure message encoding based on settings + options.MessageEncoding = this.Settings.QueueClientEncodingStrategy switch + { + QueueClientEncodingStrategy.None => QueueMessageEncoding.None, + QueueClientEncodingStrategy.Base64 => QueueMessageEncoding.Base64, + _ => throw new ArgumentException($"Unsupported encoding strategy: {this.Settings.QueueClientEncodingStrategy}") + }; + + // Base64-encoded clients will fail to decode messages sent in UTF-8 format. + // This handler catches decoding failures and update the message with its the original content, + // so that the client can successfully process it on the next attempt. + if (this.Settings.QueueClientEncodingStrategy == QueueClientEncodingStrategy.Base64) + { + options.MessageDecodingFailed += async (QueueMessageDecodingFailedEventArgs args) => + { + if (args.ReceivedMessage != null) + { + var queueMessage = args.ReceivedMessage; + + try + { + // Get the raw message content and update the message. + string originalJson = Encoding.UTF8.GetString(queueMessage.Body.ToArray()); + + // Update the message in the queue with the Base64-encoded body + if (args.IsRunningSynchronously) + { + args.Queue.UpdateMessage( + queueMessage.MessageId, + queueMessage.PopReceipt, + originalJson, + TimeSpan.FromSeconds(0)); + } + else + { + await args.Queue.UpdateMessageAsync( + queueMessage.MessageId, + queueMessage.PopReceipt, + originalJson, + TimeSpan.FromSeconds(0)); + } + } + catch (Exception ex) + { + // If re-encoding or update fails, rethrow the error to be handled upstream + throw new InvalidOperationException( + $"Failed to re-encode and update UTF-8 message as Base64. MessageId: {queueMessage.MessageId}", ex); + } + } + }; + } + + ConfigureClientPolicies(options); + } } public AzureStorageOrchestrationServiceSettings Settings { get; } diff --git a/src/DurableTask.AzureStorage/Storage/Queue.cs b/src/DurableTask.AzureStorage/Storage/Queue.cs index a31913306..84f695198 100644 --- a/src/DurableTask.AzureStorage/Storage/Queue.cs +++ b/src/DurableTask.AzureStorage/Storage/Queue.cs @@ -48,6 +48,7 @@ public async Task GetApproximateMessagesCountAsync(CancellationToken cancel public async Task AddMessageAsync(string message, TimeSpan? visibilityDelay, Guid? clientRequestId = null, CancellationToken cancellationToken = default) { using IDisposable scope = OperationContext.CreateClientRequestScope(clientRequestId); + await this.queueClient .SendMessageAsync( message, From 64912e1c07b0d36bd12a024c053b80746cadb921 Mon Sep 17 00:00:00 2001 From: "naiyuantian@microsoft.com" Date: Thu, 3 Jul 2025 15:20:42 -0700 Subject: [PATCH 2/7] udpate by comments --- ...eClientEncodingStrategyIntegrationTests.cs | 53 +++++++++++++++++-- ...zureStorageOrchestrationServiceSettings.cs | 4 +- .../QueueClientEncodingStrategy.cs | 4 +- .../Storage/AzureStorageClient.cs | 2 +- src/DurableTask.AzureStorage/Storage/Queue.cs | 1 - 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/Test/DurableTask.AzureStorage.Tests/QueueClientEncodingStrategyIntegrationTests.cs b/Test/DurableTask.AzureStorage.Tests/QueueClientEncodingStrategyIntegrationTests.cs index 23b0fb63c..61cc89e04 100644 --- a/Test/DurableTask.AzureStorage.Tests/QueueClientEncodingStrategyIntegrationTests.cs +++ b/Test/DurableTask.AzureStorage.Tests/QueueClientEncodingStrategyIntegrationTests.cs @@ -55,7 +55,7 @@ public async Task CrossEncodingCompatibility_Base64ToUtf8() { TaskHubName = testName, StorageAccountClientProvider = new StorageAccountClientProvider(TestConnectionString), - QueueClientEncodingStrategy = QueueClientEncodingStrategy.None, + QueueClientEncodingStrategy = QueueClientEncodingStrategy.UTF8, }; var utf8Service = new AzureStorageOrchestrationService(utf8Settings); @@ -99,7 +99,7 @@ public async Task CrossEncodingCompatibility_Utf8ToBase64() { TaskHubName = testName, StorageAccountClientProvider = new StorageAccountClientProvider(TestConnectionString), - QueueClientEncodingStrategy = QueueClientEncodingStrategy.None, + QueueClientEncodingStrategy = QueueClientEncodingStrategy.UTF8, }; var utf8Service = new AzureStorageOrchestrationService(utf8Settings); @@ -206,7 +206,7 @@ public async Task SameEncodingStrategy_Utf8ToUtf8() { TaskHubName = testName, StorageAccountClientProvider = new StorageAccountClientProvider(TestConnectionString), - QueueClientEncodingStrategy = QueueClientEncodingStrategy.None, + QueueClientEncodingStrategy = QueueClientEncodingStrategy.UTF8, }; var utf8Service = new AzureStorageOrchestrationService(utf8Settings); @@ -241,6 +241,53 @@ public async Task SameEncodingStrategy_Utf8ToUtf8() } } + [TestMethod] + // Verifies that Base64 encoding can handle non-UTF8 characters like 0xFFFE (Byte Order Mark) + public async Task Base64Encodig_HandlesNonUtf8Characters() + { + string testName = "Base64WithUtf8Chars"; + // Create a string with non-UTF8 characters including 0xFFFE (Byte Order Mark) + string input = "Normal text " + (char)0xFFFE + " with BOM and " + (char)0xFFFF + " other invalid chars"; + + var base64Settings = new AzureStorageOrchestrationServiceSettings + { + TaskHubName = testName, + StorageAccountClientProvider = new StorageAccountClientProvider(TestConnectionString), + QueueClientEncodingStrategy = QueueClientEncodingStrategy.Base64, + }; + + var base64Service = new AzureStorageOrchestrationService(base64Settings); + + try + { + var base64Client = new TaskHubClient(base64Service); + var worker = new TaskHubWorker(base64Service); + worker.AddTaskOrchestrations(typeof(HelloOrchestrator)); + worker.AddTaskActivities(typeof(Hello)); + + await worker.StartAsync(); + + try + { + var instance = await base64Client.CreateOrchestrationInstanceAsync(typeof(HelloOrchestrator), input); + var state = await base64Client.WaitForOrchestrationAsync(instance, TimeSpan.FromSeconds(60)); + + // Verify Base64 encoding successfully handled non-UTF8 characters + Assert.IsNotNull(state); + Assert.AreEqual(OrchestrationStatus.Completed, state.OrchestrationStatus); + Assert.AreEqual($"\"Hello, {input}!\"", state.Output); + } + finally + { + await worker.StopAsync(); + } + } + finally + { + await base64Service.DeleteAsync(); + } + } + // Test orchestrator and activity [KnownType(typeof(Hello))] internal class HelloOrchestrator : TaskOrchestration diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs index e57bb96a4..e34fc0d85 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs @@ -291,8 +291,8 @@ internal LogHelper Logger /// /// Gets or sets the encoding strategy used for Azure Storage Queue messages. - /// The default is . + /// The default is . /// - public QueueClientEncodingStrategy QueueClientEncodingStrategy { get; set; } = QueueClientEncodingStrategy.None; + public QueueClientEncodingStrategy QueueClientEncodingStrategy { get; set; } = QueueClientEncodingStrategy.UTF8; } } diff --git a/src/DurableTask.AzureStorage/QueueClientEncodingStrategy.cs b/src/DurableTask.AzureStorage/QueueClientEncodingStrategy.cs index e854d10b6..899bb4e7d 100644 --- a/src/DurableTask.AzureStorage/QueueClientEncodingStrategy.cs +++ b/src/DurableTask.AzureStorage/QueueClientEncodingStrategy.cs @@ -24,7 +24,7 @@ public enum QueueClientEncodingStrategy /// /// Use UTF8 encoding for queue messages. Maps to . /// - None = 0, + UTF8 = 0, /// /// Use Base64 encoding for queue messages. Maps to . @@ -33,4 +33,4 @@ public enum QueueClientEncodingStrategy /// Base64 = 1, } -} \ No newline at end of file +} diff --git a/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs b/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs index fd5938f06..8216549a3 100644 --- a/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs +++ b/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs @@ -71,7 +71,7 @@ void ConfigureQueueClientPolicies(QueueClientOptions options) // Configure message encoding based on settings options.MessageEncoding = this.Settings.QueueClientEncodingStrategy switch { - QueueClientEncodingStrategy.None => QueueMessageEncoding.None, + QueueClientEncodingStrategy.UTF8 => QueueMessageEncoding.None, QueueClientEncodingStrategy.Base64 => QueueMessageEncoding.Base64, _ => throw new ArgumentException($"Unsupported encoding strategy: {this.Settings.QueueClientEncodingStrategy}") }; diff --git a/src/DurableTask.AzureStorage/Storage/Queue.cs b/src/DurableTask.AzureStorage/Storage/Queue.cs index 84f695198..a31913306 100644 --- a/src/DurableTask.AzureStorage/Storage/Queue.cs +++ b/src/DurableTask.AzureStorage/Storage/Queue.cs @@ -48,7 +48,6 @@ public async Task GetApproximateMessagesCountAsync(CancellationToken cancel public async Task AddMessageAsync(string message, TimeSpan? visibilityDelay, Guid? clientRequestId = null, CancellationToken cancellationToken = default) { using IDisposable scope = OperationContext.CreateClientRequestScope(clientRequestId); - await this.queueClient .SendMessageAsync( message, From ad2d1d0998c4593617bd8e7741bb5577b279d10d Mon Sep 17 00:00:00 2001 From: "naiyuantian@microsoft.com" Date: Sun, 6 Jul 2025 20:49:04 -0700 Subject: [PATCH 3/7] udpate by comments --- ...eClientEncodingStrategyIntegrationTests.cs | 16 +++++++------- ...zureStorageOrchestrationServiceSettings.cs | 4 ++-- .../MessageManager.cs | 17 ++++++++------- .../QueueClientEncodingStrategy.cs | 2 +- .../Storage/AzureStorageClient.cs | 21 +++++++++++++------ 5 files changed, 35 insertions(+), 25 deletions(-) diff --git a/Test/DurableTask.AzureStorage.Tests/QueueClientEncodingStrategyIntegrationTests.cs b/Test/DurableTask.AzureStorage.Tests/QueueClientEncodingStrategyIntegrationTests.cs index 61cc89e04..42e0f00a0 100644 --- a/Test/DurableTask.AzureStorage.Tests/QueueClientEncodingStrategyIntegrationTests.cs +++ b/Test/DurableTask.AzureStorage.Tests/QueueClientEncodingStrategyIntegrationTests.cs @@ -21,7 +21,7 @@ namespace DurableTask.AzureStorage.Tests using System.Threading.Tasks; [TestClass] - public class QueueClientEncodingStrategyIntegrationTests + public class QueueClientMessageEncodingIntegrationTests { private const string TestConnectionString = "UseDevelopmentStorage=true"; @@ -37,7 +37,7 @@ public async Task CrossEncodingCompatibility_Base64ToUtf8() { TaskHubName = testName, StorageAccountClientProvider = new StorageAccountClientProvider(TestConnectionString), - QueueClientEncodingStrategy = QueueClientEncodingStrategy.Base64, + QueueClientMessageEncoding = QueueClientMessageEncoding.Base64, }; var base64Service = new AzureStorageOrchestrationService(base64Settings); @@ -55,7 +55,7 @@ public async Task CrossEncodingCompatibility_Base64ToUtf8() { TaskHubName = testName, StorageAccountClientProvider = new StorageAccountClientProvider(TestConnectionString), - QueueClientEncodingStrategy = QueueClientEncodingStrategy.UTF8, + QueueClientMessageEncoding = QueueClientMessageEncoding.UTF8, }; var utf8Service = new AzureStorageOrchestrationService(utf8Settings); @@ -99,7 +99,7 @@ public async Task CrossEncodingCompatibility_Utf8ToBase64() { TaskHubName = testName, StorageAccountClientProvider = new StorageAccountClientProvider(TestConnectionString), - QueueClientEncodingStrategy = QueueClientEncodingStrategy.UTF8, + QueueClientMessageEncoding = QueueClientMessageEncoding.UTF8, }; var utf8Service = new AzureStorageOrchestrationService(utf8Settings); @@ -117,7 +117,7 @@ public async Task CrossEncodingCompatibility_Utf8ToBase64() { TaskHubName = testName, StorageAccountClientProvider = new StorageAccountClientProvider(TestConnectionString), - QueueClientEncodingStrategy = QueueClientEncodingStrategy.Base64, + QueueClientMessageEncoding = QueueClientMessageEncoding.Base64, }; var base64Service = new AzureStorageOrchestrationService(base64Settings); @@ -160,7 +160,7 @@ public async Task SameEncodingStrategy_Base64ToBase64() { TaskHubName = testName, StorageAccountClientProvider = new StorageAccountClientProvider(TestConnectionString), - QueueClientEncodingStrategy = QueueClientEncodingStrategy.Base64, + QueueClientMessageEncoding = QueueClientMessageEncoding.Base64, }; var base64Service = new AzureStorageOrchestrationService(base64Settings); @@ -206,7 +206,7 @@ public async Task SameEncodingStrategy_Utf8ToUtf8() { TaskHubName = testName, StorageAccountClientProvider = new StorageAccountClientProvider(TestConnectionString), - QueueClientEncodingStrategy = QueueClientEncodingStrategy.UTF8, + QueueClientMessageEncoding = QueueClientMessageEncoding.UTF8, }; var utf8Service = new AzureStorageOrchestrationService(utf8Settings); @@ -253,7 +253,7 @@ public async Task Base64Encodig_HandlesNonUtf8Characters() { TaskHubName = testName, StorageAccountClientProvider = new StorageAccountClientProvider(TestConnectionString), - QueueClientEncodingStrategy = QueueClientEncodingStrategy.Base64, + QueueClientMessageEncoding = QueueClientMessageEncoding.Base64, }; var base64Service = new AzureStorageOrchestrationService(base64Settings); diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs index e34fc0d85..48e653868 100644 --- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs +++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs @@ -291,8 +291,8 @@ internal LogHelper Logger /// /// Gets or sets the encoding strategy used for Azure Storage Queue messages. - /// The default is . + /// The default is . /// - public QueueClientEncodingStrategy QueueClientEncodingStrategy { get; set; } = QueueClientEncodingStrategy.UTF8; + public QueueClientMessageEncoding QueueClientMessageEncoding { get; set; } = QueueClientMessageEncoding.UTF8; } } diff --git a/src/DurableTask.AzureStorage/MessageManager.cs b/src/DurableTask.AzureStorage/MessageManager.cs index 7c77d2b1f..9c05fd42d 100644 --- a/src/DurableTask.AzureStorage/MessageManager.cs +++ b/src/DurableTask.AzureStorage/MessageManager.cs @@ -158,7 +158,7 @@ public async Task DeserializeQueueMessageAsync(QueueMessage queueMe byte[] body = queueMessage.Body.ToArray(); MessageData envelope = null; - if (this.azureStorageClient.Settings.QueueClientEncodingStrategy == QueueClientEncodingStrategy.Base64) + if (this.azureStorageClient.Settings.QueueClientMessageEncoding == QueueClientMessageEncoding.Base64) { try { @@ -192,17 +192,18 @@ public async Task DeserializeQueueMessageAsync(QueueMessage queueMe // In this case, queueMessage.Body contains raw Base64 bytes try { + this.azureStorageClient.Settings.Logger.GeneralWarning( + this.azureStorageClient.QueueAccountName, + this.azureStorageClient.Settings.TaskHubName, + $"Failed to deserialize queue message using UTF-8 encoding. " + + $"First few characters are: \"{Encoding.UTF8.GetString(body, 0, Math.Min(10, body.Length))}\". " + + $"Assuming this is a Base64-encoded message created with a different message encoding configuration."); + // Convert raw bytes to Base64 string, then decode Base64 to get original message string base64String = Encoding.UTF8.GetString(body); byte[] decodedBytes = Convert.FromBase64String(base64String); string decodedMessage = Encoding.UTF8.GetString(decodedBytes); envelope = this.DeserializeMessageData(decodedMessage); - - // Log successful cross-encoding processing - this.azureStorageClient.Settings.Logger.GeneralWarning( - this.azureStorageClient.QueueAccountName, - this.azureStorageClient.Settings.TaskHubName, - $"Successfully processed Base64 message with UTF8 client. MessageId: {queueMessage.MessageId}"); } catch (Exception fallbackException) { @@ -220,7 +221,7 @@ public async Task DeserializeQueueMessageAsync(QueueMessage queueMe if (envelope == null) { - throw new InvalidOperationException($"Failed to deserialize message {queueMessage.MessageId} with encoding strategy {this.azureStorageClient.Settings.QueueClientEncodingStrategy}"); + throw new InvalidOperationException($"Failed to deserialize message {queueMessage.MessageId} with encoding strategy {this.azureStorageClient.Settings.QueueClientMessageEncoding}"); } if (!string.IsNullOrEmpty(envelope.CompressedBlobName)) diff --git a/src/DurableTask.AzureStorage/QueueClientEncodingStrategy.cs b/src/DurableTask.AzureStorage/QueueClientEncodingStrategy.cs index 899bb4e7d..4516ea7c7 100644 --- a/src/DurableTask.AzureStorage/QueueClientEncodingStrategy.cs +++ b/src/DurableTask.AzureStorage/QueueClientEncodingStrategy.cs @@ -19,7 +19,7 @@ namespace DurableTask.AzureStorage /// Specifies the encoding strategy used for Azure Storage Queue messages. /// This enum maps to the Azure Storage SDK's values. /// - public enum QueueClientEncodingStrategy + public enum QueueClientMessageEncoding { /// /// Use UTF8 encoding for queue messages. Maps to . diff --git a/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs b/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs index 8216549a3..76b8a2678 100644 --- a/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs +++ b/src/DurableTask.AzureStorage/Storage/AzureStorageClient.cs @@ -69,20 +69,29 @@ void ConfigureClientPolicies(TClientOptions options) where TClie void ConfigureQueueClientPolicies(QueueClientOptions options) { // Configure message encoding based on settings - options.MessageEncoding = this.Settings.QueueClientEncodingStrategy switch + options.MessageEncoding = this.Settings.QueueClientMessageEncoding switch { - QueueClientEncodingStrategy.UTF8 => QueueMessageEncoding.None, - QueueClientEncodingStrategy.Base64 => QueueMessageEncoding.Base64, - _ => throw new ArgumentException($"Unsupported encoding strategy: {this.Settings.QueueClientEncodingStrategy}") + QueueClientMessageEncoding.UTF8 => QueueMessageEncoding.None, + QueueClientMessageEncoding.Base64 => QueueMessageEncoding.Base64, + _ => throw new ArgumentException($"Unsupported encoding strategy: {this.Settings.QueueClientMessageEncoding}") }; // Base64-encoded clients will fail to decode messages sent in UTF-8 format. // This handler catches decoding failures and update the message with its the original content, // so that the client can successfully process it on the next attempt. - if (this.Settings.QueueClientEncodingStrategy == QueueClientEncodingStrategy.Base64) + if (this.Settings.QueueClientMessageEncoding == QueueClientMessageEncoding.Base64) { options.MessageDecodingFailed += async (QueueMessageDecodingFailedEventArgs args) => { + this.Settings.Logger.GeneralWarning( + this.QueueAccountName, + this.Settings.TaskHubName, + $"Base64-encoded queue client failed to decode message with ID: {args.ReceivedMessage.MessageId}. " + + "The message appears to have been originally sent using UTF-8 encoding. " + + "Will attempt to re-encode the message content as Base64 and update the queue message in-place " + + "so it can be successfully processed on the next attempt." + ); + if (args.ReceivedMessage != null) { var queueMessage = args.ReceivedMessage; @@ -120,7 +129,7 @@ await args.Queue.UpdateMessageAsync( }; } - ConfigureClientPolicies(options); + ConfigureClientPolicies(options); } } From 2e0f2ba36137ba5dfd15f5768298fcfddfac2d0c Mon Sep 17 00:00:00 2001 From: "naiyuantian@microsoft.com" Date: Tue, 8 Jul 2025 11:55:15 -0700 Subject: [PATCH 4/7] update by comment --- .../MessageManager.cs | 91 ++++++++----------- 1 file changed, 39 insertions(+), 52 deletions(-) diff --git a/src/DurableTask.AzureStorage/MessageManager.cs b/src/DurableTask.AzureStorage/MessageManager.cs index 9c05fd42d..094659564 100644 --- a/src/DurableTask.AzureStorage/MessageManager.cs +++ b/src/DurableTask.AzureStorage/MessageManager.cs @@ -154,74 +154,61 @@ internal static bool TryGetLargeMessageReference(string messagePayload, out Uri public async Task DeserializeQueueMessageAsync(QueueMessage queueMessage, string queueName, CancellationToken cancellationToken = default) { - // TODO: Deserialize with Stream? - byte[] body = queueMessage.Body.ToArray(); + // No matter what the queue client encoding is (Base64 or UTF8), + // we should always successfully get the string here. + string bodyAsString = queueMessage.Body.ToString(); MessageData envelope = null; - if (this.azureStorageClient.Settings.QueueClientMessageEncoding == QueueClientMessageEncoding.Base64) + try + { + envelope = this.DeserializeMessageData(bodyAsString); + } + catch (JsonReaderException) { + // This catch block is especially for the case where the queue client encoding is UTF8 + // but it receives a message that was sent by a queue client using Base64 encoding. + // In this case, queueMessage.Body contains raw Base64 bytes try { - // For a queue client using the Base64 encoding strategy, error handling is configured via the options. - // Therefore, we should always receive messages encoded in Base64 here. - string messageText = queueMessage.Body.ToString(); - envelope = this.DeserializeMessageData(messageText); + this.azureStorageClient.Settings.Logger.GeneralWarning( + this.azureStorageClient.QueueAccountName, + this.azureStorageClient.Settings.TaskHubName, + $"Failed to deserialize queue message using UTF-8 encoding. " + + $"First few characters are: \"{bodyAsString.Substring(0, Math.Min(10, bodyAsString.Length))}\". " + + $"Assuming this is a Base64-encoded message created with a different message encoding configuration."); + + // Convert raw bytes to Base64 string, then decode Base64 to get original message + byte[] decodedBytes = Convert.FromBase64String(bodyAsString); + string decodedMessage = Encoding.UTF8.GetString(decodedBytes); + envelope = this.DeserializeMessageData(decodedMessage); } - catch (Exception ex) + catch (Exception fallbackException) { + // Log the fallback failure this.azureStorageClient.Settings.Logger.GeneralWarning( this.azureStorageClient.QueueAccountName, this.azureStorageClient.Settings.TaskHubName, - $"Failed to process message with Base64 client fallback. MessageId: {queueMessage.MessageId}, Error: {ex.Message}"); - - // Re-throw the original exception to maintain the original behavior - throw; + $"Failed to process message with UTF8 client fallback. MessageId: {queueMessage.MessageId}, Error: {fallbackException.Message}"); + + // Re-throw the original exception to maintain the original behavior + throw; } } - else // queue client use Utf8-encoding. + catch (Exception ex) { - try - { - // For UTF8 encoding strategy, try to decode as UTF8 first - envelope = this.DeserializeMessageData(Encoding.UTF8.GetString(body)); - } - catch (JsonReaderException) - { - // If UTF8 decode fails, the message might be Base64 encoded - // This can happen when MessageDecodingFailed event was triggered - // In this case, queueMessage.Body contains raw Base64 bytes - try - { - this.azureStorageClient.Settings.Logger.GeneralWarning( - this.azureStorageClient.QueueAccountName, - this.azureStorageClient.Settings.TaskHubName, - $"Failed to deserialize queue message using UTF-8 encoding. " + - $"First few characters are: \"{Encoding.UTF8.GetString(body, 0, Math.Min(10, body.Length))}\". " + - $"Assuming this is a Base64-encoded message created with a different message encoding configuration."); - - // Convert raw bytes to Base64 string, then decode Base64 to get original message - string base64String = Encoding.UTF8.GetString(body); - byte[] decodedBytes = Convert.FromBase64String(base64String); - string decodedMessage = Encoding.UTF8.GetString(decodedBytes); - envelope = this.DeserializeMessageData(decodedMessage); - } - catch (Exception fallbackException) - { - // Log the fallback failure - this.azureStorageClient.Settings.Logger.GeneralWarning( - this.azureStorageClient.QueueAccountName, - this.azureStorageClient.Settings.TaskHubName, - $"Failed to process message with UTF8 client fallback. MessageId: {queueMessage.MessageId}, Error: {fallbackException.Message}"); - - // Re-throw the original exception to maintain the original behavior - throw; - } - } + // Note: For a Base64 queue client, it is supposed to always receive a Base64-encoded queue message. + // Becasue error handling for the case where a Base64 queue client receives a UTF8 message is implemented in AzureStorageClient.cs. + // This exception catch block handles any other unexpected errors during deserialization. + this.azureStorageClient.Settings.Logger.GeneralWarning( + this.azureStorageClient.QueueAccountName, + this.azureStorageClient.Settings.TaskHubName, + $"Failed to process message. MessageId: {queueMessage.MessageId}, Error: {ex.Message}"); + throw; } if (envelope == null) { - throw new InvalidOperationException($"Failed to deserialize message {queueMessage.MessageId} with encoding strategy {this.azureStorageClient.Settings.QueueClientMessageEncoding}"); + throw new InvalidOperationException($"Failed to deserialize message {queueMessage.MessageId}"); } if (!string.IsNullOrEmpty(envelope.CompressedBlobName)) @@ -233,7 +220,7 @@ public async Task DeserializeQueueMessageAsync(QueueMessage queueMe } else { - envelope.TotalMessageSizeBytes = body.Length; + envelope.TotalMessageSizeBytes = Encoding.UTF8.GetByteCount(bodyAsString); } envelope.OriginalQueueMessage = queueMessage; From 181b7e0311ac79152215219e0db8734e226f4612 Mon Sep 17 00:00:00 2001 From: "naiyuantian@microsoft.com" Date: Wed, 9 Jul 2025 10:35:51 -0700 Subject: [PATCH 5/7] update --- .../MessageManager.cs | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/src/DurableTask.AzureStorage/MessageManager.cs b/src/DurableTask.AzureStorage/MessageManager.cs index 094659564..a95164943 100644 --- a/src/DurableTask.AzureStorage/MessageManager.cs +++ b/src/DurableTask.AzureStorage/MessageManager.cs @@ -161,15 +161,19 @@ public async Task DeserializeQueueMessageAsync(QueueMessage queueMe try { - envelope = this.DeserializeMessageData(bodyAsString); - } - catch (JsonReaderException) - { - // This catch block is especially for the case where the queue client encoding is UTF8 - // but it receives a message that was sent by a queue client using Base64 encoding. - // In this case, queueMessage.Body contains raw Base64 bytes - try + // Check if the message starts with '{' which indicates it's a JSON message + // If so, deserialize it directly. Otherwise, try Base64 decoding strategy. + if (bodyAsString.TrimStart().StartsWith("{")) { + envelope = this.DeserializeMessageData(bodyAsString); + } + else + { + // The message we got is not a valid json message (doesn't start with '{'). + // This could happen in the case where our queue client is UTF8 encoding + // while receiving a message sent by a Base64 queue client. + // So we try to re-decode it as Base64. + this.azureStorageClient.Settings.Logger.GeneralWarning( this.azureStorageClient.QueueAccountName, this.azureStorageClient.Settings.TaskHubName, @@ -177,22 +181,10 @@ public async Task DeserializeQueueMessageAsync(QueueMessage queueMe $"First few characters are: \"{bodyAsString.Substring(0, Math.Min(10, bodyAsString.Length))}\". " + $"Assuming this is a Base64-encoded message created with a different message encoding configuration."); - // Convert raw bytes to Base64 string, then decode Base64 to get original message byte[] decodedBytes = Convert.FromBase64String(bodyAsString); string decodedMessage = Encoding.UTF8.GetString(decodedBytes); envelope = this.DeserializeMessageData(decodedMessage); } - catch (Exception fallbackException) - { - // Log the fallback failure - this.azureStorageClient.Settings.Logger.GeneralWarning( - this.azureStorageClient.QueueAccountName, - this.azureStorageClient.Settings.TaskHubName, - $"Failed to process message with UTF8 client fallback. MessageId: {queueMessage.MessageId}, Error: {fallbackException.Message}"); - - // Re-throw the original exception to maintain the original behavior - throw; - } } catch (Exception ex) { From 20bc287e0fb2c24b9645170b056702715e5739f7 Mon Sep 17 00:00:00 2001 From: "naiyuantian@microsoft.com" Date: Wed, 9 Jul 2025 10:42:32 -0700 Subject: [PATCH 6/7] remove unnecessary trimstart --- src/DurableTask.AzureStorage/MessageManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.AzureStorage/MessageManager.cs b/src/DurableTask.AzureStorage/MessageManager.cs index a95164943..1c834a545 100644 --- a/src/DurableTask.AzureStorage/MessageManager.cs +++ b/src/DurableTask.AzureStorage/MessageManager.cs @@ -163,7 +163,7 @@ public async Task DeserializeQueueMessageAsync(QueueMessage queueMe { // Check if the message starts with '{' which indicates it's a JSON message // If so, deserialize it directly. Otherwise, try Base64 decoding strategy. - if (bodyAsString.TrimStart().StartsWith("{")) + if (bodyAsString.StartsWith("{")) { envelope = this.DeserializeMessageData(bodyAsString); } From 2e5388cf548e62eb9d20897cc6c32d38eb48884d Mon Sep 17 00:00:00 2001 From: "naiyuantian@microsoft.com" Date: Thu, 10 Jul 2025 09:31:51 -0700 Subject: [PATCH 7/7] remove logging --- src/DurableTask.AzureStorage/MessageManager.cs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/DurableTask.AzureStorage/MessageManager.cs b/src/DurableTask.AzureStorage/MessageManager.cs index 1c834a545..ff48d507f 100644 --- a/src/DurableTask.AzureStorage/MessageManager.cs +++ b/src/DurableTask.AzureStorage/MessageManager.cs @@ -173,14 +173,6 @@ public async Task DeserializeQueueMessageAsync(QueueMessage queueMe // This could happen in the case where our queue client is UTF8 encoding // while receiving a message sent by a Base64 queue client. // So we try to re-decode it as Base64. - - this.azureStorageClient.Settings.Logger.GeneralWarning( - this.azureStorageClient.QueueAccountName, - this.azureStorageClient.Settings.TaskHubName, - $"Failed to deserialize queue message using UTF-8 encoding. " + - $"First few characters are: \"{bodyAsString.Substring(0, Math.Min(10, bodyAsString.Length))}\". " + - $"Assuming this is a Base64-encoded message created with a different message encoding configuration."); - byte[] decodedBytes = Convert.FromBase64String(bodyAsString); string decodedMessage = Encoding.UTF8.GetString(decodedBytes); envelope = this.DeserializeMessageData(decodedMessage);