Reinitialise on connect, bound the auth callback, and complete the teardown

RTN11d - fixed. Only the channel half ran, so Connection.errorReason, msgSerial and
the channel's channelSerial all survived into the new connection.
RTN11b - fixed. connect() while CLOSING applied no part of RTN11d.
RTN12b - fixed. The close timeout was a hardcoded 1s, not realtimeRequestTimeout,
so on a slow link close() could reach CLOSED unacknowledged.
TO3l11 - implemented. realtimeRequestTimeout was internal and could not be set.
Validated at both ends; the upper bound is Int32.MaxValue ms, the tightest limit
across every timer sink on every framework this package ships.
RSA4c - fixed. The bound was applied to the task the callback returned, so a
callback whose body runs synchronously was never bounded at all.
RSA4c1 - fixed. cause was set as InnerException, which is not the spec's field.
RTN8d, RTN9d - fixed. A throwing transition skipped the key clear and the transport
teardown, leaving a terminal state holding a resumable key and a live transport.

Also carries RealtimeWorkflowSpecs.cs in full: its additions are one contiguous
insertion of complete test classes and cannot be split across the commits whose
spec points they cover.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: AndyTWF

src/IO.Ably.Shared/AblyAuth.cs

@@ -305,11 +305,27 @@ public virtual async Task<TokenDetails> RequestTokenAsync(TokenParams tokenParam
                 bool shouldCatch = true;
                 try
                 {
-                    var callbackResult = await authOptions.AuthCallback(tokenParams);
+                    // RSA4c bounds an auth attempt by realtimeRequestTimeout. This one is awaited on
+                    // the realtime workflow's single reader thread, so an authCallback that never
+                    // returns would stall channel state, inbound messages and the idle monitor.
+                    //
+                    // Task.Run rather than awaiting the delegate directly: TimeoutAfter extends an
+                    // already-created Task, so a callback whose body runs synchronously - the most
+                    // ordinary shape in C# - would block before there is anything to bound. It
+                    // cannot cancel the callback; an abandoned one runs to completion.
+                    var callbackResult = await Task.Run(() => authOptions.AuthCallback(tokenParams))
+                        .TimeoutAfter(Options.RealtimeRequestTimeout, null);
 
                     if (callbackResult == null)
                     {
-                        throw new AblyException("AuthCallback returned null", ErrorCodes.ClientAuthProviderRequestFailed);
+                        // Covers a callback that returned null and one that timed out - TimeoutAfter
+                        // yields null for both, and the wrapper below replaces the message anyway.
+                        // ClientCallbackError, not ClientAuthProviderRequestFailed: the wrapper
+                        // rethrows as 80019 per RSA4c1 with this as the cause, so 80019 here would
+                        // duplicate its own parent. ably-js uses 40170 for the same case.
+                        throw new AblyException(
+                            $"AuthCallback returned null or did not return within {Options.RealtimeRequestTimeout.TotalSeconds}s",
+                            ErrorCodes.ClientCallbackError);
                     }
 
                     if (callbackResult is TokenDetails)
@@ -341,12 +357,17 @@ public virtual async Task<TokenDetails> RequestTokenAsync(TokenParams tokenParam
                             : HttpStatusCode.Unauthorized;
                     }
 
+                    // RSA4c1 wants the cause "set to the underlying cause". The four argument overload
+                    // assigns InnerException instead, which is not the spec's field and is not
+                    // serialised as one.
                     throw new AblyException(
                         new ErrorInfo(
                             "Error calling AuthCallback, token request failed. See inner exception for details.",
                             ErrorCodes.ClientAuthProviderRequestFailed,
                             statusCode,
-                            ex),
+                            href: null,
+                            cause: (ex as AblyException)?.ErrorInfo ?? new ErrorInfo(ex.Message),
+                            innerException: ex),
                         ex);
                 }
             }

src/IO.Ably.Shared/AuthOptions.cs

@@ -13,6 +13,13 @@ public class AuthOptions
         /// <summary>
         /// The callback used to get a new <see cref="IO.Ably.TokenDetails"/> or <see cref="IO.Ably.TokenRequest"/>.
         /// AuthCallback is used by internally by <see cref="IO.Ably.AblyAuth"/>.RequestTokenAsync.
+        /// <para>
+        /// The callback is bounded by <see cref="ClientOptions.RealtimeRequestTimeout"/> per RSA4c,
+        /// but cannot be cancelled: one that overruns is abandoned and keeps running, so a later auth
+        /// attempt may invoke it again concurrently and implementations must tolerate that. A result
+        /// returned after the bound is discarded, and each abandoned invocation holds a thread pool
+        /// worker until it returns.
+        /// </para>
         /// </summary>
         public Func<TokenParams, Task<object>> AuthCallback { get; set; }
 

src/IO.Ably.Shared/ClientOptions.cs

@@ -295,6 +295,51 @@ public bool UseBinaryProtocol
         /// </summary>
         public TimeSpan ChannelRetryTimeout { get; set; } = Defaults.ChannelRetryTimeout;
 
+        private TimeSpan _realtimeRequestTimeout = Defaults.RealtimeRequestTimeout;
+
+        /// <summary>
+        /// How long the library waits for Ably to answer before treating a realtime request as
+        /// having failed. Applies while establishing a connection, while awaiting a response to a
+        /// Heartbeat, Connect, Attach, Detach or Close, and as part of the RTN23a idle timeout.
+        /// Default: 10s. Must be a positive interval, and values beyond a minute or so are rarely
+        /// useful. Disabling the timeout is not supported - the timeouts this value drives are all
+        /// required to fire - so both Timeout.InfiniteTimeSpan and TimeSpan.MaxValue are rejected.
+        /// TO3l11 - https://sdk.ably.com/builds/ably/specification/main/features/#TO3l11.
+        /// </summary>
+        /// <exception cref="ArgumentOutOfRangeException">when set to zero, a negative value, or an
+        /// interval too large for the underlying timers to fire.</exception>
+        public TimeSpan RealtimeRequestTimeout
+        {
+            get => _realtimeRequestTimeout;
+
+            set
+            {
+                // A non-positive value fails quietly rather than loudly: zero turns the RTN14c
+                // connect timeout into a hot reconnect loop, and a negative one stops the timer
+                // firing at all. Timeout.InfiniteTimeSpan is -1ms, so the lower bound catches it -
+                // which matters, because both Task.Delay and System.Threading.Timer accept it as
+                // genuine infinity and it would disable RTN14c, RTN12b, RTL4f, RTL5f and RSA4c.
+                //
+                // The upper bound is the tightest limit across the sinks this value reaches on every
+                // framework shipped: Task.Delay allows uint.MaxValue - 1 ms on .NET 6+ but only
+                // Int32.MaxValue on .NET Framework, Mono and Xamarin, and CountdownTimer casts to
+                // int for System.Threading.Timer. So Int32.MaxValue ms - a bound on the arithmetic,
+                // not a supported configuration, which is why the message names the useful range.
+                if (value <= TimeSpan.Zero || value.TotalMilliseconds > int.MaxValue)
+                {
+                    throw new ArgumentOutOfRangeException(
+                        nameof(RealtimeRequestTimeout),
+                        value,
+                        "RealtimeRequestTimeout must be a positive interval. The default is 10s; " +
+                        "values beyond a minute or so are rarely useful. Disabling the timeout is " +
+                        "not supported - the connect, close, attach, detach and auth timeouts it " +
+                        "drives are all required to fire.");
+                }
+
+                _realtimeRequestTimeout = value;
+            }
+        }
+
         /// <summary>
         /// Timeout for opening an http request.
         /// Default: 4s.
@@ -436,8 +481,6 @@ internal Func<DateTimeOffset> NowFunc
 
         internal bool SkipInternetCheck { get; set; }
 
-        internal TimeSpan RealtimeRequestTimeout { get; set; } = Defaults.RealtimeRequestTimeout;
-
         /// <summary>
         /// Default constructor for ClientOptions.
         /// </summary>

src/IO.Ably.Shared/Realtime/Workflows/RealtimeWorkflow.cs

@@ -360,6 +360,17 @@ private async Task<RealtimeCommand> ProcessCommandInner(RealtimeCommand command)
             switch (command)
             {
                 case ConnectCommand _:
+
+                    // RTN11d - connect() out of CLOSED or FAILED starts afresh. The channel half,
+                    // back to INITIALIZED with errorReason unset, is done per channel by the command
+                    // queued below; Id and Key are already emptied on entering CLOSED or FAILED.
+                    if (State.Connection.State == ConnectionState.Closed ||
+                        State.Connection.State == ConnectionState.Failed)
+                    {
+                        State.Connection.ErrorReason = null;
+                        State.Connection.MessageSerial = 0;
+                    }
+
                     var nextCommand = ConnectionManager.Connect();
                     var initFailedChannelsOnConnect =
                         ChannelCommand.CreateForAllChannels(InitialiseFailedChannelsOnConnect.Create().TriggeredBy(command));
@@ -949,14 +960,36 @@ private async Task<RealtimeCommand> HandleSetStateCommand(RealtimeCommand comman
 
                     case SetFailedStateCommand cmd:
 
-                        ClearAckQueueAndFailMessages(ErrorInfo.ReasonFailed);
-
                         var error = TransformIfTokenErrorAndNotRetryable();
                         var failedState = new ConnectionFailedState(ConnectionManager, error, Logger);
-                        SetState(failedState);
-                        State.Connection.ClearKeyAndId(); // RTN8c, RTN9c
 
-                        ConnectionManager.DestroyTransport();
+                        // RTN7e - the queued messages are failed with "an error representing the
+                        // reason for the state change", taken off the state object so it is this
+                        // transition's reason even if SetState early-returns. In the finally, after
+                        // the transition, so a publisher's callback sees the state it is being told
+                        // about and a throwing transition cannot strand the messages uncalled.
+                        // ably-js orders it the same way: enactStateChange then failQueuedMessages.
+                        //
+                        // RTN7e - the queued messages are failed with "an error representing the
+                        // reason for the state change", taken off the state object so it is this
+                        // transition's reason even if SetState early-returns. In the finally, after
+                        // the transition, so a publisher's callback sees the state it is being told
+                        // about and a throwing transition cannot strand the messages uncalled.
+                        // ably-js orders it the same way: enactStateChange then failQueuedMessages.
+                        //
+                        // RTN8d and RTN9d share that finally: the connection has entered the state
+                        // by the time SetState rethrows, so a throw must not leave it reporting a
+                        // terminal state while still holding a resumable key and a live transport.
+                        try
+                        {
+                            SetState(failedState);
+                        }
+                        finally
+                        {
+                            ClearAckQueueAndFailMessages(failedState.Error);
+                            State.Connection.ClearKeyAndId(); // RTN8d, RTN9d
+                            ConnectionManager.DestroyTransport();
+                        }
 
                         ErrorInfo TransformIfTokenErrorAndNotRetryable()
                         {
@@ -1108,7 +1141,7 @@ async Task<bool> CheckInstantRetryFlag()
 
                         var closingState = new ConnectionClosingState(ConnectionManager, connectedTransport, Logger);
                         SetState(closingState);
-                        State.Connection.ClearKeyAndId(); // RTN8c, RTN9c
+                        State.Connection.ClearKeyAndId(); // RTN8d, RTN9d
 
                         if (connectedTransport)
                         {
@@ -1126,27 +1159,49 @@ async Task<bool> CheckInstantRetryFlag()
                             State.Connection.ClearKey();
                         }
 
-                        ClearAckQueueAndFailMessages(ErrorInfo.ReasonSuspended);
-
                         var suspendedState = new ConnectionSuspendedState(ConnectionManager, cmd.Error, Logger);
-                        SetState(suspendedState);
-                        State.Connection.ClearKeyAndId(); // RTN8c, RTN9c
+
+                        // RTN7e and the teardown - see the note on the FAILED case.
+                        try
+                        {
+                            SetState(suspendedState);
+                        }
+                        finally
+                        {
+                            ClearAckQueueAndFailMessages(suspendedState.Error);
+
+                            // Deliberately NOT RTN8d/RTN9d, which name only CLOSED, CLOSING and
+                            // FAILED. Clearing here is this library's pre-6.1.0 behaviour, kept
+                            // because it is coupled to the connectionStateTtl freshness check that
+                            // also predates 6.1.0.
+                            State.Connection.ClearKeyAndId();
+
+                            // Needed here as well as in the DISCONNECTED handler, which diverts to
+                            // this case before reaching its own DestroyTransport. A surviving
+                            // transport keeps its listener for up to suspendedRetryTimeout.
+                            ConnectionManager.DestroyTransport();
+                        }
 
                         break;
 
                     case SetClosedStateCommand cmd:
 
-                        ClearAckQueueAndFailMessages(ErrorInfo.ReasonClosed);
-
                         var closedState = new ConnectionClosedState(ConnectionManager, cmd.Error, Logger)
                         {
                             Exception = cmd.Exception,
                         };
 
-                        SetState(closedState);
-                        State.Connection.ClearKeyAndId(); // RTN8c, RTN9c
-
-                        ConnectionManager.DestroyTransport();
+                        // RTN7e and the teardown - see the note on the FAILED case.
+                        try
+                        {
+                            SetState(closedState);
+                        }
+                        finally
+                        {
+                            ClearAckQueueAndFailMessages(closedState.Error);
+                            State.Connection.ClearKeyAndId(); // RTN8d, RTN9d
+                            ConnectionManager.DestroyTransport();
+                        }
 
                         break;
                 }
@@ -1177,6 +1232,8 @@ public void SetState(ConnectionStateBase newState, bool skipTimer = false)
                 Logger.Debug(message);
             }
 
+            var notified = false;
+
             try
             {
                 if (newState.IsUpdate == false)
@@ -1204,13 +1261,28 @@ public void SetState(ConnectionStateBase newState, bool skipTimer = false)
                     Logger.Debug($"xx {newState.State}: Skipping attaching.");
                 }
 
+                notified = true;
                 UpdateStateAndNotifyConnection(newState);
             }
-            catch (AblyException ex)
+            catch (Exception ex)
             {
-                Logger.Error("Error attaching to context", ex);
-
-                UpdateStateAndNotifyConnection(newState);
+                // Everything, not just AblyException: anything else thrown by StartTimer or the
+                // state object would reach the command loop, which logs and drops it, leaving the
+                // connection with no transport, no timer and no state change emitted. The transition
+                // is still completed below and the exception still rethrown.
+                Logger.Error($"Error attaching to context while changing state to {newState.State}", ex);
+
+                // Only if the notify has not already happened. A throw during the transition lands
+                // here after the state change has been emitted - StartTimer is one source, and a
+                // negative retry timeout reaches System.Threading.Timer. Not a channel's
+                // ConnectionStateChanged, which RealtimeChannels guards per channel. Re-emitting is harmless for an ordinary transition, which the
+                // same-state check swallows, but an RTN24 update has no such check and was emitted
+                // twice. The flag is set before the call so a throw from inside it does not trigger
+                // a second attempt either.
+                if (notified == false)
+                {
+                    UpdateStateAndNotifyConnection(newState);
+                }
 
                 newState.AbortTimer();
 

src/IO.Ably.Shared/Transport/ConnectionManager.cs

@@ -172,12 +172,14 @@ internal async Task OnAuthUpdated(TokenDetails tokenDetails, bool wait)
                 {
                     while (true)
                     {
-                        var (success, newState) = await waiter.Wait(Defaults.RealtimeRequestTimeout);
+                        // Options rather than Defaults, since TO3l11 makes realtimeRequestTimeout a
+                        // client option.
+                        var (success, newState) = await waiter.Wait(Options.RealtimeRequestTimeout);
                         if (success == false)
                         {
                             throw new AblyException(
                                 new ErrorInfo(
-                                $"Connection state didn't change after Auth updated within {Defaults.RealtimeRequestTimeout}",
+                                $"Connection state didn't change after Auth updated within {Options.RealtimeRequestTimeout}",
                                 40140));
                         }
 

src/IO.Ably.Shared/Transport/States/Connection/ConnectionClosingState.cs

@@ -10,7 +10,6 @@ internal class ConnectionClosingState : ConnectionStateBase
     {
         public override ErrorInfo DefaultErrorInfo => ErrorInfo.ReasonClosed;
 
-        private const int CloseTimeout = 1000;
         private readonly bool _connectedTransport;
         private readonly ICountdownTimer _timer;
 
@@ -62,7 +61,9 @@ public override void StartTimer()
         {
             if (_connectedTransport)
             {
-                _timer.Start(TimeSpan.FromMilliseconds(CloseTimeout), OnTimeOut);
+                // RTN12b - the wait for the CLOSED message is realtimeRequestTimeout, which TO3l11
+                // makes a client option.
+                _timer.Start(Context.DefaultTimeout, OnTimeOut);
             }
         }