Skip to content

Report client connection events to the tunnel service #546

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Jason Ginchereau (jasongin) merged 3 commits into from
Aug 11, 2025
Merged

Report client connection events to the tunnel service #546

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4abe8cd
Report connection events to the tunnel service
jasongin Aug 5, 2025
f869e51
Bump package version
jasongin Aug 11, 2025
751b6c6
Fix tunnel client constructors
jasongin Aug 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 14 additions & 16 deletions cs/src/Connections/RelayTunnelConnector.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ public async Task ConnectSessionAsync(
}
catch (UnauthorizedAccessException uaex) // Tunnel access token validation failed.
{
if (!IsRetryAllowed(uaex, SshDisconnectReason.AuthCancelledByUser, delayNeeded: false))
if (!IsRetryAllowed(uaex, attempt, SshDisconnectReason.AuthCancelledByUser, delayNeeded: false))
{
throw;
}
Expand All @@ -88,7 +88,7 @@ public async Task ConnectSessionAsync(
}
catch (SshReconnectException srex)
{
if (!IsRetryAllowed(srex, SshDisconnectReason.ProtocolError, delayNeeded: false))
if (!IsRetryAllowed(srex, attempt, SshDisconnectReason.ProtocolError, delayNeeded: false))
{
throw;
}
Expand All @@ -99,7 +99,7 @@ public async Task ConnectSessionAsync(
when (scex.DisconnectReason == SshDisconnectReason.ConnectionLost)
{
// Recoverable
if (!IsRetryAllowed(scex, scex.DisconnectReason))
if (!IsRetryAllowed(scex, attempt, scex.DisconnectReason))
{
throw;
}
Expand Down Expand Up @@ -127,7 +127,7 @@ public async Task ConnectSessionAsync(
$"Unauthorized (401). Provide a fresh tunnel access token with '{this.relayClient.TunnelAccessScope}' scope.",
wse);

ThrowIfRetryNotAllowed(exception, SshDisconnectReason.AuthCancelledByUser, delayNeeded: false);
ThrowIfRetryNotAllowed(exception, attempt, SshDisconnectReason.AuthCancelledByUser, delayNeeded: false);

// Unauthorized error may happen when the tunnel access token is no longer valid, e.g. expired.
// Try refreshing it.
Expand Down Expand Up @@ -166,7 +166,7 @@ public async Task ConnectSessionAsync(
attemptDelayMs = RetryMaxDelayMs / 2;
}

if (!IsRetryAllowed(exception, SshDisconnectReason.ServiceNotAvailable) ||
if (!IsRetryAllowed(exception, attempt, SshDisconnectReason.ServiceNotAvailable) ||
attempt > 3)
{
throw exception;
Expand All @@ -181,7 +181,7 @@ public async Task ConnectSessionAsync(
}

// Other web socket errors may be recoverable
else if (!IsRetryAllowed(wse))
else if (!IsRetryAllowed(wse, attempt))
{
throw;
}
Expand Down Expand Up @@ -222,7 +222,7 @@ ex is ArgumentNullException ||
throw;
}

if (!IsRetryAllowed(ex, SshDisconnectReason.ProtocolError))
if (!IsRetryAllowed(ex, attempt, SshDisconnectReason.ProtocolError))
{
throw;
}
Expand Down Expand Up @@ -265,8 +265,8 @@ ex is ArgumentNullException ||
}
}

var retryTiming = isDelayNeeded ? $" in {(attemptDelayMs < 1000 ? $"0.{attemptDelayMs / 100}s" : $"{attemptDelayMs / 1000}s")}" : string.Empty;
Trace.Verbose($"Error connecting to tunnel SSH session, retrying{retryTiming}{(errorDescription != null ? $": {errorDescription}" : string.Empty)}");
var retryTiming = isDelayNeeded ? $"{(attemptDelayMs < 1000 ? $"0.{attemptDelayMs / 100}s" : $"{attemptDelayMs / 1000}s")}" : string.Empty;
Trace.Verbose($"Error connecting to tunnel SSH session, retrying in {retryTiming}{(errorDescription != null ? $": {errorDescription}" : string.Empty)}");

if (isDelayNeeded)
{
Expand Down Expand Up @@ -361,30 +361,28 @@ async Task RefreshTunnelAccessTokenAsync(Exception exception)
return null;
}

void ThrowIfRetryNotAllowed(Exception ex, SshDisconnectReason reason, bool delayNeeded = true)
void ThrowIfRetryNotAllowed(Exception ex, int attemptNumber, SshDisconnectReason reason, bool delayNeeded = true)
{
if (!IsRetryAllowed(ex,reason, delayNeeded))
if (!IsRetryAllowed(ex, attemptNumber, reason, delayNeeded))
{
throw ex;
}
}

bool IsRetryAllowed(
Exception ex,
int attemptNumber,
SshDisconnectReason reason = SshDisconnectReason.ConnectionLost,
bool delayNeeded = true)
{
disconnectReason = reason;
errorDescription = ex.Message;
exception = ex;
if (options?.EnableRetry == false)
{
return false;
}

isDelayNeeded = delayNeeded;
var retryDelay = TimeSpan.FromMilliseconds(isDelayNeeded ? attemptDelayMs : 0);
var retryingArgs = new RetryingTunnelConnectionEventArgs(ex, retryDelay);
var retryingArgs = new RetryingTunnelConnectionEventArgs(ex, attemptNumber, retryDelay);
retryingArgs.Retry = options?.EnableRetry ?? true;
this.relayClient.OnRetrying(retryingArgs);
if (!retryingArgs.Retry)
{
Expand Down
8 changes: 7 additions & 1 deletion cs/src/Connections/RetryingTunnelConnectionEventArgs.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,10 @@ public class RetryingTunnelConnectionEventArgs : EventArgs
/// <summary>
/// Creates a new instance of <see cref="RetryingTunnelConnectionEventArgs"/> class.
/// </summary>
public RetryingTunnelConnectionEventArgs(Exception exception, TimeSpan delay)
public RetryingTunnelConnectionEventArgs(Exception exception, int attemptNumber, TimeSpan delay)
{
Exception = Requires.NotNull(exception, nameof(exception));
AttemptNumber = attemptNumber;
Retry = true;
Delay = delay;
}
Expand All @@ -30,6 +31,11 @@ public RetryingTunnelConnectionEventArgs(Exception exception, TimeSpan delay)
/// </remarks>
public Exception Exception { get; }

/// <summary>
/// Gets the attempt number for the retry.
/// </summary>
public int AttemptNumber { get; }

/// <summary>
/// Gets the amount of time to wait before retrying. An event handler may change this value
/// to adjust the delay.
Expand Down
44 changes: 42 additions & 2 deletions cs/src/Connections/TunnelConnection.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
// </copyright>

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
Expand All @@ -21,6 +22,7 @@ public abstract class TunnelConnection : IAsyncDisposable
{
private readonly CancellationTokenSource disposeCts = new();
private ConnectionStatus connectionStatus;
private Stopwatch connectionTimer = new();
private Tunnel? tunnel;

/// <summary>
Expand Down Expand Up @@ -417,8 +419,29 @@ public async ValueTask DisposeAsync()
/// <summary>
/// Event fired when the connection status has changed.
/// </summary>
protected virtual void OnConnectionStatusChanged(ConnectionStatus previousConnectionStatus, ConnectionStatus connectionStatus)
protected virtual void OnConnectionStatusChanged(
ConnectionStatus previousConnectionStatus,
ConnectionStatus connectionStatus)
{
TimeSpan duration = this.connectionTimer.Elapsed;
this.connectionTimer.Restart();

if (Tunnel != null)
{
var statusEvent = new TunnelEvent($"{ConnectionRole}_connection_status");
statusEvent.Properties = new Dictionary<string, string>
{
[nameof(ConnectionStatus)] = connectionStatus.ToString(),
[$"Previous{nameof(ConnectionStatus)}"] = previousConnectionStatus.ToString(),
};
if (previousConnectionStatus != ConnectionStatus.None)
{
statusEvent.Properties[$"{previousConnectionStatus}Duration"] = duration.ToString();
}

ManagementClient?.ReportEvent(Tunnel, statusEvent);
}

var handler = ConnectionStatusChanged;
if (handler != null)
{
Expand All @@ -441,7 +464,24 @@ protected virtual void OnConnectionStatusChanged(ConnectionStatus previousConnec
/// </summary>
internal void OnRetrying(RetryingTunnelConnectionEventArgs e)
{
RetryingTunnelConnection?.Invoke(this, e);
if (e.Retry)
{
RetryingTunnelConnection?.Invoke(this, e);
}

if (Tunnel != null)
{
var retryingEvent = new TunnelEvent($"{ConnectionRole}_connect_retrying");
retryingEvent.Severity = TunnelEvent.Warning;
retryingEvent.Details = e.Exception?.ToString();
retryingEvent.Properties = new Dictionary<string, string>
{
[nameof(e.Retry)] = e.Retry.ToString(),
[nameof(e.AttemptNumber)] = e.AttemptNumber.ToString(),
[nameof(e.Delay)] = ((int)e.Delay.TotalMilliseconds).ToString(),
};
ManagementClient?.ReportEvent(Tunnel, retryingEvent);
}
}

/// <summary>
Expand Down
40 changes: 38 additions & 2 deletions cs/src/Connections/TunnelRelayConnection.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,19 @@ protected async Task ConnectTunnelSessionAsync(
var isReconnect = Tunnel != null;
Tunnel = tunnel;
this.connector ??= await CreateTunnelConnectorAsync(cancellation);
await this.connector.ConnectSessionAsync(options, isReconnect, cancellation);

try
{
await this.connector.ConnectSessionAsync(options, isReconnect, cancellation);
}
catch (Exception ex)
{
var connectFailedEvent = new TunnelEvent($"{ConnectionRole}_connect_failed");
connectFailedEvent.Severity = TunnelEvent.Error;
connectFailedEvent.Details = ex.ToString();
ManagementClient?.ReportEvent(tunnel, connectFailedEvent);
throw;
}
}

/// <summary>
Expand Down Expand Up @@ -197,12 +209,28 @@ protected void MaybeStartReconnecting(
this.connector != null && // The connector may be null if the tunnel client/host was created directly from a stream.
reason == SshDisconnectReason.ConnectionLost) // Only reconnect if it's connection lost.
{
if (Tunnel != null)
{
var reconnectEvent = new TunnelEvent($"{ConnectionRole}_reconnect");
reconnectEvent.Severity = TunnelEvent.Warning;
reconnectEvent.Details = exception?.ToString() ?? traceMessage;
ManagementClient?.ReportEvent(Tunnel, reconnectEvent);
}

Trace.TraceInformation($"{traceMessage}. Reconnecting.");
var task = ReconnectAsync(DisposeToken);
this.reconnectTask = !task.IsCompleted ? task : null;
}
else
{
if (Tunnel != null)
{
var disconnectEvent = new TunnelEvent($"{ConnectionRole}_disconnect");
disconnectEvent.Severity = TunnelEvent.Warning;
disconnectEvent.Details = exception?.ToString() ?? traceMessage;
ManagementClient?.ReportEvent(Tunnel, disconnectEvent);
}

Trace.TraceInformation(traceMessage);
ConnectionStatus = ConnectionStatus.Disconnected;
}
Expand Down Expand Up @@ -530,8 +558,16 @@ await this.connector.ConnectSessionAsync(
isReconnect: true,
cancellation);
}
catch
catch (Exception ex)
{
if (Tunnel != null)
{
var connectFailedEvent = new TunnelEvent($"{ConnectionRole}_reconnect_failed");
connectFailedEvent.Severity = TunnelEvent.Error;
connectFailedEvent.Details = ex.ToString();
ManagementClient?.ReportEvent(Tunnel, connectFailedEvent);
}

// Tracing of the exception has already been done by ConnectSessionAsync.
// As reconnection is an async process, there is nobody watching it throw.
// The exception, if it was not cancellation, is stored in DisconnectException property.
Expand Down
23 changes: 19 additions & 4 deletions cs/src/Connections/TunnelRelayTunnelClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,10 +119,25 @@ protected async Task ConnectAsync(
this.accessToken = accessToken;
this.HostPublicKeys = hostPublicKeys;
this.connector = new RelayTunnelConnector(this);
await this.connector.ConnectSessionAsync(
options,
isReconnect: false,
cancellation);

try
{
await this.connector.ConnectSessionAsync(
options,
isReconnect: false,
cancellation);
}
catch (Exception ex)
{
if (Tunnel != null)
{
var connectFailedEvent = new TunnelEvent($"{ConnectionRole}_connect_failed");
connectFailedEvent.Severity = TunnelEvent.Error;
connectFailedEvent.Details = ex.ToString();
ManagementClient?.ReportEvent(Tunnel, connectFailedEvent);
}
throw;
}
}

/// <summary>
Expand Down
Loading