Skip to content

Commit fb6f074

Browse files
jasonginjasongin
committed
Report connection events to the tunnel service
1 parent 507a98f commit fb6f074

32 files changed

Lines changed: 2065 additions & 69 deletions

‎cs/src/Connections/RelayTunnelConnector.cs‎

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ public async Task ConnectSessionAsync(
7979
}
8080
catch (UnauthorizedAccessException uaex) // Tunnel access token validation failed.
8181
{
82-
if (!IsRetryAllowed(uaex, SshDisconnectReason.AuthCancelledByUser, delayNeeded: false))
82+
if (!IsRetryAllowed(uaex, attempt, SshDisconnectReason.AuthCancelledByUser, delayNeeded: false))
8383
{
8484
throw;
8585
}
@@ -88,7 +88,7 @@ public async Task ConnectSessionAsync(
8888
}
8989
catch (SshReconnectException srex)
9090
{
91-
if (!IsRetryAllowed(srex, SshDisconnectReason.ProtocolError, delayNeeded: false))
91+
if (!IsRetryAllowed(srex, attempt, SshDisconnectReason.ProtocolError, delayNeeded: false))
9292
{
9393
throw;
9494
}
@@ -99,7 +99,7 @@ public async Task ConnectSessionAsync(
9999
when (scex.DisconnectReason == SshDisconnectReason.ConnectionLost)
100100
{
101101
// Recoverable
102-
if (!IsRetryAllowed(scex, scex.DisconnectReason))
102+
if (!IsRetryAllowed(scex, attempt, scex.DisconnectReason))
103103
{
104104
throw;
105105
}
@@ -127,7 +127,7 @@ public async Task ConnectSessionAsync(
127127
$"Unauthorized (401). Provide a fresh tunnel access token with '{this.relayClient.TunnelAccessScope}' scope.",
128128
wse);
129129

130-
ThrowIfRetryNotAllowed(exception, SshDisconnectReason.AuthCancelledByUser, delayNeeded: false);
130+
ThrowIfRetryNotAllowed(exception, attempt, SshDisconnectReason.AuthCancelledByUser, delayNeeded: false);
131131

132132
// Unauthorized error may happen when the tunnel access token is no longer valid, e.g. expired.
133133
// Try refreshing it.
@@ -166,7 +166,7 @@ public async Task ConnectSessionAsync(
166166
attemptDelayMs = RetryMaxDelayMs / 2;
167167
}
168168

169-
if (!IsRetryAllowed(exception, SshDisconnectReason.ServiceNotAvailable) ||
169+
if (!IsRetryAllowed(exception, attempt, SshDisconnectReason.ServiceNotAvailable) ||
170170
attempt > 3)
171171
{
172172
throw exception;
@@ -181,7 +181,7 @@ public async Task ConnectSessionAsync(
181181
}
182182

183183
// Other web socket errors may be recoverable
184-
else if (!IsRetryAllowed(wse))
184+
else if (!IsRetryAllowed(wse, attempt))
185185
{
186186
throw;
187187
}
@@ -222,7 +222,7 @@ ex is ArgumentNullException ||
222222
throw;
223223
}
224224

225-
if (!IsRetryAllowed(ex, SshDisconnectReason.ProtocolError))
225+
if (!IsRetryAllowed(ex, attempt, SshDisconnectReason.ProtocolError))
226226
{
227227
throw;
228228
}
@@ -265,8 +265,8 @@ ex is ArgumentNullException ||
265265
}
266266
}
267267

268-
var retryTiming = isDelayNeeded ? $" in {(attemptDelayMs < 1000 ? $"0.{attemptDelayMs / 100}s" : $"{attemptDelayMs / 1000}s")}" : string.Empty;
269-
Trace.Verbose($"Error connecting to tunnel SSH session, retrying{retryTiming}{(errorDescription != null ? $": {errorDescription}" : string.Empty)}");
268+
var retryTiming = isDelayNeeded ? $"{(attemptDelayMs < 1000 ? $"0.{attemptDelayMs / 100}s" : $"{attemptDelayMs / 1000}s")}" : string.Empty;
269+
Trace.Verbose($"Error connecting to tunnel SSH session, retrying in {retryTiming}{(errorDescription != null ? $": {errorDescription}" : string.Empty)}");
270270

271271
if (isDelayNeeded)
272272
{
@@ -361,30 +361,28 @@ async Task RefreshTunnelAccessTokenAsync(Exception exception)
361361
return null;
362362
}
363363

364-
void ThrowIfRetryNotAllowed(Exception ex, SshDisconnectReason reason, bool delayNeeded = true)
364+
void ThrowIfRetryNotAllowed(Exception ex, int attemptNumber, SshDisconnectReason reason, bool delayNeeded = true)
365365
{
366-
if (!IsRetryAllowed(ex,reason, delayNeeded))
366+
if (!IsRetryAllowed(ex, attemptNumber, reason, delayNeeded))
367367
{
368368
throw ex;
369369
}
370370
}
371371

372372
bool IsRetryAllowed(
373373
Exception ex,
374+
int attemptNumber,
374375
SshDisconnectReason reason = SshDisconnectReason.ConnectionLost,
375376
bool delayNeeded = true)
376377
{
377378
disconnectReason = reason;
378379
errorDescription = ex.Message;
379380
exception = ex;
380-
if (options?.EnableRetry == false)
381-
{
382-
return false;
383-
}
384381

385382
isDelayNeeded = delayNeeded;
386383
var retryDelay = TimeSpan.FromMilliseconds(isDelayNeeded ? attemptDelayMs : 0);
387-
var retryingArgs = new RetryingTunnelConnectionEventArgs(ex, retryDelay);
384+
var retryingArgs = new RetryingTunnelConnectionEventArgs(ex, attemptNumber, retryDelay);
385+
retryingArgs.Retry = options?.EnableRetry ?? true;
388386
this.relayClient.OnRetrying(retryingArgs);
389387
if (!retryingArgs.Retry)
390388
{

‎cs/src/Connections/RetryingTunnelConnectionEventArgs.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,10 @@ public class RetryingTunnelConnectionEventArgs : EventArgs
1515
/// <summary>
1616
/// Creates a new instance of <see cref="RetryingTunnelConnectionEventArgs"/> class.
1717
/// </summary>
18-
public RetryingTunnelConnectionEventArgs(Exception exception, TimeSpan delay)
18+
public RetryingTunnelConnectionEventArgs(Exception exception, int attemptNumber, TimeSpan delay)
1919
{
2020
Exception = Requires.NotNull(exception, nameof(exception));
21+
AttemptNumber = attemptNumber;
2122
Retry = true;
2223
Delay = delay;
2324
}
@@ -30,6 +31,11 @@ public RetryingTunnelConnectionEventArgs(Exception exception, TimeSpan delay)
3031
/// </remarks>
3132
public Exception Exception { get; }
3233

34+
/// <summary>
35+
/// Gets the attempt number for the retry.
36+
/// </summary>
37+
public int AttemptNumber { get; }
38+
3339
/// <summary>
3440
/// Gets the amount of time to wait before retrying. An event handler may change this value
3541
/// to adjust the delay.

‎cs/src/Connections/TunnelConnection.cs‎

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
// </copyright>
55

66
using System;
7+
using System.Collections.Generic;
78
using System.Diagnostics;
89
using System.Threading;
910
using System.Threading.Tasks;
@@ -21,6 +22,7 @@ public abstract class TunnelConnection : IAsyncDisposable
2122
{
2223
private readonly CancellationTokenSource disposeCts = new();
2324
private ConnectionStatus connectionStatus;
25+
private Stopwatch connectionTimer = new();
2426
private Tunnel? tunnel;
2527

2628
/// <summary>
@@ -417,8 +419,29 @@ public async ValueTask DisposeAsync()
417419
/// <summary>
418420
/// Event fired when the connection status has changed.
419421
/// </summary>
420-
protected virtual void OnConnectionStatusChanged(ConnectionStatus previousConnectionStatus, ConnectionStatus connectionStatus)
422+
protected virtual void OnConnectionStatusChanged(
423+
ConnectionStatus previousConnectionStatus,
424+
ConnectionStatus connectionStatus)
421425
{
426+
TimeSpan duration = this.connectionTimer.Elapsed;
427+
this.connectionTimer.Restart();
428+
429+
if (Tunnel != null)
430+
{
431+
var statusEvent = new TunnelEvent($"{ConnectionRole}_connection_status");
432+
statusEvent.Properties = new Dictionary<string, string>
433+
{
434+
[nameof(ConnectionStatus)] = connectionStatus.ToString(),
435+
[$"Previous{nameof(ConnectionStatus)}"] = previousConnectionStatus.ToString(),
436+
};
437+
if (previousConnectionStatus != ConnectionStatus.None)
438+
{
439+
statusEvent.Properties[$"{previousConnectionStatus}Duration"] = duration.ToString();
440+
}
441+
442+
ManagementClient?.ReportEvent(Tunnel, statusEvent);
443+
}
444+
422445
var handler = ConnectionStatusChanged;
423446
if (handler != null)
424447
{
@@ -441,7 +464,24 @@ protected virtual void OnConnectionStatusChanged(ConnectionStatus previousConnec
441464
/// </summary>
442465
internal void OnRetrying(RetryingTunnelConnectionEventArgs e)
443466
{
444-
RetryingTunnelConnection?.Invoke(this, e);
467+
if (e.Retry)
468+
{
469+
RetryingTunnelConnection?.Invoke(this, e);
470+
}
471+
472+
if (Tunnel != null)
473+
{
474+
var retryingEvent = new TunnelEvent($"{ConnectionRole}_connect_retrying");
475+
retryingEvent.Severity = TunnelEvent.Warning;
476+
retryingEvent.Details = e.Exception?.ToString();
477+
retryingEvent.Properties = new Dictionary<string, string>
478+
{
479+
[nameof(e.Retry)] = e.Retry.ToString(),
480+
[nameof(e.AttemptNumber)] = e.AttemptNumber.ToString(),
481+
[nameof(e.Delay)] = ((int)e.Delay.TotalMilliseconds).ToString(),
482+
};
483+
ManagementClient?.ReportEvent(Tunnel, retryingEvent);
484+
}
445485
}
446486

447487
/// <summary>

‎cs/src/Connections/TunnelRelayConnection.cs‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,19 @@ protected async Task ConnectTunnelSessionAsync(
155155
var isReconnect = Tunnel != null;
156156
Tunnel = tunnel;
157157
this.connector ??= await CreateTunnelConnectorAsync(cancellation);
158-
await this.connector.ConnectSessionAsync(options, isReconnect, cancellation);
158+
159+
try
160+
{
161+
await this.connector.ConnectSessionAsync(options, isReconnect, cancellation);
162+
}
163+
catch (Exception ex)
164+
{
165+
var connectFailedEvent = new TunnelEvent($"{ConnectionRole}_connect_failed");
166+
connectFailedEvent.Severity = TunnelEvent.Error;
167+
connectFailedEvent.Details = ex.ToString();
168+
ManagementClient?.ReportEvent(tunnel, connectFailedEvent);
169+
throw;
170+
}
159171
}
160172

161173
/// <summary>
@@ -197,12 +209,28 @@ protected void MaybeStartReconnecting(
197209
this.connector != null && // The connector may be null if the tunnel client/host was created directly from a stream.
198210
reason == SshDisconnectReason.ConnectionLost) // Only reconnect if it's connection lost.
199211
{
212+
if (Tunnel != null)
213+
{
214+
var reconnectEvent = new TunnelEvent($"{ConnectionRole}_reconnect");
215+
reconnectEvent.Severity = TunnelEvent.Warning;
216+
reconnectEvent.Details = exception?.ToString() ?? traceMessage;
217+
ManagementClient?.ReportEvent(Tunnel, reconnectEvent);
218+
}
219+
200220
Trace.TraceInformation($"{traceMessage}. Reconnecting.");
201221
var task = ReconnectAsync(DisposeToken);
202222
this.reconnectTask = !task.IsCompleted ? task : null;
203223
}
204224
else
205225
{
226+
if (Tunnel != null)
227+
{
228+
var disconnectEvent = new TunnelEvent($"{ConnectionRole}_disconnect");
229+
disconnectEvent.Severity = TunnelEvent.Warning;
230+
disconnectEvent.Details = exception?.ToString() ?? traceMessage;
231+
ManagementClient?.ReportEvent(Tunnel, disconnectEvent);
232+
}
233+
206234
Trace.TraceInformation(traceMessage);
207235
ConnectionStatus = ConnectionStatus.Disconnected;
208236
}
@@ -530,8 +558,16 @@ await this.connector.ConnectSessionAsync(
530558
isReconnect: true,
531559
cancellation);
532560
}
533-
catch
561+
catch (Exception ex)
534562
{
563+
if (Tunnel != null)
564+
{
565+
var connectFailedEvent = new TunnelEvent($"{ConnectionRole}_reconnect_failed");
566+
connectFailedEvent.Severity = TunnelEvent.Error;
567+
connectFailedEvent.Details = ex.ToString();
568+
ManagementClient?.ReportEvent(Tunnel, connectFailedEvent);
569+
}
570+
535571
// Tracing of the exception has already been done by ConnectSessionAsync.
536572
// As reconnection is an async process, there is nobody watching it throw.
537573
// The exception, if it was not cancellation, is stored in DisconnectException property.

‎cs/src/Connections/TunnelRelayTunnelClient.cs‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -119,10 +119,25 @@ protected async Task ConnectAsync(
119119
this.accessToken = accessToken;
120120
this.HostPublicKeys = hostPublicKeys;
121121
this.connector = new RelayTunnelConnector(this);
122-
await this.connector.ConnectSessionAsync(
123-
options,
124-
isReconnect: false,
125-
cancellation);
122+
123+
try
124+
{
125+
await this.connector.ConnectSessionAsync(
126+
options,
127+
isReconnect: false,
128+
cancellation);
129+
}
130+
catch (Exception ex)
131+
{
132+
if (Tunnel != null)
133+
{
134+
var connectFailedEvent = new TunnelEvent($"{ConnectionRole}_connect_failed");
135+
connectFailedEvent.Severity = TunnelEvent.Error;
136+
connectFailedEvent.Details = ex.ToString();
137+
ManagementClient?.ReportEvent(Tunnel, connectFailedEvent);
138+
}
139+
throw;
140+
}
126141
}
127142

128143
/// <summary>

0 commit comments

Comments
 (0)