From 8093c2d053c02c077ca73c03a7928fcdb303fc9b Mon Sep 17 00:00:00 2001 From: Marci Date: Sun, 30 Aug 2026 01:13:55 +0800 Subject: [PATCH] =?UTF-8?q?ci:=20=E5=A2=9E=E5=8A=A0=E5=8F=AF=E5=A4=8D?= =?UTF-8?q?=E7=8E=B0=E4=BC=9A=E8=AF=9D=E6=95=85=E9=9A=9C=20Soak?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/session-soak.yml | 71 ++++++ CHANGELOG.md | 6 + README.md | 2 + SecsFrame.slnx | 1 + docs/DEVELOPMENT-STATUS.md | 11 +- docs/HSMS-DATA-TRANSACTIONS.md | 7 + docs/HSMS-SOAK.md | 68 ++++++ docs/ROADMAP.md | 2 + test/SecsFrame.Soak/HsmsConnectionPair.cs | 216 ++++++++++++++++++ test/SecsFrame.Soak/HsmsEndpointProbe.cs | 127 ++++++++++ test/SecsFrame.Soak/HsmsSoakRunner.cs | 183 +++++++++++++++ test/SecsFrame.Soak/Program.cs | 33 +++ test/SecsFrame.Soak/SecsFrame.Soak.csproj | 13 ++ .../SoakConfigurationException.cs | 9 + test/SecsFrame.Soak/SoakFaultMode.cs | 9 + test/SecsFrame.Soak/SoakOptions.cs | 99 ++++++++ test/SecsFrame.Soak/SoakReportWriter.cs | 92 ++++++++ 17 files changed, 946 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/session-soak.yml create mode 100644 docs/HSMS-SOAK.md create mode 100644 test/SecsFrame.Soak/HsmsConnectionPair.cs create mode 100644 test/SecsFrame.Soak/HsmsEndpointProbe.cs create mode 100644 test/SecsFrame.Soak/HsmsSoakRunner.cs create mode 100644 test/SecsFrame.Soak/Program.cs create mode 100644 test/SecsFrame.Soak/SecsFrame.Soak.csproj create mode 100644 test/SecsFrame.Soak/SoakConfigurationException.cs create mode 100644 test/SecsFrame.Soak/SoakFaultMode.cs create mode 100644 test/SecsFrame.Soak/SoakOptions.cs create mode 100644 test/SecsFrame.Soak/SoakReportWriter.cs diff --git a/.github/workflows/session-soak.yml b/.github/workflows/session-soak.yml new file mode 100644 index 0000000..addf5f6 --- /dev/null +++ b/.github/workflows/session-soak.yml @@ -0,0 +1,71 @@ +name: session-soak + +on: + workflow_dispatch: + inputs: + duration_minutes: + description: Bounded soak duration + type: choice + options: ["1", "5", "15", "20"] + default: "5" + seed: + description: Positive 32-bit seed; 0 derives one from the run ID + type: string + default: "0" + schedule: + - cron: "43 2 * * 3" + +permissions: + contents: read + +concurrency: + group: session-soak + cancel-in-progress: false + +jobs: + session-soak: + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-dotnet@v6 + with: + dotnet-version: 8.0.x + - name: Resolve bounded run parameters + id: parameters + shell: bash + env: + INPUT_DURATION_MINUTES: ${{ inputs.duration_minutes || '15' }} + INPUT_SEED: ${{ inputs.seed || '0' }} + run: | + set -euo pipefail + if ! [[ "$INPUT_SEED" =~ ^[0-9]+$ ]] || (( INPUT_SEED > 2147483647 )); then + echo "seed must be between 0 and 2147483647" >&2 + exit 2 + fi + seed="$INPUT_SEED" + if (( seed == 0 )); then + seed=$((GITHUB_RUN_ID % 2147483647)) + if (( seed == 0 )); then seed=1; fi + fi + duration_seconds=$((INPUT_DURATION_MINUTES * 60)) + echo "seed=$seed" >> "$GITHUB_OUTPUT" + echo "duration_seconds=$duration_seconds" >> "$GITHUB_OUTPUT" + - name: Build soak harness + run: dotnet build test/SecsFrame.Soak/SecsFrame.Soak.csproj -c Release + - name: Run session fault soak + run: >- + dotnet run --project test/SecsFrame.Soak/SecsFrame.Soak.csproj + -c Release --no-build -- + --seed ${{ steps.parameters.outputs.seed }} + --duration-seconds ${{ steps.parameters.outputs.duration_seconds }} + --max-cycles 10000 + --output artifacts/soak/session-soak.jsonl + - name: Upload reproduction report + if: always() + uses: actions/upload-artifact@v6 + with: + name: session-soak-${{ github.run_id }}-${{ steps.parameters.outputs.seed }} + path: artifacts/soak/session-soak.jsonl + if-no-files-found: error + retention-days: 14 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dd4f7c..b262aa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,12 @@ ### 新增 +- 增加独立 SecsFrame.Soak 可执行程序，在真实 TCP 待处理事务中 + 按显式随机种子选择双方 Separate 或端点销毁重建，恢复 Selected 后立即 + 验证新事务；普通 dotnet test 不执行该长时负载。 +- 增加手动和每周定时的 session-soak 工作流，固定 20 分钟程序 + 上限、25 分钟作业上限和每周期 20 秒上限；成功或失败均上传包含 commit、 + 运行时、seed、故障序列和结果的 JSONL，保留 14 天。 - 增加真实 TCP Active/Passive 重复会话抖动故障注入测试；固定三轮断开与 重选并复用相同 System Bytes，验证待处理事务确定中断、旧回复与强制迟到 T3 回调不污染替换会话，且新事务可成功完成。 diff --git a/README.md b/README.md index 8b45b3d..e171009 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,8 @@ Item 用法与边界见 [docs/HSMS-SESSION-STATE-MACHINE.md](docs/HSMS-SESSION-STATE-MACHINE.md)， 数据事务边界见 [docs/HSMS-DATA-TRANSACTIONS.md](docs/HSMS-DATA-TRANSACTIONS.md)， +独立长时会话故障注入见 +[docs/HSMS-SOAK.md](docs/HSMS-SOAK.md)， 公共连接用法见 [docs/HSMS-CONNECTION.md](docs/HSMS-CONNECTION.md)， 结构化故障分类见 diff --git a/SecsFrame.slnx b/SecsFrame.slnx index 273bb9e..2283f00 100644 --- a/SecsFrame.slnx +++ b/SecsFrame.slnx @@ -7,6 +7,7 @@ + diff --git a/docs/DEVELOPMENT-STATUS.md b/docs/DEVELOPMENT-STATUS.md index 9243744..975d062 100644 --- a/docs/DEVELOPMENT-STATUS.md +++ b/docs/DEVELOPMENT-STATUS.md @@ -36,6 +36,9 @@ GitHub Issue/Actions： - 真实 TCP Active/Passive 事务测试固定重复三次断开和重选，复用相同 System Bytes 并强制投递旧 T3 回调，验证旧回复、旧计时器和待处理事务 不会污染替换会话。 +- 独立 SecsFrame.Soak 和 session-soak 工作流提供 + 显式 seed、四类真实 TCP 故障、程序/周期/作业三层上限及 14 天 JSONL + 产物；普通 CI 只编译 harness，不运行长时负载。 - 最近一次完整本地验证中，核心测试在 `net48`、`net8.0`、`net10.0` 各 通过 289 项；官方 secs4net 互操作测试在 `net8.0`、`net10.0` 各通过 2 项。 @@ -85,9 +88,11 @@ Primary/Secondary、断开和重选，并验证： - 测试时间有严格上限，可稳定进入现有普通 CI。 这个切片只增加故障注入测试和必要的测试支撑，不改变协议默认值或公共 API。 -下一可执行项是评估不进入普通 CI 的独立定时工作流：先固定运行上限、随机 -种子、故障重现信息和产物保留边界，再增加长时间 soak 与随机网络故障。 -该工作流不得引入新的协议默认值，也不能替代确定性回归和完整普通 CI。 +独立定时工作流已经落地。合并后应先以 workflow_dispatch 使用 +固定 seed 12345 运行 5 分钟，核对四种故障均出现、JSONL 可下载 +且失败定位信息完整；随后观察至少三次每周定时运行，再决定是否增加 TCP +half-close、reset 或跨平台矩阵。扩大故障范围前必须保持固定运行上限和可 +重现输入，也不能用 soak 替代确定性回归和完整普通 CI。 ## 状态更新规则 diff --git a/docs/HSMS-DATA-TRANSACTIONS.md b/docs/HSMS-DATA-TRANSACTIONS.md index ad498ab..c7f7525 100644 --- a/docs/HSMS-DATA-TRANSACTIONS.md +++ b/docs/HSMS-DATA-TRANSACTIONS.md @@ -85,6 +85,13 @@ Active/Passive 双端真实 TCP 回环另行覆盖 Select 后的 Primary/Seconda 会话回复，验证待处理事务确定中断且替换会话仍可完成本代事务。同步只使用 状态事件、计时器武装信号和事务任务，并由统一取消令牌严格封顶。 +独立 SecsFrame.Soak 可执行程序进一步在真实 TCP 待处理事务中 +按 seed 随机选择 Active Separate、Passive Separate、Active 端点销毁重建 +或 Passive 端点销毁重建。每次故障后必须重新进入 Selected 并完成新事务， +逐周期 JSONL 保留故障类型、中断异常和新旧 System Bytes。该负载只由手动/ +定时工作流运行，不进入普通 dotnet test；完整运行与复现边界见 +[HSMS-SOAK.md](HSMS-SOAK.md)。 + 这些测试是自主构造的工程向量，不是 SEMI 一致性测试。 ## 标准核对边界 diff --git a/docs/HSMS-SOAK.md b/docs/HSMS-SOAK.md new file mode 100644 index 0000000..c137223 --- /dev/null +++ b/docs/HSMS-SOAK.md @@ -0,0 +1,68 @@ +# HSMS 会话故障 Soak + +## 范围 + +SecsFrame.Soak 是独立的真实 TCP 工程故障注入程序。它加入 +solution 以便普通构建持续检查，但不是测试项目，因此 +dotnet test SecsFrame.slnx 不会执行长时负载。 + +每个周期先从 Active 端发送带 W-Bit 的 Primary，并在 Passive 端收到但尚未 +回复时，按显式随机 seed 选择一种故障： + +- Active 发送 Separate； +- Passive 发送 Separate； +- 销毁并重建 Active 公共连接； +- 销毁并重建 Passive 公共连接。 + +原事务必须以运输/会话中断结束。双方重新进入 Selected 后，程序立即执行 +新的 Primary/Secondary 往返并核对动态 Body。程序按 seed 对每组四种故障 +洗牌，因此每四个完整周期覆盖全部模式且顺序可复现；程序不使用设备输入、 +厂商兼容开关或新的协议默认值。 + +## 运行边界 + +独立 [session-soak 工作流](../.github/workflows/session-soak.yml) 支持： + +- 每周三 UTC 02:43 定时运行 15 分钟； +- 手动选择 1、5、15 或 20 分钟； +- 手动指定正 32 位 seed，0 从 GitHub Run ID 确定性派生； +- 程序最长 20 分钟、单周期最长 20 秒、作业最长 25 分钟； +- 同一时间只运行一个 soak 作业，后续触发排队而不取消已有运行。 + +普通 PR/push CI 不触发该工作流。它仍会通过 solution 的 Release 构建检查 +harness 是否可编译；协议向量、状态机测试和 secs4net 互操作门禁保持不变。 + +## 本地复现 + +~~~bash +dotnet build test/SecsFrame.Soak/SecsFrame.Soak.csproj -c Release +dotnet run --project test/SecsFrame.Soak/SecsFrame.Soak.csproj \ + -c Release --no-build -- \ + --seed 12345 \ + --duration-seconds 300 \ + --max-cycles 10000 \ + --output artifacts/soak/session-soak.jsonl +~~~ + +--seed 必须显式提供；duration 接受 1 到 1200 秒，max-cycles +接受 1 到 100000。达到任一上限即成功停止。若总时限在周期中到达，程序 +取消该周期、释放双方连接并写出 durationElapsed 完成记录；若 +时限内没有任何周期完成则失败，避免初始化停滞被误记为成功。 + +## 产物与复现 + +JSONL 在启动、每个周期开始、每个周期完成、失败和正常结束时立即刷新。 +启动记录包含 Git commit、.NET 运行时、OS、seed、duration 与 max-cycles； +周期记录包含故障类型、中断异常类型、原事务和恢复事务的 System Bytes 及 +耗时。失败记录包含 seed、周期、故障类型和异常文本。 + +工作流无论成功失败都上传该文件，保留 14 天。复现时应检出启动记录中的 +commit，使用相同运行时、OS、seed、duration 和 max-cycles；最后一个 +cycleStarted 给出失败前选择的确切故障。报告只包含程序生成的 +周期标记和协议/运行期元数据，不捕获设备 Body、原始 TCP 字节或异常帧。 + +## 证据边界 + +固定 seed 的本地短时验证必须至少覆盖四种故障并确认 duration 上限正常 +收口。定时 soak 用于发现长期资源、监听恢复和竞态问题，不替代普通 CI、 +确定性状态机/协议向量或跨实现测试，也不构成 SEMI 一致性或生产稳定性声明。 diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 0768343..4cbcad3 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -21,6 +21,8 @@ 确定的取消/释放语义。 - [x] 使用 nuget.org 官方 Secs4Net 3.1.0 的 Active/Passive、Select、 双方 Linktest 和双方 Primary/Secondary 跨实现测试。 +- [x] 不进入普通测试的可复现会话 soak：手动/定时触发、固定上限、显式 + seed、四类真实 TCP 故障和失败产物保留。 - [x] 公共结构化诊断模型，覆盖运输、协议、会话、事务和解码故障。 - [x] 运行期 Primary 路由与可释放处理器注册，不依赖预生成消息类型。 - [ ] 使用授权 E37/E37.1 核对 T5/T8 默认值与精确启停边界。 diff --git a/test/SecsFrame.Soak/HsmsConnectionPair.cs b/test/SecsFrame.Soak/HsmsConnectionPair.cs new file mode 100644 index 0000000..bdcfa4e --- /dev/null +++ b/test/SecsFrame.Soak/HsmsConnectionPair.cs @@ -0,0 +1,216 @@ +using System.Net; +using System.Net.Sockets; + +namespace SecsFrame.Soak; + +internal sealed class HsmsConnectionPair : IAsyncDisposable +{ + private const ushort ProtocolSessionId = 10; + private readonly int _port; + private HsmsEndpointProbe _passive; + private HsmsEndpointProbe _active; + + private HsmsConnectionPair(int port) + { + _port = port; + _passive = CreateEndpoint(HsmsConnectionMode.Passive); + _active = CreateEndpoint(HsmsConnectionMode.Active); + } + + public static async Task CreateAsync( + CancellationToken cancellationToken) + { + var pair = new HsmsConnectionPair(GetFreePort()); + try + { + await pair.WaitForRecoveryAsync( + new RecoveryTargets(0, 0), + cancellationToken).ConfigureAwait(false); + return pair; + } + catch + { + await pair.DisposeAsync().ConfigureAwait(false); + throw; + } + } + + public async Task RunCycleAsync( + int cycle, + SoakFaultMode faultMode, + CancellationToken cancellationToken) + { + var pending = _active.Connection.SendAsync( + CreatePrimary(cycle, marker: 0), + cancellationToken); + var interrupted = await _passive.NextIncomingAsync(cancellationToken) + .ConfigureAwait(false); + ValidateMessage(interrupted.DataMessage, cycle, marker: 0); + var targets = await InjectFaultAsync(faultMode, cancellationToken) + .ConfigureAwait(false); + var interruption = await ObserveInterruptionAsync( + pending, + cancellationToken).ConfigureAwait(false); + + await WaitForRecoveryAsync(targets, cancellationToken) + .ConfigureAwait(false); + var recoveredSystemBytes = await RoundTripAsync( + cycle, + cancellationToken).ConfigureAwait(false); + return new CycleResult( + interrupted.DataMessage.SystemBytes, + interruption, + recoveredSystemBytes); + } + + public async ValueTask DisposeAsync() + { + try + { + await _active.DisposeAsync().ConfigureAwait(false); + } + finally + { + await _passive.DisposeAsync().ConfigureAwait(false); + } + } + + private async Task InjectFaultAsync( + SoakFaultMode faultMode, + CancellationToken cancellationToken) + { + var targets = new RecoveryTargets( + _passive.SelectionGeneration, + _active.SelectionGeneration); + switch (faultMode) + { + case SoakFaultMode.ActiveSeparate: + await _active.Connection.SeparateAsync(cancellationToken) + .ConfigureAwait(false); + break; + case SoakFaultMode.PassiveSeparate: + await _passive.Connection.SeparateAsync(cancellationToken) + .ConfigureAwait(false); + break; + case SoakFaultMode.RestartActiveEndpoint: + await _active.DisposeAsync().ConfigureAwait(false); + _active = CreateEndpoint(HsmsConnectionMode.Active); + targets = targets with { Active = 0 }; + break; + case SoakFaultMode.RestartPassiveEndpoint: + await _passive.DisposeAsync().ConfigureAwait(false); + _passive = CreateEndpoint(HsmsConnectionMode.Passive); + targets = targets with { Passive = 0 }; + break; + default: + throw new ArgumentOutOfRangeException( + nameof(faultMode), + faultMode, + "Unknown soak fault mode."); + } + + return targets; + } + + private async Task WaitForRecoveryAsync( + RecoveryTargets targets, + CancellationToken cancellationToken) + { + await Task.WhenAll( + _passive.WaitForSelectionAfterAsync( + targets.Passive, + cancellationToken), + _active.WaitForSelectionAfterAsync( + targets.Active, + cancellationToken)).ConfigureAwait(false); + } + + private async Task RoundTripAsync( + int cycle, + CancellationToken cancellationToken) + { + var expected = CreateBody(cycle, marker: 1); + var send = _active.Connection.SendAsync( + CreatePrimary(cycle, marker: 1), + cancellationToken); + var incoming = await _passive.NextIncomingAsync(cancellationToken) + .ConfigureAwait(false); + ValidateMessage(incoming.DataMessage, cycle, marker: 1); + await _passive.Connection.ReplyAsync( + incoming, + new SecsMessage(6, 12, rootItem: expected), + cancellationToken).ConfigureAwait(false); + var secondary = await send.WaitAsync(cancellationToken) + .ConfigureAwait(false); + if (!Equals(secondary?.Message.RootItem, expected)) + throw new InvalidOperationException("The replacement transaction returned an unexpected secondary."); + return secondary.SystemBytes; + } + + private HsmsEndpointProbe CreateEndpoint(HsmsConnectionMode mode) + => HsmsEndpointProbe.Start(new HsmsConnectionOptions( + IPAddress.Loopback, + _port, + mode, + ProtocolSessionId, + t3: TimeSpan.FromSeconds(3), + t5: TimeSpan.FromMilliseconds(50), + t6: TimeSpan.FromSeconds(3), + t7: TimeSpan.FromSeconds(5), + t8: TimeSpan.FromSeconds(3))); + + private static async Task ObserveInterruptionAsync( + Task pending, + CancellationToken cancellationToken) + { + try + { + _ = await pending.WaitAsync(cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException("The pending transaction survived the injected session fault."); + } + catch (OperationCanceledException) + when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + when (ex is IOException or ObjectDisposedException or OperationCanceledException) + { + return ex.GetType().FullName ?? ex.GetType().Name; + } + } + + private static SecsMessage CreatePrimary(int cycle, byte marker) + => new(6, 11, true, CreateBody(cycle, marker)); + + private static SecsItem CreateBody(int cycle, byte marker) + => SecsItem.List(SecsItem.U4((uint)cycle), SecsItem.U1(marker)); + + private static void ValidateMessage( + HsmsDataMessage message, + int cycle, + byte marker) + { + if (message.SessionId != ProtocolSessionId || + !Equals(message.Message.RootItem, CreateBody(cycle, marker))) + { + throw new InvalidOperationException("The received primary did not match the current soak cycle."); + } + } + + private static int GetFreePort() + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; + } + + private sealed record RecoveryTargets(long Passive, long Active); + + internal sealed record CycleResult( + uint InterruptedSystemBytes, + string InterruptionException, + uint RecoveredSystemBytes); +} diff --git a/test/SecsFrame.Soak/HsmsEndpointProbe.cs b/test/SecsFrame.Soak/HsmsEndpointProbe.cs new file mode 100644 index 0000000..0d16b74 --- /dev/null +++ b/test/SecsFrame.Soak/HsmsEndpointProbe.cs @@ -0,0 +1,127 @@ +using System.Threading.Channels; + +namespace SecsFrame.Soak; + +internal sealed class HsmsEndpointProbe : IAsyncDisposable +{ + private readonly HsmsConnection _connection; + private readonly Channel _incoming = + Channel.CreateUnbounded(); + private readonly CancellationTokenSource _eventCancellation = new(); + private readonly object _selectionGate = new(); + private TaskCompletionSource _nextSelection = CreateSelectionSignal(); + private readonly Task _eventPump; + private long _selectionGeneration; + + private HsmsEndpointProbe(HsmsConnectionOptions options) + { + _connection = new HsmsConnection(options); + _connection.Start(); + _eventPump = PumpEventsAsync(); + } + + public HsmsConnection Connection => _connection; + + public long SelectionGeneration + { + get + { + lock (_selectionGate) + return _selectionGeneration; + } + } + + public static HsmsEndpointProbe Start(HsmsConnectionOptions options) + => new(options); + + public ValueTask NextIncomingAsync( + CancellationToken cancellationToken) + => _incoming.Reader.ReadAsync(cancellationToken); + + public async Task WaitForSelectionAfterAsync( + long previousGeneration, + CancellationToken cancellationToken) + { + while (true) + { + Task signal; + lock (_selectionGate) + { + if (_selectionGeneration > previousGeneration) + return _selectionGeneration; + signal = _nextSelection.Task; + } + + await signal.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + public async ValueTask DisposeAsync() + { + _eventCancellation.Cancel(); + await _connection.DisposeAsync().ConfigureAwait(false); + try + { + await _eventPump.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + } + _eventCancellation.Dispose(); + } + + private async Task PumpEventsAsync() + { + Exception? completionError = null; + try + { + await foreach (var item in _connection + .GetEventsAsync(_eventCancellation.Token) + .ConfigureAwait(false)) + { + ProcessEvent(item); + } + } + catch (OperationCanceledException) + when (_eventCancellation.IsCancellationRequested) + { + } + catch (Exception ex) + { + completionError = ex; + throw; + } + finally + { + _incoming.Writer.TryComplete(completionError); + } + } + + private void ProcessEvent(HsmsConnectionEvent item) + { + if (item.Kind == HsmsConnectionEventKind.DataMessageReceived) + { + _incoming.Writer.TryWrite(item.IncomingMessage!); + return; + } + + if (item.Kind != HsmsConnectionEventKind.StateChanged || + item.State != HsmsSessionState.Selected) + { + return; + } + + TaskCompletionSource signal; + long generation; + lock (_selectionGate) + { + generation = ++_selectionGeneration; + signal = _nextSelection; + _nextSelection = CreateSelectionSignal(); + } + signal.TrySetResult(generation); + } + + private static TaskCompletionSource CreateSelectionSignal() + => new(TaskCreationOptions.RunContinuationsAsynchronously); +} diff --git a/test/SecsFrame.Soak/HsmsSoakRunner.cs b/test/SecsFrame.Soak/HsmsSoakRunner.cs new file mode 100644 index 0000000..2dd57f2 --- /dev/null +++ b/test/SecsFrame.Soak/HsmsSoakRunner.cs @@ -0,0 +1,183 @@ +using System.Diagnostics; +using System.Runtime.InteropServices; + +namespace SecsFrame.Soak; + +internal sealed class HsmsSoakRunner +{ + private static readonly SoakFaultMode[] FaultModes = + Enum.GetValues(); + private static readonly TimeSpan CycleTimeout = TimeSpan.FromSeconds(20); + private readonly SoakOptions _options; + private readonly SoakReportWriter _report; + private readonly Random _random; + private readonly SoakFaultMode[] _faultOrder = + Enum.GetValues(); + private readonly Dictionary _faultCounts = + FaultModes.ToDictionary(static item => item, static _ => 0); + private int _faultIndex = FaultModes.Length; + + public HsmsSoakRunner(SoakOptions options, SoakReportWriter report) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _report = report ?? throw new ArgumentNullException(nameof(report)); + _random = new Random(options.Seed); + } + + public async Task RunAsync() + { + var stopwatch = Stopwatch.StartNew(); + var progress = new RunProgress(); + await WriteStartedAsync().ConfigureAwait(false); + Console.WriteLine( + $"SecsFrame session soak seed={_options.Seed} " + + $"duration={_options.Duration} report={_report.OutputPath}"); + + using var lifetime = new CancellationTokenSource(_options.Duration); + try + { + await ExecuteCyclesAsync(stopwatch, progress, lifetime.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (lifetime.IsCancellationRequested) + { + } + catch (Exception ex) + { + await WriteFailureAsync(progress, ex).ConfigureAwait(false); + throw; + } + + if (progress.CompletedCycles == 0) + { + var error = new InvalidOperationException( + "The soak duration elapsed before any fault cycle completed."); + await WriteFailureAsync(progress, error).ConfigureAwait(false); + throw error; + } + + await WriteCompletedAsync(progress, stopwatch).ConfigureAwait(false); + } + + private async Task ExecuteCyclesAsync( + Stopwatch stopwatch, + RunProgress progress, + CancellationToken cancellationToken) + { + var pair = await HsmsConnectionPair + .CreateAsync(cancellationToken).ConfigureAwait(false); + await using var pairScope = pair.ConfigureAwait(false); + for (; progress.CompletedCycles < _options.MaxCycles; + progress.CompletedCycles++) + { + cancellationToken.ThrowIfCancellationRequested(); + progress.CurrentCycle = progress.CompletedCycles; + progress.CurrentFault = NextFaultMode(); + await RunCycleAsync( + pair, + progress.CurrentCycle, + progress.CurrentFault.Value, + stopwatch, + cancellationToken).ConfigureAwait(false); + } + } + + private ValueTask WriteFailureAsync(RunProgress progress, Exception error) + => _report.WriteAsync(new SoakReportWriter.RunFailed( + "runFailed", + DateTimeOffset.UtcNow, + _options.Seed, + progress.CurrentCycle, + progress.CurrentFault, + error.ToString())); + + private async Task WriteCompletedAsync( + RunProgress progress, + Stopwatch stopwatch) + { + var reason = progress.CompletedCycles >= _options.MaxCycles + ? "maxCyclesReached" + : "durationElapsed"; + await _report.WriteAsync(new SoakReportWriter.RunCompleted( + "runCompleted", + DateTimeOffset.UtcNow, + _options.Seed, + progress.CompletedCycles, + reason, + stopwatch.ElapsedMilliseconds, + _faultCounts)).ConfigureAwait(false); + Console.WriteLine( + $"Completed {progress.CompletedCycles} cycles in {stopwatch.Elapsed}; reason={reason}."); + } + + private async Task RunCycleAsync( + HsmsConnectionPair pair, + int cycle, + SoakFaultMode faultMode, + Stopwatch runStopwatch, + CancellationToken lifetimeToken) + { + await _report.WriteAsync(new SoakReportWriter.CycleStarted( + "cycleStarted", + DateTimeOffset.UtcNow, + cycle, + faultMode)).ConfigureAwait(false); + using var cycleCancellation = CancellationTokenSource + .CreateLinkedTokenSource(lifetimeToken); + cycleCancellation.CancelAfter(CycleTimeout); + var startedAt = runStopwatch.ElapsedMilliseconds; + var result = await pair.RunCycleAsync( + cycle, + faultMode, + cycleCancellation.Token).ConfigureAwait(false); + _faultCounts[faultMode]++; + await _report.WriteAsync(new SoakReportWriter.CycleCompleted( + "cycleCompleted", + DateTimeOffset.UtcNow, + cycle, + faultMode, + result.InterruptedSystemBytes, + result.InterruptionException, + result.RecoveredSystemBytes, + runStopwatch.ElapsedMilliseconds - startedAt)).ConfigureAwait(false); + + if (cycle % 25 == 0) + Console.WriteLine($"cycle={cycle} fault={faultMode}"); + } + + private ValueTask WriteStartedAsync() + => _report.WriteAsync(new SoakReportWriter.RunStarted( + "runStarted", + DateTimeOffset.UtcNow, + _options.Seed, + checked((int)_options.Duration.TotalSeconds), + _options.MaxCycles, + Environment.GetEnvironmentVariable("GITHUB_SHA") ?? "local", + RuntimeInformation.FrameworkDescription, + RuntimeInformation.OSDescription)); + + private SoakFaultMode NextFaultMode() + { + if (_faultIndex >= _faultOrder.Length) + { + for (var index = _faultOrder.Length - 1; index > 0; index--) + { + var target = _random.Next(index + 1); + (_faultOrder[index], _faultOrder[target]) = + (_faultOrder[target], _faultOrder[index]); + } + _faultIndex = 0; + } + + return _faultOrder[_faultIndex++]; + } + + private sealed class RunProgress + { + public int CompletedCycles { get; set; } + + public int CurrentCycle { get; set; } = -1; + + public SoakFaultMode? CurrentFault { get; set; } + } +} diff --git a/test/SecsFrame.Soak/Program.cs b/test/SecsFrame.Soak/Program.cs new file mode 100644 index 0000000..b58f2e4 --- /dev/null +++ b/test/SecsFrame.Soak/Program.cs @@ -0,0 +1,33 @@ +namespace SecsFrame.Soak; + +internal static class Program +{ + public static async Task Main(string[] args) + { + if (SoakOptions.IsHelpRequested(args)) + { + Console.WriteLine(SoakOptions.Usage); + return 0; + } + + try + { + var options = SoakOptions.Parse(args); + var report = SoakReportWriter.Create(options.OutputPath); + await using var reportScope = report.ConfigureAwait(false); + await new HsmsSoakRunner(options, report).RunAsync().ConfigureAwait(false); + return 0; + } + catch (SoakConfigurationException ex) + { + Console.Error.WriteLine(ex.Message); + Console.Error.WriteLine(SoakOptions.Usage); + return 2; + } + catch (Exception ex) + { + Console.Error.WriteLine(ex); + return 1; + } + } +} diff --git a/test/SecsFrame.Soak/SecsFrame.Soak.csproj b/test/SecsFrame.Soak/SecsFrame.Soak.csproj new file mode 100644 index 0000000..1752317 --- /dev/null +++ b/test/SecsFrame.Soak/SecsFrame.Soak.csproj @@ -0,0 +1,13 @@ + + + Exe + net8.0 + false + + + + + + + + diff --git a/test/SecsFrame.Soak/SoakConfigurationException.cs b/test/SecsFrame.Soak/SoakConfigurationException.cs new file mode 100644 index 0000000..e2530d9 --- /dev/null +++ b/test/SecsFrame.Soak/SoakConfigurationException.cs @@ -0,0 +1,9 @@ +namespace SecsFrame.Soak; + +internal sealed class SoakConfigurationException : Exception +{ + public SoakConfigurationException(string message) + : base(message) + { + } +} diff --git a/test/SecsFrame.Soak/SoakFaultMode.cs b/test/SecsFrame.Soak/SoakFaultMode.cs new file mode 100644 index 0000000..e96655e --- /dev/null +++ b/test/SecsFrame.Soak/SoakFaultMode.cs @@ -0,0 +1,9 @@ +namespace SecsFrame.Soak; + +internal enum SoakFaultMode +{ + ActiveSeparate, + PassiveSeparate, + RestartActiveEndpoint, + RestartPassiveEndpoint, +} diff --git a/test/SecsFrame.Soak/SoakOptions.cs b/test/SecsFrame.Soak/SoakOptions.cs new file mode 100644 index 0000000..f9a4821 --- /dev/null +++ b/test/SecsFrame.Soak/SoakOptions.cs @@ -0,0 +1,99 @@ +using System.Globalization; + +namespace SecsFrame.Soak; + +internal sealed record SoakOptions( + int Seed, + TimeSpan Duration, + int MaxCycles, + string OutputPath) +{ + public const string Usage = + "Usage: SecsFrame.Soak --seed [--duration-seconds <1..1200>] " + + "[--max-cycles <1..100000>] [--output ]"; + + public static bool IsHelpRequested(string[] args) + => args.Any(static item => item is "--help" or "-h"); + + public static SoakOptions Parse(string[] args) + { + ArgumentNullException.ThrowIfNull(args); + int? seed = null; + var durationSeconds = 900; + var maxCycles = 10_000; + var outputPath = Path.Combine("artifacts", "soak", "session-soak.jsonl"); + + for (var index = 0; index < args.Length; index++) + { + switch (args[index]) + { + case "--seed": + seed = ReadInteger(args, ref index, "--seed"); + break; + case "--duration-seconds": + durationSeconds = ReadInteger(args, ref index, "--duration-seconds"); + break; + case "--max-cycles": + maxCycles = ReadInteger(args, ref index, "--max-cycles"); + break; + case "--output": + outputPath = ReadValue(args, ref index, "--output"); + break; + default: + throw new SoakConfigurationException($"Unknown option: {args[index]}"); + } + } + + ValidateRange(durationSeconds, 1, 1200, "--duration-seconds"); + ValidateRange(maxCycles, 1, 100_000, "--max-cycles"); + if (seed is null) + throw new SoakConfigurationException("--seed is required for reproducibility."); + + return new SoakOptions( + seed.Value, + TimeSpan.FromSeconds(durationSeconds), + maxCycles, + Path.GetFullPath(outputPath)); + } + + private static int ReadInteger( + string[] args, + ref int index, + string option) + { + var value = ReadValue(args, ref index, option); + if (!int.TryParse( + value, + NumberStyles.Integer, + CultureInfo.InvariantCulture, + out var parsed)) + { + throw new SoakConfigurationException($"{option} must be a 32-bit integer."); + } + return parsed; + } + + private static string ReadValue( + string[] args, + ref int index, + string option) + { + index++; + if (index >= args.Length || string.IsNullOrWhiteSpace(args[index])) + throw new SoakConfigurationException($"{option} requires a value."); + return args[index]; + } + + private static void ValidateRange( + int value, + int minimum, + int maximum, + string option) + { + if (value < minimum || value > maximum) + { + throw new SoakConfigurationException( + $"{option} must be between {minimum} and {maximum}."); + } + } +} diff --git a/test/SecsFrame.Soak/SoakReportWriter.cs b/test/SecsFrame.Soak/SoakReportWriter.cs new file mode 100644 index 0000000..71af595 --- /dev/null +++ b/test/SecsFrame.Soak/SoakReportWriter.cs @@ -0,0 +1,92 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace SecsFrame.Soak; + +internal sealed class SoakReportWriter : IAsyncDisposable +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + Converters = { new JsonStringEnumConverter() }, + }; + + private readonly StreamWriter _writer; + + private SoakReportWriter(string outputPath, StreamWriter writer) + { + OutputPath = outputPath; + _writer = writer; + } + + public string OutputPath { get; } + + public static SoakReportWriter Create(string outputPath) + { + var directory = Path.GetDirectoryName(outputPath); + if (string.IsNullOrEmpty(directory)) + throw new SoakConfigurationException("The report path requires a directory."); + + Directory.CreateDirectory(directory); + var stream = new FileStream( + outputPath, + FileMode.Create, + FileAccess.Write, + FileShare.Read, + bufferSize: 4096, + useAsync: true); + return new SoakReportWriter(outputPath, new StreamWriter(stream)); + } + + public async ValueTask WriteAsync(T value) + { + var json = JsonSerializer.Serialize(value, SerializerOptions); + await _writer.WriteLineAsync(json).ConfigureAwait(false); + await _writer.FlushAsync().ConfigureAwait(false); + } + + public ValueTask DisposeAsync() => _writer.DisposeAsync(); + + internal sealed record RunStarted( + string Event, + DateTimeOffset TimestampUtc, + int Seed, + int DurationSeconds, + int MaxCycles, + string GitCommit, + string Framework, + string OperatingSystem); + + internal sealed record CycleStarted( + string Event, + DateTimeOffset TimestampUtc, + int Cycle, + SoakFaultMode FaultMode); + + internal sealed record CycleCompleted( + string Event, + DateTimeOffset TimestampUtc, + int Cycle, + SoakFaultMode FaultMode, + uint InterruptedSystemBytes, + string InterruptionException, + uint RecoveredSystemBytes, + long ElapsedMilliseconds); + + internal sealed record RunFailed( + string Event, + DateTimeOffset TimestampUtc, + int Seed, + int Cycle, + SoakFaultMode? FaultMode, + string Exception); + + internal sealed record RunCompleted( + string Event, + DateTimeOffset TimestampUtc, + int Seed, + int CompletedCycles, + string CompletionReason, + long ElapsedMilliseconds, + IReadOnlyDictionary FaultCounts); +}