diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4fa3d1..42adc09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: build-and-test: name: ${{ matrix.os }} runs-on: ${{ matrix.os }} - timeout-minutes: 10 + timeout-minutes: 15 strategy: fail-fast: false matrix: diff --git a/test/Pants.Tests/Cloud/PantsCloudCrashRecoveryTests.cs b/test/Pants.Tests/Cloud/PantsCloudCrashRecoveryTests.cs index 41a52ab..b2f3cda 100644 --- a/test/Pants.Tests/Cloud/PantsCloudCrashRecoveryTests.cs +++ b/test/Pants.Tests/Cloud/PantsCloudCrashRecoveryTests.cs @@ -331,7 +331,8 @@ static async Task WaitForChildReadinessAsync(Process child, string databasePath) var readyPath = Path.Combine(databasePath, ReadyFileName); while (!File.Exists(readyPath)) { - if (child.HasExited) + // The child can publish readiness and exit between these two observations. + if (child.HasExited && !File.Exists(readyPath)) { throw new XunitException( $"Crash child exited with code {child.ExitCode} before readiness."); diff --git a/test/Pants.Tests/Observability/PantsWalDurabilityMetricsTests.cs b/test/Pants.Tests/Observability/PantsWalDurabilityMetricsTests.cs index f5e791f..d745bf3 100644 --- a/test/Pants.Tests/Observability/PantsWalDurabilityMetricsTests.cs +++ b/test/Pants.Tests/Observability/PantsWalDurabilityMetricsTests.cs @@ -290,10 +290,8 @@ public async Task ShouldMeasureAppendAndFsyncAtSeparatePhysicalBoundaries() var metrics = await database.Diagnostics.GetRuntimeMetricsAsync(); Assert.True(metrics.WalAppendNanosecondsTotal >= 50_000_000); - Assert.True( - metrics.WalFsyncNanosecondsTotal < metrics.WalAppendNanosecondsTotal, - $"Expected fsync {metrics.WalFsyncNanosecondsTotal} ns to exclude " + - $"the delayed append {metrics.WalAppendNanosecondsTotal} ns."); + Assert.Equal(1, metrics.WalFsyncCount); + Assert.InRange(metrics.WalFsyncNanosecondsTotal, 1, failpoints.FsyncWindowNanoseconds); } static async ValueTask CommitAsync( diff --git a/test/Pants.Tests/Observability/WalAppendDelayFailpointHandler.cs b/test/Pants.Tests/Observability/WalAppendDelayFailpointHandler.cs index 4ce8d80..06942f3 100644 --- a/test/Pants.Tests/Observability/WalAppendDelayFailpointHandler.cs +++ b/test/Pants.Tests/Observability/WalAppendDelayFailpointHandler.cs @@ -1,12 +1,27 @@ +using System.Diagnostics; + namespace Cntryl.Pants.Observability; sealed class WalAppendDelayFailpointHandler(TimeSpan delay) : IFailpointHandler { + long _flushStarted; + + public long FsyncWindowNanoseconds { get; private set; } + public void Hit(Failpoint failpoint) { if (failpoint == Failpoint.MidWalAppend) { Thread.Sleep(delay); } + + if (failpoint == Failpoint.BeforeWalFlush) + { + _flushStarted = Stopwatch.GetTimestamp(); + } + else if (failpoint == Failpoint.AfterWalFlush) + { + FsyncWindowNanoseconds = checked(Stopwatch.GetElapsedTime(_flushStarted).Ticks * 100); + } } } diff --git a/test/Pants.Tests/Runtime/PantsBackgroundFlushPipelineTests.cs b/test/Pants.Tests/Runtime/PantsBackgroundFlushPipelineTests.cs index 8007a1a..9c117a2 100644 --- a/test/Pants.Tests/Runtime/PantsBackgroundFlushPipelineTests.cs +++ b/test/Pants.Tests/Runtime/PantsBackgroundFlushPipelineTests.cs @@ -2623,7 +2623,7 @@ await CommitAsync( { Assert.True(await database.Maintenance.WaitForWriteStallClearAsync( family, - AssertionTimeout)); + BackgroundWorkTimeout)); } } } @@ -2651,7 +2651,7 @@ async Task ReadLayoutAsync() // legitimately take longer than five seconds while eight 132 KiB writes are flushed // alongside repeated manifest reads. await Task.WhenAll(work).WaitAsync(BackgroundWorkTimeout); - await database.Maintenance.FlushAsync(family).AsTask().WaitAsync(AssertionTimeout); + await database.Maintenance.FlushAsync(family).AsTask().WaitAsync(BackgroundWorkTimeout); var layout = await database.Diagnostics.GetStorageLayoutAsync(); var names = layout.Levels.SelectMany(static level => level.Files) diff --git a/test/Pants.Tests/Storage/PantsDiskResidentDifferentialTests.cs b/test/Pants.Tests/Storage/PantsDiskResidentDifferentialTests.cs index 0af2018..ca994e8 100644 --- a/test/Pants.Tests/Storage/PantsDiskResidentDifferentialTests.cs +++ b/test/Pants.Tests/Storage/PantsDiskResidentDifferentialTests.cs @@ -312,7 +312,8 @@ static async Task WaitForCrashChildAsync(Process child, string path) var readyPath = Path.Combine(path, CrashReadyFileName); while (!File.Exists(readyPath)) { - if (child.HasExited) + // The child can publish readiness and exit between these two observations. + if (child.HasExited && !File.Exists(readyPath)) { throw new XunitException( $"Differential crash child exited with {child.ExitCode} before readiness."); diff --git a/test/Pants.Tests/Transactions/PantsCommitCoalescingCrashRecoveryTests.cs b/test/Pants.Tests/Transactions/PantsCommitCoalescingCrashRecoveryTests.cs index fa22ee2..1ea83f9 100644 --- a/test/Pants.Tests/Transactions/PantsCommitCoalescingCrashRecoveryTests.cs +++ b/test/Pants.Tests/Transactions/PantsCommitCoalescingCrashRecoveryTests.cs @@ -366,7 +366,8 @@ static async Task WaitForChildReadinessAsync(Process child, string databasePath) { while (!File.Exists(readyPath)) { - if (child.HasExited) + // The child can publish readiness and exit between these two observations. + if (child.HasExited && !File.Exists(readyPath)) { throw new XunitException( $"Coalesced-commit crash child exited with code {child.ExitCode} " + diff --git a/test/Pants.Tests/Transactions/Spill/PantsTransactionSpillCrashRecoveryTests.cs b/test/Pants.Tests/Transactions/Spill/PantsTransactionSpillCrashRecoveryTests.cs index 9ba3f8b..45b847e 100644 --- a/test/Pants.Tests/Transactions/Spill/PantsTransactionSpillCrashRecoveryTests.cs +++ b/test/Pants.Tests/Transactions/Spill/PantsTransactionSpillCrashRecoveryTests.cs @@ -177,7 +177,8 @@ static async Task WaitForChildReadinessAsync(Process child, string databasePath) { while (!File.Exists(readyPath)) { - if (child.HasExited) + // The child can publish readiness and exit between these two observations. + if (child.HasExited && !File.Exists(readyPath)) { throw new XunitException( $"Transaction spill crash child exited with code {child.ExitCode} before readiness.");