Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ jobs:
build-and-test:
name: ${{ matrix.os }}
runs-on: ${{ matrix.os }}
timeout-minutes: 10
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
Expand Down
3 changes: 2 additions & 1 deletion test/Pants.Tests/Cloud/PantsCloudCrashRecoveryTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,8 @@ static async Task WaitForChildReadinessAsync(Process child, string databasePath)
var readyPath = Path.Combine(databasePath, ReadyFileName);
while (!File.Exists(readyPath))
{
if (child.HasExited)
// The child can publish readiness and exit between these two observations.
if (child.HasExited && !File.Exists(readyPath))
{
throw new XunitException(
$"Crash child exited with code {child.ExitCode} before readiness.");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -290,10 +290,8 @@ public async Task ShouldMeasureAppendAndFsyncAtSeparatePhysicalBoundaries()

var metrics = await database.Diagnostics.GetRuntimeMetricsAsync();
Assert.True(metrics.WalAppendNanosecondsTotal >= 50_000_000);
Assert.True(
metrics.WalFsyncNanosecondsTotal < metrics.WalAppendNanosecondsTotal,
$"Expected fsync {metrics.WalFsyncNanosecondsTotal} ns to exclude " +
$"the delayed append {metrics.WalAppendNanosecondsTotal} ns.");
Assert.Equal(1, metrics.WalFsyncCount);
Assert.InRange(metrics.WalFsyncNanosecondsTotal, 1, failpoints.FsyncWindowNanoseconds);
}

static async ValueTask CommitAsync(
Expand Down
15 changes: 15 additions & 0 deletions test/Pants.Tests/Observability/WalAppendDelayFailpointHandler.cs
Original file line number Diff line number Diff line change
@@ -1,12 +1,27 @@
using System.Diagnostics;

namespace Cntryl.Pants.Observability;

sealed class WalAppendDelayFailpointHandler(TimeSpan delay) : IFailpointHandler
{
long _flushStarted;

public long FsyncWindowNanoseconds { get; private set; }

public void Hit(Failpoint failpoint)
{
if (failpoint == Failpoint.MidWalAppend)
{
Thread.Sleep(delay);
}

if (failpoint == Failpoint.BeforeWalFlush)
{
_flushStarted = Stopwatch.GetTimestamp();
}
else if (failpoint == Failpoint.AfterWalFlush)
{
FsyncWindowNanoseconds = checked(Stopwatch.GetElapsedTime(_flushStarted).Ticks * 100);
}
}
}
4 changes: 2 additions & 2 deletions test/Pants.Tests/Runtime/PantsBackgroundFlushPipelineTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2623,7 +2623,7 @@ await CommitAsync(
{
Assert.True(await database.Maintenance.WaitForWriteStallClearAsync(
family,
AssertionTimeout));
BackgroundWorkTimeout));
}
}
}
Expand Down Expand Up @@ -2651,7 +2651,7 @@ async Task ReadLayoutAsync()
// legitimately take longer than five seconds while eight 132 KiB writes are flushed
// alongside repeated manifest reads.
await Task.WhenAll(work).WaitAsync(BackgroundWorkTimeout);
await database.Maintenance.FlushAsync(family).AsTask().WaitAsync(AssertionTimeout);
await database.Maintenance.FlushAsync(family).AsTask().WaitAsync(BackgroundWorkTimeout);

var layout = await database.Diagnostics.GetStorageLayoutAsync();
var names = layout.Levels.SelectMany(static level => level.Files)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,8 @@ static async Task WaitForCrashChildAsync(Process child, string path)
var readyPath = Path.Combine(path, CrashReadyFileName);
while (!File.Exists(readyPath))
{
if (child.HasExited)
// The child can publish readiness and exit between these two observations.
if (child.HasExited && !File.Exists(readyPath))
{
throw new XunitException(
$"Differential crash child exited with {child.ExitCode} before readiness.");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -366,7 +366,8 @@ static async Task WaitForChildReadinessAsync(Process child, string databasePath)
{
while (!File.Exists(readyPath))
{
if (child.HasExited)
// The child can publish readiness and exit between these two observations.
if (child.HasExited && !File.Exists(readyPath))
{
throw new XunitException(
$"Coalesced-commit crash child exited with code {child.ExitCode} " +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,8 @@ static async Task WaitForChildReadinessAsync(Process child, string databasePath)
{
while (!File.Exists(readyPath))
{
if (child.HasExited)
// The child can publish readiness and exit between these two observations.
if (child.HasExited && !File.Exists(readyPath))
{
throw new XunitException(
$"Transaction spill crash child exited with code {child.ExitCode} before readiness.");
Expand Down
Loading