Skip to content

test(audit): fix flaky 412-retry backoff tests with setTimeout stub #298

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kptdobe merged 5 commits into from
Jul 7, 2026
+149 −130
Merged

test(audit): fix flaky 412-retry backoff tests with setTimeout stub #298

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0d1332e
fix(test): stub setTimeout in audit 412-retry test to prevent timeout…
shsteimer Jul 6, 2026
ed6e089
refactor(test): consolidate 412 retry backoff tests under shared setT…
shsteimer Jul 6, 2026
6e9f78e
style(test): trim verbose comments in audit 412-retry tests
shsteimer Jul 6, 2026
03354de
style(test): shorten 412 retry backoff comment to one line
shsteimer Jul 6, 2026
5b58501
style(test): reword Math.random stub comment
shsteimer Jul 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
279 changes: 149 additions & 130 deletions test/storage/version/audit.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -814,70 +814,27 @@ describe('Version Audit', () => {
assert.strictEqual(lines.length, 2, 'both entries must be present (no collapse across mismatched users)');
});

it('returns status 500 when PUT 412 persists across all 7 attempts', async () => {
let getCallCount = 0;
let putCallCount = 0;
describe('412 retry backoff', () => {
// stub/restore setTimeout to avoid exceeding mocha 2000ms timeout
let originalSetTimeout;

const makeBody = () => new ReadableStream({
start(controller) {
controller.enqueue(new TextEncoder().encode(''));
controller.close();
},
beforeEach(() => {
originalSetTimeout = globalThis.setTimeout;
});

const { writeAuditEntry } = await esmock(
'../../../src/storage/version/audit.js',
{
'@aws-sdk/client-s3': {
S3Client: function S3Client() {
this.send = async (cmd) => {
if (cmd instanceof GetObjectCommand) {
getCallCount += 1;
return { Body: makeBody(), ETag: '"etag-x"' };
}
if (cmd instanceof PutObjectCommand) {
putCallCount += 1;
const err = new Error('precondition failed');
err.$metadata = { httpStatusCode: 412 };
throw err;
}
return { $metadata: { httpStatusCode: 200 } };
};
},
GetObjectCommand,
PutObjectCommand,
},
'../../../src/storage/utils/config.js': { default: () => ({}) },
},
);

const result = await writeAuditEntry({}, { bucket: 'b', org: 'o' }, 'repo', 'fid', {
timestamp: '5000',
users: '[{"email":"u@x.com"}]',
path: 'repo/doc.html',
afterEach(() => {
globalThis.setTimeout = originalSetTimeout;
});

assert.strictEqual(result.status, 500, 'persistent 412 must surface as 500 after 7 total attempts');
assert.strictEqual(result.error, 'precondition failed');
assert.strictEqual(putCallCount, 7, 'must attempt PUT 7 times total (1 initial + 6 retries)');
assert.strictEqual(getCallCount, 7, 'must re-read on each attempt');
});
it('returns status 500 when PUT 412 persists across all 7 attempts', async () => {
globalThis.setTimeout = (fn) => {
fn();
return 0;
};

it('uses exponential jitter backoff (0-50, 0-100, 0-200, 0-400, 0-800, 0-1600 ms) across six 412 retries', async () => {
// Stubs setTimeout and Math.random to capture per-retry delays.
// Asserts per-attempt exponential upper bounds for the 6-retry ladder
// (50, 100, 200, 400, 800, 1600 ms) and total elapsed sleep ~3050 ms.
const originalSetTimeout = globalThis.setTimeout;
const originalRandom = Math.random;
const delays = [];
Math.random = () => 0.999999;
globalThis.setTimeout = (fn, ms) => {
delays.push(ms);
fn();
return 0;
};
let getCallCount = 0;
let putCallCount = 0;

try {
const makeBody = () => new ReadableStream({
start(controller) {
controller.enqueue(new TextEncoder().encode(''));
Expand All @@ -892,9 +849,11 @@ describe('Version Audit', () => {
S3Client: function S3Client() {
this.send = async (cmd) => {
if (cmd instanceof GetObjectCommand) {
getCallCount += 1;
return { Body: makeBody(), ETag: '"etag-x"' };
}
if (cmd instanceof PutObjectCommand) {
putCallCount += 1;
const err = new Error('precondition failed');
err.$metadata = { httpStatusCode: 412 };
throw err;
Expand All @@ -909,96 +868,156 @@ describe('Version Audit', () => {
},
);

await writeAuditEntry({}, { bucket: 'b', org: 'o' }, 'repo', 'fid', {
const result = await writeAuditEntry({}, { bucket: 'b', org: 'o' }, 'repo', 'fid', {
timestamp: '5000',
users: '[{"email":"u@x.com"}]',
path: 'repo/doc.html',
});
} finally {
globalThis.setTimeout = originalSetTimeout;
Math.random = originalRandom;
}

assert.strictEqual(delays.length, 6, 'must sleep between each of the 6 retries');
const upperBounds = [50, 100, 200, 400, 800, 1600];
delays.forEach((ms, i) => {
assert.strictEqual(result.status, 500, 'persistent 412 must surface as 500 after 7 total attempts');
assert.strictEqual(result.error, 'precondition failed');
assert.strictEqual(putCallCount, 7, 'must attempt PUT 7 times total (1 initial + 6 retries)');
assert.strictEqual(getCallCount, 7, 'must re-read on each attempt');
});

it('uses exponential jitter backoff (0-50, 0-100, 0-200, 0-400, 0-800, 0-1600 ms) across six 412 retries', async () => {
// stub Math.random so we can capture per-retry delays
const originalRandom = Math.random;
const delays = [];
Math.random = () => 0.999999;
globalThis.setTimeout = (fn, ms) => {
delays.push(ms);
fn();
return 0;
};

try {
const makeBody = () => new ReadableStream({
start(controller) {
controller.enqueue(new TextEncoder().encode(''));
controller.close();
},
});

const { writeAuditEntry } = await esmock(
'../../../src/storage/version/audit.js',
{
'@aws-sdk/client-s3': {
S3Client: function S3Client() {
this.send = async (cmd) => {
if (cmd instanceof GetObjectCommand) {
return { Body: makeBody(), ETag: '"etag-x"' };
}
if (cmd instanceof PutObjectCommand) {
const err = new Error('precondition failed');
err.$metadata = { httpStatusCode: 412 };
throw err;
}
return { $metadata: { httpStatusCode: 200 } };
};
},
GetObjectCommand,
PutObjectCommand,
},
'../../../src/storage/utils/config.js': { default: () => ({}) },
},
);

await writeAuditEntry({}, { bucket: 'b', org: 'o' }, 'repo', 'fid', {
timestamp: '5000',
users: '[{"email":"u@x.com"}]',
path: 'repo/doc.html',
});
} finally {
Math.random = originalRandom;
}

assert.strictEqual(delays.length, 6, 'must sleep between each of the 6 retries');
const upperBounds = [50, 100, 200, 400, 800, 1600];
delays.forEach((ms, i) => {
assert.ok(
ms >= 0 && ms < upperBounds[i],
`delay[${i}]=${ms} must be within [0, ${upperBounds[i]})`,
);
});
assert.ok(
delays[2] >= 150,
`delay[2]=${delays[2]} must exceed the prior linear cap of 150 ms`,
);
assert.ok(
delays[3] >= 200,
`delay[3]=${delays[3]} must exceed the prior linear cap of 200 ms`,
);
assert.ok(
delays[5] >= 800,
`delay[5]=${delays[5]} must reach the new 6-retry exponential ceiling (>=800 ms)`,
);
const total = delays.reduce((a, b) => a + b, 0);
assert.ok(
ms >= 0 && ms < upperBounds[i],
`delay[${i}]=${ms} must be within [0, ${upperBounds[i]})`,
total > 1500,
`total elapsed sleep ${total} must exceed prior 4-retry worst-case (~750 ms)`,
);
assert.ok(
total < 3200,
`total elapsed sleep ${total} must stay within the new 6-retry worst-case (~3050 ms)`,
);
});
assert.ok(
delays[2] >= 150,
`delay[2]=${delays[2]} must exceed the prior linear cap of 150 ms`,
);
assert.ok(
delays[3] >= 200,
`delay[3]=${delays[3]} must exceed the prior linear cap of 200 ms`,
);
assert.ok(
delays[5] >= 800,
`delay[5]=${delays[5]} must reach the new 6-retry exponential ceiling (>=800 ms)`,
);
const total = delays.reduce((a, b) => a + b, 0);
assert.ok(
total > 1500,
`total elapsed sleep ${total} must exceed prior 4-retry worst-case (~750 ms)`,
);
assert.ok(
total < 3200,
`total elapsed sleep ${total} must stay within the new 6-retry worst-case (~3050 ms)`,
);
});

it('succeeds on the 5th attempt after four 412s', async () => {
let getCallCount = 0;
let putCallCount = 0;
it('succeeds on the 5th attempt after four 412s', async () => {
globalThis.setTimeout = (fn) => {
fn();
return 0;
};

const makeBody = () => new ReadableStream({
start(controller) {
controller.enqueue(new TextEncoder().encode(''));
controller.close();
},
});
let getCallCount = 0;
let putCallCount = 0;

const { writeAuditEntry } = await esmock(
'../../../src/storage/version/audit.js',
{
'@aws-sdk/client-s3': {
S3Client: function S3Client() {
this.send = async (cmd) => {
if (cmd instanceof GetObjectCommand) {
getCallCount += 1;
return { Body: makeBody(), ETag: `"etag-${getCallCount}"` };
}
if (cmd instanceof PutObjectCommand) {
putCallCount += 1;
if (putCallCount < 5) {
const err = new Error('precondition failed');
err.$metadata = { httpStatusCode: 412 };
throw err;
const makeBody = () => new ReadableStream({
start(controller) {
controller.enqueue(new TextEncoder().encode(''));
controller.close();
},
});

const { writeAuditEntry } = await esmock(
'../../../src/storage/version/audit.js',
{
'@aws-sdk/client-s3': {
S3Client: function S3Client() {
this.send = async (cmd) => {
if (cmd instanceof GetObjectCommand) {
getCallCount += 1;
return { Body: makeBody(), ETag: `"etag-${getCallCount}"` };
}
if (cmd instanceof PutObjectCommand) {
putCallCount += 1;
if (putCallCount < 5) {
const err = new Error('precondition failed');
err.$metadata = { httpStatusCode: 412 };
throw err;
}
return { $metadata: { httpStatusCode: 200 } };
}
return { $metadata: { httpStatusCode: 200 } };
}
return { $metadata: { httpStatusCode: 200 } };
};
};
},
GetObjectCommand,
PutObjectCommand,
},
GetObjectCommand,
PutObjectCommand,
'../../../src/storage/utils/config.js': { default: () => ({}) },
},
'../../../src/storage/utils/config.js': { default: () => ({}) },
},
);
);

const result = await writeAuditEntry({}, { bucket: 'b', org: 'o' }, 'repo', 'fid', {
timestamp: '5000',
users: '[{"email":"u@x.com"}]',
path: 'repo/doc.html',
});
const result = await writeAuditEntry({}, { bucket: 'b', org: 'o' }, 'repo', 'fid', {
timestamp: '5000',
users: '[{"email":"u@x.com"}]',
path: 'repo/doc.html',
});

assert.strictEqual(result.status, 200, 'must succeed on the 5th attempt');
assert.strictEqual(putCallCount, 5, 'must have attempted PUT 5 times');
assert.strictEqual(getCallCount, 5, 'must re-read on each attempt');
assert.strictEqual(result.status, 200, 'must succeed on the 5th attempt');
assert.strictEqual(putCallCount, 5, 'must have attempted PUT 5 times');
assert.strictEqual(getCallCount, 5, 'must re-read on each attempt');
});
});
});
});
Loading