From 01bc2eed6faf9629d07677eb9d7d65ca24e10389 Mon Sep 17 00:00:00 2001 From: sleep3r Date: Wed, 14 Oct 2026 12:42:27 +0300 Subject: [PATCH 1/2] test(runners): the drain test holds its run until it has read the hidden test The stand-in run ended at once, so on a fast runner the agent could settle it and remove its tree between the wait for the run and the read of its /data: ENOENT on test.csv in the release PR's CI. The run now waits for the read, as the sync test's run does. --- tests/runner-agent-lifecycle.test.mts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/runner-agent-lifecycle.test.mts b/tests/runner-agent-lifecycle.test.mts index 548caa5c..5417c1fd 100644 --- a/tests/runner-agent-lifecycle.test.mts +++ b/tests/runner-agent-lifecycle.test.mts @@ -618,6 +618,8 @@ test('a drain lets a held hidden-test run finish: what it reads stays until it h return { json: syncReply({ seq: lease.seq, leases, trusted: true, want: { revisions: [{ id: REVISION.id, ref: REVISION.ref, layers: LAYERS, gpu: false }], data, bundles: [] } }), delayMs: leases.length > 0 ? 0 : 20 } } const { agent, seam, docker } = agentFor(home) + // The run waits for the read of its /data below: one that ended at once could have its tree removed under the read. + seam.next = 'wait' // The lease's image is checked once more right before its run (images.ts · verify), before it links the hidden test: held there, where a drain can come. let verifying = 0 let release!: () => void @@ -636,12 +638,14 @@ test('a drain lets a held hidden-test run finish: what it reads stays until it h await waitFor(() => seam.runs.length === 1, 'the hidden-test run, after the drain') assert.equal(seam.runs[0]!.sealedInputs, true) assert.equal(fs.readFileSync(path.join(seam.runs[0]!.dataDir, 'test.csv'), 'utf8'), 'id\n9\n') + seam.release() await waitFor(() => !fs.existsSync(path.join(layout.leasesDir, `${lease.attemptId}.json`)), 'the run settled') assert.ok(!home.calls.some((call) => call.path.endsWith(`/attempts/${lease.attemptId}/outcome`) && (call.body as { kind: string }).kind === 'declined'), 'the drain declined a run it was to finish') // Settled, it reads nothing more: a draining machine keeps none of the hidden test. await waitFor(() => fs.readdirSync(path.join(competitionsDir, '.sealed')).length === 0, 'the hidden test gone once the run is done') } finally { release() + seam.release() await agent.stop() await running await home.close() From f1da91bf99ee7385a6b4bc95a8a34b302a9d6579 Mon Sep 17 00:00:00 2001 From: sleep3r Date: Wed, 14 Oct 2026 12:53:58 +0300 Subject: [PATCH 2/2] test(runners): the mid-body refusal is counted in chunks, not milliseconds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Answered before the sixty-four megabytes were sent" was a 1.5 s bound, and six suites at once on one VM stretched it to 1.6–1.7 s in three runs of six. What the test means is that home does not drain the body: it now counts the chunks sent before the answer (24 of 1024 here; a socket's buffers hold a few megabytes, a drained body all of them) and keeps a 10 s bound against an answer that waits for a timeout. --- tests/remote-runners-doors.test.mts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/remote-runners-doors.test.mts b/tests/remote-runners-doors.test.mts index f9a6d057..23cd1a1f 100644 --- a/tests/remote-runners-doors.test.mts +++ b/tests/remote-runners-doors.test.mts @@ -483,12 +483,16 @@ test('an upload refused in the middle of its body is answered, marked: never a r send(c) const { m, lease } = await runningLease(c) /** A PUT whose body is megabytes of `chunk()`, sent as it is read: still on its way when home finds what is wrong with it. */ + let pulled = 0 const streamed = (headers: Record, chunk: () => Uint8Array, chunks: number) => { let sent = 0 const body = new ReadableStream({ pull(controller) { if (sent++ >= chunks) controller.close() - else controller.enqueue(chunk()) + else { + pulled++ + controller.enqueue(chunk()) + } }, }) return fetch(`${m.home}/api/runners/v1/attempts/${lease.attemptId}/files/submission.csv`, { @@ -526,9 +530,13 @@ test('an upload refused in the middle of its body is answered, marked: never a r const { cancelRemote } = await import('../server/src/competitions/remote/release.js') assert.equal(cancelRemote(queueRow(lease.submissionId)!, 'teacher'), true) const started = Date.now() + pulled = 0 const late = await streamed({ 'content-encoding': 'gzip', 'x-colloq-sha256': 'c'.repeat(64), 'x-colloq-bytes': '100' }, noise, 1024) assert.equal(late.status, 409) - assert.ok(Date.now() - started < 1_500, `answered before the sixty-four megabytes were sent (${Date.now() - started} ms)`) + // Counted in chunks, not milliseconds, which a loaded runner stretches: a body home does not read stops at what the + // sockets' buffers hold, a few megabytes, while one it drained would have been sent whole before the answer. + assert.ok(pulled < 512, `answered before the sixty-four megabytes were sent (${pulled} of 1024 chunks, ${Date.now() - started} ms)`) + assert.ok(Date.now() - started < 10_000, `answered at once, not after a timeout (${Date.now() - started} ms)`) assert.equal(parseRemoteErrorReply(await late.json()).error, 'revoked') })