From eef192c883ec8e910b7a9ba72322404d75969941 Mon Sep 17 00:00:00 2001 From: dkijania Date: Sun, 28 Jun 2026 15:23:41 +0200 Subject: [PATCH 1/4] =?UTF-8?q?feat(server):=20graceful=20shutdown=20?= =?UTF-8?q?=E2=80=94=20drain,=20flush=20traces,=20uncaught=20handlers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shutdown previously called `server.close()` then `process.exit(0)` from the close event. It didn't bound how long draining could take, never flushed OpenTelemetry spans (losing the tail of traces on deploy), and had no handlers for uncaughtException / unhandledRejection. Add a small, unit-tested `createGracefulShutdown` orchestrator and wire it into the entry point: - Drain in-flight requests via `server.close()`, then run teardown steps (flush the tracer provider, close the Postgres pool) in order. - A hard `SHUTDOWN_TIMEOUT_MS` deadline (default 10s) forces exit if draining or teardown hangs; the process exits at most once. - The handler is idempotent, so a second signal is ignored. - SIGINT/SIGTERM/SIGQUIT plus uncaughtException/unhandledRejection all route through it. `buildPlugins` now returns the tracer provider so the entry point can flush it. Unit tests cover ordering, idempotency, a failing teardown step, and the timeout-forces-exit path with an injected exit hook. Closes #170. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01QSuak9smCHbp4N17xjjLF6 --- docs/getting-started.md | 1 + src/envionment.d.ts | 1 + src/index.ts | 33 +++++++++-- src/server/graceful-shutdown.ts | 72 +++++++++++++++++++++++ src/server/plugins.ts | 7 ++- tests/unit/graceful-shutdown.test.ts | 87 ++++++++++++++++++++++++++++ 6 files changed, 194 insertions(+), 7 deletions(-) create mode 100644 src/server/graceful-shutdown.ts create mode 100644 tests/unit/graceful-shutdown.test.ts diff --git a/docs/getting-started.md b/docs/getting-started.md index f1841680..139281e7 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -176,6 +176,7 @@ The server reads config from environment variables. `PG_CONN` is the only requir | --- | --- | --- | | `PG_CONN` | *(required)* | Postgres connection string for the archive-node DB | | `PORT` | `8080` | Port the GraphQL server listens on | +| `SHUTDOWN_TIMEOUT_MS` | `10000` | Max ms to drain in-flight requests on SIGTERM before forcing exit | | `LOG_LEVEL` | `info` | `debug` \| `info` \| `warn` \| `error` | | `CORS_ORIGIN` | `*` | CORS allowed origin | | `READINESS_PING_TIMEOUT_MS` | `2000` | Upper bound on the `/readiness` database ping. Exceeding it returns 503 rather than leaving the probe to hang. Keep it below the orchestrator's probe `timeoutSeconds` | diff --git a/src/envionment.d.ts b/src/envionment.d.ts index 73d7a967..ea2dac06 100644 --- a/src/envionment.d.ts +++ b/src/envionment.d.ts @@ -4,6 +4,7 @@ declare global { LOG_LEVEL: string; PORT?: string; PG_CONN: string; + SHUTDOWN_TIMEOUT_MS?: string; CORS_ORIGIN?: string; READINESS_PING_TIMEOUT_MS?: string; ENABLE_LOGGING?: bool; diff --git a/src/index.ts b/src/index.ts index 96125ad4..f6a76c4b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,26 +3,49 @@ import { buildContext } from './context.js'; import { buildServer } from './server/server.js'; import { buildPlugins } from './server/plugins.js'; +import { createGracefulShutdown } from './server/graceful-shutdown.js'; const PORT = process.env.PORT || 8080; +const SHUTDOWN_TIMEOUT_MS = Number(process.env.SHUTDOWN_TIMEOUT_MS) || 10000; (async function main() { try { const context = await buildContext(process.env.PG_CONN); - const plugins = await buildPlugins(); + const { plugins, provider } = await buildPlugins(); const server = buildServer(context, plugins); server.listen(PORT, () => { console.info(`Server is running on port: ${PORT}`); }); + const shutdown = createGracefulShutdown({ + timeoutMs: SHUTDOWN_TIMEOUT_MS, + // Stop accepting connections and wait for in-flight requests to drain. + closeServer: () => + new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }), + closers: [ + // Flush any buffered OpenTelemetry spans before exit. + async () => { + if (provider) await provider.shutdown(); + }, + // Close the Postgres connection pool. + () => context.db_client.close(), + ], + }); + ['SIGINT', 'SIGTERM', 'SIGQUIT'].forEach((signal) => { - process.on(signal, () => server.close()); + process.on(signal, () => void shutdown(signal)); }); - server.on('close', async () => { - await context.db_client.close(); - process.exit(0); // normal termination + process.on('uncaughtException', (error) => { + console.error('Uncaught exception:', error); + void shutdown('uncaughtException'); + }); + process.on('unhandledRejection', (reason) => { + console.error('Unhandled rejection:', reason); + void shutdown('unhandledRejection'); }); } catch (error) { console.error('An error occurred:', error); diff --git a/src/server/graceful-shutdown.ts b/src/server/graceful-shutdown.ts new file mode 100644 index 00000000..50ef04a0 --- /dev/null +++ b/src/server/graceful-shutdown.ts @@ -0,0 +1,72 @@ +export { createGracefulShutdown }; +export type { GracefulShutdownOptions }; + +interface GracefulShutdownOptions { + /** Stop accepting connections and resolve once in-flight requests drain. */ + closeServer: () => Promise; + /** Resource teardown to run after the server closes (flush traces, close DB). */ + closers?: Array<() => Promise>; + /** Hard deadline; if draining/teardown exceeds it, force exit. */ + timeoutMs: number; + /** Process exit hook — injectable for tests. */ + onExit?: (code: number) => void; + /** Log sink — injectable for tests. */ + log?: (message: string, error?: unknown) => void; +} + +/** + * Build an idempotent shutdown handler. On the first invocation it drains the + * server, runs each closer (a failing closer is logged but doesn't abort the + * rest), then exits 0. A hard timeout guarantees the process exits even if a + * connection or teardown step hangs, and `onExit` is invoked at most once. + * + * Subsequent invocations (e.g. a second signal) are ignored. + */ +function createGracefulShutdown(options: GracefulShutdownOptions) { + const { + closeServer, + closers = [], + timeoutMs, + onExit = (code) => process.exit(code), + log = (message, error) => + error !== undefined ? console.error(message, error) : console.info(message), + } = options; + + let started = false; + + return async function shutdown(reason: string): Promise { + if (started) return; + started = true; + log(`Shutting down (${reason})…`); + + let exited = false; + const exitOnce = (code: number) => { + if (exited) return; + exited = true; + onExit(code); + }; + + const forceTimer = setTimeout(() => { + log('Graceful shutdown timed out; forcing exit.'); + exitOnce(1); + }, timeoutMs); + if (typeof forceTimer.unref === 'function') forceTimer.unref(); + + try { + await closeServer(); + for (const close of closers) { + try { + await close(); + } catch (error) { + log('Error during shutdown step', error); + } + } + exitOnce(0); + } catch (error) { + log('Error closing server', error); + exitOnce(1); + } finally { + clearTimeout(forceTimer); + } + }; +} diff --git a/src/server/plugins.ts b/src/server/plugins.ts index db28e82d..583b902f 100644 --- a/src/server/plugins.ts +++ b/src/server/plugins.ts @@ -4,6 +4,7 @@ import { useDisableIntrospection } from '@envelop/disable-introspection'; import { useOpenTelemetry } from '@envelop/opentelemetry'; import { inspect } from 'node:util'; +import type { BasicTracerProvider } from '@opentelemetry/sdk-trace-base'; import { initJaegerProvider } from '../tracing/jaeger-tracing.js'; import { useMetrics } from './metrics.js'; @@ -20,8 +21,10 @@ async function buildPlugins() { plugins.push(useGraphQlJit()); + // Returned so the entry point can flush spans on shutdown. + let provider: BasicTracerProvider | undefined; if (process.env.ENABLE_LOGGING) { - const provider = await initJaegerProvider(); + provider = await initJaegerProvider(); plugins.push( useOpenTelemetry( { @@ -62,5 +65,5 @@ async function buildPlugins() { }, }) ); - return plugins; + return { plugins, provider }; } diff --git a/tests/unit/graceful-shutdown.test.ts b/tests/unit/graceful-shutdown.test.ts new file mode 100644 index 00000000..e5f895f4 --- /dev/null +++ b/tests/unit/graceful-shutdown.test.ts @@ -0,0 +1,87 @@ +import { describe, test } from 'node:test'; +import assert from 'node:assert'; +import { createGracefulShutdown } from '../../src/server/graceful-shutdown.js'; + +const silent = () => {}; + +describe('Graceful shutdown', () => { + test('drains the server, runs closers in order, then exits 0', async () => { + const calls: string[] = []; + const exits: number[] = []; + const shutdown = createGracefulShutdown({ + closeServer: async () => { + calls.push('server'); + }, + closers: [ + async () => { + calls.push('traces'); + }, + async () => { + calls.push('db'); + }, + ], + timeoutMs: 1000, + onExit: (code) => exits.push(code), + log: silent, + }); + + await shutdown('SIGTERM'); + assert.deepStrictEqual(calls, ['server', 'traces', 'db']); + assert.deepStrictEqual(exits, [0]); + }); + + test('is idempotent — a second signal does nothing', async () => { + let serverCloses = 0; + const exits: number[] = []; + const shutdown = createGracefulShutdown({ + closeServer: async () => { + serverCloses += 1; + }, + timeoutMs: 1000, + onExit: (code) => exits.push(code), + log: silent, + }); + + await shutdown('SIGTERM'); + await shutdown('SIGINT'); + assert.strictEqual(serverCloses, 1); + assert.deepStrictEqual(exits, [0]); + }); + + test('a failing closer is logged but does not abort the rest', async () => { + const calls: string[] = []; + const exits: number[] = []; + const shutdown = createGracefulShutdown({ + closeServer: async () => {}, + closers: [ + async () => { + throw new Error('flush failed'); + }, + async () => { + calls.push('db'); + }, + ], + timeoutMs: 1000, + onExit: (code) => exits.push(code), + log: silent, + }); + + await shutdown('SIGTERM'); + assert.deepStrictEqual(calls, ['db']); + assert.deepStrictEqual(exits, [0]); + }); + + test('exits 1 when draining exceeds the timeout, and only once', async () => { + const exits: number[] = []; + const shutdown = createGracefulShutdown({ + closeServer: () => new Promise(() => {}), // never resolves + timeoutMs: 20, + onExit: (code) => exits.push(code), + log: silent, + }); + + shutdown('SIGTERM'); + await new Promise((resolve) => setTimeout(resolve, 60)); + assert.deepStrictEqual(exits, [1]); + }); +}); From 87be16abccc4e056f974c173e5ca0b59c2e8c6da Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 17 Jul 2026 00:22:33 +0200 Subject: [PATCH 2/4] fix(shutdown): exit non-zero on crash-initiated shutdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit uncaughtException/unhandledRejection routed through shutdown(), whose happy path always exited 0. A crash that drained cleanly therefore exited 0, which Kubernetes and systemd read as a clean stop — suppressing OnFailure restarts and non-zero-exit alerting, and reducing crash signal below Node's own default of 1. shutdown() now takes an exitCode used on the success path; signals keep 0 and the crash handlers pass 1. Also drop idle keep-alive sockets when draining: server.close() waits on them, so browser clients holding connections open could push the drain past SHUTDOWN_TIMEOUT_MS and skip the trace flush and pool close entirely. Addresses review feedback on #188. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/index.ts | 10 ++++++++-- src/server/graceful-shutdown.ts | 13 +++++++++---- tests/unit/graceful-shutdown.test.ts | 23 +++++++++++++++++++++++ 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/index.ts b/src/index.ts index f6a76c4b..43f4debe 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,6 +24,10 @@ const SHUTDOWN_TIMEOUT_MS = Number(process.env.SHUTDOWN_TIMEOUT_MS) || 10000; closeServer: () => new Promise((resolve, reject) => { server.close((error) => (error ? reject(error) : resolve())); + // `close` also waits on idle keep-alive sockets, which browser + // clients hold open for keepAliveTimeout; dropping them keeps the + // drain prompt so the closers below still run inside the timeout. + server.closeIdleConnections(); }), closers: [ // Flush any buffered OpenTelemetry spans before exit. @@ -39,13 +43,15 @@ const SHUTDOWN_TIMEOUT_MS = Number(process.env.SHUTDOWN_TIMEOUT_MS) || 10000; process.on(signal, () => void shutdown(signal)); }); + // Crashes exit non-zero: an exit 0 reads as a clean stop to Kubernetes and + // systemd, suppressing restarts and non-zero-exit alerting. process.on('uncaughtException', (error) => { console.error('Uncaught exception:', error); - void shutdown('uncaughtException'); + void shutdown('uncaughtException', 1); }); process.on('unhandledRejection', (reason) => { console.error('Unhandled rejection:', reason); - void shutdown('unhandledRejection'); + void shutdown('unhandledRejection', 1); }); } catch (error) { console.error('An error occurred:', error); diff --git a/src/server/graceful-shutdown.ts b/src/server/graceful-shutdown.ts index 50ef04a0..be53646b 100644 --- a/src/server/graceful-shutdown.ts +++ b/src/server/graceful-shutdown.ts @@ -17,8 +17,9 @@ interface GracefulShutdownOptions { /** * Build an idempotent shutdown handler. On the first invocation it drains the * server, runs each closer (a failing closer is logged but doesn't abort the - * rest), then exits 0. A hard timeout guarantees the process exits even if a - * connection or teardown step hangs, and `onExit` is invoked at most once. + * rest), then exits with the caller's code. A hard timeout guarantees the + * process exits even if a connection or teardown step hangs, and `onExit` is + * invoked at most once. * * Subsequent invocations (e.g. a second signal) are ignored. */ @@ -34,7 +35,11 @@ function createGracefulShutdown(options: GracefulShutdownOptions) { let started = false; - return async function shutdown(reason: string): Promise { + /** + * `exitCode` is the code used when the drain succeeds; a crash-initiated + * shutdown must pass non-zero so supervisors still see a failed exit. + */ + return async function shutdown(reason: string, exitCode = 0): Promise { if (started) return; started = true; log(`Shutting down (${reason})…`); @@ -61,7 +66,7 @@ function createGracefulShutdown(options: GracefulShutdownOptions) { log('Error during shutdown step', error); } } - exitOnce(0); + exitOnce(exitCode); } catch (error) { log('Error closing server', error); exitOnce(1); diff --git a/tests/unit/graceful-shutdown.test.ts b/tests/unit/graceful-shutdown.test.ts index e5f895f4..75d35758 100644 --- a/tests/unit/graceful-shutdown.test.ts +++ b/tests/unit/graceful-shutdown.test.ts @@ -71,6 +71,29 @@ describe('Graceful shutdown', () => { assert.deepStrictEqual(exits, [0]); }); + test('a crash-initiated shutdown drains cleanly but still exits non-zero', async () => { + const calls: string[] = []; + const exits: number[] = []; + const shutdown = createGracefulShutdown({ + closeServer: async () => { + calls.push('server'); + }, + closers: [ + async () => { + calls.push('traces'); + }, + ], + timeoutMs: 1000, + onExit: (code) => exits.push(code), + log: silent, + }); + + await shutdown('uncaughtException', 1); + // The drain still runs in full — only the exit code differs from a signal. + assert.deepStrictEqual(calls, ['server', 'traces']); + assert.deepStrictEqual(exits, [1]); + }); + test('exits 1 when draining exceeds the timeout, and only once', async () => { const exits: number[] = []; const shutdown = createGracefulShutdown({ From 216efd653a4d74ce03303460f54c61a5e282d9fa Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 19 Aug 2026 13:10:38 +0200 Subject: [PATCH 3/4] fix(server): bound shutdown teardown steps --- .env.example.compose | 1 + .env.example.lightnet | 1 + docs/getting-started.md | 2 +- src/index.ts | 39 +++++++++++++++++++++++++++------ src/server/graceful-shutdown.ts | 1 - 5 files changed, 35 insertions(+), 9 deletions(-) diff --git a/.env.example.compose b/.env.example.compose index 89299ebc..6eb2734b 100644 --- a/.env.example.compose +++ b/.env.example.compose @@ -27,6 +27,7 @@ JAEGER=jaegertracing/all-in-one:latest # Fields for App (Required) PORT=8080 +SHUTDOWN_TIMEOUT_MS=20000 LOG_LEVEL="info" CORS_ORIGIN="*" READINESS_PING_TIMEOUT_MS=2000 diff --git a/.env.example.lightnet b/.env.example.lightnet index 60d2751c..7b3665d0 100644 --- a/.env.example.lightnet +++ b/.env.example.lightnet @@ -1,4 +1,5 @@ PORT=8080 +SHUTDOWN_TIMEOUT_MS=20000 LOG_LEVEL="info" CORS_ORIGIN="*" READINESS_PING_TIMEOUT_MS=2000 diff --git a/docs/getting-started.md b/docs/getting-started.md index 139281e7..3ab6544f 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -176,7 +176,7 @@ The server reads config from environment variables. `PG_CONN` is the only requir | --- | --- | --- | | `PG_CONN` | *(required)* | Postgres connection string for the archive-node DB | | `PORT` | `8080` | Port the GraphQL server listens on | -| `SHUTDOWN_TIMEOUT_MS` | `10000` | Max ms to drain in-flight requests on SIGTERM before forcing exit | +| `SHUTDOWN_TIMEOUT_MS` | `20000` | Max ms to drain in-flight requests on SIGTERM before forcing exit. Keep Kubernetes `terminationGracePeriodSeconds` above this value | | `LOG_LEVEL` | `info` | `debug` \| `info` \| `warn` \| `error` | | `CORS_ORIGIN` | `*` | CORS allowed origin | | `READINESS_PING_TIMEOUT_MS` | `2000` | Upper bound on the `/readiness` database ping. Exceeding it returns 503 rather than leaving the probe to hang. Keep it below the orchestrator's probe `timeoutSeconds` | diff --git a/src/index.ts b/src/index.ts index 43f4debe..5978deaa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,7 +6,28 @@ import { buildPlugins } from './server/plugins.js'; import { createGracefulShutdown } from './server/graceful-shutdown.js'; const PORT = process.env.PORT || 8080; -const SHUTDOWN_TIMEOUT_MS = Number(process.env.SHUTDOWN_TIMEOUT_MS) || 10000; +const SHUTDOWN_TIMEOUT_MS = Number(process.env.SHUTDOWN_TIMEOUT_MS) || 20000; + +function withTimeout( + label: string, + ms: number, + run: () => Promise +): Promise { + let timer: ReturnType | undefined; + return Promise.race([ + run(), + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`${label} timed out after ${ms}ms`)), + ms + ); + }), + ]) + .then(() => undefined) + .finally(() => { + if (timer !== undefined) clearTimeout(timer); + }); +} (async function main() { try { @@ -30,12 +51,16 @@ const SHUTDOWN_TIMEOUT_MS = Number(process.env.SHUTDOWN_TIMEOUT_MS) || 10000; server.closeIdleConnections(); }), closers: [ - // Flush any buffered OpenTelemetry spans before exit. - async () => { - if (provider) await provider.shutdown(); - }, - // Close the Postgres connection pool. - () => context.db_client.close(), + // Flush buffered spans without letting an unreachable collector consume + // the whole shutdown budget. + () => + withTimeout('trace flush', 3000, async () => { + if (provider) await provider.shutdown(); + }), + // Close the Postgres pool. postgres.js waits forever by default for + // open connections, so bound the wait and let the force timer handle any + // still-running queries. + () => withTimeout('pg pool close', 5000, () => context.db_client.close()), ], }); diff --git a/src/server/graceful-shutdown.ts b/src/server/graceful-shutdown.ts index be53646b..91bcc2e2 100644 --- a/src/server/graceful-shutdown.ts +++ b/src/server/graceful-shutdown.ts @@ -55,7 +55,6 @@ function createGracefulShutdown(options: GracefulShutdownOptions) { log('Graceful shutdown timed out; forcing exit.'); exitOnce(1); }, timeoutMs); - if (typeof forceTimer.unref === 'function') forceTimer.unref(); try { await closeServer(); From ee5887c83edd8b7f45724c3a4d3a0e39e483d41e Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 25 Aug 2026 00:07:30 +0200 Subject: [PATCH 4/4] test(metrics): use plugins from buildPlugins result --- tests/unit/metrics.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/metrics.test.ts b/tests/unit/metrics.test.ts index c2112899..73a612d9 100644 --- a/tests/unit/metrics.test.ts +++ b/tests/unit/metrics.test.ts @@ -23,10 +23,11 @@ async function graphql(yoga: ReturnType) { } async function serverWithBuiltPlugins() { + const built = await buildPlugins(); return createYoga({ schema, graphqlEndpoint: '/', - plugins: await buildPlugins(), + plugins: built.plugins, }); }