From 4816a7fb251bdfddfd461863e363d62a08766452 Mon Sep 17 00:00:00 2001 From: Tom Taylor Date: Sat, 2 May 2026 01:05:57 +0100 Subject: [PATCH] filter: fix silent truncation at 131072 bytes for inputs >ZSTD_CStreamInSize MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #25. When `ZSTD_compressStream` returns rc>0 (a hint that ~rc bytes are still pending in libzstd's internal buffer, the natural value being `ZSTD_CStreamInSize() == 131072`), the body filter flips action to FLUSH and re-enters. After `ZSTD_flushStream` returns rc=0 (drained), the existing block if (rc == 0 && (ctx->flush || ctx->last)) { b->last_buf = ctx->last; ctx->done = ctx->last; } unconditionally marks the chain buffer as last and the context as done — even though `ctx->buffer_in.pos < ctx->buffer_in.size` and more chain links may still be queued. The next filter sees `last_buf=1`, stops reading, and `ZSTD_endStream` is never invoked on the remaining input. Symptom: any single-buf body filter call carrying a chunk larger than `ZSTD_CStreamInSize` and `last_buf=1` silently terminates the zstd frame at exactly 131072 decompressed bytes. Multi-buf streams (where the first chunk has `last_buf=0`) dodge the bug because the FLUSH-rc=0 branch is gated on `ctx->last`, so the action is restored to COMPRESS and the next chain link is consumed. Reproduced live with nginx 1.30.0 + libzstd 1.4.4 against a 141186-byte CSS file via `Accept-Encoding: zstd`: wire 84969 bytes, decompressed to exactly 131072 bytes. After patch: wire 92265 bytes, decompressed to the full 141186. Verified across input sizes 5 KB / 141 KB / 208 KB / 290 KB / 1.5 MB; all decompress to the full original size. Two-part fix: 1. Gate the END transition on the current `ZSTD_inBuffer` being fully drained AND no more chain links queued, so we don't finalize the frame while libzstd still has unconsumed input: } else if (ctx->last && ctx->action != END && ctx->buffer_in.pos >= ctx->buffer_in.size && ctx->in == NULL) { ctx->action = END; return NGX_AGAIN; } else if (ctx->action != END) { ctx->action = COMPRESS; /* restore */ } The new `else if (... != END)` clause is required because the original unconditional `action = COMPRESS` restore would reset the END action immediately after `ZSTD_endStream` returned rc=0, defeating the done-flag gate below and causing infinite output (observed: 422 MB output for 141 KB input during patch development). 2. Gate the EOF marker on `action == END` so `b->last_buf=1` and `ctx->done=1` only fire after `ZSTD_endStream` has emitted the frame epilogue: if (rc == 0 && (ctx->flush || (ctx->last && ctx->action == END))) { ... } The mid-stream flush case (`ctx->flush`) is unaffected — that is the client-flush path, not the EOF path. --- filter/ngx_http_zstd_filter_module.c | 40 +++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/filter/ngx_http_zstd_filter_module.c b/filter/ngx_http_zstd_filter_module.c index 50ec55f..ea83cb4 100644 --- a/filter/ngx_http_zstd_filter_module.c +++ b/filter/ngx_http_zstd_filter_module.c @@ -447,7 +447,21 @@ ngx_http_zstd_filter_compress(ngx_http_request_t *r, ngx_http_zstd_ctx_t *ctx) ctx->redo = 1; - } else if (ctx->last && ctx->action != NGX_HTTP_ZSTD_FILTER_END) { + } else if (ctx->last && ctx->action != NGX_HTTP_ZSTD_FILTER_END + && ctx->buffer_in.pos >= ctx->buffer_in.size + && ctx->in == NULL) + { + /* + * 2026-05-01 (tommytaylor.co.uk): only transition to END once the + * current ZSTD_inBuffer is fully drained AND no more chain links + * are queued. Without these gates, libzstd's "131072 bytes still + * pending" hint return on the same body-filter call that carried + * last_buf=1 jumps straight to END. ZSTD_endStream emits a clean + * frame epilogue at the partial-input boundary, and any input + * beyond 131072 is silently truncated. Surfaces on single-buf + * static-file responses sized 131073..output_buffers_size. + * Upstream issue tokers/zstd-nginx-module#25. + */ ctx->redo = 1; ctx->action = NGX_HTTP_ZSTD_FILTER_END; @@ -455,7 +469,11 @@ ngx_http_zstd_filter_compress(ngx_http_request_t *r, ngx_http_zstd_ctx_t *ctx) return NGX_AGAIN; - } else { + } else if (ctx->action != NGX_HTTP_ZSTD_FILTER_END) { + /* preserve END after ZSTD_endStream completes (rc==0) so the + * done-flag gate below sees action==END and marks last_buf=1. + * Otherwise an unconditional restore-to-COMPRESS would loop + * forever re-running endStream without ever setting done. */ ctx->action = NGX_HTTP_ZSTD_FILTER_COMPRESS; /* restore */ } @@ -470,7 +488,23 @@ ngx_http_zstd_filter_compress(ngx_http_request_t *r, ngx_http_zstd_ctx_t *ctx) b = ctx->out_buf; - if (rc == 0 && (ctx->flush || ctx->last)) { + /* + * 2026-05-01 (tommytaylor.co.uk truncation fix): only mark the response + * "done" once ZSTD_endStream has actually run (action == END). The + * original code set b->last_buf=1 + done=1 after a FLUSH-rc=0 cycle + * with last==1, even when buffer_in still had unconsumed bytes. + * Mechanism: input streams >131072 bytes (libzstd's ZSTD_CStreamInSize + * default) trigger ZSTD_compressStream to return rc=131072 (still- + * pending hint) → state machine flips action to FLUSH → FLUSH returns + * rc=0 → premature last_buf flag → next filter stops reading → + * 131072-byte silent truncation. Gate the EOF marker on action==END so + * it only fires after ZSTD_endStream has emitted the frame epilogue. + * Flush case (mid-stream client flush) preserves original behaviour. + */ + if (rc == 0 + && (ctx->flush + || (ctx->last && ctx->action == NGX_HTTP_ZSTD_FILTER_END))) + { r->connection->buffered &= ~NGX_HTTP_GZIP_BUFFERED; b->flush = ctx->flush;