From b943109bf164e978d91a029ce145e4bb232d82ab Mon Sep 17 00:00:00 2001 From: Oliver Lambson Date: Mon, 2 Feb 2026 09:22:40 -0500 Subject: [PATCH] Add precompressed static file serving Adds support for serving pre-compressed static files with .br, .gz, or .zst extensions. When enabled via --static-path-precompressed, the server will: - Check Accept-Encoding header for supported encodings (br, gzip, zstd) - Serve compressed sidecar files if they exist (e.g., file.js.br for file.js) - Respect client q-value priorities for encoding selection - Fall back to uncompressed file if no sidecar exists - Use lazy caching to avoid repeated filesystem checks for sidecars Works with the existing multi-mount and dir-to-file rewrite features. --- granian/_granian.pyi | 7 +- granian/cli.py | 7 + granian/files.py | 18 +++ granian/server/common.py | 23 ++-- granian/server/embed.py | 17 ++- granian/server/mp.py | 19 +-- granian/server/mt.py | 19 +-- src/asgi/serve.rs | 6 +- src/conversion.rs | 19 ++- src/files.rs | 207 +++++++++++++++++++++++++--- src/rsgi/serve.rs | 6 +- src/workers.rs | 50 +++---- src/wsgi/serve.rs | 6 +- tests/conftest.py | 8 ++ tests/fixtures/static/media.png.br | Bin 0 -> 100 bytes tests/fixtures/static/media.png.gz | Bin 0 -> 112 bytes tests/fixtures/static/media.png.zst | Bin 0 -> 108 bytes tests/test_static_files.py | 180 ++++++++++++++++++++++++ 18 files changed, 496 insertions(+), 96 deletions(-) create mode 100644 granian/files.py create mode 100644 tests/fixtures/static/media.png.br create mode 100644 tests/fixtures/static/media.png.gz create mode 100644 tests/fixtures/static/media.png.zst diff --git a/granian/_granian.pyi b/granian/_granian.pyi index d156bc76..641db6c6 100644 --- a/granian/_granian.pyi +++ b/granian/_granian.pyi @@ -3,6 +3,7 @@ import threading from typing import Any from ._types import WebsocketMessage +from .files import StaticFilesSettings from .http import HTTP1Settings, HTTP2Settings __version__: str @@ -71,7 +72,7 @@ class ASGIWorker: http1_opts: HTTP1Settings | None, http2_opts: HTTP2Settings | None, websockets_enabled: bool, - static_files: tuple[str, str, str | None, str | None] | None, + static_files: StaticFilesSettings | None, ssl_enabled: bool, ssl_cert: str | None, ssl_key: str | None, @@ -95,7 +96,7 @@ class WSGIWorker: http_mode: str, http1_opts: HTTP1Settings | None, http2_opts: HTTP2Settings | None, - static_files: tuple[str, str, str | None, str | None] | None, + static_files: StaticFilesSettings | None, ssl_enabled: bool, ssl_cert: str | None, ssl_key: str | None, @@ -120,7 +121,7 @@ class RSGIWorker: http1_opts: HTTP1Settings | None, http2_opts: HTTP2Settings | None, websockets_enabled: bool, - static_files: tuple[str, str, str | None, str | None] | None, + static_files: StaticFilesSettings | None, ssl_enabled: bool, ssl_cert: str | None, ssl_key: str | None, diff --git a/granian/cli.py b/granian/cli.py index 06b672f3..7bdb3bcf 100644 --- a/granian/cli.py +++ b/granian/cli.py @@ -367,6 +367,11 @@ def option(*param_decls: str, cls: type[click.Option] | None = None, **attrs: An default=86400, help='Cache headers expiration (in seconds or a human-readable duration) for static file serving. 0 to disable.', ) +@option( + '--static-path-precompressed/--no-static-path-precompressed', + default=False, + help='Enable serving precompressed static files with .br, .gz, or .zst extensions', +) @option('--metrics/--no-metrics', 'metrics_enabled', default=False, help='Enable the prometheus metrics exporter.') @option( '--metrics-scrape-interval', default=15, type=Duration(1, 60), help='Configure the interval for metrics collection.' @@ -491,6 +496,7 @@ def cli( static_path_mount: list[pathlib.Path], static_path_dir_to_file: str | None, static_path_expires: int, + static_path_precompressed: bool, metrics_enabled: bool, metrics_scrape_interval: int, metrics_address: str, @@ -581,6 +587,7 @@ def cli( static_path_mount=static_path_mount, static_path_dir_to_file=static_path_dir_to_file, static_path_expires=static_path_expires, + static_path_precompressed=static_path_precompressed, metrics_enabled=metrics_enabled, metrics_scrape_interval=metrics_scrape_interval, metrics_address=metrics_address, diff --git a/granian/files.py b/granian/files.py new file mode 100644 index 00000000..10aa7156 --- /dev/null +++ b/granian/files.py @@ -0,0 +1,18 @@ +from dataclasses import dataclass + + +@dataclass +class StaticFilesSettings: + """Configuration for static file serving. + + Attributes: + mounts: List of (route_prefix, filesystem_path) tuples for serving static files. + dir_to_file: Optional filename to serve when a directory is requested (e.g., 'index.html'). + expires: Cache-Control max-age value in seconds (as string), or None to disable. + precompressed: Whether to serve pre-compressed sidecar files (.br, .gz, .zst). + """ + + mounts: list[tuple[str, str]] + dir_to_file: str | None = None + expires: str | None = None + precompressed: bool = False diff --git a/granian/server/common.py b/granian/server/common.py index 1369a22e..e66fba6f 100644 --- a/granian/server/common.py +++ b/granian/server/common.py @@ -19,6 +19,7 @@ from .._signals import set_main_signals from ..constants import HTTPModes, Interfaces, Loops, RuntimeModes, SSLProtocols, TaskImpl from ..errors import ConfigurationError, PidFileError +from ..files import StaticFilesSettings from ..http import HTTP1Settings, HTTP2Settings from ..log import DEFAULT_ACCESSLOG_FMT, LogLevels, configure_logging, logger from ..net import SocketSpec, UnixSocketSpec @@ -129,6 +130,7 @@ def __init__( static_path_mount: Sequence[Path] | None = None, static_path_dir_to_file: str | None = None, static_path_expires: int = 86400, + static_path_precompressed: bool = False, metrics_enabled: bool = False, metrics_scrape_interval: int = 15, metrics_address: str = '127.0.0.1', @@ -186,7 +188,8 @@ def __init__( self.factory = factory self.working_dir = working_dir self.env_files = env_files or () - self.static_path = None + self.static_files = None + self.static_path_precompressed = static_path_precompressed self.metrics_enabled = metrics_enabled self.metrics_scrape_interval = metrics_scrape_interval self.metrics_address = metrics_address @@ -243,19 +246,21 @@ def _init_static_mounts( if not paths: return if len(paths) == 1 and not routes: - self.static_path = ( - [('/static', str(paths[0].resolve()))], - dir_to_file, - expires, + self.static_files = StaticFilesSettings( + mounts=[('/static', str(paths[0].resolve()))], + dir_to_file=dir_to_file, + expires=expires, + precompressed=self.static_path_precompressed, ) return if len(paths) != len(routes): logger.error('Static path routes and mounts should have the same length') raise ConfigurationError('static_path') - self.static_path = ( - [(routes[idx], str(path.resolve())) for idx, path in enumerate(paths)], - dir_to_file, - expires, + self.static_files = StaticFilesSettings( + mounts=[(routes[idx], str(path.resolve())) for idx, path in enumerate(paths)], + dir_to_file=dir_to_file, + expires=expires, + precompressed=self.static_path_precompressed, ) def build_ssl_context( diff --git a/granian/server/embed.py b/granian/server/embed.py index 1028a6a1..a31786ef 100644 --- a/granian/server/embed.py +++ b/granian/server/embed.py @@ -14,6 +14,7 @@ from .._types import SSLCtx from ..asgi import LifespanProtocol, _callback_wrapper as _asgi_call_wrap from ..errors import ConfigurationError, FatalError +from ..files import StaticFilesSettings from ..rsgi import _callback_wrapper as _rsgi_call_wrap, _callbacks_from_target as _rsgi_cbs_from_target from .common import ( _PY_312, @@ -128,6 +129,7 @@ def __init__( static_path_mount: Sequence[Path] | None = None, static_path_dir_to_file: str | None = None, static_path_expires: int = 86400, + static_path_precompressed: bool = False, ): super().__init__( target=target, @@ -164,6 +166,7 @@ def __init__( static_path_mount=static_path_mount, static_path_dir_to_file=static_path_dir_to_file, static_path_expires=static_path_expires, + static_path_precompressed=static_path_precompressed, ) self.main_loop_interrupt = asyncio.Event() @@ -189,7 +192,7 @@ def _spawn_worker(self, idx, target, callback_loader) -> AsyncWorker: self.http1_settings, self.http2_settings, self.websockets, - self.static_path, + self.static_files, self.log_access_format if self.log_access else None, self.ssl_ctx, {'url_path_prefix': self.url_path_prefix}, @@ -215,7 +218,7 @@ async def _spawn_asgi_worker( http1_settings: HTTP1Settings | None, http2_settings: HTTP2Settings | None, websockets: bool, - static_path: tuple[str, str, str | None, str | None] | None, + static_files: StaticFilesSettings | None, log_access_fmt: str | None, ssl_ctx: SSLCtx, scope_opts: dict[str, Any], @@ -235,7 +238,7 @@ async def _spawn_asgi_worker( http1_settings, http2_settings, websockets, - static_path, + static_files, *ssl_ctx, (None, None), ) @@ -261,7 +264,7 @@ async def _spawn_asgi_lifespan_worker( http1_settings: HTTP1Settings | None, http2_settings: HTTP2Settings | None, websockets: bool, - static_path: tuple[str, str, str | None, str | None] | None, + static_files: StaticFilesSettings | None, log_access_fmt: str | None, ssl_ctx: SSLCtx, scope_opts: dict[str, Any], @@ -289,7 +292,7 @@ async def _spawn_asgi_lifespan_worker( http1_settings, http2_settings, websockets, - static_path, + static_files, *ssl_ctx, (None, None), ) @@ -316,7 +319,7 @@ async def _spawn_rsgi_worker( http1_settings: HTTP1Settings | None, http2_settings: HTTP2Settings | None, websockets: bool, - static_path: tuple[str, str, str | None, str | None] | None, + static_files: StaticFilesSettings | None, log_access_fmt: str | None, ssl_ctx: SSLCtx, scope_opts: dict[str, Any], @@ -338,7 +341,7 @@ async def _spawn_rsgi_worker( http1_settings, http2_settings, websockets, - static_path, + static_files, *ssl_ctx, (None, None), ) diff --git a/granian/server/mp.py b/granian/server/mp.py index 4d59ca14..65eae665 100644 --- a/granian/server/mp.py +++ b/granian/server/mp.py @@ -20,6 +20,7 @@ from .._internal import load_env from .._types import SSLCtx from ..asgi import LifespanProtocol, _callback_wrapper as _asgi_call_wrap +from ..files import StaticFilesSettings from ..rsgi import _callback_wrapper as _rsgi_call_wrap, _callbacks_from_target as _rsgi_cbs_from_target from ..wsgi import _callback_wrapper as _wsgi_call_wrap from .common import ( @@ -129,7 +130,7 @@ def _spawn_asgi_worker( http1_settings: HTTP1Settings | None, http2_settings: HTTP2Settings | None, websockets: bool, - static_path: tuple[str, str, str | None, str | None] | None, + static_files: StaticFilesSettings | None, log_access_fmt: str | None, ssl_ctx: SSLCtx, scope_opts: dict[str, Any], @@ -153,7 +154,7 @@ def _spawn_asgi_worker( http1_settings, http2_settings, websockets, - static_path, + static_files, *ssl_ctx, metrics, ) @@ -180,7 +181,7 @@ def _spawn_asgi_lifespan_worker( http1_settings: HTTP1Settings | None, http2_settings: HTTP2Settings | None, websockets: bool, - static_path: tuple[str, str, str | None] | None, + static_files: StaticFilesSettings | None, log_access_fmt: str | None, ssl_ctx: SSLCtx, scope_opts: dict[str, Any], @@ -212,7 +213,7 @@ def _spawn_asgi_lifespan_worker( http1_settings, http2_settings, websockets, - static_path, + static_files, *ssl_ctx, metrics, ) @@ -240,7 +241,7 @@ def _spawn_rsgi_worker( http1_settings: HTTP1Settings | None, http2_settings: HTTP2Settings | None, websockets: bool, - static_path: tuple[str, str, str | None] | None, + static_files: StaticFilesSettings | None, log_access_fmt: str | None, ssl_ctx: SSLCtx, scope_opts: dict[str, Any], @@ -266,7 +267,7 @@ def _spawn_rsgi_worker( http1_settings, http2_settings, websockets, - static_path, + static_files, *ssl_ctx, metrics, ) @@ -294,7 +295,7 @@ def _spawn_wsgi_worker( http1_settings: HTTP1Settings | None, http2_settings: HTTP2Settings | None, websockets: bool, - static_path: tuple[str, str, str | None] | None, + static_files: StaticFilesSettings | None, log_access_fmt: str | None, ssl_ctx: SSLCtx, scope_opts: dict[str, Any], @@ -317,7 +318,7 @@ def _spawn_wsgi_worker( http_mode, http1_settings, http2_settings, - static_path, + static_files, *ssl_ctx, metrics, ) @@ -427,7 +428,7 @@ def _spawn_worker(self, idx, target, callback_loader) -> WorkerProcess: self.http1_settings, self.http2_settings, self.websockets, - self.static_path, + self.static_files, self.log_access_format if self.log_access else None, self.ssl_ctx, {'url_path_prefix': self.url_path_prefix}, diff --git a/granian/server/mt.py b/granian/server/mt.py index 9a22acfc..e46c4347 100644 --- a/granian/server/mt.py +++ b/granian/server/mt.py @@ -10,6 +10,7 @@ from .._types import SSLCtx from ..asgi import LifespanProtocol, _callback_wrapper as _asgi_call_wrap from ..errors import ConfigurationError, FatalError +from ..files import StaticFilesSettings from ..rsgi import _callback_wrapper as _rsgi_call_wrap, _callbacks_from_target as _rsgi_cbs_from_target from ..wsgi import _callback_wrapper as _wsgi_call_wrap from .common import ( @@ -88,7 +89,7 @@ def _spawn_asgi_worker( http1_settings: HTTP1Settings | None, http2_settings: HTTP2Settings | None, websockets: bool, - static_path: tuple[str, str, str | None] | None, + static_files: StaticFilesSettings | None, log_access_fmt: str | None, ssl_ctx: SSLCtx, scope_opts: dict[str, Any], @@ -109,7 +110,7 @@ def _spawn_asgi_worker( http1_settings, http2_settings, websockets, - static_path, + static_files, *ssl_ctx, metrics, ) @@ -136,7 +137,7 @@ def _spawn_asgi_lifespan_worker( http1_settings: HTTP1Settings | None, http2_settings: HTTP2Settings | None, websockets: bool, - static_path: tuple[str, str, str | None] | None, + static_files: StaticFilesSettings | None, log_access_fmt: str | None, ssl_ctx: SSLCtx, scope_opts: dict[str, Any], @@ -165,7 +166,7 @@ def _spawn_asgi_lifespan_worker( http1_settings, http2_settings, websockets, - static_path, + static_files, *ssl_ctx, metrics, ) @@ -193,7 +194,7 @@ def _spawn_rsgi_worker( http1_settings: HTTP1Settings | None, http2_settings: HTTP2Settings | None, websockets: bool, - static_path: tuple[str, str, str | None] | None, + static_files: StaticFilesSettings | None, log_access_fmt: str | None, ssl_ctx: SSLCtx, scope_opts: dict[str, Any], @@ -216,7 +217,7 @@ def _spawn_rsgi_worker( http1_settings, http2_settings, websockets, - static_path, + static_files, *ssl_ctx, metrics, ) @@ -244,7 +245,7 @@ def _spawn_wsgi_worker( http1_settings: HTTP1Settings | None, http2_settings: HTTP2Settings | None, websockets: bool, - static_path: tuple[str, str, str | None] | None, + static_files: StaticFilesSettings | None, log_access_fmt: str | None, ssl_ctx: SSLCtx, scope_opts: dict[str, Any], @@ -264,7 +265,7 @@ def _spawn_wsgi_worker( http_mode, http1_settings, http2_settings, - static_path, + static_files, *ssl_ctx, metrics, ) @@ -296,7 +297,7 @@ def _spawn_worker(self, idx, target, callback_loader) -> WorkerThread: self.http1_settings, self.http2_settings, self.websockets, - self.static_path, + self.static_files, self.log_access_format if self.log_access else None, self.ssl_ctx, {'url_path_prefix': self.url_path_prefix}, diff --git a/src/asgi/serve.rs b/src/asgi/serve.rs index 15a4841e..5058a1b6 100644 --- a/src/asgi/serve.rs +++ b/src/asgi/serve.rs @@ -3,7 +3,7 @@ use pyo3::prelude::*; use super::http::{handle, handle_ws}; use crate::callbacks::CallbackScheduler; -use crate::conversion::{worker_http1_config_from_py, worker_http2_config_from_py}; +use crate::conversion::{worker_http1_config_from_py, worker_http2_config_from_py, worker_static_files_config_from_py}; use crate::net::SocketHolder; use crate::serve::gen_serve_match; use crate::workers::{WorkerConfig, WorkerSignal}; @@ -56,7 +56,7 @@ impl ASGIWorker { http1_opts: Option>, http2_opts: Option>, websockets_enabled: bool, - static_files: Option<(Vec<(String, String)>, Option, Option)>, + static_files: Option>, ssl_enabled: bool, ssl_cert: Option, ssl_key: Option, @@ -81,7 +81,7 @@ impl ASGIWorker { worker_http1_config_from_py(py, http1_opts)?, worker_http2_config_from_py(py, http2_opts)?, websockets_enabled, - static_files, + worker_static_files_config_from_py(py, static_files)?, ssl_enabled, ssl_cert, ssl_key, diff --git a/src/conversion.rs b/src/conversion.rs index 3f75dc0b..94a8ffbe 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -1,6 +1,6 @@ use pyo3::{IntoPyObjectExt, prelude::*}; -use crate::workers::{HTTP1Config, HTTP2Config}; +use crate::workers::{HTTP1Config, HTTP2Config, StaticFilesConfig}; pub(crate) struct Utf8BytesToPy(pub tokio_tungstenite::tungstenite::Utf8Bytes); @@ -94,3 +94,20 @@ pub(crate) fn worker_http2_config_from_py(py: Python, cfg: Option>) -> }; Ok(ret) } + +pub(crate) fn worker_static_files_config_from_py( + py: Python, + cfg: Option>, +) -> PyResult> { + let ret = match cfg { + Some(cfg) => Some(StaticFilesConfig { + mounts: cfg.getattr(py, "mounts")?.extract(py)?, + dir_to_file: cfg.getattr(py, "dir_to_file")?.extract(py)?, + expires: cfg.getattr(py, "expires")?.extract(py)?, + precompressed: cfg.getattr(py, "precompressed")?.extract(py)?, + sidecar_cache: std::sync::Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())), + }), + None => None, + }; + Ok(ret) +} diff --git a/src/files.rs b/src/files.rs index 50fb3bcb..05dbf60f 100644 --- a/src/files.rs +++ b/src/files.rs @@ -3,32 +3,182 @@ use futures::TryStreamExt; use http_body_util::BodyExt; use hyper::{ HeaderMap, StatusCode, - header::{HeaderValue, SERVER as HK_SERVER}, + header::{ACCEPT_ENCODING, CACHE_CONTROL, CONTENT_ENCODING, CONTENT_TYPE, HeaderValue, SERVER as HK_SERVER, VARY}, }; use percent_encoding::percent_decode_str; use std::{io, path::Path}; use tokio::fs::File; use tokio_util::io::ReaderStream; -use crate::http::{HTTPResponse, HV_SERVER, response_404}; +use crate::{ + http::{HTTPRequest, HTTPResponse, HV_SERVER, response_404}, + workers::StaticFilesConfig, +}; + +#[derive(Clone, Copy)] +pub(crate) enum Encoding { + Plain, + Gzip, + Brotli, + Zstd, +} + +impl Encoding { + fn as_header_value(self) -> Option<&'static str> { + match self { + Encoding::Plain => None, + Encoding::Gzip => Some("gzip"), + Encoding::Brotli => Some("br"), + Encoding::Zstd => Some("zstd"), + } + } +} + +const SUPPORTED_ENCODINGS: &[(&str, &str, Encoding)] = &[ + ("zstd", ".zst", Encoding::Zstd), + ("br", ".br", Encoding::Brotli), + ("gzip", ".gz", Encoding::Gzip), +]; + +const SIDECAR_CACHE_MAX_SIZE: usize = 10_000; + +/// Check if a sidecar file exists, using the cache to avoid repeated filesystem checks. +/// Results are cached lazily on first access. When cache exceeds max size, ~half of +/// entries are evicted randomly to prevent full cache flush attacks. +#[inline] +fn sidecar_exists(config: &StaticFilesConfig, path: &str) -> bool { + { + let cache = config.sidecar_cache.read().unwrap(); + if let Some(&exists) = cache.get(path) { + return exists; + } + } + + let exists = Path::new(path).exists(); + { + let mut cache = config.sidecar_cache.write().unwrap(); + if cache.len() >= SIDECAR_CACHE_MAX_SIZE { + use std::hash::{Hash, Hasher}; + cache.retain(|k, _| { + let mut h = std::collections::hash_map::DefaultHasher::new(); + k.hash(&mut h); + h.finish().is_multiple_of(2) + }); + } + cache.insert(path.to_string(), exists); + } + exists +} + +/// Parse Accept-Encoding header and return encodings sorted by client priority (q-value). +/// Returns a Vec of (`encoding_name`, `q_value`) sorted by `q_value` descending. +/// Stable sort preserves client order for equal q-values. +/// Q-values are clamped to 0.0-1.0 per RFC 7231. +fn parse_accept_encoding(header_value: &str) -> Vec<(&str, f32)> { + let mut encodings: Vec<(&str, f32)> = header_value + .split(',') + .filter_map(|part| { + let part = part.trim(); + if part.is_empty() { + return None; + } + + let mut iter = part.splitn(2, ';'); + let encoding = iter.next()?.trim(); + + let q_value = iter + .next() + .and_then(|params| { + params.split(';').find_map(|p| { + let p = p.trim(); + if let Some(q_str) = p.strip_prefix("q=") { + q_str.trim().parse::().ok() + } else { + None + } + }) + }) + .unwrap_or(1.0) + .clamp(0.0, 1.0); + + Some((encoding, q_value)) + }) + .collect(); + + encodings.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + + encodings +} + +/// Resolve the static file path and encoding based on Accept-Encoding header. +/// Returns the path to serve and the encoding to use. +/// Respects client q-value priorities, with client order as tiebreaker for equal q-values. +/// Uses lazy caching to avoid repeated filesystem checks. +fn resolve_precompressed(config: &StaticFilesConfig, base_path: &str, req: &HTTPRequest) -> (String, Encoding) { + let accept_encoding = req + .headers() + .get(ACCEPT_ENCODING) + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + + if accept_encoding.is_empty() { + return (base_path.to_string(), Encoding::Plain); + } + + let client_prefs = parse_accept_encoding(accept_encoding); + + let rejected: Vec<&str> = client_prefs + .iter() + .filter(|(_, q)| *q == 0.0) + .map(|(name, _)| *name) + .collect(); + + for (encoding_name, q_value) in &client_prefs { + if *q_value == 0.0 { + continue; + } + + if *encoding_name == "*" { + for (supported_name, ext, encoding) in SUPPORTED_ENCODINGS { + if rejected.iter().any(|r| r.eq_ignore_ascii_case(supported_name)) { + continue; + } + let compressed_path = format!("{base_path}{ext}"); + if sidecar_exists(config, &compressed_path) { + return (compressed_path, *encoding); + } + } + continue; + } + + for (supported_name, ext, encoding) in SUPPORTED_ENCODINGS { + if encoding_name.eq_ignore_ascii_case(supported_name) { + let compressed_path = format!("{base_path}{ext}"); + if sidecar_exists(config, &compressed_path) { + return (compressed_path, *encoding); + } + break; + } + } + } + + (base_path.to_string(), Encoding::Plain) +} #[inline(always)] -pub(crate) fn match_static_file( - uri_path: &str, - mounts: &Vec<(String, String)>, - dir_to_file: Option<&String>, -) -> Option> { - let decoded_uri_path = percent_decode_str(uri_path).decode_utf8_lossy(); - for (prefix, mount_point) in mounts { +pub(crate) fn match_static_file(config: &StaticFilesConfig, req: &HTTPRequest) -> Option> { + let decoded_uri_path = percent_decode_str(req.uri().path()).decode_utf8_lossy(); + for (prefix, mount_point) in &config.mounts { if let Some(file_path) = decoded_uri_path.strip_prefix(prefix) { #[cfg(not(windows))] let fpath = format!("{mount_point}{file_path}"); #[cfg(windows)] let fpath = format!("{mount_point}{}", file_path.replace("/", "\\")); + // PERF: sync syscall below is in hot path match Path::new(&fpath).canonicalize() { Ok(mut full_path) => { if full_path.is_dir() { - match dir_to_file { + match &config.dir_to_file { Some(rewrite_file) => { full_path = full_path.join(rewrite_file); } @@ -36,12 +186,17 @@ pub(crate) fn match_static_file( } } #[cfg(windows)] - let full_path = &full_path.display().to_string()[4..]; - if full_path.starts_with(mount_point) { - #[cfg(not(windows))] - return full_path.to_str().map(ToOwned::to_owned).map(Ok); - #[cfg(windows)] - return Some(Ok(full_path.to_owned())); + let full_path_str = &full_path.display().to_string()[4..]; + #[cfg(not(windows))] + let full_path_str = full_path.to_str()?; + if full_path_str.starts_with(mount_point) { + let base_path = full_path_str.to_owned(); + let (path, encoding) = if config.precompressed { + resolve_precompressed(config, &base_path, req) + } else { + (base_path, Encoding::Plain) + }; + return Some(Ok((path, encoding))); } return Some(Err(anyhow::anyhow!("outside mount path"))); } @@ -55,26 +210,36 @@ pub(crate) fn match_static_file( None } -pub(crate) async fn serve_static_file(path: String, expires: Option) -> HTTPResponse { +pub(crate) async fn serve_static_file(config: &StaticFilesConfig, path: String, encoding: Encoding) -> HTTPResponse { match File::open(&path).await { Ok(file) => { - let mime = mime_guess::from_path(path).first(); + let mime_path = match encoding { + Encoding::Brotli => path.strip_suffix(".br").unwrap_or(&path), + Encoding::Gzip => path.strip_suffix(".gz").unwrap_or(&path), + Encoding::Zstd => path.strip_suffix(".zst").unwrap_or(&path), + Encoding::Plain => &path, + }; + let mime = mime_guess::from_path(mime_path).first(); let stream = ReaderStream::with_capacity(file, 131_072); let stream_body = http_body_util::StreamBody::new(stream.map_ok(hyper::body::Frame::data)); let mut headers = HeaderMap::new(); let mut res = hyper::Response::new(BodyExt::map_err(stream_body, std::convert::Into::into).boxed()); headers.insert(HK_SERVER, HV_SERVER); - if let Some(expires) = expires { + if let Some(expires) = &config.expires { headers.insert( - "cache-control", + CACHE_CONTROL, HeaderValue::from_str(&format!("max-age={expires}")).unwrap(), ); } if let Some(mime) = mime && let Ok(hv) = HeaderValue::from_str(mime.essence_str()) { - headers.insert("content-type", hv); + headers.insert(CONTENT_TYPE, hv); + } + if let Some(encoding_value) = encoding.as_header_value() { + headers.insert(CONTENT_ENCODING, HeaderValue::from_static(encoding_value)); + headers.append(VARY, HeaderValue::from_name(ACCEPT_ENCODING)); } *res.status_mut() = StatusCode::from_u16(200).unwrap(); diff --git a/src/rsgi/serve.rs b/src/rsgi/serve.rs index 98d7f37c..cb03e4ee 100644 --- a/src/rsgi/serve.rs +++ b/src/rsgi/serve.rs @@ -3,7 +3,7 @@ use pyo3::prelude::*; use super::http::{handle, handle_ws}; use crate::callbacks::CallbackScheduler; -use crate::conversion::{worker_http1_config_from_py, worker_http2_config_from_py}; +use crate::conversion::{worker_http1_config_from_py, worker_http2_config_from_py, worker_static_files_config_from_py}; use crate::net::SocketHolder; use crate::serve::gen_serve_match; use crate::workers::{WorkerConfig, WorkerSignal}; @@ -56,7 +56,7 @@ impl RSGIWorker { http1_opts: Option>, http2_opts: Option>, websockets_enabled: bool, - static_files: Option<(Vec<(String, String)>, Option, Option)>, + static_files: Option>, ssl_enabled: bool, ssl_cert: Option, ssl_key: Option, @@ -81,7 +81,7 @@ impl RSGIWorker { worker_http1_config_from_py(py, http1_opts)?, worker_http2_config_from_py(py, http2_opts)?, websockets_enabled, - static_files, + worker_static_files_config_from_py(py, static_files)?, ssl_enabled, ssl_cert, ssl_key, diff --git a/src/workers.rs b/src/workers.rs index 383c8c2a..ab080c7a 100644 --- a/src/workers.rs +++ b/src/workers.rs @@ -88,6 +88,15 @@ pub(crate) struct HTTP2Config { pub max_send_buffer_size: usize, } +#[derive(Clone)] +pub(crate) struct StaticFilesConfig { + pub mounts: Vec<(String, String)>, + pub dir_to_file: Option, + pub expires: Option, + pub precompressed: bool, + pub sidecar_cache: std::sync::Arc>>, +} + pub(crate) struct WorkerConfig { pub id: i32, sock: Py, @@ -102,7 +111,7 @@ pub(crate) struct WorkerConfig { pub http1_opts: HTTP1Config, pub http2_opts: HTTP2Config, pub websockets_enabled: bool, - pub static_files: Option<(Vec<(String, String)>, Option, Option)>, + pub static_files: Option, pub tls_opts: Option, pub metrics: ( Option, @@ -134,7 +143,7 @@ impl WorkerConfig { http1_opts: HTTP1Config, http2_opts: HTTP2Config, websockets_enabled: bool, - static_files: Option<(Vec<(String, String)>, Option, Option)>, + static_files: Option, ssl_enabled: bool, ssl_cert: Option, ssl_key: Option, @@ -264,24 +273,15 @@ impl WorkerCTXBase { pub(crate) struct WorkerCTXFiles { pub callback: crate::callbacks::ArcCBScheduler, pub metrics: M, - pub static_mounts: Vec<(String, String)>, - pub static_dir_to_file: Option, - pub static_expires: Option, + pub config: StaticFilesConfig, } impl WorkerCTXFiles { - pub fn new( - callback: crate::callbacks::PyCBScheduler, - metrics: M, - files: Option<(Vec<(String, String)>, Option, Option)>, - ) -> Self { - let (static_mounts, static_dir_to_file, static_expires) = files.unwrap(); + pub fn new(callback: crate::callbacks::PyCBScheduler, metrics: M, config: Option) -> Self { Self { callback: Arc::new(callback), metrics, - static_mounts, - static_dir_to_file, - static_expires, + config: config.unwrap(), } } } @@ -402,17 +402,14 @@ macro_rules! service_impl { type Future = Pin> + Send>>; fn call(&self, req: crate::http::HTTPRequest) -> Self::Future { - if let Some(static_match) = crate::files::match_static_file( - req.uri().path(), - &self.ctx.static_mounts, - self.ctx.static_dir_to_file.as_ref(), - ) { + let config = self.ctx.config.clone(); + if let Some(static_match) = crate::files::match_static_file(&config, &req) { if static_match.is_err() { return Box::pin(async move { Ok::<_, hyper::Error>(crate::http::response_404()) }); } - let expires = self.ctx.static_expires.clone(); + let (path, encoding) = static_match.unwrap(); return Box::pin(async move { - Ok::<_, hyper::Error>(crate::files::serve_static_file(static_match.unwrap(), expires).await) + Ok::<_, hyper::Error>(crate::files::serve_static_file(&config, path, encoding).await) }); } @@ -479,11 +476,8 @@ macro_rules! service_impl { .req_handled .fetch_add(1, std::sync::atomic::Ordering::Release); - if let Some(static_match) = crate::files::match_static_file( - req.uri().path(), - &self.ctx.static_mounts, - self.ctx.static_dir_to_file.as_ref(), - ) { + let config = self.ctx.config.clone(); + if let Some(static_match) = crate::files::match_static_file(&config, &req) { self.ctx .metrics .req_static_handled @@ -495,9 +489,9 @@ macro_rules! service_impl { .fetch_add(1, std::sync::atomic::Ordering::Release); return Box::pin(async move { Ok::<_, hyper::Error>(crate::http::response_404()) }); } - let expires = self.ctx.static_expires.clone(); + let (path, encoding) = static_match.unwrap(); return Box::pin(async move { - Ok::<_, hyper::Error>(crate::files::serve_static_file(static_match.unwrap(), expires).await) + Ok::<_, hyper::Error>(crate::files::serve_static_file(&config, path, encoding).await) }); } diff --git a/src/wsgi/serve.rs b/src/wsgi/serve.rs index c5f817f4..19c382bf 100644 --- a/src/wsgi/serve.rs +++ b/src/wsgi/serve.rs @@ -6,7 +6,7 @@ use super::http::handle; use crate::{ callbacks::CallbackScheduler, - conversion::{worker_http1_config_from_py, worker_http2_config_from_py}, + conversion::{worker_http1_config_from_py, worker_http2_config_from_py, worker_static_files_config_from_py}, http::HTTPProto, net::SocketHolder, serve::gen_serve_match, @@ -59,7 +59,7 @@ impl WSGIWorker { http_mode: &str, http1_opts: Option>, http2_opts: Option>, - static_files: Option<(Vec<(String, String)>, Option, Option)>, + static_files: Option>, ssl_enabled: bool, ssl_cert: Option, ssl_key: Option, @@ -84,7 +84,7 @@ impl WSGIWorker { worker_http1_config_from_py(py, http1_opts)?, worker_http2_config_from_py(py, http2_opts)?, false, - static_files, + worker_static_files_config_from_py(py, static_files)?, ssl_enabled, ssl_cert, ssl_key, diff --git a/tests/conftest.py b/tests/conftest.py index ab87d0c7..487d4620 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,6 +26,7 @@ async def _server( task_impl='asyncio', static_mount=False, static_rewrite=False, + static_precompressed=False, ): certs_path = Path.cwd() / 'tests' / 'fixtures' / 'tls' kwargs = { @@ -57,6 +58,8 @@ async def _server( kwargs['static_path_mount'] = [(Path.cwd() / 'tests' / 'fixtures' / v[1]) for v in static_mount] if static_rewrite: kwargs['static_path_dir_to_file'] = 'index.txt' + if static_precompressed: + kwargs['static_path_precompressed'] = True succeeded, spawn_failures = False, 0 while spawn_failures < 3: @@ -130,3 +133,8 @@ def server_tls(server_port, request, tls=True, **extras): @pytest.fixture(scope='function') def server_static_files(server_port, request, static_mount=True, **extras): return partial(_server, request.param, server_port, static_mount=static_mount, **extras) + + +@pytest.fixture(scope='function') +def server_static_files_precompressed(server_port, request): + return partial(_server, request.param, server_port, static_mount=True, static_precompressed=True) diff --git a/tests/fixtures/static/media.png.br b/tests/fixtures/static/media.png.br new file mode 100644 index 0000000000000000000000000000000000000000..6a839dba860ec420b6cfabffbe8ecc961a8d07fe GIT binary patch literal 100 zcmY$aU}WhG@N?(olHy`uVBq!ia0vp^j3CU&3?x-=hn)ga%mF?ju0VQumF+E%TuG2$ qFoVOh8)-lem#2$k2*>s01R$Gz9%CSj!PC{xWt~$(6EgshDioFg literal 0 HcmV?d00001 diff --git a/tests/fixtures/static/media.png.gz b/tests/fixtures/static/media.png.gz new file mode 100644 index 0000000000000000000000000000000000000000..8ede58f492c7214e92ba4646762f526eb893bd4c GIT binary patch literal 112 zcmV-$0FVD4iwFoRu7GI(18rqwX<;sKZf5}N4DfU3<&xrJU|`_&^l%9R(u^R?$P6S^ zZ-<=%Qp^E9A+A7rd6n%gkX%WSUoeBivm0qZ4wt8kV+hCf? S&t;ucLK6UbK}q>v0001^OE9(o literal 0 HcmV?d00001 diff --git a/tests/fixtures/static/media.png.zst b/tests/fixtures/static/media.png.zst new file mode 100644 index 0000000000000000000000000000000000000000..6c4861d91d274e331fa6dc8bb6617df15f0793ba GIT binary patch literal 108 zcmdPcs{dCd{wEVdXMmqOFP9V-0|NuEr-w@rkY)s7MrI(XdOPeCkYWz-32_C|%d2c} zf#gbp{DK)Ap4~_Ta=1KQ978y+Cno^eObm>V6!#bdSqz@8elF{r5}J;!-qrvB1(zBl literal 0 HcmV?d00001 diff --git a/tests/test_static_files.py b/tests/test_static_files.py index 1b3fbe02..d011d351 100644 --- a/tests/test_static_files.py +++ b/tests/test_static_files.py @@ -40,6 +40,7 @@ async def test_static_files_dir_no_rewrite(server_static_files, runtime_mode): @pytest.mark.parametrize('runtime_mode', ['mt', 'st']) async def test_static_files_outsidemount(monkeypatch, server_static_files, runtime_mode): monkeypatch.setattr(httpx._urlparse, 'normalize_path', lambda v: v) + # otherwise httpx will convert /static/../conftest.py to /conftest.py before request async with server_static_files(runtime_mode, ws=False) as port: res = httpx.get(f'http://localhost:{port}/static/../conftest.py') @@ -91,3 +92,182 @@ async def test_static_files_multi(server_static_files, runtime_mode): assert res2.status_code == 200 assert res1.headers.get('content-type') == 'image/png' assert res2.headers.get('content-type') == 'application/x-x509-ca-cert' + + +@pytest.mark.asyncio +@pytest.mark.parametrize('server_static_files', ['asgi', 'rsgi', 'wsgi'], indirect=True) +@pytest.mark.parametrize('runtime_mode', ['mt', 'st']) +async def test_static_files_multi_precompressed(server_static_files, runtime_mode): + async with server_static_files( + runtime_mode, ws=False, static_mount=[('/static', 'static'), ('/other', 'tls')], static_precompressed=True + ) as port: + # Precompressed from first mount + headers = {'Accept-Encoding': 'br'} + res1 = httpx.get(f'http://localhost:{port}/static/media.png', headers=headers) + # Plain file from second mount (no sidecars exist) + res2 = httpx.get(f'http://localhost:{port}/other/cert.pem', headers=headers) + + assert res1.status_code == 200 + assert res1.headers.get('content-type') == 'image/png' + assert res1.headers.get('content-encoding') == 'br' + assert res2.status_code == 200 + assert res2.headers.get('content-type') == 'application/x-x509-ca-cert' + assert res2.headers.get('content-encoding') is None # no sidecar, falls back to plain + + +@pytest.mark.asyncio +@pytest.mark.parametrize('server_static_files_precompressed', ['asgi', 'rsgi', 'wsgi'], indirect=True) +@pytest.mark.parametrize('runtime_mode', ['mt', 'st']) +@pytest.mark.parametrize( + 'accept_encoding,expected_encoding', + [ + ('br', 'br'), + ('gzip', 'gzip'), + ('zstd', 'zstd'), + ], +) +async def test_static_files_precompressed( + server_static_files_precompressed, runtime_mode, accept_encoding, expected_encoding +): + async with server_static_files_precompressed(runtime_mode, ws=False) as port: + headers = {'Accept-Encoding': accept_encoding} + res = httpx.get(f'http://localhost:{port}/static/media.png', headers=headers) + + assert res.status_code == 200 + assert res.headers.get('content-type') == 'image/png' + assert res.headers.get('content-encoding') == expected_encoding + assert res.headers.get('vary') == 'accept-encoding' + + +# only use one interface here since the negotiation logic in Rust is +# interface-agnostic and tests take long to run +@pytest.mark.asyncio +@pytest.mark.parametrize('server_static_files_precompressed', ['rsgi'], indirect=True) +@pytest.mark.parametrize( + 'accept_encoding,expected_encoding', + [ + # multiple encodings - client order wins when q-values equal (first one wins) + ('br, gzip', 'br'), + ('gzip, br', 'gzip'), + ('zstd, gzip', 'zstd'), + ('gzip, zstd', 'gzip'), + ('br, zstd', 'br'), + ('zstd, br', 'zstd'), + ('br, zstd, gzip', 'br'), + # q-value priority - client preference respected + ('gzip;q=1.0, br;q=0.5', 'gzip'), + ('br;q=0.5, gzip;q=1.0', 'gzip'), + ('zstd;q=0.5, br;q=1.0', 'br'), + ('gzip;q=0.9, br;q=0.8, zstd;q=1.0', 'zstd'), + ('gzip;q=1.0, br;q=0.9, zstd;q=0.8', 'gzip'), + # q=0 means explicitly rejected + ('gzip;q=0, br', 'br'), + ('zstd;q=0, br;q=0, gzip', 'gzip'), + # unsupported encodings + ('deflate', None), + ('identity', None), + ], +) +async def test_static_files_precompressed_negotiation( + server_static_files_precompressed, accept_encoding, expected_encoding +): + async with server_static_files_precompressed('mt', ws=False) as port: + headers = {'Accept-Encoding': accept_encoding} + res = httpx.get(f'http://localhost:{port}/static/media.png', headers=headers) + + assert res.status_code == 200 + assert res.headers.get('content-type') == 'image/png' + if expected_encoding: + assert res.headers.get('content-encoding') == expected_encoding + assert res.headers.get('vary') == 'accept-encoding' + else: + assert res.headers.get('content-encoding') is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize('server_static_files_precompressed', ['rsgi'], indirect=True) +async def test_static_files_precompressed_no_accept_encoding(server_static_files_precompressed): + async with server_static_files_precompressed('mt', ws=False) as port: + res = httpx.get(f'http://localhost:{port}/static/media.png', headers={'Accept-Encoding': ''}) + + assert res.status_code == 200 + assert res.headers.get('content-type') == 'image/png' + assert res.headers.get('content-encoding') is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize('server_static_files_precompressed', ['rsgi'], indirect=True) +@pytest.mark.parametrize('accept_encoding', ['br', 'gzip', 'zstd', 'br, gzip, zstd']) +async def test_static_files_precompressed_fallback_to_plain(server_static_files_precompressed, accept_encoding): + async with server_static_files_precompressed('mt', ws=False) as port: + headers = {'Accept-Encoding': accept_encoding} + res = httpx.get(f'http://localhost:{port}/static/こんにちは.png', headers=headers) + # こんにちは.png has no compressed sidecars, so should always serve plain + + assert res.status_code == 200 + assert res.headers.get('content-type') == 'image/png' + assert res.headers.get('content-encoding') is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize('server_static_files_precompressed', ['rsgi'], indirect=True) +@pytest.mark.parametrize( + 'accept_encoding,expected_encoding', + [ + ('gzip;q=invalid', 'gzip'), # invalid defaults to 1.0 + ('gzip;q=', 'gzip'), # empty defaults to 1.0 + ('gzip;q=2.0', 'gzip'), # q>1.0 clamped to 1.0 + ('gzip;q=-0.5', None), # negative clamped to 0.0 (rejected) + (',,,gzip,,,', 'gzip'), # multiple commas handled + # note: can't test leading/trailing whitespace (' gzip ') - httpx rejects it + ], +) +async def test_static_files_precompressed_malformed_qvalues( + server_static_files_precompressed, accept_encoding, expected_encoding +): + async with server_static_files_precompressed('mt', ws=False) as port: + headers = {'Accept-Encoding': accept_encoding} + res = httpx.get(f'http://localhost:{port}/static/media.png', headers=headers) + + assert res.status_code == 200 + assert res.headers.get('content-type') == 'image/png' + if expected_encoding: + assert res.headers.get('content-encoding') == expected_encoding + else: + assert res.headers.get('content-encoding') is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize('server_static_files_precompressed', ['rsgi'], indirect=True) +@pytest.mark.parametrize( + 'accept_encoding,expected_encoding', + [ + ('*', 'zstd'), # matches best available (zstd > br > gzip) + ('gzip, *', 'gzip'), # explicit gzip before wildcard + ('*;q=0.5, gzip;q=1.0', 'gzip'), # q-value on wildcard + ('zstd;q=0, *', 'br'), # reject zstd, wildcard returns next best (br) + ('zstd;q=0, br;q=0, *', 'gzip'), # reject zstd and br, wildcard returns gzip + ], +) +async def test_static_files_precompressed_wildcard( + server_static_files_precompressed, accept_encoding, expected_encoding +): + async with server_static_files_precompressed('mt', ws=False) as port: + headers = {'Accept-Encoding': accept_encoding} + res = httpx.get(f'http://localhost:{port}/static/media.png', headers=headers) + + assert res.status_code == 200 + assert res.headers.get('content-type') == 'image/png' + assert res.headers.get('content-encoding') == expected_encoding + assert res.headers.get('vary') == 'accept-encoding' + + +@pytest.mark.asyncio +@pytest.mark.parametrize('server_static_files_precompressed', ['rsgi'], indirect=True) +async def test_static_files_precompressed_unicode_filename(server_static_files_precompressed): + async with server_static_files_precompressed('mt', ws=False) as port: + headers = {'Accept-Encoding': 'br, gzip, zstd'} + res = httpx.get(f'http://localhost:{port}/static/こんにちは.png', headers=headers) + + assert res.status_code == 200 + assert res.headers.get('content-type') == 'image/png'