diff --git a/.github/workflows/docker-integration.yml b/.github/workflows/docker-integration.yml index cf89f8461..625b6451e 100644 --- a/.github/workflows/docker-integration.yml +++ b/.github/workflows/docker-integration.yml @@ -10,6 +10,7 @@ on: - 'gunicorn/asgi/**' - 'gunicorn/workers/**' - 'tests/docker/h2spec/**' + - 'tests/docker/stress/**' - '.github/workflows/docker-integration.yml' pull_request: paths: @@ -19,6 +20,7 @@ on: - 'gunicorn/asgi/**' - 'gunicorn/workers/**' - 'tests/docker/h2spec/**' + - 'tests/docker/stress/**' - '.github/workflows/docker-integration.yml' permissions: @@ -69,3 +71,21 @@ jobs: - name: Run h2spec against each HTTP/2 worker run: | pytest tests/docker/h2spec/ -v --tb=short + + stress-smoke: + name: Stress smoke (k6) + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v7 + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Install test dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pytest pytest-cov requests + - name: Run the k6 stress smoke matrix + run: | + pytest tests/docker/stress/test_stress.py -v --tb=short diff --git a/.github/workflows/stress-nightly.yml b/.github/workflows/stress-nightly.yml new file mode 100644 index 000000000..b19c2758b --- /dev/null +++ b/.github/workflows/stress-nightly.yml @@ -0,0 +1,41 @@ +name: Stress Nightly + +on: + schedule: + - cron: '0 3 * * *' # 03:00 UTC daily + workflow_dispatch: + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + +jobs: + stress-heavy: + name: Stress and resilience (k6 + toxiproxy) + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v7 + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Install test dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pytest pytest-cov requests + - name: Run the full stress suite (smoke + resilience + faults) + env: + GUNICORN_STRESS_HEAVY: "1" + run: | + pytest tests/docker/stress -v --tb=short + - name: Upload k6 results and container logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: stress-results + path: | + tests/docker/stress/_results/** + if-no-files-found: ignore diff --git a/tests/docker/README.md b/tests/docker/README.md index 0931fe0f3..1a816159e 100644 --- a/tests/docker/README.md +++ b/tests/docker/README.md @@ -47,6 +47,7 @@ to 8000 and collides with `asgi_compliance`. | `asgi_framework_compat` | 8001 to 8006 | | `http2` | 8443, 8444 | | `h2spec` | 8451, 8452, 8453 | +| `stress` | 8460 to 8466, 8474, 8475 | | `uwsgi` | 8080 | If a suite fails everywhere at once, check for a port collision before looking @@ -66,6 +67,7 @@ lsof -nP -iTCP:8000 -sTCP:LISTEN | `dirty_ttin_ttou` | scaling dirty workers with TTIN/TTOU | | `http2` | HTTP/2 over TLS, direct and behind nginx | | `per_app_allocation` | per-app worker allocation end to end | +| `stress` | k6 load and resilience across workers, direct and behind nginx | | `uwsgi` | uWSGI binary protocol behind nginx | `asgi/` and `test_asgi_uwsgi/` are shell-driven demos (`test_asgi.sh`, @@ -73,9 +75,10 @@ lsof -nP -iTCP:8000 -sTCP:LISTEN ## CI -`.github/workflows/docker-integration.yml` runs `tests/docker/uwsgi/` only. The -other suites are not gated, so run them locally before changing the areas they -cover. +`.github/workflows/docker-integration.yml` runs `uwsgi`, `h2spec`, and the +`stress` smoke matrix. The heavier `stress` resilience and fault scenarios run +in `.github/workflows/stress-nightly.yml`. The other suites are not gated, so +run them locally before changing the areas they cover. ## Certificates diff --git a/tests/docker/stress/.gitignore b/tests/docker/stress/.gitignore new file mode 100644 index 000000000..8aa95551d --- /dev/null +++ b/tests/docker/stress/.gitignore @@ -0,0 +1,3 @@ +certs/ +_results/ +*.log diff --git a/tests/docker/stress/Dockerfile.gunicorn b/tests/docker/stress/Dockerfile.gunicorn new file mode 100644 index 000000000..8baafe541 --- /dev/null +++ b/tests/docker/stress/Dockerfile.gunicorn @@ -0,0 +1,17 @@ +# syntax=docker/dockerfile:1 +FROM python:3.14-slim + +RUN apt-get update && apt-get install -y --no-install-recommends gcc curl procps \ + && rm -rf /var/lib/apt/lists/* + +COPY . /gunicorn-src +RUN pip install --no-cache-dir "/gunicorn-src/[http2,gevent,testing]" \ + && pip install --no-cache-dir "starlette>=0.35.0" "websockets>=12.0" + +WORKDIR /app +COPY tests/docker/stress/apps /app +COPY tests/docker/stress/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +EXPOSE 8000 8443 +ENTRYPOINT ["/entrypoint.sh"] diff --git a/tests/docker/stress/Dockerfile.nginx b/tests/docker/stress/Dockerfile.nginx new file mode 100644 index 000000000..03fae11fe --- /dev/null +++ b/tests/docker/stress/Dockerfile.nginx @@ -0,0 +1,6 @@ +FROM nginx:1.29-alpine +RUN apk add --no-cache curl +RUN rm -f /etc/nginx/conf.d/default.conf +COPY tests/docker/stress/nginx.conf /etc/nginx/nginx.conf +COPY tests/docker/stress/uwsgi_params /etc/nginx/uwsgi_params +EXPOSE 8461 8463 8464 8466 diff --git a/tests/docker/stress/README.md b/tests/docker/stress/README.md new file mode 100644 index 000000000..9c189a2d8 --- /dev/null +++ b/tests/docker/stress/README.md @@ -0,0 +1,97 @@ +# Stress and resilience suite + +Drives real load at gunicorn with [k6](https://k6.io) and asserts the server +stays correct under it: no failed requests, intact bodies and checksums, the +negotiated protocol, and no worker tracebacks. Use it when a change touches the +workers, the ASGI/HTTP-2 paths, or process management and you need to know it +holds up under concurrency, not just on a single request. + +Load runs as a pinned k6 container inside the compose network; network faults +run through a pinned [Toxiproxy](https://github.com/Shopify/toxiproxy). Nothing +is installed on the host and Grafana Cloud is not used. HTTP/2 and HPACK +conformance stay with the `h2spec` suite; k6 is a load driver, not a fuzzer. + +## Run it + +```sh +PYTEST=".venv/bin/python -m pytest" scripts/run_docker_tests.sh stress +``` + +This runs the smoke matrix only. The heavier resilience and Toxiproxy scenarios +are opt-in: + +```sh +GUNICORN_STRESS_HEAVY=1 .venv/bin/python -m pytest tests/docker/stress -v +``` + +Requires a running Docker daemon. The first run builds the gunicorn and nginx +images and pulls `grafana/k6` and `ghcr.io/shopify/toxiproxy`; later runs reuse +them. + +## What it covers + +Smoke matrix (wired and verified): + +| Config | Worker | Path | Protocol | +| --- | --- | --- | --- | +| `sync-direct-h1` | sync | direct | HTTP/1.1 | +| `gthread-nginx-h1` | gthread | behind nginx | HTTP/1.1 | +| `asgi-direct-h1` | asgi | direct | HTTP/1.1 | +| `asgi-nginx-h1` | asgi | behind nginx | HTTP/1.1 | +| `asgi-ws-nginx` | asgi | behind nginx | WebSocket | +| `h2-direct-tls` | asgi | direct | HTTP/2 (TLS) | + +nginx terminates HTTP/2 and the uWSGI protocol downstream only; the gunicorn +upstream always sees HTTP/1.1 (proxy) or the uWSGI protocol (`uwsgi_pass`). The +`uwsgi_pass` topology targets the sync worker, gunicorn's supported uWSGI path; +the asgi worker's uWSGI parser is not exercised here. + +Heavy scenarios (`GUNICORN_STRESS_HEAVY=1`): kill a worker, HUP reload, TTIN/TTOU +scaling, and Toxiproxy latency, bandwidth, and connection-reset faults, each +applied while load runs. + +## Selecting worker, topology, protocol, and profile + +Every combination is reachable by environment. The k6 scenario is chosen with +`SCENARIO` (`smoke`, `constant`, `ramping`, `spike`, `churn`, `soak`) and tuned +with `RATE`, `DURATION`, `VUS`, `MAXVUS`, `FAIL_BUDGET`, `MAX_P95`, `MAX_P99`. +Gunicorn services are shaped by the compose environment: `MAX_REQUESTS`, +`ASGI_LOOP` (`auto`/`asyncio`/`uvloop`), and `H2_WORKER` (which worker backs the +HTTP/2 service). For example, to run the asgi worker on uvloop with request +recycling: + +```sh +ASGI_LOOP=uvloop MAX_REQUESTS=1000 docker compose -p gunicorn_stress \ + -f tests/docker/stress/docker-compose.yml up -d --build +``` + +## Ports + +Fixed host ports (run one docker suite at a time; a collision answers from the +wrong stack): + +| Port | Target | +| --- | --- | +| 8460 | sync, direct | +| 8461 | gthread, behind nginx | +| 8462 | asgi, direct | +| 8463 | asgi, behind nginx (HTTP + WebSocket) | +| 8464 | HTTP/2 (TLS) behind nginx | +| 8465 | asgi HTTP/2 (TLS), direct | +| 8466 | sync behind nginx `uwsgi_pass` | +| 8474 | Toxiproxy admin API | +| 8475 | Toxiproxy proxy | + +## Resource expectations + +The smoke matrix runs in a few minutes on a laptop. The nightly profiles +(`ramping`, `spike`, `soak`) and the heavy resilience/fault tests run longer and +push more concurrency; give them a host with a few spare cores and ~2 GB free. +The `soak` scenario defaults to 30 minutes (`DURATION` overrides it). + +## Evidence + +k6 writes a machine-readable summary per run to `_results/-.json` +(git-ignored) and prints error rate, check rate, p95/p99/max latency, and dropped +iterations. Container logs are available with +`docker compose -p gunicorn_stress logs `. diff --git a/tests/docker/stress/apps/__init__.py b/tests/docker/stress/apps/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/docker/stress/apps/stress_app.py b/tests/docker/stress/apps/stress_app.py new file mode 100644 index 000000000..629ef09a1 --- /dev/null +++ b/tests/docker/stress/apps/stress_app.py @@ -0,0 +1,290 @@ +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. + +"""Deterministic apps for the stress suite. + +One module exposes a WSGI callable (``wsgi``) and an ASGI callable (``asgi``) +that answer the same endpoints, so the same load scenario drives every worker. +The ASGI callable also serves WebSocket routes, which only the asgi worker +supports. + +Every response is verifiable under load: + +* ``/echo`` returns the body it received and an ``X-Body-SHA256`` header of + those exact bytes, so a client can prove the body was neither truncated nor + corrupted. +* Every response echoes the request's ``X-Request-Id`` header, so a client can + prove one request's response never carries another request's identifier. +""" + +import asyncio +import hashlib +import json +import time + +PATTERN = b"0123456789abcdef" +SMALL = b"gunicorn-stress-ok\n" +MAX_SIZE = 100 * 1024 * 1024 +CHUNK = 65536 + + +def _filled(size): + """A deterministic body of ``size`` bytes.""" + if size <= 0: + return b"" + reps = size // len(PATTERN) + 1 + return (PATTERN * reps)[:size] + + +def _int(value, default, cap): + try: + n = int(value) + except (TypeError, ValueError): + return default + if n < 0: + return default + return min(n, cap) + + +# --------------------------------------------------------------------------- # +# WSGI +# --------------------------------------------------------------------------- # + +def wsgi(environ, start_response): + from urllib.parse import parse_qs + + path = environ.get("PATH_INFO", "/") + query = parse_qs(environ.get("QUERY_STRING", "")) + req_id = environ.get("HTTP_X_REQUEST_ID", "") + base = [("x-request-id", req_id)] + + def respond(status, headers, body): + start_response(status, base + headers) + return body + + if path == "/health": + return respond("200 OK", [("content-type", "text/plain")], [b"OK"]) + + if path == "/small": + return respond("200 OK", [("content-type", "text/plain")], [SMALL]) + + if path == "/echo": + length = int(environ.get("CONTENT_LENGTH") or 0) + body = environ["wsgi.input"].read(length) if length else b"" + digest = hashlib.sha256(body).hexdigest() + return respond( + "200 OK", + [("content-type", "application/octet-stream"), ("x-body-sha256", digest)], + [body]) + + if path == "/large": + size = _int(query.get("size", [None])[0], 1024 * 1024, MAX_SIZE) + body = _filled(size) + digest = hashlib.sha256(body).hexdigest() + start_response("200 OK", base + [ + ("content-type", "application/octet-stream"), + ("content-length", str(size)), ("x-body-sha256", digest)]) + return (body[i:i + CHUNK] for i in range(0, len(body), CHUNK)) + + if path == "/stream": + chunks = _int(query.get("chunks", [None])[0], 10, 100000) + start_response("200 OK", base + [("content-type", "text/plain")]) + + def gen(): + for i in range(chunks): + yield b"%d\n" % i + return gen() + + if path == "/slow": + ms = _int(query.get("ms", [None])[0], 100, 60000) + time.sleep(ms / 1000.0) + return respond("200 OK", [("content-type", "text/plain")], [b"slept"]) + + if path == "/error": + code = _int(query.get("code", [None])[0], 500, 599) + return respond("%d Error" % code, [("content-type", "text/plain")], + [b"error"]) + + if path == "/meta": + payload = json.dumps({ + "method": environ.get("REQUEST_METHOD"), + "path": path, + "query": environ.get("QUERY_STRING", ""), + "protocol": environ.get("SERVER_PROTOCOL"), + "forwarded_for": environ.get("HTTP_X_FORWARDED_FOR", ""), + "forwarded_proto": environ.get("HTTP_X_FORWARDED_PROTO", ""), + "real_ip": environ.get("HTTP_X_REAL_IP", ""), + "host": environ.get("HTTP_HOST", ""), + }).encode() + return respond("200 OK", [("content-type", "application/json")], [payload]) + + return respond("404 Not Found", [("content-type", "text/plain")], [b"not found"]) + + +# --------------------------------------------------------------------------- # +# ASGI +# --------------------------------------------------------------------------- # + +async def asgi(scope, receive, send): + if scope["type"] == "lifespan": + await _lifespan(scope, receive, send) + return + if scope["type"] == "websocket": + await _websocket(scope, receive, send) + return + if scope["type"] == "http": + await _http(scope, receive, send) + + +async def _lifespan(scope, receive, send): + while True: + message = await receive() + if message["type"] == "lifespan.startup": + await send({"type": "lifespan.startup.complete"}) + elif message["type"] == "lifespan.shutdown": + await send({"type": "lifespan.shutdown.complete"}) + return + + +def _query(scope): + from urllib.parse import parse_qs + return parse_qs(scope.get("query_string", b"").decode()) + + +def _header(scope, name): + key = name.lower().encode() + for k, v in scope.get("headers", []): + if k.lower() == key: + return v.decode() + return "" + + +async def _read_body(receive): + body = b"" + while True: + message = await receive() + body += message.get("body", b"") + if not message.get("more_body"): + break + return body + + +async def _http(scope, receive, send): + path = scope["path"] + query = _query(scope) + req_id = _header(scope, "x-request-id") + base = [(b"x-request-id", req_id.encode())] + + async def start(status, headers): + await send({"type": "http.response.start", "status": status, + "headers": base + headers}) + + async def body(data, more=False): + await send({"type": "http.response.body", "body": data, "more_body": more}) + + if path == "/health": + await start(200, [(b"content-type", b"text/plain")]) + await body(b"OK") + return + + if path == "/small": + await start(200, [(b"content-type", b"text/plain")]) + await body(SMALL) + return + + if path == "/echo": + data = await _read_body(receive) + digest = hashlib.sha256(data).hexdigest() + await start(200, [(b"content-type", b"application/octet-stream"), + (b"x-body-sha256", digest.encode())]) + await body(data) + return + + if path == "/large": + size = _int(query.get("size", [None])[0], 1024 * 1024, MAX_SIZE) + data = _filled(size) + digest = hashlib.sha256(data).hexdigest() + await start(200, [(b"content-type", b"application/octet-stream"), + (b"content-length", str(size).encode()), + (b"x-body-sha256", digest.encode())]) + for i in range(0, len(data), CHUNK): + await body(data[i:i + CHUNK], more=i + CHUNK < len(data)) + if not data: + await body(b"") + return + + if path == "/stream": + chunks = _int(query.get("chunks", [None])[0], 10, 100000) + await start(200, [(b"content-type", b"text/plain")]) + for i in range(chunks): + await body(b"%d\n" % i, more=True) + await body(b"") + return + + if path == "/slow": + ms = _int(query.get("ms", [None])[0], 100, 60000) + await asyncio.sleep(ms / 1000.0) + await start(200, [(b"content-type", b"text/plain")]) + await body(b"slept") + return + + if path == "/error": + code = _int(query.get("code", [None])[0], 500, 599) + await start(code, [(b"content-type", b"text/plain")]) + await body(b"error") + return + + if path == "/meta": + payload = json.dumps({ + "method": scope.get("method"), + "path": path, + "query": scope.get("query_string", b"").decode(), + "protocol": scope.get("http_version"), + "forwarded_for": _header(scope, "x-forwarded-for"), + "forwarded_proto": _header(scope, "x-forwarded-proto"), + "real_ip": _header(scope, "x-real-ip"), + "host": _header(scope, "host"), + }).encode() + await start(200, [(b"content-type", b"application/json")]) + await body(payload) + return + + await start(404, [(b"content-type", b"text/plain")]) + await body(b"not found") + + +async def _websocket(scope, receive, send): + path = scope["path"] + connect = await receive() + if connect["type"] != "websocket.connect": + return + await send({"type": "websocket.accept"}) + + if path == "/ws/close": + await send({"type": "websocket.close", "code": 1000}) + return + + while True: + message = await receive() + if message["type"] == "websocket.disconnect": + return + if message["type"] != "websocket.receive": + continue + if path == "/ws/echo-binary": + await send({"type": "websocket.send", + "bytes": message.get("bytes") or b""}) + elif path == "/ws/ping": + await send({"type": "websocket.send", "text": "pong"}) + else: # /ws/echo and /ws/long + if message.get("text") is not None: + await send({"type": "websocket.send", "text": message["text"]}) + else: + await send({"type": "websocket.send", + "bytes": message.get("bytes") or b""}) + + +# Module-level aliases so ``gunicorn stress_app:app`` works for either worker; +# the entrypoint picks ``stress_app:wsgi`` or ``stress_app:asgi`` by worker. +app = wsgi +application = wsgi diff --git a/tests/docker/stress/conftest.py b/tests/docker/stress/conftest.py new file mode 100644 index 000000000..ad0e43f8c --- /dev/null +++ b/tests/docker/stress/conftest.py @@ -0,0 +1,241 @@ +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. + +"""Fixtures and helpers for the k6 stress and resilience suite. + +The suite brings up one gunicorn image serving several worker/topology targets +behind nginx, drives load with k6 (run as a pinned container inside the compose +network), and asserts correctness through k6 thresholds. Resilience and +Toxiproxy fault tests are gated behind ``@pytest.mark.stress_heavy`` and the +``GUNICORN_STRESS_HEAVY`` environment flag. +""" + +import json +import os +import shutil +import ssl +import subprocess +import time +from collections import namedtuple +from pathlib import Path + +import pytest + +requests = pytest.importorskip("requests") + +STRESS_DIR = Path(__file__).parent +COMPOSE_FILE = STRESS_DIR / "docker-compose.yml" +CERTS_DIR = STRESS_DIR / "certs" +RESULTS_DIR = STRESS_DIR / "_results" +PROJECT = "gunicorn_stress" +HEAVY = os.environ.get("GUNICORN_STRESS_HEAVY") == "1" + +# config id -> how k6 (inside the network) and pytest (from the host) reach it. +CONFIGS = { + "sync-direct-h1": dict( + target="http://gunicorn-sync:8000", host="http://127.0.0.1:8460", + proto="HTTP/1.1", service="gunicorn-sync", kind="http"), + "gthread-nginx-h1": dict( + target="http://nginx:8461", host="http://127.0.0.1:8461", + proto="HTTP/1.1", service="gunicorn-gthread", kind="http"), + "asgi-direct-h1": dict( + target="http://gunicorn-asgi:8000", host="http://127.0.0.1:8462", + proto="HTTP/1.1", service="gunicorn-asgi", kind="http"), + "asgi-nginx-h1": dict( + target="http://nginx:8463", host="http://127.0.0.1:8463", + proto="HTTP/1.1", service="gunicorn-asgi", kind="http"), + "asgi-ws-nginx": dict( + target="ws://nginx:8463", host="http://127.0.0.1:8463", + proto="", service="gunicorn-asgi", kind="ws"), + "h2-direct-tls": dict( + target="https://gunicorn-h2:8443", host="https://127.0.0.1:8465", + proto="HTTP/2.0", service="gunicorn-h2", kind="http"), +} + +K6Result = namedtuple("K6Result", "returncode summary stdout") + + +def _docker_available(): + if not shutil.which("docker"): + return False + try: + subprocess.run(["docker", "info"], check=True, capture_output=True, timeout=20) + subprocess.run(["docker", "compose", "version"], check=True, capture_output=True, timeout=20) + return True + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError): + return False + + +def _generate_cert(): + CERTS_DIR.mkdir(exist_ok=True) + crt, key = CERTS_DIR / "server.crt", CERTS_DIR / "server.key" + if crt.exists() and key.exists() and (time.time() - crt.stat().st_mtime) < 86400: + return + subprocess.run([ + "openssl", "req", "-x509", "-newkey", "rsa:2048", + "-keyout", str(key), "-out", str(crt), "-days", "1", "-nodes", + "-subj", "/CN=localhost/O=Gunicorn Stress/C=US", + "-addext", "subjectAltName=DNS:localhost,DNS:gunicorn-h2,DNS:nginx,IP:127.0.0.1", + ], check=True, capture_output=True) + os.chmod(crt, 0o644) + os.chmod(key, 0o644) + + +def _compose(*args, env=None, **kw): + full = ["docker", "compose", "-p", PROJECT, "-f", str(COMPOSE_FILE), *args] + merged = {**os.environ, **(env or {})} + return subprocess.run(full, cwd=STRESS_DIR, env=merged, **kw) + + +def _wait_ready(timeout=240): + session = requests.Session() + session.verify = False + tcp = [("127.0.0.1", 8460), ("127.0.0.1", 8462), ("127.0.0.1", 8465)] + http = ["http://127.0.0.1:8461/small", "http://127.0.0.1:8463/small", + "http://127.0.0.1:8466/small", "https://127.0.0.1:8464/small"] + import socket + deadline = time.time() + timeout + import warnings + warnings.filterwarnings("ignore") + while time.time() < deadline: + try: + for host, port in tcp: + if port == 8465: + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + with socket.create_connection((host, port), 2) as s: + ctx.wrap_socket(s, server_hostname=host).close() + else: + socket.create_connection((host, port), 2).close() + for url in http: + if session.get(url, timeout=3).status_code != 200: + raise OSError(url) + return + except (OSError, requests.RequestException): + time.sleep(1) + _compose("logs", "--no-color") + raise RuntimeError("stress stack did not become ready") + + +class Stack: + """A running compose stack with load and control-plane helpers.""" + + def run_k6(self, config, scenario, extra_env=None, timeout=1200): + cfg = CONFIGS[config] + rid = f"{config}-{scenario}.json" + (RESULTS_DIR / rid).unlink(missing_ok=True) + env_flags = {"SCENARIO": scenario, "EXPECT_PROTO": cfg["proto"], + "RESULT_FILE": rid, "INSECURE": "1", **(extra_env or {})} + if cfg["kind"] == "ws": + env_flags["WS_TARGET"] = cfg["target"] + script = "ws.js" + else: + env_flags["TARGET"] = cfg["target"] + script = "main.js" + args = ["--profile", "tools", "run", "--rm"] + for k, v in env_flags.items(): + args += ["-e", f"{k}={v}"] + args += ["k6", "run", f"/scripts/{script}"] + proc = _compose(*args, capture_output=True, text=True, timeout=timeout) + summary = {} + path = RESULTS_DIR / rid + if path.exists(): + summary = json.loads(path.read_text()) + return K6Result(proc.returncode, summary, proc.stdout + proc.stderr) + + def worker_pids(self, service): + out = _compose("exec", "-T", service, "pgrep", "-P", "1", + capture_output=True, text=True) + return [int(p) for p in out.stdout.split() if p.strip()] + + def signal_master(self, service, sig): + _compose("exec", "-T", service, "kill", f"-{sig}", "1", check=True) + + def kill_one_worker(self, service): + pids = self.worker_pids(service) + if not pids: + return None + _compose("exec", "-T", service, "kill", "-9", str(pids[0]), check=True) + return pids[0] + + def logs(self, service): + out = _compose("logs", "--no-color", service, capture_output=True, text=True) + return out.stdout + out.stderr + + def tracebacks(self, service): + return self.logs(service).count("Traceback (most recent call last)") + + def rss_kb(self, service): + total = 0 + for pid in [1] + self.worker_pids(service): + out = _compose("exec", "-T", service, "cat", f"/proc/{pid}/status", + capture_output=True, text=True) + for line in out.stdout.splitlines(): + if line.startswith("VmRSS:"): + total += int(line.split()[1]) + return total + + +@pytest.fixture(scope="session") +def stack(): + if not _docker_available(): + pytest.skip("docker not available") + _generate_cert() + RESULTS_DIR.mkdir(exist_ok=True) + os.chmod(RESULTS_DIR, 0o777) + _compose("--profile", "faults", "--profile", "tools", + "down", "-v", "--remove-orphans") # clean any leftover stack first + _compose("build", check=True) + _compose("up", "-d", check=True) + try: + _wait_ready() + _compose("pull", "k6", check=False) + yield Stack() + finally: + _compose("--profile", "faults", "--profile", "tools", + "down", "-v", "--remove-orphans") + + +@pytest.fixture(scope="session") +def faults_stack(stack): + """The smoke stack with a Toxiproxy front for gunicorn-asgi.""" + _compose("--profile", "faults", "up", "-d", "toxiproxy", check=True) + # toxiproxy admin on host 8474; proxy on 8475 -> gunicorn-asgi:8000 + for _ in range(30): + try: + if requests.get("http://127.0.0.1:8474/version", timeout=2).ok: + break + except requests.RequestException: + time.sleep(1) + yield stack + + +def toxiproxy_reset(): + requests.post("http://127.0.0.1:8474/reset", timeout=5) + + +def add_toxic(name, toxic_type, attributes, stream="downstream", toxicity=1.0): + body = {"name": name, "type": toxic_type, "stream": stream, + "toxicity": toxicity, "attributes": attributes} + r = requests.post("http://127.0.0.1:8474/proxies/asgi/toxics", json=body, timeout=5) + r.raise_for_status() + return r.json() + + +def pytest_configure(config): + config.addinivalue_line("markers", "docker: requires a docker daemon") + config.addinivalue_line("markers", "integration: end-to-end integration test") + config.addinivalue_line("markers", "stress: k6 load/correctness scenarios") + config.addinivalue_line("markers", + "stress_heavy: resilience and fault scenarios (opt-in)") + + +def pytest_collection_modifyitems(config, items): + if HEAVY: + return + skip = pytest.mark.skip(reason="set GUNICORN_STRESS_HEAVY=1 to run heavy scenarios") + for item in items: + if "stress_heavy" in item.keywords: + item.add_marker(skip) diff --git a/tests/docker/stress/docker-compose.yml b/tests/docker/stress/docker-compose.yml new file mode 100644 index 000000000..be8dea5ec --- /dev/null +++ b/tests/docker/stress/docker-compose.yml @@ -0,0 +1,86 @@ +# Stress suite. One gunicorn image (built once, shared via the anchor) serves +# every worker/topology combination; each service differs only by environment, +# which entrypoint.sh turns into the gunicorn argv. k6 (load) and toxiproxy +# (network faults) are pinned tool images behind profiles, so a plain `up` +# starts only gunicorn + nginx. + +x-gunicorn: &gunicorn + build: + context: ../../.. + dockerfile: tests/docker/stress/Dockerfile.gunicorn + image: gunicorn-stress:local + volumes: + - ./certs:/certs:ro + healthcheck: + test: ["CMD", "python", "-c", + "import socket,os; p=8443 if os.environ.get('USE_SSL')=='1' else 8000; socket.create_connection(('localhost',p),2).close()"] + interval: 2s + timeout: 5s + retries: 20 + start_period: 5s + +services: + gunicorn-sync: + <<: *gunicorn + environment: { WORKER: sync, WORKERS: "2", MAX_REQUESTS: "${MAX_REQUESTS:-0}" } + ports: ["8460:8000"] + + gunicorn-gthread: + <<: *gunicorn + environment: { WORKER: gthread, WORKERS: "2", THREADS: "4", MAX_REQUESTS: "${MAX_REQUESTS:-0}" } + + gunicorn-asgi: + <<: *gunicorn + environment: { WORKER: asgi, WORKERS: "2", ASGI_LOOP: "${ASGI_LOOP:-auto}", MAX_REQUESTS: "${MAX_REQUESTS:-0}", GRACEFUL_TIMEOUT: "5" } + ports: ["8462:8000"] + + gunicorn-h2: + <<: *gunicorn + environment: { WORKER: "${H2_WORKER:-asgi}", WORKERS: "2", USE_SSL: "1", HTTP_PROTOCOLS: "h2,h1" } + ports: ["8465:8443"] + + gunicorn-uwsgi: + <<: *gunicorn + environment: { WORKER: sync, WORKERS: "2", PROTOCOL: uwsgi } + + nginx: + build: + context: ../../.. + dockerfile: tests/docker/stress/Dockerfile.nginx + image: gunicorn-stress-nginx:local + volumes: + - ./certs:/certs:ro + ports: ["8461:8461", "8463:8463", "8464:8464", "8466:8466"] + depends_on: + gunicorn-gthread: { condition: service_healthy } + gunicorn-asgi: { condition: service_healthy } + gunicorn-h2: { condition: service_healthy } + gunicorn-uwsgi: { condition: service_healthy } + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8461/health"] + interval: 2s + timeout: 5s + retries: 20 + start_period: 5s + + toxiproxy: + image: ghcr.io/shopify/toxiproxy:2.11.0 + profiles: ["faults"] + command: ["-host", "0.0.0.0", "-config", "/config/toxiproxy.json"] + volumes: + - ./toxiproxy.json:/config/toxiproxy.json:ro + ports: ["8474:8474", "8475:8475"] + depends_on: + gunicorn-asgi: { condition: service_healthy } + + k6: + image: grafana/k6:0.54.0 + profiles: ["tools"] + volumes: + - ./k6:/scripts:ro + - ./_results:/results + working_dir: /scripts + +networks: + default: + driver: bridge diff --git a/tests/docker/stress/entrypoint.sh b/tests/docker/stress/entrypoint.sh new file mode 100755 index 000000000..5e8fef7bb --- /dev/null +++ b/tests/docker/stress/entrypoint.sh @@ -0,0 +1,64 @@ +#!/bin/sh +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. +# +# Build the gunicorn argv from environment variables so one image serves every +# worker/topology/protocol combination in the stress suite. +set -eu + +WORKER="${WORKER:-asgi}" +USE_SSL="${USE_SSL:-0}" +PROTOCOL="${PROTOCOL:-http}" +WORKERS="${WORKERS:-2}" +THREADS="${THREADS:-4}" +WORKER_CONNECTIONS="${WORKER_CONNECTIONS:-1000}" +MAX_REQUESTS="${MAX_REQUESTS:-0}" +MAX_REQUESTS_JITTER="${MAX_REQUESTS_JITTER:-0}" +HTTP_PROTOCOLS="${HTTP_PROTOCOLS:-h1}" +ASGI_LOOP="${ASGI_LOOP:-auto}" +GRACEFUL_TIMEOUT="${GRACEFUL_TIMEOUT:-30}" + +if [ "$WORKER" = "asgi" ]; then + APP="stress_app:asgi" +else + APP="stress_app:wsgi" +fi + +if [ "$USE_SSL" = "1" ]; then + BIND="[::]:8443" +elif [ "$PROTOCOL" = "uwsgi" ]; then + BIND="0.0.0.0:8000" +else + BIND="[::]:8000" +fi + +set -- gunicorn "$APP" \ + --bind "$BIND" \ + --worker-class "$WORKER" \ + --workers "$WORKERS" \ + --worker-connections "$WORKER_CONNECTIONS" \ + --max-requests "$MAX_REQUESTS" \ + --max-requests-jitter "$MAX_REQUESTS_JITTER" \ + --graceful-timeout "$GRACEFUL_TIMEOUT" \ + --access-logfile - --error-logfile - --log-level "${LOG_LEVEL:-info}" + +if [ "$WORKER" = "gthread" ]; then + set -- "$@" --threads "$THREADS" +fi + +if [ "$WORKER" = "asgi" ]; then + set -- "$@" --asgi-loop "$ASGI_LOOP" --asgi-disconnect-grace-period 0 +fi + +if [ "$PROTOCOL" = "uwsgi" ]; then + set -- "$@" --protocol uwsgi --uwsgi-allow-from '*' +fi + +if [ "$USE_SSL" = "1" ]; then + set -- "$@" --certfile /certs/server.crt --keyfile /certs/server.key \ + --http-protocols "$HTTP_PROTOCOLS" +fi + +echo "stress entrypoint: $*" >&2 +exec "$@" diff --git a/tests/docker/stress/k6/lib/checks.js b/tests/docker/stress/k6/lib/checks.js new file mode 100644 index 000000000..295a714e0 --- /dev/null +++ b/tests/docker/stress/k6/lib/checks.js @@ -0,0 +1,124 @@ +// Shared checks, thresholds, and scenario builders for the stress suite. +import http from 'k6/http'; +import { check } from 'k6'; +import crypto from 'k6/crypto'; + +const TARGET = __ENV.TARGET || 'http://localhost:8000'; +const EXPECT_PROTO = __ENV.EXPECT_PROTO || ''; // e.g. HTTP/1.1, HTTP/2.0 +const INSECURE = (__ENV.INSECURE || '1') === '1'; +const FAIL_BUDGET = parseFloat(__ENV.FAIL_BUDGET || '0'); // allowed error rate +const MAX_P95 = parseInt(__ENV.MAX_P95 || '3000', 10); // ms +const MAX_P99 = parseInt(__ENV.MAX_P99 || '8000', 10); // ms + +export const params = { insecureSkipTLSVerify: INSECURE }; + +// A request id unique per iteration so a response carrying a different id +// proves cross-request data leakage. +function reqId() { + return `${__VU}-${__ITER}-${Math.random().toString(36).slice(2)}`; +} + +export function thresholds() { + return { + http_req_failed: [`rate<=${FAIL_BUDGET}`], + checks: [`rate>=${1 - FAIL_BUDGET}`], + http_req_duration: [`p(95)<${MAX_P95}`, `p(99)<${MAX_P99}`], + dropped_iterations: ['count>=0'], // reported so LG saturation is visible + }; +} + +// One representative request mix. Every response is checked for status, the +// echoed request id, the negotiated protocol, and (for bodies we can predict) +// an exact checksum. +export function drive() { + const id = reqId(); + const headers = { 'X-Request-Id': id }; + const roll = Math.random(); + + if (roll < 0.4) { + const res = http.get(`${TARGET}/small`, { headers, ...params }); + verify(res, id, 200); + } else if (roll < 0.7) { + const body = crypto.randomBytes(1024 + Math.floor(Math.random() * 8192)); + const res = http.post(`${TARGET}/echo`, body, { + headers: { ...headers, 'Content-Type': 'application/octet-stream' }, + responseType: 'binary', ...params, + }); + const ok = verify(res, id, 200); + check(res, { + 'echo body intact': (r) => new Uint8Array(r.body).length === body.byteLength, + 'echo checksum matches': (r) => + r.headers['X-Body-Sha256'] === crypto.sha256(body, 'hex'), + }) && ok; + } else if (roll < 0.85) { + const size = 65536; + const res = http.get(`${TARGET}/large?size=${size}`, { headers, responseType: 'binary', ...params }); + verify(res, id, 200); + check(res, { + 'large size matches': (r) => new Uint8Array(r.body).length === size, + 'large checksum matches': (r) => + r.headers['X-Body-Sha256'] === crypto.sha256(r.body, 'hex'), + }); + } else { + const res = http.get(`${TARGET}/meta?probe=1`, { headers, ...params }); + verify(res, id, 200); + check(res, { 'meta is json': (r) => (r.headers['Content-Type'] || '').includes('json') }); + } +} + +function verify(res, id, status) { + return check(res, { + 'status is expected': (r) => r.status === status, + 'request id echoed': (r) => r.headers['X-Request-Id'] === id, + 'protocol as negotiated': (r) => !EXPECT_PROTO || r.proto === EXPECT_PROTO, + }); +} + +// Scenario executor definitions, selected by name. +export function scenario(name) { + const RATE = parseInt(__ENV.RATE || '50', 10); + const DURATION = __ENV.DURATION || '30s'; + const VUS = parseInt(__ENV.VUS || '10', 10); + const MAXVUS = parseInt(__ENV.MAXVUS || '200', 10); + + switch (name) { + case 'smoke': + return { executor: 'constant-vus', vus: 2, duration: __ENV.DURATION || '10s' }; + case 'constant': + return { + executor: 'constant-arrival-rate', rate: RATE, timeUnit: '1s', + duration: DURATION, preAllocatedVUs: VUS, maxVUs: MAXVUS, + }; + case 'ramping': + return { + executor: 'ramping-arrival-rate', startRate: 10, timeUnit: '1s', + preAllocatedVUs: VUS, maxVUs: MAXVUS, + stages: [ + { target: RATE, duration: '15s' }, + { target: RATE * 2, duration: '30s' }, + { target: RATE * 4, duration: '30s' }, + { target: 0, duration: '10s' }, + ], + }; + case 'spike': + return { + executor: 'ramping-arrival-rate', startRate: RATE, timeUnit: '1s', + preAllocatedVUs: VUS, maxVUs: MAXVUS, + stages: [ + { target: RATE, duration: '10s' }, + { target: RATE * 10, duration: '10s' }, + { target: RATE, duration: '10s' }, + ], + }; + case 'churn': + // New connection per iteration exercises accept/keepalive churn. + return { executor: 'constant-vus', vus: VUS, duration: DURATION }; + case 'soak': + return { + executor: 'constant-arrival-rate', rate: RATE, timeUnit: '1s', + duration: __ENV.DURATION || '30m', preAllocatedVUs: VUS, maxVUs: MAXVUS, + }; + default: + throw new Error(`unknown scenario ${name}`); + } +} diff --git a/tests/docker/stress/k6/main.js b/tests/docker/stress/k6/main.js new file mode 100644 index 000000000..8bfdb09a6 --- /dev/null +++ b/tests/docker/stress/k6/main.js @@ -0,0 +1,42 @@ +// HTTP stress driver. Select the scenario with SCENARIO and the target with +// TARGET; thresholds make the run exit non-zero if correctness or latency +// budgets are breached. +import { drive, scenario, thresholds } from './lib/checks.js'; + +const NAME = __ENV.SCENARIO || 'smoke'; +const CHURN = NAME === 'churn'; + +export const options = { + noConnectionReuse: CHURN, + insecureSkipTLSVerify: (__ENV.INSECURE || '1') === '1', + scenarios: { [NAME]: scenario(NAME) }, + thresholds: thresholds(), + summaryTrendStats: ['avg', 'min', 'med', 'p(95)', 'p(99)', 'max'], +}; + +export default function () { + drive(); +} + +export function handleSummary(data) { + const out = {}; + out.stdout = textSummary(data); + const file = __ENV.RESULT_FILE; + if (file) out[`/results/${file}`] = JSON.stringify(data, null, 2); + return out; +} + +function textSummary(data) { + const m = data.metrics; + const failed = m.http_req_failed ? m.http_req_failed.values.rate : 0; + const checks = m.checks ? m.checks.values.rate : 1; + const dur = m.http_req_duration ? m.http_req_duration.values : {}; + const dropped = m.dropped_iterations ? m.dropped_iterations.values.count : 0; + return [ + `scenario=${__ENV.SCENARIO} target=${__ENV.TARGET}`, + `http_req_failed=${(failed * 100).toFixed(3)}% checks=${(checks * 100).toFixed(3)}%`, + `p95=${(dur['p(95)'] || 0).toFixed(1)}ms p99=${(dur['p(99)'] || 0).toFixed(1)}ms max=${(dur.max || 0).toFixed(1)}ms`, + `dropped_iterations=${dropped}`, + '', + ].join('\n'); +} diff --git a/tests/docker/stress/k6/ws.js b/tests/docker/stress/k6/ws.js new file mode 100644 index 000000000..f3ab03f39 --- /dev/null +++ b/tests/docker/stress/k6/ws.js @@ -0,0 +1,45 @@ +// WebSocket stress driver for the asgi worker. Opens connections, echoes text, +// checks integrity, and paces itself so the smoke load is a realistic handshake +// rate rather than a connection flood. Raise VUS / drop SLEEP for nightly load. +import ws from 'k6/ws'; +import { check, sleep } from 'k6'; + +const TARGET = __ENV.WS_TARGET || 'ws://localhost:8463'; +const DURATION = __ENV.DURATION || '15s'; +const VUS = parseInt(__ENV.VUS || '10', 10); +const SLEEP = parseFloat(__ENV.WS_SLEEP || '0.3'); +const FAIL_BUDGET = parseFloat(__ENV.FAIL_BUDGET || '0'); + +export const options = { + scenarios: { ws: { executor: 'constant-vus', vus: VUS, duration: DURATION } }, + thresholds: { checks: [`rate>=${1 - FAIL_BUDGET}`] }, +}; + +export default function () { + const text = `hello-${__VU}-${__ITER}-${Math.random().toString(36).slice(2)}`; + const res = ws.connect(`${TARGET}/ws/echo`, {}, (socket) => { + let got = 0; + socket.on('open', () => socket.send(text)); + socket.on('message', (msg) => { + check(msg, { 'ws text echoed intact': (m) => m === text }); + got += 1; + socket.close(); + }); + socket.setTimeout(() => { + check(got, { 'ws received a reply': (g) => g > 0 }); + socket.close(); + }, 5000); + }); + check(res, { 'ws handshake 101': (r) => r && r.status === 101 }); + if (SLEEP > 0) sleep(SLEEP); +} + +export function handleSummary(data) { + const out = { stdout: '' }; + const c = data.metrics.checks ? data.metrics.checks.values.rate : 1; + out.stdout = `ws checks=${(c * 100).toFixed(3)}% sessions=${ + data.metrics.ws_sessions ? data.metrics.ws_sessions.values.count : 0}\n`; + const file = __ENV.RESULT_FILE; + if (file) out[`/results/${file}`] = JSON.stringify(data, null, 2); + return out; +} diff --git a/tests/docker/stress/nginx.conf b/tests/docker/stress/nginx.conf new file mode 100644 index 000000000..3973b2a67 --- /dev/null +++ b/tests/docker/stress/nginx.conf @@ -0,0 +1,87 @@ +# Stress-suite nginx. Each server block is one topology target: +# :8461 proxy_pass to gthread over HTTP/1.1 +# :8463 proxy_pass to the asgi worker over HTTP/1.1 (HTTP, WS, streaming) +# :8464 terminate TLS + HTTP/2 downstream, proxy to asgi over HTTP/1.1 +# :8466 uwsgi_pass to the asgi worker (uWSGI binary protocol upstream) +# HTTP/2 and uWSGI are downstream-only: the gunicorn upstream always sees +# HTTP/1.1 on the proxy blocks and the uWSGI protocol on the uwsgi block. + +worker_processes auto; +events { worker_connections 4096; } + +http { + access_log off; + error_log /dev/stderr warn; + resolver 127.0.0.11 ipv6=off valid=10s; + + map $http_upgrade $connection_upgrade { default upgrade; '' ''; } + + upstream gunicorn_gthread { server gunicorn-gthread:8000 max_fails=0; keepalive 32; } + upstream gunicorn_asgi { server gunicorn-asgi:8000 max_fails=0; keepalive 32; } + upstream gunicorn_h2 { server gunicorn-h2:8443 max_fails=0; keepalive 32; } + upstream gunicorn_uwsgi { server gunicorn-uwsgi:8000 max_fails=0; } + + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + client_max_body_size 200m; + + # gthread behind nginx (HTTP/1.1) + server { + listen 8461; + server_name localhost; + location /health { return 200 'OK'; add_header Content-Type text/plain; } + location /stream/ { proxy_buffering off; proxy_set_header Connection ''; proxy_pass http://gunicorn_gthread/; } + location / { proxy_pass http://gunicorn_gthread; } + } + + # asgi behind nginx (HTTP/1.1) with WebSocket + streaming + server { + listen 8463; + server_name localhost; + location /health { return 200 'OK'; add_header Content-Type text/plain; } + location /ws/ { + proxy_pass http://gunicorn_asgi; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + } + location /stream/ { proxy_buffering off; proxy_set_header Connection ''; proxy_pass http://gunicorn_asgi/; } + location / { proxy_pass http://gunicorn_asgi; } + } + + # HTTP/2 (TLS) downstream, HTTP/1.1 upstream to asgi + server { + listen 8464 ssl; + http2 on; + server_name localhost; + ssl_certificate /certs/server.crt; + ssl_certificate_key /certs/server.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + http2_max_concurrent_streams 128; + location /health { return 200 'OK'; add_header Content-Type text/plain; } + location / { + proxy_pass https://gunicorn_h2; + proxy_ssl_verify off; + proxy_ssl_server_name on; + } + } + + # uwsgi_pass to the asgi worker (uWSGI protocol upstream) + server { + listen 8466; + server_name localhost; + uwsgi_read_timeout 300s; + location /health { return 200 'OK'; add_header Content-Type text/plain; } + location / { + uwsgi_pass gunicorn_uwsgi; + include uwsgi_params; + uwsgi_param HTTP_X_FORWARDED_FOR $proxy_add_x_forwarded_for; + uwsgi_param HTTP_X_REAL_IP $remote_addr; + } + } +} diff --git a/tests/docker/stress/test_faults.py b/tests/docker/stress/test_faults.py new file mode 100644 index 000000000..be1db318d --- /dev/null +++ b/tests/docker/stress/test_faults.py @@ -0,0 +1,50 @@ +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. + +"""Network faults via Toxiproxy in front of gunicorn-asgi, with k6 driving load +through the toxic proxy. Gated behind GUNICORN_STRESS_HEAVY=1.""" + +import pytest + +from conftest import add_toxic, toxiproxy_reset + +pytestmark = [pytest.mark.docker, pytest.mark.stress_heavy, pytest.mark.integration] + +# k6 reaches gunicorn-asgi through the toxiproxy container inside the network. +TOXIC_TARGET_ENV = {"TARGET": "http://toxiproxy:8475"} + + +def _run(faults_stack, env): + # Reuse the asgi-direct config's checks but point k6 at the toxic proxy. + return faults_stack.run_k6("asgi-direct-h1", "constant", + extra_env={**TOXIC_TARGET_ENV, "RATE": "30", + "DURATION": "15s", **env}) + + +def test_latency_toxic(faults_stack): + toxiproxy_reset() + add_toxic("latency", "latency", {"latency": 200, "jitter": 100}) + res = _run(faults_stack, {"FAIL_BUDGET": "0.02", "MAX_P95": "60000", "MAX_P99": "60000"}) + toxiproxy_reset() + assert res.returncode == 0, res.stdout + + +def test_bandwidth_toxic(faults_stack): + toxiproxy_reset() + add_toxic("bandwidth", "bandwidth", {"rate": 512}) # KB/s + res = _run(faults_stack, {"FAIL_BUDGET": "0.05", "MAX_P95": "60000", "MAX_P99": "60000"}) + toxiproxy_reset() + assert res.returncode == 0, res.stdout + + +def test_reset_peer_toxic_is_survived(faults_stack): + toxiproxy_reset() + # Reset a fraction of connections; the worker must not crash or leak. + add_toxic("reset", "reset_peer", {"timeout": 200}, toxicity=0.3) + res = _run(faults_stack, {"FAIL_BUDGET": "0.6"}) # transport errors expected + toxiproxy_reset() + # We assert survival, not a clean run: the worker is still serving after. + assert faults_stack.tracebacks("gunicorn-asgi") == 0 + ok = faults_stack.run_k6("asgi-direct-h1", "smoke") + assert ok.returncode == 0, ok.stdout diff --git a/tests/docker/stress/test_resilience.py b/tests/docker/stress/test_resilience.py new file mode 100644 index 000000000..7f31a6945 --- /dev/null +++ b/tests/docker/stress/test_resilience.py @@ -0,0 +1,101 @@ +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. + +"""Resilience: apply a control-plane fault while k6 load runs and assert the +worker recovers and keeps serving. Gated behind GUNICORN_STRESS_HEAVY=1.""" + +import threading +import time + +import pytest + +pytestmark = [pytest.mark.docker, pytest.mark.stress_heavy, pytest.mark.integration] + +SERVICE = "gunicorn-asgi" +CONFIG = "asgi-nginx-h1" + + +def _load(stack, results, extra_env): + results.append(stack.run_k6(CONFIG, "constant", extra_env=extra_env)) + + +def _wait_workers(stack, predicate, timeout=15): + """Poll the worker count until ``predicate(count)`` holds or time runs out. + + Worker counts settle asynchronously after a signal, and the session stack is + shared across tests, so poll rather than assert after a fixed sleep. + """ + deadline = time.time() + timeout + count = len(stack.worker_pids(SERVICE)) + while time.time() < deadline: + count = len(stack.worker_pids(SERVICE)) + if predicate(count): + return count + time.sleep(0.5) + return count + + +def _wait_stable(stack, timeout=20, samples=4, interval=0.5): + """Return the worker count once it holds steady across ``samples`` reads. + + A prior HUP reload leaves old and new workers briefly co-parented under the + master, so the raw count is inflated until reaping finishes; wait it out + before capturing a baseline. + """ + deadline = time.time() + timeout + last, streak = None, 0 + while time.time() < deadline: + count = len(stack.worker_pids(SERVICE)) + streak = streak + 1 if count == last else 1 + last = count + if streak >= samples: + return count + time.sleep(interval) + return last + + +def _run_under_load(stack, fault, fail_budget="0.05"): + """Start a constant-rate load, fire ``fault`` mid-run, return the k6 result.""" + results = [] + env = {"RATE": "40", "DURATION": "20s", "FAIL_BUDGET": fail_budget} + t = threading.Thread(target=_load, args=(stack, results, env)) + t.start() + time.sleep(6) + fault() + t.join() + assert results, "load thread produced no result" + return results[0] + + +def test_worker_kill_recovers(stack): + before = stack.worker_pids(SERVICE) + res = _run_under_load(stack, lambda: stack.kill_one_worker(SERVICE)) + # A direct-connection casualty is allowed; the budget covers it. + assert res.returncode == 0, res.stdout + after = _wait_workers(stack, lambda c: c >= len(before)) + assert after >= len(before), f"workers not respawned: {len(before)} -> {after}" + + +def test_hup_reload_under_load(stack): + res = _run_under_load(stack, lambda: stack.signal_master(SERVICE, "HUP")) + assert res.returncode == 0, res.stdout + time.sleep(3) + assert stack.worker_pids(SERVICE), "no workers after HUP reload" + + +def test_ttin_ttou_scaling_under_load(stack): + base = _wait_stable(stack) + + res = _run_under_load(stack, lambda: stack.signal_master(SERVICE, "TTIN")) + assert res.returncode == 0, res.stdout + grown = _wait_workers(stack, lambda c: c >= base + 1) + assert grown >= base + 1, f"TTIN did not add a worker: {base} -> {grown}" + + stack.signal_master(SERVICE, "TTOU") + restored = _wait_workers(stack, lambda c: c <= grown - 1) + assert restored <= grown - 1, f"TTOU did not remove a worker: {grown} -> {restored}" + + +def test_no_tracebacks_after_faults(stack): + assert stack.tracebacks(SERVICE) == 0, stack.logs(SERVICE)[-4000:] diff --git a/tests/docker/stress/test_stress.py b/tests/docker/stress/test_stress.py new file mode 100644 index 000000000..592416af7 --- /dev/null +++ b/tests/docker/stress/test_stress.py @@ -0,0 +1,35 @@ +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. + +"""Smoke matrix: drive each representative config under a short k6 load and +assert the server stayed correct (thresholds passed, no worker traceback).""" + +import pytest + +from conftest import CONFIGS + +pytestmark = [pytest.mark.docker, pytest.mark.stress, pytest.mark.integration] + + +def _rate(summary, metric): + try: + return summary["metrics"][metric]["values"]["rate"] + except KeyError: + return None + + +@pytest.mark.parametrize("config", list(CONFIGS)) +def test_smoke(stack, config): + res = stack.run_k6(config, "smoke") + assert res.returncode == 0, f"k6 thresholds failed:\n{res.stdout}" + + checks = _rate(res.summary, "checks") + assert checks == 1.0, f"checks rate {checks}\n{res.stdout}" + + if CONFIGS[config]["kind"] == "http": + failed = _rate(res.summary, "http_req_failed") + assert failed == 0.0, f"request failure rate {failed}\n{res.stdout}" + + service = CONFIGS[config]["service"] + assert stack.tracebacks(service) == 0, stack.logs(service)[-4000:] diff --git a/tests/docker/stress/toxiproxy.json b/tests/docker/stress/toxiproxy.json new file mode 100644 index 000000000..55e9f8cb6 --- /dev/null +++ b/tests/docker/stress/toxiproxy.json @@ -0,0 +1,8 @@ +[ + { + "name": "asgi", + "listen": "0.0.0.0:8475", + "upstream": "gunicorn-asgi:8000", + "enabled": true + } +] diff --git a/tests/docker/stress/uwsgi_params b/tests/docker/stress/uwsgi_params new file mode 100644 index 000000000..228780178 --- /dev/null +++ b/tests/docker/stress/uwsgi_params @@ -0,0 +1,14 @@ +uwsgi_param QUERY_STRING $query_string; +uwsgi_param REQUEST_METHOD $request_method; +uwsgi_param CONTENT_TYPE $content_type; +uwsgi_param CONTENT_LENGTH $content_length; +uwsgi_param REQUEST_URI $request_uri; +uwsgi_param PATH_INFO $document_uri; +uwsgi_param DOCUMENT_ROOT $document_root; +uwsgi_param SERVER_PROTOCOL $server_protocol; +uwsgi_param REQUEST_SCHEME $scheme; +uwsgi_param HTTPS $https if_not_empty; +uwsgi_param REMOTE_ADDR $remote_addr; +uwsgi_param REMOTE_PORT $remote_port; +uwsgi_param SERVER_PORT $server_port; +uwsgi_param SERVER_NAME $server_name;