From fc12e97cf9f4b03ae3024bbf8a37f1f63b8eadfd Mon Sep 17 00:00:00 2001 From: Benoit Chesneau Date: Mon, 31 Aug 2026 11:27:50 +0200 Subject: [PATCH] tests: add k6 stress and resilience suite Drive real load at the WSGI (sync, gthread, gevent) and native ASGI workers with k6, direct and behind nginx, over HTTP/1.1, TLS HTTP/2, WebSocket, and uwsgi_pass, and assert correctness under load: zero failed requests, intact bodies and checksums, the negotiated protocol, echoed request ids, and no worker tracebacks. k6 and toxiproxy run as pinned container images. The smoke matrix (six representative configs) is wired and verified and gates CI; every worker/topology/protocol/profile is selectable by environment. Resilience (worker kill, HUP, TTIN/TTOU, graceful) and toxiproxy fault scenarios are gated behind GUNICORN_STRESS_HEAVY and run nightly. No gunicorn code changes; the uwsgi_pass topology uses the supported sync worker. --- .github/workflows/docker-integration.yml | 20 ++ .github/workflows/stress-nightly.yml | 41 ++++ tests/docker/README.md | 9 +- tests/docker/stress/.gitignore | 3 + tests/docker/stress/Dockerfile.gunicorn | 17 ++ tests/docker/stress/Dockerfile.nginx | 6 + tests/docker/stress/README.md | 97 ++++++++ tests/docker/stress/apps/__init__.py | 0 tests/docker/stress/apps/stress_app.py | 290 +++++++++++++++++++++++ tests/docker/stress/conftest.py | 241 +++++++++++++++++++ tests/docker/stress/docker-compose.yml | 86 +++++++ tests/docker/stress/entrypoint.sh | 64 +++++ tests/docker/stress/k6/lib/checks.js | 124 ++++++++++ tests/docker/stress/k6/main.js | 42 ++++ tests/docker/stress/k6/ws.js | 45 ++++ tests/docker/stress/nginx.conf | 87 +++++++ tests/docker/stress/test_faults.py | 50 ++++ tests/docker/stress/test_resilience.py | 101 ++++++++ tests/docker/stress/test_stress.py | 35 +++ tests/docker/stress/toxiproxy.json | 8 + tests/docker/stress/uwsgi_params | 14 ++ 21 files changed, 1377 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/stress-nightly.yml create mode 100644 tests/docker/stress/.gitignore create mode 100644 tests/docker/stress/Dockerfile.gunicorn create mode 100644 tests/docker/stress/Dockerfile.nginx create mode 100644 tests/docker/stress/README.md create mode 100644 tests/docker/stress/apps/__init__.py create mode 100644 tests/docker/stress/apps/stress_app.py create mode 100644 tests/docker/stress/conftest.py create mode 100644 tests/docker/stress/docker-compose.yml create mode 100755 tests/docker/stress/entrypoint.sh create mode 100644 tests/docker/stress/k6/lib/checks.js create mode 100644 tests/docker/stress/k6/main.js create mode 100644 tests/docker/stress/k6/ws.js create mode 100644 tests/docker/stress/nginx.conf create mode 100644 tests/docker/stress/test_faults.py create mode 100644 tests/docker/stress/test_resilience.py create mode 100644 tests/docker/stress/test_stress.py create mode 100644 tests/docker/stress/toxiproxy.json create mode 100644 tests/docker/stress/uwsgi_params diff --git a/.github/workflows/docker-integration.yml b/.github/workflows/docker-integration.yml index cf89f8461e..625b6451e9 100644 --- a/.github/workflows/docker-integration.yml +++ b/.github/workflows/docker-integration.yml @@ -10,6 +10,7 @@ on: - 'gunicorn/asgi/**' - 'gunicorn/workers/**' - 'tests/docker/h2spec/**' + - 'tests/docker/stress/**' - '.github/workflows/docker-integration.yml' pull_request: paths: @@ -19,6 +20,7 @@ on: - 'gunicorn/asgi/**' - 'gunicorn/workers/**' - 'tests/docker/h2spec/**' + - 'tests/docker/stress/**' - '.github/workflows/docker-integration.yml' permissions: @@ -69,3 +71,21 @@ jobs: - name: Run h2spec against each HTTP/2 worker run: | pytest tests/docker/h2spec/ -v --tb=short + + stress-smoke: + name: Stress smoke (k6) + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v7 + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Install test dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pytest pytest-cov requests + - name: Run the k6 stress smoke matrix + run: | + pytest tests/docker/stress/test_stress.py -v --tb=short diff --git a/.github/workflows/stress-nightly.yml b/.github/workflows/stress-nightly.yml new file mode 100644 index 0000000000..b19c2758bc --- /dev/null +++ b/.github/workflows/stress-nightly.yml @@ -0,0 +1,41 @@ +name: Stress Nightly + +on: + schedule: + - cron: '0 3 * * *' # 03:00 UTC daily + workflow_dispatch: + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + +jobs: + stress-heavy: + name: Stress and resilience (k6 + toxiproxy) + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v7 + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Install test dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pytest pytest-cov requests + - name: Run the full stress suite (smoke + resilience + faults) + env: + GUNICORN_STRESS_HEAVY: "1" + run: | + pytest tests/docker/stress -v --tb=short + - name: Upload k6 results and container logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: stress-results + path: | + tests/docker/stress/_results/** + if-no-files-found: ignore diff --git a/tests/docker/README.md b/tests/docker/README.md index 0931fe0f35..1a816159e0 100644 --- a/tests/docker/README.md +++ b/tests/docker/README.md @@ -47,6 +47,7 @@ to 8000 and collides with `asgi_compliance`. | `asgi_framework_compat` | 8001 to 8006 | | `http2` | 8443, 8444 | | `h2spec` | 8451, 8452, 8453 | +| `stress` | 8460 to 8466, 8474, 8475 | | `uwsgi` | 8080 | If a suite fails everywhere at once, check for a port collision before looking @@ -66,6 +67,7 @@ lsof -nP -iTCP:8000 -sTCP:LISTEN | `dirty_ttin_ttou` | scaling dirty workers with TTIN/TTOU | | `http2` | HTTP/2 over TLS, direct and behind nginx | | `per_app_allocation` | per-app worker allocation end to end | +| `stress` | k6 load and resilience across workers, direct and behind nginx | | `uwsgi` | uWSGI binary protocol behind nginx | `asgi/` and `test_asgi_uwsgi/` are shell-driven demos (`test_asgi.sh`, @@ -73,9 +75,10 @@ lsof -nP -iTCP:8000 -sTCP:LISTEN ## CI -`.github/workflows/docker-integration.yml` runs `tests/docker/uwsgi/` only. The -other suites are not gated, so run them locally before changing the areas they -cover. +`.github/workflows/docker-integration.yml` runs `uwsgi`, `h2spec`, and the +`stress` smoke matrix. The heavier `stress` resilience and fault scenarios run +in `.github/workflows/stress-nightly.yml`. The other suites are not gated, so +run them locally before changing the areas they cover. ## Certificates diff --git a/tests/docker/stress/.gitignore b/tests/docker/stress/.gitignore new file mode 100644 index 0000000000..8aa95551df --- /dev/null +++ b/tests/docker/stress/.gitignore @@ -0,0 +1,3 @@ +certs/ +_results/ +*.log diff --git a/tests/docker/stress/Dockerfile.gunicorn b/tests/docker/stress/Dockerfile.gunicorn new file mode 100644 index 0000000000..8baafe5418 --- /dev/null +++ b/tests/docker/stress/Dockerfile.gunicorn @@ -0,0 +1,17 @@ +# syntax=docker/dockerfile:1 +FROM python:3.14-slim + +RUN apt-get update && apt-get install -y --no-install-recommends gcc curl procps \ + && rm -rf /var/lib/apt/lists/* + +COPY . /gunicorn-src +RUN pip install --no-cache-dir "/gunicorn-src/[http2,gevent,testing]" \ + && pip install --no-cache-dir "starlette>=0.35.0" "websockets>=12.0" + +WORKDIR /app +COPY tests/docker/stress/apps /app +COPY tests/docker/stress/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +EXPOSE 8000 8443 +ENTRYPOINT ["/entrypoint.sh"] diff --git a/tests/docker/stress/Dockerfile.nginx b/tests/docker/stress/Dockerfile.nginx new file mode 100644 index 0000000000..03fae11fe6 --- /dev/null +++ b/tests/docker/stress/Dockerfile.nginx @@ -0,0 +1,6 @@ +FROM nginx:1.29-alpine +RUN apk add --no-cache curl +RUN rm -f /etc/nginx/conf.d/default.conf +COPY tests/docker/stress/nginx.conf /etc/nginx/nginx.conf +COPY tests/docker/stress/uwsgi_params /etc/nginx/uwsgi_params +EXPOSE 8461 8463 8464 8466 diff --git a/tests/docker/stress/README.md b/tests/docker/stress/README.md new file mode 100644 index 0000000000..9c189a2d85 --- /dev/null +++ b/tests/docker/stress/README.md @@ -0,0 +1,97 @@ +# Stress and resilience suite + +Drives real load at gunicorn with [k6](https://k6.io) and asserts the server +stays correct under it: no failed requests, intact bodies and checksums, the +negotiated protocol, and no worker tracebacks. Use it when a change touches the +workers, the ASGI/HTTP-2 paths, or process management and you need to know it +holds up under concurrency, not just on a single request. + +Load runs as a pinned k6 container inside the compose network; network faults +run through a pinned [Toxiproxy](https://github.com/Shopify/toxiproxy). Nothing +is installed on the host and Grafana Cloud is not used. HTTP/2 and HPACK +conformance stay with the `h2spec` suite; k6 is a load driver, not a fuzzer. + +## Run it + +```sh +PYTEST=".venv/bin/python -m pytest" scripts/run_docker_tests.sh stress +``` + +This runs the smoke matrix only. The heavier resilience and Toxiproxy scenarios +are opt-in: + +```sh +GUNICORN_STRESS_HEAVY=1 .venv/bin/python -m pytest tests/docker/stress -v +``` + +Requires a running Docker daemon. The first run builds the gunicorn and nginx +images and pulls `grafana/k6` and `ghcr.io/shopify/toxiproxy`; later runs reuse +them. + +## What it covers + +Smoke matrix (wired and verified): + +| Config | Worker | Path | Protocol | +| --- | --- | --- | --- | +| `sync-direct-h1` | sync | direct | HTTP/1.1 | +| `gthread-nginx-h1` | gthread | behind nginx | HTTP/1.1 | +| `asgi-direct-h1` | asgi | direct | HTTP/1.1 | +| `asgi-nginx-h1` | asgi | behind nginx | HTTP/1.1 | +| `asgi-ws-nginx` | asgi | behind nginx | WebSocket | +| `h2-direct-tls` | asgi | direct | HTTP/2 (TLS) | + +nginx terminates HTTP/2 and the uWSGI protocol downstream only; the gunicorn +upstream always sees HTTP/1.1 (proxy) or the uWSGI protocol (`uwsgi_pass`). The +`uwsgi_pass` topology targets the sync worker, gunicorn's supported uWSGI path; +the asgi worker's uWSGI parser is not exercised here. + +Heavy scenarios (`GUNICORN_STRESS_HEAVY=1`): kill a worker, HUP reload, TTIN/TTOU +scaling, and Toxiproxy latency, bandwidth, and connection-reset faults, each +applied while load runs. + +## Selecting worker, topology, protocol, and profile + +Every combination is reachable by environment. The k6 scenario is chosen with +`SCENARIO` (`smoke`, `constant`, `ramping`, `spike`, `churn`, `soak`) and tuned +with `RATE`, `DURATION`, `VUS`, `MAXVUS`, `FAIL_BUDGET`, `MAX_P95`, `MAX_P99`. +Gunicorn services are shaped by the compose environment: `MAX_REQUESTS`, +`ASGI_LOOP` (`auto`/`asyncio`/`uvloop`), and `H2_WORKER` (which worker backs the +HTTP/2 service). For example, to run the asgi worker on uvloop with request +recycling: + +```sh +ASGI_LOOP=uvloop MAX_REQUESTS=1000 docker compose -p gunicorn_stress \ + -f tests/docker/stress/docker-compose.yml up -d --build +``` + +## Ports + +Fixed host ports (run one docker suite at a time; a collision answers from the +wrong stack): + +| Port | Target | +| --- | --- | +| 8460 | sync, direct | +| 8461 | gthread, behind nginx | +| 8462 | asgi, direct | +| 8463 | asgi, behind nginx (HTTP + WebSocket) | +| 8464 | HTTP/2 (TLS) behind nginx | +| 8465 | asgi HTTP/2 (TLS), direct | +| 8466 | sync behind nginx `uwsgi_pass` | +| 8474 | Toxiproxy admin API | +| 8475 | Toxiproxy proxy | + +## Resource expectations + +The smoke matrix runs in a few minutes on a laptop. The nightly profiles +(`ramping`, `spike`, `soak`) and the heavy resilience/fault tests run longer and +push more concurrency; give them a host with a few spare cores and ~2 GB free. +The `soak` scenario defaults to 30 minutes (`DURATION` overrides it). + +## Evidence + +k6 writes a machine-readable summary per run to `_results/-.json` +(git-ignored) and prints error rate, check rate, p95/p99/max latency, and dropped +iterations. Container logs are available with +`docker compose -p gunicorn_stress logs `. diff --git a/tests/docker/stress/apps/__init__.py b/tests/docker/stress/apps/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/docker/stress/apps/stress_app.py b/tests/docker/stress/apps/stress_app.py new file mode 100644 index 0000000000..629ef09a1f --- /dev/null +++ b/tests/docker/stress/apps/stress_app.py @@ -0,0 +1,290 @@ +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. + +"""Deterministic apps for the stress suite. + +One module exposes a WSGI callable (``wsgi``) and an ASGI callable (``asgi``) +that answer the same endpoints, so the same load scenario drives every worker. +The ASGI callable also serves WebSocket routes, which only the asgi worker +supports. + +Every response is verifiable under load: + +* ``/echo`` returns the body it received and an ``X-Body-SHA256`` header of + those exact bytes, so a client can prove the body was neither truncated nor + corrupted. +* Every response echoes the request's ``X-Request-Id`` header, so a client can + prove one request's response never carries another request's identifier. +""" + +import asyncio +import hashlib +import json +import time + +PATTERN = b"0123456789abcdef" +SMALL = b"gunicorn-stress-ok\n" +MAX_SIZE = 100 * 1024 * 1024 +CHUNK = 65536 + + +def _filled(size): + """A deterministic body of ``size`` bytes.""" + if size <= 0: + return b"" + reps = size // len(PATTERN) + 1 + return (PATTERN * reps)[:size] + + +def _int(value, default, cap): + try: + n = int(value) + except (TypeError, ValueError): + return default + if n < 0: + return default + return min(n, cap) + + +# --------------------------------------------------------------------------- # +# WSGI +# --------------------------------------------------------------------------- # + +def wsgi(environ, start_response): + from urllib.parse import parse_qs + + path = environ.get("PATH_INFO", "/") + query = parse_qs(environ.get("QUERY_STRING", "")) + req_id = environ.get("HTTP_X_REQUEST_ID", "") + base = [("x-request-id", req_id)] + + def respond(status, headers, body): + start_response(status, base + headers) + return body + + if path == "/health": + return respond("200 OK", [("content-type", "text/plain")], [b"OK"]) + + if path == "/small": + return respond("200 OK", [("content-type", "text/plain")], [SMALL]) + + if path == "/echo": + length = int(environ.get("CONTENT_LENGTH") or 0) + body = environ["wsgi.input"].read(length) if length else b"" + digest = hashlib.sha256(body).hexdigest() + return respond( + "200 OK", + [("content-type", "application/octet-stream"), ("x-body-sha256", digest)], + [body]) + + if path == "/large": + size = _int(query.get("size", [None])[0], 1024 * 1024, MAX_SIZE) + body = _filled(size) + digest = hashlib.sha256(body).hexdigest() + start_response("200 OK", base + [ + ("content-type", "application/octet-stream"), + ("content-length", str(size)), ("x-body-sha256", digest)]) + return (body[i:i + CHUNK] for i in range(0, len(body), CHUNK)) + + if path == "/stream": + chunks = _int(query.get("chunks", [None])[0], 10, 100000) + start_response("200 OK", base + [("content-type", "text/plain")]) + + def gen(): + for i in range(chunks): + yield b"%d\n" % i + return gen() + + if path == "/slow": + ms = _int(query.get("ms", [None])[0], 100, 60000) + time.sleep(ms / 1000.0) + return respond("200 OK", [("content-type", "text/plain")], [b"slept"]) + + if path == "/error": + code = _int(query.get("code", [None])[0], 500, 599) + return respond("%d Error" % code, [("content-type", "text/plain")], + [b"error"]) + + if path == "/meta": + payload = json.dumps({ + "method": environ.get("REQUEST_METHOD"), + "path": path, + "query": environ.get("QUERY_STRING", ""), + "protocol": environ.get("SERVER_PROTOCOL"), + "forwarded_for": environ.get("HTTP_X_FORWARDED_FOR", ""), + "forwarded_proto": environ.get("HTTP_X_FORWARDED_PROTO", ""), + "real_ip": environ.get("HTTP_X_REAL_IP", ""), + "host": environ.get("HTTP_HOST", ""), + }).encode() + return respond("200 OK", [("content-type", "application/json")], [payload]) + + return respond("404 Not Found", [("content-type", "text/plain")], [b"not found"]) + + +# --------------------------------------------------------------------------- # +# ASGI +# --------------------------------------------------------------------------- # + +async def asgi(scope, receive, send): + if scope["type"] == "lifespan": + await _lifespan(scope, receive, send) + return + if scope["type"] == "websocket": + await _websocket(scope, receive, send) + return + if scope["type"] == "http": + await _http(scope, receive, send) + + +async def _lifespan(scope, receive, send): + while True: + message = await receive() + if message["type"] == "lifespan.startup": + await send({"type": "lifespan.startup.complete"}) + elif message["type"] == "lifespan.shutdown": + await send({"type": "lifespan.shutdown.complete"}) + return + + +def _query(scope): + from urllib.parse import parse_qs + return parse_qs(scope.get("query_string", b"").decode()) + + +def _header(scope, name): + key = name.lower().encode() + for k, v in scope.get("headers", []): + if k.lower() == key: + return v.decode() + return "" + + +async def _read_body(receive): + body = b"" + while True: + message = await receive() + body += message.get("body", b"") + if not message.get("more_body"): + break + return body + + +async def _http(scope, receive, send): + path = scope["path"] + query = _query(scope) + req_id = _header(scope, "x-request-id") + base = [(b"x-request-id", req_id.encode())] + + async def start(status, headers): + await send({"type": "http.response.start", "status": status, + "headers": base + headers}) + + async def body(data, more=False): + await send({"type": "http.response.body", "body": data, "more_body": more}) + + if path == "/health": + await start(200, [(b"content-type", b"text/plain")]) + await body(b"OK") + return + + if path == "/small": + await start(200, [(b"content-type", b"text/plain")]) + await body(SMALL) + return + + if path == "/echo": + data = await _read_body(receive) + digest = hashlib.sha256(data).hexdigest() + await start(200, [(b"content-type", b"application/octet-stream"), + (b"x-body-sha256", digest.encode())]) + await body(data) + return + + if path == "/large": + size = _int(query.get("size", [None])[0], 1024 * 1024, MAX_SIZE) + data = _filled(size) + digest = hashlib.sha256(data).hexdigest() + await start(200, [(b"content-type", b"application/octet-stream"), + (b"content-length", str(size).encode()), + (b"x-body-sha256", digest.encode())]) + for i in range(0, len(data), CHUNK): + await body(data[i:i + CHUNK], more=i + CHUNK < len(data)) + if not data: + await body(b"") + return + + if path == "/stream": + chunks = _int(query.get("chunks", [None])[0], 10, 100000) + await start(200, [(b"content-type", b"text/plain")]) + for i in range(chunks): + await body(b"%d\n" % i, more=True) + await body(b"") + return + + if path == "/slow": + ms = _int(query.get("ms", [None])[0], 100, 60000) + await asyncio.sleep(ms / 1000.0) + await start(200, [(b"content-type", b"text/plain")]) + await body(b"slept") + return + + if path == "/error": + code = _int(query.get("code", [None])[0], 500, 599) + await start(code, [(b"content-type", b"text/plain")]) + await body(b"error") + return + + if path == "/meta": + payload = json.dumps({ + "method": scope.get("method"), + "path": path, + "query": scope.get("query_string", b"").decode(), + "protocol": scope.get("http_version"), + "forwarded_for": _header(scope, "x-forwarded-for"), + "forwarded_proto": _header(scope, "x-forwarded-proto"), + "real_ip": _header(scope, "x-real-ip"), + "host": _header(scope, "host"), + }).encode() + await start(200, [(b"content-type", b"application/json")]) + await body(payload) + return + + await start(404, [(b"content-type", b"text/plain")]) + await body(b"not found") + + +async def _websocket(scope, receive, send): + path = scope["path"] + connect = await receive() + if connect["type"] != "websocket.connect": + return + await send({"type": "websocket.accept"}) + + if path == "/ws/close": + await send({"type": "websocket.close", "code": 1000}) + return + + while True: + message = await receive() + if message["type"] == "websocket.disconnect": + return + if message["type"] != "websocket.receive": + continue + if path == "/ws/echo-binary": + await send({"type": "websocket.send", + "bytes": message.get("bytes") or b""}) + elif path == "/ws/ping": + await send({"type": "websocket.send", "text": "pong"}) + else: # /ws/echo and /ws/long + if message.get("text") is not None: + await send({"type": "websocket.send", "text": message["text"]}) + else: + await send({"type": "websocket.send", + "bytes": message.get("bytes") or b""}) + + +# Module-level aliases so ``gunicorn stress_app:app`` works for either worker; +# the entrypoint picks ``stress_app:wsgi`` or ``stress_app:asgi`` by worker. +app = wsgi +application = wsgi diff --git a/tests/docker/stress/conftest.py b/tests/docker/stress/conftest.py new file mode 100644 index 0000000000..ad0e43f8cc --- /dev/null +++ b/tests/docker/stress/conftest.py @@ -0,0 +1,241 @@ +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. + +"""Fixtures and helpers for the k6 stress and resilience suite. + +The suite brings up one gunicorn image serving several worker/topology targets +behind nginx, drives load with k6 (run as a pinned container inside the compose +network), and asserts correctness through k6 thresholds. Resilience and +Toxiproxy fault tests are gated behind ``@pytest.mark.stress_heavy`` and the +``GUNICORN_STRESS_HEAVY`` environment flag. +""" + +import json +import os +import shutil +import ssl +import subprocess +import time +from collections import namedtuple +from pathlib import Path + +import pytest + +requests = pytest.importorskip("requests") + +STRESS_DIR = Path(__file__).parent +COMPOSE_FILE = STRESS_DIR / "docker-compose.yml" +CERTS_DIR = STRESS_DIR / "certs" +RESULTS_DIR = STRESS_DIR / "_results" +PROJECT = "gunicorn_stress" +HEAVY = os.environ.get("GUNICORN_STRESS_HEAVY") == "1" + +# config id -> how k6 (inside the network) and pytest (from the host) reach it. +CONFIGS = { + "sync-direct-h1": dict( + target="http://gunicorn-sync:8000", host="http://127.0.0.1:8460", + proto="HTTP/1.1", service="gunicorn-sync", kind="http"), + "gthread-nginx-h1": dict( + target="http://nginx:8461", host="http://127.0.0.1:8461", + proto="HTTP/1.1", service="gunicorn-gthread", kind="http"), + "asgi-direct-h1": dict( + target="http://gunicorn-asgi:8000", host="http://127.0.0.1:8462", + proto="HTTP/1.1", service="gunicorn-asgi", kind="http"), + "asgi-nginx-h1": dict( + target="http://nginx:8463", host="http://127.0.0.1:8463", + proto="HTTP/1.1", service="gunicorn-asgi", kind="http"), + "asgi-ws-nginx": dict( + target="ws://nginx:8463", host="http://127.0.0.1:8463", + proto="", service="gunicorn-asgi", kind="ws"), + "h2-direct-tls": dict( + target="https://gunicorn-h2:8443", host="https://127.0.0.1:8465", + proto="HTTP/2.0", service="gunicorn-h2", kind="http"), +} + +K6Result = namedtuple("K6Result", "returncode summary stdout") + + +def _docker_available(): + if not shutil.which("docker"): + return False + try: + subprocess.run(["docker", "info"], check=True, capture_output=True, timeout=20) + subprocess.run(["docker", "compose", "version"], check=True, capture_output=True, timeout=20) + return True + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError): + return False + + +def _generate_cert(): + CERTS_DIR.mkdir(exist_ok=True) + crt, key = CERTS_DIR / "server.crt", CERTS_DIR / "server.key" + if crt.exists() and key.exists() and (time.time() - crt.stat().st_mtime) < 86400: + return + subprocess.run([ + "openssl", "req", "-x509", "-newkey", "rsa:2048", + "-keyout", str(key), "-out", str(crt), "-days", "1", "-nodes", + "-subj", "/CN=localhost/O=Gunicorn Stress/C=US", + "-addext", "subjectAltName=DNS:localhost,DNS:gunicorn-h2,DNS:nginx,IP:127.0.0.1", + ], check=True, capture_output=True) + os.chmod(crt, 0o644) + os.chmod(key, 0o644) + + +def _compose(*args, env=None, **kw): + full = ["docker", "compose", "-p", PROJECT, "-f", str(COMPOSE_FILE), *args] + merged = {**os.environ, **(env or {})} + return subprocess.run(full, cwd=STRESS_DIR, env=merged, **kw) + + +def _wait_ready(timeout=240): + session = requests.Session() + session.verify = False + tcp = [("127.0.0.1", 8460), ("127.0.0.1", 8462), ("127.0.0.1", 8465)] + http = ["http://127.0.0.1:8461/small", "http://127.0.0.1:8463/small", + "http://127.0.0.1:8466/small", "https://127.0.0.1:8464/small"] + import socket + deadline = time.time() + timeout + import warnings + warnings.filterwarnings("ignore") + while time.time() < deadline: + try: + for host, port in tcp: + if port == 8465: + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + with socket.create_connection((host, port), 2) as s: + ctx.wrap_socket(s, server_hostname=host).close() + else: + socket.create_connection((host, port), 2).close() + for url in http: + if session.get(url, timeout=3).status_code != 200: + raise OSError(url) + return + except (OSError, requests.RequestException): + time.sleep(1) + _compose("logs", "--no-color") + raise RuntimeError("stress stack did not become ready") + + +class Stack: + """A running compose stack with load and control-plane helpers.""" + + def run_k6(self, config, scenario, extra_env=None, timeout=1200): + cfg = CONFIGS[config] + rid = f"{config}-{scenario}.json" + (RESULTS_DIR / rid).unlink(missing_ok=True) + env_flags = {"SCENARIO": scenario, "EXPECT_PROTO": cfg["proto"], + "RESULT_FILE": rid, "INSECURE": "1", **(extra_env or {})} + if cfg["kind"] == "ws": + env_flags["WS_TARGET"] = cfg["target"] + script = "ws.js" + else: + env_flags["TARGET"] = cfg["target"] + script = "main.js" + args = ["--profile", "tools", "run", "--rm"] + for k, v in env_flags.items(): + args += ["-e", f"{k}={v}"] + args += ["k6", "run", f"/scripts/{script}"] + proc = _compose(*args, capture_output=True, text=True, timeout=timeout) + summary = {} + path = RESULTS_DIR / rid + if path.exists(): + summary = json.loads(path.read_text()) + return K6Result(proc.returncode, summary, proc.stdout + proc.stderr) + + def worker_pids(self, service): + out = _compose("exec", "-T", service, "pgrep", "-P", "1", + capture_output=True, text=True) + return [int(p) for p in out.stdout.split() if p.strip()] + + def signal_master(self, service, sig): + _compose("exec", "-T", service, "kill", f"-{sig}", "1", check=True) + + def kill_one_worker(self, service): + pids = self.worker_pids(service) + if not pids: + return None + _compose("exec", "-T", service, "kill", "-9", str(pids[0]), check=True) + return pids[0] + + def logs(self, service): + out = _compose("logs", "--no-color", service, capture_output=True, text=True) + return out.stdout + out.stderr + + def tracebacks(self, service): + return self.logs(service).count("Traceback (most recent call last)") + + def rss_kb(self, service): + total = 0 + for pid in [1] + self.worker_pids(service): + out = _compose("exec", "-T", service, "cat", f"/proc/{pid}/status", + capture_output=True, text=True) + for line in out.stdout.splitlines(): + if line.startswith("VmRSS:"): + total += int(line.split()[1]) + return total + + +@pytest.fixture(scope="session") +def stack(): + if not _docker_available(): + pytest.skip("docker not available") + _generate_cert() + RESULTS_DIR.mkdir(exist_ok=True) + os.chmod(RESULTS_DIR, 0o777) + _compose("--profile", "faults", "--profile", "tools", + "down", "-v", "--remove-orphans") # clean any leftover stack first + _compose("build", check=True) + _compose("up", "-d", check=True) + try: + _wait_ready() + _compose("pull", "k6", check=False) + yield Stack() + finally: + _compose("--profile", "faults", "--profile", "tools", + "down", "-v", "--remove-orphans") + + +@pytest.fixture(scope="session") +def faults_stack(stack): + """The smoke stack with a Toxiproxy front for gunicorn-asgi.""" + _compose("--profile", "faults", "up", "-d", "toxiproxy", check=True) + # toxiproxy admin on host 8474; proxy on 8475 -> gunicorn-asgi:8000 + for _ in range(30): + try: + if requests.get("http://127.0.0.1:8474/version", timeout=2).ok: + break + except requests.RequestException: + time.sleep(1) + yield stack + + +def toxiproxy_reset(): + requests.post("http://127.0.0.1:8474/reset", timeout=5) + + +def add_toxic(name, toxic_type, attributes, stream="downstream", toxicity=1.0): + body = {"name": name, "type": toxic_type, "stream": stream, + "toxicity": toxicity, "attributes": attributes} + r = requests.post("http://127.0.0.1:8474/proxies/asgi/toxics", json=body, timeout=5) + r.raise_for_status() + return r.json() + + +def pytest_configure(config): + config.addinivalue_line("markers", "docker: requires a docker daemon") + config.addinivalue_line("markers", "integration: end-to-end integration test") + config.addinivalue_line("markers", "stress: k6 load/correctness scenarios") + config.addinivalue_line("markers", + "stress_heavy: resilience and fault scenarios (opt-in)") + + +def pytest_collection_modifyitems(config, items): + if HEAVY: + return + skip = pytest.mark.skip(reason="set GUNICORN_STRESS_HEAVY=1 to run heavy scenarios") + for item in items: + if "stress_heavy" in item.keywords: + item.add_marker(skip) diff --git a/tests/docker/stress/docker-compose.yml b/tests/docker/stress/docker-compose.yml new file mode 100644 index 0000000000..be8dea5ecf --- /dev/null +++ b/tests/docker/stress/docker-compose.yml @@ -0,0 +1,86 @@ +# Stress suite. One gunicorn image (built once, shared via the anchor) serves +# every worker/topology combination; each service differs only by environment, +# which entrypoint.sh turns into the gunicorn argv. k6 (load) and toxiproxy +# (network faults) are pinned tool images behind profiles, so a plain `up` +# starts only gunicorn + nginx. + +x-gunicorn: &gunicorn + build: + context: ../../.. + dockerfile: tests/docker/stress/Dockerfile.gunicorn + image: gunicorn-stress:local + volumes: + - ./certs:/certs:ro + healthcheck: + test: ["CMD", "python", "-c", + "import socket,os; p=8443 if os.environ.get('USE_SSL')=='1' else 8000; socket.create_connection(('localhost',p),2).close()"] + interval: 2s + timeout: 5s + retries: 20 + start_period: 5s + +services: + gunicorn-sync: + <<: *gunicorn + environment: { WORKER: sync, WORKERS: "2", MAX_REQUESTS: "${MAX_REQUESTS:-0}" } + ports: ["8460:8000"] + + gunicorn-gthread: + <<: *gunicorn + environment: { WORKER: gthread, WORKERS: "2", THREADS: "4", MAX_REQUESTS: "${MAX_REQUESTS:-0}" } + + gunicorn-asgi: + <<: *gunicorn + environment: { WORKER: asgi, WORKERS: "2", ASGI_LOOP: "${ASGI_LOOP:-auto}", MAX_REQUESTS: "${MAX_REQUESTS:-0}", GRACEFUL_TIMEOUT: "5" } + ports: ["8462:8000"] + + gunicorn-h2: + <<: *gunicorn + environment: { WORKER: "${H2_WORKER:-asgi}", WORKERS: "2", USE_SSL: "1", HTTP_PROTOCOLS: "h2,h1" } + ports: ["8465:8443"] + + gunicorn-uwsgi: + <<: *gunicorn + environment: { WORKER: sync, WORKERS: "2", PROTOCOL: uwsgi } + + nginx: + build: + context: ../../.. + dockerfile: tests/docker/stress/Dockerfile.nginx + image: gunicorn-stress-nginx:local + volumes: + - ./certs:/certs:ro + ports: ["8461:8461", "8463:8463", "8464:8464", "8466:8466"] + depends_on: + gunicorn-gthread: { condition: service_healthy } + gunicorn-asgi: { condition: service_healthy } + gunicorn-h2: { condition: service_healthy } + gunicorn-uwsgi: { condition: service_healthy } + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8461/health"] + interval: 2s + timeout: 5s + retries: 20 + start_period: 5s + + toxiproxy: + image: ghcr.io/shopify/toxiproxy:2.11.0 + profiles: ["faults"] + command: ["-host", "0.0.0.0", "-config", "/config/toxiproxy.json"] + volumes: + - ./toxiproxy.json:/config/toxiproxy.json:ro + ports: ["8474:8474", "8475:8475"] + depends_on: + gunicorn-asgi: { condition: service_healthy } + + k6: + image: grafana/k6:0.54.0 + profiles: ["tools"] + volumes: + - ./k6:/scripts:ro + - ./_results:/results + working_dir: /scripts + +networks: + default: + driver: bridge diff --git a/tests/docker/stress/entrypoint.sh b/tests/docker/stress/entrypoint.sh new file mode 100755 index 0000000000..5e8fef7bbe --- /dev/null +++ b/tests/docker/stress/entrypoint.sh @@ -0,0 +1,64 @@ +#!/bin/sh +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. +# +# Build the gunicorn argv from environment variables so one image serves every +# worker/topology/protocol combination in the stress suite. +set -eu + +WORKER="${WORKER:-asgi}" +USE_SSL="${USE_SSL:-0}" +PROTOCOL="${PROTOCOL:-http}" +WORKERS="${WORKERS:-2}" +THREADS="${THREADS:-4}" +WORKER_CONNECTIONS="${WORKER_CONNECTIONS:-1000}" +MAX_REQUESTS="${MAX_REQUESTS:-0}" +MAX_REQUESTS_JITTER="${MAX_REQUESTS_JITTER:-0}" +HTTP_PROTOCOLS="${HTTP_PROTOCOLS:-h1}" +ASGI_LOOP="${ASGI_LOOP:-auto}" +GRACEFUL_TIMEOUT="${GRACEFUL_TIMEOUT:-30}" + +if [ "$WORKER" = "asgi" ]; then + APP="stress_app:asgi" +else + APP="stress_app:wsgi" +fi + +if [ "$USE_SSL" = "1" ]; then + BIND="[::]:8443" +elif [ "$PROTOCOL" = "uwsgi" ]; then + BIND="0.0.0.0:8000" +else + BIND="[::]:8000" +fi + +set -- gunicorn "$APP" \ + --bind "$BIND" \ + --worker-class "$WORKER" \ + --workers "$WORKERS" \ + --worker-connections "$WORKER_CONNECTIONS" \ + --max-requests "$MAX_REQUESTS" \ + --max-requests-jitter "$MAX_REQUESTS_JITTER" \ + --graceful-timeout "$GRACEFUL_TIMEOUT" \ + --access-logfile - --error-logfile - --log-level "${LOG_LEVEL:-info}" + +if [ "$WORKER" = "gthread" ]; then + set -- "$@" --threads "$THREADS" +fi + +if [ "$WORKER" = "asgi" ]; then + set -- "$@" --asgi-loop "$ASGI_LOOP" --asgi-disconnect-grace-period 0 +fi + +if [ "$PROTOCOL" = "uwsgi" ]; then + set -- "$@" --protocol uwsgi --uwsgi-allow-from '*' +fi + +if [ "$USE_SSL" = "1" ]; then + set -- "$@" --certfile /certs/server.crt --keyfile /certs/server.key \ + --http-protocols "$HTTP_PROTOCOLS" +fi + +echo "stress entrypoint: $*" >&2 +exec "$@" diff --git a/tests/docker/stress/k6/lib/checks.js b/tests/docker/stress/k6/lib/checks.js new file mode 100644 index 0000000000..295a714e0c --- /dev/null +++ b/tests/docker/stress/k6/lib/checks.js @@ -0,0 +1,124 @@ +// Shared checks, thresholds, and scenario builders for the stress suite. +import http from 'k6/http'; +import { check } from 'k6'; +import crypto from 'k6/crypto'; + +const TARGET = __ENV.TARGET || 'http://localhost:8000'; +const EXPECT_PROTO = __ENV.EXPECT_PROTO || ''; // e.g. HTTP/1.1, HTTP/2.0 +const INSECURE = (__ENV.INSECURE || '1') === '1'; +const FAIL_BUDGET = parseFloat(__ENV.FAIL_BUDGET || '0'); // allowed error rate +const MAX_P95 = parseInt(__ENV.MAX_P95 || '3000', 10); // ms +const MAX_P99 = parseInt(__ENV.MAX_P99 || '8000', 10); // ms + +export const params = { insecureSkipTLSVerify: INSECURE }; + +// A request id unique per iteration so a response carrying a different id +// proves cross-request data leakage. +function reqId() { + return `${__VU}-${__ITER}-${Math.random().toString(36).slice(2)}`; +} + +export function thresholds() { + return { + http_req_failed: [`rate<=${FAIL_BUDGET}`], + checks: [`rate>=${1 - FAIL_BUDGET}`], + http_req_duration: [`p(95)<${MAX_P95}`, `p(99)<${MAX_P99}`], + dropped_iterations: ['count>=0'], // reported so LG saturation is visible + }; +} + +// One representative request mix. Every response is checked for status, the +// echoed request id, the negotiated protocol, and (for bodies we can predict) +// an exact checksum. +export function drive() { + const id = reqId(); + const headers = { 'X-Request-Id': id }; + const roll = Math.random(); + + if (roll < 0.4) { + const res = http.get(`${TARGET}/small`, { headers, ...params }); + verify(res, id, 200); + } else if (roll < 0.7) { + const body = crypto.randomBytes(1024 + Math.floor(Math.random() * 8192)); + const res = http.post(`${TARGET}/echo`, body, { + headers: { ...headers, 'Content-Type': 'application/octet-stream' }, + responseType: 'binary', ...params, + }); + const ok = verify(res, id, 200); + check(res, { + 'echo body intact': (r) => new Uint8Array(r.body).length === body.byteLength, + 'echo checksum matches': (r) => + r.headers['X-Body-Sha256'] === crypto.sha256(body, 'hex'), + }) && ok; + } else if (roll < 0.85) { + const size = 65536; + const res = http.get(`${TARGET}/large?size=${size}`, { headers, responseType: 'binary', ...params }); + verify(res, id, 200); + check(res, { + 'large size matches': (r) => new Uint8Array(r.body).length === size, + 'large checksum matches': (r) => + r.headers['X-Body-Sha256'] === crypto.sha256(r.body, 'hex'), + }); + } else { + const res = http.get(`${TARGET}/meta?probe=1`, { headers, ...params }); + verify(res, id, 200); + check(res, { 'meta is json': (r) => (r.headers['Content-Type'] || '').includes('json') }); + } +} + +function verify(res, id, status) { + return check(res, { + 'status is expected': (r) => r.status === status, + 'request id echoed': (r) => r.headers['X-Request-Id'] === id, + 'protocol as negotiated': (r) => !EXPECT_PROTO || r.proto === EXPECT_PROTO, + }); +} + +// Scenario executor definitions, selected by name. +export function scenario(name) { + const RATE = parseInt(__ENV.RATE || '50', 10); + const DURATION = __ENV.DURATION || '30s'; + const VUS = parseInt(__ENV.VUS || '10', 10); + const MAXVUS = parseInt(__ENV.MAXVUS || '200', 10); + + switch (name) { + case 'smoke': + return { executor: 'constant-vus', vus: 2, duration: __ENV.DURATION || '10s' }; + case 'constant': + return { + executor: 'constant-arrival-rate', rate: RATE, timeUnit: '1s', + duration: DURATION, preAllocatedVUs: VUS, maxVUs: MAXVUS, + }; + case 'ramping': + return { + executor: 'ramping-arrival-rate', startRate: 10, timeUnit: '1s', + preAllocatedVUs: VUS, maxVUs: MAXVUS, + stages: [ + { target: RATE, duration: '15s' }, + { target: RATE * 2, duration: '30s' }, + { target: RATE * 4, duration: '30s' }, + { target: 0, duration: '10s' }, + ], + }; + case 'spike': + return { + executor: 'ramping-arrival-rate', startRate: RATE, timeUnit: '1s', + preAllocatedVUs: VUS, maxVUs: MAXVUS, + stages: [ + { target: RATE, duration: '10s' }, + { target: RATE * 10, duration: '10s' }, + { target: RATE, duration: '10s' }, + ], + }; + case 'churn': + // New connection per iteration exercises accept/keepalive churn. + return { executor: 'constant-vus', vus: VUS, duration: DURATION }; + case 'soak': + return { + executor: 'constant-arrival-rate', rate: RATE, timeUnit: '1s', + duration: __ENV.DURATION || '30m', preAllocatedVUs: VUS, maxVUs: MAXVUS, + }; + default: + throw new Error(`unknown scenario ${name}`); + } +} diff --git a/tests/docker/stress/k6/main.js b/tests/docker/stress/k6/main.js new file mode 100644 index 0000000000..8bfdb09a68 --- /dev/null +++ b/tests/docker/stress/k6/main.js @@ -0,0 +1,42 @@ +// HTTP stress driver. Select the scenario with SCENARIO and the target with +// TARGET; thresholds make the run exit non-zero if correctness or latency +// budgets are breached. +import { drive, scenario, thresholds } from './lib/checks.js'; + +const NAME = __ENV.SCENARIO || 'smoke'; +const CHURN = NAME === 'churn'; + +export const options = { + noConnectionReuse: CHURN, + insecureSkipTLSVerify: (__ENV.INSECURE || '1') === '1', + scenarios: { [NAME]: scenario(NAME) }, + thresholds: thresholds(), + summaryTrendStats: ['avg', 'min', 'med', 'p(95)', 'p(99)', 'max'], +}; + +export default function () { + drive(); +} + +export function handleSummary(data) { + const out = {}; + out.stdout = textSummary(data); + const file = __ENV.RESULT_FILE; + if (file) out[`/results/${file}`] = JSON.stringify(data, null, 2); + return out; +} + +function textSummary(data) { + const m = data.metrics; + const failed = m.http_req_failed ? m.http_req_failed.values.rate : 0; + const checks = m.checks ? m.checks.values.rate : 1; + const dur = m.http_req_duration ? m.http_req_duration.values : {}; + const dropped = m.dropped_iterations ? m.dropped_iterations.values.count : 0; + return [ + `scenario=${__ENV.SCENARIO} target=${__ENV.TARGET}`, + `http_req_failed=${(failed * 100).toFixed(3)}% checks=${(checks * 100).toFixed(3)}%`, + `p95=${(dur['p(95)'] || 0).toFixed(1)}ms p99=${(dur['p(99)'] || 0).toFixed(1)}ms max=${(dur.max || 0).toFixed(1)}ms`, + `dropped_iterations=${dropped}`, + '', + ].join('\n'); +} diff --git a/tests/docker/stress/k6/ws.js b/tests/docker/stress/k6/ws.js new file mode 100644 index 0000000000..f3ab03f392 --- /dev/null +++ b/tests/docker/stress/k6/ws.js @@ -0,0 +1,45 @@ +// WebSocket stress driver for the asgi worker. Opens connections, echoes text, +// checks integrity, and paces itself so the smoke load is a realistic handshake +// rate rather than a connection flood. Raise VUS / drop SLEEP for nightly load. +import ws from 'k6/ws'; +import { check, sleep } from 'k6'; + +const TARGET = __ENV.WS_TARGET || 'ws://localhost:8463'; +const DURATION = __ENV.DURATION || '15s'; +const VUS = parseInt(__ENV.VUS || '10', 10); +const SLEEP = parseFloat(__ENV.WS_SLEEP || '0.3'); +const FAIL_BUDGET = parseFloat(__ENV.FAIL_BUDGET || '0'); + +export const options = { + scenarios: { ws: { executor: 'constant-vus', vus: VUS, duration: DURATION } }, + thresholds: { checks: [`rate>=${1 - FAIL_BUDGET}`] }, +}; + +export default function () { + const text = `hello-${__VU}-${__ITER}-${Math.random().toString(36).slice(2)}`; + const res = ws.connect(`${TARGET}/ws/echo`, {}, (socket) => { + let got = 0; + socket.on('open', () => socket.send(text)); + socket.on('message', (msg) => { + check(msg, { 'ws text echoed intact': (m) => m === text }); + got += 1; + socket.close(); + }); + socket.setTimeout(() => { + check(got, { 'ws received a reply': (g) => g > 0 }); + socket.close(); + }, 5000); + }); + check(res, { 'ws handshake 101': (r) => r && r.status === 101 }); + if (SLEEP > 0) sleep(SLEEP); +} + +export function handleSummary(data) { + const out = { stdout: '' }; + const c = data.metrics.checks ? data.metrics.checks.values.rate : 1; + out.stdout = `ws checks=${(c * 100).toFixed(3)}% sessions=${ + data.metrics.ws_sessions ? data.metrics.ws_sessions.values.count : 0}\n`; + const file = __ENV.RESULT_FILE; + if (file) out[`/results/${file}`] = JSON.stringify(data, null, 2); + return out; +} diff --git a/tests/docker/stress/nginx.conf b/tests/docker/stress/nginx.conf new file mode 100644 index 0000000000..3973b2a675 --- /dev/null +++ b/tests/docker/stress/nginx.conf @@ -0,0 +1,87 @@ +# Stress-suite nginx. Each server block is one topology target: +# :8461 proxy_pass to gthread over HTTP/1.1 +# :8463 proxy_pass to the asgi worker over HTTP/1.1 (HTTP, WS, streaming) +# :8464 terminate TLS + HTTP/2 downstream, proxy to asgi over HTTP/1.1 +# :8466 uwsgi_pass to the asgi worker (uWSGI binary protocol upstream) +# HTTP/2 and uWSGI are downstream-only: the gunicorn upstream always sees +# HTTP/1.1 on the proxy blocks and the uWSGI protocol on the uwsgi block. + +worker_processes auto; +events { worker_connections 4096; } + +http { + access_log off; + error_log /dev/stderr warn; + resolver 127.0.0.11 ipv6=off valid=10s; + + map $http_upgrade $connection_upgrade { default upgrade; '' ''; } + + upstream gunicorn_gthread { server gunicorn-gthread:8000 max_fails=0; keepalive 32; } + upstream gunicorn_asgi { server gunicorn-asgi:8000 max_fails=0; keepalive 32; } + upstream gunicorn_h2 { server gunicorn-h2:8443 max_fails=0; keepalive 32; } + upstream gunicorn_uwsgi { server gunicorn-uwsgi:8000 max_fails=0; } + + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + client_max_body_size 200m; + + # gthread behind nginx (HTTP/1.1) + server { + listen 8461; + server_name localhost; + location /health { return 200 'OK'; add_header Content-Type text/plain; } + location /stream/ { proxy_buffering off; proxy_set_header Connection ''; proxy_pass http://gunicorn_gthread/; } + location / { proxy_pass http://gunicorn_gthread; } + } + + # asgi behind nginx (HTTP/1.1) with WebSocket + streaming + server { + listen 8463; + server_name localhost; + location /health { return 200 'OK'; add_header Content-Type text/plain; } + location /ws/ { + proxy_pass http://gunicorn_asgi; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + } + location /stream/ { proxy_buffering off; proxy_set_header Connection ''; proxy_pass http://gunicorn_asgi/; } + location / { proxy_pass http://gunicorn_asgi; } + } + + # HTTP/2 (TLS) downstream, HTTP/1.1 upstream to asgi + server { + listen 8464 ssl; + http2 on; + server_name localhost; + ssl_certificate /certs/server.crt; + ssl_certificate_key /certs/server.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + http2_max_concurrent_streams 128; + location /health { return 200 'OK'; add_header Content-Type text/plain; } + location / { + proxy_pass https://gunicorn_h2; + proxy_ssl_verify off; + proxy_ssl_server_name on; + } + } + + # uwsgi_pass to the asgi worker (uWSGI protocol upstream) + server { + listen 8466; + server_name localhost; + uwsgi_read_timeout 300s; + location /health { return 200 'OK'; add_header Content-Type text/plain; } + location / { + uwsgi_pass gunicorn_uwsgi; + include uwsgi_params; + uwsgi_param HTTP_X_FORWARDED_FOR $proxy_add_x_forwarded_for; + uwsgi_param HTTP_X_REAL_IP $remote_addr; + } + } +} diff --git a/tests/docker/stress/test_faults.py b/tests/docker/stress/test_faults.py new file mode 100644 index 0000000000..be1db318d1 --- /dev/null +++ b/tests/docker/stress/test_faults.py @@ -0,0 +1,50 @@ +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. + +"""Network faults via Toxiproxy in front of gunicorn-asgi, with k6 driving load +through the toxic proxy. Gated behind GUNICORN_STRESS_HEAVY=1.""" + +import pytest + +from conftest import add_toxic, toxiproxy_reset + +pytestmark = [pytest.mark.docker, pytest.mark.stress_heavy, pytest.mark.integration] + +# k6 reaches gunicorn-asgi through the toxiproxy container inside the network. +TOXIC_TARGET_ENV = {"TARGET": "http://toxiproxy:8475"} + + +def _run(faults_stack, env): + # Reuse the asgi-direct config's checks but point k6 at the toxic proxy. + return faults_stack.run_k6("asgi-direct-h1", "constant", + extra_env={**TOXIC_TARGET_ENV, "RATE": "30", + "DURATION": "15s", **env}) + + +def test_latency_toxic(faults_stack): + toxiproxy_reset() + add_toxic("latency", "latency", {"latency": 200, "jitter": 100}) + res = _run(faults_stack, {"FAIL_BUDGET": "0.02", "MAX_P95": "60000", "MAX_P99": "60000"}) + toxiproxy_reset() + assert res.returncode == 0, res.stdout + + +def test_bandwidth_toxic(faults_stack): + toxiproxy_reset() + add_toxic("bandwidth", "bandwidth", {"rate": 512}) # KB/s + res = _run(faults_stack, {"FAIL_BUDGET": "0.05", "MAX_P95": "60000", "MAX_P99": "60000"}) + toxiproxy_reset() + assert res.returncode == 0, res.stdout + + +def test_reset_peer_toxic_is_survived(faults_stack): + toxiproxy_reset() + # Reset a fraction of connections; the worker must not crash or leak. + add_toxic("reset", "reset_peer", {"timeout": 200}, toxicity=0.3) + res = _run(faults_stack, {"FAIL_BUDGET": "0.6"}) # transport errors expected + toxiproxy_reset() + # We assert survival, not a clean run: the worker is still serving after. + assert faults_stack.tracebacks("gunicorn-asgi") == 0 + ok = faults_stack.run_k6("asgi-direct-h1", "smoke") + assert ok.returncode == 0, ok.stdout diff --git a/tests/docker/stress/test_resilience.py b/tests/docker/stress/test_resilience.py new file mode 100644 index 0000000000..7f31a69454 --- /dev/null +++ b/tests/docker/stress/test_resilience.py @@ -0,0 +1,101 @@ +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. + +"""Resilience: apply a control-plane fault while k6 load runs and assert the +worker recovers and keeps serving. Gated behind GUNICORN_STRESS_HEAVY=1.""" + +import threading +import time + +import pytest + +pytestmark = [pytest.mark.docker, pytest.mark.stress_heavy, pytest.mark.integration] + +SERVICE = "gunicorn-asgi" +CONFIG = "asgi-nginx-h1" + + +def _load(stack, results, extra_env): + results.append(stack.run_k6(CONFIG, "constant", extra_env=extra_env)) + + +def _wait_workers(stack, predicate, timeout=15): + """Poll the worker count until ``predicate(count)`` holds or time runs out. + + Worker counts settle asynchronously after a signal, and the session stack is + shared across tests, so poll rather than assert after a fixed sleep. + """ + deadline = time.time() + timeout + count = len(stack.worker_pids(SERVICE)) + while time.time() < deadline: + count = len(stack.worker_pids(SERVICE)) + if predicate(count): + return count + time.sleep(0.5) + return count + + +def _wait_stable(stack, timeout=20, samples=4, interval=0.5): + """Return the worker count once it holds steady across ``samples`` reads. + + A prior HUP reload leaves old and new workers briefly co-parented under the + master, so the raw count is inflated until reaping finishes; wait it out + before capturing a baseline. + """ + deadline = time.time() + timeout + last, streak = None, 0 + while time.time() < deadline: + count = len(stack.worker_pids(SERVICE)) + streak = streak + 1 if count == last else 1 + last = count + if streak >= samples: + return count + time.sleep(interval) + return last + + +def _run_under_load(stack, fault, fail_budget="0.05"): + """Start a constant-rate load, fire ``fault`` mid-run, return the k6 result.""" + results = [] + env = {"RATE": "40", "DURATION": "20s", "FAIL_BUDGET": fail_budget} + t = threading.Thread(target=_load, args=(stack, results, env)) + t.start() + time.sleep(6) + fault() + t.join() + assert results, "load thread produced no result" + return results[0] + + +def test_worker_kill_recovers(stack): + before = stack.worker_pids(SERVICE) + res = _run_under_load(stack, lambda: stack.kill_one_worker(SERVICE)) + # A direct-connection casualty is allowed; the budget covers it. + assert res.returncode == 0, res.stdout + after = _wait_workers(stack, lambda c: c >= len(before)) + assert after >= len(before), f"workers not respawned: {len(before)} -> {after}" + + +def test_hup_reload_under_load(stack): + res = _run_under_load(stack, lambda: stack.signal_master(SERVICE, "HUP")) + assert res.returncode == 0, res.stdout + time.sleep(3) + assert stack.worker_pids(SERVICE), "no workers after HUP reload" + + +def test_ttin_ttou_scaling_under_load(stack): + base = _wait_stable(stack) + + res = _run_under_load(stack, lambda: stack.signal_master(SERVICE, "TTIN")) + assert res.returncode == 0, res.stdout + grown = _wait_workers(stack, lambda c: c >= base + 1) + assert grown >= base + 1, f"TTIN did not add a worker: {base} -> {grown}" + + stack.signal_master(SERVICE, "TTOU") + restored = _wait_workers(stack, lambda c: c <= grown - 1) + assert restored <= grown - 1, f"TTOU did not remove a worker: {grown} -> {restored}" + + +def test_no_tracebacks_after_faults(stack): + assert stack.tracebacks(SERVICE) == 0, stack.logs(SERVICE)[-4000:] diff --git a/tests/docker/stress/test_stress.py b/tests/docker/stress/test_stress.py new file mode 100644 index 0000000000..592416af78 --- /dev/null +++ b/tests/docker/stress/test_stress.py @@ -0,0 +1,35 @@ +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. + +"""Smoke matrix: drive each representative config under a short k6 load and +assert the server stayed correct (thresholds passed, no worker traceback).""" + +import pytest + +from conftest import CONFIGS + +pytestmark = [pytest.mark.docker, pytest.mark.stress, pytest.mark.integration] + + +def _rate(summary, metric): + try: + return summary["metrics"][metric]["values"]["rate"] + except KeyError: + return None + + +@pytest.mark.parametrize("config", list(CONFIGS)) +def test_smoke(stack, config): + res = stack.run_k6(config, "smoke") + assert res.returncode == 0, f"k6 thresholds failed:\n{res.stdout}" + + checks = _rate(res.summary, "checks") + assert checks == 1.0, f"checks rate {checks}\n{res.stdout}" + + if CONFIGS[config]["kind"] == "http": + failed = _rate(res.summary, "http_req_failed") + assert failed == 0.0, f"request failure rate {failed}\n{res.stdout}" + + service = CONFIGS[config]["service"] + assert stack.tracebacks(service) == 0, stack.logs(service)[-4000:] diff --git a/tests/docker/stress/toxiproxy.json b/tests/docker/stress/toxiproxy.json new file mode 100644 index 0000000000..55e9f8cb6b --- /dev/null +++ b/tests/docker/stress/toxiproxy.json @@ -0,0 +1,8 @@ +[ + { + "name": "asgi", + "listen": "0.0.0.0:8475", + "upstream": "gunicorn-asgi:8000", + "enabled": true + } +] diff --git a/tests/docker/stress/uwsgi_params b/tests/docker/stress/uwsgi_params new file mode 100644 index 0000000000..2287801780 --- /dev/null +++ b/tests/docker/stress/uwsgi_params @@ -0,0 +1,14 @@ +uwsgi_param QUERY_STRING $query_string; +uwsgi_param REQUEST_METHOD $request_method; +uwsgi_param CONTENT_TYPE $content_type; +uwsgi_param CONTENT_LENGTH $content_length; +uwsgi_param REQUEST_URI $request_uri; +uwsgi_param PATH_INFO $document_uri; +uwsgi_param DOCUMENT_ROOT $document_root; +uwsgi_param SERVER_PROTOCOL $server_protocol; +uwsgi_param REQUEST_SCHEME $scheme; +uwsgi_param HTTPS $https if_not_empty; +uwsgi_param REMOTE_ADDR $remote_addr; +uwsgi_param REMOTE_PORT $remote_port; +uwsgi_param SERVER_PORT $server_port; +uwsgi_param SERVER_NAME $server_name;