fulmine: gateway-h3 edge on nginx with QUIC, and the HTTP/2 profiles

frameworks/fulmine/README.md

@@ -43,3 +43,8 @@ and why they are there are in the framework's
 - `tls_check`: the listener on :9000 reads `/certs-tls`, and a rotation is a new listener on the
   shared port followed by `close()` on the old one, since uWS reads the pair only when it builds
   the SSL context.
+- HTTP/2 (`baseline-h2`, `static-h2` on :8443, `baseline-h2c`, `json-h2c` on :8082): uWS has no
+  HTTP/2, so the same app is handed to `node:http2` with its defaults, one socket per worker with
+  `SO_REUSEPORT`: [node:http2](https://fulmine.sndesign.it/examples/node-http#http-2).
+- `/json` on the TLS listener has no `compression()`: json-tls never sends `Accept-Encoding`.
+  The production-stack item cache lives one second, the profile's limit.

frameworks/fulmine/app.js

@@ -85,7 +85,8 @@ app.get('/delay/:ms', (req, res) => {
     setTimeout(() => res.type('text/plain').send(String(ms)), ms);
 });
 
-const registerJsonRoute = (target, path = '/json/:count') => target.get(path, compress, (req, res) => {
+// json-tls never sends Accept-Encoding, so the TLS listener registers it without compress.
+const registerJsonRoute = (target, path = '/json/:count', ...before) => target.get(path, ...before, (req, res) => {
     const count = Math.min(Math.max(parseInt(req.params.count, 10) || 0, 0), dataset.length);
     const m = parseInt(req.query.m, 10) || 1;
     const items = new Array(count);
@@ -114,7 +115,7 @@ const registerStaticRoute = (target) => target.use('/static', express.static('/d
     fallthrough: false
 }));
 
-registerJsonRoute(app);
+registerJsonRoute(app, '/json/:count', compress);
 registerEchoRoute(app);
 registerStaticRoute(app);
 
@@ -163,7 +164,7 @@ app.get('/fortunes', async (req, res) => {
     app.get('/public/baseline', (req, res) => {
         res.type('text/plain').send(String(sumQuery(req.query)));
     });
-    registerJsonRoute(app, '/public/json/:count');
+    registerJsonRoute(app, '/public/json/:count', compress);
 
     app.get('/api/items/:id', async (req, res) => {
         const id = parseInt(req.params.id, 10);
@@ -177,7 +178,8 @@ app.get('/fortunes', async (req, res) => {
         });
         if (rows.length === 0) return res.status(404).end();
         const json = JSON.stringify(itemShape(rows[0]));
-        cache().set('crud:' + id, json, 'PX', 200);
+        // the profile allows a TTL up to one second
+        cache().set('crud:' + id, json, 'PX', 1000);
         res.set(CACHE_MISS).type('application/json').send(json);
     });
 
@@ -238,6 +240,19 @@ const hasPair = (dir) => fs.existsSync(dir + '/server.key') && fs.existsSync(dir
 
 if (hasPair('/certs')) tlsApp('/certs').listen(8081);
 
+// HTTP/2: µWS has none, so the same app goes to node:http2 with its defaults, h2 with TLS on 8443
+// and h2c on 8082. Every worker binds its own socket with SO_REUSEPORT, as the uWS listeners do.
+// https://fulmine.sndesign.it/examples/node-http#http-2
+if (require('cluster').isWorker) {
+    const http2 = require('http2');
+    const own = { reusePort: true, exclusive: true };
+    http2.createServer(app).listen({ port: 8082, ...own });
+    if (hasPair('/certs')) {
+        const pair = { key: fs.readFileSync('/certs/server.key'), cert: fs.readFileSync('/certs/server.crt') };
+        http2.createSecureServer(pair, app).listen({ port: 8443, ...own });
+    }
+}
+
 // tls_check, mounted only by validate.sh. uWS reads the pair once per SSL context, so a rotation
 // is a new listener on the shared port, then close() on the old one, which drains it.
 if (hasPair('/certs-tls')) {

frameworks/fulmine/compose.gateway-h3.yml

@@ -1,11 +1,10 @@
 # gateway-h3 - the same two services as compose.gateway.yml, reached over QUIC
-# instead. Stock Caddy already binds 8443 on tcp and udp with `protocols h1 h2
-# h3`, so the proxy config is identical and only the load generator differs.
-# The proxy to server hop stays loopback h1 either way.
+# instead. Stock nginx with http3 on udp 8443, the same edge as ./proxy-nginx
+# plus the quic listen. The proxy to server hop stays loopback h1.
 services:
   proxy:
     build:
-      context: ./proxy
+      context: ./proxy-h3
     network_mode: host
     cpuset: "${PROXY_CPUSET:-0-15,64-79}"
     ulimits:

frameworks/fulmine/compose.gateway.yml

@@ -1,7 +1,6 @@
 # gateway-64 - stock nginx (TLS + h2 on 8443) in front of the fulmine server,
 # forwarding the dynamic endpoints over loopback h1 with keepalive. The server
-# gets no certs here, so it comes up cleartext on 8080 only. Caddy stays on
-# the h3 profile only: here the edge is what saturates, and the Caddy one
+# gets no certs here, so it comes up cleartext on 8080 only. The Caddy edge
 # topped out near a third of what nginx moves on the same split.
 #
 # The CPU split is the knob worth sweeping, via PROXY_CPUSET / SERVER_CPUSET.

frameworks/fulmine/meta.json

@@ -31,7 +31,11 @@
     "gateway-64",
     "gateway-h3",
     "production-stack",
-    "fortunes"
+    "fortunes",
+    "baseline-h2",
+    "static-h2",
+    "baseline-h2c",
+    "json-h2c"
   ],
   "maintainers": [
     "nigrosimone"

frameworks/fulmine/package.json

@@ -2,7 +2,7 @@
   "name": "httparena-fulmine",
   "private": true,
   "dependencies": {
-    "fulmine.js": "^5.25.10",
+    "fulmine.js": "^5.26.0",
     "pg": "^8.13.0",
     "pg-native": "^3.8.0",
     "ejs": "^3.1.10",

frameworks/fulmine/proxy-h3/Dockerfile

@@ -0,0 +1,11 @@
+# Stock nginx, its official build has http_v3_module. Same edge as ../proxy-nginx, with QUIC.
+FROM nginx:1.27-alpine
+
+COPY nginx.conf /etc/nginx/nginx.conf
+# run by the image's entrypoint before nginx starts
+COPY workers.sh /docker-entrypoint.d/30-workers.sh
+RUN chmod +x /docker-entrypoint.d/30-workers.sh
+
+EXPOSE 8443/tcp 8443/udp
+
+CMD ["nginx", "-g", "daemon off;"]

frameworks/fulmine/proxy-h3/nginx.conf

@@ -0,0 +1,74 @@
+# fulmine gateway-h3 edge, nginx.
+#
+# nginx terminates QUIC and h3 on udp 8443 (TLS and h2 on tcp 8443), serves /static/* from disk and
+# forwards the rest to the fulmine server over loopback h1 through a keepalive pool. Caddy was here
+# before: every entry behind it lands at 56-63k at 256 connections, whatever the backend.
+
+# set to the cpuset size by workers.sh
+worker_processes 1;
+worker_rlimit_nofile 1048576;
+
+events {
+    worker_connections 65535;
+    multi_accept on;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+
+    # precompressed .gz sidecars straight from disk, nothing is compressed per request
+    gzip_static on;
+    gzip_vary on;
+
+    keepalive_timeout 300s;
+    # the generator holds its connections for the whole run, so never close one on a request count
+    keepalive_requests 1000000;
+
+    access_log off;
+    error_log /dev/stderr warn;
+
+    upstream backend {
+        server 127.0.0.1:8080;
+        keepalive 2048;
+        keepalive_requests 1000000;
+        keepalive_timeout 300s;
+    }
+
+    server {
+        # reuseport: one QUIC socket per worker, so the kernel spreads the connections
+        listen 8443 quic reuseport;
+        # Loopback, not the wildcard: validate.sh keeps the standalone container up beside the stack,
+        # and its h2 server holds *:8443 with SO_REUSEPORT. A specific address wins the lookup, the
+        # same fix as anole, plod and libuvcpp.
+        listen 127.0.0.1:8443 ssl backlog=65535 reuseport;
+        listen [::1]:8443 ssl backlog=65535 reuseport;
+        http2 on;
+        http3 on;
+        add_header Alt-Svc 'h3=":8443"; ma=86400' always;
+
+        ssl_certificate /certs/server.crt;
+        ssl_certificate_key /certs/server.key;
+        ssl_protocols TLSv1.2 TLSv1.3;
+        ssl_prefer_server_ciphers off;
+
+        # the profile asks the proxy to serve these from disk rather than forward them
+        location /static/ {
+            alias /data/static/;
+            expires off;
+            add_header Cache-Control "no-cache";
+        }
+
+        # everything else is the fulmine server
+        location / {
+            proxy_pass http://backend;
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            proxy_set_header Host $host;
+        }
+    }
+}

frameworks/fulmine/proxy-h3/workers.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+# One nginx worker per CPU of the container's cpuset: nproc honours it, `auto` may count the host.
+sed -i "s/^worker_processes .*/worker_processes $(nproc);/" /etc/nginx/nginx.conf

frameworks/fulmine/proxy-nginx/nginx.conf

@@ -1,9 +1,8 @@
 # fulmine gateway edge, nginx.
 #
 # nginx terminates TLS and h2 on 8443, serves /static/* from disk and forwards the rest to the
-# fulmine server over loopback h1 through a keepalive pool. gateway-h3 keeps the Caddy edge in
-# ./proxy: nginx does QUIC too, but on that profile every edge measures about the same, and on
-# this one the edge is what saturates, so it is the one worth changing.
+# fulmine server over loopback h1 through a keepalive pool. gateway-h3 uses the same edge with
+# QUIC, in ../proxy-h3.
 
 worker_processes auto;
 worker_rlimit_nofile 1048576;
@@ -40,7 +39,11 @@ http {
     }
 
     server {
-        listen 8443 ssl;
+        # Loopback, not the wildcard: validate.sh keeps the standalone container up beside the stack,
+        # and its h2 server holds *:8443 with SO_REUSEPORT. A specific address wins the lookup, the
+        # same fix as anole, plod and libuvcpp. reuseport is needed to bind beside that socket.
+        listen 127.0.0.1:8443 ssl reuseport;
+        listen [::1]:8443 ssl reuseport;
         http2 on;
 
         ssl_certificate /certs/server.crt;

frameworks/fulmine/proxy-production/nginx.conf

@@ -45,7 +45,11 @@ http {
     }
 
     server {
-        listen 8443 ssl;
+        # Loopback, not the wildcard: validate.sh keeps the standalone container up beside the stack,
+        # and its h2 server holds *:8443 with SO_REUSEPORT. A specific address wins the lookup, the
+        # same fix as anole, plod and libuvcpp. reuseport is needed to bind beside that socket.
+        listen 127.0.0.1:8443 ssl reuseport;
+        listen [::1]:8443 ssl reuseport;
         http2 on;
 
         ssl_certificate /certs/server.crt;