Add SECURITY.md

README.md

@@ -99,6 +99,30 @@ cd bench && node api.mjs --only=pyxle,hono --generator=oha --reps=5
 cd bench && node scaling.mjs ../results/<1w>.json ../results/<16w>.json
 ```
 
+### Pyxle-only subsystem runners
+
+Two runners measure a Pyxle internal rather than comparing frameworks, so they
+**spawn their own `pyxle serve`** to control its configuration across runs (via
+`bench/server-control.mjs`). Both need the `pyxle-ssr` app built first
+(`cd frameworks/pyxle-ssr && pyxle build`, or one `./bench.sh --suite=ssr`):
+
+```bash
+# SSR worker-pool scaling — sweep --ssr-workers, hold --workers 1, report the
+# 1→N scale factor per page. Same-box + relative; use oha so the generator
+# isn't the ceiling that masks scaling.
+cd bench && node pool-scaling-ab.mjs --ssr-workers=1,2,4,8 --generator=oha
+
+# Page-cache HIT vs MISS latency on the cacheable /cached route. Each pass is
+# header-verified (x-pyxle-cache) before it counts.
+cd bench && node cache-hitmiss.mjs --generator=oha            # query-bypass MISS
+cd bench && node cache-hitmiss.mjs --miss-mode=backend-off    # cache-disabled MISS server
+```
+
+These report a **relative** measurement (scale factor; HIT-vs-MISS delta), not a
+publishable cross-framework absolute — the script spawns the server locally, so
+`meta.publishable` stays false unless you run `--skip-spawn` against an off-box
+host with `--generator=oha`.
+
 ## Methodology
 
 A benchmark is only useful if it's fair. This suite is built around five properties:
@@ -168,6 +192,9 @@ benchmarks/
 │   ├── api.mjs               # API throughput suite
 │   ├── ssr.mjs               # SSR + full-stack suite (Pyxle SSR vs Next.js)
 │   ├── scaling.mjs           # per-core / scaling / aggregate report
+│   ├── pool-scaling-ab.mjs   # Pyxle SSR worker-pool scaling (A/B across --ssr-workers)
+│   ├── cache-hitmiss.mjs     # Pyxle page-cache HIT vs MISS latency
+│   ├── server-control.mjs    # spawn/health-check/teardown for the self-spawning runners
 │   └── verify-parity.mjs     # pre-run parity + payload-size check
 ├── frameworks/               # One self-contained app per framework
 │   ├── pyxle/  pyxle-ssr/  fastapi/  django/  flask/  express/  hono/  nextjs/

SECURITY.md

@@ -0,0 +1,26 @@
+# Security Policy
+
+This repository contains the Pyxle benchmark suite — reproducible load tests and
+the apps under test. It is not a published package, but we still want private
+disclosure for anything sensitive.
+
+## Reporting a vulnerability
+
+**Please do not open a public issue for security reports.**
+
+Report privately through either channel:
+
+- **GitHub private advisory (preferred):** [Report a vulnerability](https://github.com/pyxle-dev/benchmarks/security/advisories/new)
+- **Email:** **security@pyxle.dev**
+
+You will get an acknowledgement within **72 hours**.
+
+## Scope
+
+- Issues in the benchmark harness or fixtures that could execute untrusted code
+  on a machine running the suite are in scope.
+- A framework vulnerability surfaced by the benchmarks belongs in that
+  framework's repository — please report it against [`pyxle`](https://github.com/pyxle-dev/pyxle/security/advisories/new) instead.
+- The published numbers are meant to be honest and reproducible; if you can
+  reproduce a materially different result, please open a regular issue with your
+  methodology — that's a correctness report, not a security one.

bench/cache-hitmiss.mjs

@@ -0,0 +1,296 @@
+#!/usr/bin/env node
+/**
+ * Pyxle page-cache HIT vs MISS latency.
+ *
+ * Measures the value of the server-side page cache: a cacheable route served
+ * from the cache store (HIT — loader and Node SSR render both skipped) versus
+ * the same route rendered fresh (MISS). The delta is what the cache buys you.
+ *
+ * The page cache is active only under `pyxle serve`. The bench route is
+ * `/cached` (frameworks/pyxle-ssr/pages/cached.pyxl) — a 300-row table whose
+ * loader returns a `{"data": ..., "revalidate": N}` envelope.
+ *
+ *   HIT  — warm the route, then measure GET /cached → every response x-pyxle-cache: HIT.
+ *   MISS — two honest ways, selected by --miss-mode:
+ *           query-bypass (default): GET /cached?<q> is non-cacheable (a query
+ *             string disables caching for that request), so it always live-renders
+ *             on the SAME server — no cache store involved.
+ *           backend-off: a second server started with PYXLE_PAGE_CACHE_BACKEND=off
+ *             renders every request fresh, including the cache-disabled bookkeeping
+ *             a real first request pays. Higher fidelity; needs a second spawn.
+ *
+ * Each pass is verified with a header probe before measuring (fail loud if the
+ * HIT pass isn't actually a HIT), so a pass can't silently claim the wrong thing.
+ *
+ * License: MIT
+ */
+
+import { writeFileSync, mkdirSync, existsSync, readFileSync } from "fs";
+import { join, dirname } from "path";
+import { fileURLToPath } from "url";
+import { execSync } from "child_process";
+import { runLoad, invalidReasons, ohaAvailable } from "./loadgen.mjs";
+import { summarizeReps } from "./stats.mjs";
+import { spawnServer, waitForHealthy, stopServer, probeCacheHeader, isLocal } from "./server-control.mjs";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const RESULTS_DIR = join(__dirname, "..", "results");
+const PYXLE_SSR_DIR = join(__dirname, "..", "frameworks", "pyxle-ssr");
+mkdirSync(RESULTS_DIR, { recursive: true });
+
+// ── CLI ──────────────────────────────────────────────────────────
+
+function parseArgs() {
+  const cfg = {
+    host: "127.0.0.1",
+    port: 8011,
+    pyxleUrl: null,            // with --skip-spawn, target a running server
+    pyxleBin: "pyxle",
+    page: "/cached",
+    missMode: "query-bypass",  // query-bypass | backend-off
+    connections: 50,
+    duration: 10,
+    reps: 1,
+    warmupDuration: 5,
+    generator: "autocannon",
+    skipSpawn: false,
+    output: null,
+  };
+  for (const arg of process.argv.slice(2)) {
+    if (arg.startsWith("--host=")) cfg.host = arg.split("=")[1];
+    else if (arg.startsWith("--port=")) cfg.port = parseInt(arg.split("=")[1], 10);
+    else if (arg.startsWith("--pyxle-url=")) cfg.pyxleUrl = arg.split("=")[1];
+    else if (arg.startsWith("--pyxle-bin=")) cfg.pyxleBin = arg.split("=")[1];
+    else if (arg.startsWith("--page=")) cfg.page = arg.split("=")[1];
+    else if (arg.startsWith("--miss-mode=")) cfg.missMode = arg.split("=")[1];
+    else if (arg.startsWith("--connections=")) cfg.connections = parseInt(arg.split("=")[1], 10);
+    else if (arg.startsWith("--duration=")) cfg.duration = parseInt(arg.split("=")[1], 10);
+    else if (arg.startsWith("--reps=")) cfg.reps = Math.max(1, parseInt(arg.split("=")[1], 10));
+    else if (arg.startsWith("--warmup-duration=")) cfg.warmupDuration = parseInt(arg.split("=")[1], 10);
+    else if (arg.startsWith("--generator=")) cfg.generator = arg.split("=")[1];
+    else if (arg === "--skip-spawn") cfg.skipSpawn = true;
+    else if (arg.startsWith("--output=")) cfg.output = arg.split("=")[1];
+    else if (arg === "--help" || arg === "-h") {
+      console.log(`
+Pyxle page-cache HIT vs MISS latency
+
+Usage: node cache-hitmiss.mjs [options]
+
+Options:
+  --page=/cached         Cacheable route to measure       (default /cached)
+  --miss-mode=MODE       query-bypass | backend-off       (default query-bypass)
+  --connections=N        Concurrency                      (default 50)
+  --duration=N           Measured seconds per pass        (default 10)
+  --reps=N               Repetitions per pass; median     (default 1)
+  --warmup-duration=N    Warmup seconds before measuring  (default 5)
+  --generator=NAME       autocannon | oha
+  --port=N               Port to serve/measure on         (default 8011)
+  --pyxle-bin=PATH       pyxle executable                 (default 'pyxle')
+  --skip-spawn           Don't spawn; measure --pyxle-url (query-bypass only)
+  --pyxle-url=URL        Server URL for --skip-spawn      (default http://host:port)
+  --output=FILE          Save JSON results to file
+  --help                 Show this help
+
+Requires the pyxle-ssr app to be built (run \`pyxle build\` in
+frameworks/pyxle-ssr, or \`./bench.sh --suite=ssr\` once).
+`);
+      process.exit(0);
+    }
+  }
+  if (!cfg.pyxleUrl) cfg.pyxleUrl = `http://${cfg.host}:${cfg.port}`;
+  return cfg;
+}
+
+const CONFIG = parseArgs();
+const fix = (x, d = 2) => (typeof x === "number" ? x.toFixed(d) : "—");
+
+// ── Measurement ──────────────────────────────────────────────────
+
+let INVALID = 0;
+function gate(label, m) {
+  if (m && !m.error && !m.valid) {
+    INVALID++;
+    console.log(`     \x1b[31m✗ INVALID\x1b[0m ${label}: ${invalidReasons(m).join(", ")} — excluded`);
+  }
+}
+
+async function measure(url) {
+  const opts = { method: "GET", connections: CONFIG.connections, generator: CONFIG.generator };
+  if (CONFIG.warmupDuration > 0) {
+    try { await runLoad({ url, ...opts, duration: CONFIG.warmupDuration }); }
+    catch (e) { console.warn(`     ⚠ warmup failed ${url}: ${e.message}`); }
+  }
+  const reps = [];
+  let err = null;
+  for (let i = 0; i < CONFIG.reps; i++) {
+    try { reps.push(await runLoad({ url, ...opts, duration: CONFIG.duration })); }
+    catch (e) { err = e.message; }
+  }
+  return reps.length ? summarizeReps(reps) : { error: err || "no successful measurement" };
+}
+
+async function buildMeta() {
+  const os = (await import("os")).default;
+  const meta = {
+    platform: process.platform, arch: process.arch, nodeVersion: process.version,
+    cpus: os.cpus().length, cpuModel: os.cpus()[0]?.model || "unknown",
+    generator: CONFIG.generator, missMode: CONFIG.missMode,
+    localSpawn: !CONFIG.skipSpawn,
+    offBox: CONFIG.skipSpawn ? !isLocal(CONFIG.pyxleUrl) : false,
+    instanceType: process.env.BENCH_INSTANCE_TYPE || null,
+  };
+  meta.publishable = meta.generator === "oha" && meta.offBox === true && CONFIG.skipSpawn;
+  try { meta.pythonVersion = execSync("python3 --version 2>&1", { encoding: "utf-8" }).trim(); } catch {}
+  const versionsFile = join(RESULTS_DIR, ".framework-versions.json");
+  if (existsSync(versionsFile)) {
+    try { meta.frameworkVersions = JSON.parse(readFileSync(versionsFile, "utf-8")); } catch {}
+  }
+  return meta;
+}
+
+// ── Passes ───────────────────────────────────────────────────────
+
+// Measure the HIT pass on an already-healthy server at baseUrl.
+async function measureHit(baseUrl) {
+  const url = `${baseUrl}${CONFIG.page}`;
+  // Populate + assert the cache is warm before measuring.
+  await probeCacheHeader(url); // first hit renders + stores
+  const status = await probeCacheHeader(url);
+  console.log(`\n  ── HIT · ${CONFIG.page} (x-pyxle-cache: ${status ?? "—"}) ──`);
+  if (status !== "HIT") {
+    console.error(`     ✗ expected x-pyxle-cache: HIT but got "${status}". Is the page cacheable and \`pyxle serve\` (not dev)? Aborting HIT pass.`);
+    return { error: `cache not warm (got ${status})` };
+  }
+  const m = await measure(url);
+  gate(`HIT ${CONFIG.page}`, m);
+  if (!m.error) console.log(`     req/s ${fix(m.reqPerSec, 0)} · p50 ${fix(m.latency.p50)} · p99 ${fix(m.latency.p99)} · max ${fix(m.latency.max)} ms`);
+  return m;
+}
+
+// Measure the MISS pass. query-bypass appends a query string (non-cacheable);
+// backend-off measures a server already started with PYXLE_PAGE_CACHE_BACKEND=off.
+async function measureMiss(baseUrl, { expectHeaderNot = "HIT" } = {}) {
+  const path = CONFIG.missMode === "query-bypass" ? `${CONFIG.page}?nocache=1` : CONFIG.page;
+  const url = `${baseUrl}${path}`;
+  const status = await probeCacheHeader(url);
+  console.log(`\n  ── MISS (${CONFIG.missMode}) · ${path} (x-pyxle-cache: ${status ?? "—"}) ──`);
+  // A genuine MISS must not be served from cache. query-bypass requests are
+  // non-cacheable so they carry no HIT header; backend-off renders fresh.
+  if (status === expectHeaderNot) {
+    console.error(`     ✗ MISS pass returned x-pyxle-cache: ${status} — it is being served from cache. Aborting MISS pass.`);
+    return { error: `served from cache (${status})` };
+  }
+  const m = await measure(url);
+  gate(`MISS ${path}`, m);
+  if (!m.error) console.log(`     req/s ${fix(m.reqPerSec, 0)} · p50 ${fix(m.latency.p50)} · p99 ${fix(m.latency.p99)} · max ${fix(m.latency.max)} ms`);
+  return m;
+}
+
+// ── Main ─────────────────────────────────────────────────────────
+
+async function main() {
+  console.log("\n" + "═".repeat(80));
+  console.log("  PAGE-CACHE HIT vs MISS — Pyxle");
+  console.log("═".repeat(80));
+  console.log(`  page ${CONFIG.page} · miss-mode ${CONFIG.missMode} · ${CONFIG.connections} conn · ${CONFIG.duration}s × ${CONFIG.reps} rep(s) · generator ${CONFIG.generator}`);
+
+  if (CONFIG.generator === "oha" && !(await ohaAvailable())) {
+    console.error("\n  --generator=oha requested but `oha` is not on PATH (brew install oha / cargo install oha).\n");
+    process.exit(1);
+  }
+  if (CONFIG.missMode !== "query-bypass" && CONFIG.missMode !== "backend-off") {
+    console.error(`\n  unknown --miss-mode "${CONFIG.missMode}" (expected query-bypass or backend-off)\n`);
+    process.exit(1);
+  }
+  if (CONFIG.skipSpawn && CONFIG.missMode === "backend-off") {
+    console.error("\n  --skip-spawn can't use backend-off (it needs to spawn the cache-disabled server). Use --miss-mode=query-bypass.\n");
+    process.exit(1);
+  }
+
+  let hit, miss;
+
+  if (CONFIG.skipSpawn) {
+    // Both passes against one running server (query-bypass MISS).
+    const baseUrl = CONFIG.pyxleUrl;
+    console.log(`\n  --skip-spawn: measuring the running server at ${baseUrl}.`);
+    if (!(await waitForHealthy(baseUrl, { timeoutMs: 5000 }))) {
+      console.error(`  ✗ ${baseUrl}/api/healthz not responding. Is the server up?\n`);
+      process.exit(1);
+    }
+    hit = await measureHit(baseUrl);
+    miss = await measureMiss(baseUrl);
+  } else {
+    const baseUrl = `http://${CONFIG.host}:${CONFIG.port}`;
+    // HIT server: default cache backend.
+    console.log(`\n  ── Spawning Pyxle SSR (cache on) on ${baseUrl} ──`);
+    let child = spawnServer({ bin: CONFIG.pyxleBin, cwd: PYXLE_SSR_DIR, host: CONFIG.host, port: CONFIG.port, workers: 1, ssrWorkers: 1 });
+    if (!(await waitForHealthy(baseUrl, { timeoutMs: 40000, child }))) {
+      console.error("  ✗ cache-on server never became healthy.\n");
+      await stopServer(child);
+      process.exit(1);
+    }
+    hit = await measureHit(baseUrl);
+    if (CONFIG.missMode === "query-bypass") {
+      miss = await measureMiss(baseUrl);
+      await stopServer(child);
+    } else {
+      // backend-off: stop the cache-on server, start a cache-disabled one.
+      await stopServer(child);
+      console.log(`\n  ── Spawning Pyxle SSR (PYXLE_PAGE_CACHE_BACKEND=off) on ${baseUrl} ──`);
+      child = spawnServer({
+        bin: CONFIG.pyxleBin, cwd: PYXLE_SSR_DIR, host: CONFIG.host, port: CONFIG.port,
+        workers: 1, ssrWorkers: 1, env: { PYXLE_PAGE_CACHE_BACKEND: "off" },
+      });
+      if (!(await waitForHealthy(baseUrl, { timeoutMs: 40000, child }))) {
+        console.error("  ✗ cache-off server never became healthy.\n");
+        await stopServer(child);
+        process.exit(1);
+      }
+      // With caching off, /cached carries no HIT header, so don't reject on it.
+      miss = await measureMiss(baseUrl, { expectHeaderNot: "HIT" });
+      await stopServer(child);
+    }
+  }
+
+  // ── Delta report ────────────────────────────────────────────────
+  if (!hit.error && !miss.error && hit.reqPerSec > 0 && miss.reqPerSec > 0) {
+    console.log("\n" + "═".repeat(80) + "\n  CACHE VALUE (HIT vs MISS)\n" + "═".repeat(80));
+    console.log(`  Throughput: HIT ${fix(hit.reqPerSec, 0)} req/s vs MISS ${fix(miss.reqPerSec, 0)} req/s  →  ${(hit.reqPerSec / miss.reqPerSec).toFixed(2)}× more`);
+    console.log(`  Latency p50: HIT ${fix(hit.latency.p50)} ms vs MISS ${fix(miss.latency.p50)} ms  →  ${(miss.latency.p50 / hit.latency.p50).toFixed(2)}× faster`);
+    console.log(`  Latency p99: HIT ${fix(hit.latency.p99)} ms vs MISS ${fix(miss.latency.p99)} ms  →  ${(miss.latency.p99 / hit.latency.p99).toFixed(2)}× faster`);
+  } else {
+    console.log("\n  (delta omitted — a pass failed or was aborted)");
+  }
+
+  const meta = await buildMeta();
+  const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+  const file = CONFIG.output || join(RESULTS_DIR, `cache-hitmiss-${timestamp}.json`);
+  writeFileSync(file, JSON.stringify({
+    version: "1.0.0",
+    suite: "cache-hitmiss",
+    timestamp: new Date().toISOString(),
+    config: {
+      page: CONFIG.page, missMode: CONFIG.missMode, connections: CONFIG.connections,
+      duration: CONFIG.duration, reps: CONFIG.reps, warmupDuration: CONFIG.warmupDuration,
+      generator: CONFIG.generator, skipSpawn: CONFIG.skipSpawn,
+    },
+    meta,
+    results: [
+      { mode: "hit", page: CONFIG.page, metrics: hit },
+      { mode: "miss", page: CONFIG.page, metrics: miss },
+    ],
+  }, null, 2));
+  console.log(`\n  Results saved: ${file}`);
+
+  if (INVALID > 0) {
+    console.error(`\n  \x1b[31m✗ ${INVALID} cell(s) INVALID (non-2xx / errors / timeouts) — excluded, MUST NOT be published.\x1b[0m\n`);
+    process.exit(2);
+  }
+  if (hit.error || miss.error) {
+    console.error(`\n  ✗ a pass failed: HIT ${hit.error || "ok"} · MISS ${miss.error || "ok"}\n`);
+    process.exit(2);
+  }
+  console.log(`\n  ✓ Both passes valid.\n`);
+}
+
+main().catch((err) => { console.error("Fatal:", err); process.exit(1); });

bench/package.json

@@ -3,10 +3,12 @@
   "version": "2.0.0",
   "private": true,
   "type": "module",
-  "description": "Load-test runners for the Pyxle benchmark suite: api.mjs (API throughput across frameworks) and ssr.mjs (SSR latency, Pyxle vs Next.js).",
+  "description": "Load-test runners for the Pyxle benchmark suite: api.mjs (API throughput across frameworks), ssr.mjs (SSR latency, Pyxle vs Next.js), plus Pyxle-only subsystem runners (pool-scaling-ab.mjs, cache-hitmiss.mjs).",
   "scripts": {
     "api": "node api.mjs",
-    "ssr": "node ssr.mjs"
+    "ssr": "node ssr.mjs",
+    "pool-scaling": "node pool-scaling-ab.mjs",
+    "cache-hitmiss": "node cache-hitmiss.mjs"
   },
   "dependencies": {
     "autocannon": "^8.0.0"