diff --git a/README.md b/README.md index d009f33..fa4240f 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,30 @@ cd bench && node api.mjs --only=pyxle,hono --generator=oha --reps=5 cd bench && node scaling.mjs ../results/<1w>.json ../results/<16w>.json ``` +### Pyxle-only subsystem runners + +Two runners measure a Pyxle internal rather than comparing frameworks, so they +**spawn their own `pyxle serve`** to control its configuration across runs (via +`bench/server-control.mjs`). Both need the `pyxle-ssr` app built first +(`cd frameworks/pyxle-ssr && pyxle build`, or one `./bench.sh --suite=ssr`): + +```bash +# SSR worker-pool scaling — sweep --ssr-workers, hold --workers 1, report the +# 1→N scale factor per page. Same-box + relative; use oha so the generator +# isn't the ceiling that masks scaling. +cd bench && node pool-scaling-ab.mjs --ssr-workers=1,2,4,8 --generator=oha + +# Page-cache HIT vs MISS latency on the cacheable /cached route. Each pass is +# header-verified (x-pyxle-cache) before it counts. +cd bench && node cache-hitmiss.mjs --generator=oha # query-bypass MISS +cd bench && node cache-hitmiss.mjs --miss-mode=backend-off # cache-disabled MISS server +``` + +These report a **relative** measurement (scale factor; HIT-vs-MISS delta), not a +publishable cross-framework absolute — the script spawns the server locally, so +`meta.publishable` stays false unless you run `--skip-spawn` against an off-box +host with `--generator=oha`. + ## Methodology A benchmark is only useful if it's fair. This suite is built around five properties: @@ -168,6 +192,9 @@ benchmarks/ │ ├── api.mjs # API throughput suite │ ├── ssr.mjs # SSR + full-stack suite (Pyxle SSR vs Next.js) │ ├── scaling.mjs # per-core / scaling / aggregate report +│ ├── pool-scaling-ab.mjs # Pyxle SSR worker-pool scaling (A/B across --ssr-workers) +│ ├── cache-hitmiss.mjs # Pyxle page-cache HIT vs MISS latency +│ ├── server-control.mjs # spawn/health-check/teardown for the self-spawning runners │ └── verify-parity.mjs # pre-run parity + payload-size check ├── frameworks/ # One self-contained app per framework │ ├── pyxle/ pyxle-ssr/ fastapi/ django/ flask/ express/ hono/ nextjs/ diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..5a6f001 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,26 @@ +# Security Policy + +This repository contains the Pyxle benchmark suite — reproducible load tests and +the apps under test. It is not a published package, but we still want private +disclosure for anything sensitive. + +## Reporting a vulnerability + +**Please do not open a public issue for security reports.** + +Report privately through either channel: + +- **GitHub private advisory (preferred):** [Report a vulnerability](https://github.com/pyxle-dev/benchmarks/security/advisories/new) +- **Email:** **security@pyxle.dev** + +You will get an acknowledgement within **72 hours**. + +## Scope + +- Issues in the benchmark harness or fixtures that could execute untrusted code + on a machine running the suite are in scope. +- A framework vulnerability surfaced by the benchmarks belongs in that + framework's repository — please report it against [`pyxle`](https://github.com/pyxle-dev/pyxle/security/advisories/new) instead. +- The published numbers are meant to be honest and reproducible; if you can + reproduce a materially different result, please open a regular issue with your + methodology — that's a correctness report, not a security one. diff --git a/bench/cache-hitmiss.mjs b/bench/cache-hitmiss.mjs new file mode 100644 index 0000000..d3a3d47 --- /dev/null +++ b/bench/cache-hitmiss.mjs @@ -0,0 +1,296 @@ +#!/usr/bin/env node +/** + * Pyxle page-cache HIT vs MISS latency. + * + * Measures the value of the server-side page cache: a cacheable route served + * from the cache store (HIT — loader and Node SSR render both skipped) versus + * the same route rendered fresh (MISS). The delta is what the cache buys you. + * + * The page cache is active only under `pyxle serve`. The bench route is + * `/cached` (frameworks/pyxle-ssr/pages/cached.pyxl) — a 300-row table whose + * loader returns a `{"data": ..., "revalidate": N}` envelope. + * + * HIT — warm the route, then measure GET /cached → every response x-pyxle-cache: HIT. + * MISS — two honest ways, selected by --miss-mode: + * query-bypass (default): GET /cached? is non-cacheable (a query + * string disables caching for that request), so it always live-renders + * on the SAME server — no cache store involved. + * backend-off: a second server started with PYXLE_PAGE_CACHE_BACKEND=off + * renders every request fresh, including the cache-disabled bookkeeping + * a real first request pays. Higher fidelity; needs a second spawn. + * + * Each pass is verified with a header probe before measuring (fail loud if the + * HIT pass isn't actually a HIT), so a pass can't silently claim the wrong thing. + * + * License: MIT + */ + +import { writeFileSync, mkdirSync, existsSync, readFileSync } from "fs"; +import { join, dirname } from "path"; +import { fileURLToPath } from "url"; +import { execSync } from "child_process"; +import { runLoad, invalidReasons, ohaAvailable } from "./loadgen.mjs"; +import { summarizeReps } from "./stats.mjs"; +import { spawnServer, waitForHealthy, stopServer, probeCacheHeader, isLocal } from "./server-control.mjs"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const RESULTS_DIR = join(__dirname, "..", "results"); +const PYXLE_SSR_DIR = join(__dirname, "..", "frameworks", "pyxle-ssr"); +mkdirSync(RESULTS_DIR, { recursive: true }); + +// ── CLI ────────────────────────────────────────────────────────── + +function parseArgs() { + const cfg = { + host: "127.0.0.1", + port: 8011, + pyxleUrl: null, // with --skip-spawn, target a running server + pyxleBin: "pyxle", + page: "/cached", + missMode: "query-bypass", // query-bypass | backend-off + connections: 50, + duration: 10, + reps: 1, + warmupDuration: 5, + generator: "autocannon", + skipSpawn: false, + output: null, + }; + for (const arg of process.argv.slice(2)) { + if (arg.startsWith("--host=")) cfg.host = arg.split("=")[1]; + else if (arg.startsWith("--port=")) cfg.port = parseInt(arg.split("=")[1], 10); + else if (arg.startsWith("--pyxle-url=")) cfg.pyxleUrl = arg.split("=")[1]; + else if (arg.startsWith("--pyxle-bin=")) cfg.pyxleBin = arg.split("=")[1]; + else if (arg.startsWith("--page=")) cfg.page = arg.split("=")[1]; + else if (arg.startsWith("--miss-mode=")) cfg.missMode = arg.split("=")[1]; + else if (arg.startsWith("--connections=")) cfg.connections = parseInt(arg.split("=")[1], 10); + else if (arg.startsWith("--duration=")) cfg.duration = parseInt(arg.split("=")[1], 10); + else if (arg.startsWith("--reps=")) cfg.reps = Math.max(1, parseInt(arg.split("=")[1], 10)); + else if (arg.startsWith("--warmup-duration=")) cfg.warmupDuration = parseInt(arg.split("=")[1], 10); + else if (arg.startsWith("--generator=")) cfg.generator = arg.split("=")[1]; + else if (arg === "--skip-spawn") cfg.skipSpawn = true; + else if (arg.startsWith("--output=")) cfg.output = arg.split("=")[1]; + else if (arg === "--help" || arg === "-h") { + console.log(` +Pyxle page-cache HIT vs MISS latency + +Usage: node cache-hitmiss.mjs [options] + +Options: + --page=/cached Cacheable route to measure (default /cached) + --miss-mode=MODE query-bypass | backend-off (default query-bypass) + --connections=N Concurrency (default 50) + --duration=N Measured seconds per pass (default 10) + --reps=N Repetitions per pass; median (default 1) + --warmup-duration=N Warmup seconds before measuring (default 5) + --generator=NAME autocannon | oha + --port=N Port to serve/measure on (default 8011) + --pyxle-bin=PATH pyxle executable (default 'pyxle') + --skip-spawn Don't spawn; measure --pyxle-url (query-bypass only) + --pyxle-url=URL Server URL for --skip-spawn (default http://host:port) + --output=FILE Save JSON results to file + --help Show this help + +Requires the pyxle-ssr app to be built (run \`pyxle build\` in +frameworks/pyxle-ssr, or \`./bench.sh --suite=ssr\` once). +`); + process.exit(0); + } + } + if (!cfg.pyxleUrl) cfg.pyxleUrl = `http://${cfg.host}:${cfg.port}`; + return cfg; +} + +const CONFIG = parseArgs(); +const fix = (x, d = 2) => (typeof x === "number" ? x.toFixed(d) : "—"); + +// ── Measurement ────────────────────────────────────────────────── + +let INVALID = 0; +function gate(label, m) { + if (m && !m.error && !m.valid) { + INVALID++; + console.log(` \x1b[31m✗ INVALID\x1b[0m ${label}: ${invalidReasons(m).join(", ")} — excluded`); + } +} + +async function measure(url) { + const opts = { method: "GET", connections: CONFIG.connections, generator: CONFIG.generator }; + if (CONFIG.warmupDuration > 0) { + try { await runLoad({ url, ...opts, duration: CONFIG.warmupDuration }); } + catch (e) { console.warn(` ⚠ warmup failed ${url}: ${e.message}`); } + } + const reps = []; + let err = null; + for (let i = 0; i < CONFIG.reps; i++) { + try { reps.push(await runLoad({ url, ...opts, duration: CONFIG.duration })); } + catch (e) { err = e.message; } + } + return reps.length ? summarizeReps(reps) : { error: err || "no successful measurement" }; +} + +async function buildMeta() { + const os = (await import("os")).default; + const meta = { + platform: process.platform, arch: process.arch, nodeVersion: process.version, + cpus: os.cpus().length, cpuModel: os.cpus()[0]?.model || "unknown", + generator: CONFIG.generator, missMode: CONFIG.missMode, + localSpawn: !CONFIG.skipSpawn, + offBox: CONFIG.skipSpawn ? !isLocal(CONFIG.pyxleUrl) : false, + instanceType: process.env.BENCH_INSTANCE_TYPE || null, + }; + meta.publishable = meta.generator === "oha" && meta.offBox === true && CONFIG.skipSpawn; + try { meta.pythonVersion = execSync("python3 --version 2>&1", { encoding: "utf-8" }).trim(); } catch {} + const versionsFile = join(RESULTS_DIR, ".framework-versions.json"); + if (existsSync(versionsFile)) { + try { meta.frameworkVersions = JSON.parse(readFileSync(versionsFile, "utf-8")); } catch {} + } + return meta; +} + +// ── Passes ─────────────────────────────────────────────────────── + +// Measure the HIT pass on an already-healthy server at baseUrl. +async function measureHit(baseUrl) { + const url = `${baseUrl}${CONFIG.page}`; + // Populate + assert the cache is warm before measuring. + await probeCacheHeader(url); // first hit renders + stores + const status = await probeCacheHeader(url); + console.log(`\n ── HIT · ${CONFIG.page} (x-pyxle-cache: ${status ?? "—"}) ──`); + if (status !== "HIT") { + console.error(` ✗ expected x-pyxle-cache: HIT but got "${status}". Is the page cacheable and \`pyxle serve\` (not dev)? Aborting HIT pass.`); + return { error: `cache not warm (got ${status})` }; + } + const m = await measure(url); + gate(`HIT ${CONFIG.page}`, m); + if (!m.error) console.log(` req/s ${fix(m.reqPerSec, 0)} · p50 ${fix(m.latency.p50)} · p99 ${fix(m.latency.p99)} · max ${fix(m.latency.max)} ms`); + return m; +} + +// Measure the MISS pass. query-bypass appends a query string (non-cacheable); +// backend-off measures a server already started with PYXLE_PAGE_CACHE_BACKEND=off. +async function measureMiss(baseUrl, { expectHeaderNot = "HIT" } = {}) { + const path = CONFIG.missMode === "query-bypass" ? `${CONFIG.page}?nocache=1` : CONFIG.page; + const url = `${baseUrl}${path}`; + const status = await probeCacheHeader(url); + console.log(`\n ── MISS (${CONFIG.missMode}) · ${path} (x-pyxle-cache: ${status ?? "—"}) ──`); + // A genuine MISS must not be served from cache. query-bypass requests are + // non-cacheable so they carry no HIT header; backend-off renders fresh. + if (status === expectHeaderNot) { + console.error(` ✗ MISS pass returned x-pyxle-cache: ${status} — it is being served from cache. Aborting MISS pass.`); + return { error: `served from cache (${status})` }; + } + const m = await measure(url); + gate(`MISS ${path}`, m); + if (!m.error) console.log(` req/s ${fix(m.reqPerSec, 0)} · p50 ${fix(m.latency.p50)} · p99 ${fix(m.latency.p99)} · max ${fix(m.latency.max)} ms`); + return m; +} + +// ── Main ───────────────────────────────────────────────────────── + +async function main() { + console.log("\n" + "═".repeat(80)); + console.log(" PAGE-CACHE HIT vs MISS — Pyxle"); + console.log("═".repeat(80)); + console.log(` page ${CONFIG.page} · miss-mode ${CONFIG.missMode} · ${CONFIG.connections} conn · ${CONFIG.duration}s × ${CONFIG.reps} rep(s) · generator ${CONFIG.generator}`); + + if (CONFIG.generator === "oha" && !(await ohaAvailable())) { + console.error("\n --generator=oha requested but `oha` is not on PATH (brew install oha / cargo install oha).\n"); + process.exit(1); + } + if (CONFIG.missMode !== "query-bypass" && CONFIG.missMode !== "backend-off") { + console.error(`\n unknown --miss-mode "${CONFIG.missMode}" (expected query-bypass or backend-off)\n`); + process.exit(1); + } + if (CONFIG.skipSpawn && CONFIG.missMode === "backend-off") { + console.error("\n --skip-spawn can't use backend-off (it needs to spawn the cache-disabled server). Use --miss-mode=query-bypass.\n"); + process.exit(1); + } + + let hit, miss; + + if (CONFIG.skipSpawn) { + // Both passes against one running server (query-bypass MISS). + const baseUrl = CONFIG.pyxleUrl; + console.log(`\n --skip-spawn: measuring the running server at ${baseUrl}.`); + if (!(await waitForHealthy(baseUrl, { timeoutMs: 5000 }))) { + console.error(` ✗ ${baseUrl}/api/healthz not responding. Is the server up?\n`); + process.exit(1); + } + hit = await measureHit(baseUrl); + miss = await measureMiss(baseUrl); + } else { + const baseUrl = `http://${CONFIG.host}:${CONFIG.port}`; + // HIT server: default cache backend. + console.log(`\n ── Spawning Pyxle SSR (cache on) on ${baseUrl} ──`); + let child = spawnServer({ bin: CONFIG.pyxleBin, cwd: PYXLE_SSR_DIR, host: CONFIG.host, port: CONFIG.port, workers: 1, ssrWorkers: 1 }); + if (!(await waitForHealthy(baseUrl, { timeoutMs: 40000, child }))) { + console.error(" ✗ cache-on server never became healthy.\n"); + await stopServer(child); + process.exit(1); + } + hit = await measureHit(baseUrl); + if (CONFIG.missMode === "query-bypass") { + miss = await measureMiss(baseUrl); + await stopServer(child); + } else { + // backend-off: stop the cache-on server, start a cache-disabled one. + await stopServer(child); + console.log(`\n ── Spawning Pyxle SSR (PYXLE_PAGE_CACHE_BACKEND=off) on ${baseUrl} ──`); + child = spawnServer({ + bin: CONFIG.pyxleBin, cwd: PYXLE_SSR_DIR, host: CONFIG.host, port: CONFIG.port, + workers: 1, ssrWorkers: 1, env: { PYXLE_PAGE_CACHE_BACKEND: "off" }, + }); + if (!(await waitForHealthy(baseUrl, { timeoutMs: 40000, child }))) { + console.error(" ✗ cache-off server never became healthy.\n"); + await stopServer(child); + process.exit(1); + } + // With caching off, /cached carries no HIT header, so don't reject on it. + miss = await measureMiss(baseUrl, { expectHeaderNot: "HIT" }); + await stopServer(child); + } + } + + // ── Delta report ──────────────────────────────────────────────── + if (!hit.error && !miss.error && hit.reqPerSec > 0 && miss.reqPerSec > 0) { + console.log("\n" + "═".repeat(80) + "\n CACHE VALUE (HIT vs MISS)\n" + "═".repeat(80)); + console.log(` Throughput: HIT ${fix(hit.reqPerSec, 0)} req/s vs MISS ${fix(miss.reqPerSec, 0)} req/s → ${(hit.reqPerSec / miss.reqPerSec).toFixed(2)}× more`); + console.log(` Latency p50: HIT ${fix(hit.latency.p50)} ms vs MISS ${fix(miss.latency.p50)} ms → ${(miss.latency.p50 / hit.latency.p50).toFixed(2)}× faster`); + console.log(` Latency p99: HIT ${fix(hit.latency.p99)} ms vs MISS ${fix(miss.latency.p99)} ms → ${(miss.latency.p99 / hit.latency.p99).toFixed(2)}× faster`); + } else { + console.log("\n (delta omitted — a pass failed or was aborted)"); + } + + const meta = await buildMeta(); + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + const file = CONFIG.output || join(RESULTS_DIR, `cache-hitmiss-${timestamp}.json`); + writeFileSync(file, JSON.stringify({ + version: "1.0.0", + suite: "cache-hitmiss", + timestamp: new Date().toISOString(), + config: { + page: CONFIG.page, missMode: CONFIG.missMode, connections: CONFIG.connections, + duration: CONFIG.duration, reps: CONFIG.reps, warmupDuration: CONFIG.warmupDuration, + generator: CONFIG.generator, skipSpawn: CONFIG.skipSpawn, + }, + meta, + results: [ + { mode: "hit", page: CONFIG.page, metrics: hit }, + { mode: "miss", page: CONFIG.page, metrics: miss }, + ], + }, null, 2)); + console.log(`\n Results saved: ${file}`); + + if (INVALID > 0) { + console.error(`\n \x1b[31m✗ ${INVALID} cell(s) INVALID (non-2xx / errors / timeouts) — excluded, MUST NOT be published.\x1b[0m\n`); + process.exit(2); + } + if (hit.error || miss.error) { + console.error(`\n ✗ a pass failed: HIT ${hit.error || "ok"} · MISS ${miss.error || "ok"}\n`); + process.exit(2); + } + console.log(`\n ✓ Both passes valid.\n`); +} + +main().catch((err) => { console.error("Fatal:", err); process.exit(1); }); diff --git a/bench/package.json b/bench/package.json index 6a8ea04..fa17cd8 100644 --- a/bench/package.json +++ b/bench/package.json @@ -3,10 +3,12 @@ "version": "2.0.0", "private": true, "type": "module", - "description": "Load-test runners for the Pyxle benchmark suite: api.mjs (API throughput across frameworks) and ssr.mjs (SSR latency, Pyxle vs Next.js).", + "description": "Load-test runners for the Pyxle benchmark suite: api.mjs (API throughput across frameworks), ssr.mjs (SSR latency, Pyxle vs Next.js), plus Pyxle-only subsystem runners (pool-scaling-ab.mjs, cache-hitmiss.mjs).", "scripts": { "api": "node api.mjs", - "ssr": "node ssr.mjs" + "ssr": "node ssr.mjs", + "pool-scaling": "node pool-scaling-ab.mjs", + "cache-hitmiss": "node cache-hitmiss.mjs" }, "dependencies": { "autocannon": "^8.0.0" diff --git a/bench/pool-scaling-ab.mjs b/bench/pool-scaling-ab.mjs new file mode 100644 index 0000000..7c6ed6f --- /dev/null +++ b/bench/pool-scaling-ab.mjs @@ -0,0 +1,267 @@ +#!/usr/bin/env node +/** + * Pyxle SSR worker-pool scaling — A/B across pool sizes. + * + * Measures how SSR throughput and latency scale with the size of the Node SSR + * worker pool (`pyxle serve --ssr-workers N`), holding the server-process count + * fixed at `--workers 1` so the swept axis is the pool, not Uvicorn workers. + * ("ab" = the A/B pool-size comparison — NOT ApacheBench; load is generated by + * the shared loadgen, autocannon or oha, like every other suite.) + * + * For each pool size N this script spawns its own Pyxle SSR server + * (`pyxle serve --skip-build --workers 1 --ssr-workers N`), health-checks it, + * drives load through the same dynamic SSR pages the ssr suite uses, tears the + * server down, and reports — per page — the N=1 baseline, each N's req/s and + * latency, and the 1→N scale factor (the honest "does the pool actually use the + * extra workers" number). + * + * Because the script must spawn the server to set --ssr-workers, this is a + * same-box, RELATIVE measurement: the scale FACTOR is the headline, not a + * publishable absolute req/s. Use --generator=oha even same-box so the load + * generator isn't the ceiling that masks scaling. Use --skip-spawn to measure a + * single already-running server instead (e.g. an off-box host you configured). + * + * License: MIT + */ + +import { writeFileSync, mkdirSync, existsSync, readFileSync } from "fs"; +import { join, dirname } from "path"; +import { fileURLToPath } from "url"; +import { execSync } from "child_process"; +import { runLoad, invalidReasons, ohaAvailable } from "./loadgen.mjs"; +import { summarizeReps } from "./stats.mjs"; +import { spawnServer, waitForHealthy, stopServer, isLocal } from "./server-control.mjs"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const RESULTS_DIR = join(__dirname, "..", "results"); +const PYXLE_SSR_DIR = join(__dirname, "..", "frameworks", "pyxle-ssr"); +mkdirSync(RESULTS_DIR, { recursive: true }); + +// ── CLI ────────────────────────────────────────────────────────── + +function parseArgs() { + const cfg = { + host: "127.0.0.1", + port: 8011, + pyxleUrl: null, // set with --skip-spawn to target a running server + pyxleBin: "pyxle", + ssrWorkers: [1, 2, 4, 8], + pages: ["/", "/heavy", "/complex"], + connections: 100, + duration: 10, + reps: 1, + warmupDuration: 5, + generator: "autocannon", + skipSpawn: false, + output: null, + }; + for (const arg of process.argv.slice(2)) { + if (arg.startsWith("--host=")) cfg.host = arg.split("=")[1]; + else if (arg.startsWith("--port=")) cfg.port = parseInt(arg.split("=")[1], 10); + else if (arg.startsWith("--pyxle-url=")) cfg.pyxleUrl = arg.split("=")[1]; + else if (arg.startsWith("--pyxle-bin=")) cfg.pyxleBin = arg.split("=")[1]; + else if (arg.startsWith("--ssr-workers=")) cfg.ssrWorkers = arg.split("=")[1].split(",").map(Number); + else if (arg.startsWith("--pages=")) cfg.pages = arg.split("=")[1].split(","); + else if (arg.startsWith("--connections=")) cfg.connections = parseInt(arg.split("=")[1], 10); + else if (arg.startsWith("--duration=")) cfg.duration = parseInt(arg.split("=")[1], 10); + else if (arg.startsWith("--reps=")) cfg.reps = Math.max(1, parseInt(arg.split("=")[1], 10)); + else if (arg.startsWith("--warmup-duration=")) cfg.warmupDuration = parseInt(arg.split("=")[1], 10); + else if (arg.startsWith("--generator=")) cfg.generator = arg.split("=")[1]; + else if (arg === "--skip-spawn") cfg.skipSpawn = true; + else if (arg.startsWith("--output=")) cfg.output = arg.split("=")[1]; + else if (arg === "--help" || arg === "-h") { + console.log(` +Pyxle SSR worker-pool scaling (A/B across --ssr-workers) + +Usage: node pool-scaling-ab.mjs [options] + +Options: + --ssr-workers=1,2,4,8 Pool sizes to sweep (default 1,2,4,8) + --pages=/,/heavy SSR pages to measure (default /,/heavy,/complex) + --connections=N Saturating concurrency (default 100) + --duration=N Measured seconds per cell (default 10) + --reps=N Repetitions per cell; median (default 1) + --warmup-duration=N Warmup seconds before measuring(default 5) + --generator=NAME autocannon | oha (use oha so the generator isn't the ceiling) + --port=N Port to serve/measure on (default 8011) + --pyxle-bin=PATH pyxle executable (default 'pyxle' on PATH) + --skip-spawn Don't spawn; measure the running server at --pyxle-url + once (no sweep — for off-box / a pre-configured host) + --pyxle-url=URL Server URL for --skip-spawn (default http://host:port) + --output=FILE Save JSON results to file + --help Show this help + +Requires the pyxle-ssr app to be built (run \`pyxle build\` in +frameworks/pyxle-ssr, or \`./bench.sh --suite=ssr\` once). +`); + process.exit(0); + } + } + if (!cfg.pyxleUrl) cfg.pyxleUrl = `http://${cfg.host}:${cfg.port}`; + return cfg; +} + +const CONFIG = parseArgs(); +const fix = (x, d = 2) => (typeof x === "number" ? x.toFixed(d) : "—"); +const pad = (s, n = 14) => String(s).padEnd(n); + +// ── Measurement ────────────────────────────────────────────────── + +let INVALID = 0; +function gate(label, m) { + if (m && !m.error && !m.valid) { + INVALID++; + console.log(` \x1b[31m✗ INVALID\x1b[0m ${label}: ${invalidReasons(m).join(", ")} — excluded`); + } +} + +async function measureCell(url) { + const opts = { method: "GET", connections: CONFIG.connections, generator: CONFIG.generator }; + if (CONFIG.warmupDuration > 0) { + try { await runLoad({ url, ...opts, duration: CONFIG.warmupDuration }); } + catch (e) { console.warn(` ⚠ warmup failed ${url}: ${e.message}`); } + } + const reps = []; + let err = null; + for (let i = 0; i < CONFIG.reps; i++) { + try { reps.push(await runLoad({ url, ...opts, duration: CONFIG.duration })); } + catch (e) { err = e.message; } + } + return reps.length ? summarizeReps(reps) : { error: err || "no successful measurement" }; +} + +// ── Server lifecycle for one pool size ─────────────────────────── + +async function withPoolSize(n, fn) { + if (CONFIG.skipSpawn) return fn(CONFIG.pyxleUrl); + const baseUrl = `http://${CONFIG.host}:${CONFIG.port}`; + console.log(`\n ── Spawning Pyxle SSR · --workers 1 --ssr-workers ${n} on ${baseUrl} ──`); + const child = spawnServer({ + bin: CONFIG.pyxleBin, cwd: PYXLE_SSR_DIR, + host: CONFIG.host, port: CONFIG.port, workers: 1, ssrWorkers: n, + }); + try { + const healthy = await waitForHealthy(baseUrl, { timeoutMs: 40000, child }); + if (!healthy) { + console.error(` ✗ server with --ssr-workers ${n} never became healthy — skipping`); + return null; + } + return await fn(baseUrl); + } finally { + await stopServer(child); + } +} + +async function buildMeta() { + const os = (await import("os")).default; + const meta = { + platform: process.platform, arch: process.arch, nodeVersion: process.version, + cpus: os.cpus().length, cpuModel: os.cpus()[0]?.model || "unknown", + generator: CONFIG.generator, + localSpawn: !CONFIG.skipSpawn, + offBox: CONFIG.skipSpawn ? !isLocal(CONFIG.pyxleUrl) : false, + workers: 1, + ssrWorkersSwept: CONFIG.ssrWorkers, + instanceType: process.env.BENCH_INSTANCE_TYPE || null, + }; + // A self-spawned sweep is same-box and relative by construction; only a + // --skip-spawn run against an off-box host with oha yields a publishable + // absolute. The scale FACTOR is meaningful regardless. + meta.publishable = meta.generator === "oha" && meta.offBox === true && CONFIG.skipSpawn; + try { meta.pythonVersion = execSync("python3 --version 2>&1", { encoding: "utf-8" }).trim(); } catch {} + const versionsFile = join(RESULTS_DIR, ".framework-versions.json"); + if (existsSync(versionsFile)) { + try { meta.frameworkVersions = JSON.parse(readFileSync(versionsFile, "utf-8")); } catch {} + } + return meta; +} + +// ── Main ───────────────────────────────────────────────────────── + +async function main() { + console.log("\n" + "═".repeat(80)); + console.log(" SSR WORKER-POOL SCALING — Pyxle (A/B across --ssr-workers)"); + console.log("═".repeat(80)); + console.log(` Pool sizes: ${CONFIG.ssrWorkers.join(", ")} · pages: ${CONFIG.pages.join(", ")}`); + console.log(` ${CONFIG.connections} conn · ${CONFIG.duration}s × ${CONFIG.reps} rep(s) · ${CONFIG.warmupDuration}s warmup · generator ${CONFIG.generator}`); + + if (CONFIG.generator === "oha" && !(await ohaAvailable())) { + console.error("\n --generator=oha requested but `oha` is not on PATH (brew install oha / cargo install oha).\n"); + process.exit(1); + } + if (CONFIG.generator !== "oha") { + console.log("\n \x1b[33m⚠ autocannon (in-process) can become the throughput ceiling and mask"); + console.log(" pool scaling. Prefer --generator=oha for a faithful scale factor.\x1b[0m"); + } + if (CONFIG.skipSpawn) { + console.log(`\n --skip-spawn: measuring the running server at ${CONFIG.pyxleUrl} once (no sweep).`); + } + + // results[page] = [{ ssrWorkers, metrics }] + const byPage = Object.fromEntries(CONFIG.pages.map((p) => [p, []])); + const sweep = CONFIG.skipSpawn ? ["running"] : CONFIG.ssrWorkers; + + for (const n of sweep) { + const measured = await withPoolSize(n, async (baseUrl) => { + const cells = {}; + for (const page of CONFIG.pages) { + console.log(`\n ── ssr-workers=${n} · ${page} (${CONFIG.connections} conn) ──`); + const m = await measureCell(`${baseUrl}${page}`); + gate(`ssr-workers=${n} ${page}`, m); + if (!m.error) { + console.log(` req/s ${fix(m.reqPerSec, 0)} · p50 ${fix(m.latency.p50)} · p99 ${fix(m.latency.p99)} · max ${fix(m.latency.max)} ms · ${fix(m.bytesPerReq / 1024, 1)}KB/resp`); + } else { + console.log(` ${m.error}`); + } + cells[page] = m; + } + return cells; + }); + if (!measured) continue; + for (const page of CONFIG.pages) byPage[page].push({ ssrWorkers: n, metrics: measured[page] }); + } + + // ── Scale-factor report (per page, vs the N=1 baseline) ────────── + if (!CONFIG.skipSpawn) { + console.log("\n" + "═".repeat(80) + "\n SCALE FACTOR vs --ssr-workers=1 (req/s)\n" + "═".repeat(80)); + for (const page of CONFIG.pages) { + const cells = byPage[page].filter((c) => !c.metrics.error && c.metrics.valid); + const base = cells.find((c) => c.ssrWorkers === 1); + if (!base || !(base.metrics.reqPerSec > 0)) { + console.log(`\n ${page}: no valid --ssr-workers=1 baseline — scale factors omitted.`); + continue; + } + console.log(`\n ${pad(page, 12)} ${"req/s".padStart(12)} ${"p99 ms".padStart(10)} ${"scale".padStart(8)}`); + console.log(" " + "─".repeat(44)); + for (const c of cells) { + const scale = c.metrics.reqPerSec / base.metrics.reqPerSec; + console.log(` ${pad(`ssr=${c.ssrWorkers}`, 12)} ${fix(c.metrics.reqPerSec, 0).padStart(12)} ${fix(c.metrics.latency.p99).padStart(10)} ${(scale.toFixed(2) + "×").padStart(8)}`); + } + } + } + + const meta = await buildMeta(); + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + const file = CONFIG.output || join(RESULTS_DIR, `pool-scaling-${timestamp}.json`); + writeFileSync(file, JSON.stringify({ + version: "1.0.0", + suite: "pool-scaling", + timestamp: new Date().toISOString(), + config: { + ssrWorkers: CONFIG.ssrWorkers, workers: 1, pages: CONFIG.pages, + connections: CONFIG.connections, duration: CONFIG.duration, reps: CONFIG.reps, + warmupDuration: CONFIG.warmupDuration, generator: CONFIG.generator, skipSpawn: CONFIG.skipSpawn, + }, + meta, + results: byPage, + }, null, 2)); + console.log(`\n Results saved: ${file}`); + + if (INVALID > 0) { + console.error(`\n \x1b[31m✗ ${INVALID} cell(s) INVALID (non-2xx / errors / timeouts) — excluded, MUST NOT be published.\x1b[0m\n`); + process.exit(2); + } + console.log(`\n ✓ All cells valid (0 non-2xx / errors / timeouts).\n`); +} + +main().catch((err) => { console.error("Fatal:", err); process.exit(1); }); diff --git a/bench/server-control.mjs b/bench/server-control.mjs new file mode 100644 index 0000000..38c66e5 --- /dev/null +++ b/bench/server-control.mjs @@ -0,0 +1,93 @@ +/** + * Server control helpers for the self-spawning benchmark runners. + * + * Most suites (api.mjs / ssr.mjs) measure servers that bench.sh started. Two + * suites need to control the server's OWN configuration across runs — the + * SSR worker-pool size (pool-scaling-ab.mjs) and the page cache backend + * (cache-hitmiss.mjs) — so they spawn `pyxle serve` themselves. These helpers + * keep that spawn/health-check/teardown identical between them. + * + * License: MIT + */ + +import { spawn } from "child_process"; + +const isLocal = (url) => /127\.0\.0\.1|localhost|::1/.test(url); + +/** + * Spawn `pyxle serve` for a framework app. Returns the child process. + * Mirrors bench.sh's start(): production serve, --skip-build (the app must be + * built already), output suppressed. `env` is merged over the inherited + * environment so callers can set e.g. PYXLE_PAGE_CACHE_BACKEND. + */ +export function spawnServer({ + bin = "pyxle", + cwd, + host = "127.0.0.1", + port, + workers = 1, + ssrWorkers = 1, + env = {}, +}) { + const args = [ + "serve", + "--host", host, + "--port", String(port), + "--skip-build", + "--workers", String(workers), + "--ssr-workers", String(ssrWorkers), + ]; + return spawn(bin, args, { + cwd, + env: { ...process.env, ...env }, + stdio: "ignore", + }); +} + +/** + * Poll `${baseUrl}/api/healthz` until it returns 2xx or the deadline passes. + * Resolves true when healthy, false on timeout (the caller decides how loud to + * be). `child`, when given, short-circuits to false if the process exits early. + */ +export async function waitForHealthy(baseUrl, { timeoutMs = 30000, intervalMs = 300, child = null } = {}) { + const deadline = Date.now() + timeoutMs; + let exited = false; + if (child) child.once("exit", () => { exited = true; }); + while (Date.now() < deadline) { + if (exited) return false; + try { + const res = await fetch(`${baseUrl}/api/healthz`, { signal: AbortSignal.timeout(2000) }); + if (res.ok) return true; + } catch { + /* not up yet */ + } + await sleep(intervalMs); + } + return false; +} + +/** SIGTERM, wait, then SIGKILL if still alive. */ +export async function stopServer(child, { graceMs = 1500 } = {}) { + if (!child || child.exitCode !== null || child.signalCode !== null) return; + try { child.kill("SIGTERM"); } catch { /* already gone */ } + await sleep(graceMs); + if (child.exitCode === null && child.signalCode === null) { + try { child.kill("SIGKILL"); } catch { /* already gone */ } + await sleep(250); + } +} + +/** Read one response's `x-pyxle-cache` header (HIT|STALE|MISS) or null. */ +export async function probeCacheHeader(url) { + try { + const res = await fetch(url, { signal: AbortSignal.timeout(3000) }); + // Drain so the connection is reusable and the render actually ran. + await res.text(); + return res.headers.get("x-pyxle-cache"); + } catch { + return null; + } +} + +export const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +export { isLocal }; diff --git a/frameworks/pyxle-ssr/pages/cached.pyxl b/frameworks/pyxle-ssr/pages/cached.pyxl new file mode 100644 index 0000000..b6d8797 --- /dev/null +++ b/frameworks/pyxle-ssr/pages/cached.pyxl @@ -0,0 +1,71 @@ +# pages/cached.pyxl — a CACHEABLE SSR page for the cache HIT/MISS benchmark. +# Same render workload as heavy.pyxl (a 300-row table) but the loader returns the +# {"data": ..., "revalidate": N} envelope, so under `pyxle serve` the page cache +# stores the rendered HTML and serves it (HIT, skipping the React SSR render). +# cache-hitmiss.mjs measures HIT (warm /cached) vs MISS (uncached render) latency. + +_CATEGORIES = ["electronics", "tools", "parts", "clothing", "food"] +_STATUSES = ["active", "pending", "archived"] + + +def _gen_rows(n): + return [ + { + "id": i, + "name": f"Item {i}", + "sku": f"SKU-{i:05d}", + "value": f"{(i * 37 % 5000) / 10:.2f}", + "category": _CATEGORIES[i % 5], + "status": _STATUSES[i % 3], + } + for i in range(1, n + 1) + ] + + +@server +async def load(request): + # The envelope opts this page into the server page cache: the first request + # renders + stores (MISS), later requests within the window are served from + # the store (HIT) without re-running this loader or the Node SSR render. + return {"data": {"rows": _gen_rows(300)}, "revalidate": 3600} + + +import './styles/tailwind.css'; +import React from 'react'; +import { Head } from 'pyxle/client'; + +export default function CachedPage({ data }) { + return ( +
+ Cached Table · Pyxle +

Inventory (cached)

+

{data.rows.length} server-rendered rows, page-cached.

+ + + + + + + + + + + + + {data.rows.map((row) => ( + + + + + + + + + ))} + +
IDSKUNameCategoryStatusValue
{row.id}{row.sku}{row.name} + {row.category} + {row.status}{row.value}
+
+ ); +}