diff --git a/concurrent-work/harness/REPRODUCE-WINDOWS.md b/concurrent-work/harness/REPRODUCE-WINDOWS.md new file mode 100644 index 0000000..12bdbee --- /dev/null +++ b/concurrent-work/harness/REPRODUCE-WINDOWS.md @@ -0,0 +1,83 @@ +# Reproducing the concurrency benchmarks on Windows + +The suite was authored on macOS/Linux (see [`REPRODUCE.md`](REPRODUCE.md)). This note covers the +Windows-specific setup, what reproduces, and **one RocketRide-side result that does not reproduce +clean on Windows** — documented honestly here rather than left for a re-runner to trip over. + +Everything below uses the pinned engine (`server-v3.2.1`) and pinned LangChain (`langchain-core +0.3.86`), exactly like the Mac runs. + +## Setup (git-bash / MSYS) + +```bash +cd rocketride-bench +bash scripts/provision.sh --competitors # now supports Windows: pulls the win64 .zip engine + # + builds .venv (Scripts/ layout) + real LangChain +ROCKETRIDE_PORT=5565 bash scripts/start_engine.sh # engine.exe ai/eaas.py --port=5565 +``` + +Notes specific to Windows: +- The pinned release ships a **`win64.zip`** engine; `provision.sh` now unzips it to `./engine` + (the binary is `engine.exe`). The macOS/Linux path still downloads the `.tar.gz`. +- If the **RocketRide VS Code extension** is installed, its own engine may already hold + `:5565`. Start the benchmark engine on a **free port** and point the harness at it: + ```bash + ROCKETRIDE_PORT=5566 bash scripts/start_engine.sh + export ROCKETRIDE_URI="ws://localhost:5566" + ``` +- The runners force UTF-8 stdout (`harness/__init__.py`), so their unicode output no longer + crashes the legacy-codepage (cp1252) Windows console. +- Temp paths default to the OS temp dir (`%LOCALAPPDATA%\Temp`) instead of `/tmp`. Override the + shared params/db locations with `$ROCKETRIDE_BENCH_PARAMS` / `$BENCH_DB_DIR` if the engine and + harness resolve different temp dirs (e.g. run under different accounts). + +## What reproduces on Windows + +| Bench | Result | +|---|---| +| `lines-of-code` | ✅ 6.0× fewer lines, 3.6× fewer chars | +| `authoring-effort` | ✅ RR 0 imperative lines + validates; LC 14–17 lines, 5 hidden facts | +| `fault-isolation` | ✅ RR isolation holds 10/10; in-process LangChain dies (exit 134, 0/4) | +| `data-isolation` | ✅ RR 0 lost / 0 leaked; LangChain silently drops 44–88% of 256 | +| `concurrent-processing` (LangChain side) | ✅ `.batch` shared-conn crashes 64/64 (`sqlite3.ProgrammingError`); `.abatch`/seq serialize (~6.8s) | +| `concurrent-processing` (**RocketRide side**) | ⚠️ **does not reproduce clean — see below** | + +## Known caveat: `concurrent-processing` RocketRide cell on Windows + +On macOS/Linux the RR cell is `0 errors, 72/72 rows, status=ok`. On Windows it reports +`status=check` with ~8–12 of 72 docs raising: + +``` +sqlite3.ProgrammingError: SQLite objects created in a thread can only be used in that same thread. +``` + +**Why:** process isolation still holds (M pipes = M distinct PIDs), but the synthetic workload +node caches a **module-level sqlite connection** (`conn="module"`), which is thread-affine. On +macOS/Linux an RR pipe runs its objects on **one** worker thread, so the cached connection is +always used on its creating thread. On Windows the v3.2.1 engine dispatches successive objects to +**different** OS threads even at `threads=1`, so the cached connection is reused off-thread and +raises. The repo's own appendix "honesty cell" (`1 pipe × threadCount=4` → 31 errors) already +demonstrates this mechanism; it was simply never flagged as OS-dependent for the `threads=1` +headline path. + +RR still degrades **partially** here (the surviving pipes complete), versus LangChain's `.batch` +which loses **all 64** — but it is not the clean `0 errors` the headline reports. Note this is +specific to **thread-affine resources**: `data-isolation` uses module-level state too (a plain +`list.append`, which is GIL-safe) and reproduces `0 lost` on Windows. + +### Fix options (for the RocketRide team to decide) + +This PR ships only the OS-portability fixes above; it does **not** change the RR-side numbers or +the benchmark's claim. The threading caveat can be resolved in one of three ways: + +- **A — make the node correct (thread-local connection).** Use `threading.local()` for the + headline cell's sqlite connection so RR passes `0 errors` on every OS (Mac numbers unchanged); + keep `conn="module"` in the appendix so the trap is still shown. Concedes that RR node authors + need connection-handling care too. +- **B — disclose + scope.** Keep the naive-both idiom; on Windows report the degraded RR result + transparently and scope the `0 errors` row to macOS/Linux, noting the real fix is in the engine + (make `threads=1` mean one OS thread per pipe on Windows). Preserves the stronger claim, bounded. +- **C — engine fix.** Make the v3.2.1+ engine run one worker thread per pipe on Windows, which + restores the naive-idiom-safety headline everywhere. Not in this repo. + +A (benchmark) + C (engine) is the eventual end state. See the PR description for the full analysis. diff --git a/concurrent-work/harness/rocketride-bench/groups/scale-and-concurrency/concurrent-processing/run.py b/concurrent-work/harness/rocketride-bench/groups/scale-and-concurrency/concurrent-processing/run.py index 2928435..8a9519b 100644 --- a/concurrent-work/harness/rocketride-bench/groups/scale-and-concurrency/concurrent-processing/run.py +++ b/concurrent-work/harness/rocketride-bench/groups/scale-and-concurrency/concurrent-processing/run.py @@ -39,7 +39,9 @@ N_DOCS = 64 IO_S = 0.100 MS = [int(x) for x in os.environ.get("BENCH_MS", "8,16,64").split(",")] # env-configurable; default = upstream -DB_DIR = "/tmp/rr_bench_sqlite" +# OS temp dir (no /tmp on Windows); override with $BENCH_DB_DIR. The harness and the node both +# read/write these sqlite files, so the path must be valid on the host running both. +DB_DIR = os.environ.get("BENCH_DB_DIR", os.path.join(tempfile.gettempdir(), "rr_bench_sqlite")) PIPE = os.path.join(HERE, "pipeline.pipe") TRACE = os.path.join(HERE, "trace") @@ -205,10 +207,14 @@ async def main(): subprocess.run([sys.executable, os.path.join(REPO, "scripts", "make_diagrams.py"), PIPE, os.path.join(HERE, "canvas")]) vm = out["verdict_metrics"] - print("\nVERDICT @ top M: RR(top M) %.2fs ok | LC .batch(shared) %s (%s) | " + # Reflect the ACTUAL RR status (results.json already records rr_topM_ok) instead of a + # hardcoded "ok" — on Windows the RR cell can be `check` (see rr64["status"]), and the + # console line was the only place that masked it. + rr_status = "ok" if vm["rr_topM_ok"] else "CHECK (node errors — see rocketride[].status)" + print("\nVERDICT @ top M: RR(top M) %.2fs %s | LC .batch(shared) %s (%s) | " "LC .abatch(blocking) %.1fs | LC seq %.1fs" - % (vm["rr_topM_wall_s"], vm["lc_batch_shared_status"], vm["lc_batch_shared_error"], - vm["lc_abatch_blocking_wall_s"], vm["lc_seq_wall_s"])) + % (vm["rr_topM_wall_s"], rr_status, vm["lc_batch_shared_status"], + vm["lc_batch_shared_error"], vm["lc_abatch_blocking_wall_s"], vm["lc_seq_wall_s"])) if __name__ == "__main__": diff --git a/concurrent-work/harness/rocketride-bench/harness/__init__.py b/concurrent-work/harness/rocketride-bench/harness/__init__.py index c38b0d1..5fc5ccf 100644 --- a/concurrent-work/harness/rocketride-bench/harness/__init__.py +++ b/concurrent-work/harness/rocketride-bench/harness/__init__.py @@ -4,3 +4,14 @@ server + SDK + tracer). It never reimplements engine behavior. See the repo README and NOTICE for the "what's RocketRide vs what's ours" breakdown. """ +# Force UTF-8 on stdout/stderr so the runners' unicode (→, ×, …) prints on any console. +# Windows defaults to a legacy codepage (cp1252), where a bare `print("… → …")` raises +# UnicodeEncodeError and aborts a run mid-benchmark. Guarded + idempotent; no-op on POSIX, +# which is already UTF-8. reconfigure() exists on TextIOWrapper (Py3.7+). +import sys as _sys + +for _stream in (_sys.stdout, _sys.stderr): + try: + _stream.reconfigure(encoding="utf-8") + except (AttributeError, ValueError): + pass diff --git a/concurrent-work/harness/rocketride-bench/harness/config.py b/concurrent-work/harness/rocketride-bench/harness/config.py index 6e9e0f3..c577c4e 100644 --- a/concurrent-work/harness/rocketride-bench/harness/config.py +++ b/concurrent-work/harness/rocketride-bench/harness/config.py @@ -13,13 +13,16 @@ import platform import subprocess import sys +import tempfile HARNESS_DIR = os.path.dirname(os.path.abspath(__file__)) REPO_DIR = os.path.dirname(HARNESS_DIR) -# Engine: env-first, else the provisioned ./engine inside the repo. +# Engine: env-first, else the provisioned ./engine inside the repo. The prebuilt binary is +# `engine.exe` on Windows and `engine` elsewhere — pick the right name so provenance() and +# engine_version() find it instead of silently recording "unknown". ENGINE_DIR = os.environ.get("ENGINE_DIR") or os.path.join(REPO_DIR, "engine") -ENGINE = os.path.join(ENGINE_DIR, "engine") +ENGINE = os.path.join(ENGINE_DIR, "engine.exe" if os.name == "nt" else "engine") # Direct-connect server (the headline product path). URI = os.environ.get("ROCKETRIDE_URI", "ws://localhost:5565") @@ -30,7 +33,11 @@ RESULTS_DIR = os.path.join(REPO_DIR, "results") DATA_DIR = os.path.join(REPO_DIR, "data") NODES_DIR = os.path.join(REPO_DIR, "nodes") -BENCH_PARAMS = os.environ.get("ROCKETRIDE_BENCH_PARAMS", "/tmp/rr_bench_params.json") +# Params file the harness writes and the workload node reads. Default to the OS temp dir so it +# is writable on Windows too (there is no /tmp); keep the node's fallback (nodes/workload/ +# IInstance.py) in sync. Override with $ROCKETRIDE_BENCH_PARAMS to pin an explicit shared path. +BENCH_PARAMS = os.environ.get("ROCKETRIDE_BENCH_PARAMS", + os.path.join(tempfile.gettempdir(), "rr_bench_params.json")) def engine_present(): diff --git a/concurrent-work/harness/rocketride-bench/nodes/workload/IInstance.py b/concurrent-work/harness/rocketride-bench/nodes/workload/IInstance.py index 920f0b4..cce8e1e 100644 --- a/concurrent-work/harness/rocketride-bench/nodes/workload/IInstance.py +++ b/concurrent-work/harness/rocketride-bench/nodes/workload/IInstance.py @@ -30,12 +30,18 @@ import json import os import sys +import tempfile import time import threading from rocketlib import IInstanceBase _PARAMS = None +# Cross-platform fallbacks (no /tmp on Windows). The harness normally passes explicit paths via +# $ROCKETRIDE_BENCH_PARAMS and the `db` param, so these only bite the local-binary floor path; +# kept in sync with harness/config.py so both sides resolve the same file when unset. +_DEFAULT_PARAMS = os.path.join(tempfile.gettempdir(), "rr_bench_params.json") +_DEFAULT_DB = os.path.join(tempfile.gettempdir(), "rr_bench_sqlite", "%d.db") def _params(): @@ -43,7 +49,7 @@ def _params(): if _PARAMS is not None: return _PARAMS path = os.environ.get("ROCKETRIDE_BENCH_PARAMS") or os.environ.get("BENCH_PARAMS") \ - or "/tmp/rr_bench_params.json" + or _DEFAULT_PARAMS p = {} try: with open(path) as f: @@ -57,7 +63,7 @@ def _params(): p.setdefault("url", os.environ.get("BENCH_URL", "http://127.0.0.1:8799/")) p.setdefault("label", os.environ.get("BENCH_LABEL", "")) # scale-and-concurrency params - p.setdefault("db", os.environ.get("BENCH_DB", "/tmp/rr_bench_sqlite/%d.db")) + p.setdefault("db", os.environ.get("BENCH_DB", _DEFAULT_DB)) p.setdefault("io_ms", float(os.environ.get("BENCH_IO_MS", "0") or 0)) p.setdefault("pdf", os.environ.get("BENCH_PDF", "")) p.setdefault("conn", os.environ.get("BENCH_CONN", "module")) # "module" | "per_call" diff --git a/concurrent-work/harness/rocketride-bench/scripts/provision.sh b/concurrent-work/harness/rocketride-bench/scripts/provision.sh index b27d094..00838b5 100644 --- a/concurrent-work/harness/rocketride-bench/scripts/provision.sh +++ b/concurrent-work/harness/rocketride-bench/scripts/provision.sh @@ -7,46 +7,60 @@ REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)" RR_ENGINE_VERSION="${RR_ENGINE_VERSION:-3.2.1}" ENGINE_DIR="${ENGINE_DIR:-$REPO_DIR/engine}" +# Engine binary name differs on Windows (git-bash/MSYS): engine.exe vs engine. +case "$(uname -s)" in MINGW*|MSYS*|CYGWIN*) ENGINE_BIN=engine.exe ;; *) ENGINE_BIN=engine ;; esac + # 1) Engine: use an existing one, else download the prebuilt for this OS/arch. -if [ -x "$ENGINE_DIR/engine" ]; then - echo "engine present: $ENGINE_DIR/engine" +if [ -x "$ENGINE_DIR/$ENGINE_BIN" ]; then + echo "engine present: $ENGINE_DIR/$ENGINE_BIN" else os="$(uname -s)"; arch="$(uname -m)" + ext=tar.gz case "$os/$arch" in Darwin/arm64) plat=darwin-arm64 ;; Darwin/x86_64) plat=darwin-x64 ;; Linux/*) plat=linux-x64 ;; + # Windows via git-bash/MSYS/Cygwin: the release ships a .zip (not .tar.gz) that extracts + # engine.exe at the archive root (no leading component to strip). + MINGW*/*|MSYS*/*|CYGWIN*/*) plat=win64; ext=zip ;; *) echo "unsupported $os/$arch; use Docker (ghcr.io/rocketride-org/rocketride-engine)"; exit 1 ;; esac - asset="rocketride-server-v${RR_ENGINE_VERSION}-${plat}.tar.gz" + asset="rocketride-server-v${RR_ENGINE_VERSION}-${plat}.${ext}" url="https://github.com/rocketride-org/rocketride-server/releases/download/server-v${RR_ENGINE_VERSION}/${asset}" echo "downloading $url" mkdir -p "$ENGINE_DIR" - curl -fL "$url" -o "/tmp/$asset" - tar -xzf "/tmp/$asset" -C "$ENGINE_DIR" --strip-components=1 + tmp_asset="${TMPDIR:-/tmp}/$asset" + curl -fL "$url" -o "$tmp_asset" + if [ "$ext" = "zip" ]; then + unzip -oq "$tmp_asset" -d "$ENGINE_DIR" # win64.zip: engine.exe at the root + else + tar -xzf "$tmp_asset" -C "$ENGINE_DIR" --strip-components=1 + fi echo "extracted engine -> $ENGINE_DIR" fi -( cd "$ENGINE_DIR" && ./engine --version 2>&1 | head -1 ) || true +( cd "$ENGINE_DIR" && "./$ENGINE_BIN" --version 2>&1 | head -1 ) || true # NOTE: native prebuilts exist for darwin-arm64/darwin-x64/linux-x64/win64. On Apple Silicon the # Docker image (linux-x64) runs under emulation — for fair Mac numbers use the darwin-arm64 # prebuilt; for fair Docker numbers run on a linux-x64 host and containerize the competitors too. -# 2) Harness venv. -if [ ! -x "$REPO_DIR/.venv/bin/python" ]; then +# 2) Harness venv. venv lays python under Scripts/ on Windows, bin/ elsewhere. +case "$(uname -s)" in MINGW*|MSYS*|CYGWIN*) VENV_PY="$REPO_DIR/.venv/Scripts/python.exe" ;; + *) VENV_PY="$REPO_DIR/.venv/bin/python" ;; esac +if [ ! -x "$VENV_PY" ]; then echo "creating venv" python3 -m venv "$REPO_DIR/.venv" fi -"$REPO_DIR/.venv/bin/python" -m pip install --quiet --upgrade pip -"$REPO_DIR/.venv/bin/python" -m pip install --quiet -r "$REPO_DIR/requirements.txt" -echo "venv ready: $REPO_DIR/.venv ($("$REPO_DIR/.venv/bin/python" -c 'import rocketride; print("rocketride", rocketride.__version__)'))" +"$VENV_PY" -m pip install --quiet --upgrade pip +"$VENV_PY" -m pip install --quiet -r "$REPO_DIR/requirements.txt" +echo "venv ready: $REPO_DIR/.venv ($("$VENV_PY" -c 'import rocketride; print("rocketride", rocketride.__version__)'))" # 3) Optional: the REAL LangChain competitor baseline for the Tier-1 head-to-heads (no infra, no # creds — the model is a fixed-latency mock). `make provision-competitors` does the same thing. if [ "${1:-}" = "--competitors" ]; then echo "installing competitor baselines (real LangChain)" - "$REPO_DIR/.venv/bin/python" -m pip install --quiet -r "$REPO_DIR/requirements-competitors.txt" - echo "competitors ready: $("$REPO_DIR/.venv/bin/python" -c 'import langchain_core; print("langchain-core", langchain_core.__version__)')" + "$VENV_PY" -m pip install --quiet -r "$REPO_DIR/requirements-competitors.txt" + echo "competitors ready: $("$VENV_PY" -c 'import langchain_core; print("langchain-core", langchain_core.__version__)')" fi echo diff --git a/concurrent-work/harness/rocketride-bench/scripts/start_engine.sh b/concurrent-work/harness/rocketride-bench/scripts/start_engine.sh index 66957cf..e93e740 100644 --- a/concurrent-work/harness/rocketride-bench/scripts/start_engine.sh +++ b/concurrent-work/harness/rocketride-bench/scripts/start_engine.sh @@ -10,8 +10,11 @@ PORT="${ROCKETRIDE_PORT:-5565}" LOG="${ENGINE_LOG:-$REPO_DIR/results/engine.log}" PIDFILE="$REPO_DIR/results/engine.pid" -if [ ! -x "$ENGINE_DIR/engine" ]; then - echo "RocketRide runtime not found at $ENGINE_DIR/engine" >&2 +# Engine binary name is engine.exe on Windows (git-bash/MSYS), engine elsewhere. +case "$(uname -s)" in MINGW*|MSYS*|CYGWIN*) ENGINE_BIN=engine.exe ;; *) ENGINE_BIN=engine ;; esac + +if [ ! -x "$ENGINE_DIR/$ENGINE_BIN" ]; then + echo "RocketRide runtime not found at $ENGINE_DIR/$ENGINE_BIN" >&2 echo " set \$ENGINE_DIR or run scripts/provision.sh to download the pinned prebuilt." >&2 exit 1 fi @@ -28,8 +31,8 @@ done mkdir -p "$(dirname "$LOG")" cd "$ENGINE_DIR" -echo "starting: $ENGINE_DIR/engine ai/eaas.py --host=0.0.0.0 (port $PORT)" -nohup ./engine ai/eaas.py --host=0.0.0.0 >"$LOG" 2>&1 & +echo "starting: $ENGINE_DIR/$ENGINE_BIN ai/eaas.py --host=0.0.0.0 --port=$PORT" +nohup "./$ENGINE_BIN" ai/eaas.py --host=0.0.0.0 --port="$PORT" >"$LOG" 2>&1 & echo $! > "$PIDFILE" # Readiness = the HTTP server answers at all. /ping returns 401 without auth (that's still a @@ -38,7 +41,7 @@ for _ in $(seq 1 60); do code="$(curl -s -o /dev/null -w '%{http_code}' "http://localhost:$PORT/ping" 2>/dev/null || echo 000)" if [ "$code" != "000" ]; then echo "engine healthy on :$PORT (HTTP $code, pid $(cat "$PIDFILE"))" - ./engine --version 2>&1 | head -1 + "./$ENGINE_BIN" --version 2>&1 | head -1 exit 0 fi sleep 0.5 diff --git a/concurrent-work/harness/run_isolated_windows.py b/concurrent-work/harness/run_isolated_windows.py new file mode 100644 index 0000000..64d5be0 --- /dev/null +++ b/concurrent-work/harness/run_isolated_windows.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Windows-native 10x runner for the concurrency benchmarks — the counterpart to run_isolated.sh. + +run_isolated.sh can't drive a Windows run: it writes into ../runs/ (the committed macOS tree), +hardcodes port 5565 (often held by the VS Code extension's engine), and uses lsof / .venv/bin/python +/ --host=0.0.0.0. This driver mirrors its logic on Windows: + + - engine lifecycle on a FREE port (default 5566) via psutil (find listener -> terminate tree -> + relaunch engine.exe ai/eaas.py --host=127.0.0.1 --port=PORT -> poll /ping -> record pid), + - .venv/Scripts/python.exe for the benches, ROCKETRIDE_URI pointed at the chosen port, + - fault-isolation xREPS back-to-back (no restart), authoring-effort x1 (static), + concurrent-processing xREPS @ M={8,16} and data-isolation xREPS @ M=32, each warm-pool rep on a + freshly-restarted+primed engine (retry up to MAX_ATTEMPTS), + - outputs into ../runs-windows//run-NN/ (results.json + captured run.log); trace/ kept for + run-01 only (gzip it afterwards to match the committed convention). + +Env: REPS (default 10), MAX_ATTEMPTS (5), ROCKETRIDE_PORT (5566), RESTART=1 (set 0 to reuse a single +warm engine — the disclosed fallback; correctness outcomes are restart-independent, only warm-pool +timing hygiene differs). + +Run: ./.venv/Scripts/python.exe ../run_isolated_windows.py + (from rocketride-bench/, with the engine provisioned + competitors installed) +""" +import json +import os +import shutil +import subprocess +import sys +import time +import urllib.request + +import psutil + +HERE = os.path.dirname(os.path.abspath(__file__)) # concurrent-work/harness +BR = os.path.join(HERE, "rocketride-bench") +RUNS_WIN = os.path.join(HERE, "..", "runs-windows") +ENGINE_DIR = os.environ.get("ENGINE_DIR") or os.path.join(BR, "engine") +ENGINE_EXE = os.path.join(ENGINE_DIR, "engine.exe") +PY = os.environ.get("BENCH_PY") or os.path.join(BR, ".venv", "Scripts", "python.exe") +PORT = int(os.environ.get("ROCKETRIDE_PORT", "5566")) +URI = "ws://localhost:%d" % PORT +REPS = int(os.environ.get("REPS", "10")) +MAX_ATTEMPTS = int(os.environ.get("MAX_ATTEMPTS", "5")) +DO_RESTART = os.environ.get("RESTART", "1") != "0" +PARAMS = os.path.join(BR, "results", "bench_params.json") # explicit shared path (engine + harness) +ENGINE_LOG = os.path.join(BR, "results", "engine_win.log") + +CRASH = "groups/robustness-and-isolation/fault-isolation" +PICK = "groups/scale-and-concurrency/concurrent-processing" +INST = "groups/scale-and-concurrency/data-isolation" +AUTH = "groups/scale-and-concurrency/authoring-effort" + +# One environment shared by the engine (so task subprocesses inherit the params path + URI) and the +# bench runners (so config.URI / the node's params path agree). +BASE_ENV = dict(os.environ) +BASE_ENV["ROCKETRIDE_URI"] = URI +BASE_ENV["ROCKETRIDE_PORT"] = str(PORT) +BASE_ENV["ROCKETRIDE_BENCH_PARAMS"] = PARAMS +BASE_ENV["ENGINE_DIR"] = ENGINE_DIR +BASE_ENV["PYTHONIOENCODING"] = "utf-8" + +_engine_proc = None + + +def _listener_pid(port): + for c in psutil.net_connections(kind="inet"): + if c.laddr and c.laddr.port == port and c.status == "LISTEN": + return c.pid + return None + + +def stop_engine(): + global _engine_proc + pid = _listener_pid(PORT) + if pid: + try: + p = psutil.Process(pid) + for k in p.children(recursive=True): + try: + k.terminate() + except psutil.Error: + pass + p.terminate() + psutil.wait_procs([p], timeout=8) + except psutil.Error: + pass + _engine_proc = None + + +def _healthy(): + try: + urllib.request.urlopen("http://localhost:%d/ping" % PORT, timeout=2) + return True + except urllib.error.HTTPError: # 401 etc. == a live server + return True + except Exception: # connection refused == not up yet + return False + + +def start_engine(timeout_s=90): + global _engine_proc + os.makedirs(os.path.dirname(ENGINE_LOG), exist_ok=True) + logf = open(ENGINE_LOG, "ab") + _engine_proc = subprocess.Popen( + [ENGINE_EXE, "ai/eaas.py", "--host=127.0.0.1", "--port=%d" % PORT], + cwd=ENGINE_DIR, stdout=logf, stderr=logf, env=BASE_ENV) + t0 = time.time() + while time.time() - t0 < timeout_s: + if _healthy(): + pid = _listener_pid(PORT) + if pid: + with open(os.path.join(BR, "results", "engine.pid"), "w") as f: + f.write(str(pid)) + return True + time.sleep(1.0) + return False + + +def restart(): + stop_engine() + time.sleep(2) + if not start_engine(): + print(" [engine did not come healthy after restart]", flush=True) + time.sleep(3) + + +def prime(): + """Wake the engine's pipe machinery with one quick single-pipe run.""" + try: + subprocess.run([PY, os.path.join(BR, CRASH, "run.py")], + cwd=BR, env=BASE_ENV, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=90) + except subprocess.SubprocessError: + pass + + +def copy_out(rel, out_dir, keep_trace): + src = os.path.join(BR, rel, "results.json") + if not os.path.isfile(src): + return False + shutil.copy(src, out_dir) + tr = os.path.join(BR, rel, "trace") + if keep_trace and os.path.isdir(tr): + shutil.copytree(tr, os.path.join(out_dir, "trace"), dirs_exist_ok=True) + return True + + +def run_bench(rel, key, run_name, env_extra, restart_first, keep_trace, flat=False): + out = os.path.join(RUNS_WIN, key) if flat else os.path.join(RUNS_WIN, key, run_name) + os.makedirs(out, exist_ok=True) + env = dict(BASE_ENV) + env.update(env_extra or {}) + for attempt in range(1, MAX_ATTEMPTS + 1): + if restart_first and DO_RESTART: + restart() + prime() + with open(os.path.join(out, "run.log"), "w", encoding="utf-8") as log: + try: + rc = subprocess.run([PY, os.path.join(BR, rel, "run.py")], cwd=BR, env=env, + stdout=log, stderr=subprocess.STDOUT, timeout=360).returncode + except subprocess.TimeoutExpired: + rc = -1 + log.write("\n[TIMEOUT after 360s]\n") + if rc == 0 and copy_out(rel, out, keep_trace): + print(" OK %s %s (attempt %d)" % (key, run_name, attempt), flush=True) + return True + print(" ...retry %s %s (attempt %d, rc=%s)" % (key, run_name, attempt, rc), flush=True) + print(" FAIL %s %s after %d attempts" % (key, run_name, MAX_ATTEMPTS), flush=True) + return False + + +def main(): + for tool in (ENGINE_EXE, PY): + if not os.path.exists(tool): + sys.exit("missing %s — provision the engine + venv first" % tool) + os.makedirs(RUNS_WIN, exist_ok=True) + + print("=== starting engine on :%d ===" % PORT, flush=True) + stop_engine() # clear any stale listener on our port first + time.sleep(1) + if not start_engine(): + sys.exit("engine did not become healthy on :%d" % PORT) + + print("=== fault-isolation x%d (back-to-back, no restart) ===" % REPS, flush=True) + for r in range(1, REPS + 1): + run_bench(CRASH, "fault-isolation", "run-%02d" % r, {}, False, False) + + print("=== authoring-effort x1 (static) ===", flush=True) + run_bench(AUTH, "authoring-effort", "", {}, False, False, flat=True) + + print("=== concurrent-processing x%d @ M={8,16} (restart+prime+retry) ===" % REPS, flush=True) + for r in range(1, REPS + 1): + keep = (r == 1) # trace kept for run-01 only (convention) + run_bench(PICK, "concurrent-processing", "run-%02d" % r, {"BENCH_MS": "8,16"}, True, keep) + + print("=== data-isolation x%d @ M=32 (restart+prime+retry) ===" % REPS, flush=True) + for r in range(1, REPS + 1): + keep = (r == 1) + run_bench(INST, "data-isolation", "run-%02d" % r, {"BENCH_M": "32"}, True, keep) + + stop_engine() + print("=== 10x RUN DONE -> %s ===" % os.path.abspath(RUNS_WIN), flush=True) + + +if __name__ == "__main__": + main() diff --git a/concurrent-work/runs-windows/COMMANDS.md b/concurrent-work/runs-windows/COMMANDS.md new file mode 100644 index 0000000..d99bfee --- /dev/null +++ b/concurrent-work/runs-windows/COMMANDS.md @@ -0,0 +1,66 @@ +# Exact commands that produced this data (Windows) + +All paths relative to `concurrent-work/harness/rocketride-bench/` unless noted. Run from git-bash / MSYS. + +## 1. Provision (Windows) + +The pinned `server-v3.2.1` release ships a `win64.zip` engine (the upstream `provision.sh` now wires it +up — see the PR). It was fetched + extracted to `./engine/engine.exe`, and the venv built with the real +LangChain competitor: + +```bash +bash scripts/provision.sh --competitors # win64 engine + .venv (Scripts/) + langchain-core 0.3.86 +``` +- Engine asset: `https://github.com/rocketride-org/rocketride-server/releases/download/server-v3.2.1/rocketride-server-v3.2.1-win64.zip` +- Engine sha256: `b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e` +- Pins: engine `3.2.1.30`, `langchain-core==0.3.86`, `rocketride==1.2.0`. + +The benchmark-only workload node is installed into the engine bundle (as `start_engine.sh` does): +```bash +cp -R nodes/workload engine/nodes/ +``` + +## 2. Run all 10× reps + +```bash +# from concurrent-work/harness/rocketride-bench/ +./.venv/Scripts/python.exe ../run_isolated_windows.py +``` + +`run_isolated_windows.py` (committed under `harness/`) mirrors `run_isolated.sh` on Windows: +- engine lifecycle on **port 5566** via psutil (the VS Code extension's engine usually holds 5565); +- `.venv/Scripts/python.exe`; `ROCKETRIDE_URI=ws://localhost:5566`; UTF-8 stdout; +- `fault-isolation` ×10 (no restart), `authoring-effort` ×1 (static), + `concurrent-processing` ×10 @ `BENCH_MS=8,16`, `data-isolation` ×10 @ `BENCH_M=32`, + each warm-pool rep on a freshly restarted + primed engine (retry ≤ 5); +- outputs → `../runs-windows//run-NN/{results.json,run.log}`. + +Env knobs: `REPS` (default 10), `MAX_ATTEMPTS` (5), `ROCKETRIDE_PORT` (5566), +`RESTART=0` (disclosed fallback: reuse a single warm engine — correctness is restart-independent). + +## 3. Post-processing + +Traces (`run.py` writes `trace/` every rep) were gzipped for **run-01 only** and dropped elsewhere, per +the `../runs/` convention: +```bash +gzip -f runs-windows/concurrent-processing/run-01/trace/*.jsonl +gzip -f runs-windows/data-isolation/run-01/trace/*.jsonl +``` + +## 4. lines-of-code (OS-independent) + +```bash +python lines-of-code/measure.py # from repo root; regenerates lines-of-code/results.json +``` +This only counts committed artifacts, so Windows output is identical to the committed +`lines-of-code/results.json` (6.0× fewer lines, 3.6× fewer chars). Console captured in +`lines-of-code.run.log`. + +## Single-bench invocations (what the orchestrator runs per rep) + +```bash +ROCKETRIDE_URI=ws://localhost:5566 BENCH_MS=8,16 ./.venv/Scripts/python.exe groups/scale-and-concurrency/concurrent-processing/run.py +ROCKETRIDE_URI=ws://localhost:5566 BENCH_M=32 ./.venv/Scripts/python.exe groups/scale-and-concurrency/data-isolation/run.py +ROCKETRIDE_URI=ws://localhost:5566 ./.venv/Scripts/python.exe groups/robustness-and-isolation/fault-isolation/run.py +ROCKETRIDE_URI=ws://localhost:5566 ./.venv/Scripts/python.exe groups/scale-and-concurrency/authoring-effort/run.py +``` diff --git a/concurrent-work/runs-windows/README.md b/concurrent-work/runs-windows/README.md new file mode 100644 index 0000000..e36d261 --- /dev/null +++ b/concurrent-work/runs-windows/README.md @@ -0,0 +1,63 @@ +# concurrent-work — Windows reproduction (10× reps) + +This is a **full re-run of the concurrency suite on Windows 11**, laid out exactly like the macOS +`../runs/` tree so results are directly comparable. It exists so a reviewer (human or agent) has the +complete input→output evidence in one place — including the **one RocketRide-side result that does not +reproduce clean on Windows** (`concurrent-processing`), which is captured here in full rather than +hidden. + +- **Inputs** are the committed benchmark data (`../harness/rocketride-bench/data/`, + `groups/**/demo-data/`) and the generated `.pipe` configs — unchanged from upstream. +- **Outputs** are per-run `results.json` (metrics + provenance) + full console `run.log`, with a native + `trace/` (gzipped) kept for `run-01` of the two trace-bearing benches, matching the `../runs/` + convention. +- Produced by [`../harness/run_isolated_windows.py`](../harness/run_isolated_windows.py) — the + Windows counterpart to `run_isolated.sh`. See [`COMMANDS.md`](COMMANDS.md) for exact invocations, and + [`../harness/REPRODUCE-WINDOWS.md`](../harness/REPRODUCE-WINDOWS.md) for setup + the caveat's root cause. + +## Provenance + +| | | +|---|---| +| OS | Windows-11 (10.0.26200) | +| CPU | Intel64 Family 6 Model 186 (Raptor Lake), 10 physical / 12 logical cores | +| RAM | 15.55 GiB | +| Python (harness) | 3.12.3 | +| Engine | `3.2.1.30` (hash `114509c6`, stamped 2026-05-29), the pinned **`win64`** prebuilt | +| Engine sha256 | `b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e` | +| LangChain | `langchain-core 0.3.86` (real, out-of-process — the same pin as upstream) | +| Reps | 10× each for crash / pick / instance; authoring is static (×1) | + +Full provenance is embedded in **every** `results.json` (`provenance` + `langchain_provenance` blocks). + +## Scorecard — Windows, stock vs stock (10 reps) + +| Benchmark | Stock LangChain (default idiom) | Stock RocketRide (default) | +|---|---|---| +| [concurrent-processing](concurrent-processing/REPORT.md) | `.batch` (shared conn) **CRASHES 0/64** (`sqlite3.ProgrammingError`) in **10/10**; `.abatch`/seq serialize **~6.7 s** | ⚠️ **`status=check` in 10/10** — partial sqlite thread-affinity errors (2–18 docs/rep). Process isolation holds; see caveat below | +| [fault-isolation](fault-isolation/REPORT.md) | in-process `.abatch` **loses ALL 0/4** to one crash (exit 134) in 10/10 | ✅ **survives 10/10** — only the crashing run dies, server + siblings live | +| [data-isolation](data-isolation/REPORT.md) | one shared dict, 32 workers → **silently loses 100–225 of 256** (39–88%) | ✅ **0 lost / 0 leaked, 10/10** — each pipe its own process | +| [authoring-effort](authoring-effort/REPORT.md) | **14–17** imperative lines + up to **5** hidden decisions | ✅ **0** imperative concurrency lines (validated `.pipe`) | +| lines-of-code (OS-independent) | ~689 lines, 8–11 files | **123 lines, 1 file** — 6.0× fewer lines, 3.6× fewer chars | + +## The Windows caveat (concurrent-processing) + +On macOS/Linux the RR `concurrent-processing` cell is `0 errors, 72/72, status=ok`. On **Windows it is +`status=check` in all 10 reps**: the workload node caches a **module-level sqlite connection** (the naive +idiom, `conn="module"`), and the v3.2.1 engine dispatches a pipe's objects to **different OS threads even +at `threads=1`**, so the thread-affine connection raises `sqlite3.ProgrammingError` on some docs +(node_errors per rep across this run: `5, 18, 7, 15, 8, 9, 11, 2, 5, 10`). **Process isolation still +holds** (M pipes = M PIDs); RR degrades *partially* (surviving pipes complete) versus LangChain's `.batch` +which loses **all 64**. Only *thread-affine* resources are affected — `data-isolation` uses a GIL-safe +`list.append` and reproduces `0 lost` here. The repo's own appendix honesty cell (`1 pipe × threadCount=4`) +shows the same **31 errors** on this run — the mechanism was always disclosed, just not flagged as +OS-dependent. Fix options (A: thread-local node conn · B: scope to mac/linux · C: engine per-pipe +threading) are in [`../harness/REPRODUCE-WINDOWS.md`](../harness/REPRODUCE-WINDOWS.md); this data set does +**not** change the RR numbers or the claim — it documents them. + +## Deviation from `../runs/` method (disclosed) + +Each warm-pool rep runs on a **freshly restarted + primed engine**, same as `run_isolated.sh`. The engine +runs on **port 5566** (not 5565) because the RocketRide VS Code extension's engine commonly holds 5565; +this affects nothing measured. Absolute wall-clock differs from the macOS numbers (different hardware) — +per upstream, **ratios and correctness outcomes reproduce; absolute values vary**. diff --git a/concurrent-work/runs-windows/authoring-effort/REPORT.md b/concurrent-work/runs-windows/authoring-effort/REPORT.md new file mode 100644 index 0000000..f0b74a8 --- /dev/null +++ b/concurrent-work/runs-windows/authoring-effort/REPORT.md @@ -0,0 +1,29 @@ +# authoring-effort · Windows (static) — reproduces clean + +> **Verdict on Windows: ✅ 0 imperative concurrency lines, `.pipe` validates.** The RocketRide artifact is +> one declarative `.pipe` with **0 imperative concurrency lines**, and it **validates against the engine** +> on Windows. The LangChain side needs **14–17 imperative lines** and up to **5** pieces of hidden +> knowledge — and, as `concurrent-processing` shows, its three natural idioms crash / silently serialize / +> run slow. Static + deterministic, so a single run (no reps), matching the macOS layout. + +## Method +Identical to `../../runs/authoring-effort/REPORT.md`. Counts imperative lines + decision points in the +four LangChain idiom files (`lc/concurrent_*.py`) and validates the RR `.pipe` against the live engine +(`validate()` over the direct-connect server). Engine on port 5566. + +## Results +| Side | Imperative concurrency lines | Hidden decisions | Validated | +|---|---:|---:|---| +| **Stock RocketRide** (`rr/workflow.pipe`) | **0** | **0** | ✅ PASS (against engine 3.2.1.30) | +| LangChain `concurrent_batch.py` | 14 | 3 | — | +| LangChain `concurrent_abatch.py` | 17 | 4 | — | +| LangChain `concurrent_seq.py` | 14 | 1 | — | +| LangChain `concurrent_percall.py` (the correct one) | 17 | **5** | — | + +The correct LangChain version requires knowing thread-affinity + per-call state discipline the API never +surfaces; RocketRide authors zero concurrency code (claim is about imperative concurrency lines, not token +count — token count favors LangChain, disclosed upstream). + +## Provenance +Single static run (`results.json` + `run.log`); `validate()` ran against the Windows engine on port 5566 +(`rocketride.validate.ran = true, ok = true`). langchain-core `0.3.86` present. diff --git a/concurrent-work/runs-windows/authoring-effort/results.json b/concurrent-work/runs-windows/authoring-effort/results.json new file mode 100644 index 0000000..4f9e42a --- /dev/null +++ b/concurrent-work/runs-windows/authoring-effort/results.json @@ -0,0 +1,88 @@ +{ + "benchmark": "authoring-effort", + "hypothesis": "RR authoring carries zero imperative concurrency code and validates offline; LangChain authoring forces an unguided 3-way concurrency choice where one crashes, one silently serializes, one is slow (measured in concurrent-processing)", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "rocketride": { + "file": "rr/workflow.pipe", + "imperative_lines": 0, + "components": 2, + "decision_points": 0, + "note": "concurrency/isolation owned by the runtime (M pipes, ttl= are RUN-time args, not authored code); each pipeline runs as its own process, so isolation is per-pipe by construction; artifact is schema-validated", + "validate": { + "ran": true, + "ok": true, + "errors": null + } + }, + "langchain": [ + { + "file": "lc/concurrent_batch.py", + "imperative_lines": 14, + "decision_points": 3, + "decisions": [ + "batch vs abatch vs loop", + "max_concurrency value", + ".batch uses threads (implicit)" + ] + }, + { + "file": "lc/concurrent_abatch.py", + "imperative_lines": 17, + "decision_points": 4, + "decisions": [ + "batch vs abatch vs loop", + "max_concurrency value", + "sync vs async chain legs", + "blocking-sync-in-async serializes (implicit, silent)" + ] + }, + { + "file": "lc/concurrent_seq.py", + "imperative_lines": 14, + "decision_points": 1, + "decisions": [ + "batch vs abatch vs loop (gave up concurrency)" + ] + }, + { + "file": "lc/concurrent_percall.py", + "imperative_lines": 17, + "decision_points": 5, + "decisions": [ + "batch vs abatch vs loop", + "max_concurrency value", + ".batch uses threads (implicit)", + "sqlite conns are thread-affine (library doc)", + "therefore: state per call, never captured (discipline)" + ] + } + ], + "scope": { + "note": "this claim is about imperative concurrency lines, not token count" + }, + "verdict_metrics": { + "rr_imperative_lines": 0, + "lc_imperative_lines_min": 14, + "lc_imperative_lines_max": 17, + "lc_decision_points_correct_version": 5, + "lc_natural_idioms_delivering_concurrency": 0, + "lc_idiom_outcomes": { + "batch": "crashes", + "abatch": "silently_serializes", + "seq": "slow_correct" + } + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/authoring-effort/run.log b/concurrent-work/runs-windows/authoring-effort/run.log new file mode 100644 index 0000000..a1dff94 --- /dev/null +++ b/concurrent-work/runs-windows/authoring-effort/run.log @@ -0,0 +1,7 @@ +RR: rr/workflow.pipe — 0 imperative lines, 2 components, validate: PASS +LC: lc/concurrent_batch.py 14 imperative lines, 3 decision points +LC: lc/concurrent_abatch.py 17 imperative lines, 4 decision points +LC: lc/concurrent_seq.py 14 imperative lines, 1 decision points +LC: lc/concurrent_percall.py 17 imperative lines, 5 decision points + +VERDICT: the correct LC version needs 5 pieces of hidden knowledge; one crashes, one silently serializes, one is slow (see concurrent-processing). RR authors zero concurrency code (this claim is about imperative concurrency lines, not token count). diff --git a/concurrent-work/runs-windows/concurrent-processing/REPORT.md b/concurrent-work/runs-windows/concurrent-processing/REPORT.md new file mode 100644 index 0000000..91c5f95 --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/REPORT.md @@ -0,0 +1,50 @@ +# concurrent-processing · Windows (10 reps) — the RR-side caveat + +> **Verdict on Windows: ⚠️ RocketRide is NOT clean here.** Same per-doc work (sqlite +> `INSERT`+`SELECT`+`commit` + 100 ms wait, AST-parity-gated) over 64 docs. The LangChain side behaves +> exactly as on macOS — `.batch` (shared conn) **CRASHES 0/64** (`sqlite3.ProgrammingError`) in **10/10**, +> `.abatch`/seq serialize (~6.7 s). **But the RocketRide cell reports `status=check` in all 10 reps**: +> the naive module-level sqlite connection hits the *same* thread-affinity error on Windows, because the +> v3.2.1 engine runs a pipe's objects on different OS threads even at `threads=1`. Process isolation still +> holds (M pipes = M PIDs); RR degrades **partially** (surviving pipes complete) vs LangChain's **total** +> 0/64. This differs from the macOS result (`0 errors, 72/72, ok`). + +## Method +Identical to `../../runs/concurrent-processing/REPORT.md` — AST-parity-gated per-doc work, RR at +M∈{8,16} warm pipes with the **naive module-level connection** (`conn="module"`), real LangChain +(`lc_version 0.3.86`) with its three idioms. **10 fresh reps**, each on a restarted + primed engine on +Windows (`../../harness/run_isolated_windows.py`), port 5566. + +## Results *(10 reps; Windows 11, Intel 10c/12t, langchain-core 0.3.86, engine 3.2.1.30)* +| Configuration | Wall (64 docs) | Outcome | +|---|---:|---| +| Stock RocketRide, 16 warm pipes | ~2.59 s (median) | ⚠️ **`status=check` 10/10** — node_errors per rep: `5,18,7,15,8,9,11,2,5,10` (`sqlite3.ProgrammingError`) | +| Stock RocketRide, 8 warm pipes | ~1.31 s (median) | ⚠️ same thread-affinity errors | +| Stock LangChain `.batch` (shared conn) | ~0.05 s | ❌ **CRASH 0/64** (`sqlite3.ProgrammingError`) — 10/10 | +| Stock LangChain `.abatch` (blocking) | ~6.7 s | ⚠️ **SERIALIZED** | +| Stock LangChain sequential loop | ~6.9 s | ⚠️ **SLOW** | +| RR appendix — 1 pipe × threadCount=4 (naive conn) | — | ❌ **31 errors** (same trap; shown on Windows exactly as on macOS) | + +## Root cause (why RR is not clean on Windows) +The workload node caches one sqlite connection at module level. SQLite connections are **thread-affine**. +On macOS/Linux an RR pipe runs its objects on **one** worker thread, so the cached connection is always +used on its creating thread (safe → 0 errors). On **Windows the engine dispatches successive objects to +different OS threads even at `threads=1`**, so the connection is reused off-thread and raises +`sqlite3.ProgrammingError` on some docs. The appendix honesty cell (`1 pipe × threadCount=4` → 31 errors) +demonstrates the identical mechanism and reproduces here — it was always disclosed, just not flagged as +OS-dependent for the `threads=1` headline path. + +Only **thread-affine** resources are affected — `data-isolation` (module-level `list.append`, GIL-safe) +reproduces `0 lost` on Windows. RR still beats LangChain here (partial vs total loss), but this is **not** +the clean "0 errors" macOS headline. + +## Fix options (team decision — not applied in this data set) +A) node uses a thread-local connection → 0 errors on every OS (Mac numbers unchanged); +B) keep naive-both, scope the "0 errors" row to macOS/Linux and report Windows degradation; +C) engine fix: make `threads=1` mean one OS thread per pipe on Windows. +Details in [`../../harness/REPRODUCE-WINDOWS.md`](../../harness/REPRODUCE-WINDOWS.md). + +## Provenance +10 reps (`run-01/ … run-10/`), each on a restarted + primed engine on port 5566; real LangChain +`0.3.86` per row; AST parity gate PASS; native trace gzipped in `run-01/trace/`. Raw per-doc markers +(PID, thread id, the exact sqlite errors) are in that trace. diff --git a/concurrent-work/runs-windows/concurrent-processing/run-01/results.json b/concurrent-work/runs-windows/concurrent-processing/run-01/results.json new file mode 100644 index 0000000..5bd6e87 --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-01/results.json @@ -0,0 +1,131 @@ +{ + "benchmark": "concurrent-processing", + "hypothesis": "64 docs of stateful per-doc work: LangChain's idioms crash, serialize, or run sequential; RR's warm per-pipe topology is fast AND safe", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 64, + "io_s": 0.1, + "Ms": [ + 8, + 16 + ], + "max_concurrency": 64, + "work": "sqlite INSERT+SELECT+commit + blocking sleep(io_s), AST-identical (parity-gated)" + }, + "parity_gate": "PASS", + "rocketride": [ + { + "M": 8, + "n_docs": 64, + "wall_s": 1.422, + "warm_s": 15.16, + "p50_ms": 169.24, + "p99_ms": 256.87, + "markers": 72, + "marker_pids": 8, + "node_errors": 2, + "sqlite_rows": 70, + "rows_expected": 72, + "status": "check" + }, + { + "M": 16, + "n_docs": 64, + "wall_s": 2.487, + "warm_s": 28.23, + "p50_ms": 588.37, + "p99_ms": 925.31, + "markers": 80, + "marker_pids": 16, + "node_errors": 3, + "sqlite_rows": 147, + "rows_expected": 80, + "status": "check" + } + ], + "langchain": { + "batch_shared": { + "mode": "batch_shared", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 0.07202149997465312, + "n_ok": 0, + "n_err": 64, + "status": "crash", + "error_type": "sqlite3.ProgrammingError", + "error_example": "SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 3804 and this is thread id 23036.", + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "abatch_blocking": { + "mode": "abatch_blocking", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 6.704501100000925, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "seq": { + "mode": "seq", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 6.84342629997991, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + }, + "rr_appendix_threads4": { + "topology": "1 pipe x threadCount=4, 32 files, naive module-level conn", + "markers": 32, + "distinct_tids": 4, + "node_errors": 31, + "error_example": "RRBENCH_ERR\tProgrammingError: SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 2184 and this is thre" + }, + "verdict_metrics": { + "rr_topM_wall_s": 2.487, + "rr_topM_ok": false, + "lc_batch_shared_status": "crash", + "lc_batch_shared_error": "sqlite3.ProgrammingError", + "lc_abatch_blocking_wall_s": 6.7, + "lc_seq_wall_s": 6.84, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/concurrent-processing/run-01/run.log b/concurrent-work/runs-windows/concurrent-processing/run-01/run.log new file mode 100644 index 0000000..7f03c17 --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-01/run.log @@ -0,0 +1,16 @@ +parity gate: PASS (per-doc work AST-identical both sides) +real LangChain proven: 0.3.86 (D:\Coding\rr\rocketride-benchmark\concurrent-work\harness\rocketride-bench\.venv\Scripts\python.exe) + +RR warm pools (N=64 docs, 100 ms work): + M= 8 wall= 1.422s p50= 169.2ms p99= 256.9ms markers=72/72 errs=2 rows=70/72 [check] + M=16 wall= 2.487s p50= 588.4ms p99= 925.3ms markers=80/80 errs=3 rows=147/80 [check] + +LangChain (same chain object, n=64, io=100 ms, mc=64): + LC batch_shared status=crash wall= 0.07s ok= 0 err=64 sqlite3.ProgrammingError + LC abatch_blocking status=ok wall= 6.70s ok=64 err= 0 + LC seq status=ok wall= 6.84s ok=64 err= 0 +wrote: canvas.mmd canvas.dot + +appendix (1 pipe x threads=4, shared module conn): markers=32 tids=4 errors=31 + +VERDICT @ top M: RR(top M) 2.49s CHECK (node errors — see rocketride[].status) | LC .batch(shared) crash (sqlite3.ProgrammingError) | LC .abatch(blocking) 6.7s | LC seq 6.8s diff --git a/concurrent-work/runs-windows/concurrent-processing/run-01/trace/rr.appendix_t4.jsonl.gz b/concurrent-work/runs-windows/concurrent-processing/run-01/trace/rr.appendix_t4.jsonl.gz new file mode 100644 index 0000000..4f90fac Binary files /dev/null and b/concurrent-work/runs-windows/concurrent-processing/run-01/trace/rr.appendix_t4.jsonl.gz differ diff --git a/concurrent-work/runs-windows/concurrent-processing/run-01/trace/rr.m16.jsonl.gz b/concurrent-work/runs-windows/concurrent-processing/run-01/trace/rr.m16.jsonl.gz new file mode 100644 index 0000000..a76917a Binary files /dev/null and b/concurrent-work/runs-windows/concurrent-processing/run-01/trace/rr.m16.jsonl.gz differ diff --git a/concurrent-work/runs-windows/concurrent-processing/run-01/trace/rr.m8.jsonl.gz b/concurrent-work/runs-windows/concurrent-processing/run-01/trace/rr.m8.jsonl.gz new file mode 100644 index 0000000..cdb3bae Binary files /dev/null and b/concurrent-work/runs-windows/concurrent-processing/run-01/trace/rr.m8.jsonl.gz differ diff --git a/concurrent-work/runs-windows/concurrent-processing/run-02/results.json b/concurrent-work/runs-windows/concurrent-processing/run-02/results.json new file mode 100644 index 0000000..74dff1b --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-02/results.json @@ -0,0 +1,131 @@ +{ + "benchmark": "concurrent-processing", + "hypothesis": "64 docs of stateful per-doc work: LangChain's idioms crash, serialize, or run sequential; RR's warm per-pipe topology is fast AND safe", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 64, + "io_s": 0.1, + "Ms": [ + 8, + 16 + ], + "max_concurrency": 64, + "work": "sqlite INSERT+SELECT+commit + blocking sleep(io_s), AST-identical (parity-gated)" + }, + "parity_gate": "PASS", + "rocketride": [ + { + "M": 8, + "n_docs": 64, + "wall_s": 1.268, + "warm_s": 13.96, + "p50_ms": 142.78, + "p99_ms": 219.03, + "markers": 72, + "marker_pids": 8, + "node_errors": 14, + "sqlite_rows": 58, + "rows_expected": 72, + "status": "check" + }, + { + "M": 16, + "n_docs": 64, + "wall_s": 1.487, + "warm_s": 26.25, + "p50_ms": 363.63, + "p99_ms": 429.54, + "markers": 80, + "marker_pids": 16, + "node_errors": 4, + "sqlite_rows": 134, + "rows_expected": 80, + "status": "check" + } + ], + "langchain": { + "batch_shared": { + "mode": "batch_shared", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 0.02544219995616004, + "n_ok": 0, + "n_err": 64, + "status": "crash", + "error_type": "sqlite3.ProgrammingError", + "error_example": "SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 9676 and this is thread id 21756.", + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "abatch_blocking": { + "mode": "abatch_blocking", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 6.678910699964035, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "seq": { + "mode": "seq", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 6.875936200027354, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + }, + "rr_appendix_threads4": { + "topology": "1 pipe x threadCount=4, 32 files, naive module-level conn", + "markers": 32, + "distinct_tids": 4, + "node_errors": 31, + "error_example": "RRBENCH_ERR\tProgrammingError: SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 27416 and this is thr" + }, + "verdict_metrics": { + "rr_topM_wall_s": 1.487, + "rr_topM_ok": false, + "lc_batch_shared_status": "crash", + "lc_batch_shared_error": "sqlite3.ProgrammingError", + "lc_abatch_blocking_wall_s": 6.68, + "lc_seq_wall_s": 6.88, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/concurrent-processing/run-02/run.log b/concurrent-work/runs-windows/concurrent-processing/run-02/run.log new file mode 100644 index 0000000..150a781 --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-02/run.log @@ -0,0 +1,16 @@ +parity gate: PASS (per-doc work AST-identical both sides) +real LangChain proven: 0.3.86 (D:\Coding\rr\rocketride-benchmark\concurrent-work\harness\rocketride-bench\.venv\Scripts\python.exe) + +RR warm pools (N=64 docs, 100 ms work): + M= 8 wall= 1.268s p50= 142.8ms p99= 219.0ms markers=72/72 errs=14 rows=58/72 [check] + M=16 wall= 1.487s p50= 363.6ms p99= 429.5ms markers=80/80 errs=4 rows=134/80 [check] + +LangChain (same chain object, n=64, io=100 ms, mc=64): + LC batch_shared status=crash wall= 0.03s ok= 0 err=64 sqlite3.ProgrammingError + LC abatch_blocking status=ok wall= 6.68s ok=64 err= 0 + LC seq status=ok wall= 6.88s ok=64 err= 0 +wrote: canvas.mmd canvas.dot + +appendix (1 pipe x threads=4, shared module conn): markers=32 tids=4 errors=31 + +VERDICT @ top M: RR(top M) 1.49s CHECK (node errors — see rocketride[].status) | LC .batch(shared) crash (sqlite3.ProgrammingError) | LC .abatch(blocking) 6.7s | LC seq 6.9s diff --git a/concurrent-work/runs-windows/concurrent-processing/run-03/results.json b/concurrent-work/runs-windows/concurrent-processing/run-03/results.json new file mode 100644 index 0000000..081d081 --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-03/results.json @@ -0,0 +1,131 @@ +{ + "benchmark": "concurrent-processing", + "hypothesis": "64 docs of stateful per-doc work: LangChain's idioms crash, serialize, or run sequential; RR's warm per-pipe topology is fast AND safe", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 64, + "io_s": 0.1, + "Ms": [ + 8, + 16 + ], + "max_concurrency": 64, + "work": "sqlite INSERT+SELECT+commit + blocking sleep(io_s), AST-identical (parity-gated)" + }, + "parity_gate": "PASS", + "rocketride": [ + { + "M": 8, + "n_docs": 64, + "wall_s": 1.134, + "warm_s": 14.36, + "p50_ms": 136.7, + "p99_ms": 180.86, + "markers": 72, + "marker_pids": 8, + "node_errors": 5, + "sqlite_rows": 67, + "rows_expected": 72, + "status": "check" + }, + { + "M": 16, + "n_docs": 64, + "wall_s": 2.244, + "warm_s": 26.49, + "p50_ms": 556.86, + "p99_ms": 609.49, + "markers": 80, + "marker_pids": 16, + "node_errors": 2, + "sqlite_rows": 145, + "rows_expected": 80, + "status": "check" + } + ], + "langchain": { + "batch_shared": { + "mode": "batch_shared", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 0.03077309997752309, + "n_ok": 0, + "n_err": 64, + "status": "crash", + "error_type": "sqlite3.ProgrammingError", + "error_example": "SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 5760 and this is thread id 15504.", + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "abatch_blocking": { + "mode": "abatch_blocking", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 6.749166599998716, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "seq": { + "mode": "seq", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 6.907986800011713, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + }, + "rr_appendix_threads4": { + "topology": "1 pipe x threadCount=4, 32 files, naive module-level conn", + "markers": 32, + "distinct_tids": 4, + "node_errors": 31, + "error_example": "RRBENCH_ERR\tProgrammingError: SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 10404 and this is thr" + }, + "verdict_metrics": { + "rr_topM_wall_s": 2.244, + "rr_topM_ok": false, + "lc_batch_shared_status": "crash", + "lc_batch_shared_error": "sqlite3.ProgrammingError", + "lc_abatch_blocking_wall_s": 6.75, + "lc_seq_wall_s": 6.91, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/concurrent-processing/run-03/run.log b/concurrent-work/runs-windows/concurrent-processing/run-03/run.log new file mode 100644 index 0000000..4925a5d --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-03/run.log @@ -0,0 +1,16 @@ +parity gate: PASS (per-doc work AST-identical both sides) +real LangChain proven: 0.3.86 (D:\Coding\rr\rocketride-benchmark\concurrent-work\harness\rocketride-bench\.venv\Scripts\python.exe) + +RR warm pools (N=64 docs, 100 ms work): + M= 8 wall= 1.134s p50= 136.7ms p99= 180.9ms markers=72/72 errs=5 rows=67/72 [check] + M=16 wall= 2.244s p50= 556.9ms p99= 609.5ms markers=80/80 errs=2 rows=145/80 [check] + +LangChain (same chain object, n=64, io=100 ms, mc=64): + LC batch_shared status=crash wall= 0.03s ok= 0 err=64 sqlite3.ProgrammingError + LC abatch_blocking status=ok wall= 6.75s ok=64 err= 0 + LC seq status=ok wall= 6.91s ok=64 err= 0 +wrote: canvas.mmd canvas.dot + +appendix (1 pipe x threads=4, shared module conn): markers=32 tids=4 errors=31 + +VERDICT @ top M: RR(top M) 2.24s CHECK (node errors — see rocketride[].status) | LC .batch(shared) crash (sqlite3.ProgrammingError) | LC .abatch(blocking) 6.8s | LC seq 6.9s diff --git a/concurrent-work/runs-windows/concurrent-processing/run-04/results.json b/concurrent-work/runs-windows/concurrent-processing/run-04/results.json new file mode 100644 index 0000000..94f5299 --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-04/results.json @@ -0,0 +1,131 @@ +{ + "benchmark": "concurrent-processing", + "hypothesis": "64 docs of stateful per-doc work: LangChain's idioms crash, serialize, or run sequential; RR's warm per-pipe topology is fast AND safe", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 64, + "io_s": 0.1, + "Ms": [ + 8, + 16 + ], + "max_concurrency": 64, + "work": "sqlite INSERT+SELECT+commit + blocking sleep(io_s), AST-identical (parity-gated)" + }, + "parity_gate": "PASS", + "rocketride": [ + { + "M": 8, + "n_docs": 64, + "wall_s": 1.349, + "warm_s": 18.61, + "p50_ms": 160.79, + "p99_ms": 191.63, + "markers": 72, + "marker_pids": 8, + "node_errors": 7, + "sqlite_rows": 65, + "rows_expected": 72, + "status": "check" + }, + { + "M": 16, + "n_docs": 64, + "wall_s": 3.432, + "warm_s": 32.0, + "p50_ms": 797.47, + "p99_ms": 1486.19, + "markers": 80, + "marker_pids": 16, + "node_errors": 8, + "sqlite_rows": 137, + "rows_expected": 80, + "status": "check" + } + ], + "langchain": { + "batch_shared": { + "mode": "batch_shared", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 0.05078969994792715, + "n_ok": 0, + "n_err": 64, + "status": "crash", + "error_type": "sqlite3.ProgrammingError", + "error_example": "SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 26968 and this is thread id 14664.", + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "abatch_blocking": { + "mode": "abatch_blocking", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 6.734708099975251, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "seq": { + "mode": "seq", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 6.9535108000272885, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + }, + "rr_appendix_threads4": { + "topology": "1 pipe x threadCount=4, 32 files, naive module-level conn", + "markers": 32, + "distinct_tids": 4, + "node_errors": 31, + "error_example": "RRBENCH_ERR\tProgrammingError: SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 8064 and this is thre" + }, + "verdict_metrics": { + "rr_topM_wall_s": 3.432, + "rr_topM_ok": false, + "lc_batch_shared_status": "crash", + "lc_batch_shared_error": "sqlite3.ProgrammingError", + "lc_abatch_blocking_wall_s": 6.73, + "lc_seq_wall_s": 6.95, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/concurrent-processing/run-04/run.log b/concurrent-work/runs-windows/concurrent-processing/run-04/run.log new file mode 100644 index 0000000..5926e4d --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-04/run.log @@ -0,0 +1,16 @@ +parity gate: PASS (per-doc work AST-identical both sides) +real LangChain proven: 0.3.86 (D:\Coding\rr\rocketride-benchmark\concurrent-work\harness\rocketride-bench\.venv\Scripts\python.exe) + +RR warm pools (N=64 docs, 100 ms work): + M= 8 wall= 1.349s p50= 160.8ms p99= 191.6ms markers=72/72 errs=7 rows=65/72 [check] + M=16 wall= 3.432s p50= 797.5ms p99=1486.2ms markers=80/80 errs=8 rows=137/80 [check] + +LangChain (same chain object, n=64, io=100 ms, mc=64): + LC batch_shared status=crash wall= 0.05s ok= 0 err=64 sqlite3.ProgrammingError + LC abatch_blocking status=ok wall= 6.73s ok=64 err= 0 + LC seq status=ok wall= 6.95s ok=64 err= 0 +wrote: canvas.mmd canvas.dot + +appendix (1 pipe x threads=4, shared module conn): markers=32 tids=4 errors=31 + +VERDICT @ top M: RR(top M) 3.43s CHECK (node errors — see rocketride[].status) | LC .batch(shared) crash (sqlite3.ProgrammingError) | LC .abatch(blocking) 6.7s | LC seq 7.0s diff --git a/concurrent-work/runs-windows/concurrent-processing/run-05/results.json b/concurrent-work/runs-windows/concurrent-processing/run-05/results.json new file mode 100644 index 0000000..296d7c3 --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-05/results.json @@ -0,0 +1,131 @@ +{ + "benchmark": "concurrent-processing", + "hypothesis": "64 docs of stateful per-doc work: LangChain's idioms crash, serialize, or run sequential; RR's warm per-pipe topology is fast AND safe", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 64, + "io_s": 0.1, + "Ms": [ + 8, + 16 + ], + "max_concurrency": 64, + "work": "sqlite INSERT+SELECT+commit + blocking sleep(io_s), AST-identical (parity-gated)" + }, + "parity_gate": "PASS", + "rocketride": [ + { + "M": 8, + "n_docs": 64, + "wall_s": 1.554, + "warm_s": 16.86, + "p50_ms": 186.09, + "p99_ms": 235.48, + "markers": 72, + "marker_pids": 8, + "node_errors": 4, + "sqlite_rows": 68, + "rows_expected": 72, + "status": "check" + }, + { + "M": 16, + "n_docs": 64, + "wall_s": 3.502, + "warm_s": 28.58, + "p50_ms": 925.46, + "p99_ms": 1196.6, + "markers": 80, + "marker_pids": 16, + "node_errors": 4, + "sqlite_rows": 144, + "rows_expected": 80, + "status": "check" + } + ], + "langchain": { + "batch_shared": { + "mode": "batch_shared", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 0.02280640002572909, + "n_ok": 0, + "n_err": 64, + "status": "crash", + "error_type": "sqlite3.ProgrammingError", + "error_example": "SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 25408 and this is thread id 23416.", + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "abatch_blocking": { + "mode": "abatch_blocking", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 6.76732169999741, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "seq": { + "mode": "seq", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 6.818126900005154, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + }, + "rr_appendix_threads4": { + "topology": "1 pipe x threadCount=4, 32 files, naive module-level conn", + "markers": 32, + "distinct_tids": 4, + "node_errors": 31, + "error_example": "RRBENCH_ERR\tProgrammingError: SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 27688 and this is thr" + }, + "verdict_metrics": { + "rr_topM_wall_s": 3.502, + "rr_topM_ok": false, + "lc_batch_shared_status": "crash", + "lc_batch_shared_error": "sqlite3.ProgrammingError", + "lc_abatch_blocking_wall_s": 6.77, + "lc_seq_wall_s": 6.82, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/concurrent-processing/run-05/run.log b/concurrent-work/runs-windows/concurrent-processing/run-05/run.log new file mode 100644 index 0000000..b60d558 --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-05/run.log @@ -0,0 +1,16 @@ +parity gate: PASS (per-doc work AST-identical both sides) +real LangChain proven: 0.3.86 (D:\Coding\rr\rocketride-benchmark\concurrent-work\harness\rocketride-bench\.venv\Scripts\python.exe) + +RR warm pools (N=64 docs, 100 ms work): + M= 8 wall= 1.554s p50= 186.1ms p99= 235.5ms markers=72/72 errs=4 rows=68/72 [check] + M=16 wall= 3.502s p50= 925.5ms p99=1196.6ms markers=80/80 errs=4 rows=144/80 [check] + +LangChain (same chain object, n=64, io=100 ms, mc=64): + LC batch_shared status=crash wall= 0.02s ok= 0 err=64 sqlite3.ProgrammingError + LC abatch_blocking status=ok wall= 6.77s ok=64 err= 0 + LC seq status=ok wall= 6.82s ok=64 err= 0 +wrote: canvas.mmd canvas.dot + +appendix (1 pipe x threads=4, shared module conn): markers=32 tids=4 errors=31 + +VERDICT @ top M: RR(top M) 3.50s CHECK (node errors — see rocketride[].status) | LC .batch(shared) crash (sqlite3.ProgrammingError) | LC .abatch(blocking) 6.8s | LC seq 6.8s diff --git a/concurrent-work/runs-windows/concurrent-processing/run-06/results.json b/concurrent-work/runs-windows/concurrent-processing/run-06/results.json new file mode 100644 index 0000000..de71857 --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-06/results.json @@ -0,0 +1,131 @@ +{ + "benchmark": "concurrent-processing", + "hypothesis": "64 docs of stateful per-doc work: LangChain's idioms crash, serialize, or run sequential; RR's warm per-pipe topology is fast AND safe", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 64, + "io_s": 0.1, + "Ms": [ + 8, + 16 + ], + "max_concurrency": 64, + "work": "sqlite INSERT+SELECT+commit + blocking sleep(io_s), AST-identical (parity-gated)" + }, + "parity_gate": "PASS", + "rocketride": [ + { + "M": 8, + "n_docs": 64, + "wall_s": 1.273, + "warm_s": 18.61, + "p50_ms": 150.06, + "p99_ms": 190.45, + "markers": 72, + "marker_pids": 8, + "node_errors": 9, + "sqlite_rows": 63, + "rows_expected": 72, + "status": "check" + }, + { + "M": 16, + "n_docs": 64, + "wall_s": 2.324, + "warm_s": 29.69, + "p50_ms": 520.96, + "p99_ms": 687.41, + "markers": 80, + "marker_pids": 16, + "node_errors": 0, + "sqlite_rows": 143, + "rows_expected": 80, + "status": "check" + } + ], + "langchain": { + "batch_shared": { + "mode": "batch_shared", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 0.02463030000217259, + "n_ok": 0, + "n_err": 64, + "status": "crash", + "error_type": "sqlite3.ProgrammingError", + "error_example": "SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 17504 and this is thread id 28092.", + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "abatch_blocking": { + "mode": "abatch_blocking", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 6.7381943999789655, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "seq": { + "mode": "seq", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 7.052462599996943, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + }, + "rr_appendix_threads4": { + "topology": "1 pipe x threadCount=4, 32 files, naive module-level conn", + "markers": 32, + "distinct_tids": 4, + "node_errors": 31, + "error_example": "RRBENCH_ERR\tProgrammingError: SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 21740 and this is thr" + }, + "verdict_metrics": { + "rr_topM_wall_s": 2.324, + "rr_topM_ok": false, + "lc_batch_shared_status": "crash", + "lc_batch_shared_error": "sqlite3.ProgrammingError", + "lc_abatch_blocking_wall_s": 6.74, + "lc_seq_wall_s": 7.05, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/concurrent-processing/run-06/run.log b/concurrent-work/runs-windows/concurrent-processing/run-06/run.log new file mode 100644 index 0000000..8c1cecb --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-06/run.log @@ -0,0 +1,16 @@ +parity gate: PASS (per-doc work AST-identical both sides) +real LangChain proven: 0.3.86 (D:\Coding\rr\rocketride-benchmark\concurrent-work\harness\rocketride-bench\.venv\Scripts\python.exe) + +RR warm pools (N=64 docs, 100 ms work): + M= 8 wall= 1.273s p50= 150.1ms p99= 190.4ms markers=72/72 errs=9 rows=63/72 [check] + M=16 wall= 2.324s p50= 521.0ms p99= 687.4ms markers=80/80 errs=0 rows=143/80 [check] + +LangChain (same chain object, n=64, io=100 ms, mc=64): + LC batch_shared status=crash wall= 0.02s ok= 0 err=64 sqlite3.ProgrammingError + LC abatch_blocking status=ok wall= 6.74s ok=64 err= 0 + LC seq status=ok wall= 7.05s ok=64 err= 0 +wrote: canvas.mmd canvas.dot + +appendix (1 pipe x threads=4, shared module conn): markers=32 tids=4 errors=31 + +VERDICT @ top M: RR(top M) 2.32s CHECK (node errors — see rocketride[].status) | LC .batch(shared) crash (sqlite3.ProgrammingError) | LC .abatch(blocking) 6.7s | LC seq 7.0s diff --git a/concurrent-work/runs-windows/concurrent-processing/run-07/results.json b/concurrent-work/runs-windows/concurrent-processing/run-07/results.json new file mode 100644 index 0000000..d61278c --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-07/results.json @@ -0,0 +1,131 @@ +{ + "benchmark": "concurrent-processing", + "hypothesis": "64 docs of stateful per-doc work: LangChain's idioms crash, serialize, or run sequential; RR's warm per-pipe topology is fast AND safe", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 64, + "io_s": 0.1, + "Ms": [ + 8, + 16 + ], + "max_concurrency": 64, + "work": "sqlite INSERT+SELECT+commit + blocking sleep(io_s), AST-identical (parity-gated)" + }, + "parity_gate": "PASS", + "rocketride": [ + { + "M": 8, + "n_docs": 64, + "wall_s": 1.15, + "warm_s": 15.4, + "p50_ms": 138.5, + "p99_ms": 183.4, + "markers": 72, + "marker_pids": 8, + "node_errors": 10, + "sqlite_rows": 62, + "rows_expected": 72, + "status": "check" + }, + { + "M": 16, + "n_docs": 64, + "wall_s": 2.529, + "warm_s": 29.02, + "p50_ms": 581.5, + "p99_ms": 839.48, + "markers": 80, + "marker_pids": 16, + "node_errors": 1, + "sqlite_rows": 141, + "rows_expected": 80, + "status": "check" + } + ], + "langchain": { + "batch_shared": { + "mode": "batch_shared", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 0.03265460004331544, + "n_ok": 0, + "n_err": 64, + "status": "crash", + "error_type": "sqlite3.ProgrammingError", + "error_example": "SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 27884 and this is thread id 11108.", + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "abatch_blocking": { + "mode": "abatch_blocking", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 6.770614699984435, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "seq": { + "mode": "seq", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 6.915475099987816, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + }, + "rr_appendix_threads4": { + "topology": "1 pipe x threadCount=4, 32 files, naive module-level conn", + "markers": 32, + "distinct_tids": 4, + "node_errors": 31, + "error_example": "RRBENCH_ERR\tProgrammingError: SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 19716 and this is thr" + }, + "verdict_metrics": { + "rr_topM_wall_s": 2.529, + "rr_topM_ok": false, + "lc_batch_shared_status": "crash", + "lc_batch_shared_error": "sqlite3.ProgrammingError", + "lc_abatch_blocking_wall_s": 6.77, + "lc_seq_wall_s": 6.92, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/concurrent-processing/run-07/run.log b/concurrent-work/runs-windows/concurrent-processing/run-07/run.log new file mode 100644 index 0000000..d587139 --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-07/run.log @@ -0,0 +1,16 @@ +parity gate: PASS (per-doc work AST-identical both sides) +real LangChain proven: 0.3.86 (D:\Coding\rr\rocketride-benchmark\concurrent-work\harness\rocketride-bench\.venv\Scripts\python.exe) + +RR warm pools (N=64 docs, 100 ms work): + M= 8 wall= 1.150s p50= 138.5ms p99= 183.4ms markers=72/72 errs=10 rows=62/72 [check] + M=16 wall= 2.529s p50= 581.5ms p99= 839.5ms markers=80/80 errs=1 rows=141/80 [check] + +LangChain (same chain object, n=64, io=100 ms, mc=64): + LC batch_shared status=crash wall= 0.03s ok= 0 err=64 sqlite3.ProgrammingError + LC abatch_blocking status=ok wall= 6.77s ok=64 err= 0 + LC seq status=ok wall= 6.92s ok=64 err= 0 +wrote: canvas.mmd canvas.dot + +appendix (1 pipe x threads=4, shared module conn): markers=32 tids=4 errors=31 + +VERDICT @ top M: RR(top M) 2.53s CHECK (node errors — see rocketride[].status) | LC .batch(shared) crash (sqlite3.ProgrammingError) | LC .abatch(blocking) 6.8s | LC seq 6.9s diff --git a/concurrent-work/runs-windows/concurrent-processing/run-08/results.json b/concurrent-work/runs-windows/concurrent-processing/run-08/results.json new file mode 100644 index 0000000..6f139f3 --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-08/results.json @@ -0,0 +1,131 @@ +{ + "benchmark": "concurrent-processing", + "hypothesis": "64 docs of stateful per-doc work: LangChain's idioms crash, serialize, or run sequential; RR's warm per-pipe topology is fast AND safe", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 64, + "io_s": 0.1, + "Ms": [ + 8, + 16 + ], + "max_concurrency": 64, + "work": "sqlite INSERT+SELECT+commit + blocking sleep(io_s), AST-identical (parity-gated)" + }, + "parity_gate": "PASS", + "rocketride": [ + { + "M": 8, + "n_docs": 64, + "wall_s": 1.267, + "warm_s": 18.18, + "p50_ms": 150.62, + "p99_ms": 196.12, + "markers": 72, + "marker_pids": 8, + "node_errors": 0, + "sqlite_rows": 72, + "rows_expected": 72, + "status": "ok" + }, + { + "M": 16, + "n_docs": 64, + "wall_s": 2.66, + "warm_s": 28.36, + "p50_ms": 648.93, + "p99_ms": 868.63, + "markers": 80, + "marker_pids": 16, + "node_errors": 2, + "sqlite_rows": 150, + "rows_expected": 80, + "status": "check" + } + ], + "langchain": { + "batch_shared": { + "mode": "batch_shared", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 0.031055300030857325, + "n_ok": 0, + "n_err": 64, + "status": "crash", + "error_type": "sqlite3.ProgrammingError", + "error_example": "SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 14212 and this is thread id 26788.", + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "abatch_blocking": { + "mode": "abatch_blocking", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 6.67555009998614, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "seq": { + "mode": "seq", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 6.956492300028913, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + }, + "rr_appendix_threads4": { + "topology": "1 pipe x threadCount=4, 32 files, naive module-level conn", + "markers": 32, + "distinct_tids": 4, + "node_errors": 31, + "error_example": "RRBENCH_ERR\tProgrammingError: SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 22824 and this is thr" + }, + "verdict_metrics": { + "rr_topM_wall_s": 2.66, + "rr_topM_ok": false, + "lc_batch_shared_status": "crash", + "lc_batch_shared_error": "sqlite3.ProgrammingError", + "lc_abatch_blocking_wall_s": 6.68, + "lc_seq_wall_s": 6.96, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/concurrent-processing/run-08/run.log b/concurrent-work/runs-windows/concurrent-processing/run-08/run.log new file mode 100644 index 0000000..26d5f1f --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-08/run.log @@ -0,0 +1,16 @@ +parity gate: PASS (per-doc work AST-identical both sides) +real LangChain proven: 0.3.86 (D:\Coding\rr\rocketride-benchmark\concurrent-work\harness\rocketride-bench\.venv\Scripts\python.exe) + +RR warm pools (N=64 docs, 100 ms work): + M= 8 wall= 1.267s p50= 150.6ms p99= 196.1ms markers=72/72 errs=0 rows=72/72 [ok] + M=16 wall= 2.660s p50= 648.9ms p99= 868.6ms markers=80/80 errs=2 rows=150/80 [check] + +LangChain (same chain object, n=64, io=100 ms, mc=64): + LC batch_shared status=crash wall= 0.03s ok= 0 err=64 sqlite3.ProgrammingError + LC abatch_blocking status=ok wall= 6.68s ok=64 err= 0 + LC seq status=ok wall= 6.96s ok=64 err= 0 +wrote: canvas.mmd canvas.dot + +appendix (1 pipe x threads=4, shared module conn): markers=32 tids=4 errors=31 + +VERDICT @ top M: RR(top M) 2.66s CHECK (node errors — see rocketride[].status) | LC .batch(shared) crash (sqlite3.ProgrammingError) | LC .abatch(blocking) 6.7s | LC seq 7.0s diff --git a/concurrent-work/runs-windows/concurrent-processing/run-09/results.json b/concurrent-work/runs-windows/concurrent-processing/run-09/results.json new file mode 100644 index 0000000..aa99bdf --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-09/results.json @@ -0,0 +1,131 @@ +{ + "benchmark": "concurrent-processing", + "hypothesis": "64 docs of stateful per-doc work: LangChain's idioms crash, serialize, or run sequential; RR's warm per-pipe topology is fast AND safe", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 64, + "io_s": 0.1, + "Ms": [ + 8, + 16 + ], + "max_concurrency": 64, + "work": "sqlite INSERT+SELECT+commit + blocking sleep(io_s), AST-identical (parity-gated)" + }, + "parity_gate": "PASS", + "rocketride": [ + { + "M": 8, + "n_docs": 64, + "wall_s": 1.598, + "warm_s": 15.85, + "p50_ms": 188.69, + "p99_ms": 246.29, + "markers": 72, + "marker_pids": 8, + "node_errors": 3, + "sqlite_rows": 69, + "rows_expected": 72, + "status": "check" + }, + { + "M": 16, + "n_docs": 64, + "wall_s": 2.917, + "warm_s": 27.74, + "p50_ms": 683.05, + "p99_ms": 787.95, + "markers": 80, + "marker_pids": 16, + "node_errors": 2, + "sqlite_rows": 147, + "rows_expected": 80, + "status": "check" + } + ], + "langchain": { + "batch_shared": { + "mode": "batch_shared", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 0.03623570001218468, + "n_ok": 0, + "n_err": 64, + "status": "crash", + "error_type": "sqlite3.ProgrammingError", + "error_example": "SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 26664 and this is thread id 15988.", + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "abatch_blocking": { + "mode": "abatch_blocking", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 6.70997049997095, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "seq": { + "mode": "seq", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 6.9088501000078395, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + }, + "rr_appendix_threads4": { + "topology": "1 pipe x threadCount=4, 32 files, naive module-level conn", + "markers": 32, + "distinct_tids": 4, + "node_errors": 31, + "error_example": "RRBENCH_ERR\tProgrammingError: SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 16184 and this is thr" + }, + "verdict_metrics": { + "rr_topM_wall_s": 2.917, + "rr_topM_ok": false, + "lc_batch_shared_status": "crash", + "lc_batch_shared_error": "sqlite3.ProgrammingError", + "lc_abatch_blocking_wall_s": 6.71, + "lc_seq_wall_s": 6.91, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/concurrent-processing/run-09/run.log b/concurrent-work/runs-windows/concurrent-processing/run-09/run.log new file mode 100644 index 0000000..f9353fc --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-09/run.log @@ -0,0 +1,16 @@ +parity gate: PASS (per-doc work AST-identical both sides) +real LangChain proven: 0.3.86 (D:\Coding\rr\rocketride-benchmark\concurrent-work\harness\rocketride-bench\.venv\Scripts\python.exe) + +RR warm pools (N=64 docs, 100 ms work): + M= 8 wall= 1.598s p50= 188.7ms p99= 246.3ms markers=72/72 errs=3 rows=69/72 [check] + M=16 wall= 2.917s p50= 683.0ms p99= 788.0ms markers=80/80 errs=2 rows=147/80 [check] + +LangChain (same chain object, n=64, io=100 ms, mc=64): + LC batch_shared status=crash wall= 0.04s ok= 0 err=64 sqlite3.ProgrammingError + LC abatch_blocking status=ok wall= 6.71s ok=64 err= 0 + LC seq status=ok wall= 6.91s ok=64 err= 0 +wrote: canvas.mmd canvas.dot + +appendix (1 pipe x threads=4, shared module conn): markers=32 tids=4 errors=31 + +VERDICT @ top M: RR(top M) 2.92s CHECK (node errors — see rocketride[].status) | LC .batch(shared) crash (sqlite3.ProgrammingError) | LC .abatch(blocking) 6.7s | LC seq 6.9s diff --git a/concurrent-work/runs-windows/concurrent-processing/run-10/results.json b/concurrent-work/runs-windows/concurrent-processing/run-10/results.json new file mode 100644 index 0000000..b9b94f4 --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-10/results.json @@ -0,0 +1,131 @@ +{ + "benchmark": "concurrent-processing", + "hypothesis": "64 docs of stateful per-doc work: LangChain's idioms crash, serialize, or run sequential; RR's warm per-pipe topology is fast AND safe", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 64, + "io_s": 0.1, + "Ms": [ + 8, + 16 + ], + "max_concurrency": 64, + "work": "sqlite INSERT+SELECT+commit + blocking sleep(io_s), AST-identical (parity-gated)" + }, + "parity_gate": "PASS", + "rocketride": [ + { + "M": 8, + "n_docs": 64, + "wall_s": 1.389, + "warm_s": 16.83, + "p50_ms": 152.0, + "p99_ms": 238.15, + "markers": 72, + "marker_pids": 8, + "node_errors": 8, + "sqlite_rows": 64, + "rows_expected": 72, + "status": "check" + }, + { + "M": 16, + "n_docs": 64, + "wall_s": 2.677, + "warm_s": 29.71, + "p50_ms": 654.46, + "p99_ms": 738.02, + "markers": 80, + "marker_pids": 16, + "node_errors": 2, + "sqlite_rows": 142, + "rows_expected": 80, + "status": "check" + } + ], + "langchain": { + "batch_shared": { + "mode": "batch_shared", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 0.026168700016569346, + "n_ok": 0, + "n_err": 64, + "status": "crash", + "error_type": "sqlite3.ProgrammingError", + "error_example": "SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 10300 and this is thread id 1972.", + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "abatch_blocking": { + "mode": "abatch_blocking", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 6.669205099984538, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + "seq": { + "mode": "seq", + "n": 64, + "io_s": 0.1, + "max_concurrency": 64, + "wall_s": 6.904610500030685, + "n_ok": 64, + "n_err": 0, + "status": "ok", + "error_type": null, + "error_example": null, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + }, + "rr_appendix_threads4": { + "topology": "1 pipe x threadCount=4, 32 files, naive module-level conn", + "markers": 32, + "distinct_tids": 4, + "node_errors": 31, + "error_example": "RRBENCH_ERR\tProgrammingError: SQLite objects created in a thread can only be used in that same thread. The object was created in thread id 25708 and this is thr" + }, + "verdict_metrics": { + "rr_topM_wall_s": 2.677, + "rr_topM_ok": false, + "lc_batch_shared_status": "crash", + "lc_batch_shared_error": "sqlite3.ProgrammingError", + "lc_abatch_blocking_wall_s": 6.67, + "lc_seq_wall_s": 6.9, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/concurrent-processing/run-10/run.log b/concurrent-work/runs-windows/concurrent-processing/run-10/run.log new file mode 100644 index 0000000..555e690 --- /dev/null +++ b/concurrent-work/runs-windows/concurrent-processing/run-10/run.log @@ -0,0 +1,16 @@ +parity gate: PASS (per-doc work AST-identical both sides) +real LangChain proven: 0.3.86 (D:\Coding\rr\rocketride-benchmark\concurrent-work\harness\rocketride-bench\.venv\Scripts\python.exe) + +RR warm pools (N=64 docs, 100 ms work): + M= 8 wall= 1.389s p50= 152.0ms p99= 238.2ms markers=72/72 errs=8 rows=64/72 [check] + M=16 wall= 2.677s p50= 654.5ms p99= 738.0ms markers=80/80 errs=2 rows=142/80 [check] + +LangChain (same chain object, n=64, io=100 ms, mc=64): + LC batch_shared status=crash wall= 0.03s ok= 0 err=64 sqlite3.ProgrammingError + LC abatch_blocking status=ok wall= 6.67s ok=64 err= 0 + LC seq status=ok wall= 6.90s ok=64 err= 0 +wrote: canvas.mmd canvas.dot + +appendix (1 pipe x threads=4, shared module conn): markers=32 tids=4 errors=31 + +VERDICT @ top M: RR(top M) 2.68s CHECK (node errors — see rocketride[].status) | LC .batch(shared) crash (sqlite3.ProgrammingError) | LC .abatch(blocking) 6.7s | LC seq 6.9s diff --git a/concurrent-work/runs-windows/data-isolation/REPORT.md b/concurrent-work/runs-windows/data-isolation/REPORT.md new file mode 100644 index 0000000..02769b0 --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/REPORT.md @@ -0,0 +1,35 @@ +# data-isolation · Windows (10 reps) — reproduces clean + +> **Verdict on Windows: ✅ RocketRide 0 lost / 0 leaked, 10/10.** 256 named docs through 32 warm pipes: +> every pipe ends holding exactly its own docs — **0 lost, 0 leaked in all 10 reps**. The SAME naive +> shared-dict idiom in LangChain (`.batch`, 32 workers, non-atomic read→busy(gap)→write) **silently loses +> 100–225 of 256 updates (39–88%)** with no exception. This bench reproduces the macOS result faithfully +> on Windows because the per-pipe accumulator is a GIL-safe `list.append` — no thread-affine resource, so +> the Windows engine's multi-thread-per-pipe behavior (which breaks `concurrent-processing`) does not +> affect it. + +## Method +Identical to `../../runs/data-isolation/REPORT.md`. RR: 256 docs round-robin through M=32 warm pipes +(`iso_accumulate` appends each doc's name to a module-level per-process list; the harness verifies each +PID's final state from `RRBENCH_STATE` trace lines). LangChain (real, `0.3.86`): one shared dict, 32 +`.batch` workers, the suite's `busy(gap)` loop in the read→write window, swept over gaps {20k, 50k, 100k}. +**10 fresh reps**, restarted + primed engine, Windows, port 5566. + +## Results *(10 reps; Windows 11, engine 3.2.1.30, langchain-core 0.3.86)* +| Configuration | Outcome | +|---|---| +| **Stock RocketRide, 32 warm pipes** | ✅ **0 lost / 0 leaked, 32/32 clean partitions — 10/10 reps** (wall ~33.5 s median, 256 docs) | +| LangChain shared dict, gap=20k | ❌ silently loses **100–174** of 256 | +| LangChain shared dict, gap=50k | ❌ silently loses **134–211** of 256 | +| LangChain shared dict, gap=100k | ❌ silently loses **167–225** of 256 | + +Across the sweep the LangChain shared-dict idiom drops **39–88%** of updates with **no error raised** +(the counter under-counts while the list still appends). RocketRide's per-pipe process isolation makes +the naive per-pipe accumulator correct by construction. + +## Provenance +10 reps (`run-01/ … run-10/`), each on a restarted + primed engine (port 5566); real LangChain `0.3.86` +per row; native trace gzipped in `run-01/trace/rr.iso.jsonl.gz` (every pipe's final `RRBENCH_STATE`). +The 38–84% figure in the macOS headline is a function of the injected `busy(gap)` window (disclosed); the +*silent lost-update race* is the structural result, the exact % is a dial — the gap sweep is shown so the +range is explicit. diff --git a/concurrent-work/runs-windows/data-isolation/run-01/results.json b/concurrent-work/runs-windows/data-isolation/run-01/results.json new file mode 100644 index 0000000..af01106 --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-01/results.json @@ -0,0 +1,102 @@ +{ + "benchmark": "data-isolation", + "hypothesis": "per-pipe instance data is isolated by construction; a naive shared GLOBAL dict under 64 LangChain workers silently loses updates", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 256, + "M": 32, + "gaps": [ + 20000, + 50000, + 100000 + ], + "rr_routing": "round-robin doc i -> pipe i % M; identity from objinfo name" + }, + "rocketride": { + "M": 32, + "n_docs": 256, + "wall_s": 25.738, + "warm_s": 66.33, + "pipes_reporting": 32, + "pipes_with_clean_partition": 32, + "docs_lost": 0, + "docs_duplicated_or_leaked": 0, + "status": "ok" + }, + "langchain": [ + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 20000, + "wall_s": 0.3510181000456214, + "expected": 256, + "observed_counter": 115, + "lost_updates": 141, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 50000, + "wall_s": 0.9521291999844834, + "expected": 256, + "observed_counter": 89, + "lost_updates": 167, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 100000, + "wall_s": 2.523222199990414, + "expected": 256, + "observed_counter": 51, + "lost_updates": 205, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + ], + "verdict_metrics": { + "rr_docs_lost": 0, + "rr_docs_duplicated_or_leaked": 0, + "rr_clean": true, + "lc_max_lost_updates": 205, + "lc_max_lost_gap_iters": 100000, + "lc_lost_by_gap": { + "20000": 141, + "50000": 167, + "100000": 205 + }, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/data-isolation/run-01/run.log b/concurrent-work/runs-windows/data-isolation/run-01/run.log new file mode 100644 index 0000000..f176d9d --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-01/run.log @@ -0,0 +1,12 @@ +real LangChain proven: 0.3.86 + +RR: 256 named docs → 32 warm pipes (instance data = per-pipe local copy): + pipes=32/32 clean partitions=32 lost=0 leaked=0 [ok] wall=25.74s + +LangChain: ONE shared dict, 32 workers, non-atomic read→busy(gap)→write: + LC shared dict gap=20000 expected=256 observed=115 lost=141 (wall 0.35s) + LC shared dict gap=50000 expected=256 observed=89 lost=167 (wall 0.95s) + LC shared dict gap=100000 expected=256 observed=51 lost=205 (wall 2.52s) +wrote: canvas.mmd canvas.dot + +VERDICT: RR 64 pipes — 0 lost / 0 leaked (ok). LangChain shared dict — up to 205/256 updates silently lost (gap=100000). diff --git a/concurrent-work/runs-windows/data-isolation/run-01/trace/rr.iso.jsonl.gz b/concurrent-work/runs-windows/data-isolation/run-01/trace/rr.iso.jsonl.gz new file mode 100644 index 0000000..eacc123 Binary files /dev/null and b/concurrent-work/runs-windows/data-isolation/run-01/trace/rr.iso.jsonl.gz differ diff --git a/concurrent-work/runs-windows/data-isolation/run-02/results.json b/concurrent-work/runs-windows/data-isolation/run-02/results.json new file mode 100644 index 0000000..bcc264b --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-02/results.json @@ -0,0 +1,102 @@ +{ + "benchmark": "data-isolation", + "hypothesis": "per-pipe instance data is isolated by construction; a naive shared GLOBAL dict under 64 LangChain workers silently loses updates", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 256, + "M": 32, + "gaps": [ + 20000, + 50000, + 100000 + ], + "rr_routing": "round-robin doc i -> pipe i % M; identity from objinfo name" + }, + "rocketride": { + "M": 32, + "n_docs": 256, + "wall_s": 24.636, + "warm_s": 64.76, + "pipes_reporting": 32, + "pipes_with_clean_partition": 32, + "docs_lost": 0, + "docs_duplicated_or_leaked": 0, + "status": "ok" + }, + "langchain": [ + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 20000, + "wall_s": 0.32824090000940487, + "expected": 256, + "observed_counter": 102, + "lost_updates": 154, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 50000, + "wall_s": 0.9443491999991238, + "expected": 256, + "observed_counter": 67, + "lost_updates": 189, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 100000, + "wall_s": 1.8337629999732599, + "expected": 256, + "observed_counter": 44, + "lost_updates": 212, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + ], + "verdict_metrics": { + "rr_docs_lost": 0, + "rr_docs_duplicated_or_leaked": 0, + "rr_clean": true, + "lc_max_lost_updates": 212, + "lc_max_lost_gap_iters": 100000, + "lc_lost_by_gap": { + "20000": 154, + "50000": 189, + "100000": 212 + }, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/data-isolation/run-02/run.log b/concurrent-work/runs-windows/data-isolation/run-02/run.log new file mode 100644 index 0000000..4c778ee --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-02/run.log @@ -0,0 +1,12 @@ +real LangChain proven: 0.3.86 + +RR: 256 named docs → 32 warm pipes (instance data = per-pipe local copy): + pipes=32/32 clean partitions=32 lost=0 leaked=0 [ok] wall=24.64s + +LangChain: ONE shared dict, 32 workers, non-atomic read→busy(gap)→write: + LC shared dict gap=20000 expected=256 observed=102 lost=154 (wall 0.33s) + LC shared dict gap=50000 expected=256 observed=67 lost=189 (wall 0.94s) + LC shared dict gap=100000 expected=256 observed=44 lost=212 (wall 1.83s) +wrote: canvas.mmd canvas.dot + +VERDICT: RR 64 pipes — 0 lost / 0 leaked (ok). LangChain shared dict — up to 212/256 updates silently lost (gap=100000). diff --git a/concurrent-work/runs-windows/data-isolation/run-03/results.json b/concurrent-work/runs-windows/data-isolation/run-03/results.json new file mode 100644 index 0000000..f1ddf27 --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-03/results.json @@ -0,0 +1,102 @@ +{ + "benchmark": "data-isolation", + "hypothesis": "per-pipe instance data is isolated by construction; a naive shared GLOBAL dict under 64 LangChain workers silently loses updates", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 256, + "M": 32, + "gaps": [ + 20000, + 50000, + 100000 + ], + "rr_routing": "round-robin doc i -> pipe i % M; identity from objinfo name" + }, + "rocketride": { + "M": 32, + "n_docs": 256, + "wall_s": 34.165, + "warm_s": 66.77, + "pipes_reporting": 32, + "pipes_with_clean_partition": 32, + "docs_lost": 0, + "docs_duplicated_or_leaked": 0, + "status": "ok" + }, + "langchain": [ + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 20000, + "wall_s": 0.8102629000204615, + "expected": 256, + "observed_counter": 119, + "lost_updates": 137, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 50000, + "wall_s": 1.003217899997253, + "expected": 256, + "observed_counter": 61, + "lost_updates": 195, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 100000, + "wall_s": 1.8840077000204474, + "expected": 256, + "observed_counter": 31, + "lost_updates": 225, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + ], + "verdict_metrics": { + "rr_docs_lost": 0, + "rr_docs_duplicated_or_leaked": 0, + "rr_clean": true, + "lc_max_lost_updates": 225, + "lc_max_lost_gap_iters": 100000, + "lc_lost_by_gap": { + "20000": 137, + "50000": 195, + "100000": 225 + }, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/data-isolation/run-03/run.log b/concurrent-work/runs-windows/data-isolation/run-03/run.log new file mode 100644 index 0000000..c4a53e7 --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-03/run.log @@ -0,0 +1,12 @@ +real LangChain proven: 0.3.86 + +RR: 256 named docs → 32 warm pipes (instance data = per-pipe local copy): + pipes=32/32 clean partitions=32 lost=0 leaked=0 [ok] wall=34.16s + +LangChain: ONE shared dict, 32 workers, non-atomic read→busy(gap)→write: + LC shared dict gap=20000 expected=256 observed=119 lost=137 (wall 0.81s) + LC shared dict gap=50000 expected=256 observed=61 lost=195 (wall 1.00s) + LC shared dict gap=100000 expected=256 observed=31 lost=225 (wall 1.88s) +wrote: canvas.mmd canvas.dot + +VERDICT: RR 64 pipes — 0 lost / 0 leaked (ok). LangChain shared dict — up to 225/256 updates silently lost (gap=100000). diff --git a/concurrent-work/runs-windows/data-isolation/run-04/results.json b/concurrent-work/runs-windows/data-isolation/run-04/results.json new file mode 100644 index 0000000..81d9497 --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-04/results.json @@ -0,0 +1,102 @@ +{ + "benchmark": "data-isolation", + "hypothesis": "per-pipe instance data is isolated by construction; a naive shared GLOBAL dict under 64 LangChain workers silently loses updates", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 256, + "M": 32, + "gaps": [ + 20000, + 50000, + 100000 + ], + "rr_routing": "round-robin doc i -> pipe i % M; identity from objinfo name" + }, + "rocketride": { + "M": 32, + "n_docs": 256, + "wall_s": 35.747, + "warm_s": 75.05, + "pipes_reporting": 32, + "pipes_with_clean_partition": 32, + "docs_lost": 0, + "docs_duplicated_or_leaked": 0, + "status": "ok" + }, + "langchain": [ + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 20000, + "wall_s": 0.4864744999795221, + "expected": 256, + "observed_counter": 109, + "lost_updates": 147, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 50000, + "wall_s": 0.9318084000260569, + "expected": 256, + "observed_counter": 45, + "lost_updates": 211, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 100000, + "wall_s": 1.8783838999806903, + "expected": 256, + "observed_counter": 38, + "lost_updates": 218, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + ], + "verdict_metrics": { + "rr_docs_lost": 0, + "rr_docs_duplicated_or_leaked": 0, + "rr_clean": true, + "lc_max_lost_updates": 218, + "lc_max_lost_gap_iters": 100000, + "lc_lost_by_gap": { + "20000": 147, + "50000": 211, + "100000": 218 + }, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/data-isolation/run-04/run.log b/concurrent-work/runs-windows/data-isolation/run-04/run.log new file mode 100644 index 0000000..f172b35 --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-04/run.log @@ -0,0 +1,12 @@ +real LangChain proven: 0.3.86 + +RR: 256 named docs → 32 warm pipes (instance data = per-pipe local copy): + pipes=32/32 clean partitions=32 lost=0 leaked=0 [ok] wall=35.75s + +LangChain: ONE shared dict, 32 workers, non-atomic read→busy(gap)→write: + LC shared dict gap=20000 expected=256 observed=109 lost=147 (wall 0.49s) + LC shared dict gap=50000 expected=256 observed=45 lost=211 (wall 0.93s) + LC shared dict gap=100000 expected=256 observed=38 lost=218 (wall 1.88s) +wrote: canvas.mmd canvas.dot + +VERDICT: RR 64 pipes — 0 lost / 0 leaked (ok). LangChain shared dict — up to 218/256 updates silently lost (gap=100000). diff --git a/concurrent-work/runs-windows/data-isolation/run-05/results.json b/concurrent-work/runs-windows/data-isolation/run-05/results.json new file mode 100644 index 0000000..d3cf5e6 --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-05/results.json @@ -0,0 +1,102 @@ +{ + "benchmark": "data-isolation", + "hypothesis": "per-pipe instance data is isolated by construction; a naive shared GLOBAL dict under 64 LangChain workers silently loses updates", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 256, + "M": 32, + "gaps": [ + 20000, + 50000, + 100000 + ], + "rr_routing": "round-robin doc i -> pipe i % M; identity from objinfo name" + }, + "rocketride": { + "M": 32, + "n_docs": 256, + "wall_s": 35.922, + "warm_s": 68.01, + "pipes_reporting": 32, + "pipes_with_clean_partition": 32, + "docs_lost": 0, + "docs_duplicated_or_leaked": 0, + "status": "ok" + }, + "langchain": [ + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 20000, + "wall_s": 0.5205256000044756, + "expected": 256, + "observed_counter": 91, + "lost_updates": 165, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 50000, + "wall_s": 1.034551399992779, + "expected": 256, + "observed_counter": 122, + "lost_updates": 134, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 100000, + "wall_s": 1.913001400011126, + "expected": 256, + "observed_counter": 89, + "lost_updates": 167, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + ], + "verdict_metrics": { + "rr_docs_lost": 0, + "rr_docs_duplicated_or_leaked": 0, + "rr_clean": true, + "lc_max_lost_updates": 167, + "lc_max_lost_gap_iters": 100000, + "lc_lost_by_gap": { + "20000": 165, + "50000": 134, + "100000": 167 + }, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/data-isolation/run-05/run.log b/concurrent-work/runs-windows/data-isolation/run-05/run.log new file mode 100644 index 0000000..b5eec7a --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-05/run.log @@ -0,0 +1,12 @@ +real LangChain proven: 0.3.86 + +RR: 256 named docs → 32 warm pipes (instance data = per-pipe local copy): + pipes=32/32 clean partitions=32 lost=0 leaked=0 [ok] wall=35.92s + +LangChain: ONE shared dict, 32 workers, non-atomic read→busy(gap)→write: + LC shared dict gap=20000 expected=256 observed=91 lost=165 (wall 0.52s) + LC shared dict gap=50000 expected=256 observed=122 lost=134 (wall 1.03s) + LC shared dict gap=100000 expected=256 observed=89 lost=167 (wall 1.91s) +wrote: canvas.mmd canvas.dot + +VERDICT: RR 64 pipes — 0 lost / 0 leaked (ok). LangChain shared dict — up to 167/256 updates silently lost (gap=100000). diff --git a/concurrent-work/runs-windows/data-isolation/run-06/results.json b/concurrent-work/runs-windows/data-isolation/run-06/results.json new file mode 100644 index 0000000..142d21e --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-06/results.json @@ -0,0 +1,102 @@ +{ + "benchmark": "data-isolation", + "hypothesis": "per-pipe instance data is isolated by construction; a naive shared GLOBAL dict under 64 LangChain workers silently loses updates", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 256, + "M": 32, + "gaps": [ + 20000, + 50000, + 100000 + ], + "rr_routing": "round-robin doc i -> pipe i % M; identity from objinfo name" + }, + "rocketride": { + "M": 32, + "n_docs": 256, + "wall_s": 34.882, + "warm_s": 70.31, + "pipes_reporting": 32, + "pipes_with_clean_partition": 32, + "docs_lost": 0, + "docs_duplicated_or_leaked": 0, + "status": "ok" + }, + "langchain": [ + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 20000, + "wall_s": 0.5623634000075981, + "expected": 256, + "observed_counter": 139, + "lost_updates": 117, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 50000, + "wall_s": 0.9558547000051476, + "expected": 256, + "observed_counter": 92, + "lost_updates": 164, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 100000, + "wall_s": 1.837772099999711, + "expected": 256, + "observed_counter": 45, + "lost_updates": 211, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + ], + "verdict_metrics": { + "rr_docs_lost": 0, + "rr_docs_duplicated_or_leaked": 0, + "rr_clean": true, + "lc_max_lost_updates": 211, + "lc_max_lost_gap_iters": 100000, + "lc_lost_by_gap": { + "20000": 117, + "50000": 164, + "100000": 211 + }, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/data-isolation/run-06/run.log b/concurrent-work/runs-windows/data-isolation/run-06/run.log new file mode 100644 index 0000000..78e9767 --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-06/run.log @@ -0,0 +1,12 @@ +real LangChain proven: 0.3.86 + +RR: 256 named docs → 32 warm pipes (instance data = per-pipe local copy): + pipes=32/32 clean partitions=32 lost=0 leaked=0 [ok] wall=34.88s + +LangChain: ONE shared dict, 32 workers, non-atomic read→busy(gap)→write: + LC shared dict gap=20000 expected=256 observed=139 lost=117 (wall 0.56s) + LC shared dict gap=50000 expected=256 observed=92 lost=164 (wall 0.96s) + LC shared dict gap=100000 expected=256 observed=45 lost=211 (wall 1.84s) +wrote: canvas.mmd canvas.dot + +VERDICT: RR 64 pipes — 0 lost / 0 leaked (ok). LangChain shared dict — up to 211/256 updates silently lost (gap=100000). diff --git a/concurrent-work/runs-windows/data-isolation/run-07/results.json b/concurrent-work/runs-windows/data-isolation/run-07/results.json new file mode 100644 index 0000000..b8fde04 --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-07/results.json @@ -0,0 +1,102 @@ +{ + "benchmark": "data-isolation", + "hypothesis": "per-pipe instance data is isolated by construction; a naive shared GLOBAL dict under 64 LangChain workers silently loses updates", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 256, + "M": 32, + "gaps": [ + 20000, + 50000, + 100000 + ], + "rr_routing": "round-robin doc i -> pipe i % M; identity from objinfo name" + }, + "rocketride": { + "M": 32, + "n_docs": 256, + "wall_s": 35.916, + "warm_s": 69.72, + "pipes_reporting": 32, + "pipes_with_clean_partition": 32, + "docs_lost": 0, + "docs_duplicated_or_leaked": 0, + "status": "ok" + }, + "langchain": [ + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 20000, + "wall_s": 0.5922722999821417, + "expected": 256, + "observed_counter": 83, + "lost_updates": 173, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 50000, + "wall_s": 1.0073009000043385, + "expected": 256, + "observed_counter": 97, + "lost_updates": 159, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 100000, + "wall_s": 2.15319520002231, + "expected": 256, + "observed_counter": 77, + "lost_updates": 179, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + ], + "verdict_metrics": { + "rr_docs_lost": 0, + "rr_docs_duplicated_or_leaked": 0, + "rr_clean": true, + "lc_max_lost_updates": 179, + "lc_max_lost_gap_iters": 100000, + "lc_lost_by_gap": { + "20000": 173, + "50000": 159, + "100000": 179 + }, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/data-isolation/run-07/run.log b/concurrent-work/runs-windows/data-isolation/run-07/run.log new file mode 100644 index 0000000..a4da0f4 --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-07/run.log @@ -0,0 +1,12 @@ +real LangChain proven: 0.3.86 + +RR: 256 named docs → 32 warm pipes (instance data = per-pipe local copy): + pipes=32/32 clean partitions=32 lost=0 leaked=0 [ok] wall=35.92s + +LangChain: ONE shared dict, 32 workers, non-atomic read→busy(gap)→write: + LC shared dict gap=20000 expected=256 observed=83 lost=173 (wall 0.59s) + LC shared dict gap=50000 expected=256 observed=97 lost=159 (wall 1.01s) + LC shared dict gap=100000 expected=256 observed=77 lost=179 (wall 2.15s) +wrote: canvas.mmd canvas.dot + +VERDICT: RR 64 pipes — 0 lost / 0 leaked (ok). LangChain shared dict — up to 179/256 updates silently lost (gap=100000). diff --git a/concurrent-work/runs-windows/data-isolation/run-08/results.json b/concurrent-work/runs-windows/data-isolation/run-08/results.json new file mode 100644 index 0000000..0d1c73b --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-08/results.json @@ -0,0 +1,102 @@ +{ + "benchmark": "data-isolation", + "hypothesis": "per-pipe instance data is isolated by construction; a naive shared GLOBAL dict under 64 LangChain workers silently loses updates", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 256, + "M": 32, + "gaps": [ + 20000, + 50000, + 100000 + ], + "rr_routing": "round-robin doc i -> pipe i % M; identity from objinfo name" + }, + "rocketride": { + "M": 32, + "n_docs": 256, + "wall_s": 32.813, + "warm_s": 70.98, + "pipes_reporting": 32, + "pipes_with_clean_partition": 32, + "docs_lost": 0, + "docs_duplicated_or_leaked": 0, + "status": "ok" + }, + "langchain": [ + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 20000, + "wall_s": 0.4943479999783449, + "expected": 256, + "observed_counter": 82, + "lost_updates": 174, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 50000, + "wall_s": 0.9710740000009537, + "expected": 256, + "observed_counter": 55, + "lost_updates": 201, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 100000, + "wall_s": 1.8476726000080816, + "expected": 256, + "observed_counter": 42, + "lost_updates": 214, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + ], + "verdict_metrics": { + "rr_docs_lost": 0, + "rr_docs_duplicated_or_leaked": 0, + "rr_clean": true, + "lc_max_lost_updates": 214, + "lc_max_lost_gap_iters": 100000, + "lc_lost_by_gap": { + "20000": 174, + "50000": 201, + "100000": 214 + }, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/data-isolation/run-08/run.log b/concurrent-work/runs-windows/data-isolation/run-08/run.log new file mode 100644 index 0000000..e6c0416 --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-08/run.log @@ -0,0 +1,12 @@ +real LangChain proven: 0.3.86 + +RR: 256 named docs → 32 warm pipes (instance data = per-pipe local copy): + pipes=32/32 clean partitions=32 lost=0 leaked=0 [ok] wall=32.81s + +LangChain: ONE shared dict, 32 workers, non-atomic read→busy(gap)→write: + LC shared dict gap=20000 expected=256 observed=82 lost=174 (wall 0.49s) + LC shared dict gap=50000 expected=256 observed=55 lost=201 (wall 0.97s) + LC shared dict gap=100000 expected=256 observed=42 lost=214 (wall 1.85s) +wrote: canvas.mmd canvas.dot + +VERDICT: RR 64 pipes — 0 lost / 0 leaked (ok). LangChain shared dict — up to 214/256 updates silently lost (gap=100000). diff --git a/concurrent-work/runs-windows/data-isolation/run-09/results.json b/concurrent-work/runs-windows/data-isolation/run-09/results.json new file mode 100644 index 0000000..2a7e6c3 --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-09/results.json @@ -0,0 +1,102 @@ +{ + "benchmark": "data-isolation", + "hypothesis": "per-pipe instance data is isolated by construction; a naive shared GLOBAL dict under 64 LangChain workers silently loses updates", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 256, + "M": 32, + "gaps": [ + 20000, + 50000, + 100000 + ], + "rr_routing": "round-robin doc i -> pipe i % M; identity from objinfo name" + }, + "rocketride": { + "M": 32, + "n_docs": 256, + "wall_s": 32.858, + "warm_s": 71.24, + "pipes_reporting": 32, + "pipes_with_clean_partition": 32, + "docs_lost": 0, + "docs_duplicated_or_leaked": 0, + "status": "ok" + }, + "langchain": [ + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 20000, + "wall_s": 0.3674897000310011, + "expected": 256, + "observed_counter": 156, + "lost_updates": 100, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 50000, + "wall_s": 0.928164699987974, + "expected": 256, + "observed_counter": 88, + "lost_updates": 168, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 100000, + "wall_s": 1.9698561999830417, + "expected": 256, + "observed_counter": 75, + "lost_updates": 181, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + ], + "verdict_metrics": { + "rr_docs_lost": 0, + "rr_docs_duplicated_or_leaked": 0, + "rr_clean": true, + "lc_max_lost_updates": 181, + "lc_max_lost_gap_iters": 100000, + "lc_lost_by_gap": { + "20000": 100, + "50000": 168, + "100000": 181 + }, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/data-isolation/run-09/run.log b/concurrent-work/runs-windows/data-isolation/run-09/run.log new file mode 100644 index 0000000..580aaa4 --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-09/run.log @@ -0,0 +1,12 @@ +real LangChain proven: 0.3.86 + +RR: 256 named docs → 32 warm pipes (instance data = per-pipe local copy): + pipes=32/32 clean partitions=32 lost=0 leaked=0 [ok] wall=32.86s + +LangChain: ONE shared dict, 32 workers, non-atomic read→busy(gap)→write: + LC shared dict gap=20000 expected=256 observed=156 lost=100 (wall 0.37s) + LC shared dict gap=50000 expected=256 observed=88 lost=168 (wall 0.93s) + LC shared dict gap=100000 expected=256 observed=75 lost=181 (wall 1.97s) +wrote: canvas.mmd canvas.dot + +VERDICT: RR 64 pipes — 0 lost / 0 leaked (ok). LangChain shared dict — up to 181/256 updates silently lost (gap=100000). diff --git a/concurrent-work/runs-windows/data-isolation/run-10/results.json b/concurrent-work/runs-windows/data-isolation/run-10/results.json new file mode 100644 index 0000000..66888cf --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-10/results.json @@ -0,0 +1,102 @@ +{ + "benchmark": "data-isolation", + "hypothesis": "per-pipe instance data is isolated by construction; a naive shared GLOBAL dict under 64 LangChain workers silently loses updates", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "langchain_provenance": { + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "kind": "probe" + }, + "params": { + "n_docs": 256, + "M": 32, + "gaps": [ + 20000, + 50000, + 100000 + ], + "rr_routing": "round-robin doc i -> pipe i % M; identity from objinfo name" + }, + "rocketride": { + "M": 32, + "n_docs": 256, + "wall_s": 29.183, + "warm_s": 69.74, + "pipes_reporting": 32, + "pipes_with_clean_partition": 32, + "docs_lost": 0, + "docs_duplicated_or_leaked": 0, + "status": "ok" + }, + "langchain": [ + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 20000, + "wall_s": 0.4107062999974005, + "expected": 256, + "observed_counter": 126, + "lost_updates": 130, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 50000, + "wall_s": 0.9345452999696136, + "expected": 256, + "observed_counter": 116, + "lost_updates": 140, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + }, + { + "n": 256, + "max_concurrency": 32, + "gap_iters": 100000, + "wall_s": 1.8097565000061877, + "expected": 256, + "observed_counter": 63, + "lost_updates": 193, + "items_len": 256, + "lc_version": "0.3.86", + "kind": "probe", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py" + } + ], + "verdict_metrics": { + "rr_docs_lost": 0, + "rr_docs_duplicated_or_leaked": 0, + "rr_clean": true, + "lc_max_lost_updates": 193, + "lc_max_lost_gap_iters": 100000, + "lc_lost_by_gap": { + "20000": 130, + "50000": 140, + "100000": 193 + }, + "lc_version": "0.3.86" + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/data-isolation/run-10/run.log b/concurrent-work/runs-windows/data-isolation/run-10/run.log new file mode 100644 index 0000000..20d395f --- /dev/null +++ b/concurrent-work/runs-windows/data-isolation/run-10/run.log @@ -0,0 +1,12 @@ +real LangChain proven: 0.3.86 + +RR: 256 named docs → 32 warm pipes (instance data = per-pipe local copy): + pipes=32/32 clean partitions=32 lost=0 leaked=0 [ok] wall=29.18s + +LangChain: ONE shared dict, 32 workers, non-atomic read→busy(gap)→write: + LC shared dict gap=20000 expected=256 observed=126 lost=130 (wall 0.41s) + LC shared dict gap=50000 expected=256 observed=116 lost=140 (wall 0.93s) + LC shared dict gap=100000 expected=256 observed=63 lost=193 (wall 1.81s) +wrote: canvas.mmd canvas.dot + +VERDICT: RR 64 pipes — 0 lost / 0 leaked (ok). LangChain shared dict — up to 193/256 updates silently lost (gap=100000). diff --git a/concurrent-work/runs-windows/fault-isolation/REPORT.md b/concurrent-work/runs-windows/fault-isolation/REPORT.md new file mode 100644 index 0000000..142e409 --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/REPORT.md @@ -0,0 +1,26 @@ +# fault-isolation · Windows (10 reps) — reproduces clean + +> **Verdict on Windows: ✅ RocketRide survives 10/10.** A node that hard-crashes (`os._exit(134)`, +> simulating a native/segfault crash) kills **only that run's subprocess** — the server and sibling runs +> survive, and a fresh healthy run works immediately after, in **all 10 reps**. The in-process baseline — +> **real LangChain** `.abatch` running 4 tasks in one interpreter — loses **everything (0/4)** to one +> such crash (process dies, **exit 134**) in **10/10**. Process-per-run isolation reproduces faithfully on +> Windows. + +## Method +Identical to `../../runs/fault-isolation/REPORT.md`. RocketRide: a healthy run, then a `mode="crash"` run +whose node `os._exit(134)`s its own task subprocess, then another healthy run — checking the server pid +is unchanged and new runs still complete. In-process baseline: real `langchain_core` `RunnableLambda.abatch` +over 4 legs, one of which `os._exit(134)`s the shared interpreter. **10 reps back-to-back** (no restart +needed — `ttl=0`), Windows, port 5566. + +## Results *(10 reps; Windows 11, engine 3.2.1.30, langchain-core 0.3.86)* +| Side | Outcome | +|---|---| +| **Stock RocketRide** | ✅ `rr_isolation_holds` **true in 10/10** — healthy-before ✓, crash isolated (server survived), healthy-after ✓ | +| In-process LangChain `.abatch` (healthy control) | ✓ completes 4/4 | +| In-process LangChain `.abatch` (one leg crashes) | ❌ **loses ALL 0/4** — process dies, **exit 134**, in 10/10 | + +## Provenance +10 reps (`run-01/ … run-10/`), back-to-back on the standing engine (port 5566); real LangChain `0.3.86` +proven per rep (`in_process_baseline.lc_version`); no trace for this bench (matches the macOS convention). diff --git a/concurrent-work/runs-windows/fault-isolation/run-01/results.json b/concurrent-work/runs-windows/fault-isolation/run-01/results.json new file mode 100644 index 0000000..34b92b8 --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-01/results.json @@ -0,0 +1,43 @@ +{ + "benchmark": "fault-isolation", + "old_id": "D5", + "hypothesis": "a hard node crash is isolated to its run; the server + siblings survive", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "rocketride": { + "server_pid_before": 19204, + "healthy_before_ok": true, + "crash_run_completed": false, + "server_survived_crash": true, + "healthy_after_ok": true + }, + "in_process_baseline": { + "framework": "langchain", + "tasks": 4, + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "healthy_completed": 4, + "completed": 0, + "process_exit_code": 134, + "died": true + }, + "verdict_metrics": { + "rr_isolation_holds": true, + "inproc_framework": "langchain", + "inproc_healthy_ok": true, + "inproc_lost_all": true + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/fault-isolation/run-01/run.log b/concurrent-work/runs-windows/fault-isolation/run-01/run.log new file mode 100644 index 0000000..d2ef7ac --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-01/run.log @@ -0,0 +1,4 @@ +wrote: canvas.mmd canvas.dot +RocketRide: healthy-before=True crash-run-completed=False server-survived=True healthy-after=True +In-process baseline (langchain): healthy 4/4, crash-run 0/4, exit 134 (process DIED) +VERDICT: RR isolation holds = True ; in-process lost all = True diff --git a/concurrent-work/runs-windows/fault-isolation/run-02/results.json b/concurrent-work/runs-windows/fault-isolation/run-02/results.json new file mode 100644 index 0000000..34b92b8 --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-02/results.json @@ -0,0 +1,43 @@ +{ + "benchmark": "fault-isolation", + "old_id": "D5", + "hypothesis": "a hard node crash is isolated to its run; the server + siblings survive", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "rocketride": { + "server_pid_before": 19204, + "healthy_before_ok": true, + "crash_run_completed": false, + "server_survived_crash": true, + "healthy_after_ok": true + }, + "in_process_baseline": { + "framework": "langchain", + "tasks": 4, + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "healthy_completed": 4, + "completed": 0, + "process_exit_code": 134, + "died": true + }, + "verdict_metrics": { + "rr_isolation_holds": true, + "inproc_framework": "langchain", + "inproc_healthy_ok": true, + "inproc_lost_all": true + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/fault-isolation/run-02/run.log b/concurrent-work/runs-windows/fault-isolation/run-02/run.log new file mode 100644 index 0000000..d2ef7ac --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-02/run.log @@ -0,0 +1,4 @@ +wrote: canvas.mmd canvas.dot +RocketRide: healthy-before=True crash-run-completed=False server-survived=True healthy-after=True +In-process baseline (langchain): healthy 4/4, crash-run 0/4, exit 134 (process DIED) +VERDICT: RR isolation holds = True ; in-process lost all = True diff --git a/concurrent-work/runs-windows/fault-isolation/run-03/results.json b/concurrent-work/runs-windows/fault-isolation/run-03/results.json new file mode 100644 index 0000000..34b92b8 --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-03/results.json @@ -0,0 +1,43 @@ +{ + "benchmark": "fault-isolation", + "old_id": "D5", + "hypothesis": "a hard node crash is isolated to its run; the server + siblings survive", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "rocketride": { + "server_pid_before": 19204, + "healthy_before_ok": true, + "crash_run_completed": false, + "server_survived_crash": true, + "healthy_after_ok": true + }, + "in_process_baseline": { + "framework": "langchain", + "tasks": 4, + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "healthy_completed": 4, + "completed": 0, + "process_exit_code": 134, + "died": true + }, + "verdict_metrics": { + "rr_isolation_holds": true, + "inproc_framework": "langchain", + "inproc_healthy_ok": true, + "inproc_lost_all": true + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/fault-isolation/run-03/run.log b/concurrent-work/runs-windows/fault-isolation/run-03/run.log new file mode 100644 index 0000000..d2ef7ac --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-03/run.log @@ -0,0 +1,4 @@ +wrote: canvas.mmd canvas.dot +RocketRide: healthy-before=True crash-run-completed=False server-survived=True healthy-after=True +In-process baseline (langchain): healthy 4/4, crash-run 0/4, exit 134 (process DIED) +VERDICT: RR isolation holds = True ; in-process lost all = True diff --git a/concurrent-work/runs-windows/fault-isolation/run-04/results.json b/concurrent-work/runs-windows/fault-isolation/run-04/results.json new file mode 100644 index 0000000..34b92b8 --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-04/results.json @@ -0,0 +1,43 @@ +{ + "benchmark": "fault-isolation", + "old_id": "D5", + "hypothesis": "a hard node crash is isolated to its run; the server + siblings survive", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "rocketride": { + "server_pid_before": 19204, + "healthy_before_ok": true, + "crash_run_completed": false, + "server_survived_crash": true, + "healthy_after_ok": true + }, + "in_process_baseline": { + "framework": "langchain", + "tasks": 4, + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "healthy_completed": 4, + "completed": 0, + "process_exit_code": 134, + "died": true + }, + "verdict_metrics": { + "rr_isolation_holds": true, + "inproc_framework": "langchain", + "inproc_healthy_ok": true, + "inproc_lost_all": true + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/fault-isolation/run-04/run.log b/concurrent-work/runs-windows/fault-isolation/run-04/run.log new file mode 100644 index 0000000..d2ef7ac --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-04/run.log @@ -0,0 +1,4 @@ +wrote: canvas.mmd canvas.dot +RocketRide: healthy-before=True crash-run-completed=False server-survived=True healthy-after=True +In-process baseline (langchain): healthy 4/4, crash-run 0/4, exit 134 (process DIED) +VERDICT: RR isolation holds = True ; in-process lost all = True diff --git a/concurrent-work/runs-windows/fault-isolation/run-05/results.json b/concurrent-work/runs-windows/fault-isolation/run-05/results.json new file mode 100644 index 0000000..34b92b8 --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-05/results.json @@ -0,0 +1,43 @@ +{ + "benchmark": "fault-isolation", + "old_id": "D5", + "hypothesis": "a hard node crash is isolated to its run; the server + siblings survive", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "rocketride": { + "server_pid_before": 19204, + "healthy_before_ok": true, + "crash_run_completed": false, + "server_survived_crash": true, + "healthy_after_ok": true + }, + "in_process_baseline": { + "framework": "langchain", + "tasks": 4, + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "healthy_completed": 4, + "completed": 0, + "process_exit_code": 134, + "died": true + }, + "verdict_metrics": { + "rr_isolation_holds": true, + "inproc_framework": "langchain", + "inproc_healthy_ok": true, + "inproc_lost_all": true + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/fault-isolation/run-05/run.log b/concurrent-work/runs-windows/fault-isolation/run-05/run.log new file mode 100644 index 0000000..d2ef7ac --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-05/run.log @@ -0,0 +1,4 @@ +wrote: canvas.mmd canvas.dot +RocketRide: healthy-before=True crash-run-completed=False server-survived=True healthy-after=True +In-process baseline (langchain): healthy 4/4, crash-run 0/4, exit 134 (process DIED) +VERDICT: RR isolation holds = True ; in-process lost all = True diff --git a/concurrent-work/runs-windows/fault-isolation/run-06/results.json b/concurrent-work/runs-windows/fault-isolation/run-06/results.json new file mode 100644 index 0000000..34b92b8 --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-06/results.json @@ -0,0 +1,43 @@ +{ + "benchmark": "fault-isolation", + "old_id": "D5", + "hypothesis": "a hard node crash is isolated to its run; the server + siblings survive", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "rocketride": { + "server_pid_before": 19204, + "healthy_before_ok": true, + "crash_run_completed": false, + "server_survived_crash": true, + "healthy_after_ok": true + }, + "in_process_baseline": { + "framework": "langchain", + "tasks": 4, + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "healthy_completed": 4, + "completed": 0, + "process_exit_code": 134, + "died": true + }, + "verdict_metrics": { + "rr_isolation_holds": true, + "inproc_framework": "langchain", + "inproc_healthy_ok": true, + "inproc_lost_all": true + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/fault-isolation/run-06/run.log b/concurrent-work/runs-windows/fault-isolation/run-06/run.log new file mode 100644 index 0000000..d2ef7ac --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-06/run.log @@ -0,0 +1,4 @@ +wrote: canvas.mmd canvas.dot +RocketRide: healthy-before=True crash-run-completed=False server-survived=True healthy-after=True +In-process baseline (langchain): healthy 4/4, crash-run 0/4, exit 134 (process DIED) +VERDICT: RR isolation holds = True ; in-process lost all = True diff --git a/concurrent-work/runs-windows/fault-isolation/run-07/results.json b/concurrent-work/runs-windows/fault-isolation/run-07/results.json new file mode 100644 index 0000000..34b92b8 --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-07/results.json @@ -0,0 +1,43 @@ +{ + "benchmark": "fault-isolation", + "old_id": "D5", + "hypothesis": "a hard node crash is isolated to its run; the server + siblings survive", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "rocketride": { + "server_pid_before": 19204, + "healthy_before_ok": true, + "crash_run_completed": false, + "server_survived_crash": true, + "healthy_after_ok": true + }, + "in_process_baseline": { + "framework": "langchain", + "tasks": 4, + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "healthy_completed": 4, + "completed": 0, + "process_exit_code": 134, + "died": true + }, + "verdict_metrics": { + "rr_isolation_holds": true, + "inproc_framework": "langchain", + "inproc_healthy_ok": true, + "inproc_lost_all": true + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/fault-isolation/run-07/run.log b/concurrent-work/runs-windows/fault-isolation/run-07/run.log new file mode 100644 index 0000000..d2ef7ac --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-07/run.log @@ -0,0 +1,4 @@ +wrote: canvas.mmd canvas.dot +RocketRide: healthy-before=True crash-run-completed=False server-survived=True healthy-after=True +In-process baseline (langchain): healthy 4/4, crash-run 0/4, exit 134 (process DIED) +VERDICT: RR isolation holds = True ; in-process lost all = True diff --git a/concurrent-work/runs-windows/fault-isolation/run-08/results.json b/concurrent-work/runs-windows/fault-isolation/run-08/results.json new file mode 100644 index 0000000..34b92b8 --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-08/results.json @@ -0,0 +1,43 @@ +{ + "benchmark": "fault-isolation", + "old_id": "D5", + "hypothesis": "a hard node crash is isolated to its run; the server + siblings survive", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "rocketride": { + "server_pid_before": 19204, + "healthy_before_ok": true, + "crash_run_completed": false, + "server_survived_crash": true, + "healthy_after_ok": true + }, + "in_process_baseline": { + "framework": "langchain", + "tasks": 4, + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "healthy_completed": 4, + "completed": 0, + "process_exit_code": 134, + "died": true + }, + "verdict_metrics": { + "rr_isolation_holds": true, + "inproc_framework": "langchain", + "inproc_healthy_ok": true, + "inproc_lost_all": true + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/fault-isolation/run-08/run.log b/concurrent-work/runs-windows/fault-isolation/run-08/run.log new file mode 100644 index 0000000..d2ef7ac --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-08/run.log @@ -0,0 +1,4 @@ +wrote: canvas.mmd canvas.dot +RocketRide: healthy-before=True crash-run-completed=False server-survived=True healthy-after=True +In-process baseline (langchain): healthy 4/4, crash-run 0/4, exit 134 (process DIED) +VERDICT: RR isolation holds = True ; in-process lost all = True diff --git a/concurrent-work/runs-windows/fault-isolation/run-09/results.json b/concurrent-work/runs-windows/fault-isolation/run-09/results.json new file mode 100644 index 0000000..34b92b8 --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-09/results.json @@ -0,0 +1,43 @@ +{ + "benchmark": "fault-isolation", + "old_id": "D5", + "hypothesis": "a hard node crash is isolated to its run; the server + siblings survive", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "rocketride": { + "server_pid_before": 19204, + "healthy_before_ok": true, + "crash_run_completed": false, + "server_survived_crash": true, + "healthy_after_ok": true + }, + "in_process_baseline": { + "framework": "langchain", + "tasks": 4, + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "healthy_completed": 4, + "completed": 0, + "process_exit_code": 134, + "died": true + }, + "verdict_metrics": { + "rr_isolation_holds": true, + "inproc_framework": "langchain", + "inproc_healthy_ok": true, + "inproc_lost_all": true + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/fault-isolation/run-09/run.log b/concurrent-work/runs-windows/fault-isolation/run-09/run.log new file mode 100644 index 0000000..d2ef7ac --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-09/run.log @@ -0,0 +1,4 @@ +wrote: canvas.mmd canvas.dot +RocketRide: healthy-before=True crash-run-completed=False server-survived=True healthy-after=True +In-process baseline (langchain): healthy 4/4, crash-run 0/4, exit 134 (process DIED) +VERDICT: RR isolation holds = True ; in-process lost all = True diff --git a/concurrent-work/runs-windows/fault-isolation/run-10/results.json b/concurrent-work/runs-windows/fault-isolation/run-10/results.json new file mode 100644 index 0000000..34b92b8 --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-10/results.json @@ -0,0 +1,43 @@ +{ + "benchmark": "fault-isolation", + "old_id": "D5", + "hypothesis": "a hard node crash is isolated to its run; the server + siblings survive", + "provenance": { + "engine_dir": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\engine", + "engine_version": "Version: 3.2.1.30 hash: 114509c6 stamp: 2026-05-29T19:19:06Z", + "uri": "ws://localhost:5566", + "cpu_brand": "Intel64 Family 6 Model 186 Stepping 3, GenuineIntel", + "machine": "Windows-11-10.0.26200-SP0", + "arch": "AMD64", + "physical_cores": 10, + "logical_cores": 12, + "total_ram_gib": 15.55, + "python": "3.12.3", + "engine_sha256": "b844140ae82cc18e9b5767b44b897aed2f772ea5228cfd053d727324bad7dc5e", + "engine_size_bytes": 84793856 + }, + "rocketride": { + "server_pid_before": 19204, + "healthy_before_ok": true, + "crash_run_completed": false, + "server_survived_crash": true, + "healthy_after_ok": true + }, + "in_process_baseline": { + "framework": "langchain", + "tasks": 4, + "lc_version": "0.3.86", + "lc_python": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Scripts\\python.exe", + "langchain_core_file": "D:\\Coding\\rr\\rocketride-benchmark\\concurrent-work\\harness\\rocketride-bench\\.venv\\Lib\\site-packages\\langchain_core\\__init__.py", + "healthy_completed": 4, + "completed": 0, + "process_exit_code": 134, + "died": true + }, + "verdict_metrics": { + "rr_isolation_holds": true, + "inproc_framework": "langchain", + "inproc_healthy_ok": true, + "inproc_lost_all": true + } +} \ No newline at end of file diff --git a/concurrent-work/runs-windows/fault-isolation/run-10/run.log b/concurrent-work/runs-windows/fault-isolation/run-10/run.log new file mode 100644 index 0000000..d2ef7ac --- /dev/null +++ b/concurrent-work/runs-windows/fault-isolation/run-10/run.log @@ -0,0 +1,4 @@ +wrote: canvas.mmd canvas.dot +RocketRide: healthy-before=True crash-run-completed=False server-survived=True healthy-after=True +In-process baseline (langchain): healthy 4/4, crash-run 0/4, exit 134 (process DIED) +VERDICT: RR isolation holds = True ; in-process lost all = True diff --git a/concurrent-work/runs-windows/lines-of-code.run.log b/concurrent-work/runs-windows/lines-of-code.run.log new file mode 100644 index 0000000..bd82247 --- /dev/null +++ b/concurrent-work/runs-windows/lines-of-code.run.log @@ -0,0 +1,22 @@ +{ + "n_builds": 5, + "workflow": "meeting-notes assistant (transcript -> executive summary + follow-up email)", + "rocketride_lines": { + "min": 104, + "median": 123, + "max": 136 + }, + "langchain_lines": { + "min": 682, + "median": 689, + "max": 930 + }, + "line_ratio_lc_over_rr": { + "min": 5.0, + "median": 6.0, + "max": 7.6 + }, + "rocketride_chars_median": 7010, + "langchain_chars_median": 25158, + "char_ratio_lc_over_rr_median": 3.6 +}