diff --git a/perf/runtime/README.md b/perf/runtime/README.md index 25543fe..068196a 100644 --- a/perf/runtime/README.md +++ b/perf/runtime/README.md @@ -15,12 +15,18 @@ The command reports: - repeated child lookup and delivery to one running child; - machine start-and-stop throughput; - parent-with-child start-and-stop throughput; +- Effect-only lifecycle reference points for a suspended fiber, a queue worker, + a minimal actor shell, and a two-shell family; +- Effect-only coordination reference points for an owner-only mutable snapshot, + a synchronized snapshot, and a terminal `Deferred` latch; - heap and resident-memory growth at 100, 500, and 1,000 live units, including - a raw managed process, an idle statechart, two independent statecharts, a - parent with one child, that relationship with child-registry observation - active, and an invoked child whose active snapshots are observed; + a raw generic process, a raw compiled process, an idle statechart, two + independent statecharts, a parent with one child, that relationship with + child-registry observation active, and an invoked child whose active snapshots + are observed; - lower-bound memory profiles for Effect itself: a suspended fiber, a queue - with a waiting fiber, and a minimal mailbox/state/completion actor shell. + with a waiting fiber, a minimal mailbox/state/completion actor shell, and a + minimal two-shell family. The comparison dependencies use package aliases, so XState 5 and 6 can be loaded by the same process: @@ -62,6 +68,11 @@ registry and invoked-snapshot observation. Invoked snapshot mapping uses a direct, state-scoped delivery path; its profile measures the retained callback and mapping state rather than a general `changes` stream subscription. The Effect profiles are primitive lower bounds, not feature-equivalent competitors. +The Effect throughput reference points similarly bound individual runtime +operations rather than predicting a complete machine by themselves. In +particular, the owner-only mutable snapshot is safe only when one process fiber +owns active state writes; terminal arbitration and externally visible +observation still require separate coordination. Resident memory is reported as a raw diagnostic because V8 and the operating-system allocator can reuse already committed pages. The capacity-per-GiB value is a linear estimate that excludes shared process @@ -96,6 +107,7 @@ to update the comment. The implementation lives in `scripts/runtime-performance.mjs`; the Effect Machine fixture is in `perf/runtime/counter.mjs`, and the comparison adapter is -in `perf/runtime/xstate.mjs`. Add new scenarios only when every implementation -performs equivalent observable work and the result is consumed and checked so -the JavaScript engine cannot discard it. +in `perf/runtime/xstate.mjs`. Effect runtime reference fixtures are in +`perf/runtime/effect-runtime.mjs`. Add cross-library scenarios only when every +implementation performs equivalent observable work and the result is consumed +and checked so the JavaScript engine cannot discard it. diff --git a/perf/runtime/counter.mjs b/perf/runtime/counter.mjs index be0b15e..317c8a7 100644 --- a/perf/runtime/counter.mjs +++ b/perf/runtime/counter.mjs @@ -229,6 +229,12 @@ const rawProcessLogic = { run: () => Effect.never } +const rawCompiledProcessLogic = { + [machineRuntime.compiledProcess]: true, + initial: () => Effect.succeed(0), + run: () => Effect.never +} + const startRawProcesses = (count) => Effect.runPromise( Effect.forEach( @@ -238,6 +244,15 @@ const startRawProcesses = (count) => ) ) +const startCompiledRawProcesses = (count) => + Effect.runPromise( + Effect.forEach( + Array.from({ length: count }), + () => machineRuntime.startProcess(rawCompiledProcessLogic), + { concurrency: 1 } + ) + ) + const startIndependentCounterPairs = (count) => Effect.runPromise( Effect.forEach( @@ -330,6 +345,23 @@ export const effectMachineAdapter = { stopChildCounters, stopObservedCounter, stopCounters, + runtimeBenchmarks: [ + { + id: "compiled-process-start-stop", + label: "Start and stop a raw compiled process", + unit: "processes/s", + async: true, + operations: () => 1, + run: async () => { + const refs = await startCompiledRawProcesses(1) + try { + return refs.length + } finally { + await stopCounters(refs) + } + } + } + ], memoryProfiles: { idle: { label: "Idle machine", @@ -337,10 +369,15 @@ export const effectMachineAdapter = { stop: stopCounters }, "raw-process": { - label: "Raw managed process", + label: "Raw generic managed process", start: startRawProcesses, stop: stopCounters }, + "compiled-raw-process": { + label: "Raw compiled process", + start: startCompiledRawProcesses, + stop: stopCounters + }, "two-independent": { label: "Two independent idle machines", start: startIndependentCounterPairs, diff --git a/perf/runtime/effect-memory.mjs b/perf/runtime/effect-memory.mjs deleted file mode 100644 index df329e7..0000000 --- a/perf/runtime/effect-memory.mjs +++ /dev/null @@ -1,81 +0,0 @@ -import { createRequire } from "node:module" -import { readFileSync } from "node:fs" -import { dirname, join, resolve } from "node:path" -import { fileURLToPath, pathToFileURL } from "node:url" - -const implementationRoot = resolve( - process.env.EFFECT_MACHINE_BENCHMARK_ROOT ?? fileURLToPath(new URL("../..", import.meta.url)) -) -const implementationRequire = createRequire(pathToFileURL(join(implementationRoot, "package.json"))) -const effectPackagePath = implementationRequire.resolve("effect/package.json") -const effectPackage = JSON.parse(readFileSync(effectPackagePath, "utf8")) -const effect = await import(pathToFileURL(resolve(dirname(effectPackagePath), effectPackage.exports["."])).href) -const { Deferred, Effect, Fiber, Queue, Ref } = effect - -const startUnits = (count, make) => - Effect.runPromise( - Effect.forEach(Array.from({ length: count }), make, { concurrency: 1 }) - ) - -const stopUnits = (units) => - Effect.runPromise( - Effect.forEach( - units, - (unit) => - Fiber.interruptAll(unit.fibers).pipe( - Effect.andThen(Effect.forEach(unit.queues, Queue.shutdown, { discard: true })) - ), - { concurrency: "unbounded", discard: true } - ) - ) - -const makeFiber = () => - Effect.map(Effect.forkDetach(Effect.never), (fiber) => ({ - fibers: [fiber], - queues: [] - })) - -const makeMailbox = () => - Effect.gen(function*() { - const queue = yield* Queue.unbounded() - const fiber = yield* Effect.forkDetach(Effect.forever(Queue.take(queue))) - return { fibers: [fiber], queues: [queue] } - }) - -const makeActorShell = () => - Effect.gen(function*() { - const queue = yield* Queue.unbounded() - const state = yield* Ref.make(0) - const done = yield* Deferred.make() - const fiber = yield* Effect.forkDetach( - Effect.forever( - Queue.take(queue).pipe( - Effect.flatMap((update) => Ref.update(state, update)) - ) - ) - ) - return { fibers: [fiber], queues: [queue], retained: [state, done] } - }) - -export const makeEffectMemoryAdapter = (version) => ({ - implementation: "effect-runtime", - label: "Effect runtime primitives", - version, - memoryProfiles: { - "effect-fiber": { - label: "Suspended Effect fiber", - start: (count) => startUnits(count, makeFiber), - stop: stopUnits - }, - "effect-mailbox": { - label: "Effect queue with waiting fiber", - start: (count) => startUnits(count, makeMailbox), - stop: stopUnits - }, - "effect-actor-shell": { - label: "Effect mailbox actor shell", - start: (count) => startUnits(count, makeActorShell), - stop: stopUnits - } - } -}) diff --git a/perf/runtime/effect-runtime.mjs b/perf/runtime/effect-runtime.mjs new file mode 100644 index 0000000..e62d55b --- /dev/null +++ b/perf/runtime/effect-runtime.mjs @@ -0,0 +1,198 @@ +import { createRequire } from "node:module" +import { readFileSync } from "node:fs" +import { dirname, join, resolve } from "node:path" +import { fileURLToPath, pathToFileURL } from "node:url" + +const implementationRoot = resolve( + process.env.EFFECT_MACHINE_BENCHMARK_ROOT ?? fileURLToPath(new URL("../..", import.meta.url)) +) +const implementationRequire = createRequire(pathToFileURL(join(implementationRoot, "package.json"))) +const effectPackagePath = implementationRequire.resolve("effect/package.json") +const effectPackage = JSON.parse(readFileSync(effectPackagePath, "utf8")) +const effect = await import(pathToFileURL(resolve(dirname(effectPackagePath), effectPackage.exports["."])).href) +const { Deferred, Effect, Fiber, MutableRef, Queue, Ref, SynchronizedRef } = effect + +const startUnits = (count, make) => + Effect.runPromise( + Effect.forEach(Array.from({ length: count }), make, { concurrency: 1 }) + ) + +const stopUnits = (units) => + Effect.runPromise( + Effect.forEach( + units, + (unit) => + Fiber.interruptAll(unit.fibers).pipe( + Effect.andThen(Effect.forEach(unit.queues, Queue.shutdown, { discard: true })) + ), + { concurrency: "unbounded", discard: true } + ) + ) + +const makeFiber = () => + Effect.map(Effect.forkDetach(Effect.never), (fiber) => ({ + fibers: [fiber], + queues: [] + })) + +const makeMailbox = () => + Effect.gen(function*() { + const queue = yield* Queue.unbounded() + const fiber = yield* Effect.forkDetach(Effect.forever(Queue.take(queue))) + return { fibers: [fiber], queues: [queue] } + }) + +const makeActorShell = () => + Effect.gen(function*() { + const queue = yield* Queue.unbounded() + const state = yield* Ref.make(0) + const done = yield* Deferred.make() + const fiber = yield* Effect.forkDetach( + Effect.forever( + Queue.take(queue).pipe( + Effect.flatMap((update) => Ref.update(state, update)) + ) + ) + ) + return { fibers: [fiber], queues: [queue], retained: [state, done] } + }) + +const makeActorFamily = () => + Effect.all([makeActorShell(), makeActorShell()], { concurrency: 1 }).pipe( + Effect.map((units) => ({ + fibers: units.flatMap((unit) => unit.fibers), + queues: units.flatMap((unit) => unit.queues), + retained: units + })) + ) + +const runLifecycle = async (make, expectedUnits = 1) => { + const units = await startUnits(expectedUnits, make) + try { + if (units.length !== expectedUnits) { + throw new Error(`Effect runtime lifecycle started ${units.length} units, expected ${expectedUnits}`) + } + return 1 + } finally { + await stopUnits(units) + } +} + +const mutableSnapshot = MutableRef.make(0) +const runMutableSnapshotBatch = (size) => { + const before = MutableRef.get(mutableSnapshot) + for (let index = 0; index < size; index += 1) { + MutableRef.update(mutableSnapshot, (value) => value + 1) + } + return MutableRef.get(mutableSnapshot) - before +} + +const synchronizedSnapshot = Effect.runSync(SynchronizedRef.make(0)) +const runSynchronizedSnapshotBatch = (size) => + Effect.runPromise( + Effect.gen(function*() { + const before = yield* SynchronizedRef.get(synchronizedSnapshot) + for (let index = 0; index < size; index += 1) { + yield* SynchronizedRef.update(synchronizedSnapshot, (value) => value + 1) + } + return (yield* SynchronizedRef.get(synchronizedSnapshot)) - before + }) + ) + +const runTerminalLatchBatch = (size) => + Effect.runPromise( + Effect.gen(function*() { + for (let index = 0; index < size; index += 1) { + const latch = yield* Deferred.make() + yield* Deferred.succeed(latch, undefined) + yield* Deferred.await(latch) + } + return size + }) + ) + +export const makeEffectRuntimeAdapter = (version) => ({ + implementation: "effect-runtime", + label: "Effect runtime primitives", + version, + runtimeBenchmarks: [ + { + id: "effect-fiber-start-stop", + label: "Start and interrupt a suspended fiber", + unit: "fibers/s", + async: true, + operations: () => 1, + run: () => runLifecycle(makeFiber) + }, + { + id: "effect-mailbox-start-stop", + label: "Start and stop a queue worker", + unit: "workers/s", + async: true, + operations: () => 1, + run: () => runLifecycle(makeMailbox) + }, + { + id: "effect-actor-shell-start-stop", + label: "Start and stop an actor shell", + unit: "actors/s", + async: true, + operations: () => 1, + run: () => runLifecycle(makeActorShell) + }, + { + id: "effect-actor-family-start-stop", + label: "Start and stop two actor shells", + unit: "families/s", + async: true, + operations: () => 1, + run: () => runLifecycle(makeActorFamily) + }, + { + id: "effect-mutable-snapshot-update", + label: "Update an owner-only mutable snapshot", + unit: "updates/s", + async: false, + operations: (configuration) => configuration.primitiveBatchSize, + run: runMutableSnapshotBatch + }, + { + id: "effect-synchronized-snapshot-update", + label: "Update a synchronized snapshot", + unit: "updates/s", + async: true, + operations: (configuration) => configuration.primitiveBatchSize, + run: runSynchronizedSnapshotBatch + }, + { + id: "effect-terminal-latch", + label: "Create, resolve, and await a terminal latch", + unit: "latches/s", + async: true, + operations: (configuration) => configuration.primitiveBatchSize, + run: runTerminalLatchBatch + } + ], + memoryProfiles: { + "effect-fiber": { + label: "Suspended Effect fiber", + start: (count) => startUnits(count, makeFiber), + stop: stopUnits + }, + "effect-mailbox": { + label: "Effect queue with waiting fiber", + start: (count) => startUnits(count, makeMailbox), + stop: stopUnits + }, + "effect-actor-shell": { + label: "Effect mailbox actor shell", + start: (count) => startUnits(count, makeActorShell), + stop: stopUnits + }, + "effect-actor-family": { + label: "Two Effect actor shells", + start: (count) => startUnits(count, makeActorFamily), + stop: stopUnits + } + } +}) diff --git a/perf/runtime/implementations.mjs b/perf/runtime/implementations.mjs index 2181618..486da1b 100644 --- a/perf/runtime/implementations.mjs +++ b/perf/runtime/implementations.mjs @@ -4,7 +4,7 @@ import { fileURLToPath } from "node:url" import * as XStateV5 from "xstate-v5" import * as XStateV6 from "xstate-v6" import { effectMachineAdapter } from "./counter.mjs" -import { makeEffectMemoryAdapter } from "./effect-memory.mjs" +import { makeEffectRuntimeAdapter } from "./effect-runtime.mjs" import { makeXStateAdapter } from "./xstate.mjs" const readPackageVersion = (path) => JSON.parse(readFileSync(path, "utf8")).version @@ -36,7 +36,13 @@ export const implementations = [ }) ] +export const effectRuntimeImplementation = makeEffectRuntimeAdapter(packageVersions.effect) +export const runtimeReferenceImplementations = [ + ...implementations.filter((implementation) => implementation.runtimeBenchmarks !== undefined), + effectRuntimeImplementation +] + export const memoryImplementations = [ ...implementations, - makeEffectMemoryAdapter(packageVersions.effect) + effectRuntimeImplementation ] diff --git a/scripts/compare-runtime-performance.mjs b/scripts/compare-runtime-performance.mjs index 7bbbe97..5cfbbbf 100644 --- a/scripts/compare-runtime-performance.mjs +++ b/scripts/compare-runtime-performance.mjs @@ -44,6 +44,7 @@ const validateReport = (report, path) => { !isShortString(benchmark.id, 100) || !isShortString(benchmark.label, 200) || !isShortString(benchmark.unit, 100) || + (benchmark.group !== undefined && benchmark.group !== "machine" && benchmark.group !== "effect-runtime") || !isBoundedNumber(benchmark.medianThroughput) || benchmark.medianThroughput < 0 || !isBoundedNumber(benchmark.relativeMarginOfError) || @@ -256,8 +257,17 @@ const formatDifference = (before, after) => { : `${sign}${decimal.format(percentage)}%` } +const machineBenchmarks = pullRequest.benchmarks.filter((benchmark) => benchmark.group !== "effect-runtime") +const runtimeBenchmarks = pullRequest.benchmarks.filter((benchmark) => benchmark.group === "effect-runtime") const benchmarkImplementations = [...new Map( - pullRequest.benchmarks.map((benchmark) => [benchmark.implementation, { + machineBenchmarks.map((benchmark) => [benchmark.implementation, { + id: benchmark.implementation, + label: benchmark.implementationLabel, + version: benchmark.implementationVersion + }]) +).values()] +const runtimeImplementations = [...new Map( + runtimeBenchmarks.map((benchmark) => [benchmark.implementation, { id: benchmark.implementation, label: benchmark.implementationLabel, version: benchmark.implementationVersion @@ -269,10 +279,20 @@ const memoryImplementations = pullRequest.memory.map((measurement) => ({ version: measurement.implementationVersion })) const reportedImplementations = [...new Map( - [...benchmarkImplementations, ...memoryImplementations].map((implementation) => [implementation.id, implementation]) + [...benchmarkImplementations, ...runtimeImplementations, ...memoryImplementations].map((implementation) => [ + implementation.id, + implementation + ]) ).values()] -const scenarios = [...new Map( - pullRequest.benchmarks.map((benchmark) => [benchmark.id, { +const machineScenarios = [...new Map( + machineBenchmarks.map((benchmark) => [benchmark.id, { + id: benchmark.id, + label: benchmark.label, + unit: benchmark.unit + }]) +).values()] +const runtimeScenarios = [...new Map( + runtimeBenchmarks.map((benchmark) => [benchmark.id, { id: benchmark.id, label: benchmark.label, unit: benchmark.unit @@ -290,7 +310,7 @@ const lines = [ `| --- | ${benchmarkImplementations.map(() => "---:").join(" | ")} |` ] -for (const scenario of scenarios) { +for (const scenario of machineScenarios) { lines.push( `| ${escapeCell(scenario.label)} | ${benchmarkImplementations.map((implementation) => { const benchmark = pullRequest.benchmarks.find((candidate) => @@ -301,6 +321,26 @@ for (const scenario of scenarios) { ) } +if (runtimeScenarios.length > 0) { + lines.push( + "", + "### Effect runtime reference points", + "", + `| Scenario | ${runtimeImplementations.map((implementation) => escapeCell(implementation.label)).join(" | ")} |`, + `| --- | ${runtimeImplementations.map(() => "---:").join(" | ")} |` + ) + for (const scenario of runtimeScenarios) { + lines.push( + `| ${escapeCell(scenario.label)} | ${runtimeImplementations.map((implementation) => { + const benchmark = runtimeBenchmarks.find((candidate) => + candidate.implementation === implementation.id && candidate.id === scenario.id + ) + return benchmark === undefined ? "—" : formatThroughput(benchmark.medianThroughput, benchmark.unit) + }).join(" | ")} |` + ) + } +} + const memoryProfiles = [...new Map( pullRequest.memory.flatMap((measurement) => (measurement.profiles ?? []).map((profile) => [profile.id, { id: profile.id, label: profile.label }]) @@ -349,7 +389,7 @@ if (base === undefined) { "| Metric | Base | Base variability | PR | PR variability | Difference |", "| --- | ---: | ---: | ---: | ---: | ---: |" ) - for (const scenario of scenarios) { + for (const scenario of machineScenarios) { const before = baseEffect.get(scenario.id) const after = pullRequestEffect.get(scenario.id) if (before !== undefined && after !== undefined && before.unit === after.unit) { @@ -370,6 +410,34 @@ if (base === undefined) { } } } + const baseRuntime = new Map( + base.benchmarks + .filter((benchmark) => benchmark.group === "effect-runtime") + .map((benchmark) => [benchmark.id, benchmark]) + ) + const pullRequestRuntime = new Map( + pullRequest.benchmarks + .filter((benchmark) => benchmark.group === "effect-runtime") + .map((benchmark) => [benchmark.id, benchmark]) + ) + if (baseRuntime.size > 0 && pullRequestRuntime.size > 0) { + lines.push( + "", + "### Effect runtime reference change from base", + "", + "| Metric | Base | Base variability | PR | PR variability | Difference |", + "| --- | ---: | ---: | ---: | ---: | ---: |" + ) + for (const scenario of runtimeScenarios) { + const before = baseRuntime.get(scenario.id) + const after = pullRequestRuntime.get(scenario.id) + if (before !== undefined && after !== undefined && before.unit === after.unit) { + lines.push( + `| ${escapeCell(scenario.label)} | ${formatThroughput(before.medianThroughput, before.unit)} | ${formatVariability(before.processRelativeMad)} | ${formatThroughput(after.medianThroughput, after.unit)} | ${formatVariability(after.processRelativeMad)} | ${formatDifference(before.medianThroughput, after.medianThroughput)} |` + ) + } + } + } } lines.push( diff --git a/scripts/runtime-performance.mjs b/scripts/runtime-performance.mjs index a529dcb..34534c6 100644 --- a/scripts/runtime-performance.mjs +++ b/scripts/runtime-performance.mjs @@ -5,7 +5,12 @@ import { resolve } from "node:path" import process from "node:process" import { fileURLToPath } from "node:url" import { Bench } from "tinybench" -import { implementations, memoryImplementations, packageVersions } from "../perf/runtime/implementations.mjs" +import { + implementations, + memoryImplementations, + packageVersions, + runtimeReferenceImplementations +} from "../perf/runtime/implementations.mjs" const sourceRevision = (() => { try { return { @@ -58,6 +63,7 @@ const configuration = options.quick planningBatchSize: 25, burstBatchSize: 25, childBatchSize: 10, + primitiveBatchSize: 100, memoryCounts: [25, 100] } : { @@ -68,6 +74,7 @@ const configuration = options.quick planningBatchSize: 100, burstBatchSize: 250, childBatchSize: 100, + primitiveBatchSize: 1_000, memoryCounts: [100, 500, 1_000] } @@ -87,7 +94,8 @@ for (const implementation of implementations) { const metadata = { implementation: implementation.implementation, implementationLabel: implementation.label, - implementationVersion: implementation.version + implementationVersion: implementation.version, + group: "machine" } const planningTask = `${implementation.implementation}:plan-counter` benchmarkDefinitions.set(planningTask, { @@ -257,6 +265,36 @@ for (const implementation of implementations) { ) } +for (const implementation of runtimeReferenceImplementations) { + for (const runtimeBenchmark of implementation.runtimeBenchmarks) { + const task = `${implementation.implementation}:${runtimeBenchmark.id}` + const operations = runtimeBenchmark.operations(configuration) + benchmarkDefinitions.set(task, { + implementation: implementation.implementation, + implementationLabel: implementation.label, + implementationVersion: implementation.version, + group: "effect-runtime", + id: runtimeBenchmark.id, + label: runtimeBenchmark.label, + unit: runtimeBenchmark.unit, + operationsPerIteration: operations + }) + const validate = (completed) => { + if (completed !== operations) { + throw new Error( + `${runtimeBenchmark.label} completed ${completed} operations, expected ${operations}` + ) + } + } + bench.add( + task, + runtimeBenchmark.async + ? async () => validate(await runtimeBenchmark.run(operations)) + : () => validate(runtimeBenchmark.run(operations)) + ) + } +} + try { await bench.warmup() await bench.run()