From 95806af395864cdc831d31ed95403307dfc54307 Mon Sep 17 00:00:00 2001 From: "0.7%" Date: Thu, 30 Apr 2026 01:56:56 +0700 Subject: [PATCH] feat: add internal benchmark suite --- benchmark/default.project.json | 39 ++++ benchmark/src/client/init.client.luau | 175 ++++++++++++++++++ benchmark/src/server/init.server.luau | 106 +++++++++++ benchmark/src/shared/benches/Booleans.luau | 6 + benchmark/src/shared/benches/Entities.luau | 13 ++ benchmark/src/shared/benches/Mixed.luau | 12 ++ benchmark/src/shared/benches/SingleValue.luau | 2 + benchmark/src/shared/benches/Strings.luau | 13 ++ benchmark/src/shared/benches/Vectors.luau | 6 + benchmark/src/shared/benches/init.luau | 9 + benchmark/src/shared/modes/bridgenet2.luau | 13 ++ benchmark/src/shared/modes/bytenet.luau | 42 +++++ benchmark/src/shared/modes/init.luau | 13 ++ benchmark/src/shared/modes/roblox.luau | 22 +++ benchmark/src/shared/modes/satset.luau | 102 ++++++++++ benchmark/src/shared/modes/warp.luau | 20 ++ 16 files changed, 593 insertions(+) create mode 100644 benchmark/default.project.json create mode 100644 benchmark/src/client/init.client.luau create mode 100644 benchmark/src/server/init.server.luau create mode 100644 benchmark/src/shared/benches/Booleans.luau create mode 100644 benchmark/src/shared/benches/Entities.luau create mode 100644 benchmark/src/shared/benches/Mixed.luau create mode 100644 benchmark/src/shared/benches/SingleValue.luau create mode 100644 benchmark/src/shared/benches/Strings.luau create mode 100644 benchmark/src/shared/benches/Vectors.luau create mode 100644 benchmark/src/shared/benches/init.luau create mode 100644 benchmark/src/shared/modes/bridgenet2.luau create mode 100644 benchmark/src/shared/modes/bytenet.luau create mode 100644 benchmark/src/shared/modes/init.luau create mode 100644 benchmark/src/shared/modes/roblox.luau create mode 100644 benchmark/src/shared/modes/satset.luau create mode 100644 benchmark/src/shared/modes/warp.luau diff --git a/benchmark/default.project.json b/benchmark/default.project.json new file mode 100644 index 0000000..6edd285 --- /dev/null +++ b/benchmark/default.project.json @@ -0,0 +1,39 @@ +{ + "name": "SatsetBenchmark", + "tree": { + "$className": "DataModel", + "ServerScriptService": { + "$className": "ServerScriptService", + "Server": { + "$path": "src/server" + } + }, + "StarterPlayer": { + "$className": "StarterPlayer", + "StarterPlayerScripts": { + "$className": "StarterPlayerScripts", + "Client": { + "$path": "src/client" + } + } + }, + "ReplicatedStorage": { + "$className": "ReplicatedStorage", + "Shared": { + "$path": "src/shared", + "GetRecieved": { + "$className": "RemoteFunction" + }, + "Generate": { + "$className": "RemoteEvent" + } + }, + "Packages": { + "$path": "../Packages", + "Satset": { + "$path": "../src" + } + } + } + } +} \ No newline at end of file diff --git a/benchmark/src/client/init.client.luau b/benchmark/src/client/init.client.luau new file mode 100644 index 0000000..5cc3436 --- /dev/null +++ b/benchmark/src/client/init.client.luau @@ -0,0 +1,175 @@ +-- Benchmark client. Fires 1000 events per frame for 10 seconds per tool per bench. +-- Measures FPS and bandwidth (Kbps) at percentile intervals. +local HttpService = game:GetService("HttpService") +local Players = game:GetService("Players") +local ReplicatedStorage = game:GetService("ReplicatedStorage") +local RunService = game:GetService("RunService") +local StarterGui = game:GetService("StarterGui") +local Stats = game:GetService("Stats") + +local Satset = require(ReplicatedStorage.Packages.Satset) +Satset.start() + +local MAXIMUM_FRAMERATE = 60 + +local Benches, Modes + +-- Minimize rendering overhead to isolate network performance. +local Camera = workspace.CurrentCamera +Camera.FieldOfView = 1 +Camera.CFrame = CFrame.new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) +Players.LocalPlayer.PlayerGui:ClearAllChildren() +for _, child in Players.LocalPlayer.PlayerScripts:GetChildren() do + if child ~= script then + child:Destroy() + end +end + +for _, item in Enum.CoreGuiType:GetEnumItems() do + while true do + local ok = pcall(function() + StarterGui:SetCoreGuiEnabled(item, false) + end) + if ok then + break + end + end +end + +type Benchmark = { + Sent: number, + Recieve: number, + Bandwidth: { number }, + Framerate: { number }, +} + +type Results = { + [string]: { + [string]: Benchmark, + }, +} + +local function percentile(samples: { number }, pct: number) + assert((pct // 1) == pct and pct >= 0 and pct <= 100, "Percentile must be an integer between 0 and 100") + local index = ((#samples * (pct / 100)) // 1) + index = math.max(index, 1) + return samples[index] +end + +local function waitForDrain() + print("Waiting for packets to drain.") + while Stats.DataSendKbps > 0.5 do + RunService.Heartbeat:Wait() + end + task.wait(5) +end + +local function runBenchmark(tool: string, bench: string): Benchmark + local total = 0 + local frames = 0 + local bandwidth = {} + local framerates = {} + + local sent = 0 + local data = Benches[bench] + local events = Modes[tool] + local event = events[bench] + local method + + if tool == "roblox" then + method = function(d) + event:FireServer(d) + end + elseif tool == "bridgenet2" then + -- BridgeNet2 client: bridge:Fire(data) + method = function(d) + event:Fire(d) + end + elseif tool == "warp" then + -- Warp client: client:Fire(reliable, ...) + method = function(d) + event:Fire(true, d) + end + else + -- Satset uses .fire(), ByteNet uses .send() + method = event.fire or event.Fire or event.send + end + + local connection = RunService.PostSimulation:Connect(function(dt: number) + total += dt + frames += 1 + + if total >= 1 then + total -= 1 + local scale = MAXIMUM_FRAMERATE / frames + table.insert(bandwidth, Stats.DataSendKbps * scale) + table.insert(framerates, frames) + frames = 0 + end + + for _ = 1, 1000 do + sent += 1 + method(data) + end + end) + + task.wait(10) + connection:Disconnect() + print("> Finished running with " .. tool) + + -- Sort for percentile calculation. + table.sort(bandwidth) + table.sort(framerates, function(a, b) + return a > b + end) + + local frameratePercentiles = {} + local bandwidthPercentiles = {} + + for _, pct in { 50, 0, 80, 90, 95, 100 } do + table.insert(bandwidthPercentiles, percentile(bandwidth, pct)) + table.insert(frameratePercentiles, percentile(framerates, pct)) + end + + return { + Sent = sent, + Recieve = 0, + Bandwidth = bandwidthPercentiles, + Framerate = frameratePercentiles, + } +end + +local function runAll() + Benches = require(ReplicatedStorage.Shared.benches) + Modes = require(ReplicatedStorage.Shared.modes) + local results: Results = {} + + for bench in Benches do + warn("Running " .. bench .. " benchmark") + results[bench] = {} + + for tool in Modes do + print("> Running with " .. tool) + results[bench][tool] = runBenchmark(tool, bench) + waitForDrain() + end + + waitForDrain() + end + + -- Pull server-side receive counts for loss calculation. + local serverReceived = ReplicatedStorage.Shared.GetRecieved:InvokeServer() + for bench, tools in serverReceived do + for tool, count in tools do + if results[bench] and results[bench][tool] then + results[bench][tool].Recieve = count + end + end + end + + print("Finished running benchmarks, generating results...") + ReplicatedStorage.Shared.Generate:FireServer(HttpService:JSONEncode(results)) +end + +task.wait(2) +runAll() diff --git a/benchmark/src/server/init.server.luau b/benchmark/src/server/init.server.luau new file mode 100644 index 0000000..1d48c34 --- /dev/null +++ b/benchmark/src/server/init.server.luau @@ -0,0 +1,106 @@ +-- Benchmark server. Receives data from all modes and validates correctness. +local HttpService = game:GetService("HttpService") +local ReplicatedStorage = game:GetService("ReplicatedStorage") + +local Satset = require(ReplicatedStorage.Packages.Satset) +Satset.start() + +local Benches = require(ReplicatedStorage.Shared.benches) +local Modes = require(ReplicatedStorage.Shared.modes) + +local function compareTables(a: any, b: any): boolean + if a == b then + return true + end + if type(a) ~= type(b) then + return false + end + if type(a) ~= "table" then + return false + end + + local keys = {} + for key, value in a do + local other = b[key] + if other == nil or not compareTables(value, other) then + return false + end + keys[key] = true + end + + for key in b do + if not keys[key] then + return false + end + end + + return true +end + +local function compareValues(a: any, b: any): boolean + if type(a) == "table" or type(b) == "table" then + return compareTables(a, b) + end + return a == b +end + +-- Track received counts per bench per tool for loss calculation. +local received = {} +for name, data in Benches do + local benchReceived = {} + received[name] = benchReceived + + local function onReceive(tool: string) + benchReceived[tool] = 0 + + return function(player, value) + benchReceived[tool] += 1 + if benchReceived[tool] > 1 then + return + end + + -- ByteNet swaps argument order. + if tool == "bytenet" then + player, value = value, player + end + + if not compareValues(data, value) then + warn("Received incorrect data with " .. tool .. " for " .. name) + else + print("> " .. tool .. " passed " .. name .. " validation!") + end + end + end + + for tool, events in Modes do + local event = events[name] + local callback = onReceive(tool) + + if tool == "roblox" then + event.OnServerEvent:Connect(callback) + elseif tool == "bridgenet2" then + -- BridgeNet2: bridge:Connect(callback(player, data)) + event:Connect(callback) + elseif tool == "warp" then + -- Warp: server:Connect(callback(player, ...)) + event:Connect(callback) + elseif event.listen then + -- Satset and ByteNet use .listen() + event.listen(callback) + end + end +end + +-- Client pulls received counts to compute packet loss. +ReplicatedStorage.Shared.GetRecieved.OnServerInvoke = function() + return received +end + +-- Client sends JSON results, server writes them out. +ReplicatedStorage.Shared.Generate.OnServerEvent:Connect(function(_player, json) + local output = Instance.new("StringValue") + output.Name = "Result" + output.Value = json + output.Parent = game + print("Generated results") +end) diff --git a/benchmark/src/shared/benches/Booleans.luau b/benchmark/src/shared/benches/Booleans.luau new file mode 100644 index 0000000..98860ea --- /dev/null +++ b/benchmark/src/shared/benches/Booleans.luau @@ -0,0 +1,6 @@ +-- 1000 boolean values. Tests minimal-payload throughput and packing efficiency. +local Array = {} +for _ = 1, 1000 do + table.insert(Array, math.random() > 0.5) +end +return Array diff --git a/benchmark/src/shared/benches/Entities.luau b/benchmark/src/shared/benches/Entities.luau new file mode 100644 index 0000000..cf9bb8a --- /dev/null +++ b/benchmark/src/shared/benches/Entities.luau @@ -0,0 +1,13 @@ +-- 100 entities, each with 6 u8 fields. Simulates a typical NPC/player sync payload. +local Array = {} +for _ = 1, 100 do + table.insert(Array, { + id = math.random(1, 255), + x = math.random(1, 255), + y = math.random(1, 255), + z = math.random(1, 255), + orientation = math.random(1, 255), + animation = math.random(1, 255), + }) +end +return Array diff --git a/benchmark/src/shared/benches/Mixed.luau b/benchmark/src/shared/benches/Mixed.luau new file mode 100644 index 0000000..b9b402e --- /dev/null +++ b/benchmark/src/shared/benches/Mixed.luau @@ -0,0 +1,12 @@ +-- Realistic game packet: player state update with mixed types. +-- Simulates what a typical game sends per player per frame. +return { + id = math.random(1, 255), + health = math.random(0, 100), + position = Vector3.new(math.random() * 2048 - 1024, math.random() * 512, math.random() * 2048 - 1024), + velocity = Vector3.new(math.random() * 100 - 50, math.random() * 100 - 50, math.random() * 100 - 50), + name = "Player_" .. tostring(math.random(1000, 9999)), + isRunning = math.random() > 0.5, + animation = math.random(1, 255), + team = math.random(1, 4), +} diff --git a/benchmark/src/shared/benches/SingleValue.luau b/benchmark/src/shared/benches/SingleValue.luau new file mode 100644 index 0000000..49e35b9 --- /dev/null +++ b/benchmark/src/shared/benches/SingleValue.luau @@ -0,0 +1,2 @@ +-- Single u8 value. Tests per-call overhead with minimal payload. +return math.random(1, 255) diff --git a/benchmark/src/shared/benches/Strings.luau b/benchmark/src/shared/benches/Strings.luau new file mode 100644 index 0000000..f0ed27e --- /dev/null +++ b/benchmark/src/shared/benches/Strings.luau @@ -0,0 +1,13 @@ +-- 100 short strings (8-32 chars). Tests variable-length serialization. +local CHARS = "abcdefghijklmnopqrstuvwxyz0123456789" +local Array = {} +for _ = 1, 100 do + local len = math.random(8, 32) + local chars = table.create(len) + for j = 1, len do + local idx = math.random(1, #CHARS) + chars[j] = string.sub(CHARS, idx, idx) + end + table.insert(Array, table.concat(chars)) +end +return Array diff --git a/benchmark/src/shared/benches/Vectors.luau b/benchmark/src/shared/benches/Vectors.luau new file mode 100644 index 0000000..0612e11 --- /dev/null +++ b/benchmark/src/shared/benches/Vectors.luau @@ -0,0 +1,6 @@ +-- 100 Vector3 positions. Tests float serialization throughput. +local Array = {} +for _ = 1, 100 do + table.insert(Array, Vector3.new(math.random() * 2048 - 1024, math.random() * 512, math.random() * 2048 - 1024)) +end +return Array diff --git a/benchmark/src/shared/benches/init.luau b/benchmark/src/shared/benches/init.luau new file mode 100644 index 0000000..e0e8b4c --- /dev/null +++ b/benchmark/src/shared/benches/init.luau @@ -0,0 +1,9 @@ +-- This file is part of the Satset networking library and is licensed under MIT License; see LICENSE.txt for details +-- Benchmark bench loader. Resets random seed for deterministic data across server/client. + +local Benches = {} +for _, Bench in script:GetChildren() do + math.randomseed(0) + Benches[Bench.Name] = require(Bench) +end +return Benches diff --git a/benchmark/src/shared/modes/bridgenet2.luau b/benchmark/src/shared/modes/bridgenet2.luau new file mode 100644 index 0000000..7654603 --- /dev/null +++ b/benchmark/src/shared/modes/bridgenet2.luau @@ -0,0 +1,13 @@ +-- BridgeNet2 benchmark mode (v1.0.0 API). +-- BridgeNet2 uses ReferenceBridge with string identifiers. +local BridgeNet2 = require(game:GetService("ReplicatedStorage").Packages.BridgeNet2) + +local Benches = require(game:GetService("ReplicatedStorage").Shared.benches) + +local Events = {} + +for Name in Benches do + Events[Name] = BridgeNet2.ReferenceBridge("BridgeBench_" .. Name) +end + +return Events diff --git a/benchmark/src/shared/modes/bytenet.luau b/benchmark/src/shared/modes/bytenet.luau new file mode 100644 index 0000000..44ad4b7 --- /dev/null +++ b/benchmark/src/shared/modes/bytenet.luau @@ -0,0 +1,42 @@ +-- ByteNet benchmark mode (v0.4.6 API). +local ByteNet = require(game:GetService("ReplicatedStorage").Packages.ByteNet) + +-- ByteNet 0.4.6 requires the schema to be in the 'value' field. +return ByteNet.defineNamespace("Benchmark", function() + return { + Booleans = ByteNet.definePacket({ + value = ByteNet.array(ByteNet.bool), + }), + Entities = ByteNet.definePacket({ + value = ByteNet.array(ByteNet.struct({ + id = ByteNet.uint8, + x = ByteNet.uint8, + y = ByteNet.uint8, + z = ByteNet.uint8, + orientation = ByteNet.uint8, + animation = ByteNet.uint8, + })), + }), + Vectors = ByteNet.definePacket({ + value = ByteNet.array(ByteNet.vec3), + }), + Strings = ByteNet.definePacket({ + value = ByteNet.array(ByteNet.string), + }), + Mixed = ByteNet.definePacket({ + value = ByteNet.struct({ + id = ByteNet.uint8, + health = ByteNet.uint8, + position = ByteNet.vec3, + velocity = ByteNet.vec3, + name = ByteNet.string, + isRunning = ByteNet.bool, + animation = ByteNet.uint8, + team = ByteNet.uint8, + }), + }), + SingleValue = ByteNet.definePacket({ + value = ByteNet.uint8, + }), + } +end) diff --git a/benchmark/src/shared/modes/init.luau b/benchmark/src/shared/modes/init.luau new file mode 100644 index 0000000..d409b2d --- /dev/null +++ b/benchmark/src/shared/modes/init.luau @@ -0,0 +1,13 @@ +-- Mode loader. Each mode wraps a networking library with a common interface. +-- Disabled modes are skipped (set to true to disable). +local DISABLED_MODES = {} + +local Modes = {} +for _, Mode in script:GetChildren() do + if DISABLED_MODES[Mode.Name] then + continue + end + Modes[Mode.Name] = require(Mode) +end + +return Modes diff --git a/benchmark/src/shared/modes/roblox.luau b/benchmark/src/shared/modes/roblox.luau new file mode 100644 index 0000000..afb21ab --- /dev/null +++ b/benchmark/src/shared/modes/roblox.luau @@ -0,0 +1,22 @@ +-- Native Roblox baseline. One RemoteEvent per bench, no serialization. +local ReplicatedStorage = game:GetService("ReplicatedStorage") +local RunService = game:GetService("RunService") + +local Benches = require(ReplicatedStorage.Shared.benches) + +local Events = {} +local IsServer = RunService:IsServer() + +for Name in Benches do + local Event: RemoteEvent + if IsServer then + Event = Instance.new("RemoteEvent") + Event.Name = "RobloxBench_" .. Name + Event.Parent = ReplicatedStorage + else + Event = ReplicatedStorage:WaitForChild("RobloxBench_" .. Name) + end + Events[Name] = Event +end + +return Events diff --git a/benchmark/src/shared/modes/satset.luau b/benchmark/src/shared/modes/satset.luau new file mode 100644 index 0000000..a93da39 --- /dev/null +++ b/benchmark/src/shared/modes/satset.luau @@ -0,0 +1,102 @@ +-- Satset benchmark mode. +local RunService = game:GetService("RunService") +local Satset = require(game:GetService("ReplicatedStorage").Packages.Satset) +local Types = Satset.Types + +-- We don't call Satset.start() here anymore. +-- It should be called by the main Server/Client runners. + +-- Entity struct type (6 x u8, fixed 6 bytes) +local EntityType = { + read = function(b: buffer, cursor: number) + return { + id = buffer.readu8(b, cursor), + x = buffer.readu8(b, cursor + 1), + y = buffer.readu8(b, cursor + 2), + z = buffer.readu8(b, cursor + 3), + orientation = buffer.readu8(b, cursor + 4), + animation = buffer.readu8(b, cursor + 5), + } + end, + write = function(b: buffer, cursor: number, value: any) + buffer.writeu8(b, cursor, value.id) + buffer.writeu8(b, cursor + 1, value.x) + buffer.writeu8(b, cursor + 2, value.y) + buffer.writeu8(b, cursor + 3, value.z) + buffer.writeu8(b, cursor + 4, value.orientation) + buffer.writeu8(b, cursor + 5, value.animation) + end, + size = 6, + getSize = function() + return 6 + end, + isFixed = true, +} :: any + +local Packets = { + Booleans = Satset.definePacket({ + name = "SatsetBenchBooleans", + schema = { value = Types.array(Types.bool) }, + reliable = true, + }), + Entities = Satset.definePacket({ + name = "SatsetBenchEntities", + schema = { value = Types.array(EntityType) }, + reliable = true, + }), + Vectors = Satset.definePacket({ + name = "SatsetBenchVectors", + schema = { value = Types.array(Types.Vector3) }, + reliable = true, + }), + Strings = Satset.definePacket({ + name = "SatsetBenchStrings", + schema = { value = Types.array(Types.string8) }, + reliable = true, + }), + Mixed = Satset.definePacket({ + name = "SatsetBenchMixed", + schema = { + id = Types.u8, + health = Types.u8, + position = Types.Vector3, + velocity = Types.Vector3, + name = Types.string8, + isRunning = Types.bool, + animation = Types.u8, + team = Types.u8, + }, + reliable = true, + }), + SingleValue = Satset.definePacket({ + name = "SatsetBenchSingleValue", + schema = { value = Types.u8 }, + reliable = true, + }), +} + +-- Wrap with common fire/listen interface for the benchmark runner. +local Wrapped = {} +for name, packet in Packets do + Wrapped[name] = { + fire = function(data: any) + if RunService:IsClient() then + if type(data) == "table" and data[1] ~= nil then + packet:fireServer({ value = data }) + elseif type(data) == "number" then + packet:fireServer({ value = data }) + else + packet:fireServer(data) + end + end + end, + listen = function(callback: any) + packet:listen(function(decoded, sender) + local val = decoded.value or decoded + callback(sender, val) + end) + end, + } +end + +return Wrapped diff --git a/benchmark/src/shared/modes/warp.luau b/benchmark/src/shared/modes/warp.luau new file mode 100644 index 0000000..572df2c --- /dev/null +++ b/benchmark/src/shared/modes/warp.luau @@ -0,0 +1,20 @@ +-- Warp benchmark mode (v1.0.14 API). +-- Warp uses string identifiers. Server/Client are separate constructors. +local RunService = game:GetService("RunService") +local Warp = require(game:GetService("ReplicatedStorage").Packages.Warp) + +local Benches = require(game:GetService("ReplicatedStorage").Shared.benches) + +local IsServer = RunService:IsServer() +local Events = {} + +for Name in Benches do + local eventName = "WarpBench_" .. Name + if IsServer then + Events[Name] = Warp.Server(eventName) + else + Events[Name] = Warp.Client(eventName) + end +end + +return Events