From 107a80a6d07b6938e3d945bd53e43a30e7c78414 Mon Sep 17 00:00:00 2001 From: Mikhail Agapov <118179774+mikhail-dcl@users.noreply.github.com> Date: Thu, 11 Jun 2026 18:39:02 +0300 Subject: [PATCH 1/5] Feat: Pulse with quantization (#360) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add Quantization as a protobuf plugin * pulse_comms.proto Signed-off-by: Mikhail Agapov * add server message & handshake response * add player state full/delta * add player joined message * add player joined into server message * add player state into client message * Update protobufjs to 7.5.4 to resolve the problem with namespaces * proto/google/ is no longer created or published, so both buf (CI) and protoc (consumers like unity-explorer) resolve descriptor.proto from their own built-in includes — which all have the correct csharp_namespace Signed-off-by: Mikhail Agapov * fix: exclude proto/google from npm package to fix C# codegen namespace Revert protobufjs to 7.2.4 (7.5.4 breaks buf and proto-compatibility-tool with newer reserved syntax) and restore the make install copy step for local tooling. Change "files" from "proto" to "proto/decentraland" so the stripped google/protobuf/descriptor.proto (missing csharp_namespace option) is no longer published. Consumers' protoc resolves descriptor.proto from its own built-in includes which have the correct option csharp_namespace = "Google.Protobuf.Reflection". Co-Authored-By: Claude Opus 4.6 * Adjust states encoding * Add quantization example for PlayerStateDelta Signed-off-by: Mikhail Agapov * Copy protoc-gen-bitwise to the output package * Make Quantize.cs compitable with Unity Signed-off-by: Mikhail Agapov * Isolate PlayerState as a reusable component Signed-off-by: Mikhail Agapov * Add requried quntizationfor delta Signed-off-by: Mikhail Agapov * Add RESYNC_REQUEST Signed-off-by: Mikhail Agapov * ProfileVersionAnnouncement - parcel_index change uint32 -> int32 Signed-off-by: Mikhail Agapov * add emote messages * add duration on EmoteStart for one shot emotes * Fix MovementBlend Range Signed-off-by: Mikhail Agapov * add teleport messages * Add "jump_count" Signed-off-by: Mikhail Agapov * add teleport ClientMessage & ServerMessage * Add BaselineSeq to delta Signed-off-by: Mikhail Agapov * add parcel index & player state in teleport * add head yaw and head pitch to state flags * Add `sequence` to `EmoteStarted` Signed-off-by: Mikhail Agapov * add player state into EmoteStart * Add seq and PlayerState to `EmoteStopped` Signed-off-by: Mikhail Agapov * Add "realm" field to "TeleportRequest" Signed-off-by: Mikhail Agapov * Split pulse_comms into separate files Signed-off-by: Mikhail Agapov * Add Initial State to HandshakeRequest - It's needed to properly authorize reconnection in the middle of the session Signed-off-by: Mikhail Agapov * Add support for "pointing at" Signed-off-by: Mikhail Agapov * Change "Point At" to the absolute value Signed-off-by: Mikhail Agapov * Add "realm" to the initial state Signed-off-by: Mikhail Agapov * Change head_pitch max from 180 to 360 Signed-off-by: Mikhail Agapov * emote mask --------- Signed-off-by: Mikhail Agapov Co-authored-by: Nicolas Lorusso Co-authored-by: Claude Opus 4.6 --- README.md | 198 ++++++++++++++++++ package.json | 5 +- proto/decentraland/common/options.proto | 30 +++ .../common/quantization_example.proto | 132 ++++++++++++ proto/decentraland/pulse/pulse_client.proto | 77 +++++++ proto/decentraland/pulse/pulse_server.proto | 138 ++++++++++++ proto/decentraland/pulse/pulse_shared.proto | 48 +++++ protoc-gen-bitwise/generator_csharp.py | 176 ++++++++++++++++ protoc-gen-bitwise/options_pb2.py | 171 +++++++++++++++ protoc-gen-bitwise/plugin.py | 92 ++++++++ protoc-gen-bitwise/runtime/cs/BitReader.cs | 112 ++++++++++ protoc-gen-bitwise/runtime/cs/BitWriter.cs | 117 +++++++++++ protoc-gen-bitwise/runtime/cs/Quantize.cs | 35 ++++ 13 files changed, 1329 insertions(+), 2 deletions(-) create mode 100644 proto/decentraland/common/options.proto create mode 100644 proto/decentraland/common/quantization_example.proto create mode 100644 proto/decentraland/pulse/pulse_client.proto create mode 100644 proto/decentraland/pulse/pulse_server.proto create mode 100644 proto/decentraland/pulse/pulse_shared.proto create mode 100644 protoc-gen-bitwise/generator_csharp.py create mode 100644 protoc-gen-bitwise/options_pb2.py create mode 100644 protoc-gen-bitwise/plugin.py create mode 100644 protoc-gen-bitwise/runtime/cs/BitReader.cs create mode 100644 protoc-gen-bitwise/runtime/cs/BitWriter.cs create mode 100644 protoc-gen-bitwise/runtime/cs/Quantize.cs diff --git a/README.md b/README.md index 598bdb0b..20c75605 100644 --- a/README.md +++ b/README.md @@ -67,3 +67,201 @@ In this case, there is no problem with when each PR is merged. It's recommendabl ## Comms TODO + +--- + +# Bitwise Serialization Plugin (`protoc-gen-bitwise`) + +A custom protoc plugin that generates C# partial classes with typed float +accessors for quantized `uint32` fields in high-frequency MMO networking +messages (position deltas, player input, etc.). It runs alongside +`--csharp_out` in the same protoc invocation; the two output files coexist +via C# `partial class`. + +## How it works + +Protobuf encodes `uint32` values as varints, which are already compact for +small values: a value up to 2¹⁴−1 costs 2 bytes, up to 2²¹−1 costs 3 bytes. +Rather than a separate binary packing layer, the plugin leverages this: + +1. Declare quantized fields as `uint32` in the `.proto` schema and annotate + them with `[(decentraland.common.quantized)]` to specify the float range + and bit resolution. +2. `--csharp_out` generates the standard protobuf class with the raw `uint32` + property (e.g. `PositionX`). +3. `--bitwise_out` (this plugin) generates a `partial class` extension with a + cached float accessor (e.g. `PositionXQuantized`) that encodes/decodes + transparently via `Quantize.Encode` / `Quantize.Decode`. + +The wire representation is a standard protobuf message — any protobuf-capable +client can read it without knowledge of the plugin. + +## Prerequisites + +| Requirement | Version | +|---|---| +| Python | 3.10+ | +| `protobuf` Python package | 4.x or 3.20+ | +| `protoc` | 3.19+ | + +```bash +pip install protobuf +``` + +## Step 1 — Annotate your `.proto` file + +Declare quantized fields as `uint32` and import `options.proto`: + +```protobuf +syntax = "proto3"; + +import "decentraland/common/options.proto"; + +package decentraland.kernel.comms.v3; + +message PositionDelta { + // Float range [-100, 100] quantized to 16 bits ≈ 0.003-unit precision. + // Stored as uint32 on the wire; protobuf encodes it as a 3-byte varint. + uint32 dx = 1 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dy = 2 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dz = 3 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + + // Unannotated uint32: protobuf varint encodes small values compactly by default. + uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; +} +``` + +### Annotation reference + +| Annotation | Target type | Parameters | Effect | +|---|---|---|---| +| `[(decentraland.common.quantized)]` | `uint32` | `min`, `max`, `bits` | Plugin emits a cached `float {Name}Quantized` accessor | +| `[(decentraland.common.bit_packed)]` | `uint32` | `bits` | Documents the value range; protobuf handles varint compaction automatically | + +### Wire cost at worst-case (all bits set) + +| Quantization bits | Max value | Varint bytes | Tag (field ≤ 15) | Total per field | +|---|---|---|---|---| +| 8 | 255 | 2 | 1 | 3 B | +| 12 | 4 095 | 2 | 1 | 3 B | +| 14 | 16 383 | 2 | 1 | 3 B | +| 16 | 65 535 | 3 | 1 | 4 B | +| 20 | 1 048 575 | 3 | 1 | 4 B | + +Proto3 omits fields equal to their default value (0), so average cost is lower. + +## Step 2 — Run protoc + +```bash +protoc \ + --proto_path=proto \ + --proto_path=/path/to/google/protobuf/include \ + --csharp_out=generated/cs \ + --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.py \ + --bitwise_out=generated/cs \ + proto/decentraland/kernel/comms/v3/comms.proto +``` + +The plugin emits one `*.Bitwise.cs` file (PascalCase, flat in the output +directory) for each `.proto` file that contains at least one `[(quantized)]` +field. + +## Step 3 — Copy the runtime + +Copy `Quantize.cs` into your project: + +``` +Assets/ +└── Scripts/ + └── Networking/ + └── Bitwise/ + └── Quantize.cs ← protoc-gen-bitwise/runtime/cs/Quantize.cs +``` + +`Quantize.cs` lives in the `Decentraland.Networking.Bitwise` namespace and +provides two static methods used by the generated accessors: + +```csharp +public static class Quantize +{ + public static uint Encode(float value, float min, float max, int bits); + public static float Decode(uint encoded, float min, float max, int bits); +} +``` + +## Step 4 — Use the generated code + +The plugin emits a `partial class` that adds float accessors on top of the +standard protobuf-generated `uint32` properties: + +```csharp +using Decentraland.Kernel.Comms.V3; + +// --- Build and send --- +var delta = new PositionDelta(); +delta.DxQuantized = 3.14f; // encodes to uint32, stored in delta.Dx +delta.DyQuantized = 0f; +delta.DzQuantized = -7.5f; +delta.EntityId = 42u; + +byte[] bytes = delta.ToByteArray(); // standard protobuf serialization +SendOnChannel1(bytes); + +// --- Receive and read --- +var received = PositionDelta.Parser.ParseFrom(receivedBytes); +float x = received.DxQuantized; // decoded on first access, cached thereafter +float y = received.DyQuantized; +float z = received.DzQuantized; + +// If raw uint32 fields are mutated directly after construction, invalidate the cache: +received.ResetDecodedCache(); +``` + +## Generated file example + +For the `PositionDelta` message above the plugin emits `PositionDelta.Bitwise.cs`: + +```csharp +// +// Generated by protoc-gen-bitwise. DO NOT EDIT. +// Source: decentraland/kernel/comms/v3/comms.proto +// + +using Decentraland.Networking.Bitwise; + +namespace Decentraland.Kernel.Comms.V3 +{ + public partial class PositionDelta + { + private float? _dx; + public float DxQuantized + { + get => _dx ??= Quantize.Decode(Dx, -100.0f, 100.0f, 16); + set { _dx = value; Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + private float? _dy; + public float DyQuantized + { + get => _dy ??= Quantize.Decode(Dy, -100.0f, 100.0f, 16); + set { _dy = value; Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + private float? _dz; + public float DzQuantized + { + get => _dz ??= Quantize.Decode(Dz, -100.0f, 100.0f, 16); + set { _dz = value; Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. + public void ResetDecodedCache() + { + _dx = null; + _dy = null; + _dz = null; + } + } + +} // namespace Decentraland.Kernel.Comms.V3 +``` diff --git a/package.json b/package.json index 1e5294e5..2312a78a 100644 --- a/package.json +++ b/package.json @@ -28,9 +28,10 @@ "protobufjs": "7.2.4" }, "files": [ - "proto", + "proto/decentraland", "out-ts", "out-js", - "public" + "public", + "protoc-gen-bitwise" ] } diff --git a/proto/decentraland/common/options.proto b/proto/decentraland/common/options.proto new file mode 100644 index 00000000..8e911547 --- /dev/null +++ b/proto/decentraland/common/options.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package decentraland.common; + +import "google/protobuf/descriptor.proto"; + +// Options for quantizing a float value stored as a uint32 field. +// The float is clamped to [min, max] and uniformly quantized to N bits; +// protobuf encodes the resulting uint32 as a varint on the wire. +// The generator emits a float {Name}Quantized accessor in the partial class. +message QuantizedFloatOptions { + float min = 1; + float max = 2; + uint32 bits = 3; +} + +// Options for bit-packing an integer field into fewer than 32 bits. +message BitPackedOptions { + uint32 bits = 1; +} + +extend google.protobuf.FieldOptions { + // Apply to uint32 fields to enable quantized float encoding. + // Example: uint32 dx = 1 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + QuantizedFloatOptions quantized = 50001; + + // Apply to uint32 fields to pack into fewer bits. + // Example: uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; + BitPackedOptions bit_packed = 50002; +} diff --git a/proto/decentraland/common/quantization_example.proto b/proto/decentraland/common/quantization_example.proto new file mode 100644 index 00000000..a82e430c --- /dev/null +++ b/proto/decentraland/common/quantization_example.proto @@ -0,0 +1,132 @@ +syntax = "proto3"; + +package decentraland.common; + +import "decentraland/common/options.proto"; + +// High-frequency player state messages sent on Channel 1 (unreliable sequenced). +// +// Every message below uses the protoc-gen-bitwise annotations to minimise +// wire size. Wire costs are listed per-message so the trade-offs are clear. +// +// Annotation cheat-sheet: +// [(decentraland.common.quantized) = { min: F, max: F, bits: N }] +// → stores a float as a uint32 quantized to N bits over [min, max]. +// → protobuf encodes the uint32 as a varint: values ≤ 2^14-1 cost 2 bytes, +// values ≤ 2^21-1 cost 3 bytes; the generated partial class adds a +// float {Name}Quantized accessor that encodes/decodes transparently. +// [(decentraland.common.bit_packed) = { bits: N }] +// → documents that this uint32 uses at most N bits; protobuf encodes it +// as a varint (same savings, no generated accessor needed). +// (no annotation) +// → standard protobuf encoding (bool/double/etc. at natural width). +// +// Varint byte costs at worst-case (all bits set): +// ≤ 7 bits → 1 byte (max 127) +// ≤ 14 bits → 2 bytes (max 16 383) +// ≤ 21 bits → 3 bytes (max 2 097 151) +// Proto3 omits fields equal to their default value (0), so average cost is lower. + +// --------------------------------------------------------------------------- +// PositionDelta — Δ position relative to last acknowledged full snapshot. +// +// Sent every client tick (~10 Hz) on Channel 1. +// +// Field | Type | Range | Q bits | Wire worst-case +// ------------|--------|----------------|--------|----------------------- +// dx | uint32 | [-100, 100] | 16 | tag 1B + varint 3B = 4B +// dy | uint32 | [-100, 100] | 16 | tag 1B + varint 3B = 4B +// dz | uint32 | [-100, 100] | 16 | tag 1B + varint 3B = 4B +// entity_id | uint32 | [0, 1 048 575] | 20 | tag 1B + varint 3B = 4B +// sequence | uint32 | [0, 4 095] | 12 | tag 1B + varint 2B = 3B +// --------------------------------------------------------------------------- +// Worst-case: 19 B (vs. 20 B raw: 3×float + 2×uint32) +// Step: dx/dy/dz ≈ 0.003 units +// --------------------------------------------------------------------------- +message PositionDelta { + uint32 dx = 1 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dy = 2 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dz = 3 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; + uint32 sequence = 5 [(decentraland.common.bit_packed) = { bits: 12 }]; +} + +// --------------------------------------------------------------------------- +// PlayerInput — client input snapshot for server-side reconciliation. +// +// Sent every client frame (~30 Hz) on Channel 1. +// +// Field | Type | Range | Q bits | Wire worst-case +// ------------|--------|----------------|--------|----------------------- +// move_x | uint32 | [-1, 1] | 8 | tag 1B + varint 2B = 3B +// move_z | uint32 | [-1, 1] | 8 | tag 1B + varint 2B = 3B +// yaw | uint32 | [-180, 180] | 12 | tag 1B + varint 2B = 3B +// buttons | uint32 | bitmask | 8 | tag 1B + varint 2B = 3B +// sequence | uint32 | [0, 4 095] | 12 | tag 1B + varint 2B = 3B +// --------------------------------------------------------------------------- +// Worst-case: 15 B (vs. 20 B raw: 3×float + 2×uint32) +// Step: move_x/move_z ≈ 0.008; yaw ≈ 0.088° +// --------------------------------------------------------------------------- +message PlayerInput { + // Normalised joystick axes in [-1, 1]. + uint32 move_x = 1 [(decentraland.common.quantized) = { min: -1.0, max: 1.0, bits: 8 }]; + uint32 move_z = 2 [(decentraland.common.quantized) = { min: -1.0, max: 1.0, bits: 8 }]; + + // Horizontal look direction in degrees. + uint32 yaw = 3 [(decentraland.common.quantized) = { min: -180.0, max: 180.0, bits: 12 }]; + + // Bitmask of active buttons (see ButtonFlags below). + uint32 buttons = 4 [(decentraland.common.bit_packed) = { bits: 8 }]; + + // Rolling input sequence number used by the server for reconciliation. + uint32 sequence = 5 [(decentraland.common.bit_packed) = { bits: 12 }]; +} + +// Bitmask values for PlayerInput.buttons. +enum ButtonFlags { + BF_NONE = 0; + BF_JUMP = 1; // bit 0 + BF_SPRINT = 2; // bit 1 + BF_INTERACT = 4; // bit 2 + BF_EMOTE = 8; // bit 3 + BF_CROUCH = 16; // bit 4 + // bits 5-7 reserved +} + +// --------------------------------------------------------------------------- +// AvatarStateSnapshot — full authoritative state, sent on Channel 0 (reliable) +// or on resync requests. Demonstrates wider ranges and mixed encodings. +// +// Field | Type | Range | Q bits | Wire worst-case +// -----------------|--------|------------------|--------|----------------------- +// x | uint32 | [-4096, 4096] | 16 | tag 1B + varint 3B = 4B +// y | uint32 | [-256, 256] | 14 | tag 1B + varint 2B = 3B +// z | uint32 | [-4096, 4096] | 16 | tag 1B + varint 3B = 4B +// pitch | uint32 | [-90, 90] | 10 | tag 1B + varint 2B = 3B +// yaw | uint32 | [-180, 180] | 12 | tag 1B + varint 2B = 3B +// entity_id | uint32 | [0, 1 048 575] | 20 | tag 1B + varint 3B = 4B +// animation_state | uint32 | [0, 63] | 6 | tag 1B + varint 1B = 2B +// is_grounded | bool | — | — | tag 1B + varint 1B = 2B +// timestamp | double | — | — | tag 1B + fixed64 8B = 9B +// --------------------------------------------------------------------------- +// Worst-case: 34 B (vs. 45 B raw: 5×float + 2×uint32 + bool + double) +// Step: x/z ≈ 0.125 units; y ≈ 0.031 units; pitch ≈ 0.176°; yaw ≈ 0.088° +// --------------------------------------------------------------------------- +message AvatarStateSnapshot { + // World-space position. + uint32 x = 1 [(decentraland.common.quantized) = { min: -4096.0, max: 4096.0, bits: 16 }]; + uint32 y = 2 [(decentraland.common.quantized) = { min: -256.0, max: 256.0, bits: 14 }]; + uint32 z = 3 [(decentraland.common.quantized) = { min: -4096.0, max: 4096.0, bits: 16 }]; + + // View angles. + uint32 pitch = 4 [(decentraland.common.quantized) = { min: -90.0, max: 90.0, bits: 10 }]; + uint32 yaw = 5 [(decentraland.common.quantized) = { min: -180.0, max: 180.0, bits: 12 }]; + + // Identity and animation state. + uint32 entity_id = 6 [(decentraland.common.bit_packed) = { bits: 20 }]; + uint32 animation_state = 7 [(decentraland.common.bit_packed) = { bits: 6 }]; + + // Un-annotated fields — encoded at their natural width. + bool is_grounded = 8; + double timestamp = 9; // server epoch milliseconds +} diff --git a/proto/decentraland/pulse/pulse_client.proto b/proto/decentraland/pulse/pulse_client.proto new file mode 100644 index 00000000..6036ec53 --- /dev/null +++ b/proto/decentraland/pulse/pulse_client.proto @@ -0,0 +1,77 @@ +syntax = "proto3"; + +package decentraland.pulse; + +import "decentraland/common/vectors.proto"; +import "decentraland/pulse/pulse_shared.proto"; + +message HandshakeRequest { + bytes auth_chain = 1; + int32 profile_version = 2; + optional PlayerInitialState initial_state = 3; +} + +// Describes the initial state of the player if (re-)connected in the middle of the session +message PlayerInitialState { + PlayerState state = 1; + optional string emote_id = 2; + optional uint32 emote_duration_ms = 3; + // Indicates how many milliseconds ago the emote was started + optional uint32 emote_start_offset_ms = 4; + // Non-empty realm identifier. Rejected if empty. + string realm = 5; + optional int32 emote_mask = 6; +} + +// Similarly to the LiveKit pipeline, a peer announces the version of its profile but +// it only does it when it's changed as it's sent reliably and stored on the server for other peers +message ProfileVersionAnnouncement { + int32 version = 1; +} + +// Since the server doesn't simulate the scenes state, it trusts the values from the client +message PlayerStateInput { + PlayerState state = 1; +} + +// Client sends resync request if it has a gap in the known sequences and, thus, can't apply a delta. +// It's sent reliably to prevent further desynchronization +message ResyncRequest { + uint32 subject_id = 1; + // highest seq the client actually has for this subject + uint32 known_seq = 2; +} + +// Client → Server. Client requests to start an emote. +message EmoteStart { + string emote_id = 1; + optional uint32 duration_ms = 2; + PlayerState player_state = 3; + optional int32 mask = 4; +} + +// Client → Server. Client requests to stop a looping emote. +// One-shot emotes are terminated by the server timer. +message EmoteStop { +} + +// Client → Server. Also announces the peer's realm; peers in different realms never see each +// other. Must be the first gameplay message after handshake. Same-realm re-teleports are valid. +message TeleportRequest { + int32 parcel_index = 1; + decentraland.common.Vector3 position = 2; + // Non-empty realm identifier. Rejected if empty. + string realm = 3; +} + +message ClientMessage { + oneof message { + HandshakeRequest handshake = 1; + PlayerStateInput input = 2; + ResyncRequest resync = 3; + ProfileVersionAnnouncement profile_announcement = 4; + EmoteStart emote_start = 5; + EmoteStop emote_stop = 6; + TeleportRequest teleport = 7; + } +} diff --git a/proto/decentraland/pulse/pulse_server.proto b/proto/decentraland/pulse/pulse_server.proto new file mode 100644 index 00000000..8311e737 --- /dev/null +++ b/proto/decentraland/pulse/pulse_server.proto @@ -0,0 +1,138 @@ +syntax = "proto3"; + +package decentraland.pulse; + +import "decentraland/common/options.proto"; +import "decentraland/pulse/pulse_shared.proto"; + +message HandshakeResponse { + bool success = 1; + optional string error = 2; +} + +message PlayerProfileVersionsAnnounced { + uint32 subject_id = 1; + int32 version = 2; +} + +message PlayerStateDeltaTier0 { + uint32 subject_id = 1; + + // Between two consecutive server simulation ticks, the subject may have sent multiple inputs, each incrementing Seq by 1. So + // the delta's NewSeq will naturally jump by more than 1 compared to the previous delta — even with zero packet loss. + // Without the "BaselineSeq" the client has no way to detect the package loss + uint32 baseline_seq = 2; + uint32 new_seq = 3; + uint32 server_tick = 4; + + // While the player doesn't cross the parcel, this field is omitted from diff + optional int32 parcel_index = 5; + + // X position inside the parcel + optional uint32 position_x = 6 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + + // Y position + optional uint32 position_y = 7 [(decentraland.common.quantized) = { min: 0, max: 200, bits: 13 }]; + + // Z position inside the parcel + optional uint32 position_z = 8 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + + optional uint32 velocity_x = 9 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; + optional uint32 velocity_y = 10 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; + optional uint32 velocity_z = 11 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; + + optional uint32 rotation_y = 12 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + optional uint32 movement_blend = 13 [(decentraland.common.quantized) = { min: 0, max: 3, bits: 5 }]; + optional uint32 slide_blend = 14 [(decentraland.common.quantized) = { min: 0, max: 1, bits: 4 }]; + optional uint32 head_yaw = 15 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + optional uint32 head_pitch = 16 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + + optional uint32 state_flags = 17; + optional GlideState glide_state = 18; + + optional int32 jump_count = 19; + + // Absolute world hit position the player is pointing at. + // Only meaningful when POINTING_AT is set in `state_flags`. + // + // X/Z use 17 bits over the world span (~±3000m, covers GenesisCity + border + + // raycast cutoff) which yields ~0.046m steps — at least as fine as the player's + // own parcel_index + position_x/z combined precision (0.0625m), so no separate + // parcel index is needed for point-at. + // + // Y uses 7 bits over the player altitude range (matches position_y), step ~1.6m. + // Point At is not supposed to change frequently so the wire overhead should be minimal + optional uint32 point_at_x = 20 [(decentraland.common.quantized) = { min: -3000.0, max: 3000.0, bits: 17 }]; + optional uint32 point_at_y = 21 [(decentraland.common.quantized) = { min: 0.0, max: 200.0, bits: 7 }]; + optional uint32 point_at_z = 22 [(decentraland.common.quantized) = { min: -3000.0, max: 3000.0, bits: 17 }]; +} + +// Full State is sent to the client when it is out of sync, and can't recover with a diff only +message PlayerStateFull { + uint32 subject_id = 1; + uint32 sequence = 2; + uint32 server_tick = 3; + PlayerState state = 4; +} + +// Notification to the client, that a peer has joined, it can mean connection or entering the area of interest, it's up to the server to decide +message PlayerJoined { + string user_id = 1; + int32 profile_version = 2; + PlayerStateFull state = 3; +} + +// Notification to the client, that a peer has left, it can mean disconnection or leaving the area of interest +message PlayerLeft { + uint32 subject_id = 1; +} + +enum EmoteStopReason { + COMPLETED = 0; // one-shot timer expired on the server + CANCELLED = 1; // client sent EmoteStop (looping emotes) +} + +// Server → All Observers +// Full player state is piggybacked to ensure the emote is played in the right place. +// Observers use server_tick to scrub animation forward by transit latency. +message EmoteStarted { + uint32 subject_id = 1; + uint32 sequence = 2; + uint32 server_tick = 3; + string emote_id = 4; + PlayerState player_state = 5; + optional int32 mask = 6; +} + +// Server → All Observers +// Client resumes MovementInput only after receiving this. +// Carries full PlayerState so the client can snap to the correct position on resume. +message EmoteStopped { + uint32 subject_id = 1; + uint32 server_tick = 2; + EmoteStopReason reason = 3; + uint32 sequence = 4; + PlayerState player_state = 5; +} + +// Server → All Observers +message TeleportPerformed { + uint32 subject_id = 1; + uint32 sequence = 2; + uint32 server_tick = 3; + PlayerState state = 4; +} + +message ServerMessage { + oneof message { + HandshakeResponse handshake = 1; + PlayerStateFull player_state_full = 2; + PlayerStateDeltaTier0 player_state_delta = 3; + PlayerJoined player_joined = 4; + PlayerLeft player_left = 5; + PlayerProfileVersionsAnnounced player_profile_version_announced = 6; + EmoteStarted emote_started = 7; + EmoteStopped emote_stopped = 8; + TeleportPerformed teleported = 9; + } +} diff --git a/proto/decentraland/pulse/pulse_shared.proto b/proto/decentraland/pulse/pulse_shared.proto new file mode 100644 index 00000000..6f4fba7c --- /dev/null +++ b/proto/decentraland/pulse/pulse_shared.proto @@ -0,0 +1,48 @@ +syntax = "proto3"; + +package decentraland.pulse; + +import "decentraland/common/vectors.proto"; + +enum PlayerAnimationFlags { + NONE = 0; + GROUNDED = 1; + LONG_JUMP = 2; + LONG_FALL = 4; + FALLING = 8; + STUNNED = 16; + HEAD_YAW = 32; + HEAD_PITCH = 64; + POINTING_AT = 128; +} + +enum GlideState { + PROP_CLOSED = 0; + OPENING_PROP = 1; + GLIDING = 2; + CLOSING_PROP = 3; +} + +message PlayerState { + int32 parcel_index = 1; + + decentraland.common.Vector3 position = 2; + decentraland.common.Vector3 velocity = 3; + + float rotation_y = 4; + + float movement_blend = 5; + float slide_blend = 6; + + optional float head_yaw = 7; + optional float head_pitch = 8; + + uint32 state_flags = 9; + GlideState glide_state = 10; + + int32 jump_count = 11; + + // Absolute world hit position the player is pointing at. + // Only meaningful when POINTING_AT is set in `state_flags`. + optional decentraland.common.Vector3 point_at = 12; +} diff --git a/protoc-gen-bitwise/generator_csharp.py b/protoc-gen-bitwise/generator_csharp.py new file mode 100644 index 00000000..376c0dc1 --- /dev/null +++ b/protoc-gen-bitwise/generator_csharp.py @@ -0,0 +1,176 @@ +""" +C# code generator for the protoc-gen-bitwise plugin. + +For every proto message that contains at least one uint32 field annotated with +[(quantized)], this module emits a C# partial class that adds a computed float +property named {FieldName}Quantized. The getter decodes the stored uint32 to +a float; the setter encodes a float back to a uint32. Standard protobuf +handles serialization of the uint32 wire field; this class adds a typed float +accessor on top. + +Protobuf encodes small uint32 values via varint, so a value that fits in +2^20 costs 3 bytes on the wire — cheaper than a raw IEEE 754 float (4 bytes). + +Only uint32 fields are supported. bit_packed and unannotated fields are +passed through without generating any accessor. +""" + +from google.protobuf import descriptor_pb2 + +from options_pb2 import get_field_options + +# FieldDescriptorProto type constants (aliased for readability) +_FT = descriptor_pb2.FieldDescriptorProto + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _snake_to_pascal(name: str) -> str: + """position_x → PositionX""" + return ''.join(word.capitalize() for word in name.split('_')) + + +def _package_to_namespace(package: str) -> str: + """decentraland.kernel.comms.v3 → Decentraland.Kernel.Comms.V3""" + if not package: + return 'Generated' + return '.'.join(part.capitalize() for part in package.split('.')) + + +def _format_float(value: float) -> str: + """Format a Python float as a C# float literal (e.g. -100.0f).""" + text = f'{value:.8g}' + if '.' not in text and 'e' not in text and 'E' not in text: + text += '.0' + return text + 'f' + + +def _format_step(step: float) -> str: + """Format a quantization step size for display in a doc comment (e.g. ≈ 0.003).""" + return f'\u2248 {step:.6g}' + + +# --------------------------------------------------------------------------- +# Per-message code generation +# --------------------------------------------------------------------------- + +def _gen_message(msg_proto, indent: str = ' ') -> list[str] | None: + """ + Generate a C# partial class for a proto message. + + For each uint32 field with a [(quantized)] annotation, emits a cached float + property {FieldName}Quantized backed by the raw uint32 field. + + Returns a list of lines (without trailing newline) or None if the message + has no quantized uint32 fields. + """ + props: list[tuple[str, str, str, int, float]] = [] # (prop_name, mn, mx, bits, step) + + for field in msg_proto.field: + # Repeated/map fields are not supported + if field.label == _FT.LABEL_REPEATED: + continue + + # Only uint32 fields are candidates for quantized accessors + if field.type != _FT.TYPE_UINT32: + continue + + quantized, _ = get_field_options(field.options) + if quantized is None: + continue + + step = (quantized.max - quantized.min) / ((1 << quantized.bits) - 1) + + props.append(( + _snake_to_pascal(field.name), + _format_float(quantized.min), + _format_float(quantized.max), + quantized.bits, + step, + )) + + if not props: + return None + + i = indent + backings: list[str] = [] + lines: list[str] = [] + lines.append(f'public partial class {msg_proto.name}') + lines.append('{') + + for prop_name, mn, mx, bits, step in props: + backing = '_' + prop_name[0].lower() + prop_name[1:] + backings.append(backing) + lines.append(f'{i}private float? {backing};') + lines.append(f'{i}/// Float accessor for . Range [{mn}, {mx}], {bits} bits, step {_format_step(step)}.') + lines.append(f'{i}public float {prop_name}Quantized') + lines.append(f'{i}{{') + lines.append(f'{i}{i}get => {backing} ??= Quantize.Decode({prop_name}, {mn}, {mx}, {bits});') + lines.append(f'{i}{i}set {{ {backing} = value; {prop_name} = Quantize.Encode(value, {mn}, {mx}, {bits}); }}') + lines.append(f'{i}}}') + lines.append('') + + lines.append(f'{i}/// Clears all cached decoded values. Call after mutating raw uint32 fields directly.') + lines.append(f'{i}public void ResetDecodedCache()') + lines.append(f'{i}{{') + for backing in backings: + lines.append(f'{i}{i}{backing} = null;') + lines.append(f'{i}}}') + + lines.append('}') + return lines + + +# --------------------------------------------------------------------------- +# Per-file code generation (public entry point) +# --------------------------------------------------------------------------- + +def generate_csharp(file_proto) -> dict | None: + """ + Generate a C# source file for a FileDescriptorProto. + + Returns a dict with keys 'name' (output path) and 'content' (C# source), + or None if the file contains no quantized uint32 fields. + """ + namespace = _package_to_namespace(file_proto.package) + + # Output is placed flat in the output root, matching --csharp_out convention. + # e.g. "decentraland/kernel/comms/v3/my_message.proto" + # → "MyMessage.Bitwise.cs" + proto_file = file_proto.name.rsplit('/', 1)[-1] + stem = _snake_to_pascal(proto_file.replace('.proto', '')) + out_name = f'{stem}.Bitwise.cs' + + header = [ + '// ', + '// Generated by protoc-gen-bitwise. DO NOT EDIT.', + f'// Source: {file_proto.name}', + '// ', + '', + 'using Decentraland.Networking.Bitwise;', + '', + f'namespace {namespace}', + '{', + ] + footer = [ + '', + f'}} // namespace {namespace}', + ] + + body: list[str] = [] + for msg in file_proto.message_type: + msg_lines = _gen_message(msg) + if msg_lines is None: + continue + # Indent each line by 4 spaces (inside the namespace block) + for line in msg_lines: + body.append((' ' + line) if line else '') + body.append('') + + if not body: + return None # nothing to emit + + content = '\n'.join(header + body + footer) + '\n' + return {'name': out_name, 'content': content} diff --git a/protoc-gen-bitwise/options_pb2.py b/protoc-gen-bitwise/options_pb2.py new file mode 100644 index 00000000..d3e637f2 --- /dev/null +++ b/protoc-gen-bitwise/options_pb2.py @@ -0,0 +1,171 @@ +""" +Manual parser for the custom bitwise field options defined in options.proto. + +Rather than relying on protobuf extension registration (which requires a properly +compiled _pb2 module), this module parses the raw serialized FieldOptions bytes +directly using the protobuf binary wire format. All protobuf runtimes preserve +unknown/unregistered extension bytes when round-tripping, so +field.options.SerializeToString() always contains the extension data even when +the extension is not registered in the Python runtime. + +Wire format reference: + tag = (field_number << 3) | wire_type + wire_type 0 = varint, 1 = 64-bit, 2 = length-delimited, 5 = 32-bit +""" + +import struct + +# Extension field numbers as defined in options.proto +QUANTIZED_FIELD_NUMBER = 50001 +BIT_PACKED_FIELD_NUMBER = 50002 + + +# --------------------------------------------------------------------------- +# Low-level wire-format helpers +# --------------------------------------------------------------------------- + +def _read_varint(data: bytes, pos: int): + """Decode a protobuf varint starting at *pos*. Returns (value, new_pos).""" + result = 0 + shift = 0 + while pos < len(data): + byte = data[pos] + pos += 1 + result |= (byte & 0x7F) << shift + if not (byte & 0x80): + break + shift += 7 + return result, pos + + +def _read_float32(data: bytes, pos: int): + """Decode a little-endian 32-bit float. Returns (value, new_pos).""" + value, = struct.unpack_from(' int: + """Advance *pos* past a field with the given wire_type.""" + if wire_type == 0: + _, pos = _read_varint(data, pos) + elif wire_type == 1: + pos += 8 + elif wire_type == 2: + length, pos = _read_varint(data, pos) + pos += length + elif wire_type == 5: + pos += 4 + # wire types 3 and 4 (start/end group) are deprecated; skip gracefully + return pos + + +# --------------------------------------------------------------------------- +# Option message classes +# --------------------------------------------------------------------------- + +class QuantizedFloatOptions: + """Mirrors the QuantizedFloatOptions proto message.""" + + __slots__ = ('min', 'max', 'bits') + + def __init__(self, min_val: float = 0.0, max_val: float = 0.0, bits: int = 0): + self.min = min_val + self.max = max_val + self.bits = bits + + @classmethod + def from_bytes(cls, data: bytes) -> 'QuantizedFloatOptions': + obj = cls() + pos = 0 + while pos < len(data): + tag, pos = _read_varint(data, pos) + field_num = tag >> 3 + wire_type = tag & 0x7 + if field_num == 1 and wire_type == 5: # min (float) + obj.min, pos = _read_float32(data, pos) + elif field_num == 2 and wire_type == 5: # max (float) + obj.max, pos = _read_float32(data, pos) + elif field_num == 3 and wire_type == 0: # bits (uint32) + obj.bits, pos = _read_varint(data, pos) + else: + pos = _skip_field(data, pos, wire_type) + return obj + + def __repr__(self): + return f'QuantizedFloatOptions(min={self.min}, max={self.max}, bits={self.bits})' + + +class BitPackedOptions: + """Mirrors the BitPackedOptions proto message.""" + + __slots__ = ('bits',) + + def __init__(self, bits: int = 0): + self.bits = bits + + @classmethod + def from_bytes(cls, data: bytes) -> 'BitPackedOptions': + obj = cls() + pos = 0 + while pos < len(data): + tag, pos = _read_varint(data, pos) + field_num = tag >> 3 + wire_type = tag & 0x7 + if field_num == 1 and wire_type == 0: # bits (uint32) + obj.bits, pos = _read_varint(data, pos) + else: + pos = _skip_field(data, pos, wire_type) + return obj + + def __repr__(self): + return f'BitPackedOptions(bits={self.bits})' + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def get_field_options(field_options_proto): + """ + Extract custom bitwise options from a FieldDescriptorProto.options object. + + Serialises the options message to bytes and walks the wire-format stream + looking for extension fields 50001 (quantized) and 50002 (bit_packed). + + Args: + field_options_proto: google.protobuf.descriptor_pb2.FieldOptions instance + (may be a default/empty instance when no options are set). + + Returns: + tuple[QuantizedFloatOptions | None, BitPackedOptions | None] + """ + try: + raw = field_options_proto.SerializeToString() + except Exception: + return None, None + + if not raw: + return None, None + + quantized = None + bit_packed = None + pos = 0 + + while pos < len(raw): + tag, pos = _read_varint(raw, pos) + field_num = tag >> 3 + wire_type = tag & 0x7 + + if wire_type == 2: # length-delimited + length, pos = _read_varint(raw, pos) + value_bytes = raw[pos:pos + length] + pos += length + if field_num == QUANTIZED_FIELD_NUMBER: + quantized = QuantizedFloatOptions.from_bytes(value_bytes) + elif field_num == BIT_PACKED_FIELD_NUMBER: + bit_packed = BitPackedOptions.from_bytes(value_bytes) + # else: unknown length-delimited field — already consumed + else: + pos = _skip_field(raw, pos, wire_type) + + return quantized, bit_packed diff --git a/protoc-gen-bitwise/plugin.py b/protoc-gen-bitwise/plugin.py new file mode 100644 index 00000000..09b5abe4 --- /dev/null +++ b/protoc-gen-bitwise/plugin.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +""" +protoc-gen-bitwise — protoc plugin that generates C# bitwise serialization code. + +Protocol: + 1. protoc writes a serialised CodeGeneratorRequest to this process's stdin. + 2. This plugin reads it, generates C# partial classes with Encode / Decode + methods for every message that carries [(quantized)] or [(bit_packed)] + field annotations. + 3. A serialised CodeGeneratorResponse is written to stdout. + +Usage (from project root): + protoc \\ + --proto_path=proto \\ + --bitwise_out=generated/ \\ + --plugin=protoc-gen-bitwise \\ + proto/my_messages.proto + +Windows invocation (plugin not on PATH as executable): + protoc \\ + --proto_path=proto \\ + --bitwise_out=generated/ \\ + --plugin=protoc-gen-bitwise=python protoc-gen-bitwise/plugin.py \\ + proto/my_messages.proto + +Dependencies: + pip install grpcio-tools # or: pip install protobuf +""" + +import os +import sys + +# Ensure sibling modules (generator_csharp, options_pb2) are importable +# regardless of where protoc invokes this script from. +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +# On Windows, stdin/stdout are opened in text mode by default which corrupts +# the binary protobuf payload. +if sys.platform == 'win32': + import msvcrt + msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) + msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) + +from google.protobuf.compiler import plugin_pb2 + +from generator_csharp import generate_csharp + + +def main() -> None: + request_bytes = sys.stdin.buffer.read() + + request = plugin_pb2.CodeGeneratorRequest() + request.ParseFromString(request_bytes) + + response = plugin_pb2.CodeGeneratorResponse() + # Advertise proto3-optional support so protoc does not reject the plugin. + response.supported_features = ( + plugin_pb2.CodeGeneratorResponse.FEATURE_PROTO3_OPTIONAL + ) + + # Build a lookup map for all file descriptors (needed for imports, though + # the current generator only uses the directly requested files). + file_by_name = {f.name: f for f in request.proto_file} + + for file_name in request.file_to_generate: + # Skip the options definition file itself — it has no messages to generate. + if file_name == 'decentraland/common/options.proto': + continue + + file_proto = file_by_name.get(file_name) + if file_proto is None: + continue + + try: + generated = generate_csharp(file_proto) + except Exception as exc: # noqa: BLE001 + error = response.file.add() + error.name = '' # empty name signals an error to protoc + # Append error text; protoc will print it and fail. + response.error = f'protoc-gen-bitwise: error processing {file_name}: {exc}' + continue + + if generated is not None: + out = response.file.add() + out.name = generated['name'] + out.content = generated['content'] + + sys.stdout.buffer.write(response.SerializeToString()) + + +if __name__ == '__main__': + main() diff --git a/protoc-gen-bitwise/runtime/cs/BitReader.cs b/protoc-gen-bitwise/runtime/cs/BitReader.cs new file mode 100644 index 00000000..51efa39c --- /dev/null +++ b/protoc-gen-bitwise/runtime/cs/BitReader.cs @@ -0,0 +1,112 @@ +// Decentraland.Networking.Bitwise — BitReader +// Copy this file into your Unity project alongside generated *.Bitwise.cs files. + +using System; + +namespace Decentraland.Networking.Bitwise +{ + /// + /// Reads bits from a byte buffer, MSB first within each byte (big-endian bit + /// order). Symmetric counterpart of : every + /// Write… call has a corresponding Read… call with identical arguments that + /// reproduces the original value. + /// + public sealed class BitReader + { + private readonly byte[] _buffer; + private int _bitPos; + + /// Source buffer filled by a . + public BitReader(byte[] buffer) + { + _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer)); + _bitPos = 0; + } + + /// Current read position in bits. + public int BitPosition => _bitPos; + + /// + /// Returns true when all written bits have been consumed + /// (i.e. has reached the end of the buffer). + /// + public bool IsAtEnd => _bitPos >= _buffer.Length * 8; + + // ----------------------------------------------------------------- + // Core primitive + // ----------------------------------------------------------------- + + /// + /// Reads bits and returns them as the + /// least-significant bits of a , MSB first. + /// + public uint ReadBits(int bits) + { + uint value = 0; + for (int i = bits - 1; i >= 0; i--) + { + int byteIdx = _bitPos / 8; + int bitIdx = 7 - (_bitPos % 8); + + if ((_buffer[byteIdx] >> bitIdx & 1) == 1) + value |= 1u << i; + + _bitPos++; + } + return value; + } + + // ----------------------------------------------------------------- + // Quantized float + // ----------------------------------------------------------------- + + /// + /// Reads a quantized float encoded with . + /// Arguments must match those used during encoding exactly. + /// + public float ReadQuantizedFloat(float min, float max, int bits) + { + uint maxQ = (1u << bits) - 1; + uint quantized = ReadBits(bits); + float normalized = (float)quantized / maxQ; + return min + normalized * (max - min); + } + + // ----------------------------------------------------------------- + // Standard IEEE 754 helpers + // ----------------------------------------------------------------- + + /// Reads a 32-bit IEEE 754 float written by . + public float ReadFloat() + { + uint bits = ReadBits(32); + byte[] bytes = + { + (byte)(bits & 0xFF), + (byte)((bits >> 8) & 0xFF), + (byte)((bits >> 16) & 0xFF), + (byte)((bits >> 24) & 0xFF), + }; + return BitConverter.ToSingle(bytes, 0); + } + + /// Reads a 64-bit IEEE 754 double written by . + public double ReadDouble() + { + uint hi = ReadBits(32); + uint lo = ReadBits(32); + byte[] bytes = + { + (byte)(lo & 0xFF), + (byte)((lo >> 8) & 0xFF), + (byte)((lo >> 16) & 0xFF), + (byte)((lo >> 24) & 0xFF), + (byte)(hi & 0xFF), + (byte)((hi >> 8) & 0xFF), + (byte)((hi >> 16) & 0xFF), + (byte)((hi >> 24) & 0xFF), + }; + return BitConverter.ToDouble(bytes, 0); + } + } +} diff --git a/protoc-gen-bitwise/runtime/cs/BitWriter.cs b/protoc-gen-bitwise/runtime/cs/BitWriter.cs new file mode 100644 index 00000000..19b205b8 --- /dev/null +++ b/protoc-gen-bitwise/runtime/cs/BitWriter.cs @@ -0,0 +1,117 @@ +// Decentraland.Networking.Bitwise — BitWriter +// Copy this file into your Unity project alongside generated *.Bitwise.cs files. + +using System; + +namespace Decentraland.Networking.Bitwise +{ + /// + /// Writes bits into a pre-allocated byte buffer, MSB first within each byte + /// (big-endian bit order). This matches the layout expected by + /// so that encode → decode is always a round-trip no-op. + /// + public sealed class BitWriter + { + private readonly byte[] _buffer; + private int _bitPos; + + /// Destination buffer (must be large enough for all writes). + public BitWriter(byte[] buffer) + { + _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer)); + _bitPos = 0; + } + + /// Current write position in bits. + public int BitPosition => _bitPos; + + /// Number of bytes written (rounded up to the nearest byte). + public int ByteLength => (_bitPos + 7) / 8; + + /// Returns a copy of the written bytes (trimmed to ). + public byte[] ToArray() + { + var result = new byte[ByteLength]; + Array.Copy(_buffer, result, ByteLength); + return result; + } + + // ----------------------------------------------------------------- + // Core primitive + // ----------------------------------------------------------------- + + /// + /// Writes the least-significant bits of + /// , MSB first. + /// + public void WriteBits(uint value, int bits) + { + for (int i = bits - 1; i >= 0; i--) + { + int byteIdx = _bitPos / 8; + int bitIdx = 7 - (_bitPos % 8); + + if ((value >> i & 1u) == 1u) + _buffer[byteIdx] |= (byte)(1 << bitIdx); + else + _buffer[byteIdx] &= (byte)~(1 << bitIdx); + + _bitPos++; + } + } + + // ----------------------------------------------------------------- + // Quantized float + // ----------------------------------------------------------------- + + /// + /// Quantizes into bits + /// using the range [, ] and + /// writes it to the buffer. + /// + /// Uses Math.Round (banker's rounding → ties-to-even) to guarantee + /// that encode → decode is a round-trip no-op. + /// + public void WriteQuantizedFloat(float value, float min, float max, int bits) + { + uint maxQ = (1u << bits) - 1; + float clamped = Math.Clamp(value, min, max); + float normalized = (clamped - min) / (max - min); + uint quantized = (uint)Math.Round(normalized * maxQ); + WriteBits(quantized, bits); + } + + // ----------------------------------------------------------------- + // Standard IEEE 754 helpers (used for un-annotated float/double fields) + // ----------------------------------------------------------------- + + /// Writes a 32-bit IEEE 754 float (4 bytes). + public void WriteFloat(float value) + { + byte[] bytes = BitConverter.GetBytes(value); + // GetBytes is little-endian on all platforms; write MSB first. + uint bits = (uint)bytes[0] + | ((uint)bytes[1] << 8) + | ((uint)bytes[2] << 16) + | ((uint)bytes[3] << 24); + WriteBits(bits, 32); + } + + /// Writes a 64-bit IEEE 754 double (8 bytes). + public void WriteDouble(double value) + { + byte[] bytes = BitConverter.GetBytes(value); + uint lo = (uint)bytes[0] + | ((uint)bytes[1] << 8) + | ((uint)bytes[2] << 16) + | ((uint)bytes[3] << 24); + uint hi = (uint)bytes[4] + | ((uint)bytes[5] << 8) + | ((uint)bytes[6] << 16) + | ((uint)bytes[7] << 24); + // Write high 32 bits first so the bit stream is big-endian at word level too. + WriteBits(hi, 32); + WriteBits(lo, 32); + } + } +} diff --git a/protoc-gen-bitwise/runtime/cs/Quantize.cs b/protoc-gen-bitwise/runtime/cs/Quantize.cs new file mode 100644 index 00000000..22c347fe --- /dev/null +++ b/protoc-gen-bitwise/runtime/cs/Quantize.cs @@ -0,0 +1,35 @@ +// Decentraland.Networking.Bitwise — Quantize +// Copy this file into your project alongside generated *.Bitwise.cs files. +// ReSharper disable once RedundantUsingDirective + +using System; + +namespace Decentraland.Networking.Bitwise + // ReSharper disable once ArrangeNamespaceBody +{ + /// + /// Static helpers for quantizing float values to/from unsigned integers. + /// Intended for use with protobuf uint32 fields: the integer is transmitted + /// as a protobuf varint, while the float accessor lives in a generated partial class. + /// + public static class Quantize + { + /// + /// Encodes to a quantized . + /// Values outside [, ] are clamped. + /// + public static uint Encode(float value, float min, float max, int bits) + { + var steps = (1u << bits) - 1; + var t = Math.Clamp((value - min) / (max - min), 0f, 1f); + return (uint)MathF.Round(t * steps); + } + + /// Decodes a quantized back to a float. + public static float Decode(uint encoded, float min, float max, int bits) + { + var steps = (1u << bits) - 1; + return (float)encoded / steps * (max - min) + min; + } + } +} From 4fa4c5c5ff3dd0bd68f0065f6f1a57eb8e69fe54 Mon Sep 17 00:00:00 2001 From: Mikhail Agapov <118179774+mikhail-dcl@users.noreply.github.com> Date: Mon, 15 Jun 2026 23:28:03 +0300 Subject: [PATCH 2/5] chore: port protoc-gen-bitwise to Node (drop Python dependency) (#423) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrite the protoc-gen-bitwise plugin from Python to a dependency-free Node implementation, so protocol generation needs only `node` on PATH — no Python interpreter and no `protobuf` pip package. - plugin.js stdin/stdout protoc plugin entry; advertises FEATURE_PROTO3_OPTIONAL - wire.js self-contained protobuf wire codec for CodeGeneratorRequest/Response (no runtime deps) - options.js parser for the custom quantized / bit_packed options - generator_csharp.js C# partial-class generator with faithful %g float formatting; output is byte-for-byte identical to the Python plugin - test/ + `npm run gen:test` golden-fixture parity test - remove plugin.py, generator_csharp.py, options_pb2.py - package.json ship only the .js + C# runtime (exclude test fixtures) - README.md, CLAUDE.md Python -> Node prerequisites and invocation Verified byte-identical regeneration three ways: protoc end-to-end, a Pulse clean-room rebuild (all 16 generated files), and unity-explorer build-protocol. Consumers invoke the plugin via a tiny `node plugin.js` wrapper (.cmd on Windows, shell script elsewhere). Co-authored-by: Claude Opus 4.8 (1M context) --- CLAUDE.md | 184 ++++++++++++++ README.md | 15 +- package.json | 6 +- protoc-gen-bitwise/generator_csharp.js | 215 ++++++++++++++++ protoc-gen-bitwise/generator_csharp.py | 176 ------------- protoc-gen-bitwise/options.js | 108 ++++++++ protoc-gen-bitwise/options_pb2.py | 171 ------------- protoc-gen-bitwise/plugin.js | 87 +++++++ protoc-gen-bitwise/plugin.py | 92 ------- protoc-gen-bitwise/test/generator.test.js | 180 +++++++++++++ .../test/golden/PulseServer.Bitwise.cs | 145 +++++++++++ .../golden/QuantizationExample.Bitwise.cs | 134 ++++++++++ protoc-gen-bitwise/wire.js | 239 ++++++++++++++++++ 13 files changed, 1305 insertions(+), 447 deletions(-) create mode 100644 CLAUDE.md create mode 100644 protoc-gen-bitwise/generator_csharp.js delete mode 100644 protoc-gen-bitwise/generator_csharp.py create mode 100644 protoc-gen-bitwise/options.js delete mode 100644 protoc-gen-bitwise/options_pb2.py create mode 100755 protoc-gen-bitwise/plugin.js delete mode 100644 protoc-gen-bitwise/plugin.py create mode 100644 protoc-gen-bitwise/test/generator.test.js create mode 100644 protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs create mode 100644 protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs create mode 100644 protoc-gen-bitwise/wire.js diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..130756ee --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,184 @@ +# CLAUDE.md — MMO Networking Stack + +## Project Overview + +High-performance MMO-style multiplayer networking stack. Protocol is **open** (Unity C# client + others). Infrastructure on **AWS**. Goals: high concurrency, aggressive interest management, low-latency state sync. + +| Layer | Choice | +|---|---| +| Transport | ENet (UDP, channel-based) | +| Client | Unity (C#) | +| Server | Custom server | +| Schema source of truth | `.proto` files | +| Serialization | Custom protoc plugin (bitwise encoding) | +| Auth | Decentraland ECDSA chain validation (local, on HANDSHAKE channel 0) | + +--- + +## Serialization: Custom Protoc Plugin + +### What it does +Reads `.proto` files with custom field options and generates **bitwise encode/decode code** in C#, keeping all client implementations bit-for-bit identical. + +### Custom Field Options (`options.proto`) + +```protobuf +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; + +message QuantizedFloatOptions { + float min = 1; + float max = 2; + uint32 bits = 3; +} + +message BitPackedOptions { + uint32 bits = 1; +} + +extend google.protobuf.FieldOptions { + QuantizedFloatOptions quantized = 50001; + BitPackedOptions bit_packed = 50002; +} +``` + +### Usage example + +```protobuf +message PositionDelta { + float dx = 1 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + float dy = 2 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + float dz = 3 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 entity_id = 4 [(bit_packed) = { bits: 20 }]; +} +// Total: 68 bits = 9 bytes on the wire +``` + +### Plugin structure + +``` +protoc-gen-bitwise/ +├── plugin.js # stdin -> CodeGeneratorRequest, stdout -> CodeGeneratorResponse (Node) +├── generator_csharp.js # emits C# for Unity +├── options.js # parses the custom quantized / bit_packed field options +├── wire.js # self-contained protobuf wire codec (zero runtime deps) +└── runtime/cs/ # C# runtime; Quantize.cs is copied into the generated output +``` + +Plugin contract: a protoc plugin that reads a serialized `CodeGeneratorRequest` from stdin and writes a serialized `CodeGeneratorResponse` to stdout. It is a plain Node script — **no `npm install` required, only `node` on PATH**. protoc invokes it through a tiny wrapper that runs `node plugin.js` (`.cmd` on Windows, a shell script elsewhere, since protoc cannot exec a `.js` directly). + +```bash +protoc \ + --proto_path=proto \ + --bitwise_out=generated/ \ + --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.js \ + movement.proto position.proto +``` + +Parity is locked down by `npm run gen:test` (compares generator output against golden C# fixtures in `protoc-gen-bitwise/test/`). + +--- + +## BitWriter / BitReader + +The C# implementation uses the following bit layout: **big-endian within each byte**, MSB written first. Use **`Round`** (not truncate) when quantizing to minimize error. + +### Core math — WriteQuantizedFloat + +``` +normalized = (clamp(value, min, max) - min) / (max - min) // -> [0.0, 1.0] +quantized = Round(normalized * ((1 << bits) - 1)) // -> integer +``` + +### Core math — ReadQuantizedFloat + +``` +normalized = quantized / ((1 << bits) - 1) +value = min + normalized * (max - min) +``` + +### Implementation + +```csharp +public class BitWriter +{ + private byte[] _buffer; + private int _bitPos; + + public BitWriter(byte[] buffer) { _buffer = buffer; _bitPos = 0; } + + public void WriteBits(uint value, int bits) + { + for (int i = bits - 1; i >= 0; i--) + { + int byteIdx = _bitPos / 8; + int bitIdx = 7 - (_bitPos % 8); + if ((value >> i & 1) == 1) _buffer[byteIdx] |= (byte)(1 << bitIdx); + else _buffer[byteIdx] &= (byte)~(1 << bitIdx); + _bitPos++; + } + } + + public void WriteQuantizedFloat(float value, float min, float max, int bits) + { + uint maxQ = (1u << bits) - 1; + float clamped = Math.Clamp(value, min, max); + float normalized = (clamped - min) / (max - min); + uint quantized = (uint)Math.Round(normalized * maxQ); + WriteBits(quantized, bits); + } +} + +public class BitReader +{ + private byte[] _buffer; + private int _bitPos; + + public BitReader(byte[] buffer) { _buffer = buffer; _bitPos = 0; } + + public uint ReadBits(int bits) + { + uint value = 0; + for (int i = bits - 1; i >= 0; i--) + { + int byteIdx = _bitPos / 8; + int bitIdx = 7 - (_bitPos % 8); + if ((_buffer[byteIdx] >> bitIdx & 1) == 1) value |= 1u << i; + _bitPos++; + } + return value; + } + + public float ReadQuantizedFloat(float min, float max, int bits) + { + uint maxQ = (1u << bits) - 1; + uint quantized = ReadBits(bits); + float normalized = (float)quantized / maxQ; + return min + normalized * (max - min); + } +} +``` + +--- + +## Precision Reference + +| Range | Bits | Step size | +|-------------|------|---------------| +| [-100, 100] | 16 | ~0.003 units | +| [-10, 10] | 12 | ~0.005 units | +| [-100, 100] | 12 | ~0.049 units | + +Sub-centimeter precision is achievable at 12-16 bits for position deltas. + +--- + +## Key Design Principles + +- `.proto` files are the **single source of truth** for all message schemas +- The protoc plugin generates **C#** from the schema — never hand-write serialization +- Encode -> decode is a **no-op** (round-trip safe) due to consistent use of `Round` +- Prefer **client-driven resync** over proactive server corrections +- Push complexity to clients where appropriate; server maintains authority +- Channel 0: reliable messages (STATE_FULL snapshots, ACKs, resync requests, HANDSHAKE) +- Channel 1: unreliable sequenced (high-frequency position deltas, client input) \ No newline at end of file diff --git a/README.md b/README.md index 20c75605..b11c3302 100644 --- a/README.md +++ b/README.md @@ -100,13 +100,11 @@ client can read it without knowledge of the plugin. | Requirement | Version | |---|---| -| Python | 3.10+ | -| `protobuf` Python package | 4.x or 3.20+ | +| Node.js | 16+ | | `protoc` | 3.19+ | -```bash -pip install protobuf -``` +The plugin is a dependency-free Node script — no `npm install` or extra +packages are required to run it. ## Step 1 — Annotate your `.proto` file @@ -157,11 +155,16 @@ protoc \ --proto_path=proto \ --proto_path=/path/to/google/protobuf/include \ --csharp_out=generated/cs \ - --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.py \ + --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.js \ --bitwise_out=generated/cs \ proto/decentraland/kernel/comms/v3/comms.proto ``` +> `plugin.js` carries a `#!/usr/bin/env node` shebang. On Windows, protoc +> cannot exec a `.js` directly, so point `--plugin` at a small `.cmd` wrapper +> that runs `node plugin.js`; on unix an equivalent shell script is used. Both +> consumer repos (Pulse, unity-explorer) generate this wrapper automatically. + The plugin emits one `*.Bitwise.cs` file (PascalCase, flat in the output directory) for each `.proto` file that contains at least one `[(quantized)]` field. diff --git a/package.json b/package.json index 2312a78a..bb3533f8 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "license": "Apache-2.0", "main": "index.js", "scripts": { - "test": "./scripts/check-proto-compabitility.sh" + "test": "./scripts/check-proto-compabitility.sh", + "gen:test": "node protoc-gen-bitwise/test/generator.test.js" }, "devDependencies": { "@protobuf-ts/protoc": "^2.8.1", @@ -32,6 +33,7 @@ "out-ts", "out-js", "public", - "protoc-gen-bitwise" + "protoc-gen-bitwise/*.js", + "protoc-gen-bitwise/runtime" ] } diff --git a/protoc-gen-bitwise/generator_csharp.js b/protoc-gen-bitwise/generator_csharp.js new file mode 100644 index 00000000..dcdb95a0 --- /dev/null +++ b/protoc-gen-bitwise/generator_csharp.js @@ -0,0 +1,215 @@ +'use strict' + +/** + * C# code generator for the protoc-gen-bitwise plugin. + * + * For every proto message that contains at least one uint32 field annotated + * with [(quantized)], this module emits a C# partial class that adds a computed + * float property named {FieldName}Quantized. The getter decodes the stored + * uint32 to a float; the setter encodes a float back to a uint32. Standard + * protobuf handles serialization of the uint32 wire field; this class adds a + * typed float accessor on top. + * + * Only uint32 fields are supported. bit_packed and unannotated fields are + * passed through without generating any accessor. + * + * Port of the original generator_csharp.py — output is intended to be + * byte-for-byte identical. + */ + +const { getFieldOptions } = require('./options') + +// FieldDescriptorProto type/label constants. +const TYPE_UINT32 = 13 +const LABEL_REPEATED = 3 + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** Mirrors Python str.capitalize(): upper-first, lowercase the rest. */ +function capitalize(word) { + if (word.length === 0) return '' + return word[0].toUpperCase() + word.slice(1).toLowerCase() +} + +/** position_x -> PositionX */ +function snakeToPascal(name) { + return name.split('_').map(capitalize).join('') +} + +/** decentraland.kernel.comms.v3 -> Decentraland.Kernel.Comms.V3 */ +function packageToNamespace(pkg) { + if (!pkg) return 'Generated' + return pkg.split('.').map(capitalize).join('.') +} + +/** + * Format a number with C printf %g semantics at `precision` significant digits: + * shortest of fixed/scientific, with trailing zeros and a trailing dot removed. + * Reproduces Python's `f'{value:.{precision}g}'`. + */ +function formatG(value, precision) { + if (precision <= 0) precision = 1 + if (value === 0) return '0' + if (!Number.isFinite(value)) return value > 0 ? 'inf' : 'nan' + + const negative = value < 0 + const v = Math.abs(value) + + // Correctly-rounded scientific form yields the decimal exponent X (handling + // carry such as 9.999 -> 1.0e+1). + const sci = v.toExponential(precision - 1) + const eIdx = sci.indexOf('e') + const X = parseInt(sci.slice(eIdx + 1), 10) + + let result + if (X >= -4 && X < precision) { + // Fixed notation with (precision - 1 - X) fraction digits. + const fractionDigits = precision - 1 - X + result = v.toFixed(fractionDigits >= 0 ? fractionDigits : 0) + if (result.indexOf('.') !== -1) { + result = result.replace(/0+$/, '').replace(/\.$/, '') + } + } else { + // Scientific notation; printf prints at least two exponent digits. + let mantissa = sci.slice(0, eIdx) + if (mantissa.indexOf('.') !== -1) { + mantissa = mantissa.replace(/0+$/, '').replace(/\.$/, '') + } + const expSign = X < 0 ? '-' : '+' + let expDigits = String(Math.abs(X)) + if (expDigits.length < 2) expDigits = '0' + expDigits + result = mantissa + 'e' + expSign + expDigits + } + + return (negative ? '-' : '') + result +} + +/** Format a value as a C# float literal (e.g. -100.0f). */ +function formatFloat(value) { + let text = formatG(value, 8) + if (text.indexOf('.') === -1 && text.indexOf('e') === -1 && text.indexOf('E') === -1) { + text += '.0' + } + return text + 'f' +} + +/** Format a quantization step size for a doc comment (e.g. "≈ 0.003"). */ +function formatStep(step) { + return '≈ ' + formatG(step, 6) +} + +// --------------------------------------------------------------------------- +// Per-message code generation +// --------------------------------------------------------------------------- + +/** + * Generate a C# partial class for a proto message, or null if it has no + * quantized uint32 fields. Returns an array of lines (no trailing newline). + */ +function generateMessage(msgProto, indent) { + const i = indent || ' ' + const props = [] + + for (const field of msgProto.field) { + // Repeated/map fields are not supported. + if (field.label === LABEL_REPEATED) continue + // Only uint32 fields are candidates for quantized accessors. + if (field.type !== TYPE_UINT32) continue + + const { quantized } = getFieldOptions(field.optionsRaw) + if (quantized === null) continue + + const step = (quantized.max - quantized.min) / ((1 << quantized.bits) - 1) + + props.push({ + propName: snakeToPascal(field.name), + mn: formatFloat(quantized.min), + mx: formatFloat(quantized.max), + bits: quantized.bits, + step, + }) + } + + if (props.length === 0) return null + + const backings = [] + const lines = [] + lines.push(`public partial class ${msgProto.name}`) + lines.push('{') + + for (const { propName, mn, mx, bits, step } of props) { + const backing = '_' + propName[0].toLowerCase() + propName.slice(1) + backings.push(backing) + lines.push(`${i}private float? ${backing};`) + lines.push( + `${i}/// Float accessor for . Range [${mn}, ${mx}], ${bits} bits, step ${formatStep(step)}.`, + ) + lines.push(`${i}public float ${propName}Quantized`) + lines.push(`${i}{`) + lines.push(`${i}${i}get => ${backing} ??= Quantize.Decode(${propName}, ${mn}, ${mx}, ${bits});`) + lines.push(`${i}${i}set { ${backing} = value; ${propName} = Quantize.Encode(value, ${mn}, ${mx}, ${bits}); }`) + lines.push(`${i}}`) + lines.push('') + } + + lines.push(`${i}/// Clears all cached decoded values. Call after mutating raw uint32 fields directly.`) + lines.push(`${i}public void ResetDecodedCache()`) + lines.push(`${i}{`) + for (const backing of backings) { + lines.push(`${i}${i}${backing} = null;`) + } + lines.push(`${i}}`) + + lines.push('}') + return lines +} + +// --------------------------------------------------------------------------- +// Per-file code generation (public entry point) +// --------------------------------------------------------------------------- + +/** + * Generate a C# source file for a FileDescriptorProto, or null if the file + * contains no quantized uint32 fields. + * @returns {{name: string, content: string} | null} + */ +function generateCsharp(fileProto) { + const namespace = packageToNamespace(fileProto.package) + + const protoFile = fileProto.name.split('/').pop() + const stem = snakeToPascal(protoFile.replace('.proto', '')) + const outName = `${stem}.Bitwise.cs` + + const header = [ + '// ', + '// Generated by protoc-gen-bitwise. DO NOT EDIT.', + `// Source: ${fileProto.name}`, + '// ', + '', + 'using Decentraland.Networking.Bitwise;', + '', + `namespace ${namespace}`, + '{', + ] + const footer = ['', `} // namespace ${namespace}`] + + const body = [] + for (const msg of fileProto.messageType) { + const msgLines = generateMessage(msg) + if (msgLines === null) continue + // Indent each line by 4 spaces (inside the namespace block). + for (const line of msgLines) { + body.push(line ? ' ' + line : '') + } + body.push('') + } + + if (body.length === 0) return null + + const content = header.concat(body, footer).join('\n') + '\n' + return { name: outName, content } +} + +module.exports = { generateCsharp } diff --git a/protoc-gen-bitwise/generator_csharp.py b/protoc-gen-bitwise/generator_csharp.py deleted file mode 100644 index 376c0dc1..00000000 --- a/protoc-gen-bitwise/generator_csharp.py +++ /dev/null @@ -1,176 +0,0 @@ -""" -C# code generator for the protoc-gen-bitwise plugin. - -For every proto message that contains at least one uint32 field annotated with -[(quantized)], this module emits a C# partial class that adds a computed float -property named {FieldName}Quantized. The getter decodes the stored uint32 to -a float; the setter encodes a float back to a uint32. Standard protobuf -handles serialization of the uint32 wire field; this class adds a typed float -accessor on top. - -Protobuf encodes small uint32 values via varint, so a value that fits in -2^20 costs 3 bytes on the wire — cheaper than a raw IEEE 754 float (4 bytes). - -Only uint32 fields are supported. bit_packed and unannotated fields are -passed through without generating any accessor. -""" - -from google.protobuf import descriptor_pb2 - -from options_pb2 import get_field_options - -# FieldDescriptorProto type constants (aliased for readability) -_FT = descriptor_pb2.FieldDescriptorProto - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _snake_to_pascal(name: str) -> str: - """position_x → PositionX""" - return ''.join(word.capitalize() for word in name.split('_')) - - -def _package_to_namespace(package: str) -> str: - """decentraland.kernel.comms.v3 → Decentraland.Kernel.Comms.V3""" - if not package: - return 'Generated' - return '.'.join(part.capitalize() for part in package.split('.')) - - -def _format_float(value: float) -> str: - """Format a Python float as a C# float literal (e.g. -100.0f).""" - text = f'{value:.8g}' - if '.' not in text and 'e' not in text and 'E' not in text: - text += '.0' - return text + 'f' - - -def _format_step(step: float) -> str: - """Format a quantization step size for display in a doc comment (e.g. ≈ 0.003).""" - return f'\u2248 {step:.6g}' - - -# --------------------------------------------------------------------------- -# Per-message code generation -# --------------------------------------------------------------------------- - -def _gen_message(msg_proto, indent: str = ' ') -> list[str] | None: - """ - Generate a C# partial class for a proto message. - - For each uint32 field with a [(quantized)] annotation, emits a cached float - property {FieldName}Quantized backed by the raw uint32 field. - - Returns a list of lines (without trailing newline) or None if the message - has no quantized uint32 fields. - """ - props: list[tuple[str, str, str, int, float]] = [] # (prop_name, mn, mx, bits, step) - - for field in msg_proto.field: - # Repeated/map fields are not supported - if field.label == _FT.LABEL_REPEATED: - continue - - # Only uint32 fields are candidates for quantized accessors - if field.type != _FT.TYPE_UINT32: - continue - - quantized, _ = get_field_options(field.options) - if quantized is None: - continue - - step = (quantized.max - quantized.min) / ((1 << quantized.bits) - 1) - - props.append(( - _snake_to_pascal(field.name), - _format_float(quantized.min), - _format_float(quantized.max), - quantized.bits, - step, - )) - - if not props: - return None - - i = indent - backings: list[str] = [] - lines: list[str] = [] - lines.append(f'public partial class {msg_proto.name}') - lines.append('{') - - for prop_name, mn, mx, bits, step in props: - backing = '_' + prop_name[0].lower() + prop_name[1:] - backings.append(backing) - lines.append(f'{i}private float? {backing};') - lines.append(f'{i}/// Float accessor for . Range [{mn}, {mx}], {bits} bits, step {_format_step(step)}.') - lines.append(f'{i}public float {prop_name}Quantized') - lines.append(f'{i}{{') - lines.append(f'{i}{i}get => {backing} ??= Quantize.Decode({prop_name}, {mn}, {mx}, {bits});') - lines.append(f'{i}{i}set {{ {backing} = value; {prop_name} = Quantize.Encode(value, {mn}, {mx}, {bits}); }}') - lines.append(f'{i}}}') - lines.append('') - - lines.append(f'{i}/// Clears all cached decoded values. Call after mutating raw uint32 fields directly.') - lines.append(f'{i}public void ResetDecodedCache()') - lines.append(f'{i}{{') - for backing in backings: - lines.append(f'{i}{i}{backing} = null;') - lines.append(f'{i}}}') - - lines.append('}') - return lines - - -# --------------------------------------------------------------------------- -# Per-file code generation (public entry point) -# --------------------------------------------------------------------------- - -def generate_csharp(file_proto) -> dict | None: - """ - Generate a C# source file for a FileDescriptorProto. - - Returns a dict with keys 'name' (output path) and 'content' (C# source), - or None if the file contains no quantized uint32 fields. - """ - namespace = _package_to_namespace(file_proto.package) - - # Output is placed flat in the output root, matching --csharp_out convention. - # e.g. "decentraland/kernel/comms/v3/my_message.proto" - # → "MyMessage.Bitwise.cs" - proto_file = file_proto.name.rsplit('/', 1)[-1] - stem = _snake_to_pascal(proto_file.replace('.proto', '')) - out_name = f'{stem}.Bitwise.cs' - - header = [ - '// ', - '// Generated by protoc-gen-bitwise. DO NOT EDIT.', - f'// Source: {file_proto.name}', - '// ', - '', - 'using Decentraland.Networking.Bitwise;', - '', - f'namespace {namespace}', - '{', - ] - footer = [ - '', - f'}} // namespace {namespace}', - ] - - body: list[str] = [] - for msg in file_proto.message_type: - msg_lines = _gen_message(msg) - if msg_lines is None: - continue - # Indent each line by 4 spaces (inside the namespace block) - for line in msg_lines: - body.append((' ' + line) if line else '') - body.append('') - - if not body: - return None # nothing to emit - - content = '\n'.join(header + body + footer) + '\n' - return {'name': out_name, 'content': content} diff --git a/protoc-gen-bitwise/options.js b/protoc-gen-bitwise/options.js new file mode 100644 index 00000000..150ece0b --- /dev/null +++ b/protoc-gen-bitwise/options.js @@ -0,0 +1,108 @@ +'use strict' + +/** + * Parser for the custom bitwise field options defined in options.proto. + * + * The descriptor decoder hands us the raw serialized FieldOptions bytes (it + * declares `options` as opaque bytes). We walk those bytes looking for the two + * custom extension field numbers — protobuf preserves unknown/unregistered + * extension bytes, so they are always present even though no runtime here knows + * the extension schema. This mirrors the original options_pb2.py. + * + * Wire format: tag = (field_number << 3) | wire_type + * wire_type 0 = varint, 1 = 64-bit, 2 = length-delimited, 5 = 32-bit + */ + +const { readVarint, skipField } = require('./wire') + +// Extension field numbers as defined in options.proto. +const QUANTIZED_FIELD_NUMBER = 50001 +const BIT_PACKED_FIELD_NUMBER = 50002 + +/** Parse a serialized QuantizedFloatOptions message: { min, max, bits }. */ +function parseQuantized(data) { + const opts = { min: 0.0, max: 0.0, bits: 0 } + let pos = 0 + while (pos < data.length) { + let tag + ;[tag, pos] = readVarint(data, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 5) { + // min (float) + opts.min = data.readFloatLE(pos) + pos += 4 + } else if (fieldNum === 2 && wireType === 5) { + // max (float) + opts.max = data.readFloatLE(pos) + pos += 4 + } else if (fieldNum === 3 && wireType === 0) { + // bits (uint32) + ;[opts.bits, pos] = readVarint(data, pos) + } else { + pos = skipField(data, pos, wireType) + } + } + return opts +} + +/** Parse a serialized BitPackedOptions message: { bits }. */ +function parseBitPacked(data) { + const opts = { bits: 0 } + let pos = 0 + while (pos < data.length) { + let tag + ;[tag, pos] = readVarint(data, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 0) { + // bits (uint32) + ;[opts.bits, pos] = readVarint(data, pos) + } else { + pos = skipField(data, pos, wireType) + } + } + return opts +} + +/** + * Extract custom bitwise options from raw FieldOptions bytes. + * + * @param {Buffer|null} optionsRaw serialized FieldOptions, or null when unset. + * @returns {{quantized: object|null, bitPacked: object|null}} + */ +function getFieldOptions(optionsRaw) { + if (!optionsRaw || optionsRaw.length === 0) { + return { quantized: null, bitPacked: null } + } + + let quantized = null + let bitPacked = null + let pos = 0 + + while (pos < optionsRaw.length) { + let tag + ;[tag, pos] = readVarint(optionsRaw, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + + if (wireType === 2) { + let len + ;[len, pos] = readVarint(optionsRaw, pos) + const valueBytes = optionsRaw.subarray(pos, pos + len) + pos += len + if (fieldNum === QUANTIZED_FIELD_NUMBER) { + quantized = parseQuantized(valueBytes) + } else if (fieldNum === BIT_PACKED_FIELD_NUMBER) { + bitPacked = parseBitPacked(valueBytes) + } + // else: unknown length-delimited field — already consumed. + } else { + pos = skipField(optionsRaw, pos, wireType) + } + } + + return { quantized, bitPacked } +} + +module.exports = { getFieldOptions } diff --git a/protoc-gen-bitwise/options_pb2.py b/protoc-gen-bitwise/options_pb2.py deleted file mode 100644 index d3e637f2..00000000 --- a/protoc-gen-bitwise/options_pb2.py +++ /dev/null @@ -1,171 +0,0 @@ -""" -Manual parser for the custom bitwise field options defined in options.proto. - -Rather than relying on protobuf extension registration (which requires a properly -compiled _pb2 module), this module parses the raw serialized FieldOptions bytes -directly using the protobuf binary wire format. All protobuf runtimes preserve -unknown/unregistered extension bytes when round-tripping, so -field.options.SerializeToString() always contains the extension data even when -the extension is not registered in the Python runtime. - -Wire format reference: - tag = (field_number << 3) | wire_type - wire_type 0 = varint, 1 = 64-bit, 2 = length-delimited, 5 = 32-bit -""" - -import struct - -# Extension field numbers as defined in options.proto -QUANTIZED_FIELD_NUMBER = 50001 -BIT_PACKED_FIELD_NUMBER = 50002 - - -# --------------------------------------------------------------------------- -# Low-level wire-format helpers -# --------------------------------------------------------------------------- - -def _read_varint(data: bytes, pos: int): - """Decode a protobuf varint starting at *pos*. Returns (value, new_pos).""" - result = 0 - shift = 0 - while pos < len(data): - byte = data[pos] - pos += 1 - result |= (byte & 0x7F) << shift - if not (byte & 0x80): - break - shift += 7 - return result, pos - - -def _read_float32(data: bytes, pos: int): - """Decode a little-endian 32-bit float. Returns (value, new_pos).""" - value, = struct.unpack_from(' int: - """Advance *pos* past a field with the given wire_type.""" - if wire_type == 0: - _, pos = _read_varint(data, pos) - elif wire_type == 1: - pos += 8 - elif wire_type == 2: - length, pos = _read_varint(data, pos) - pos += length - elif wire_type == 5: - pos += 4 - # wire types 3 and 4 (start/end group) are deprecated; skip gracefully - return pos - - -# --------------------------------------------------------------------------- -# Option message classes -# --------------------------------------------------------------------------- - -class QuantizedFloatOptions: - """Mirrors the QuantizedFloatOptions proto message.""" - - __slots__ = ('min', 'max', 'bits') - - def __init__(self, min_val: float = 0.0, max_val: float = 0.0, bits: int = 0): - self.min = min_val - self.max = max_val - self.bits = bits - - @classmethod - def from_bytes(cls, data: bytes) -> 'QuantizedFloatOptions': - obj = cls() - pos = 0 - while pos < len(data): - tag, pos = _read_varint(data, pos) - field_num = tag >> 3 - wire_type = tag & 0x7 - if field_num == 1 and wire_type == 5: # min (float) - obj.min, pos = _read_float32(data, pos) - elif field_num == 2 and wire_type == 5: # max (float) - obj.max, pos = _read_float32(data, pos) - elif field_num == 3 and wire_type == 0: # bits (uint32) - obj.bits, pos = _read_varint(data, pos) - else: - pos = _skip_field(data, pos, wire_type) - return obj - - def __repr__(self): - return f'QuantizedFloatOptions(min={self.min}, max={self.max}, bits={self.bits})' - - -class BitPackedOptions: - """Mirrors the BitPackedOptions proto message.""" - - __slots__ = ('bits',) - - def __init__(self, bits: int = 0): - self.bits = bits - - @classmethod - def from_bytes(cls, data: bytes) -> 'BitPackedOptions': - obj = cls() - pos = 0 - while pos < len(data): - tag, pos = _read_varint(data, pos) - field_num = tag >> 3 - wire_type = tag & 0x7 - if field_num == 1 and wire_type == 0: # bits (uint32) - obj.bits, pos = _read_varint(data, pos) - else: - pos = _skip_field(data, pos, wire_type) - return obj - - def __repr__(self): - return f'BitPackedOptions(bits={self.bits})' - - -# --------------------------------------------------------------------------- -# Public API -# --------------------------------------------------------------------------- - -def get_field_options(field_options_proto): - """ - Extract custom bitwise options from a FieldDescriptorProto.options object. - - Serialises the options message to bytes and walks the wire-format stream - looking for extension fields 50001 (quantized) and 50002 (bit_packed). - - Args: - field_options_proto: google.protobuf.descriptor_pb2.FieldOptions instance - (may be a default/empty instance when no options are set). - - Returns: - tuple[QuantizedFloatOptions | None, BitPackedOptions | None] - """ - try: - raw = field_options_proto.SerializeToString() - except Exception: - return None, None - - if not raw: - return None, None - - quantized = None - bit_packed = None - pos = 0 - - while pos < len(raw): - tag, pos = _read_varint(raw, pos) - field_num = tag >> 3 - wire_type = tag & 0x7 - - if wire_type == 2: # length-delimited - length, pos = _read_varint(raw, pos) - value_bytes = raw[pos:pos + length] - pos += length - if field_num == QUANTIZED_FIELD_NUMBER: - quantized = QuantizedFloatOptions.from_bytes(value_bytes) - elif field_num == BIT_PACKED_FIELD_NUMBER: - bit_packed = BitPackedOptions.from_bytes(value_bytes) - # else: unknown length-delimited field — already consumed - else: - pos = _skip_field(raw, pos, wire_type) - - return quantized, bit_packed diff --git a/protoc-gen-bitwise/plugin.js b/protoc-gen-bitwise/plugin.js new file mode 100755 index 00000000..b08c6a12 --- /dev/null +++ b/protoc-gen-bitwise/plugin.js @@ -0,0 +1,87 @@ +#!/usr/bin/env node +'use strict' + +/** + * protoc-gen-bitwise — protoc plugin that generates C# bitwise serialization code. + * + * Protocol: + * 1. protoc writes a serialised CodeGeneratorRequest to this process's stdin. + * 2. This plugin reads it, generates C# partial classes with float accessor + * properties for every message that carries [(quantized)] field annotations. + * 3. A serialised CodeGeneratorResponse is written to stdout. + * + * Implemented in plain Node with a self-contained protobuf wire codec (see + * wire.js) so it runs with only `node` on PATH — no npm install required, even + * when invoked directly from a sibling checkout. + * + * Usage (from project root): + * protoc \ + * --proto_path=proto \ + * --bitwise_out=generated/ \ + * --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.js \ + * proto/my_messages.proto + * + * On Windows, protoc needs an executable wrapper, e.g. a .cmd that runs: + * node "\protoc-gen-bitwise\plugin.js" %* + */ + +const { decodeRequest, encodeResponse } = require('./wire') +const { generateCsharp } = require('./generator_csharp') + +// CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL — advertised so protoc +// does not reject the plugin when the schema uses proto3 `optional` fields. +const FEATURE_PROTO3_OPTIONAL = 1 + +function readStdin() { + return new Promise((resolve, reject) => { + const chunks = [] + process.stdin.on('data', (chunk) => chunks.push(chunk)) + process.stdin.on('end', () => resolve(Buffer.concat(chunks))) + process.stdin.on('error', reject) + }) +} + +async function main() { + const requestBytes = await readStdin() + const request = decodeRequest(requestBytes) + + // Lookup map for all file descriptors (parity with the Python plugin). + const fileByName = new Map() + for (const file of request.protoFile) fileByName.set(file.name, file) + + const files = [] + let error = null + + for (const fileName of request.fileToGenerate) { + // Skip the options definition file itself — it has no messages to generate. + if (fileName === 'decentraland/common/options.proto') continue + + const fileProto = fileByName.get(fileName) + if (!fileProto) continue + + try { + const generated = generateCsharp(fileProto) + if (generated) files.push(generated) + } catch (exc) { + error = `protoc-gen-bitwise: error processing ${fileName}: ${exc && exc.message ? exc.message : exc}` + } + } + + const response = encodeResponse({ + error, + supportedFeatures: FEATURE_PROTO3_OPTIONAL, + files, + }) + + // Let the write flush before the process exits (do not call process.exit()). + process.stdout.write(response) +} + +main().catch((exc) => { + const response = encodeResponse({ + error: `protoc-gen-bitwise: ${exc && exc.stack ? exc.stack : exc}`, + supportedFeatures: FEATURE_PROTO3_OPTIONAL, + files: [], + }) + process.stdout.write(response) +}) diff --git a/protoc-gen-bitwise/plugin.py b/protoc-gen-bitwise/plugin.py deleted file mode 100644 index 09b5abe4..00000000 --- a/protoc-gen-bitwise/plugin.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python3 -""" -protoc-gen-bitwise — protoc plugin that generates C# bitwise serialization code. - -Protocol: - 1. protoc writes a serialised CodeGeneratorRequest to this process's stdin. - 2. This plugin reads it, generates C# partial classes with Encode / Decode - methods for every message that carries [(quantized)] or [(bit_packed)] - field annotations. - 3. A serialised CodeGeneratorResponse is written to stdout. - -Usage (from project root): - protoc \\ - --proto_path=proto \\ - --bitwise_out=generated/ \\ - --plugin=protoc-gen-bitwise \\ - proto/my_messages.proto - -Windows invocation (plugin not on PATH as executable): - protoc \\ - --proto_path=proto \\ - --bitwise_out=generated/ \\ - --plugin=protoc-gen-bitwise=python protoc-gen-bitwise/plugin.py \\ - proto/my_messages.proto - -Dependencies: - pip install grpcio-tools # or: pip install protobuf -""" - -import os -import sys - -# Ensure sibling modules (generator_csharp, options_pb2) are importable -# regardless of where protoc invokes this script from. -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -# On Windows, stdin/stdout are opened in text mode by default which corrupts -# the binary protobuf payload. -if sys.platform == 'win32': - import msvcrt - msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) - msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) - -from google.protobuf.compiler import plugin_pb2 - -from generator_csharp import generate_csharp - - -def main() -> None: - request_bytes = sys.stdin.buffer.read() - - request = plugin_pb2.CodeGeneratorRequest() - request.ParseFromString(request_bytes) - - response = plugin_pb2.CodeGeneratorResponse() - # Advertise proto3-optional support so protoc does not reject the plugin. - response.supported_features = ( - plugin_pb2.CodeGeneratorResponse.FEATURE_PROTO3_OPTIONAL - ) - - # Build a lookup map for all file descriptors (needed for imports, though - # the current generator only uses the directly requested files). - file_by_name = {f.name: f for f in request.proto_file} - - for file_name in request.file_to_generate: - # Skip the options definition file itself — it has no messages to generate. - if file_name == 'decentraland/common/options.proto': - continue - - file_proto = file_by_name.get(file_name) - if file_proto is None: - continue - - try: - generated = generate_csharp(file_proto) - except Exception as exc: # noqa: BLE001 - error = response.file.add() - error.name = '' # empty name signals an error to protoc - # Append error text; protoc will print it and fail. - response.error = f'protoc-gen-bitwise: error processing {file_name}: {exc}' - continue - - if generated is not None: - out = response.file.add() - out.name = generated['name'] - out.content = generated['content'] - - sys.stdout.buffer.write(response.SerializeToString()) - - -if __name__ == '__main__': - main() diff --git a/protoc-gen-bitwise/test/generator.test.js b/protoc-gen-bitwise/test/generator.test.js new file mode 100644 index 00000000..95512154 --- /dev/null +++ b/protoc-gen-bitwise/test/generator.test.js @@ -0,0 +1,180 @@ +'use strict' + +/** + * Byte-for-byte parity test for the C# generator. + * + * Builds in-memory FieldDescriptorProto structures (with real serialized + * FieldOptions extension bytes), runs them through the same options parser and + * generator the plugin uses, and asserts the output equals committed golden + * files. This locks down the %g float formatting and the emitted layout without + * needing protoc. + * + * Run with: node protoc-gen-bitwise/test/generator.test.js (or `npm run gen:test`) + */ + +const assert = require('assert') +const fs = require('fs') +const path = require('path') + +const { generateCsharp } = require('../generator_csharp') + +// FieldDescriptorProto.Type +const TYPE_DOUBLE = 1 +const TYPE_BOOL = 8 +const TYPE_UINT32 = 13 +// FieldDescriptorProto.Label +const LABEL_OPTIONAL = 1 + +// --- minimal wire encoders to build serialized FieldOptions extension bytes --- + +function encodeVarint(value) { + const bytes = [] + let v = value + while (v > 0x7f) { + bytes.push((v & 0x7f) | 0x80) + v = Math.floor(v / 128) + } + bytes.push(v & 0x7f) + return Buffer.from(bytes) +} + +function tag(fieldNum, wireType) { + return encodeVarint((fieldNum << 3) | wireType) +} + +function floatLE(value) { + const b = Buffer.alloc(4) + b.writeFloatLE(value, 0) + return b +} + +function lengthDelimited(fieldNum, payload) { + return Buffer.concat([tag(fieldNum, 2), encodeVarint(payload.length), payload]) +} + +// FieldOptions bytes carrying ext 50001 (quantized) = { min, max, bits } +function quantizedOptions(min, max, bits) { + const inner = Buffer.concat([ + tag(1, 5), floatLE(min), + tag(2, 5), floatLE(max), + tag(3, 0), encodeVarint(bits), + ]) + return lengthDelimited(50001, inner) +} + +// FieldOptions bytes carrying ext 50002 (bit_packed) = { bits } +function bitPackedOptions(bits) { + const inner = Buffer.concat([tag(1, 0), encodeVarint(bits)]) + return lengthDelimited(50002, inner) +} + +function field(name, type, optionsRaw) { + return { name, label: LABEL_OPTIONAL, type, optionsRaw: optionsRaw || null } +} + +function readGolden(name) { + return fs.readFileSync(path.join(__dirname, 'golden', name), 'utf8') +} + +// -------------------------------------------------------------------------- +// Case 1: quantization_example.proto +// -------------------------------------------------------------------------- + +const quantizationExample = { + name: 'decentraland/common/quantization_example.proto', + package: 'decentraland.common', + messageType: [ + { + name: 'PositionDelta', + field: [ + field('dx', TYPE_UINT32, quantizedOptions(-100, 100, 16)), + field('dy', TYPE_UINT32, quantizedOptions(-100, 100, 16)), + field('dz', TYPE_UINT32, quantizedOptions(-100, 100, 16)), + field('entity_id', TYPE_UINT32, bitPackedOptions(20)), + field('sequence', TYPE_UINT32, bitPackedOptions(12)), + ], + }, + { + name: 'PlayerInput', + field: [ + field('move_x', TYPE_UINT32, quantizedOptions(-1, 1, 8)), + field('move_z', TYPE_UINT32, quantizedOptions(-1, 1, 8)), + field('yaw', TYPE_UINT32, quantizedOptions(-180, 180, 12)), + field('buttons', TYPE_UINT32, bitPackedOptions(8)), + field('sequence', TYPE_UINT32, bitPackedOptions(12)), + ], + }, + { + name: 'AvatarStateSnapshot', + field: [ + field('x', TYPE_UINT32, quantizedOptions(-4096, 4096, 16)), + field('y', TYPE_UINT32, quantizedOptions(-256, 256, 14)), + field('z', TYPE_UINT32, quantizedOptions(-4096, 4096, 16)), + field('pitch', TYPE_UINT32, quantizedOptions(-90, 90, 10)), + field('yaw', TYPE_UINT32, quantizedOptions(-180, 180, 12)), + field('entity_id', TYPE_UINT32, bitPackedOptions(20)), + field('animation_state', TYPE_UINT32, bitPackedOptions(6)), + field('is_grounded', TYPE_BOOL, null), + field('timestamp', TYPE_DOUBLE, null), + ], + }, + ], +} + +// -------------------------------------------------------------------------- +// Case 2: pulse_server.proto (PlayerStateDeltaTier0) — wider bit/range coverage +// -------------------------------------------------------------------------- + +const pulseServer = { + name: 'decentraland/pulse/pulse_server.proto', + package: 'decentraland.pulse', + messageType: [ + { + name: 'PlayerStateDeltaTier0', + field: [ + field('position_x', TYPE_UINT32, quantizedOptions(0, 16, 8)), + field('position_y', TYPE_UINT32, quantizedOptions(0, 200, 13)), + field('position_z', TYPE_UINT32, quantizedOptions(0, 16, 8)), + field('velocity_x', TYPE_UINT32, quantizedOptions(-50, 50, 8)), + field('velocity_y', TYPE_UINT32, quantizedOptions(-50, 50, 8)), + field('velocity_z', TYPE_UINT32, quantizedOptions(-50, 50, 8)), + field('rotation_y', TYPE_UINT32, quantizedOptions(0, 360, 7)), + field('movement_blend', TYPE_UINT32, quantizedOptions(0, 3, 5)), + field('slide_blend', TYPE_UINT32, quantizedOptions(0, 1, 4)), + field('head_yaw', TYPE_UINT32, quantizedOptions(0, 360, 7)), + field('head_pitch', TYPE_UINT32, quantizedOptions(0, 360, 7)), + field('point_at_x', TYPE_UINT32, quantizedOptions(-3000, 3000, 17)), + field('point_at_y', TYPE_UINT32, quantizedOptions(0, 200, 7)), + field('point_at_z', TYPE_UINT32, quantizedOptions(-3000, 3000, 17)), + ], + }, + ], +} + +// -------------------------------------------------------------------------- + +const cases = [ + { proto: quantizationExample, golden: 'QuantizationExample.Bitwise.cs' }, + { proto: pulseServer, golden: 'PulseServer.Bitwise.cs' }, +] + +let failed = 0 +for (const { proto, golden } of cases) { + const result = generateCsharp(proto) + assert.ok(result, `expected output for ${golden}`) + assert.strictEqual(result.name, golden, `output filename for ${golden}`) + try { + assert.strictEqual(result.content, readGolden(golden)) + console.log(`ok - ${golden} matches golden`) + } catch (e) { + failed++ + console.error(`FAIL - ${golden} differs from golden`) + console.error(e.message) + } +} + +if (failed > 0) { + console.error(`\n${failed} golden mismatch(es)`) + process.exit(1) +} +console.log('\nAll bitwise generator golden tests passed.') diff --git a/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs b/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs new file mode 100644 index 00000000..255ba7bf --- /dev/null +++ b/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs @@ -0,0 +1,145 @@ +// +// Generated by protoc-gen-bitwise. DO NOT EDIT. +// Source: decentraland/pulse/pulse_server.proto +// + +using Decentraland.Networking.Bitwise; + +namespace Decentraland.Pulse +{ + public partial class PlayerStateDeltaTier0 + { + private float? _positionX; + /// Float accessor for . Range [0.0f, 16.0f], 8 bits, step ≈ 0.0627451. + public float PositionXQuantized + { + get => _positionX ??= Quantize.Decode(PositionX, 0.0f, 16.0f, 8); + set { _positionX = value; PositionX = Quantize.Encode(value, 0.0f, 16.0f, 8); } + } + + private float? _positionY; + /// Float accessor for . Range [0.0f, 200.0f], 13 bits, step ≈ 0.024417. + public float PositionYQuantized + { + get => _positionY ??= Quantize.Decode(PositionY, 0.0f, 200.0f, 13); + set { _positionY = value; PositionY = Quantize.Encode(value, 0.0f, 200.0f, 13); } + } + + private float? _positionZ; + /// Float accessor for . Range [0.0f, 16.0f], 8 bits, step ≈ 0.0627451. + public float PositionZQuantized + { + get => _positionZ ??= Quantize.Decode(PositionZ, 0.0f, 16.0f, 8); + set { _positionZ = value; PositionZ = Quantize.Encode(value, 0.0f, 16.0f, 8); } + } + + private float? _velocityX; + /// Float accessor for . Range [-50.0f, 50.0f], 8 bits, step ≈ 0.392157. + public float VelocityXQuantized + { + get => _velocityX ??= Quantize.Decode(VelocityX, -50.0f, 50.0f, 8); + set { _velocityX = value; VelocityX = Quantize.Encode(value, -50.0f, 50.0f, 8); } + } + + private float? _velocityY; + /// Float accessor for . Range [-50.0f, 50.0f], 8 bits, step ≈ 0.392157. + public float VelocityYQuantized + { + get => _velocityY ??= Quantize.Decode(VelocityY, -50.0f, 50.0f, 8); + set { _velocityY = value; VelocityY = Quantize.Encode(value, -50.0f, 50.0f, 8); } + } + + private float? _velocityZ; + /// Float accessor for . Range [-50.0f, 50.0f], 8 bits, step ≈ 0.392157. + public float VelocityZQuantized + { + get => _velocityZ ??= Quantize.Decode(VelocityZ, -50.0f, 50.0f, 8); + set { _velocityZ = value; VelocityZ = Quantize.Encode(value, -50.0f, 50.0f, 8); } + } + + private float? _rotationY; + /// Float accessor for . Range [0.0f, 360.0f], 7 bits, step ≈ 2.83465. + public float RotationYQuantized + { + get => _rotationY ??= Quantize.Decode(RotationY, 0.0f, 360.0f, 7); + set { _rotationY = value; RotationY = Quantize.Encode(value, 0.0f, 360.0f, 7); } + } + + private float? _movementBlend; + /// Float accessor for . Range [0.0f, 3.0f], 5 bits, step ≈ 0.0967742. + public float MovementBlendQuantized + { + get => _movementBlend ??= Quantize.Decode(MovementBlend, 0.0f, 3.0f, 5); + set { _movementBlend = value; MovementBlend = Quantize.Encode(value, 0.0f, 3.0f, 5); } + } + + private float? _slideBlend; + /// Float accessor for . Range [0.0f, 1.0f], 4 bits, step ≈ 0.0666667. + public float SlideBlendQuantized + { + get => _slideBlend ??= Quantize.Decode(SlideBlend, 0.0f, 1.0f, 4); + set { _slideBlend = value; SlideBlend = Quantize.Encode(value, 0.0f, 1.0f, 4); } + } + + private float? _headYaw; + /// Float accessor for . Range [0.0f, 360.0f], 7 bits, step ≈ 2.83465. + public float HeadYawQuantized + { + get => _headYaw ??= Quantize.Decode(HeadYaw, 0.0f, 360.0f, 7); + set { _headYaw = value; HeadYaw = Quantize.Encode(value, 0.0f, 360.0f, 7); } + } + + private float? _headPitch; + /// Float accessor for . Range [0.0f, 360.0f], 7 bits, step ≈ 2.83465. + public float HeadPitchQuantized + { + get => _headPitch ??= Quantize.Decode(HeadPitch, 0.0f, 360.0f, 7); + set { _headPitch = value; HeadPitch = Quantize.Encode(value, 0.0f, 360.0f, 7); } + } + + private float? _pointAtX; + /// Float accessor for . Range [-3000.0f, 3000.0f], 17 bits, step ≈ 0.0457767. + public float PointAtXQuantized + { + get => _pointAtX ??= Quantize.Decode(PointAtX, -3000.0f, 3000.0f, 17); + set { _pointAtX = value; PointAtX = Quantize.Encode(value, -3000.0f, 3000.0f, 17); } + } + + private float? _pointAtY; + /// Float accessor for . Range [0.0f, 200.0f], 7 bits, step ≈ 1.5748. + public float PointAtYQuantized + { + get => _pointAtY ??= Quantize.Decode(PointAtY, 0.0f, 200.0f, 7); + set { _pointAtY = value; PointAtY = Quantize.Encode(value, 0.0f, 200.0f, 7); } + } + + private float? _pointAtZ; + /// Float accessor for . Range [-3000.0f, 3000.0f], 17 bits, step ≈ 0.0457767. + public float PointAtZQuantized + { + get => _pointAtZ ??= Quantize.Decode(PointAtZ, -3000.0f, 3000.0f, 17); + set { _pointAtZ = value; PointAtZ = Quantize.Encode(value, -3000.0f, 3000.0f, 17); } + } + + /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. + public void ResetDecodedCache() + { + _positionX = null; + _positionY = null; + _positionZ = null; + _velocityX = null; + _velocityY = null; + _velocityZ = null; + _rotationY = null; + _movementBlend = null; + _slideBlend = null; + _headYaw = null; + _headPitch = null; + _pointAtX = null; + _pointAtY = null; + _pointAtZ = null; + } + } + + +} // namespace Decentraland.Pulse diff --git a/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs b/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs new file mode 100644 index 00000000..c0456936 --- /dev/null +++ b/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs @@ -0,0 +1,134 @@ +// +// Generated by protoc-gen-bitwise. DO NOT EDIT. +// Source: decentraland/common/quantization_example.proto +// + +using Decentraland.Networking.Bitwise; + +namespace Decentraland.Common +{ + public partial class PositionDelta + { + private float? _dx; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. + public float DxQuantized + { + get => _dx ??= Quantize.Decode(Dx, -100.0f, 100.0f, 16); + set { _dx = value; Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + private float? _dy; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. + public float DyQuantized + { + get => _dy ??= Quantize.Decode(Dy, -100.0f, 100.0f, 16); + set { _dy = value; Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + private float? _dz; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. + public float DzQuantized + { + get => _dz ??= Quantize.Decode(Dz, -100.0f, 100.0f, 16); + set { _dz = value; Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. + public void ResetDecodedCache() + { + _dx = null; + _dy = null; + _dz = null; + } + } + + public partial class PlayerInput + { + private float? _moveX; + /// Float accessor for . Range [-1.0f, 1.0f], 8 bits, step ≈ 0.00784314. + public float MoveXQuantized + { + get => _moveX ??= Quantize.Decode(MoveX, -1.0f, 1.0f, 8); + set { _moveX = value; MoveX = Quantize.Encode(value, -1.0f, 1.0f, 8); } + } + + private float? _moveZ; + /// Float accessor for . Range [-1.0f, 1.0f], 8 bits, step ≈ 0.00784314. + public float MoveZQuantized + { + get => _moveZ ??= Quantize.Decode(MoveZ, -1.0f, 1.0f, 8); + set { _moveZ = value; MoveZ = Quantize.Encode(value, -1.0f, 1.0f, 8); } + } + + private float? _yaw; + /// Float accessor for . Range [-180.0f, 180.0f], 12 bits, step ≈ 0.0879121. + public float YawQuantized + { + get => _yaw ??= Quantize.Decode(Yaw, -180.0f, 180.0f, 12); + set { _yaw = value; Yaw = Quantize.Encode(value, -180.0f, 180.0f, 12); } + } + + /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. + public void ResetDecodedCache() + { + _moveX = null; + _moveZ = null; + _yaw = null; + } + } + + public partial class AvatarStateSnapshot + { + private float? _x; + /// Float accessor for . Range [-4096.0f, 4096.0f], 16 bits, step ≈ 0.125002. + public float XQuantized + { + get => _x ??= Quantize.Decode(X, -4096.0f, 4096.0f, 16); + set { _x = value; X = Quantize.Encode(value, -4096.0f, 4096.0f, 16); } + } + + private float? _y; + /// Float accessor for . Range [-256.0f, 256.0f], 14 bits, step ≈ 0.0312519. + public float YQuantized + { + get => _y ??= Quantize.Decode(Y, -256.0f, 256.0f, 14); + set { _y = value; Y = Quantize.Encode(value, -256.0f, 256.0f, 14); } + } + + private float? _z; + /// Float accessor for . Range [-4096.0f, 4096.0f], 16 bits, step ≈ 0.125002. + public float ZQuantized + { + get => _z ??= Quantize.Decode(Z, -4096.0f, 4096.0f, 16); + set { _z = value; Z = Quantize.Encode(value, -4096.0f, 4096.0f, 16); } + } + + private float? _pitch; + /// Float accessor for . Range [-90.0f, 90.0f], 10 bits, step ≈ 0.175953. + public float PitchQuantized + { + get => _pitch ??= Quantize.Decode(Pitch, -90.0f, 90.0f, 10); + set { _pitch = value; Pitch = Quantize.Encode(value, -90.0f, 90.0f, 10); } + } + + private float? _yaw; + /// Float accessor for . Range [-180.0f, 180.0f], 12 bits, step ≈ 0.0879121. + public float YawQuantized + { + get => _yaw ??= Quantize.Decode(Yaw, -180.0f, 180.0f, 12); + set { _yaw = value; Yaw = Quantize.Encode(value, -180.0f, 180.0f, 12); } + } + + /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. + public void ResetDecodedCache() + { + _x = null; + _y = null; + _z = null; + _pitch = null; + _yaw = null; + } + } + + +} // namespace Decentraland.Common diff --git a/protoc-gen-bitwise/wire.js b/protoc-gen-bitwise/wire.js new file mode 100644 index 00000000..e2bd2d57 --- /dev/null +++ b/protoc-gen-bitwise/wire.js @@ -0,0 +1,239 @@ +'use strict' + +/** + * Self-contained protobuf wire-format codec for protoc-gen-bitwise. + * + * The plugin only needs a tiny slice of the descriptor/plugin schemas, so + * rather than pull in a protobuf runtime (which would force every consumer — + * including the sibling Pulse checkout that runs this file directly off disk — + * to `npm install` the protocol repo) we decode the handful of fields we care + * about by walking the wire format directly. This mirrors the original Python + * plugin, which already hand-parsed FieldOptions for the same reason. + * + * Decoded subset: + * CodeGeneratorRequest { file_to_generate = 1; proto_file = 15; } + * FileDescriptorProto { name = 1; package = 2; message_type = 4; } + * DescriptorProto { name = 1; field = 2; } + * FieldDescriptorProto { name = 1; label = 4; type = 5; options = 8 (raw bytes); } + * + * Encoded subset: + * CodeGeneratorResponse { error = 1; supported_features = 2; file = 15; } + * CodeGeneratorResponse.File { name = 1; content = 15; } + * + * Wire types: 0 = varint, 1 = 64-bit, 2 = length-delimited, 5 = 32-bit. + */ + +// --------------------------------------------------------------------------- +// Low-level readers +// --------------------------------------------------------------------------- + +/** Decode a protobuf varint at `pos`. Returns [value, newPos]. */ +function readVarint(buf, pos) { + let result = 0 + let shift = 0 + let byte + do { + byte = buf[pos++] + // Multiplication (not <<) keeps values correct past 32 bits. + result += (byte & 0x7f) * Math.pow(2, shift) + shift += 7 + } while (byte & 0x80) + return [result, pos] +} + +/** Advance `pos` past a field of the given wire type. Returns newPos. */ +function skipField(buf, pos, wireType) { + switch (wireType) { + case 0: // varint + return readVarint(buf, pos)[1] + case 1: // 64-bit + return pos + 8 + case 2: { + // length-delimited + const [length, next] = readVarint(buf, pos) + return next + length + } + case 5: // 32-bit + return pos + 4 + default: + // Groups (3/4) are deprecated and never appear in descriptors. + return pos + } +} + +// --------------------------------------------------------------------------- +// Request decoding +// --------------------------------------------------------------------------- + +function decodeFieldDescriptor(buf) { + const field = { name: '', label: 0, type: 0, optionsRaw: null } + let pos = 0 + while (pos < buf.length) { + let tag + ;[tag, pos] = readVarint(buf, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + field.name = buf.toString('utf8', pos, pos + len) + pos += len + } else if (fieldNum === 4 && wireType === 0) { + ;[field.label, pos] = readVarint(buf, pos) + } else if (fieldNum === 5 && wireType === 0) { + ;[field.type, pos] = readVarint(buf, pos) + } else if (fieldNum === 8 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + // Keep the raw FieldOptions bytes so custom extensions survive (the + // descriptor schema doesn't know about ext 50001/50002). + field.optionsRaw = buf.subarray(pos, pos + len) + pos += len + } else { + pos = skipField(buf, pos, wireType) + } + } + return field +} + +function decodeDescriptor(buf) { + const message = { name: '', field: [] } + let pos = 0 + while (pos < buf.length) { + let tag + ;[tag, pos] = readVarint(buf, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + message.name = buf.toString('utf8', pos, pos + len) + pos += len + } else if (fieldNum === 2 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + message.field.push(decodeFieldDescriptor(buf.subarray(pos, pos + len))) + pos += len + } else { + // nested_type / enum_type / etc. are intentionally ignored — the + // generator only walks top-level messages, matching the Python plugin. + pos = skipField(buf, pos, wireType) + } + } + return message +} + +function decodeFileDescriptor(buf) { + const file = { name: '', package: '', messageType: [] } + let pos = 0 + while (pos < buf.length) { + let tag + ;[tag, pos] = readVarint(buf, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + file.name = buf.toString('utf8', pos, pos + len) + pos += len + } else if (fieldNum === 2 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + file.package = buf.toString('utf8', pos, pos + len) + pos += len + } else if (fieldNum === 4 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + file.messageType.push(decodeDescriptor(buf.subarray(pos, pos + len))) + pos += len + } else { + pos = skipField(buf, pos, wireType) + } + } + return file +} + +/** Decode a serialized CodeGeneratorRequest. */ +function decodeRequest(buf) { + const request = { fileToGenerate: [], protoFile: [] } + let pos = 0 + while (pos < buf.length) { + let tag + ;[tag, pos] = readVarint(buf, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + request.fileToGenerate.push(buf.toString('utf8', pos, pos + len)) + pos += len + } else if (fieldNum === 15 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + request.protoFile.push(decodeFileDescriptor(buf.subarray(pos, pos + len))) + pos += len + } else { + pos = skipField(buf, pos, wireType) + } + } + return request +} + +// --------------------------------------------------------------------------- +// Response encoding +// --------------------------------------------------------------------------- + +/** Encode an unsigned integer as a protobuf varint Buffer. */ +function encodeVarint(value) { + const bytes = [] + let v = value + while (v > 0x7f) { + bytes.push((v & 0x7f) | 0x80) + v = Math.floor(v / 128) + } + bytes.push(v & 0x7f) + return Buffer.from(bytes) +} + +function encodeTag(fieldNum, wireType) { + return encodeVarint((fieldNum << 3) | wireType) +} + +/** Encode a length-delimited (string/bytes) field: tag + length + payload. */ +function encodeLengthDelimited(fieldNum, payload) { + const buf = Buffer.isBuffer(payload) ? payload : Buffer.from(payload, 'utf8') + return Buffer.concat([encodeTag(fieldNum, 2), encodeVarint(buf.length), buf]) +} + +function encodeFile(file) { + return Buffer.concat([ + encodeLengthDelimited(1, file.name), // name = 1 + encodeLengthDelimited(15, file.content), // content = 15 + ]) +} + +/** + * Encode a CodeGeneratorResponse. + * @param {{error?: string|null, supportedFeatures?: number, files?: Array<{name:string,content:string}>}} response + */ +function encodeResponse(response) { + const parts = [] + if (response.error != null) { + parts.push(encodeLengthDelimited(1, response.error)) // error = 1 + } + if (response.supportedFeatures != null) { + parts.push(encodeTag(2, 0)) // supported_features = 2 (varint) + parts.push(encodeVarint(response.supportedFeatures)) + } + for (const file of response.files || []) { + parts.push(encodeLengthDelimited(15, encodeFile(file))) // file = 15 + } + return Buffer.concat(parts) +} + +module.exports = { + readVarint, + skipField, + decodeRequest, + encodeResponse, +} From b147d7dfcaa4d6b0179b1c9ca8157ff0cc9510df Mon Sep 17 00:00:00 2001 From: robtfm <50659922+robtfm@users.noreply.github.com> Date: Fri, 3 Jul 2026 14:14:50 +0100 Subject: [PATCH 3/5] feat: power-law quantized float option; apply to Pulse velocity deltas (#431) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: power-law quantized float option; apply to Pulse velocity deltas Add a new `(decentraland.common.quantized_power)` field option for uint32 fields: an `(bits-1)`-bit linear unorm magnitude `u` (high bits) plus a sign (LSB), decoded as `sign * max * u^pow`. Unlike the linear `quantized` option, zero is exactly representable (`u=0` -> `0`) and `pow > 1` concentrates resolution near zero. Putting the sign in the LSB (`(magnitude << 1) | sign`) makes the varint cost track magnitude rather than direction — a small `|value|` of either sign stays in one varint byte, and a stopped axis canonicalizes to 0 (omitted by proto3). `PlayerStateDeltaTier0.velocity_x/y/z` switch from `quantized{min:-50,max:50,bits:8}` to `quantized_power{max:50,pow:2,bits:8}`. The linear quantizer could not represent an exact 0 over [-50,50] in 8 bits (±0.196 residual), which drove a steady drift on stopped foreign avatars; the power curve fixes that and spends its bits on the low speeds where avatars actually move. Implemented end to end against the Node protoc-gen-bitwise plugin: options.proto message + extension (50003), the options.js parser, generator_csharp.js dispatch, the C# runtime (Quantize.EncodePower/DecodePower), golden-fixture coverage (test helper + regenerated PulseServer/QuantizationExample goldens), plus the README, CLAUDE.md and quantization_example.proto docs. Breaking wire change for velocity_x/y/z: server (pulse), Unity, godot and bevy must regenerate and deploy in lockstep. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: robtfm <50659922+robtfm@users.noreply.github.com> * Quantize PlayerState identically to PlayerStateDeltaTier0 Signed-off-by: Mikhail Agapov * Add "step" const to the quantization gen Signed-off-by: Mikhail Agapov --------- Signed-off-by: robtfm <50659922+robtfm@users.noreply.github.com> Signed-off-by: Mikhail Agapov Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: Mikhail Agapov --- CLAUDE.md | 14 ++++- README.md | 1 + proto/decentraland/common/options.proto | 21 +++++++ .../common/quantization_example.proto | 32 +++++++++++ proto/decentraland/pulse/pulse_server.proto | 10 +++- proto/decentraland/pulse/pulse_shared.proto | 33 +++++++---- protoc-gen-bitwise/generator_csharp.js | 57 +++++++++++++------ protoc-gen-bitwise/options.js | 39 +++++++++++-- protoc-gen-bitwise/runtime/cs/Quantize.cs | 35 ++++++++++++ protoc-gen-bitwise/test/generator.test.js | 24 +++++++- .../test/golden/PulseServer.Bitwise.cs | 18 +++--- .../golden/QuantizationExample.Bitwise.cs | 35 ++++++++++++ 12 files changed, 268 insertions(+), 51 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 130756ee..03a2a374 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,13 +32,23 @@ message QuantizedFloatOptions { uint32 bits = 3; } +// Signed power-law quantizer: an (bits-1)-bit magnitude (high bits) plus a sign +// (LSB), decoded as sign * max * u^pow. Exact zero; pow>1 concentrates resolution +// near zero; sign in the LSB keeps small magnitudes in one varint byte. +message QuantizedPowerFloatOptions { + float max = 1; + float pow = 2; + uint32 bits = 3; +} + message BitPackedOptions { uint32 bits = 1; } extend google.protobuf.FieldOptions { - QuantizedFloatOptions quantized = 50001; - BitPackedOptions bit_packed = 50002; + QuantizedFloatOptions quantized = 50001; + BitPackedOptions bit_packed = 50002; + QuantizedPowerFloatOptions quantized_power = 50003; } ``` diff --git a/README.md b/README.md index b11c3302..be89895f 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,7 @@ message PositionDelta { | Annotation | Target type | Parameters | Effect | |---|---|---|---| | `[(decentraland.common.quantized)]` | `uint32` | `min`, `max`, `bits` | Plugin emits a cached `float {Name}Quantized` accessor | +| `[(decentraland.common.quantized_power)]` | `uint32` | `max`, `pow`, `bits` | Power-law quantizer over `[-max, max]`: `(bits-1)`-bit magnitude (high bits) + sign (LSB), decoded as `sign·max·u^pow`. Exact zero; `pow>1` gives fine resolution near zero, coarse near `±max`; sign in the LSB keeps small magnitudes one varint byte. Cached `float {Name}Quantized` accessor (`Quantize.EncodePower`/`DecodePower`) | | `[(decentraland.common.bit_packed)]` | `uint32` | `bits` | Documents the value range; protobuf handles varint compaction automatically | ### Wire cost at worst-case (all bits set) diff --git a/proto/decentraland/common/options.proto b/proto/decentraland/common/options.proto index 8e911547..8e4a3517 100644 --- a/proto/decentraland/common/options.proto +++ b/proto/decentraland/common/options.proto @@ -14,6 +14,22 @@ message QuantizedFloatOptions { uint32 bits = 3; } +// Options for quantizing a signed float with a power-law curve, stored as a uint32 field. +// The value is split into an (bits-1)-bit linear unorm magnitude `u in [0, 1]` plus a sign, +// and reconstructed as `value = sign * max * u^pow`. Because `u = 0` maps to exactly `0`, zero is +// representable (unlike the linear quantizer, whose codes straddle zero), and `pow > 1` concentrates +// resolution near zero — useful for fields like velocity that need fine detail at low magnitudes and +// only coarse detail near the extremes. The range is symmetric ([-max, max]); there is no `min`. +// The encoded layout puts the magnitude in the high bits and the sign in the LSB +// (`(magnitude << 1) | sign`), so the varint cost tracks magnitude, not direction: a small `|value|` +// of either sign stays in one varint byte. Zero canonicalizes to `0`, so proto3 omits a stopped field. +// The generator emits a float {Name}Quantized accessor, same as the linear option. +message QuantizedPowerFloatOptions { + float max = 1; + float pow = 2; + uint32 bits = 3; +} + // Options for bit-packing an integer field into fewer than 32 bits. message BitPackedOptions { uint32 bits = 1; @@ -27,4 +43,9 @@ extend google.protobuf.FieldOptions { // Apply to uint32 fields to pack into fewer bits. // Example: uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; BitPackedOptions bit_packed = 50002; + + // Apply to uint32 fields to enable power-law quantized float encoding (sign + magnitude curve). + // A field carries either `quantized` or `quantized_power`, not both. + // Example: uint32 velocity_x = 9 [(decentraland.common.quantized_power) = { max: 50.0, pow: 2.0, bits: 8 }]; + QuantizedPowerFloatOptions quantized_power = 50003; } diff --git a/proto/decentraland/common/quantization_example.proto b/proto/decentraland/common/quantization_example.proto index a82e430c..e8092f04 100644 --- a/proto/decentraland/common/quantization_example.proto +++ b/proto/decentraland/common/quantization_example.proto @@ -15,6 +15,13 @@ import "decentraland/common/options.proto"; // → protobuf encodes the uint32 as a varint: values ≤ 2^14-1 cost 2 bytes, // values ≤ 2^21-1 cost 3 bytes; the generated partial class adds a // float {Name}Quantized accessor that encodes/decodes transparently. +// [(decentraland.common.quantized_power) = { max: F, pow: F, bits: N }] +// → stores a signed float over [-max, max] as an (N-1)-bit linear unorm +// magnitude (high bits) plus a sign (LSB), reconstructed as sign·max·u^pow. +// Zero is exact, and pow>1 buys fine resolution near zero at the cost of +// coarse near ±max. Putting the sign in the LSB keeps a small |value| of +// either direction in one varint byte. Same generated float {Name}Quantized +// accessor as the linear option. // [(decentraland.common.bit_packed) = { bits: N }] // → documents that this uint32 uses at most N bits; protobuf encodes it // as a varint (same savings, no generated accessor needed). @@ -82,6 +89,31 @@ message PlayerInput { uint32 sequence = 5 [(decentraland.common.bit_packed) = { bits: 12 }]; } +// --------------------------------------------------------------------------- +// VelocityState — per-axis velocity, demonstrating the power-law quantizer. +// +// Velocity needs an exact zero (a stopped avatar must read 0, not a quantization +// residual, or it drifts) and fine resolution at locomotion speeds, but only +// coarse resolution near the ±50 m/s extremes. The linear quantizer can give +// none of these from 8 bits: its codes straddle zero (±0.196) and spend uniform +// resolution on speeds that never occur. quantized_power solves all three. +// +// Field | Type | Range | pow | bits | Wire worst-case +// -------|--------|------------|-----|------|----------------------- +// vx | uint32 | [-50, 50] | 2 | 8 | tag 1B + varint 1–2B = 2–3B +// vy | uint32 | [-50, 50] | 2 | 8 | tag 1B + varint 1–2B = 2–3B +// vz | uint32 | [-50, 50] | 2 | 8 | tag 1B + varint 1–2B = 2–3B +// --------------------------------------------------------------------------- +// Step: ≈ 0.003 m/s near zero, ≈ 0.78 m/s near ±50 (vs. 0.392 uniform linear) +// Wire: sign in the LSB, so |v| ≲ 12.5 m/s (magnitude code ≤ 63) costs a single +// varint byte regardless of direction; only faster axes spill to 2 bytes. +// --------------------------------------------------------------------------- +message VelocityState { + uint32 vx = 1 [(decentraland.common.quantized_power) = { max: 50.0, pow: 2.0, bits: 8 }]; + uint32 vy = 2 [(decentraland.common.quantized_power) = { max: 50.0, pow: 2.0, bits: 8 }]; + uint32 vz = 3 [(decentraland.common.quantized_power) = { max: 50.0, pow: 2.0, bits: 8 }]; +} + // Bitmask values for PlayerInput.buttons. enum ButtonFlags { BF_NONE = 0; diff --git a/proto/decentraland/pulse/pulse_server.proto b/proto/decentraland/pulse/pulse_server.proto index 8311e737..1a9ca80e 100644 --- a/proto/decentraland/pulse/pulse_server.proto +++ b/proto/decentraland/pulse/pulse_server.proto @@ -37,9 +37,13 @@ message PlayerStateDeltaTier0 { // Z position inside the parcel optional uint32 position_z = 8 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; - optional uint32 velocity_x = 9 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; - optional uint32 velocity_y = 10 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; - optional uint32 velocity_z = 11 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; + // Power-law quantized: a sign bit + 7-bit magnitude curve over [-50, 50]. Zero is exactly + // representable (a stopped peer reports an exact 0 instead of the linear quantizer's ±0.196 + // residual that drove foreign-avatar drift), and pow=2 concentrates resolution at the low speeds + // where avatars actually move, leaving only the visually-irrelevant extremes coarse. + optional uint32 velocity_x = 9 [(decentraland.common.quantized_power) = { max: 50, pow: 2, bits: 8 }]; + optional uint32 velocity_y = 10 [(decentraland.common.quantized_power) = { max: 50, pow: 2, bits: 8 }]; + optional uint32 velocity_z = 11 [(decentraland.common.quantized_power) = { max: 50, pow: 2, bits: 8 }]; optional uint32 rotation_y = 12 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; optional uint32 movement_blend = 13 [(decentraland.common.quantized) = { min: 0, max: 3, bits: 5 }]; diff --git a/proto/decentraland/pulse/pulse_shared.proto b/proto/decentraland/pulse/pulse_shared.proto index 6f4fba7c..cc027f3c 100644 --- a/proto/decentraland/pulse/pulse_shared.proto +++ b/proto/decentraland/pulse/pulse_shared.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package decentraland.pulse; -import "decentraland/common/vectors.proto"; +import "decentraland/common/options.proto"; enum PlayerAnimationFlags { NONE = 0; @@ -23,26 +23,35 @@ enum GlideState { CLOSING_PROP = 3; } +// Quantized identically to PlayerStateDeltaTier0 so a full state and a stream of deltas land on the same grid. message PlayerState { int32 parcel_index = 1; - decentraland.common.Vector3 position = 2; - decentraland.common.Vector3 velocity = 3; + // Position inside the parcel (X/Z) and world altitude (Y). + uint32 position_x = 2 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + uint32 position_y = 3 [(decentraland.common.quantized) = { min: 0, max: 200, bits: 13 }]; + uint32 position_z = 4 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; - float rotation_y = 4; + uint32 velocity_x = 5 [(decentraland.common.quantized_power) = { max: 50, pow: 2, bits: 8 }]; + uint32 velocity_y = 6 [(decentraland.common.quantized_power) = { max: 50, pow: 2, bits: 8 }]; + uint32 velocity_z = 7 [(decentraland.common.quantized_power) = { max: 50, pow: 2, bits: 8 }]; - float movement_blend = 5; - float slide_blend = 6; + uint32 rotation_y = 8 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; - optional float head_yaw = 7; - optional float head_pitch = 8; + uint32 movement_blend = 9 [(decentraland.common.quantized) = { min: 0, max: 3, bits: 5 }]; + uint32 slide_blend = 10 [(decentraland.common.quantized) = { min: 0, max: 1, bits: 4 }]; - uint32 state_flags = 9; - GlideState glide_state = 10; + optional uint32 head_yaw = 11 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + optional uint32 head_pitch = 12 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; - int32 jump_count = 11; + uint32 state_flags = 13; + GlideState glide_state = 14; + + int32 jump_count = 15; // Absolute world hit position the player is pointing at. // Only meaningful when POINTING_AT is set in `state_flags`. - optional decentraland.common.Vector3 point_at = 12; + optional uint32 point_at_x = 16 [(decentraland.common.quantized) = { min: -3000.0, max: 3000.0, bits: 17 }]; + optional uint32 point_at_y = 17 [(decentraland.common.quantized) = { min: 0.0, max: 200.0, bits: 7 }]; + optional uint32 point_at_z = 18 [(decentraland.common.quantized) = { min: -3000.0, max: 3000.0, bits: 17 }]; } diff --git a/protoc-gen-bitwise/generator_csharp.js b/protoc-gen-bitwise/generator_csharp.js index dcdb95a0..93b23c26 100644 --- a/protoc-gen-bitwise/generator_csharp.js +++ b/protoc-gen-bitwise/generator_csharp.js @@ -118,18 +118,39 @@ function generateMessage(msgProto, indent) { // Only uint32 fields are candidates for quantized accessors. if (field.type !== TYPE_UINT32) continue - const { quantized } = getFieldOptions(field.optionsRaw) - if (quantized === null) continue - - const step = (quantized.max - quantized.min) / ((1 << quantized.bits) - 1) - - props.push({ - propName: snakeToPascal(field.name), - mn: formatFloat(quantized.min), - mx: formatFloat(quantized.max), - bits: quantized.bits, - step, - }) + const { quantized, quantizedPower } = getFieldOptions(field.optionsRaw) + const propName = snakeToPascal(field.name) + + let doc, getExpr, setExpr, step + if (quantized !== null) { + const mn = formatFloat(quantized.min) + const mx = formatFloat(quantized.max) + const bits = quantized.bits + // Uniform quantizer — the step is constant across the whole range. + step = (quantized.max - quantized.min) / ((1 << bits) - 1) + doc = `Range [${mn}, ${mx}], ${bits} bits, step ${formatStep(step)}.` + getExpr = `Quantize.Decode(${propName}, ${mn}, ${mx}, ${bits})` + setExpr = `Quantize.Encode(value, ${mn}, ${mx}, ${bits})` + } else if (quantizedPower !== null) { + const mx = formatFloat(quantizedPower.max) + const pw = formatFloat(quantizedPower.pow) + const bits = quantizedPower.bits + // Power curve is non-uniform: the finest step sits next to zero (first magnitude code), + // the COARSEST at the top of the range. The coarsest step upper-bounds the error for any + // value, so that's what the exposed {Name}QuantizedStep const carries (safe as a tolerance). + const magSteps = (1 << (bits - 1)) - 1 + const nearZeroStep = quantizedPower.max * Math.pow(1 / magSteps, quantizedPower.pow) + step = quantizedPower.max * (1 - Math.pow((magSteps - 1) / magSteps, quantizedPower.pow)) + doc = + `Range [-${mx}, ${mx}], power ${pw}, ${bits} bits ` + + `(sign + ${bits - 1}-bit magnitude), near-zero step ${formatStep(nearZeroStep)}.` + getExpr = `Quantize.DecodePower(${propName}, ${mx}, ${pw}, ${bits})` + setExpr = `Quantize.EncodePower(value, ${mx}, ${pw}, ${bits})` + } else { + continue + } + + props.push({ propName, doc, getExpr, setExpr, step }) } if (props.length === 0) return null @@ -139,17 +160,17 @@ function generateMessage(msgProto, indent) { lines.push(`public partial class ${msgProto.name}`) lines.push('{') - for (const { propName, mn, mx, bits, step } of props) { + for (const { propName, doc, getExpr, setExpr, step } of props) { const backing = '_' + propName[0].toLowerCase() + propName.slice(1) backings.push(backing) lines.push(`${i}private float? ${backing};`) - lines.push( - `${i}/// Float accessor for . Range [${mn}, ${mx}], ${bits} bits, step ${formatStep(step)}.`, - ) + lines.push(`${i}/// Coarsest quantization step of . Safe as an equality tolerance.`) + lines.push(`${i}public const float ${propName}QuantizedStep = ${formatFloat(step)};`) + lines.push(`${i}/// Float accessor for . ${doc}`) lines.push(`${i}public float ${propName}Quantized`) lines.push(`${i}{`) - lines.push(`${i}${i}get => ${backing} ??= Quantize.Decode(${propName}, ${mn}, ${mx}, ${bits});`) - lines.push(`${i}${i}set { ${backing} = value; ${propName} = Quantize.Encode(value, ${mn}, ${mx}, ${bits}); }`) + lines.push(`${i}${i}get => ${backing} ??= ${getExpr};`) + lines.push(`${i}${i}set { ${backing} = value; ${propName} = ${setExpr}; }`) lines.push(`${i}}`) lines.push('') } diff --git a/protoc-gen-bitwise/options.js b/protoc-gen-bitwise/options.js index 150ece0b..e4a788cc 100644 --- a/protoc-gen-bitwise/options.js +++ b/protoc-gen-bitwise/options.js @@ -4,7 +4,7 @@ * Parser for the custom bitwise field options defined in options.proto. * * The descriptor decoder hands us the raw serialized FieldOptions bytes (it - * declares `options` as opaque bytes). We walk those bytes looking for the two + * declares `options` as opaque bytes). We walk those bytes looking for the * custom extension field numbers — protobuf preserves unknown/unregistered * extension bytes, so they are always present even though no runtime here knows * the extension schema. This mirrors the original options_pb2.py. @@ -18,6 +18,7 @@ const { readVarint, skipField } = require('./wire') // Extension field numbers as defined in options.proto. const QUANTIZED_FIELD_NUMBER = 50001 const BIT_PACKED_FIELD_NUMBER = 50002 +const QUANTIZED_POWER_FIELD_NUMBER = 50003 /** Parse a serialized QuantizedFloatOptions message: { min, max, bits }. */ function parseQuantized(data) { @@ -46,6 +47,33 @@ function parseQuantized(data) { return opts } +/** Parse a serialized QuantizedPowerFloatOptions message: { max, pow, bits }. */ +function parseQuantizedPower(data) { + const opts = { max: 0.0, pow: 0.0, bits: 0 } + let pos = 0 + while (pos < data.length) { + let tag + ;[tag, pos] = readVarint(data, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 5) { + // max (float) + opts.max = data.readFloatLE(pos) + pos += 4 + } else if (fieldNum === 2 && wireType === 5) { + // pow (float) + opts.pow = data.readFloatLE(pos) + pos += 4 + } else if (fieldNum === 3 && wireType === 0) { + // bits (uint32) + ;[opts.bits, pos] = readVarint(data, pos) + } else { + pos = skipField(data, pos, wireType) + } + } + return opts +} + /** Parse a serialized BitPackedOptions message: { bits }. */ function parseBitPacked(data) { const opts = { bits: 0 } @@ -69,15 +97,16 @@ function parseBitPacked(data) { * Extract custom bitwise options from raw FieldOptions bytes. * * @param {Buffer|null} optionsRaw serialized FieldOptions, or null when unset. - * @returns {{quantized: object|null, bitPacked: object|null}} + * @returns {{quantized: object|null, bitPacked: object|null, quantizedPower: object|null}} */ function getFieldOptions(optionsRaw) { if (!optionsRaw || optionsRaw.length === 0) { - return { quantized: null, bitPacked: null } + return { quantized: null, bitPacked: null, quantizedPower: null } } let quantized = null let bitPacked = null + let quantizedPower = null let pos = 0 while (pos < optionsRaw.length) { @@ -95,6 +124,8 @@ function getFieldOptions(optionsRaw) { quantized = parseQuantized(valueBytes) } else if (fieldNum === BIT_PACKED_FIELD_NUMBER) { bitPacked = parseBitPacked(valueBytes) + } else if (fieldNum === QUANTIZED_POWER_FIELD_NUMBER) { + quantizedPower = parseQuantizedPower(valueBytes) } // else: unknown length-delimited field — already consumed. } else { @@ -102,7 +133,7 @@ function getFieldOptions(optionsRaw) { } } - return { quantized, bitPacked } + return { quantized, bitPacked, quantizedPower } } module.exports = { getFieldOptions } diff --git a/protoc-gen-bitwise/runtime/cs/Quantize.cs b/protoc-gen-bitwise/runtime/cs/Quantize.cs index 22c347fe..80323615 100644 --- a/protoc-gen-bitwise/runtime/cs/Quantize.cs +++ b/protoc-gen-bitwise/runtime/cs/Quantize.cs @@ -31,5 +31,40 @@ public static float Decode(uint encoded, float min, float max, int bits) var steps = (1u << bits) - 1; return (float)encoded / steps * (max - min) + min; } + + /// + /// Encodes with a power-law curve into an + /// ( - 1)-bit linear unorm magnitude plus a sign bit. The magnitude is + /// (|value| / max)^(1/pow), so the inverse reconstructs + /// sign * max * u^pow. Zero is representable exactly; > 1 + /// concentrates resolution near zero. Magnitudes outside [0, ] are + /// clamped. + /// + /// The encoded layout puts the magnitude in the high bits and the sign in the LSB + /// ((magnitude << 1) | sign). This keeps the varint cost a function of + /// magnitude rather than direction — a small |value| of either sign stays in a + /// single varint byte. Zero canonicalizes to 0 (a zero magnitude never sets the + /// sign bit), so proto3 still omits a stopped field entirely. + /// + /// + public static uint EncodePower(float value, float max, float pow, int bits) + { + var magnitudeSteps = (1u << (bits - 1)) - 1; + var t = Math.Clamp(MathF.Abs(value) / max, 0f, 1f); + var u = MathF.Pow(t, 1f / pow); + var magnitude = (uint)MathF.Round(u * magnitudeSteps); + var sign = value < 0f && magnitude != 0u ? 1u : 0u; + return (magnitude << 1) | sign; + } + + /// Decodes a power-law quantized back to a float (inverse of + /// ). + public static float DecodePower(uint encoded, float max, float pow, int bits) + { + var magnitudeSteps = (1u << (bits - 1)) - 1; + var u = (float)(encoded >> 1) / magnitudeSteps; + var magnitude = max * MathF.Pow(u, pow); + return (encoded & 1u) != 0 ? -magnitude : magnitude; + } } } diff --git a/protoc-gen-bitwise/test/generator.test.js b/protoc-gen-bitwise/test/generator.test.js index 95512154..563aaca5 100644 --- a/protoc-gen-bitwise/test/generator.test.js +++ b/protoc-gen-bitwise/test/generator.test.js @@ -68,6 +68,16 @@ function bitPackedOptions(bits) { return lengthDelimited(50002, inner) } +// FieldOptions bytes carrying ext 50003 (quantized_power) = { max, pow, bits } +function quantizedPowerOptions(max, pow, bits) { + const inner = Buffer.concat([ + tag(1, 5), floatLE(max), + tag(2, 5), floatLE(pow), + tag(3, 0), encodeVarint(bits), + ]) + return lengthDelimited(50003, inner) +} + function field(name, type, optionsRaw) { return { name, label: LABEL_OPTIONAL, type, optionsRaw: optionsRaw || null } } @@ -104,6 +114,14 @@ const quantizationExample = { field('sequence', TYPE_UINT32, bitPackedOptions(12)), ], }, + { + name: 'VelocityState', + field: [ + field('vx', TYPE_UINT32, quantizedPowerOptions(50, 2, 8)), + field('vy', TYPE_UINT32, quantizedPowerOptions(50, 2, 8)), + field('vz', TYPE_UINT32, quantizedPowerOptions(50, 2, 8)), + ], + }, { name: 'AvatarStateSnapshot', field: [ @@ -135,9 +153,9 @@ const pulseServer = { field('position_x', TYPE_UINT32, quantizedOptions(0, 16, 8)), field('position_y', TYPE_UINT32, quantizedOptions(0, 200, 13)), field('position_z', TYPE_UINT32, quantizedOptions(0, 16, 8)), - field('velocity_x', TYPE_UINT32, quantizedOptions(-50, 50, 8)), - field('velocity_y', TYPE_UINT32, quantizedOptions(-50, 50, 8)), - field('velocity_z', TYPE_UINT32, quantizedOptions(-50, 50, 8)), + field('velocity_x', TYPE_UINT32, quantizedPowerOptions(50, 2, 8)), + field('velocity_y', TYPE_UINT32, quantizedPowerOptions(50, 2, 8)), + field('velocity_z', TYPE_UINT32, quantizedPowerOptions(50, 2, 8)), field('rotation_y', TYPE_UINT32, quantizedOptions(0, 360, 7)), field('movement_blend', TYPE_UINT32, quantizedOptions(0, 3, 5)), field('slide_blend', TYPE_UINT32, quantizedOptions(0, 1, 4)), diff --git a/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs b/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs index 255ba7bf..f121577e 100644 --- a/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs +++ b/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs @@ -34,27 +34,27 @@ public float PositionZQuantized } private float? _velocityX; - /// Float accessor for . Range [-50.0f, 50.0f], 8 bits, step ≈ 0.392157. + /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. public float VelocityXQuantized { - get => _velocityX ??= Quantize.Decode(VelocityX, -50.0f, 50.0f, 8); - set { _velocityX = value; VelocityX = Quantize.Encode(value, -50.0f, 50.0f, 8); } + get => _velocityX ??= Quantize.DecodePower(VelocityX, 50.0f, 2.0f, 8); + set { _velocityX = value; VelocityX = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } } private float? _velocityY; - /// Float accessor for . Range [-50.0f, 50.0f], 8 bits, step ≈ 0.392157. + /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. public float VelocityYQuantized { - get => _velocityY ??= Quantize.Decode(VelocityY, -50.0f, 50.0f, 8); - set { _velocityY = value; VelocityY = Quantize.Encode(value, -50.0f, 50.0f, 8); } + get => _velocityY ??= Quantize.DecodePower(VelocityY, 50.0f, 2.0f, 8); + set { _velocityY = value; VelocityY = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } } private float? _velocityZ; - /// Float accessor for . Range [-50.0f, 50.0f], 8 bits, step ≈ 0.392157. + /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. public float VelocityZQuantized { - get => _velocityZ ??= Quantize.Decode(VelocityZ, -50.0f, 50.0f, 8); - set { _velocityZ = value; VelocityZ = Quantize.Encode(value, -50.0f, 50.0f, 8); } + get => _velocityZ ??= Quantize.DecodePower(VelocityZ, 50.0f, 2.0f, 8); + set { _velocityZ = value; VelocityZ = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } } private float? _rotationY; diff --git a/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs b/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs index c0456936..596301b6 100644 --- a/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs +++ b/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs @@ -77,6 +77,41 @@ public void ResetDecodedCache() } } + public partial class VelocityState + { + private float? _vx; + /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. + public float VxQuantized + { + get => _vx ??= Quantize.DecodePower(Vx, 50.0f, 2.0f, 8); + set { _vx = value; Vx = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } + } + + private float? _vy; + /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. + public float VyQuantized + { + get => _vy ??= Quantize.DecodePower(Vy, 50.0f, 2.0f, 8); + set { _vy = value; Vy = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } + } + + private float? _vz; + /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. + public float VzQuantized + { + get => _vz ??= Quantize.DecodePower(Vz, 50.0f, 2.0f, 8); + set { _vz = value; Vz = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } + } + + /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. + public void ResetDecodedCache() + { + _vx = null; + _vy = null; + _vz = null; + } + } + public partial class AvatarStateSnapshot { private float? _x; From c2e8777b472b97f8f85fea5a1568ec5655903697 Mon Sep 17 00:00:00 2001 From: Mikhail Agapov <118179774+mikhail-dcl@users.noreply.github.com> Date: Mon, 6 Jul 2026 19:12:53 +0300 Subject: [PATCH 4/5] fix: additional quantization improvements (#435) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Quantize TeleportRequest * Add Quantized Values Validation Signed-off-by: Mikhail Agapov * Import "decentraland/common/vectors.proto" is unused Signed-off-by: Mikhail Agapov * protoc-gen-bitwise: add range check, drop decode cache Emit AreQuantizedFieldsInRange() (per-field integer bounds check, used by the server to reject out-of-range client codes) and remove the per-field decoded- float cache — decode on get, encode on set. Regenerate goldens; add UPDATE_GOLDEN mode to the golden test so future generator changes are one command. Co-Authored-By: Claude Fable 5 --------- Signed-off-by: Mikhail Agapov Co-authored-by: Claude Fable 5 --- README.md | 28 +-- proto/decentraland/pulse/pulse_client.proto | 8 +- protoc-gen-bitwise/generator_csharp.js | 46 +++-- protoc-gen-bitwise/test/generator.test.js | 16 ++ .../test/golden/PulseServer.Bitwise.cs | 138 +++++++------ .../golden/QuantizationExample.Bitwise.cs | 190 ++++++++++-------- 6 files changed, 245 insertions(+), 181 deletions(-) diff --git a/README.md b/README.md index be89895f..a968d863 100644 --- a/README.md +++ b/README.md @@ -213,12 +213,9 @@ SendOnChannel1(bytes); // --- Receive and read --- var received = PositionDelta.Parser.ParseFrom(receivedBytes); -float x = received.DxQuantized; // decoded on first access, cached thereafter +float x = received.DxQuantized; // decoded from the stored uint32 on each access float y = received.DyQuantized; float z = received.DzQuantized; - -// If raw uint32 fields are mutated directly after construction, invalidate the cache: -received.ResetDecodedCache(); ``` ## Generated file example @@ -237,33 +234,22 @@ namespace Decentraland.Kernel.Comms.V3 { public partial class PositionDelta { - private float? _dx; public float DxQuantized { - get => _dx ??= Quantize.Decode(Dx, -100.0f, 100.0f, 16); - set { _dx = value; Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); } + get => Quantize.Decode(Dx, -100.0f, 100.0f, 16); + set => Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); } - private float? _dy; public float DyQuantized { - get => _dy ??= Quantize.Decode(Dy, -100.0f, 100.0f, 16); - set { _dy = value; Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); } + get => Quantize.Decode(Dy, -100.0f, 100.0f, 16); + set => Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); } - private float? _dz; public float DzQuantized { - get => _dz ??= Quantize.Decode(Dz, -100.0f, 100.0f, 16); - set { _dz = value; Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); } - } - - /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. - public void ResetDecodedCache() - { - _dx = null; - _dy = null; - _dz = null; + get => Quantize.Decode(Dz, -100.0f, 100.0f, 16); + set => Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); } } diff --git a/proto/decentraland/pulse/pulse_client.proto b/proto/decentraland/pulse/pulse_client.proto index 6036ec53..09760dab 100644 --- a/proto/decentraland/pulse/pulse_client.proto +++ b/proto/decentraland/pulse/pulse_client.proto @@ -2,8 +2,8 @@ syntax = "proto3"; package decentraland.pulse; -import "decentraland/common/vectors.proto"; import "decentraland/pulse/pulse_shared.proto"; +import "decentraland/common/options.proto"; message HandshakeRequest { bytes auth_chain = 1; @@ -59,9 +59,11 @@ message EmoteStop { // other. Must be the first gameplay message after handshake. Same-realm re-teleports are valid. message TeleportRequest { int32 parcel_index = 1; - decentraland.common.Vector3 position = 2; + uint32 position_x = 2 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + uint32 position_y = 3 [(decentraland.common.quantized) = { min: 0, max: 200, bits: 13 }]; + uint32 position_z = 4 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; // Non-empty realm identifier. Rejected if empty. - string realm = 3; + string realm = 5; } message ClientMessage { diff --git a/protoc-gen-bitwise/generator_csharp.js b/protoc-gen-bitwise/generator_csharp.js index 93b23c26..4cb3a2ee 100644 --- a/protoc-gen-bitwise/generator_csharp.js +++ b/protoc-gen-bitwise/generator_csharp.js @@ -121,11 +121,11 @@ function generateMessage(msgProto, indent) { const { quantized, quantizedPower } = getFieldOptions(field.optionsRaw) const propName = snakeToPascal(field.name) - let doc, getExpr, setExpr, step + let doc, getExpr, setExpr, step, bits if (quantized !== null) { const mn = formatFloat(quantized.min) const mx = formatFloat(quantized.max) - const bits = quantized.bits + bits = quantized.bits // Uniform quantizer — the step is constant across the whole range. step = (quantized.max - quantized.min) / ((1 << bits) - 1) doc = `Range [${mn}, ${mx}], ${bits} bits, step ${formatStep(step)}.` @@ -134,7 +134,7 @@ function generateMessage(msgProto, indent) { } else if (quantizedPower !== null) { const mx = formatFloat(quantizedPower.max) const pw = formatFloat(quantizedPower.pow) - const bits = quantizedPower.bits + bits = quantizedPower.bits // Power curve is non-uniform: the finest step sits next to zero (first magnitude code), // the COARSEST at the top of the range. The coarsest step upper-bounds the error for any // value, so that's what the exposed {Name}QuantizedStep const carries (safe as a tolerance). @@ -150,38 +150,50 @@ function generateMessage(msgProto, indent) { continue } - props.push({ propName, doc, getExpr, setExpr, step }) + // Highest code the encoder can emit for this field. Both the linear quantizer (top code + // `2^bits - 1`) and the power quantizer (`(magnitude << 1) | sign` with an `bits-1`-bit + // magnitude, so top code `((2^(bits-1)-1) << 1) | 1 == 2^bits - 1`) share this bound. + const maxCode = 2 ** bits - 1 + + props.push({ propName, doc, getExpr, setExpr, step, maxCode }) } if (props.length === 0) return null - const backings = [] const lines = [] lines.push(`public partial class ${msgProto.name}`) lines.push('{') + // Decode on every get, encode on every set — no backing cache. The raw uint32 property is the + // single source of truth, so the float accessor can never disagree with the code on the wire + // (get-after-set returns the on-grid value a receiver decodes) and there is no stale-cache + // hazard when the raw field is mutated directly. Decode is a multiply-add; only power fields + // pay a MathF.Pow. for (const { propName, doc, getExpr, setExpr, step } of props) { - const backing = '_' + propName[0].toLowerCase() + propName.slice(1) - backings.push(backing) - lines.push(`${i}private float? ${backing};`) lines.push(`${i}/// Coarsest quantization step of . Safe as an equality tolerance.`) lines.push(`${i}public const float ${propName}QuantizedStep = ${formatFloat(step)};`) lines.push(`${i}/// Float accessor for . ${doc}`) lines.push(`${i}public float ${propName}Quantized`) lines.push(`${i}{`) - lines.push(`${i}${i}get => ${backing} ??= ${getExpr};`) - lines.push(`${i}${i}set { ${backing} = value; ${propName} = ${setExpr}; }`) + lines.push(`${i}${i}get => ${getExpr};`) + lines.push(`${i}${i}set => ${propName} = ${setExpr};`) lines.push(`${i}}`) lines.push('') } - lines.push(`${i}/// Clears all cached decoded values. Call after mutating raw uint32 fields directly.`) - lines.push(`${i}public void ResetDecodedCache()`) - lines.push(`${i}{`) - for (const backing of backings) { - lines.push(`${i}${i}${backing} = null;`) - } - lines.push(`${i}}`) + lines.push(`${i}/// `) + lines.push(`${i}/// True when every quantized field holds a wire code within its declared bit width`) + lines.push(`${i}/// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger`) + lines.push(`${i}/// value is a malformed/hostile message: decoding it would land far outside the field's`) + lines.push(`${i}/// [min, max] and, since the server relays raw codes verbatim, poison every observer.`) + lines.push(`${i}/// Reject before storing or relaying. Pure integer comparison — no decode.`) + lines.push(`${i}/// `) + lines.push(`${i}public bool AreQuantizedFieldsInRange() =>`) + props.forEach(({ propName, maxCode }, idx) => { + const prefix = idx === 0 ? '' : '&& ' + const suffix = idx === props.length - 1 ? ';' : '' + lines.push(`${i}${i}${prefix}${propName} <= ${maxCode}u${suffix}`) + }) lines.push('}') return lines diff --git a/protoc-gen-bitwise/test/generator.test.js b/protoc-gen-bitwise/test/generator.test.js index 563aaca5..dc20b39e 100644 --- a/protoc-gen-bitwise/test/generator.test.js +++ b/protoc-gen-bitwise/test/generator.test.js @@ -176,11 +176,22 @@ const cases = [ { proto: pulseServer, golden: 'PulseServer.Bitwise.cs' }, ] +// Set UPDATE_GOLDEN=1 to rewrite the golden files from the current generator output +// (after an intentional generator change), then re-run without it to verify. +const update = process.env.UPDATE_GOLDEN === '1' + let failed = 0 for (const { proto, golden } of cases) { const result = generateCsharp(proto) assert.ok(result, `expected output for ${golden}`) assert.strictEqual(result.name, golden, `output filename for ${golden}`) + + if (update) { + fs.writeFileSync(path.join(__dirname, 'golden', golden), result.content) + console.log(`updated - ${golden}`) + continue + } + try { assert.strictEqual(result.content, readGolden(golden)) console.log(`ok - ${golden} matches golden`) @@ -191,6 +202,11 @@ for (const { proto, golden } of cases) { } } +if (update) { + console.log('\nGolden files updated. Re-run `npm run gen:test` to verify.') + process.exit(0) +} + if (failed > 0) { console.error(`\n${failed} golden mismatch(es)`) process.exit(1) diff --git a/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs b/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs index f121577e..ea3c3d1a 100644 --- a/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs +++ b/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs @@ -9,136 +9,154 @@ namespace Decentraland.Pulse { public partial class PlayerStateDeltaTier0 { - private float? _positionX; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PositionXQuantizedStep = 0.062745098f; /// Float accessor for . Range [0.0f, 16.0f], 8 bits, step ≈ 0.0627451. public float PositionXQuantized { - get => _positionX ??= Quantize.Decode(PositionX, 0.0f, 16.0f, 8); - set { _positionX = value; PositionX = Quantize.Encode(value, 0.0f, 16.0f, 8); } + get => Quantize.Decode(PositionX, 0.0f, 16.0f, 8); + set => PositionX = Quantize.Encode(value, 0.0f, 16.0f, 8); } - private float? _positionY; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PositionYQuantizedStep = 0.024417043f; /// Float accessor for . Range [0.0f, 200.0f], 13 bits, step ≈ 0.024417. public float PositionYQuantized { - get => _positionY ??= Quantize.Decode(PositionY, 0.0f, 200.0f, 13); - set { _positionY = value; PositionY = Quantize.Encode(value, 0.0f, 200.0f, 13); } + get => Quantize.Decode(PositionY, 0.0f, 200.0f, 13); + set => PositionY = Quantize.Encode(value, 0.0f, 200.0f, 13); } - private float? _positionZ; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PositionZQuantizedStep = 0.062745098f; /// Float accessor for . Range [0.0f, 16.0f], 8 bits, step ≈ 0.0627451. public float PositionZQuantized { - get => _positionZ ??= Quantize.Decode(PositionZ, 0.0f, 16.0f, 8); - set { _positionZ = value; PositionZ = Quantize.Encode(value, 0.0f, 16.0f, 8); } + get => Quantize.Decode(PositionZ, 0.0f, 16.0f, 8); + set => PositionZ = Quantize.Encode(value, 0.0f, 16.0f, 8); } - private float? _velocityX; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float VelocityXQuantizedStep = 0.78430157f; /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. public float VelocityXQuantized { - get => _velocityX ??= Quantize.DecodePower(VelocityX, 50.0f, 2.0f, 8); - set { _velocityX = value; VelocityX = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } + get => Quantize.DecodePower(VelocityX, 50.0f, 2.0f, 8); + set => VelocityX = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } - private float? _velocityY; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float VelocityYQuantizedStep = 0.78430157f; /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. public float VelocityYQuantized { - get => _velocityY ??= Quantize.DecodePower(VelocityY, 50.0f, 2.0f, 8); - set { _velocityY = value; VelocityY = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } + get => Quantize.DecodePower(VelocityY, 50.0f, 2.0f, 8); + set => VelocityY = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } - private float? _velocityZ; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float VelocityZQuantizedStep = 0.78430157f; /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. public float VelocityZQuantized { - get => _velocityZ ??= Quantize.DecodePower(VelocityZ, 50.0f, 2.0f, 8); - set { _velocityZ = value; VelocityZ = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } + get => Quantize.DecodePower(VelocityZ, 50.0f, 2.0f, 8); + set => VelocityZ = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } - private float? _rotationY; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float RotationYQuantizedStep = 2.8346457f; /// Float accessor for . Range [0.0f, 360.0f], 7 bits, step ≈ 2.83465. public float RotationYQuantized { - get => _rotationY ??= Quantize.Decode(RotationY, 0.0f, 360.0f, 7); - set { _rotationY = value; RotationY = Quantize.Encode(value, 0.0f, 360.0f, 7); } + get => Quantize.Decode(RotationY, 0.0f, 360.0f, 7); + set => RotationY = Quantize.Encode(value, 0.0f, 360.0f, 7); } - private float? _movementBlend; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float MovementBlendQuantizedStep = 0.096774194f; /// Float accessor for . Range [0.0f, 3.0f], 5 bits, step ≈ 0.0967742. public float MovementBlendQuantized { - get => _movementBlend ??= Quantize.Decode(MovementBlend, 0.0f, 3.0f, 5); - set { _movementBlend = value; MovementBlend = Quantize.Encode(value, 0.0f, 3.0f, 5); } + get => Quantize.Decode(MovementBlend, 0.0f, 3.0f, 5); + set => MovementBlend = Quantize.Encode(value, 0.0f, 3.0f, 5); } - private float? _slideBlend; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float SlideBlendQuantizedStep = 0.066666667f; /// Float accessor for . Range [0.0f, 1.0f], 4 bits, step ≈ 0.0666667. public float SlideBlendQuantized { - get => _slideBlend ??= Quantize.Decode(SlideBlend, 0.0f, 1.0f, 4); - set { _slideBlend = value; SlideBlend = Quantize.Encode(value, 0.0f, 1.0f, 4); } + get => Quantize.Decode(SlideBlend, 0.0f, 1.0f, 4); + set => SlideBlend = Quantize.Encode(value, 0.0f, 1.0f, 4); } - private float? _headYaw; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float HeadYawQuantizedStep = 2.8346457f; /// Float accessor for . Range [0.0f, 360.0f], 7 bits, step ≈ 2.83465. public float HeadYawQuantized { - get => _headYaw ??= Quantize.Decode(HeadYaw, 0.0f, 360.0f, 7); - set { _headYaw = value; HeadYaw = Quantize.Encode(value, 0.0f, 360.0f, 7); } + get => Quantize.Decode(HeadYaw, 0.0f, 360.0f, 7); + set => HeadYaw = Quantize.Encode(value, 0.0f, 360.0f, 7); } - private float? _headPitch; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float HeadPitchQuantizedStep = 2.8346457f; /// Float accessor for . Range [0.0f, 360.0f], 7 bits, step ≈ 2.83465. public float HeadPitchQuantized { - get => _headPitch ??= Quantize.Decode(HeadPitch, 0.0f, 360.0f, 7); - set { _headPitch = value; HeadPitch = Quantize.Encode(value, 0.0f, 360.0f, 7); } + get => Quantize.Decode(HeadPitch, 0.0f, 360.0f, 7); + set => HeadPitch = Quantize.Encode(value, 0.0f, 360.0f, 7); } - private float? _pointAtX; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PointAtXQuantizedStep = 0.045776716f; /// Float accessor for . Range [-3000.0f, 3000.0f], 17 bits, step ≈ 0.0457767. public float PointAtXQuantized { - get => _pointAtX ??= Quantize.Decode(PointAtX, -3000.0f, 3000.0f, 17); - set { _pointAtX = value; PointAtX = Quantize.Encode(value, -3000.0f, 3000.0f, 17); } + get => Quantize.Decode(PointAtX, -3000.0f, 3000.0f, 17); + set => PointAtX = Quantize.Encode(value, -3000.0f, 3000.0f, 17); } - private float? _pointAtY; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PointAtYQuantizedStep = 1.5748031f; /// Float accessor for . Range [0.0f, 200.0f], 7 bits, step ≈ 1.5748. public float PointAtYQuantized { - get => _pointAtY ??= Quantize.Decode(PointAtY, 0.0f, 200.0f, 7); - set { _pointAtY = value; PointAtY = Quantize.Encode(value, 0.0f, 200.0f, 7); } + get => Quantize.Decode(PointAtY, 0.0f, 200.0f, 7); + set => PointAtY = Quantize.Encode(value, 0.0f, 200.0f, 7); } - private float? _pointAtZ; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PointAtZQuantizedStep = 0.045776716f; /// Float accessor for . Range [-3000.0f, 3000.0f], 17 bits, step ≈ 0.0457767. public float PointAtZQuantized { - get => _pointAtZ ??= Quantize.Decode(PointAtZ, -3000.0f, 3000.0f, 17); - set { _pointAtZ = value; PointAtZ = Quantize.Encode(value, -3000.0f, 3000.0f, 17); } + get => Quantize.Decode(PointAtZ, -3000.0f, 3000.0f, 17); + set => PointAtZ = Quantize.Encode(value, -3000.0f, 3000.0f, 17); } - /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. - public void ResetDecodedCache() - { - _positionX = null; - _positionY = null; - _positionZ = null; - _velocityX = null; - _velocityY = null; - _velocityZ = null; - _rotationY = null; - _movementBlend = null; - _slideBlend = null; - _headYaw = null; - _headPitch = null; - _pointAtX = null; - _pointAtY = null; - _pointAtZ = null; - } + /// + /// True when every quantized field holds a wire code within its declared bit width + /// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger + /// value is a malformed/hostile message: decoding it would land far outside the field's + /// [min, max] and, since the server relays raw codes verbatim, poison every observer. + /// Reject before storing or relaying. Pure integer comparison — no decode. + /// + public bool AreQuantizedFieldsInRange() => + PositionX <= 255u + && PositionY <= 8191u + && PositionZ <= 255u + && VelocityX <= 255u + && VelocityY <= 255u + && VelocityZ <= 255u + && RotationY <= 127u + && MovementBlend <= 31u + && SlideBlend <= 15u + && HeadYaw <= 127u + && HeadPitch <= 127u + && PointAtX <= 131071u + && PointAtY <= 127u + && PointAtZ <= 131071u; } diff --git a/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs b/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs index 596301b6..62f157e6 100644 --- a/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs +++ b/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs @@ -9,160 +9,190 @@ namespace Decentraland.Common { public partial class PositionDelta { - private float? _dx; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float DxQuantizedStep = 0.0030518044f; /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. public float DxQuantized { - get => _dx ??= Quantize.Decode(Dx, -100.0f, 100.0f, 16); - set { _dx = value; Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); } + get => Quantize.Decode(Dx, -100.0f, 100.0f, 16); + set => Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); } - private float? _dy; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float DyQuantizedStep = 0.0030518044f; /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. public float DyQuantized { - get => _dy ??= Quantize.Decode(Dy, -100.0f, 100.0f, 16); - set { _dy = value; Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); } + get => Quantize.Decode(Dy, -100.0f, 100.0f, 16); + set => Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); } - private float? _dz; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float DzQuantizedStep = 0.0030518044f; /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. public float DzQuantized { - get => _dz ??= Quantize.Decode(Dz, -100.0f, 100.0f, 16); - set { _dz = value; Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); } - } - - /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. - public void ResetDecodedCache() - { - _dx = null; - _dy = null; - _dz = null; - } + get => Quantize.Decode(Dz, -100.0f, 100.0f, 16); + set => Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); + } + + /// + /// True when every quantized field holds a wire code within its declared bit width + /// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger + /// value is a malformed/hostile message: decoding it would land far outside the field's + /// [min, max] and, since the server relays raw codes verbatim, poison every observer. + /// Reject before storing or relaying. Pure integer comparison — no decode. + /// + public bool AreQuantizedFieldsInRange() => + Dx <= 65535u + && Dy <= 65535u + && Dz <= 65535u; } public partial class PlayerInput { - private float? _moveX; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float MoveXQuantizedStep = 0.0078431373f; /// Float accessor for . Range [-1.0f, 1.0f], 8 bits, step ≈ 0.00784314. public float MoveXQuantized { - get => _moveX ??= Quantize.Decode(MoveX, -1.0f, 1.0f, 8); - set { _moveX = value; MoveX = Quantize.Encode(value, -1.0f, 1.0f, 8); } + get => Quantize.Decode(MoveX, -1.0f, 1.0f, 8); + set => MoveX = Quantize.Encode(value, -1.0f, 1.0f, 8); } - private float? _moveZ; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float MoveZQuantizedStep = 0.0078431373f; /// Float accessor for . Range [-1.0f, 1.0f], 8 bits, step ≈ 0.00784314. public float MoveZQuantized { - get => _moveZ ??= Quantize.Decode(MoveZ, -1.0f, 1.0f, 8); - set { _moveZ = value; MoveZ = Quantize.Encode(value, -1.0f, 1.0f, 8); } + get => Quantize.Decode(MoveZ, -1.0f, 1.0f, 8); + set => MoveZ = Quantize.Encode(value, -1.0f, 1.0f, 8); } - private float? _yaw; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float YawQuantizedStep = 0.087912088f; /// Float accessor for . Range [-180.0f, 180.0f], 12 bits, step ≈ 0.0879121. public float YawQuantized { - get => _yaw ??= Quantize.Decode(Yaw, -180.0f, 180.0f, 12); - set { _yaw = value; Yaw = Quantize.Encode(value, -180.0f, 180.0f, 12); } - } - - /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. - public void ResetDecodedCache() - { - _moveX = null; - _moveZ = null; - _yaw = null; - } + get => Quantize.Decode(Yaw, -180.0f, 180.0f, 12); + set => Yaw = Quantize.Encode(value, -180.0f, 180.0f, 12); + } + + /// + /// True when every quantized field holds a wire code within its declared bit width + /// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger + /// value is a malformed/hostile message: decoding it would land far outside the field's + /// [min, max] and, since the server relays raw codes verbatim, poison every observer. + /// Reject before storing or relaying. Pure integer comparison — no decode. + /// + public bool AreQuantizedFieldsInRange() => + MoveX <= 255u + && MoveZ <= 255u + && Yaw <= 4095u; } public partial class VelocityState { - private float? _vx; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float VxQuantizedStep = 0.78430157f; /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. public float VxQuantized { - get => _vx ??= Quantize.DecodePower(Vx, 50.0f, 2.0f, 8); - set { _vx = value; Vx = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } + get => Quantize.DecodePower(Vx, 50.0f, 2.0f, 8); + set => Vx = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } - private float? _vy; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float VyQuantizedStep = 0.78430157f; /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. public float VyQuantized { - get => _vy ??= Quantize.DecodePower(Vy, 50.0f, 2.0f, 8); - set { _vy = value; Vy = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } + get => Quantize.DecodePower(Vy, 50.0f, 2.0f, 8); + set => Vy = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } - private float? _vz; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float VzQuantizedStep = 0.78430157f; /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. public float VzQuantized { - get => _vz ??= Quantize.DecodePower(Vz, 50.0f, 2.0f, 8); - set { _vz = value; Vz = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } - } - - /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. - public void ResetDecodedCache() - { - _vx = null; - _vy = null; - _vz = null; - } + get => Quantize.DecodePower(Vz, 50.0f, 2.0f, 8); + set => Vz = Quantize.EncodePower(value, 50.0f, 2.0f, 8); + } + + /// + /// True when every quantized field holds a wire code within its declared bit width + /// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger + /// value is a malformed/hostile message: decoding it would land far outside the field's + /// [min, max] and, since the server relays raw codes verbatim, poison every observer. + /// Reject before storing or relaying. Pure integer comparison — no decode. + /// + public bool AreQuantizedFieldsInRange() => + Vx <= 255u + && Vy <= 255u + && Vz <= 255u; } public partial class AvatarStateSnapshot { - private float? _x; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float XQuantizedStep = 0.12500191f; /// Float accessor for . Range [-4096.0f, 4096.0f], 16 bits, step ≈ 0.125002. public float XQuantized { - get => _x ??= Quantize.Decode(X, -4096.0f, 4096.0f, 16); - set { _x = value; X = Quantize.Encode(value, -4096.0f, 4096.0f, 16); } + get => Quantize.Decode(X, -4096.0f, 4096.0f, 16); + set => X = Quantize.Encode(value, -4096.0f, 4096.0f, 16); } - private float? _y; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float YQuantizedStep = 0.031251907f; /// Float accessor for . Range [-256.0f, 256.0f], 14 bits, step ≈ 0.0312519. public float YQuantized { - get => _y ??= Quantize.Decode(Y, -256.0f, 256.0f, 14); - set { _y = value; Y = Quantize.Encode(value, -256.0f, 256.0f, 14); } + get => Quantize.Decode(Y, -256.0f, 256.0f, 14); + set => Y = Quantize.Encode(value, -256.0f, 256.0f, 14); } - private float? _z; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float ZQuantizedStep = 0.12500191f; /// Float accessor for . Range [-4096.0f, 4096.0f], 16 bits, step ≈ 0.125002. public float ZQuantized { - get => _z ??= Quantize.Decode(Z, -4096.0f, 4096.0f, 16); - set { _z = value; Z = Quantize.Encode(value, -4096.0f, 4096.0f, 16); } + get => Quantize.Decode(Z, -4096.0f, 4096.0f, 16); + set => Z = Quantize.Encode(value, -4096.0f, 4096.0f, 16); } - private float? _pitch; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PitchQuantizedStep = 0.17595308f; /// Float accessor for . Range [-90.0f, 90.0f], 10 bits, step ≈ 0.175953. public float PitchQuantized { - get => _pitch ??= Quantize.Decode(Pitch, -90.0f, 90.0f, 10); - set { _pitch = value; Pitch = Quantize.Encode(value, -90.0f, 90.0f, 10); } + get => Quantize.Decode(Pitch, -90.0f, 90.0f, 10); + set => Pitch = Quantize.Encode(value, -90.0f, 90.0f, 10); } - private float? _yaw; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float YawQuantizedStep = 0.087912088f; /// Float accessor for . Range [-180.0f, 180.0f], 12 bits, step ≈ 0.0879121. public float YawQuantized { - get => _yaw ??= Quantize.Decode(Yaw, -180.0f, 180.0f, 12); - set { _yaw = value; Yaw = Quantize.Encode(value, -180.0f, 180.0f, 12); } - } - - /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. - public void ResetDecodedCache() - { - _x = null; - _y = null; - _z = null; - _pitch = null; - _yaw = null; - } + get => Quantize.Decode(Yaw, -180.0f, 180.0f, 12); + set => Yaw = Quantize.Encode(value, -180.0f, 180.0f, 12); + } + + /// + /// True when every quantized field holds a wire code within its declared bit width + /// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger + /// value is a malformed/hostile message: decoding it would land far outside the field's + /// [min, max] and, since the server relays raw codes verbatim, poison every observer. + /// Reject before storing or relaying. Pure integer comparison — no decode. + /// + public bool AreQuantizedFieldsInRange() => + X <= 65535u + && Y <= 16383u + && Z <= 65535u + && Pitch <= 1023u + && Yaw <= 4095u; } From 69f9051fb6a41f902afa089fe16ee7f8df8f4bd3 Mon Sep 17 00:00:00 2001 From: Mikhail Agapov <118179774+mikhail-dcl@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:58:23 +0300 Subject: [PATCH 5/5] chore: sync pulse-prd (#441) * docs: describe protoc-gen-bitwise as it actually works; drop dead BitReader/BitWriter CLAUDE.md still documented the original bit-stream design (BitWriter/ BitReader classes, float fields, "68 bits = 9 bytes on the wire"). The plugin actually emits quantized-accessor partials (*.Bitwise.cs) over plain uint32 fields sent as standard protobuf varints, backed only by Quantize.cs; runtime BitReader.cs/BitWriter.cs were referenced by nothing and are removed. Also sync README with current generator output (QuantizedStep consts, AreQuantizedFieldsInRange, EncodePower/ DecodePower, per-proto-file output naming) and drop the incorrect "cached" accessor wording. Co-Authored-By: Claude Fable 5 * fix: normalize CRLF when comparing gen:test golden fixtures With core.autocrlf=true (and no .gitattributes) git materializes the golden .cs fixtures with CRLF on Windows while the generator always emits LF, so the strict byte comparison failed on any fresh Windows checkout. Normalize line endings when reading the goldens. Co-Authored-By: Claude Fable 5 * fix: add realm in pulse PlayerJoined message * added realm to teleport performed message --------- Co-authored-by: Claude Fable 5 Co-authored-by: Lorenzo Ranciaffi --- CLAUDE.md | 128 ++++++++------------ README.md | 41 ++++++- proto/decentraland/pulse/pulse_server.proto | 2 + protoc-gen-bitwise/runtime/cs/BitReader.cs | 112 ----------------- protoc-gen-bitwise/runtime/cs/BitWriter.cs | 117 ------------------ protoc-gen-bitwise/test/generator.test.js | 4 +- 6 files changed, 89 insertions(+), 315 deletions(-) delete mode 100644 protoc-gen-bitwise/runtime/cs/BitReader.cs delete mode 100644 protoc-gen-bitwise/runtime/cs/BitWriter.cs diff --git a/CLAUDE.md b/CLAUDE.md index 03a2a374..8810c93f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,7 +10,7 @@ High-performance MMO-style multiplayer networking stack. Protocol is **open** (U | Client | Unity (C#) | | Server | Custom server | | Schema source of truth | `.proto` files | -| Serialization | Custom protoc plugin (bitwise encoding) | +| Serialization | Standard protobuf wire format + custom protoc plugin (quantized float accessors) | | Auth | Decentraland ECDSA chain validation (local, on HANDSHAKE channel 0) | --- @@ -18,7 +18,15 @@ High-performance MMO-style multiplayer networking stack. Protocol is **open** (U ## Serialization: Custom Protoc Plugin ### What it does -Reads `.proto` files with custom field options and generates **bitwise encode/decode code** in C#, keeping all client implementations bit-for-bit identical. +Reads `.proto` files with custom field options and generates C# **partial classes** (`*.Bitwise.cs`) that add typed float accessors on top of quantized `uint32` fields, keeping the quantization math bit-for-bit identical across all client implementations. + +The wire format is **standard protobuf** — a quantized value lives in a plain `uint32` field and travels as an ordinary varint. There is no custom bit stream; any protobuf-capable client can parse the messages without this plugin. Per annotated field the plugin emits: + +- `float {Field}Quantized` — computed accessor (no backing cache): the getter decodes the stored `uint32`, the setter encodes a float back into it, via the static `Quantize` helpers +- `const float {Field}QuantizedStep` — the coarsest quantization step of the field, safe as an equality tolerance +- per message: `bool AreQuantizedFieldsInRange()` — pure-integer check that every stored code fits its declared bit width (`0 .. 2^bits-1`); reject malformed/hostile messages before storing or relaying + +Only non-repeated `uint32` fields get accessors; `bit_packed` and unannotated fields pass through with no generated code. ### Custom Field Options (`options.proto`) @@ -54,25 +62,31 @@ extend google.protobuf.FieldOptions { ### Usage example +Quantized fields are declared **`uint32`** (not `float`) — the float type exists only in the generated accessor: + ```protobuf message PositionDelta { - float dx = 1 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; - float dy = 2 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; - float dz = 3 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dx = 1 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dy = 2 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dz = 3 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; uint32 entity_id = 4 [(bit_packed) = { bits: 20 }]; + uint32 sequence = 5 [(bit_packed) = { bits: 12 }]; } -// Total: 68 bits = 9 bytes on the wire +// Varint wire cost: dx/dy/dz/entity_id ≤ 4 B each (1 B tag + ≤ 3 B varint), +// sequence ≤ 3 B — worst-case 19 B, less when proto3 omits zero-valued fields. ``` +`proto/decentraland/common/quantization_example.proto` is the fully worked reference: per-field wire costs for the linear, power-law, and bit-packed annotations. + ### Plugin structure ``` protoc-gen-bitwise/ ├── plugin.js # stdin -> CodeGeneratorRequest, stdout -> CodeGeneratorResponse (Node) -├── generator_csharp.js # emits C# for Unity -├── options.js # parses the custom quantized / bit_packed field options +├── generator_csharp.js # emits the *.Bitwise.cs accessor partials for Unity +├── options.js # parses the custom quantized / quantized_power / bit_packed field options ├── wire.js # self-contained protobuf wire codec (zero runtime deps) -└── runtime/cs/ # C# runtime; Quantize.cs is copied into the generated output +└── runtime/cs/ # C# runtime: Quantize.cs — consumers copy it next to the generated files ``` Plugin contract: a protoc plugin that reads a serialized `CodeGeneratorRequest` from stdin and writes a serialized `CodeGeneratorResponse` to stdout. It is a plain Node script — **no `npm install` required, only `node` on PATH**. protoc invokes it through a tiny wrapper that runs `node plugin.js` (`.cmd` on Windows, a shell script elsewhere, since protoc cannot exec a `.js` directly). @@ -89,85 +103,30 @@ Parity is locked down by `npm run gen:test` (compares generator output against g --- -## BitWriter / BitReader +## Quantize Runtime -The C# implementation uses the following bit layout: **big-endian within each byte**, MSB written first. Use **`Round`** (not truncate) when quantizing to minimize error. +Generated accessors call the static `Quantize` class (`protoc-gen-bitwise/runtime/cs/Quantize.cs`, namespace `Decentraland.Networking.Bitwise`) — the only C# runtime file; consumers copy it next to the generated `*.Bitwise.cs` partials. Quantization uses **`Round`** (not truncate) to minimize error; identical rounding on both sides makes encode -> decode a round-trip no-op. -### Core math — WriteQuantizedFloat +### Core math — linear (`Quantize.Encode` / `Quantize.Decode`) ``` -normalized = (clamp(value, min, max) - min) / (max - min) // -> [0.0, 1.0] -quantized = Round(normalized * ((1 << bits) - 1)) // -> integer +encoded = Round(clamp01((value - min) / (max - min)) * (2^bits - 1)) +decoded = encoded / (2^bits - 1) * (max - min) + min ``` -### Core math — ReadQuantizedFloat +### Core math — power-law (`Quantize.EncodePower` / `Quantize.DecodePower`) + +For signed fields like velocity that need an exact zero and fine resolution near zero: ``` -normalized = quantized / ((1 << bits) - 1) -value = min + normalized * (max - min) +u = clamp01(|value| / max) ^ (1 / pow) +encoded = (Round(u * (2^(bits-1) - 1)) << 1) | sign // magnitude in high bits, sign in LSB +decoded = sign * max * ((encoded >> 1) / (2^(bits-1) - 1)) ^ pow ``` -### Implementation - -```csharp -public class BitWriter -{ - private byte[] _buffer; - private int _bitPos; - - public BitWriter(byte[] buffer) { _buffer = buffer; _bitPos = 0; } - - public void WriteBits(uint value, int bits) - { - for (int i = bits - 1; i >= 0; i--) - { - int byteIdx = _bitPos / 8; - int bitIdx = 7 - (_bitPos % 8); - if ((value >> i & 1) == 1) _buffer[byteIdx] |= (byte)(1 << bitIdx); - else _buffer[byteIdx] &= (byte)~(1 << bitIdx); - _bitPos++; - } - } - - public void WriteQuantizedFloat(float value, float min, float max, int bits) - { - uint maxQ = (1u << bits) - 1; - float clamped = Math.Clamp(value, min, max); - float normalized = (clamped - min) / (max - min); - uint quantized = (uint)Math.Round(normalized * maxQ); - WriteBits(quantized, bits); - } -} - -public class BitReader -{ - private byte[] _buffer; - private int _bitPos; - - public BitReader(byte[] buffer) { _buffer = buffer; _bitPos = 0; } - - public uint ReadBits(int bits) - { - uint value = 0; - for (int i = bits - 1; i >= 0; i--) - { - int byteIdx = _bitPos / 8; - int bitIdx = 7 - (_bitPos % 8); - if ((_buffer[byteIdx] >> bitIdx & 1) == 1) value |= 1u << i; - _bitPos++; - } - return value; - } - - public float ReadQuantizedFloat(float min, float max, int bits) - { - uint maxQ = (1u << bits) - 1; - uint quantized = ReadBits(bits); - float normalized = (float)quantized / maxQ; - return min + normalized * (max - min); - } -} -``` +- Zero encodes exactly to code `0` (a zero magnitude never sets the sign bit), so proto3 omits a stopped field entirely +- `pow > 1` concentrates resolution near zero, coarse near `±max` +- Sign in the LSB makes the varint cost track magnitude, not direction — a small `|value|` of either sign stays in one varint byte --- @@ -181,13 +140,24 @@ public class BitReader Sub-centimeter precision is achievable at 12-16 bits for position deltas. +Wire cost is varint-based — 1 tag byte per present field (field numbers ≤ 15) plus: + +| Code bits | Worst-case varint | Worst-case field total | +|-----------|-------------------|------------------------| +| ≤ 7 | 1 B | 2 B | +| ≤ 14 | 2 B | 3 B | +| ≤ 21 | 3 B | 4 B | + +Proto3 omits fields equal to 0, so typical cost is lower than worst-case. + --- ## Key Design Principles - `.proto` files are the **single source of truth** for all message schemas -- The protoc plugin generates **C#** from the schema — never hand-write serialization +- The protoc plugin generates the **C# quantized accessors** from the schema — never hand-write quantization math; standard protobuf handles the wire encoding - Encode -> decode is a **no-op** (round-trip safe) due to consistent use of `Round` +- Validate inbound quantized messages with `AreQuantizedFieldsInRange()` before storing or relaying — the server relays raw codes verbatim - Prefer **client-driven resync** over proactive server corrections - Push complexity to clients where appropriate; server maintains authority - Channel 0: reliable messages (STATE_FULL snapshots, ACKs, resync requests, HANDSHAKE) diff --git a/README.md b/README.md index a968d863..a846d9a2 100644 --- a/README.md +++ b/README.md @@ -90,8 +90,10 @@ Rather than a separate binary packing layer, the plugin leverages this: 2. `--csharp_out` generates the standard protobuf class with the raw `uint32` property (e.g. `PositionX`). 3. `--bitwise_out` (this plugin) generates a `partial class` extension with a - cached float accessor (e.g. `PositionXQuantized`) that encodes/decodes - transparently via `Quantize.Encode` / `Quantize.Decode`. + computed float accessor (e.g. `PositionXQuantized`) that encodes/decodes + transparently via `Quantize.Encode` / `Quantize.Decode` on every access — + the raw `uint32` property remains the single source of truth (no cache to + go stale when the raw field is mutated directly). The wire representation is a standard protobuf message — any protobuf-capable client can read it without knowledge of the plugin. @@ -133,8 +135,8 @@ message PositionDelta { | Annotation | Target type | Parameters | Effect | |---|---|---|---| -| `[(decentraland.common.quantized)]` | `uint32` | `min`, `max`, `bits` | Plugin emits a cached `float {Name}Quantized` accessor | -| `[(decentraland.common.quantized_power)]` | `uint32` | `max`, `pow`, `bits` | Power-law quantizer over `[-max, max]`: `(bits-1)`-bit magnitude (high bits) + sign (LSB), decoded as `sign·max·u^pow`. Exact zero; `pow>1` gives fine resolution near zero, coarse near `±max`; sign in the LSB keeps small magnitudes one varint byte. Cached `float {Name}Quantized` accessor (`Quantize.EncodePower`/`DecodePower`) | +| `[(decentraland.common.quantized)]` | `uint32` | `min`, `max`, `bits` | Plugin emits a `float {Name}Quantized` accessor and a `{Name}QuantizedStep` const | +| `[(decentraland.common.quantized_power)]` | `uint32` | `max`, `pow`, `bits` | Power-law quantizer over `[-max, max]`: `(bits-1)`-bit magnitude (high bits) + sign (LSB), decoded as `sign·max·u^pow`. Exact zero; `pow>1` gives fine resolution near zero, coarse near `±max`; sign in the LSB keeps small magnitudes one varint byte. `float {Name}Quantized` accessor (`Quantize.EncodePower`/`DecodePower`) | | `[(decentraland.common.bit_packed)]` | `uint32` | `bits` | Documents the value range; protobuf handles varint compaction automatically | ### Wire cost at worst-case (all bits set) @@ -183,13 +185,15 @@ Assets/ ``` `Quantize.cs` lives in the `Decentraland.Networking.Bitwise` namespace and -provides two static methods used by the generated accessors: +provides the static encode/decode methods used by the generated accessors: ```csharp public static class Quantize { public static uint Encode(float value, float min, float max, int bits); public static float Decode(uint encoded, float min, float max, int bits); + public static uint EncodePower(float value, float max, float pow, int bits); + public static float DecodePower(uint encoded, float max, float pow, int bits); } ``` @@ -213,6 +217,7 @@ SendOnChannel1(bytes); // --- Receive and read --- var received = PositionDelta.Parser.ParseFrom(receivedBytes); +if (!received.AreQuantizedFieldsInRange()) return; // reject malformed/hostile codes float x = received.DxQuantized; // decoded from the stored uint32 on each access float y = received.DyQuantized; float z = received.DzQuantized; @@ -220,7 +225,10 @@ float z = received.DzQuantized; ## Generated file example -For the `PositionDelta` message above the plugin emits `PositionDelta.Bitwise.cs`: +For the `comms.proto` file above the plugin emits `Comms.Bitwise.cs` (one file +per `.proto`, named after the proto file). Each quantized field gets a +`{Name}QuantizedStep` const and a float accessor; each message gets an +`AreQuantizedFieldsInRange()` guard for validating inbound wire codes: ```csharp // @@ -234,23 +242,44 @@ namespace Decentraland.Kernel.Comms.V3 { public partial class PositionDelta { + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float DxQuantizedStep = 0.0030518044f; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. public float DxQuantized { get => Quantize.Decode(Dx, -100.0f, 100.0f, 16); set => Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); } + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float DyQuantizedStep = 0.0030518044f; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. public float DyQuantized { get => Quantize.Decode(Dy, -100.0f, 100.0f, 16); set => Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); } + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float DzQuantizedStep = 0.0030518044f; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. public float DzQuantized { get => Quantize.Decode(Dz, -100.0f, 100.0f, 16); set => Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); } + + /// + /// True when every quantized field holds a wire code within its declared bit width + /// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger + /// value is a malformed/hostile message: decoding it would land far outside the field's + /// [min, max] and, since the server relays raw codes verbatim, poison every observer. + /// Reject before storing or relaying. Pure integer comparison — no decode. + /// + public bool AreQuantizedFieldsInRange() => + Dx <= 65535u + && Dy <= 65535u + && Dz <= 65535u; } } // namespace Decentraland.Kernel.Comms.V3 diff --git a/proto/decentraland/pulse/pulse_server.proto b/proto/decentraland/pulse/pulse_server.proto index 1a9ca80e..9957ac32 100644 --- a/proto/decentraland/pulse/pulse_server.proto +++ b/proto/decentraland/pulse/pulse_server.proto @@ -84,6 +84,7 @@ message PlayerJoined { string user_id = 1; int32 profile_version = 2; PlayerStateFull state = 3; + string realm = 4; } // Notification to the client, that a peer has left, it can mean disconnection or leaving the area of interest @@ -125,6 +126,7 @@ message TeleportPerformed { uint32 sequence = 2; uint32 server_tick = 3; PlayerState state = 4; + string realm = 5; } message ServerMessage { diff --git a/protoc-gen-bitwise/runtime/cs/BitReader.cs b/protoc-gen-bitwise/runtime/cs/BitReader.cs deleted file mode 100644 index 51efa39c..00000000 --- a/protoc-gen-bitwise/runtime/cs/BitReader.cs +++ /dev/null @@ -1,112 +0,0 @@ -// Decentraland.Networking.Bitwise — BitReader -// Copy this file into your Unity project alongside generated *.Bitwise.cs files. - -using System; - -namespace Decentraland.Networking.Bitwise -{ - /// - /// Reads bits from a byte buffer, MSB first within each byte (big-endian bit - /// order). Symmetric counterpart of : every - /// Write… call has a corresponding Read… call with identical arguments that - /// reproduces the original value. - /// - public sealed class BitReader - { - private readonly byte[] _buffer; - private int _bitPos; - - /// Source buffer filled by a . - public BitReader(byte[] buffer) - { - _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer)); - _bitPos = 0; - } - - /// Current read position in bits. - public int BitPosition => _bitPos; - - /// - /// Returns true when all written bits have been consumed - /// (i.e. has reached the end of the buffer). - /// - public bool IsAtEnd => _bitPos >= _buffer.Length * 8; - - // ----------------------------------------------------------------- - // Core primitive - // ----------------------------------------------------------------- - - /// - /// Reads bits and returns them as the - /// least-significant bits of a , MSB first. - /// - public uint ReadBits(int bits) - { - uint value = 0; - for (int i = bits - 1; i >= 0; i--) - { - int byteIdx = _bitPos / 8; - int bitIdx = 7 - (_bitPos % 8); - - if ((_buffer[byteIdx] >> bitIdx & 1) == 1) - value |= 1u << i; - - _bitPos++; - } - return value; - } - - // ----------------------------------------------------------------- - // Quantized float - // ----------------------------------------------------------------- - - /// - /// Reads a quantized float encoded with . - /// Arguments must match those used during encoding exactly. - /// - public float ReadQuantizedFloat(float min, float max, int bits) - { - uint maxQ = (1u << bits) - 1; - uint quantized = ReadBits(bits); - float normalized = (float)quantized / maxQ; - return min + normalized * (max - min); - } - - // ----------------------------------------------------------------- - // Standard IEEE 754 helpers - // ----------------------------------------------------------------- - - /// Reads a 32-bit IEEE 754 float written by . - public float ReadFloat() - { - uint bits = ReadBits(32); - byte[] bytes = - { - (byte)(bits & 0xFF), - (byte)((bits >> 8) & 0xFF), - (byte)((bits >> 16) & 0xFF), - (byte)((bits >> 24) & 0xFF), - }; - return BitConverter.ToSingle(bytes, 0); - } - - /// Reads a 64-bit IEEE 754 double written by . - public double ReadDouble() - { - uint hi = ReadBits(32); - uint lo = ReadBits(32); - byte[] bytes = - { - (byte)(lo & 0xFF), - (byte)((lo >> 8) & 0xFF), - (byte)((lo >> 16) & 0xFF), - (byte)((lo >> 24) & 0xFF), - (byte)(hi & 0xFF), - (byte)((hi >> 8) & 0xFF), - (byte)((hi >> 16) & 0xFF), - (byte)((hi >> 24) & 0xFF), - }; - return BitConverter.ToDouble(bytes, 0); - } - } -} diff --git a/protoc-gen-bitwise/runtime/cs/BitWriter.cs b/protoc-gen-bitwise/runtime/cs/BitWriter.cs deleted file mode 100644 index 19b205b8..00000000 --- a/protoc-gen-bitwise/runtime/cs/BitWriter.cs +++ /dev/null @@ -1,117 +0,0 @@ -// Decentraland.Networking.Bitwise — BitWriter -// Copy this file into your Unity project alongside generated *.Bitwise.cs files. - -using System; - -namespace Decentraland.Networking.Bitwise -{ - /// - /// Writes bits into a pre-allocated byte buffer, MSB first within each byte - /// (big-endian bit order). This matches the layout expected by - /// so that encode → decode is always a round-trip no-op. - /// - public sealed class BitWriter - { - private readonly byte[] _buffer; - private int _bitPos; - - /// Destination buffer (must be large enough for all writes). - public BitWriter(byte[] buffer) - { - _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer)); - _bitPos = 0; - } - - /// Current write position in bits. - public int BitPosition => _bitPos; - - /// Number of bytes written (rounded up to the nearest byte). - public int ByteLength => (_bitPos + 7) / 8; - - /// Returns a copy of the written bytes (trimmed to ). - public byte[] ToArray() - { - var result = new byte[ByteLength]; - Array.Copy(_buffer, result, ByteLength); - return result; - } - - // ----------------------------------------------------------------- - // Core primitive - // ----------------------------------------------------------------- - - /// - /// Writes the least-significant bits of - /// , MSB first. - /// - public void WriteBits(uint value, int bits) - { - for (int i = bits - 1; i >= 0; i--) - { - int byteIdx = _bitPos / 8; - int bitIdx = 7 - (_bitPos % 8); - - if ((value >> i & 1u) == 1u) - _buffer[byteIdx] |= (byte)(1 << bitIdx); - else - _buffer[byteIdx] &= (byte)~(1 << bitIdx); - - _bitPos++; - } - } - - // ----------------------------------------------------------------- - // Quantized float - // ----------------------------------------------------------------- - - /// - /// Quantizes into bits - /// using the range [, ] and - /// writes it to the buffer. - /// - /// Uses Math.Round (banker's rounding → ties-to-even) to guarantee - /// that encode → decode is a round-trip no-op. - /// - public void WriteQuantizedFloat(float value, float min, float max, int bits) - { - uint maxQ = (1u << bits) - 1; - float clamped = Math.Clamp(value, min, max); - float normalized = (clamped - min) / (max - min); - uint quantized = (uint)Math.Round(normalized * maxQ); - WriteBits(quantized, bits); - } - - // ----------------------------------------------------------------- - // Standard IEEE 754 helpers (used for un-annotated float/double fields) - // ----------------------------------------------------------------- - - /// Writes a 32-bit IEEE 754 float (4 bytes). - public void WriteFloat(float value) - { - byte[] bytes = BitConverter.GetBytes(value); - // GetBytes is little-endian on all platforms; write MSB first. - uint bits = (uint)bytes[0] - | ((uint)bytes[1] << 8) - | ((uint)bytes[2] << 16) - | ((uint)bytes[3] << 24); - WriteBits(bits, 32); - } - - /// Writes a 64-bit IEEE 754 double (8 bytes). - public void WriteDouble(double value) - { - byte[] bytes = BitConverter.GetBytes(value); - uint lo = (uint)bytes[0] - | ((uint)bytes[1] << 8) - | ((uint)bytes[2] << 16) - | ((uint)bytes[3] << 24); - uint hi = (uint)bytes[4] - | ((uint)bytes[5] << 8) - | ((uint)bytes[6] << 16) - | ((uint)bytes[7] << 24); - // Write high 32 bits first so the bit stream is big-endian at word level too. - WriteBits(hi, 32); - WriteBits(lo, 32); - } - } -} diff --git a/protoc-gen-bitwise/test/generator.test.js b/protoc-gen-bitwise/test/generator.test.js index dc20b39e..d4b26b4b 100644 --- a/protoc-gen-bitwise/test/generator.test.js +++ b/protoc-gen-bitwise/test/generator.test.js @@ -83,7 +83,9 @@ function field(name, type, optionsRaw) { } function readGolden(name) { - return fs.readFileSync(path.join(__dirname, 'golden', name), 'utf8') + // Git may materialize the goldens with CRLF on Windows (core.autocrlf=true, + // no .gitattributes); the generator always emits LF, so normalize first. + return fs.readFileSync(path.join(__dirname, 'golden', name), 'utf8').replace(/\r\n/g, '\n') } // --------------------------------------------------------------------------