diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..8810c93f --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,164 @@ +# CLAUDE.md — MMO Networking Stack + +## Project Overview + +High-performance MMO-style multiplayer networking stack. Protocol is **open** (Unity C# client + others). Infrastructure on **AWS**. Goals: high concurrency, aggressive interest management, low-latency state sync. + +| Layer | Choice | +|---|---| +| Transport | ENet (UDP, channel-based) | +| Client | Unity (C#) | +| Server | Custom server | +| Schema source of truth | `.proto` files | +| Serialization | Standard protobuf wire format + custom protoc plugin (quantized float accessors) | +| Auth | Decentraland ECDSA chain validation (local, on HANDSHAKE channel 0) | + +--- + +## Serialization: Custom Protoc Plugin + +### What it does +Reads `.proto` files with custom field options and generates C# **partial classes** (`*.Bitwise.cs`) that add typed float accessors on top of quantized `uint32` fields, keeping the quantization math bit-for-bit identical across all client implementations. + +The wire format is **standard protobuf** — a quantized value lives in a plain `uint32` field and travels as an ordinary varint. There is no custom bit stream; any protobuf-capable client can parse the messages without this plugin. Per annotated field the plugin emits: + +- `float {Field}Quantized` — computed accessor (no backing cache): the getter decodes the stored `uint32`, the setter encodes a float back into it, via the static `Quantize` helpers +- `const float {Field}QuantizedStep` — the coarsest quantization step of the field, safe as an equality tolerance +- per message: `bool AreQuantizedFieldsInRange()` — pure-integer check that every stored code fits its declared bit width (`0 .. 2^bits-1`); reject malformed/hostile messages before storing or relaying + +Only non-repeated `uint32` fields get accessors; `bit_packed` and unannotated fields pass through with no generated code. + +### Custom Field Options (`options.proto`) + +```protobuf +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; + +message QuantizedFloatOptions { + float min = 1; + float max = 2; + uint32 bits = 3; +} + +// Signed power-law quantizer: an (bits-1)-bit magnitude (high bits) plus a sign +// (LSB), decoded as sign * max * u^pow. Exact zero; pow>1 concentrates resolution +// near zero; sign in the LSB keeps small magnitudes in one varint byte. +message QuantizedPowerFloatOptions { + float max = 1; + float pow = 2; + uint32 bits = 3; +} + +message BitPackedOptions { + uint32 bits = 1; +} + +extend google.protobuf.FieldOptions { + QuantizedFloatOptions quantized = 50001; + BitPackedOptions bit_packed = 50002; + QuantizedPowerFloatOptions quantized_power = 50003; +} +``` + +### Usage example + +Quantized fields are declared **`uint32`** (not `float`) — the float type exists only in the generated accessor: + +```protobuf +message PositionDelta { + uint32 dx = 1 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dy = 2 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dz = 3 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 entity_id = 4 [(bit_packed) = { bits: 20 }]; + uint32 sequence = 5 [(bit_packed) = { bits: 12 }]; +} +// Varint wire cost: dx/dy/dz/entity_id ≤ 4 B each (1 B tag + ≤ 3 B varint), +// sequence ≤ 3 B — worst-case 19 B, less when proto3 omits zero-valued fields. +``` + +`proto/decentraland/common/quantization_example.proto` is the fully worked reference: per-field wire costs for the linear, power-law, and bit-packed annotations. + +### Plugin structure + +``` +protoc-gen-bitwise/ +├── plugin.js # stdin -> CodeGeneratorRequest, stdout -> CodeGeneratorResponse (Node) +├── generator_csharp.js # emits the *.Bitwise.cs accessor partials for Unity +├── options.js # parses the custom quantized / quantized_power / bit_packed field options +├── wire.js # self-contained protobuf wire codec (zero runtime deps) +└── runtime/cs/ # C# runtime: Quantize.cs — consumers copy it next to the generated files +``` + +Plugin contract: a protoc plugin that reads a serialized `CodeGeneratorRequest` from stdin and writes a serialized `CodeGeneratorResponse` to stdout. It is a plain Node script — **no `npm install` required, only `node` on PATH**. protoc invokes it through a tiny wrapper that runs `node plugin.js` (`.cmd` on Windows, a shell script elsewhere, since protoc cannot exec a `.js` directly). + +```bash +protoc \ + --proto_path=proto \ + --bitwise_out=generated/ \ + --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.js \ + movement.proto position.proto +``` + +Parity is locked down by `npm run gen:test` (compares generator output against golden C# fixtures in `protoc-gen-bitwise/test/`). + +--- + +## Quantize Runtime + +Generated accessors call the static `Quantize` class (`protoc-gen-bitwise/runtime/cs/Quantize.cs`, namespace `Decentraland.Networking.Bitwise`) — the only C# runtime file; consumers copy it next to the generated `*.Bitwise.cs` partials. Quantization uses **`Round`** (not truncate) to minimize error; identical rounding on both sides makes encode -> decode a round-trip no-op. + +### Core math — linear (`Quantize.Encode` / `Quantize.Decode`) + +``` +encoded = Round(clamp01((value - min) / (max - min)) * (2^bits - 1)) +decoded = encoded / (2^bits - 1) * (max - min) + min +``` + +### Core math — power-law (`Quantize.EncodePower` / `Quantize.DecodePower`) + +For signed fields like velocity that need an exact zero and fine resolution near zero: + +``` +u = clamp01(|value| / max) ^ (1 / pow) +encoded = (Round(u * (2^(bits-1) - 1)) << 1) | sign // magnitude in high bits, sign in LSB +decoded = sign * max * ((encoded >> 1) / (2^(bits-1) - 1)) ^ pow +``` + +- Zero encodes exactly to code `0` (a zero magnitude never sets the sign bit), so proto3 omits a stopped field entirely +- `pow > 1` concentrates resolution near zero, coarse near `±max` +- Sign in the LSB makes the varint cost track magnitude, not direction — a small `|value|` of either sign stays in one varint byte + +--- + +## Precision Reference + +| Range | Bits | Step size | +|-------------|------|---------------| +| [-100, 100] | 16 | ~0.003 units | +| [-10, 10] | 12 | ~0.005 units | +| [-100, 100] | 12 | ~0.049 units | + +Sub-centimeter precision is achievable at 12-16 bits for position deltas. + +Wire cost is varint-based — 1 tag byte per present field (field numbers ≤ 15) plus: + +| Code bits | Worst-case varint | Worst-case field total | +|-----------|-------------------|------------------------| +| ≤ 7 | 1 B | 2 B | +| ≤ 14 | 2 B | 3 B | +| ≤ 21 | 3 B | 4 B | + +Proto3 omits fields equal to 0, so typical cost is lower than worst-case. + +--- + +## Key Design Principles + +- `.proto` files are the **single source of truth** for all message schemas +- The protoc plugin generates the **C# quantized accessors** from the schema — never hand-write quantization math; standard protobuf handles the wire encoding +- Encode -> decode is a **no-op** (round-trip safe) due to consistent use of `Round` +- Validate inbound quantized messages with `AreQuantizedFieldsInRange()` before storing or relaying — the server relays raw codes verbatim +- Prefer **client-driven resync** over proactive server corrections +- Push complexity to clients where appropriate; server maintains authority +- Channel 0: reliable messages (STATE_FULL snapshots, ACKs, resync requests, HANDSHAKE) +- Channel 1: unreliable sequenced (high-frequency position deltas, client input) \ No newline at end of file diff --git a/README.md b/README.md index 598bdb0b..a846d9a2 100644 --- a/README.md +++ b/README.md @@ -67,3 +67,220 @@ In this case, there is no problem with when each PR is merged. It's recommendabl ## Comms TODO + +--- + +# Bitwise Serialization Plugin (`protoc-gen-bitwise`) + +A custom protoc plugin that generates C# partial classes with typed float +accessors for quantized `uint32` fields in high-frequency MMO networking +messages (position deltas, player input, etc.). It runs alongside +`--csharp_out` in the same protoc invocation; the two output files coexist +via C# `partial class`. + +## How it works + +Protobuf encodes `uint32` values as varints, which are already compact for +small values: a value up to 2¹⁴−1 costs 2 bytes, up to 2²¹−1 costs 3 bytes. +Rather than a separate binary packing layer, the plugin leverages this: + +1. Declare quantized fields as `uint32` in the `.proto` schema and annotate + them with `[(decentraland.common.quantized)]` to specify the float range + and bit resolution. +2. `--csharp_out` generates the standard protobuf class with the raw `uint32` + property (e.g. `PositionX`). +3. `--bitwise_out` (this plugin) generates a `partial class` extension with a + computed float accessor (e.g. `PositionXQuantized`) that encodes/decodes + transparently via `Quantize.Encode` / `Quantize.Decode` on every access — + the raw `uint32` property remains the single source of truth (no cache to + go stale when the raw field is mutated directly). + +The wire representation is a standard protobuf message — any protobuf-capable +client can read it without knowledge of the plugin. + +## Prerequisites + +| Requirement | Version | +|---|---| +| Node.js | 16+ | +| `protoc` | 3.19+ | + +The plugin is a dependency-free Node script — no `npm install` or extra +packages are required to run it. + +## Step 1 — Annotate your `.proto` file + +Declare quantized fields as `uint32` and import `options.proto`: + +```protobuf +syntax = "proto3"; + +import "decentraland/common/options.proto"; + +package decentraland.kernel.comms.v3; + +message PositionDelta { + // Float range [-100, 100] quantized to 16 bits ≈ 0.003-unit precision. + // Stored as uint32 on the wire; protobuf encodes it as a 3-byte varint. + uint32 dx = 1 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dy = 2 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dz = 3 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + + // Unannotated uint32: protobuf varint encodes small values compactly by default. + uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; +} +``` + +### Annotation reference + +| Annotation | Target type | Parameters | Effect | +|---|---|---|---| +| `[(decentraland.common.quantized)]` | `uint32` | `min`, `max`, `bits` | Plugin emits a `float {Name}Quantized` accessor and a `{Name}QuantizedStep` const | +| `[(decentraland.common.quantized_power)]` | `uint32` | `max`, `pow`, `bits` | Power-law quantizer over `[-max, max]`: `(bits-1)`-bit magnitude (high bits) + sign (LSB), decoded as `sign·max·u^pow`. Exact zero; `pow>1` gives fine resolution near zero, coarse near `±max`; sign in the LSB keeps small magnitudes one varint byte. `float {Name}Quantized` accessor (`Quantize.EncodePower`/`DecodePower`) | +| `[(decentraland.common.bit_packed)]` | `uint32` | `bits` | Documents the value range; protobuf handles varint compaction automatically | + +### Wire cost at worst-case (all bits set) + +| Quantization bits | Max value | Varint bytes | Tag (field ≤ 15) | Total per field | +|---|---|---|---|---| +| 8 | 255 | 2 | 1 | 3 B | +| 12 | 4 095 | 2 | 1 | 3 B | +| 14 | 16 383 | 2 | 1 | 3 B | +| 16 | 65 535 | 3 | 1 | 4 B | +| 20 | 1 048 575 | 3 | 1 | 4 B | + +Proto3 omits fields equal to their default value (0), so average cost is lower. + +## Step 2 — Run protoc + +```bash +protoc \ + --proto_path=proto \ + --proto_path=/path/to/google/protobuf/include \ + --csharp_out=generated/cs \ + --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.js \ + --bitwise_out=generated/cs \ + proto/decentraland/kernel/comms/v3/comms.proto +``` + +> `plugin.js` carries a `#!/usr/bin/env node` shebang. On Windows, protoc +> cannot exec a `.js` directly, so point `--plugin` at a small `.cmd` wrapper +> that runs `node plugin.js`; on unix an equivalent shell script is used. Both +> consumer repos (Pulse, unity-explorer) generate this wrapper automatically. + +The plugin emits one `*.Bitwise.cs` file (PascalCase, flat in the output +directory) for each `.proto` file that contains at least one `[(quantized)]` +field. + +## Step 3 — Copy the runtime + +Copy `Quantize.cs` into your project: + +``` +Assets/ +└── Scripts/ + └── Networking/ + └── Bitwise/ + └── Quantize.cs ← protoc-gen-bitwise/runtime/cs/Quantize.cs +``` + +`Quantize.cs` lives in the `Decentraland.Networking.Bitwise` namespace and +provides the static encode/decode methods used by the generated accessors: + +```csharp +public static class Quantize +{ + public static uint Encode(float value, float min, float max, int bits); + public static float Decode(uint encoded, float min, float max, int bits); + public static uint EncodePower(float value, float max, float pow, int bits); + public static float DecodePower(uint encoded, float max, float pow, int bits); +} +``` + +## Step 4 — Use the generated code + +The plugin emits a `partial class` that adds float accessors on top of the +standard protobuf-generated `uint32` properties: + +```csharp +using Decentraland.Kernel.Comms.V3; + +// --- Build and send --- +var delta = new PositionDelta(); +delta.DxQuantized = 3.14f; // encodes to uint32, stored in delta.Dx +delta.DyQuantized = 0f; +delta.DzQuantized = -7.5f; +delta.EntityId = 42u; + +byte[] bytes = delta.ToByteArray(); // standard protobuf serialization +SendOnChannel1(bytes); + +// --- Receive and read --- +var received = PositionDelta.Parser.ParseFrom(receivedBytes); +if (!received.AreQuantizedFieldsInRange()) return; // reject malformed/hostile codes +float x = received.DxQuantized; // decoded from the stored uint32 on each access +float y = received.DyQuantized; +float z = received.DzQuantized; +``` + +## Generated file example + +For the `comms.proto` file above the plugin emits `Comms.Bitwise.cs` (one file +per `.proto`, named after the proto file). Each quantized field gets a +`{Name}QuantizedStep` const and a float accessor; each message gets an +`AreQuantizedFieldsInRange()` guard for validating inbound wire codes: + +```csharp +// +// Generated by protoc-gen-bitwise. DO NOT EDIT. +// Source: decentraland/kernel/comms/v3/comms.proto +// + +using Decentraland.Networking.Bitwise; + +namespace Decentraland.Kernel.Comms.V3 +{ + public partial class PositionDelta + { + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float DxQuantizedStep = 0.0030518044f; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. + public float DxQuantized + { + get => Quantize.Decode(Dx, -100.0f, 100.0f, 16); + set => Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float DyQuantizedStep = 0.0030518044f; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. + public float DyQuantized + { + get => Quantize.Decode(Dy, -100.0f, 100.0f, 16); + set => Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float DzQuantizedStep = 0.0030518044f; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. + public float DzQuantized + { + get => Quantize.Decode(Dz, -100.0f, 100.0f, 16); + set => Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); + } + + /// + /// True when every quantized field holds a wire code within its declared bit width + /// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger + /// value is a malformed/hostile message: decoding it would land far outside the field's + /// [min, max] and, since the server relays raw codes verbatim, poison every observer. + /// Reject before storing or relaying. Pure integer comparison — no decode. + /// + public bool AreQuantizedFieldsInRange() => + Dx <= 65535u + && Dy <= 65535u + && Dz <= 65535u; + } + +} // namespace Decentraland.Kernel.Comms.V3 +``` diff --git a/package.json b/package.json index 1e5294e5..bb3533f8 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "license": "Apache-2.0", "main": "index.js", "scripts": { - "test": "./scripts/check-proto-compabitility.sh" + "test": "./scripts/check-proto-compabitility.sh", + "gen:test": "node protoc-gen-bitwise/test/generator.test.js" }, "devDependencies": { "@protobuf-ts/protoc": "^2.8.1", @@ -28,9 +29,11 @@ "protobufjs": "7.2.4" }, "files": [ - "proto", + "proto/decentraland", "out-ts", "out-js", - "public" + "public", + "protoc-gen-bitwise/*.js", + "protoc-gen-bitwise/runtime" ] } diff --git a/proto/decentraland/common/options.proto b/proto/decentraland/common/options.proto new file mode 100644 index 00000000..8e4a3517 --- /dev/null +++ b/proto/decentraland/common/options.proto @@ -0,0 +1,51 @@ +syntax = "proto3"; + +package decentraland.common; + +import "google/protobuf/descriptor.proto"; + +// Options for quantizing a float value stored as a uint32 field. +// The float is clamped to [min, max] and uniformly quantized to N bits; +// protobuf encodes the resulting uint32 as a varint on the wire. +// The generator emits a float {Name}Quantized accessor in the partial class. +message QuantizedFloatOptions { + float min = 1; + float max = 2; + uint32 bits = 3; +} + +// Options for quantizing a signed float with a power-law curve, stored as a uint32 field. +// The value is split into an (bits-1)-bit linear unorm magnitude `u in [0, 1]` plus a sign, +// and reconstructed as `value = sign * max * u^pow`. Because `u = 0` maps to exactly `0`, zero is +// representable (unlike the linear quantizer, whose codes straddle zero), and `pow > 1` concentrates +// resolution near zero — useful for fields like velocity that need fine detail at low magnitudes and +// only coarse detail near the extremes. The range is symmetric ([-max, max]); there is no `min`. +// The encoded layout puts the magnitude in the high bits and the sign in the LSB +// (`(magnitude << 1) | sign`), so the varint cost tracks magnitude, not direction: a small `|value|` +// of either sign stays in one varint byte. Zero canonicalizes to `0`, so proto3 omits a stopped field. +// The generator emits a float {Name}Quantized accessor, same as the linear option. +message QuantizedPowerFloatOptions { + float max = 1; + float pow = 2; + uint32 bits = 3; +} + +// Options for bit-packing an integer field into fewer than 32 bits. +message BitPackedOptions { + uint32 bits = 1; +} + +extend google.protobuf.FieldOptions { + // Apply to uint32 fields to enable quantized float encoding. + // Example: uint32 dx = 1 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + QuantizedFloatOptions quantized = 50001; + + // Apply to uint32 fields to pack into fewer bits. + // Example: uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; + BitPackedOptions bit_packed = 50002; + + // Apply to uint32 fields to enable power-law quantized float encoding (sign + magnitude curve). + // A field carries either `quantized` or `quantized_power`, not both. + // Example: uint32 velocity_x = 9 [(decentraland.common.quantized_power) = { max: 50.0, pow: 2.0, bits: 8 }]; + QuantizedPowerFloatOptions quantized_power = 50003; +} diff --git a/proto/decentraland/common/quantization_example.proto b/proto/decentraland/common/quantization_example.proto new file mode 100644 index 00000000..e8092f04 --- /dev/null +++ b/proto/decentraland/common/quantization_example.proto @@ -0,0 +1,164 @@ +syntax = "proto3"; + +package decentraland.common; + +import "decentraland/common/options.proto"; + +// High-frequency player state messages sent on Channel 1 (unreliable sequenced). +// +// Every message below uses the protoc-gen-bitwise annotations to minimise +// wire size. Wire costs are listed per-message so the trade-offs are clear. +// +// Annotation cheat-sheet: +// [(decentraland.common.quantized) = { min: F, max: F, bits: N }] +// → stores a float as a uint32 quantized to N bits over [min, max]. +// → protobuf encodes the uint32 as a varint: values ≤ 2^14-1 cost 2 bytes, +// values ≤ 2^21-1 cost 3 bytes; the generated partial class adds a +// float {Name}Quantized accessor that encodes/decodes transparently. +// [(decentraland.common.quantized_power) = { max: F, pow: F, bits: N }] +// → stores a signed float over [-max, max] as an (N-1)-bit linear unorm +// magnitude (high bits) plus a sign (LSB), reconstructed as sign·max·u^pow. +// Zero is exact, and pow>1 buys fine resolution near zero at the cost of +// coarse near ±max. Putting the sign in the LSB keeps a small |value| of +// either direction in one varint byte. Same generated float {Name}Quantized +// accessor as the linear option. +// [(decentraland.common.bit_packed) = { bits: N }] +// → documents that this uint32 uses at most N bits; protobuf encodes it +// as a varint (same savings, no generated accessor needed). +// (no annotation) +// → standard protobuf encoding (bool/double/etc. at natural width). +// +// Varint byte costs at worst-case (all bits set): +// ≤ 7 bits → 1 byte (max 127) +// ≤ 14 bits → 2 bytes (max 16 383) +// ≤ 21 bits → 3 bytes (max 2 097 151) +// Proto3 omits fields equal to their default value (0), so average cost is lower. + +// --------------------------------------------------------------------------- +// PositionDelta — Δ position relative to last acknowledged full snapshot. +// +// Sent every client tick (~10 Hz) on Channel 1. +// +// Field | Type | Range | Q bits | Wire worst-case +// ------------|--------|----------------|--------|----------------------- +// dx | uint32 | [-100, 100] | 16 | tag 1B + varint 3B = 4B +// dy | uint32 | [-100, 100] | 16 | tag 1B + varint 3B = 4B +// dz | uint32 | [-100, 100] | 16 | tag 1B + varint 3B = 4B +// entity_id | uint32 | [0, 1 048 575] | 20 | tag 1B + varint 3B = 4B +// sequence | uint32 | [0, 4 095] | 12 | tag 1B + varint 2B = 3B +// --------------------------------------------------------------------------- +// Worst-case: 19 B (vs. 20 B raw: 3×float + 2×uint32) +// Step: dx/dy/dz ≈ 0.003 units +// --------------------------------------------------------------------------- +message PositionDelta { + uint32 dx = 1 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dy = 2 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dz = 3 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; + uint32 sequence = 5 [(decentraland.common.bit_packed) = { bits: 12 }]; +} + +// --------------------------------------------------------------------------- +// PlayerInput — client input snapshot for server-side reconciliation. +// +// Sent every client frame (~30 Hz) on Channel 1. +// +// Field | Type | Range | Q bits | Wire worst-case +// ------------|--------|----------------|--------|----------------------- +// move_x | uint32 | [-1, 1] | 8 | tag 1B + varint 2B = 3B +// move_z | uint32 | [-1, 1] | 8 | tag 1B + varint 2B = 3B +// yaw | uint32 | [-180, 180] | 12 | tag 1B + varint 2B = 3B +// buttons | uint32 | bitmask | 8 | tag 1B + varint 2B = 3B +// sequence | uint32 | [0, 4 095] | 12 | tag 1B + varint 2B = 3B +// --------------------------------------------------------------------------- +// Worst-case: 15 B (vs. 20 B raw: 3×float + 2×uint32) +// Step: move_x/move_z ≈ 0.008; yaw ≈ 0.088° +// --------------------------------------------------------------------------- +message PlayerInput { + // Normalised joystick axes in [-1, 1]. + uint32 move_x = 1 [(decentraland.common.quantized) = { min: -1.0, max: 1.0, bits: 8 }]; + uint32 move_z = 2 [(decentraland.common.quantized) = { min: -1.0, max: 1.0, bits: 8 }]; + + // Horizontal look direction in degrees. + uint32 yaw = 3 [(decentraland.common.quantized) = { min: -180.0, max: 180.0, bits: 12 }]; + + // Bitmask of active buttons (see ButtonFlags below). + uint32 buttons = 4 [(decentraland.common.bit_packed) = { bits: 8 }]; + + // Rolling input sequence number used by the server for reconciliation. + uint32 sequence = 5 [(decentraland.common.bit_packed) = { bits: 12 }]; +} + +// --------------------------------------------------------------------------- +// VelocityState — per-axis velocity, demonstrating the power-law quantizer. +// +// Velocity needs an exact zero (a stopped avatar must read 0, not a quantization +// residual, or it drifts) and fine resolution at locomotion speeds, but only +// coarse resolution near the ±50 m/s extremes. The linear quantizer can give +// none of these from 8 bits: its codes straddle zero (±0.196) and spend uniform +// resolution on speeds that never occur. quantized_power solves all three. +// +// Field | Type | Range | pow | bits | Wire worst-case +// -------|--------|------------|-----|------|----------------------- +// vx | uint32 | [-50, 50] | 2 | 8 | tag 1B + varint 1–2B = 2–3B +// vy | uint32 | [-50, 50] | 2 | 8 | tag 1B + varint 1–2B = 2–3B +// vz | uint32 | [-50, 50] | 2 | 8 | tag 1B + varint 1–2B = 2–3B +// --------------------------------------------------------------------------- +// Step: ≈ 0.003 m/s near zero, ≈ 0.78 m/s near ±50 (vs. 0.392 uniform linear) +// Wire: sign in the LSB, so |v| ≲ 12.5 m/s (magnitude code ≤ 63) costs a single +// varint byte regardless of direction; only faster axes spill to 2 bytes. +// --------------------------------------------------------------------------- +message VelocityState { + uint32 vx = 1 [(decentraland.common.quantized_power) = { max: 50.0, pow: 2.0, bits: 8 }]; + uint32 vy = 2 [(decentraland.common.quantized_power) = { max: 50.0, pow: 2.0, bits: 8 }]; + uint32 vz = 3 [(decentraland.common.quantized_power) = { max: 50.0, pow: 2.0, bits: 8 }]; +} + +// Bitmask values for PlayerInput.buttons. +enum ButtonFlags { + BF_NONE = 0; + BF_JUMP = 1; // bit 0 + BF_SPRINT = 2; // bit 1 + BF_INTERACT = 4; // bit 2 + BF_EMOTE = 8; // bit 3 + BF_CROUCH = 16; // bit 4 + // bits 5-7 reserved +} + +// --------------------------------------------------------------------------- +// AvatarStateSnapshot — full authoritative state, sent on Channel 0 (reliable) +// or on resync requests. Demonstrates wider ranges and mixed encodings. +// +// Field | Type | Range | Q bits | Wire worst-case +// -----------------|--------|------------------|--------|----------------------- +// x | uint32 | [-4096, 4096] | 16 | tag 1B + varint 3B = 4B +// y | uint32 | [-256, 256] | 14 | tag 1B + varint 2B = 3B +// z | uint32 | [-4096, 4096] | 16 | tag 1B + varint 3B = 4B +// pitch | uint32 | [-90, 90] | 10 | tag 1B + varint 2B = 3B +// yaw | uint32 | [-180, 180] | 12 | tag 1B + varint 2B = 3B +// entity_id | uint32 | [0, 1 048 575] | 20 | tag 1B + varint 3B = 4B +// animation_state | uint32 | [0, 63] | 6 | tag 1B + varint 1B = 2B +// is_grounded | bool | — | — | tag 1B + varint 1B = 2B +// timestamp | double | — | — | tag 1B + fixed64 8B = 9B +// --------------------------------------------------------------------------- +// Worst-case: 34 B (vs. 45 B raw: 5×float + 2×uint32 + bool + double) +// Step: x/z ≈ 0.125 units; y ≈ 0.031 units; pitch ≈ 0.176°; yaw ≈ 0.088° +// --------------------------------------------------------------------------- +message AvatarStateSnapshot { + // World-space position. + uint32 x = 1 [(decentraland.common.quantized) = { min: -4096.0, max: 4096.0, bits: 16 }]; + uint32 y = 2 [(decentraland.common.quantized) = { min: -256.0, max: 256.0, bits: 14 }]; + uint32 z = 3 [(decentraland.common.quantized) = { min: -4096.0, max: 4096.0, bits: 16 }]; + + // View angles. + uint32 pitch = 4 [(decentraland.common.quantized) = { min: -90.0, max: 90.0, bits: 10 }]; + uint32 yaw = 5 [(decentraland.common.quantized) = { min: -180.0, max: 180.0, bits: 12 }]; + + // Identity and animation state. + uint32 entity_id = 6 [(decentraland.common.bit_packed) = { bits: 20 }]; + uint32 animation_state = 7 [(decentraland.common.bit_packed) = { bits: 6 }]; + + // Un-annotated fields — encoded at their natural width. + bool is_grounded = 8; + double timestamp = 9; // server epoch milliseconds +} diff --git a/proto/decentraland/pulse/pulse_client.proto b/proto/decentraland/pulse/pulse_client.proto new file mode 100644 index 00000000..09760dab --- /dev/null +++ b/proto/decentraland/pulse/pulse_client.proto @@ -0,0 +1,79 @@ +syntax = "proto3"; + +package decentraland.pulse; + +import "decentraland/pulse/pulse_shared.proto"; +import "decentraland/common/options.proto"; + +message HandshakeRequest { + bytes auth_chain = 1; + int32 profile_version = 2; + optional PlayerInitialState initial_state = 3; +} + +// Describes the initial state of the player if (re-)connected in the middle of the session +message PlayerInitialState { + PlayerState state = 1; + optional string emote_id = 2; + optional uint32 emote_duration_ms = 3; + // Indicates how many milliseconds ago the emote was started + optional uint32 emote_start_offset_ms = 4; + // Non-empty realm identifier. Rejected if empty. + string realm = 5; + optional int32 emote_mask = 6; +} + +// Similarly to the LiveKit pipeline, a peer announces the version of its profile but +// it only does it when it's changed as it's sent reliably and stored on the server for other peers +message ProfileVersionAnnouncement { + int32 version = 1; +} + +// Since the server doesn't simulate the scenes state, it trusts the values from the client +message PlayerStateInput { + PlayerState state = 1; +} + +// Client sends resync request if it has a gap in the known sequences and, thus, can't apply a delta. +// It's sent reliably to prevent further desynchronization +message ResyncRequest { + uint32 subject_id = 1; + // highest seq the client actually has for this subject + uint32 known_seq = 2; +} + +// Client → Server. Client requests to start an emote. +message EmoteStart { + string emote_id = 1; + optional uint32 duration_ms = 2; + PlayerState player_state = 3; + optional int32 mask = 4; +} + +// Client → Server. Client requests to stop a looping emote. +// One-shot emotes are terminated by the server timer. +message EmoteStop { +} + +// Client → Server. Also announces the peer's realm; peers in different realms never see each +// other. Must be the first gameplay message after handshake. Same-realm re-teleports are valid. +message TeleportRequest { + int32 parcel_index = 1; + uint32 position_x = 2 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + uint32 position_y = 3 [(decentraland.common.quantized) = { min: 0, max: 200, bits: 13 }]; + uint32 position_z = 4 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + // Non-empty realm identifier. Rejected if empty. + string realm = 5; +} + +message ClientMessage { + oneof message { + HandshakeRequest handshake = 1; + PlayerStateInput input = 2; + ResyncRequest resync = 3; + ProfileVersionAnnouncement profile_announcement = 4; + EmoteStart emote_start = 5; + EmoteStop emote_stop = 6; + TeleportRequest teleport = 7; + } +} diff --git a/proto/decentraland/pulse/pulse_server.proto b/proto/decentraland/pulse/pulse_server.proto new file mode 100644 index 00000000..9957ac32 --- /dev/null +++ b/proto/decentraland/pulse/pulse_server.proto @@ -0,0 +1,144 @@ +syntax = "proto3"; + +package decentraland.pulse; + +import "decentraland/common/options.proto"; +import "decentraland/pulse/pulse_shared.proto"; + +message HandshakeResponse { + bool success = 1; + optional string error = 2; +} + +message PlayerProfileVersionsAnnounced { + uint32 subject_id = 1; + int32 version = 2; +} + +message PlayerStateDeltaTier0 { + uint32 subject_id = 1; + + // Between two consecutive server simulation ticks, the subject may have sent multiple inputs, each incrementing Seq by 1. So + // the delta's NewSeq will naturally jump by more than 1 compared to the previous delta — even with zero packet loss. + // Without the "BaselineSeq" the client has no way to detect the package loss + uint32 baseline_seq = 2; + uint32 new_seq = 3; + uint32 server_tick = 4; + + // While the player doesn't cross the parcel, this field is omitted from diff + optional int32 parcel_index = 5; + + // X position inside the parcel + optional uint32 position_x = 6 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + + // Y position + optional uint32 position_y = 7 [(decentraland.common.quantized) = { min: 0, max: 200, bits: 13 }]; + + // Z position inside the parcel + optional uint32 position_z = 8 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + + // Power-law quantized: a sign bit + 7-bit magnitude curve over [-50, 50]. Zero is exactly + // representable (a stopped peer reports an exact 0 instead of the linear quantizer's ±0.196 + // residual that drove foreign-avatar drift), and pow=2 concentrates resolution at the low speeds + // where avatars actually move, leaving only the visually-irrelevant extremes coarse. + optional uint32 velocity_x = 9 [(decentraland.common.quantized_power) = { max: 50, pow: 2, bits: 8 }]; + optional uint32 velocity_y = 10 [(decentraland.common.quantized_power) = { max: 50, pow: 2, bits: 8 }]; + optional uint32 velocity_z = 11 [(decentraland.common.quantized_power) = { max: 50, pow: 2, bits: 8 }]; + + optional uint32 rotation_y = 12 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + optional uint32 movement_blend = 13 [(decentraland.common.quantized) = { min: 0, max: 3, bits: 5 }]; + optional uint32 slide_blend = 14 [(decentraland.common.quantized) = { min: 0, max: 1, bits: 4 }]; + optional uint32 head_yaw = 15 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + optional uint32 head_pitch = 16 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + + optional uint32 state_flags = 17; + optional GlideState glide_state = 18; + + optional int32 jump_count = 19; + + // Absolute world hit position the player is pointing at. + // Only meaningful when POINTING_AT is set in `state_flags`. + // + // X/Z use 17 bits over the world span (~±3000m, covers GenesisCity + border + + // raycast cutoff) which yields ~0.046m steps — at least as fine as the player's + // own parcel_index + position_x/z combined precision (0.0625m), so no separate + // parcel index is needed for point-at. + // + // Y uses 7 bits over the player altitude range (matches position_y), step ~1.6m. + // Point At is not supposed to change frequently so the wire overhead should be minimal + optional uint32 point_at_x = 20 [(decentraland.common.quantized) = { min: -3000.0, max: 3000.0, bits: 17 }]; + optional uint32 point_at_y = 21 [(decentraland.common.quantized) = { min: 0.0, max: 200.0, bits: 7 }]; + optional uint32 point_at_z = 22 [(decentraland.common.quantized) = { min: -3000.0, max: 3000.0, bits: 17 }]; +} + +// Full State is sent to the client when it is out of sync, and can't recover with a diff only +message PlayerStateFull { + uint32 subject_id = 1; + uint32 sequence = 2; + uint32 server_tick = 3; + PlayerState state = 4; +} + +// Notification to the client, that a peer has joined, it can mean connection or entering the area of interest, it's up to the server to decide +message PlayerJoined { + string user_id = 1; + int32 profile_version = 2; + PlayerStateFull state = 3; + string realm = 4; +} + +// Notification to the client, that a peer has left, it can mean disconnection or leaving the area of interest +message PlayerLeft { + uint32 subject_id = 1; +} + +enum EmoteStopReason { + COMPLETED = 0; // one-shot timer expired on the server + CANCELLED = 1; // client sent EmoteStop (looping emotes) +} + +// Server → All Observers +// Full player state is piggybacked to ensure the emote is played in the right place. +// Observers use server_tick to scrub animation forward by transit latency. +message EmoteStarted { + uint32 subject_id = 1; + uint32 sequence = 2; + uint32 server_tick = 3; + string emote_id = 4; + PlayerState player_state = 5; + optional int32 mask = 6; +} + +// Server → All Observers +// Client resumes MovementInput only after receiving this. +// Carries full PlayerState so the client can snap to the correct position on resume. +message EmoteStopped { + uint32 subject_id = 1; + uint32 server_tick = 2; + EmoteStopReason reason = 3; + uint32 sequence = 4; + PlayerState player_state = 5; +} + +// Server → All Observers +message TeleportPerformed { + uint32 subject_id = 1; + uint32 sequence = 2; + uint32 server_tick = 3; + PlayerState state = 4; + string realm = 5; +} + +message ServerMessage { + oneof message { + HandshakeResponse handshake = 1; + PlayerStateFull player_state_full = 2; + PlayerStateDeltaTier0 player_state_delta = 3; + PlayerJoined player_joined = 4; + PlayerLeft player_left = 5; + PlayerProfileVersionsAnnounced player_profile_version_announced = 6; + EmoteStarted emote_started = 7; + EmoteStopped emote_stopped = 8; + TeleportPerformed teleported = 9; + } +} diff --git a/proto/decentraland/pulse/pulse_shared.proto b/proto/decentraland/pulse/pulse_shared.proto new file mode 100644 index 00000000..cc027f3c --- /dev/null +++ b/proto/decentraland/pulse/pulse_shared.proto @@ -0,0 +1,57 @@ +syntax = "proto3"; + +package decentraland.pulse; + +import "decentraland/common/options.proto"; + +enum PlayerAnimationFlags { + NONE = 0; + GROUNDED = 1; + LONG_JUMP = 2; + LONG_FALL = 4; + FALLING = 8; + STUNNED = 16; + HEAD_YAW = 32; + HEAD_PITCH = 64; + POINTING_AT = 128; +} + +enum GlideState { + PROP_CLOSED = 0; + OPENING_PROP = 1; + GLIDING = 2; + CLOSING_PROP = 3; +} + +// Quantized identically to PlayerStateDeltaTier0 so a full state and a stream of deltas land on the same grid. +message PlayerState { + int32 parcel_index = 1; + + // Position inside the parcel (X/Z) and world altitude (Y). + uint32 position_x = 2 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + uint32 position_y = 3 [(decentraland.common.quantized) = { min: 0, max: 200, bits: 13 }]; + uint32 position_z = 4 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + + uint32 velocity_x = 5 [(decentraland.common.quantized_power) = { max: 50, pow: 2, bits: 8 }]; + uint32 velocity_y = 6 [(decentraland.common.quantized_power) = { max: 50, pow: 2, bits: 8 }]; + uint32 velocity_z = 7 [(decentraland.common.quantized_power) = { max: 50, pow: 2, bits: 8 }]; + + uint32 rotation_y = 8 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + + uint32 movement_blend = 9 [(decentraland.common.quantized) = { min: 0, max: 3, bits: 5 }]; + uint32 slide_blend = 10 [(decentraland.common.quantized) = { min: 0, max: 1, bits: 4 }]; + + optional uint32 head_yaw = 11 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + optional uint32 head_pitch = 12 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + + uint32 state_flags = 13; + GlideState glide_state = 14; + + int32 jump_count = 15; + + // Absolute world hit position the player is pointing at. + // Only meaningful when POINTING_AT is set in `state_flags`. + optional uint32 point_at_x = 16 [(decentraland.common.quantized) = { min: -3000.0, max: 3000.0, bits: 17 }]; + optional uint32 point_at_y = 17 [(decentraland.common.quantized) = { min: 0.0, max: 200.0, bits: 7 }]; + optional uint32 point_at_z = 18 [(decentraland.common.quantized) = { min: -3000.0, max: 3000.0, bits: 17 }]; +} diff --git a/protoc-gen-bitwise/generator_csharp.js b/protoc-gen-bitwise/generator_csharp.js new file mode 100644 index 00000000..4cb3a2ee --- /dev/null +++ b/protoc-gen-bitwise/generator_csharp.js @@ -0,0 +1,248 @@ +'use strict' + +/** + * C# code generator for the protoc-gen-bitwise plugin. + * + * For every proto message that contains at least one uint32 field annotated + * with [(quantized)], this module emits a C# partial class that adds a computed + * float property named {FieldName}Quantized. The getter decodes the stored + * uint32 to a float; the setter encodes a float back to a uint32. Standard + * protobuf handles serialization of the uint32 wire field; this class adds a + * typed float accessor on top. + * + * Only uint32 fields are supported. bit_packed and unannotated fields are + * passed through without generating any accessor. + * + * Port of the original generator_csharp.py — output is intended to be + * byte-for-byte identical. + */ + +const { getFieldOptions } = require('./options') + +// FieldDescriptorProto type/label constants. +const TYPE_UINT32 = 13 +const LABEL_REPEATED = 3 + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** Mirrors Python str.capitalize(): upper-first, lowercase the rest. */ +function capitalize(word) { + if (word.length === 0) return '' + return word[0].toUpperCase() + word.slice(1).toLowerCase() +} + +/** position_x -> PositionX */ +function snakeToPascal(name) { + return name.split('_').map(capitalize).join('') +} + +/** decentraland.kernel.comms.v3 -> Decentraland.Kernel.Comms.V3 */ +function packageToNamespace(pkg) { + if (!pkg) return 'Generated' + return pkg.split('.').map(capitalize).join('.') +} + +/** + * Format a number with C printf %g semantics at `precision` significant digits: + * shortest of fixed/scientific, with trailing zeros and a trailing dot removed. + * Reproduces Python's `f'{value:.{precision}g}'`. + */ +function formatG(value, precision) { + if (precision <= 0) precision = 1 + if (value === 0) return '0' + if (!Number.isFinite(value)) return value > 0 ? 'inf' : 'nan' + + const negative = value < 0 + const v = Math.abs(value) + + // Correctly-rounded scientific form yields the decimal exponent X (handling + // carry such as 9.999 -> 1.0e+1). + const sci = v.toExponential(precision - 1) + const eIdx = sci.indexOf('e') + const X = parseInt(sci.slice(eIdx + 1), 10) + + let result + if (X >= -4 && X < precision) { + // Fixed notation with (precision - 1 - X) fraction digits. + const fractionDigits = precision - 1 - X + result = v.toFixed(fractionDigits >= 0 ? fractionDigits : 0) + if (result.indexOf('.') !== -1) { + result = result.replace(/0+$/, '').replace(/\.$/, '') + } + } else { + // Scientific notation; printf prints at least two exponent digits. + let mantissa = sci.slice(0, eIdx) + if (mantissa.indexOf('.') !== -1) { + mantissa = mantissa.replace(/0+$/, '').replace(/\.$/, '') + } + const expSign = X < 0 ? '-' : '+' + let expDigits = String(Math.abs(X)) + if (expDigits.length < 2) expDigits = '0' + expDigits + result = mantissa + 'e' + expSign + expDigits + } + + return (negative ? '-' : '') + result +} + +/** Format a value as a C# float literal (e.g. -100.0f). */ +function formatFloat(value) { + let text = formatG(value, 8) + if (text.indexOf('.') === -1 && text.indexOf('e') === -1 && text.indexOf('E') === -1) { + text += '.0' + } + return text + 'f' +} + +/** Format a quantization step size for a doc comment (e.g. "≈ 0.003"). */ +function formatStep(step) { + return '≈ ' + formatG(step, 6) +} + +// --------------------------------------------------------------------------- +// Per-message code generation +// --------------------------------------------------------------------------- + +/** + * Generate a C# partial class for a proto message, or null if it has no + * quantized uint32 fields. Returns an array of lines (no trailing newline). + */ +function generateMessage(msgProto, indent) { + const i = indent || ' ' + const props = [] + + for (const field of msgProto.field) { + // Repeated/map fields are not supported. + if (field.label === LABEL_REPEATED) continue + // Only uint32 fields are candidates for quantized accessors. + if (field.type !== TYPE_UINT32) continue + + const { quantized, quantizedPower } = getFieldOptions(field.optionsRaw) + const propName = snakeToPascal(field.name) + + let doc, getExpr, setExpr, step, bits + if (quantized !== null) { + const mn = formatFloat(quantized.min) + const mx = formatFloat(quantized.max) + bits = quantized.bits + // Uniform quantizer — the step is constant across the whole range. + step = (quantized.max - quantized.min) / ((1 << bits) - 1) + doc = `Range [${mn}, ${mx}], ${bits} bits, step ${formatStep(step)}.` + getExpr = `Quantize.Decode(${propName}, ${mn}, ${mx}, ${bits})` + setExpr = `Quantize.Encode(value, ${mn}, ${mx}, ${bits})` + } else if (quantizedPower !== null) { + const mx = formatFloat(quantizedPower.max) + const pw = formatFloat(quantizedPower.pow) + bits = quantizedPower.bits + // Power curve is non-uniform: the finest step sits next to zero (first magnitude code), + // the COARSEST at the top of the range. The coarsest step upper-bounds the error for any + // value, so that's what the exposed {Name}QuantizedStep const carries (safe as a tolerance). + const magSteps = (1 << (bits - 1)) - 1 + const nearZeroStep = quantizedPower.max * Math.pow(1 / magSteps, quantizedPower.pow) + step = quantizedPower.max * (1 - Math.pow((magSteps - 1) / magSteps, quantizedPower.pow)) + doc = + `Range [-${mx}, ${mx}], power ${pw}, ${bits} bits ` + + `(sign + ${bits - 1}-bit magnitude), near-zero step ${formatStep(nearZeroStep)}.` + getExpr = `Quantize.DecodePower(${propName}, ${mx}, ${pw}, ${bits})` + setExpr = `Quantize.EncodePower(value, ${mx}, ${pw}, ${bits})` + } else { + continue + } + + // Highest code the encoder can emit for this field. Both the linear quantizer (top code + // `2^bits - 1`) and the power quantizer (`(magnitude << 1) | sign` with an `bits-1`-bit + // magnitude, so top code `((2^(bits-1)-1) << 1) | 1 == 2^bits - 1`) share this bound. + const maxCode = 2 ** bits - 1 + + props.push({ propName, doc, getExpr, setExpr, step, maxCode }) + } + + if (props.length === 0) return null + + const lines = [] + lines.push(`public partial class ${msgProto.name}`) + lines.push('{') + + // Decode on every get, encode on every set — no backing cache. The raw uint32 property is the + // single source of truth, so the float accessor can never disagree with the code on the wire + // (get-after-set returns the on-grid value a receiver decodes) and there is no stale-cache + // hazard when the raw field is mutated directly. Decode is a multiply-add; only power fields + // pay a MathF.Pow. + for (const { propName, doc, getExpr, setExpr, step } of props) { + lines.push(`${i}/// Coarsest quantization step of . Safe as an equality tolerance.`) + lines.push(`${i}public const float ${propName}QuantizedStep = ${formatFloat(step)};`) + lines.push(`${i}/// Float accessor for . ${doc}`) + lines.push(`${i}public float ${propName}Quantized`) + lines.push(`${i}{`) + lines.push(`${i}${i}get => ${getExpr};`) + lines.push(`${i}${i}set => ${propName} = ${setExpr};`) + lines.push(`${i}}`) + lines.push('') + } + + lines.push(`${i}/// `) + lines.push(`${i}/// True when every quantized field holds a wire code within its declared bit width`) + lines.push(`${i}/// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger`) + lines.push(`${i}/// value is a malformed/hostile message: decoding it would land far outside the field's`) + lines.push(`${i}/// [min, max] and, since the server relays raw codes verbatim, poison every observer.`) + lines.push(`${i}/// Reject before storing or relaying. Pure integer comparison — no decode.`) + lines.push(`${i}/// `) + lines.push(`${i}public bool AreQuantizedFieldsInRange() =>`) + props.forEach(({ propName, maxCode }, idx) => { + const prefix = idx === 0 ? '' : '&& ' + const suffix = idx === props.length - 1 ? ';' : '' + lines.push(`${i}${i}${prefix}${propName} <= ${maxCode}u${suffix}`) + }) + + lines.push('}') + return lines +} + +// --------------------------------------------------------------------------- +// Per-file code generation (public entry point) +// --------------------------------------------------------------------------- + +/** + * Generate a C# source file for a FileDescriptorProto, or null if the file + * contains no quantized uint32 fields. + * @returns {{name: string, content: string} | null} + */ +function generateCsharp(fileProto) { + const namespace = packageToNamespace(fileProto.package) + + const protoFile = fileProto.name.split('/').pop() + const stem = snakeToPascal(protoFile.replace('.proto', '')) + const outName = `${stem}.Bitwise.cs` + + const header = [ + '// ', + '// Generated by protoc-gen-bitwise. DO NOT EDIT.', + `// Source: ${fileProto.name}`, + '// ', + '', + 'using Decentraland.Networking.Bitwise;', + '', + `namespace ${namespace}`, + '{', + ] + const footer = ['', `} // namespace ${namespace}`] + + const body = [] + for (const msg of fileProto.messageType) { + const msgLines = generateMessage(msg) + if (msgLines === null) continue + // Indent each line by 4 spaces (inside the namespace block). + for (const line of msgLines) { + body.push(line ? ' ' + line : '') + } + body.push('') + } + + if (body.length === 0) return null + + const content = header.concat(body, footer).join('\n') + '\n' + return { name: outName, content } +} + +module.exports = { generateCsharp } diff --git a/protoc-gen-bitwise/options.js b/protoc-gen-bitwise/options.js new file mode 100644 index 00000000..e4a788cc --- /dev/null +++ b/protoc-gen-bitwise/options.js @@ -0,0 +1,139 @@ +'use strict' + +/** + * Parser for the custom bitwise field options defined in options.proto. + * + * The descriptor decoder hands us the raw serialized FieldOptions bytes (it + * declares `options` as opaque bytes). We walk those bytes looking for the + * custom extension field numbers — protobuf preserves unknown/unregistered + * extension bytes, so they are always present even though no runtime here knows + * the extension schema. This mirrors the original options_pb2.py. + * + * Wire format: tag = (field_number << 3) | wire_type + * wire_type 0 = varint, 1 = 64-bit, 2 = length-delimited, 5 = 32-bit + */ + +const { readVarint, skipField } = require('./wire') + +// Extension field numbers as defined in options.proto. +const QUANTIZED_FIELD_NUMBER = 50001 +const BIT_PACKED_FIELD_NUMBER = 50002 +const QUANTIZED_POWER_FIELD_NUMBER = 50003 + +/** Parse a serialized QuantizedFloatOptions message: { min, max, bits }. */ +function parseQuantized(data) { + const opts = { min: 0.0, max: 0.0, bits: 0 } + let pos = 0 + while (pos < data.length) { + let tag + ;[tag, pos] = readVarint(data, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 5) { + // min (float) + opts.min = data.readFloatLE(pos) + pos += 4 + } else if (fieldNum === 2 && wireType === 5) { + // max (float) + opts.max = data.readFloatLE(pos) + pos += 4 + } else if (fieldNum === 3 && wireType === 0) { + // bits (uint32) + ;[opts.bits, pos] = readVarint(data, pos) + } else { + pos = skipField(data, pos, wireType) + } + } + return opts +} + +/** Parse a serialized QuantizedPowerFloatOptions message: { max, pow, bits }. */ +function parseQuantizedPower(data) { + const opts = { max: 0.0, pow: 0.0, bits: 0 } + let pos = 0 + while (pos < data.length) { + let tag + ;[tag, pos] = readVarint(data, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 5) { + // max (float) + opts.max = data.readFloatLE(pos) + pos += 4 + } else if (fieldNum === 2 && wireType === 5) { + // pow (float) + opts.pow = data.readFloatLE(pos) + pos += 4 + } else if (fieldNum === 3 && wireType === 0) { + // bits (uint32) + ;[opts.bits, pos] = readVarint(data, pos) + } else { + pos = skipField(data, pos, wireType) + } + } + return opts +} + +/** Parse a serialized BitPackedOptions message: { bits }. */ +function parseBitPacked(data) { + const opts = { bits: 0 } + let pos = 0 + while (pos < data.length) { + let tag + ;[tag, pos] = readVarint(data, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 0) { + // bits (uint32) + ;[opts.bits, pos] = readVarint(data, pos) + } else { + pos = skipField(data, pos, wireType) + } + } + return opts +} + +/** + * Extract custom bitwise options from raw FieldOptions bytes. + * + * @param {Buffer|null} optionsRaw serialized FieldOptions, or null when unset. + * @returns {{quantized: object|null, bitPacked: object|null, quantizedPower: object|null}} + */ +function getFieldOptions(optionsRaw) { + if (!optionsRaw || optionsRaw.length === 0) { + return { quantized: null, bitPacked: null, quantizedPower: null } + } + + let quantized = null + let bitPacked = null + let quantizedPower = null + let pos = 0 + + while (pos < optionsRaw.length) { + let tag + ;[tag, pos] = readVarint(optionsRaw, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + + if (wireType === 2) { + let len + ;[len, pos] = readVarint(optionsRaw, pos) + const valueBytes = optionsRaw.subarray(pos, pos + len) + pos += len + if (fieldNum === QUANTIZED_FIELD_NUMBER) { + quantized = parseQuantized(valueBytes) + } else if (fieldNum === BIT_PACKED_FIELD_NUMBER) { + bitPacked = parseBitPacked(valueBytes) + } else if (fieldNum === QUANTIZED_POWER_FIELD_NUMBER) { + quantizedPower = parseQuantizedPower(valueBytes) + } + // else: unknown length-delimited field — already consumed. + } else { + pos = skipField(optionsRaw, pos, wireType) + } + } + + return { quantized, bitPacked, quantizedPower } +} + +module.exports = { getFieldOptions } diff --git a/protoc-gen-bitwise/plugin.js b/protoc-gen-bitwise/plugin.js new file mode 100755 index 00000000..b08c6a12 --- /dev/null +++ b/protoc-gen-bitwise/plugin.js @@ -0,0 +1,87 @@ +#!/usr/bin/env node +'use strict' + +/** + * protoc-gen-bitwise — protoc plugin that generates C# bitwise serialization code. + * + * Protocol: + * 1. protoc writes a serialised CodeGeneratorRequest to this process's stdin. + * 2. This plugin reads it, generates C# partial classes with float accessor + * properties for every message that carries [(quantized)] field annotations. + * 3. A serialised CodeGeneratorResponse is written to stdout. + * + * Implemented in plain Node with a self-contained protobuf wire codec (see + * wire.js) so it runs with only `node` on PATH — no npm install required, even + * when invoked directly from a sibling checkout. + * + * Usage (from project root): + * protoc \ + * --proto_path=proto \ + * --bitwise_out=generated/ \ + * --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.js \ + * proto/my_messages.proto + * + * On Windows, protoc needs an executable wrapper, e.g. a .cmd that runs: + * node "\protoc-gen-bitwise\plugin.js" %* + */ + +const { decodeRequest, encodeResponse } = require('./wire') +const { generateCsharp } = require('./generator_csharp') + +// CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL — advertised so protoc +// does not reject the plugin when the schema uses proto3 `optional` fields. +const FEATURE_PROTO3_OPTIONAL = 1 + +function readStdin() { + return new Promise((resolve, reject) => { + const chunks = [] + process.stdin.on('data', (chunk) => chunks.push(chunk)) + process.stdin.on('end', () => resolve(Buffer.concat(chunks))) + process.stdin.on('error', reject) + }) +} + +async function main() { + const requestBytes = await readStdin() + const request = decodeRequest(requestBytes) + + // Lookup map for all file descriptors (parity with the Python plugin). + const fileByName = new Map() + for (const file of request.protoFile) fileByName.set(file.name, file) + + const files = [] + let error = null + + for (const fileName of request.fileToGenerate) { + // Skip the options definition file itself — it has no messages to generate. + if (fileName === 'decentraland/common/options.proto') continue + + const fileProto = fileByName.get(fileName) + if (!fileProto) continue + + try { + const generated = generateCsharp(fileProto) + if (generated) files.push(generated) + } catch (exc) { + error = `protoc-gen-bitwise: error processing ${fileName}: ${exc && exc.message ? exc.message : exc}` + } + } + + const response = encodeResponse({ + error, + supportedFeatures: FEATURE_PROTO3_OPTIONAL, + files, + }) + + // Let the write flush before the process exits (do not call process.exit()). + process.stdout.write(response) +} + +main().catch((exc) => { + const response = encodeResponse({ + error: `protoc-gen-bitwise: ${exc && exc.stack ? exc.stack : exc}`, + supportedFeatures: FEATURE_PROTO3_OPTIONAL, + files: [], + }) + process.stdout.write(response) +}) diff --git a/protoc-gen-bitwise/runtime/cs/Quantize.cs b/protoc-gen-bitwise/runtime/cs/Quantize.cs new file mode 100644 index 00000000..80323615 --- /dev/null +++ b/protoc-gen-bitwise/runtime/cs/Quantize.cs @@ -0,0 +1,70 @@ +// Decentraland.Networking.Bitwise — Quantize +// Copy this file into your project alongside generated *.Bitwise.cs files. +// ReSharper disable once RedundantUsingDirective + +using System; + +namespace Decentraland.Networking.Bitwise + // ReSharper disable once ArrangeNamespaceBody +{ + /// + /// Static helpers for quantizing float values to/from unsigned integers. + /// Intended for use with protobuf uint32 fields: the integer is transmitted + /// as a protobuf varint, while the float accessor lives in a generated partial class. + /// + public static class Quantize + { + /// + /// Encodes to a quantized . + /// Values outside [, ] are clamped. + /// + public static uint Encode(float value, float min, float max, int bits) + { + var steps = (1u << bits) - 1; + var t = Math.Clamp((value - min) / (max - min), 0f, 1f); + return (uint)MathF.Round(t * steps); + } + + /// Decodes a quantized back to a float. + public static float Decode(uint encoded, float min, float max, int bits) + { + var steps = (1u << bits) - 1; + return (float)encoded / steps * (max - min) + min; + } + + /// + /// Encodes with a power-law curve into an + /// ( - 1)-bit linear unorm magnitude plus a sign bit. The magnitude is + /// (|value| / max)^(1/pow), so the inverse reconstructs + /// sign * max * u^pow. Zero is representable exactly; > 1 + /// concentrates resolution near zero. Magnitudes outside [0, ] are + /// clamped. + /// + /// The encoded layout puts the magnitude in the high bits and the sign in the LSB + /// ((magnitude << 1) | sign). This keeps the varint cost a function of + /// magnitude rather than direction — a small |value| of either sign stays in a + /// single varint byte. Zero canonicalizes to 0 (a zero magnitude never sets the + /// sign bit), so proto3 still omits a stopped field entirely. + /// + /// + public static uint EncodePower(float value, float max, float pow, int bits) + { + var magnitudeSteps = (1u << (bits - 1)) - 1; + var t = Math.Clamp(MathF.Abs(value) / max, 0f, 1f); + var u = MathF.Pow(t, 1f / pow); + var magnitude = (uint)MathF.Round(u * magnitudeSteps); + var sign = value < 0f && magnitude != 0u ? 1u : 0u; + return (magnitude << 1) | sign; + } + + /// Decodes a power-law quantized back to a float (inverse of + /// ). + public static float DecodePower(uint encoded, float max, float pow, int bits) + { + var magnitudeSteps = (1u << (bits - 1)) - 1; + var u = (float)(encoded >> 1) / magnitudeSteps; + var magnitude = max * MathF.Pow(u, pow); + return (encoded & 1u) != 0 ? -magnitude : magnitude; + } + } +} diff --git a/protoc-gen-bitwise/test/generator.test.js b/protoc-gen-bitwise/test/generator.test.js new file mode 100644 index 00000000..d4b26b4b --- /dev/null +++ b/protoc-gen-bitwise/test/generator.test.js @@ -0,0 +1,216 @@ +'use strict' + +/** + * Byte-for-byte parity test for the C# generator. + * + * Builds in-memory FieldDescriptorProto structures (with real serialized + * FieldOptions extension bytes), runs them through the same options parser and + * generator the plugin uses, and asserts the output equals committed golden + * files. This locks down the %g float formatting and the emitted layout without + * needing protoc. + * + * Run with: node protoc-gen-bitwise/test/generator.test.js (or `npm run gen:test`) + */ + +const assert = require('assert') +const fs = require('fs') +const path = require('path') + +const { generateCsharp } = require('../generator_csharp') + +// FieldDescriptorProto.Type +const TYPE_DOUBLE = 1 +const TYPE_BOOL = 8 +const TYPE_UINT32 = 13 +// FieldDescriptorProto.Label +const LABEL_OPTIONAL = 1 + +// --- minimal wire encoders to build serialized FieldOptions extension bytes --- + +function encodeVarint(value) { + const bytes = [] + let v = value + while (v > 0x7f) { + bytes.push((v & 0x7f) | 0x80) + v = Math.floor(v / 128) + } + bytes.push(v & 0x7f) + return Buffer.from(bytes) +} + +function tag(fieldNum, wireType) { + return encodeVarint((fieldNum << 3) | wireType) +} + +function floatLE(value) { + const b = Buffer.alloc(4) + b.writeFloatLE(value, 0) + return b +} + +function lengthDelimited(fieldNum, payload) { + return Buffer.concat([tag(fieldNum, 2), encodeVarint(payload.length), payload]) +} + +// FieldOptions bytes carrying ext 50001 (quantized) = { min, max, bits } +function quantizedOptions(min, max, bits) { + const inner = Buffer.concat([ + tag(1, 5), floatLE(min), + tag(2, 5), floatLE(max), + tag(3, 0), encodeVarint(bits), + ]) + return lengthDelimited(50001, inner) +} + +// FieldOptions bytes carrying ext 50002 (bit_packed) = { bits } +function bitPackedOptions(bits) { + const inner = Buffer.concat([tag(1, 0), encodeVarint(bits)]) + return lengthDelimited(50002, inner) +} + +// FieldOptions bytes carrying ext 50003 (quantized_power) = { max, pow, bits } +function quantizedPowerOptions(max, pow, bits) { + const inner = Buffer.concat([ + tag(1, 5), floatLE(max), + tag(2, 5), floatLE(pow), + tag(3, 0), encodeVarint(bits), + ]) + return lengthDelimited(50003, inner) +} + +function field(name, type, optionsRaw) { + return { name, label: LABEL_OPTIONAL, type, optionsRaw: optionsRaw || null } +} + +function readGolden(name) { + // Git may materialize the goldens with CRLF on Windows (core.autocrlf=true, + // no .gitattributes); the generator always emits LF, so normalize first. + return fs.readFileSync(path.join(__dirname, 'golden', name), 'utf8').replace(/\r\n/g, '\n') +} + +// -------------------------------------------------------------------------- +// Case 1: quantization_example.proto +// -------------------------------------------------------------------------- + +const quantizationExample = { + name: 'decentraland/common/quantization_example.proto', + package: 'decentraland.common', + messageType: [ + { + name: 'PositionDelta', + field: [ + field('dx', TYPE_UINT32, quantizedOptions(-100, 100, 16)), + field('dy', TYPE_UINT32, quantizedOptions(-100, 100, 16)), + field('dz', TYPE_UINT32, quantizedOptions(-100, 100, 16)), + field('entity_id', TYPE_UINT32, bitPackedOptions(20)), + field('sequence', TYPE_UINT32, bitPackedOptions(12)), + ], + }, + { + name: 'PlayerInput', + field: [ + field('move_x', TYPE_UINT32, quantizedOptions(-1, 1, 8)), + field('move_z', TYPE_UINT32, quantizedOptions(-1, 1, 8)), + field('yaw', TYPE_UINT32, quantizedOptions(-180, 180, 12)), + field('buttons', TYPE_UINT32, bitPackedOptions(8)), + field('sequence', TYPE_UINT32, bitPackedOptions(12)), + ], + }, + { + name: 'VelocityState', + field: [ + field('vx', TYPE_UINT32, quantizedPowerOptions(50, 2, 8)), + field('vy', TYPE_UINT32, quantizedPowerOptions(50, 2, 8)), + field('vz', TYPE_UINT32, quantizedPowerOptions(50, 2, 8)), + ], + }, + { + name: 'AvatarStateSnapshot', + field: [ + field('x', TYPE_UINT32, quantizedOptions(-4096, 4096, 16)), + field('y', TYPE_UINT32, quantizedOptions(-256, 256, 14)), + field('z', TYPE_UINT32, quantizedOptions(-4096, 4096, 16)), + field('pitch', TYPE_UINT32, quantizedOptions(-90, 90, 10)), + field('yaw', TYPE_UINT32, quantizedOptions(-180, 180, 12)), + field('entity_id', TYPE_UINT32, bitPackedOptions(20)), + field('animation_state', TYPE_UINT32, bitPackedOptions(6)), + field('is_grounded', TYPE_BOOL, null), + field('timestamp', TYPE_DOUBLE, null), + ], + }, + ], +} + +// -------------------------------------------------------------------------- +// Case 2: pulse_server.proto (PlayerStateDeltaTier0) — wider bit/range coverage +// -------------------------------------------------------------------------- + +const pulseServer = { + name: 'decentraland/pulse/pulse_server.proto', + package: 'decentraland.pulse', + messageType: [ + { + name: 'PlayerStateDeltaTier0', + field: [ + field('position_x', TYPE_UINT32, quantizedOptions(0, 16, 8)), + field('position_y', TYPE_UINT32, quantizedOptions(0, 200, 13)), + field('position_z', TYPE_UINT32, quantizedOptions(0, 16, 8)), + field('velocity_x', TYPE_UINT32, quantizedPowerOptions(50, 2, 8)), + field('velocity_y', TYPE_UINT32, quantizedPowerOptions(50, 2, 8)), + field('velocity_z', TYPE_UINT32, quantizedPowerOptions(50, 2, 8)), + field('rotation_y', TYPE_UINT32, quantizedOptions(0, 360, 7)), + field('movement_blend', TYPE_UINT32, quantizedOptions(0, 3, 5)), + field('slide_blend', TYPE_UINT32, quantizedOptions(0, 1, 4)), + field('head_yaw', TYPE_UINT32, quantizedOptions(0, 360, 7)), + field('head_pitch', TYPE_UINT32, quantizedOptions(0, 360, 7)), + field('point_at_x', TYPE_UINT32, quantizedOptions(-3000, 3000, 17)), + field('point_at_y', TYPE_UINT32, quantizedOptions(0, 200, 7)), + field('point_at_z', TYPE_UINT32, quantizedOptions(-3000, 3000, 17)), + ], + }, + ], +} + +// -------------------------------------------------------------------------- + +const cases = [ + { proto: quantizationExample, golden: 'QuantizationExample.Bitwise.cs' }, + { proto: pulseServer, golden: 'PulseServer.Bitwise.cs' }, +] + +// Set UPDATE_GOLDEN=1 to rewrite the golden files from the current generator output +// (after an intentional generator change), then re-run without it to verify. +const update = process.env.UPDATE_GOLDEN === '1' + +let failed = 0 +for (const { proto, golden } of cases) { + const result = generateCsharp(proto) + assert.ok(result, `expected output for ${golden}`) + assert.strictEqual(result.name, golden, `output filename for ${golden}`) + + if (update) { + fs.writeFileSync(path.join(__dirname, 'golden', golden), result.content) + console.log(`updated - ${golden}`) + continue + } + + try { + assert.strictEqual(result.content, readGolden(golden)) + console.log(`ok - ${golden} matches golden`) + } catch (e) { + failed++ + console.error(`FAIL - ${golden} differs from golden`) + console.error(e.message) + } +} + +if (update) { + console.log('\nGolden files updated. Re-run `npm run gen:test` to verify.') + process.exit(0) +} + +if (failed > 0) { + console.error(`\n${failed} golden mismatch(es)`) + process.exit(1) +} +console.log('\nAll bitwise generator golden tests passed.') diff --git a/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs b/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs new file mode 100644 index 00000000..ea3c3d1a --- /dev/null +++ b/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs @@ -0,0 +1,163 @@ +// +// Generated by protoc-gen-bitwise. DO NOT EDIT. +// Source: decentraland/pulse/pulse_server.proto +// + +using Decentraland.Networking.Bitwise; + +namespace Decentraland.Pulse +{ + public partial class PlayerStateDeltaTier0 + { + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PositionXQuantizedStep = 0.062745098f; + /// Float accessor for . Range [0.0f, 16.0f], 8 bits, step ≈ 0.0627451. + public float PositionXQuantized + { + get => Quantize.Decode(PositionX, 0.0f, 16.0f, 8); + set => PositionX = Quantize.Encode(value, 0.0f, 16.0f, 8); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PositionYQuantizedStep = 0.024417043f; + /// Float accessor for . Range [0.0f, 200.0f], 13 bits, step ≈ 0.024417. + public float PositionYQuantized + { + get => Quantize.Decode(PositionY, 0.0f, 200.0f, 13); + set => PositionY = Quantize.Encode(value, 0.0f, 200.0f, 13); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PositionZQuantizedStep = 0.062745098f; + /// Float accessor for . Range [0.0f, 16.0f], 8 bits, step ≈ 0.0627451. + public float PositionZQuantized + { + get => Quantize.Decode(PositionZ, 0.0f, 16.0f, 8); + set => PositionZ = Quantize.Encode(value, 0.0f, 16.0f, 8); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float VelocityXQuantizedStep = 0.78430157f; + /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. + public float VelocityXQuantized + { + get => Quantize.DecodePower(VelocityX, 50.0f, 2.0f, 8); + set => VelocityX = Quantize.EncodePower(value, 50.0f, 2.0f, 8); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float VelocityYQuantizedStep = 0.78430157f; + /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. + public float VelocityYQuantized + { + get => Quantize.DecodePower(VelocityY, 50.0f, 2.0f, 8); + set => VelocityY = Quantize.EncodePower(value, 50.0f, 2.0f, 8); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float VelocityZQuantizedStep = 0.78430157f; + /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. + public float VelocityZQuantized + { + get => Quantize.DecodePower(VelocityZ, 50.0f, 2.0f, 8); + set => VelocityZ = Quantize.EncodePower(value, 50.0f, 2.0f, 8); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float RotationYQuantizedStep = 2.8346457f; + /// Float accessor for . Range [0.0f, 360.0f], 7 bits, step ≈ 2.83465. + public float RotationYQuantized + { + get => Quantize.Decode(RotationY, 0.0f, 360.0f, 7); + set => RotationY = Quantize.Encode(value, 0.0f, 360.0f, 7); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float MovementBlendQuantizedStep = 0.096774194f; + /// Float accessor for . Range [0.0f, 3.0f], 5 bits, step ≈ 0.0967742. + public float MovementBlendQuantized + { + get => Quantize.Decode(MovementBlend, 0.0f, 3.0f, 5); + set => MovementBlend = Quantize.Encode(value, 0.0f, 3.0f, 5); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float SlideBlendQuantizedStep = 0.066666667f; + /// Float accessor for . Range [0.0f, 1.0f], 4 bits, step ≈ 0.0666667. + public float SlideBlendQuantized + { + get => Quantize.Decode(SlideBlend, 0.0f, 1.0f, 4); + set => SlideBlend = Quantize.Encode(value, 0.0f, 1.0f, 4); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float HeadYawQuantizedStep = 2.8346457f; + /// Float accessor for . Range [0.0f, 360.0f], 7 bits, step ≈ 2.83465. + public float HeadYawQuantized + { + get => Quantize.Decode(HeadYaw, 0.0f, 360.0f, 7); + set => HeadYaw = Quantize.Encode(value, 0.0f, 360.0f, 7); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float HeadPitchQuantizedStep = 2.8346457f; + /// Float accessor for . Range [0.0f, 360.0f], 7 bits, step ≈ 2.83465. + public float HeadPitchQuantized + { + get => Quantize.Decode(HeadPitch, 0.0f, 360.0f, 7); + set => HeadPitch = Quantize.Encode(value, 0.0f, 360.0f, 7); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PointAtXQuantizedStep = 0.045776716f; + /// Float accessor for . Range [-3000.0f, 3000.0f], 17 bits, step ≈ 0.0457767. + public float PointAtXQuantized + { + get => Quantize.Decode(PointAtX, -3000.0f, 3000.0f, 17); + set => PointAtX = Quantize.Encode(value, -3000.0f, 3000.0f, 17); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PointAtYQuantizedStep = 1.5748031f; + /// Float accessor for . Range [0.0f, 200.0f], 7 bits, step ≈ 1.5748. + public float PointAtYQuantized + { + get => Quantize.Decode(PointAtY, 0.0f, 200.0f, 7); + set => PointAtY = Quantize.Encode(value, 0.0f, 200.0f, 7); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PointAtZQuantizedStep = 0.045776716f; + /// Float accessor for . Range [-3000.0f, 3000.0f], 17 bits, step ≈ 0.0457767. + public float PointAtZQuantized + { + get => Quantize.Decode(PointAtZ, -3000.0f, 3000.0f, 17); + set => PointAtZ = Quantize.Encode(value, -3000.0f, 3000.0f, 17); + } + + /// + /// True when every quantized field holds a wire code within its declared bit width + /// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger + /// value is a malformed/hostile message: decoding it would land far outside the field's + /// [min, max] and, since the server relays raw codes verbatim, poison every observer. + /// Reject before storing or relaying. Pure integer comparison — no decode. + /// + public bool AreQuantizedFieldsInRange() => + PositionX <= 255u + && PositionY <= 8191u + && PositionZ <= 255u + && VelocityX <= 255u + && VelocityY <= 255u + && VelocityZ <= 255u + && RotationY <= 127u + && MovementBlend <= 31u + && SlideBlend <= 15u + && HeadYaw <= 127u + && HeadPitch <= 127u + && PointAtX <= 131071u + && PointAtY <= 127u + && PointAtZ <= 131071u; + } + + +} // namespace Decentraland.Pulse diff --git a/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs b/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs new file mode 100644 index 00000000..62f157e6 --- /dev/null +++ b/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs @@ -0,0 +1,199 @@ +// +// Generated by protoc-gen-bitwise. DO NOT EDIT. +// Source: decentraland/common/quantization_example.proto +// + +using Decentraland.Networking.Bitwise; + +namespace Decentraland.Common +{ + public partial class PositionDelta + { + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float DxQuantizedStep = 0.0030518044f; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. + public float DxQuantized + { + get => Quantize.Decode(Dx, -100.0f, 100.0f, 16); + set => Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float DyQuantizedStep = 0.0030518044f; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. + public float DyQuantized + { + get => Quantize.Decode(Dy, -100.0f, 100.0f, 16); + set => Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float DzQuantizedStep = 0.0030518044f; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. + public float DzQuantized + { + get => Quantize.Decode(Dz, -100.0f, 100.0f, 16); + set => Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); + } + + /// + /// True when every quantized field holds a wire code within its declared bit width + /// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger + /// value is a malformed/hostile message: decoding it would land far outside the field's + /// [min, max] and, since the server relays raw codes verbatim, poison every observer. + /// Reject before storing or relaying. Pure integer comparison — no decode. + /// + public bool AreQuantizedFieldsInRange() => + Dx <= 65535u + && Dy <= 65535u + && Dz <= 65535u; + } + + public partial class PlayerInput + { + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float MoveXQuantizedStep = 0.0078431373f; + /// Float accessor for . Range [-1.0f, 1.0f], 8 bits, step ≈ 0.00784314. + public float MoveXQuantized + { + get => Quantize.Decode(MoveX, -1.0f, 1.0f, 8); + set => MoveX = Quantize.Encode(value, -1.0f, 1.0f, 8); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float MoveZQuantizedStep = 0.0078431373f; + /// Float accessor for . Range [-1.0f, 1.0f], 8 bits, step ≈ 0.00784314. + public float MoveZQuantized + { + get => Quantize.Decode(MoveZ, -1.0f, 1.0f, 8); + set => MoveZ = Quantize.Encode(value, -1.0f, 1.0f, 8); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float YawQuantizedStep = 0.087912088f; + /// Float accessor for . Range [-180.0f, 180.0f], 12 bits, step ≈ 0.0879121. + public float YawQuantized + { + get => Quantize.Decode(Yaw, -180.0f, 180.0f, 12); + set => Yaw = Quantize.Encode(value, -180.0f, 180.0f, 12); + } + + /// + /// True when every quantized field holds a wire code within its declared bit width + /// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger + /// value is a malformed/hostile message: decoding it would land far outside the field's + /// [min, max] and, since the server relays raw codes verbatim, poison every observer. + /// Reject before storing or relaying. Pure integer comparison — no decode. + /// + public bool AreQuantizedFieldsInRange() => + MoveX <= 255u + && MoveZ <= 255u + && Yaw <= 4095u; + } + + public partial class VelocityState + { + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float VxQuantizedStep = 0.78430157f; + /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. + public float VxQuantized + { + get => Quantize.DecodePower(Vx, 50.0f, 2.0f, 8); + set => Vx = Quantize.EncodePower(value, 50.0f, 2.0f, 8); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float VyQuantizedStep = 0.78430157f; + /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. + public float VyQuantized + { + get => Quantize.DecodePower(Vy, 50.0f, 2.0f, 8); + set => Vy = Quantize.EncodePower(value, 50.0f, 2.0f, 8); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float VzQuantizedStep = 0.78430157f; + /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. + public float VzQuantized + { + get => Quantize.DecodePower(Vz, 50.0f, 2.0f, 8); + set => Vz = Quantize.EncodePower(value, 50.0f, 2.0f, 8); + } + + /// + /// True when every quantized field holds a wire code within its declared bit width + /// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger + /// value is a malformed/hostile message: decoding it would land far outside the field's + /// [min, max] and, since the server relays raw codes verbatim, poison every observer. + /// Reject before storing or relaying. Pure integer comparison — no decode. + /// + public bool AreQuantizedFieldsInRange() => + Vx <= 255u + && Vy <= 255u + && Vz <= 255u; + } + + public partial class AvatarStateSnapshot + { + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float XQuantizedStep = 0.12500191f; + /// Float accessor for . Range [-4096.0f, 4096.0f], 16 bits, step ≈ 0.125002. + public float XQuantized + { + get => Quantize.Decode(X, -4096.0f, 4096.0f, 16); + set => X = Quantize.Encode(value, -4096.0f, 4096.0f, 16); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float YQuantizedStep = 0.031251907f; + /// Float accessor for . Range [-256.0f, 256.0f], 14 bits, step ≈ 0.0312519. + public float YQuantized + { + get => Quantize.Decode(Y, -256.0f, 256.0f, 14); + set => Y = Quantize.Encode(value, -256.0f, 256.0f, 14); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float ZQuantizedStep = 0.12500191f; + /// Float accessor for . Range [-4096.0f, 4096.0f], 16 bits, step ≈ 0.125002. + public float ZQuantized + { + get => Quantize.Decode(Z, -4096.0f, 4096.0f, 16); + set => Z = Quantize.Encode(value, -4096.0f, 4096.0f, 16); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PitchQuantizedStep = 0.17595308f; + /// Float accessor for . Range [-90.0f, 90.0f], 10 bits, step ≈ 0.175953. + public float PitchQuantized + { + get => Quantize.Decode(Pitch, -90.0f, 90.0f, 10); + set => Pitch = Quantize.Encode(value, -90.0f, 90.0f, 10); + } + + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float YawQuantizedStep = 0.087912088f; + /// Float accessor for . Range [-180.0f, 180.0f], 12 bits, step ≈ 0.0879121. + public float YawQuantized + { + get => Quantize.Decode(Yaw, -180.0f, 180.0f, 12); + set => Yaw = Quantize.Encode(value, -180.0f, 180.0f, 12); + } + + /// + /// True when every quantized field holds a wire code within its declared bit width + /// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger + /// value is a malformed/hostile message: decoding it would land far outside the field's + /// [min, max] and, since the server relays raw codes verbatim, poison every observer. + /// Reject before storing or relaying. Pure integer comparison — no decode. + /// + public bool AreQuantizedFieldsInRange() => + X <= 65535u + && Y <= 16383u + && Z <= 65535u + && Pitch <= 1023u + && Yaw <= 4095u; + } + + +} // namespace Decentraland.Common diff --git a/protoc-gen-bitwise/wire.js b/protoc-gen-bitwise/wire.js new file mode 100644 index 00000000..e2bd2d57 --- /dev/null +++ b/protoc-gen-bitwise/wire.js @@ -0,0 +1,239 @@ +'use strict' + +/** + * Self-contained protobuf wire-format codec for protoc-gen-bitwise. + * + * The plugin only needs a tiny slice of the descriptor/plugin schemas, so + * rather than pull in a protobuf runtime (which would force every consumer — + * including the sibling Pulse checkout that runs this file directly off disk — + * to `npm install` the protocol repo) we decode the handful of fields we care + * about by walking the wire format directly. This mirrors the original Python + * plugin, which already hand-parsed FieldOptions for the same reason. + * + * Decoded subset: + * CodeGeneratorRequest { file_to_generate = 1; proto_file = 15; } + * FileDescriptorProto { name = 1; package = 2; message_type = 4; } + * DescriptorProto { name = 1; field = 2; } + * FieldDescriptorProto { name = 1; label = 4; type = 5; options = 8 (raw bytes); } + * + * Encoded subset: + * CodeGeneratorResponse { error = 1; supported_features = 2; file = 15; } + * CodeGeneratorResponse.File { name = 1; content = 15; } + * + * Wire types: 0 = varint, 1 = 64-bit, 2 = length-delimited, 5 = 32-bit. + */ + +// --------------------------------------------------------------------------- +// Low-level readers +// --------------------------------------------------------------------------- + +/** Decode a protobuf varint at `pos`. Returns [value, newPos]. */ +function readVarint(buf, pos) { + let result = 0 + let shift = 0 + let byte + do { + byte = buf[pos++] + // Multiplication (not <<) keeps values correct past 32 bits. + result += (byte & 0x7f) * Math.pow(2, shift) + shift += 7 + } while (byte & 0x80) + return [result, pos] +} + +/** Advance `pos` past a field of the given wire type. Returns newPos. */ +function skipField(buf, pos, wireType) { + switch (wireType) { + case 0: // varint + return readVarint(buf, pos)[1] + case 1: // 64-bit + return pos + 8 + case 2: { + // length-delimited + const [length, next] = readVarint(buf, pos) + return next + length + } + case 5: // 32-bit + return pos + 4 + default: + // Groups (3/4) are deprecated and never appear in descriptors. + return pos + } +} + +// --------------------------------------------------------------------------- +// Request decoding +// --------------------------------------------------------------------------- + +function decodeFieldDescriptor(buf) { + const field = { name: '', label: 0, type: 0, optionsRaw: null } + let pos = 0 + while (pos < buf.length) { + let tag + ;[tag, pos] = readVarint(buf, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + field.name = buf.toString('utf8', pos, pos + len) + pos += len + } else if (fieldNum === 4 && wireType === 0) { + ;[field.label, pos] = readVarint(buf, pos) + } else if (fieldNum === 5 && wireType === 0) { + ;[field.type, pos] = readVarint(buf, pos) + } else if (fieldNum === 8 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + // Keep the raw FieldOptions bytes so custom extensions survive (the + // descriptor schema doesn't know about ext 50001/50002). + field.optionsRaw = buf.subarray(pos, pos + len) + pos += len + } else { + pos = skipField(buf, pos, wireType) + } + } + return field +} + +function decodeDescriptor(buf) { + const message = { name: '', field: [] } + let pos = 0 + while (pos < buf.length) { + let tag + ;[tag, pos] = readVarint(buf, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + message.name = buf.toString('utf8', pos, pos + len) + pos += len + } else if (fieldNum === 2 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + message.field.push(decodeFieldDescriptor(buf.subarray(pos, pos + len))) + pos += len + } else { + // nested_type / enum_type / etc. are intentionally ignored — the + // generator only walks top-level messages, matching the Python plugin. + pos = skipField(buf, pos, wireType) + } + } + return message +} + +function decodeFileDescriptor(buf) { + const file = { name: '', package: '', messageType: [] } + let pos = 0 + while (pos < buf.length) { + let tag + ;[tag, pos] = readVarint(buf, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + file.name = buf.toString('utf8', pos, pos + len) + pos += len + } else if (fieldNum === 2 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + file.package = buf.toString('utf8', pos, pos + len) + pos += len + } else if (fieldNum === 4 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + file.messageType.push(decodeDescriptor(buf.subarray(pos, pos + len))) + pos += len + } else { + pos = skipField(buf, pos, wireType) + } + } + return file +} + +/** Decode a serialized CodeGeneratorRequest. */ +function decodeRequest(buf) { + const request = { fileToGenerate: [], protoFile: [] } + let pos = 0 + while (pos < buf.length) { + let tag + ;[tag, pos] = readVarint(buf, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + request.fileToGenerate.push(buf.toString('utf8', pos, pos + len)) + pos += len + } else if (fieldNum === 15 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + request.protoFile.push(decodeFileDescriptor(buf.subarray(pos, pos + len))) + pos += len + } else { + pos = skipField(buf, pos, wireType) + } + } + return request +} + +// --------------------------------------------------------------------------- +// Response encoding +// --------------------------------------------------------------------------- + +/** Encode an unsigned integer as a protobuf varint Buffer. */ +function encodeVarint(value) { + const bytes = [] + let v = value + while (v > 0x7f) { + bytes.push((v & 0x7f) | 0x80) + v = Math.floor(v / 128) + } + bytes.push(v & 0x7f) + return Buffer.from(bytes) +} + +function encodeTag(fieldNum, wireType) { + return encodeVarint((fieldNum << 3) | wireType) +} + +/** Encode a length-delimited (string/bytes) field: tag + length + payload. */ +function encodeLengthDelimited(fieldNum, payload) { + const buf = Buffer.isBuffer(payload) ? payload : Buffer.from(payload, 'utf8') + return Buffer.concat([encodeTag(fieldNum, 2), encodeVarint(buf.length), buf]) +} + +function encodeFile(file) { + return Buffer.concat([ + encodeLengthDelimited(1, file.name), // name = 1 + encodeLengthDelimited(15, file.content), // content = 15 + ]) +} + +/** + * Encode a CodeGeneratorResponse. + * @param {{error?: string|null, supportedFeatures?: number, files?: Array<{name:string,content:string}>}} response + */ +function encodeResponse(response) { + const parts = [] + if (response.error != null) { + parts.push(encodeLengthDelimited(1, response.error)) // error = 1 + } + if (response.supportedFeatures != null) { + parts.push(encodeTag(2, 0)) // supported_features = 2 (varint) + parts.push(encodeVarint(response.supportedFeatures)) + } + for (const file of response.files || []) { + parts.push(encodeLengthDelimited(15, encodeFile(file))) // file = 15 + } + return Buffer.concat(parts) +} + +module.exports = { + readVarint, + skipField, + decodeRequest, + encodeResponse, +}