From 5fbeb3c9a0c74f466f8c6e8bf6d4c44bc8554ff7 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Melucci <163010988+AlejandroAlvarezMelucciDCL@users.noreply.github.com> Date: Tue, 4 Feb 2025 16:56:36 -0300 Subject: [PATCH 01/54] feat: dynamic lights (#234) * Initial draft of LightSource component * Fixed conflicting ids * Refactor structure * Fixed ids * Removed unused property and adjusted ids again * Fixed color reference * Fixed lowecase to oneof name * Test to fix proto builds * Renamed file to properly generate classes with the intended name * Updated documentation * Added cookie field * Added missing import * Fixed proper type reference * Set optional properties, improved documentation * Updated shadow mask type to TextureUnion * Reordered properties to support different defaults on different messages --- .../sdk/components/light_source.proto | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 proto/decentraland/sdk/components/light_source.proto diff --git a/proto/decentraland/sdk/components/light_source.proto b/proto/decentraland/sdk/components/light_source.proto new file mode 100644 index 00000000..67c9a3c9 --- /dev/null +++ b/proto/decentraland/sdk/components/light_source.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; +package decentraland.sdk.components; +import "decentraland/sdk/components/common/id.proto"; +import "decentraland/common/colors.proto"; +import "decentraland/common/texture.proto"; +option (common.ecs_component_id) = 1079; + +message PBLightSource { + optional bool active = 4; // default = true, whether the lightSource is active or not. + optional decentraland.common.Color3 color = 1; // default = Color.white, the tint of the light, in RGB format where each component is a floating point value with a range from 0 to 1. + optional float brightness = 2; // default = 250, ranges from 1 (dim) to 100,000 (very bright), expressed in Lumens for Point and Spot. + optional float range = 3; // default = 10, how far the light travels, expressed in meters. + + oneof type { + Point point = 6; + Spot spot = 7; + } + + message Point { + optional ShadowType shadow = 5; // default = ShadowType.ST_NONE The type of shadow the light source supports. + } + + message Spot { + optional float inner_angle = 1; // default = 21.8. Inner angle can't be higher than outer angle, otherwise will default to same value. Min value is 0. Max value is 179. + optional float outer_angle = 2; // default = 30. Outer angle can't be lower than inner angle, otherwise will inner angle will be set to same value. Max value is 179. + optional ShadowType shadow = 5; // default = ShadowType.ST_NONE The type of shadow the light source supports. + optional decentraland.common.TextureUnion shadow_mask_texture = 8; // Texture mask through which shadows are cast to simulate caustics, soft shadows, and light shapes such as light entering from a window. + } + + enum ShadowType { + ST_NONE = 0; // No shadows are cast from this LightSource. + ST_SOFT = 1; // More realistic type of shadow that reduces block artifacts, noise or pixelation, but requires more processing. + ST_HARD = 2; // Less realistic type of shadow but more performant, uses hard edges. + } +} From e07ccb6e15f66761b3acaebdfc7ff0cd469a7e8b Mon Sep 17 00:00:00 2001 From: Kevin Szuchet <31735779+kevinszuchet@users.noreply.github.com> Date: Mon, 17 Feb 2025 15:26:44 +0000 Subject: [PATCH 02/54] feat: Social service v2 (#242) * feat: Concept of friendships * feat: Only friend requests need id * feat: change order of props for compatibility * feat: New stream FriendUpdate (online, offline, away in the future) * feat: Missing changes after merge * feat: Get Friends and Mutual responses include data from their profile * feat: Friendship request response also include profile data * fix: change string for bool on has claimed name * refactor: Rename stream friend connectivity updates subscription and response * feat: Additional data in upsert and updates responses * feat: Add None Friendship Status --- .../social_service/v2/social_service_v2.proto | 40 +++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/proto/decentraland/social_service/v2/social_service_v2.proto b/proto/decentraland/social_service/v2/social_service_v2.proto index 363ed330..ca9a466f 100644 --- a/proto/decentraland/social_service/v2/social_service_v2.proto +++ b/proto/decentraland/social_service/v2/social_service_v2.proto @@ -10,15 +10,23 @@ message InternalServerError {} // Types message User { string address = 1; } +message FriendProfile { + string address = 1; + string name = 2; + bool has_claimed_name = 3; + string profile_picture_url = 4; +} + message Pagination { int32 limit = 1; int32 offset = 2; } message FriendshipRequestResponse { - User user = 1; + FriendProfile friend = 1; int64 created_at = 2; optional string message = 3; + string id = 4; } message FriendshipRequests { @@ -33,7 +41,6 @@ enum ConnectivityStatus { message GetFriendsPayload { optional Pagination pagination = 1; - optional ConnectivityStatus status = 2; } message GetFriendshipRequestsPayload { @@ -69,8 +76,8 @@ message PaginatedResponse { int32 page = 2; } -message PaginatedUsersResponse { - repeated User users = 1; +message PaginatedFriendsProfilesResponse { + repeated FriendProfile friends = 1; PaginatedResponse pagination_data = 2; } @@ -86,6 +93,8 @@ message UpsertFriendshipResponse { message Accepted { string id = 1; int64 created_at = 2; + FriendProfile friend = 3; + optional string message = 4; } oneof response { Accepted accepted = 1; @@ -95,13 +104,19 @@ message UpsertFriendshipResponse { } message FriendshipUpdate { + message RequestResponse { + FriendProfile friend = 1; + int64 created_at = 2; + optional string message = 3; + string id = 4; + } message AcceptResponse { User user = 1; } message RejectResponse { User user = 1; } message DeleteResponse { User user = 1; } message CancelResponse { User user = 1; } oneof update { - FriendshipRequestResponse request = 1; + RequestResponse request = 1; AcceptResponse accept = 2; RejectResponse reject = 3; DeleteResponse delete = 4; @@ -109,6 +124,11 @@ message FriendshipUpdate { } } +message FriendConnectivityUpdate { + FriendProfile friend = 1; + ConnectivityStatus status = 2; +} + message GetFriendshipStatusPayload { User user = 1; } @@ -121,6 +141,7 @@ enum FriendshipStatus { REJECTED = 4; DELETED = 5; BLOCKED = 6; + NONE = 7; } message GetFriendshipStatusResponse { @@ -136,10 +157,10 @@ message GetFriendshipStatusResponse { service SocialService { // Get the list of friends for the authenticated user - rpc GetFriends(GetFriendsPayload) returns (PaginatedUsersResponse) {} + rpc GetFriends(GetFriendsPayload) returns (PaginatedFriendsProfilesResponse) {} // Get the list of mutual friends between the authenticated user and the one in the parameter - rpc GetMutualFriends(GetMutualFriendsPayload) returns (PaginatedUsersResponse) {} + rpc GetMutualFriends(GetMutualFriendsPayload) returns (PaginatedFriendsProfilesResponse) {} // Get the pending friendship requests for the authenticated user rpc GetPendingFriendshipRequests(GetFriendshipRequestsPayload) returns (PaginatedFriendshipRequestsResponse) {} @@ -155,5 +176,10 @@ service SocialService { rpc SubscribeToFriendshipUpdates(google.protobuf.Empty) returns (stream FriendshipUpdate) {} + // Get the friendship status between the authenticated user and the one in the parameter rpc GetFriendshipStatus(GetFriendshipStatusPayload) returns (GetFriendshipStatusResponse) {} + + // Subscribe to connectivity updates of friends: ONLINE, OFFLINE, AWAY + rpc SubscribeToFriendConnectivityUpdates(google.protobuf.Empty) + returns (stream FriendConnectivityUpdate) {} } From 0870a3e554a46f0d06dda40bc7597abc3e567a2f Mon Sep 17 00:00:00 2001 From: Pravus Date: Fri, 28 Feb 2025 15:16:41 +0100 Subject: [PATCH 03/54] chore: merge main into experimental (#249) feat: avatar_target param addition in movePlayerTo (#248) Added avatar_target param in movePlayerTo --- proto/decentraland/kernel/apis/restricted_actions.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/proto/decentraland/kernel/apis/restricted_actions.proto b/proto/decentraland/kernel/apis/restricted_actions.proto index b299fc41..b2dcc7b8 100644 --- a/proto/decentraland/kernel/apis/restricted_actions.proto +++ b/proto/decentraland/kernel/apis/restricted_actions.proto @@ -6,6 +6,7 @@ import "decentraland/common/vectors.proto"; message MovePlayerToRequest { decentraland.common.Vector3 new_relative_position = 1; optional decentraland.common.Vector3 camera_target = 2; + optional decentraland.common.Vector3 avatar_target = 3; } message TeleportToRequest { From 195803dba45ac137ea43ffefc313118fdf669a0f Mon Sep 17 00:00:00 2001 From: Kevin Szuchet <31735779+kevinszuchet@users.noreply.github.com> Date: Mon, 17 Mar 2025 11:29:59 +0100 Subject: [PATCH 04/54] feat: Social service blocked users (#247) * feat: RPCs for blocking feature * refactor: Enhance block unblock response * refactor: Rename Profile to UserProfile * feat: Add RPC GetBlockingStatus * refactor: GetBlockingStatus with empty payload * refactor: GetBlockingStatusResponse has two array of addresses * refactor: New type BlockedUserProfile * feat: Add SubscribeToBlockUpdates rpc * feat: New FriendshipStatus BlockedBy * feat: Add clearer error messages to block/unblock responses * feat: Add Block update to FriendshipUpdate message type --- .../social_service/v2/social_service_v2.proto | 88 ++++++++++++++++++- 1 file changed, 86 insertions(+), 2 deletions(-) diff --git a/proto/decentraland/social_service/v2/social_service_v2.proto b/proto/decentraland/social_service/v2/social_service_v2.proto index ca9a466f..229e9787 100644 --- a/proto/decentraland/social_service/v2/social_service_v2.proto +++ b/proto/decentraland/social_service/v2/social_service_v2.proto @@ -4,8 +4,18 @@ package decentraland.social_service.v2; import "google/protobuf/empty.proto"; // Errors -message InvalidFriendshipAction {} -message InternalServerError {} +message InvalidFriendshipAction { + optional string message = 1; +} +message InternalServerError { + optional string message = 1; +} +message InvalidRequest { + optional string message = 1; +} +message ProfileNotFound { + optional string message = 1; +} // Types message User { string address = 1; } @@ -17,6 +27,14 @@ message FriendProfile { string profile_picture_url = 4; } +message BlockedUserProfile { + string address = 1; + string name = 2; + bool has_claimed_name = 3; + string profile_picture_url = 4; + optional int64 blocked_at = 5; +} + message Pagination { int32 limit = 1; int32 offset = 2; @@ -114,6 +132,7 @@ message FriendshipUpdate { message RejectResponse { User user = 1; } message DeleteResponse { User user = 1; } message CancelResponse { User user = 1; } + message BlockResponse { User user = 1; } oneof update { RequestResponse request = 1; @@ -121,6 +140,7 @@ message FriendshipUpdate { RejectResponse reject = 3; DeleteResponse delete = 4; CancelResponse cancel = 5; + BlockResponse block = 6; } } @@ -142,6 +162,7 @@ enum FriendshipStatus { DELETED = 5; BLOCKED = 6; NONE = 7; + BLOCKED_BY = 8; } message GetFriendshipStatusResponse { @@ -155,6 +176,59 @@ message GetFriendshipStatusResponse { } } +message BlockUserPayload { + User user = 1; +} + +message BlockUserResponse { + message Ok { + BlockedUserProfile profile = 1; + } + + oneof response { + Ok ok = 1; + InternalServerError internal_server_error = 2; + InvalidRequest invalid_request = 3; + ProfileNotFound profile_not_found = 4; + } +} + +message UnblockUserPayload { + User user = 1; +} + +message UnblockUserResponse { + message Ok { + BlockedUserProfile profile = 1; + } + + oneof response { + Ok ok = 1; + InternalServerError internal_server_error = 2; + InvalidRequest invalid_request = 3; + ProfileNotFound profile_not_found = 4; + } +} + +message GetBlockedUsersPayload { + optional Pagination pagination = 1; +} + +message GetBlockedUsersResponse { + repeated BlockedUserProfile profiles = 1; + PaginatedResponse pagination_data = 2; +} + +message GetBlockingStatusResponse { + repeated string blocked_users = 1; + repeated string blocked_by_users = 2; +} + +message BlockUpdate { + string address = 1; + bool is_blocked = 2; +} + service SocialService { // Get the list of friends for the authenticated user rpc GetFriends(GetFriendsPayload) returns (PaginatedFriendsProfilesResponse) {} @@ -182,4 +256,14 @@ service SocialService { // Subscribe to connectivity updates of friends: ONLINE, OFFLINE, AWAY rpc SubscribeToFriendConnectivityUpdates(google.protobuf.Empty) returns (stream FriendConnectivityUpdate) {} + + rpc BlockUser(BlockUserPayload) returns (BlockUserResponse) {} + + rpc UnblockUser(UnblockUserPayload) returns (UnblockUserResponse) {} + + rpc GetBlockedUsers(GetBlockedUsersPayload) returns (GetBlockedUsersResponse) {} + + rpc GetBlockingStatus(google.protobuf.Empty) returns (GetBlockingStatusResponse) {} + + rpc SubscribeToBlockUpdates(google.protobuf.Empty) returns (stream BlockUpdate) {} } From 1f4cb5e383eccc3c284178b1c8c7e136a103dea4 Mon Sep 17 00:00:00 2001 From: Pravus Date: Tue, 18 Mar 2025 13:43:17 +0100 Subject: [PATCH 05/54] feat: virtual camera fov parameter (#250) --- proto/decentraland/sdk/components/virtual_camera.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proto/decentraland/sdk/components/virtual_camera.proto b/proto/decentraland/sdk/components/virtual_camera.proto index 94302069..a8764e68 100644 --- a/proto/decentraland/sdk/components/virtual_camera.proto +++ b/proto/decentraland/sdk/components/virtual_camera.proto @@ -10,7 +10,9 @@ option (common.ecs_component_id) = 1076; // an 'instant' transition (like using speed/time = 0) // * The lookAtEntity defines to which entity the Camera has to look at constantly (independent from // the holding entity transform). +// * The fov defines the Field of View of the virtual camera message PBVirtualCamera { optional common.CameraTransition default_transition = 1; optional uint32 look_at_entity = 2; + optional float fov = 3; // default: 60 } \ No newline at end of file From 0daf6ca7b341b3b9eb2c60e56047f758b95927ce Mon Sep 17 00:00:00 2001 From: Gon Pombo Date: Mon, 24 Mar 2025 07:43:56 -0300 Subject: [PATCH 06/54] Feat/merge main (#256) * feat: avatar_target param addition in movePlayerTo (#248) Added avatar_target param in movePlayerTo * add border properties (main) (#254) * add border properties * add border color for each edge --------- Co-authored-by: Pravus --- .../sdk/components/ui_transform.proto | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/proto/decentraland/sdk/components/ui_transform.proto b/proto/decentraland/sdk/components/ui_transform.proto index d3e0783c..f7c9de35 100644 --- a/proto/decentraland/sdk/components/ui_transform.proto +++ b/proto/decentraland/sdk/components/ui_transform.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package decentraland.sdk.components; import "decentraland/sdk/components/common/id.proto"; +import "decentraland/common/colors.proto"; option (common.ecs_component_id) = 1050; @@ -143,6 +144,32 @@ message PBUiTransform { float padding_right = 49; YGUnit padding_bottom_unit = 50; // YGUnit.YGU_UNDEFINED float padding_bottom = 51; - + optional PointerFilterMode pointer_filter = 52; // default: PointerFilterMode.PFM_NONE + + // Border Width + optional YGUnit border_left_width_unit = 53; // YGUnit.YGU_UNDEFINED + optional float border_left_width = 54; + optional YGUnit border_top_width_unit = 55; // YGUnit.YGU_UNDEFINED + optional float border_top_width = 56; + optional YGUnit border_right_width_unit = 57; // YGUnit.YGU_UNDEFINED + optional float border_right_width = 58; + optional YGUnit border_bottom_width_unit = 59; // YGUnit.YGU_UNDEFINED + optional float border_bottom_width = 60; + + // Border Radius + optional YGUnit border_top_left_radius_unit = 61; // YGUnit.YGU_UNDEFINED + optional float border_top_left_radius = 62; + optional YGUnit border_top_right_radius_unit = 63; // YGUnit.YGU_UNDEFINED + optional float border_top_right_radius = 64; + optional YGUnit border_bottom_left_radius_unit = 65; // YGUnit.YGU_UNDEFINED + optional float border_bottom_left_radius = 66; + optional YGUnit border_bottom_right_radius_unit = 67; // YGUnit.YGU_UNDEFINED + optional float border_bottom_right_radius = 68; + + // Border Color + optional decentraland.common.Color4 border_top_color = 69; + optional decentraland.common.Color4 border_bottom_color = 70; + optional decentraland.common.Color4 border_left_color = 71; + optional decentraland.common.Color4 border_right_color = 72; } From 3bf30a394732d4c5a8ae6471c6218b41a1d2bb93 Mon Sep 17 00:00:00 2001 From: Lautaro Petaccio <1120791+LautaroPetaccio@users.noreply.github.com> Date: Fri, 28 Mar 2025 10:12:10 -0300 Subject: [PATCH 07/54] feat: Social settings (#253) * feat: Social settings * fix: Missing props in the unblock response --------- Co-authored-by: Kevin Szuchet --- .../social_service/v2/social_service_v2.proto | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/proto/decentraland/social_service/v2/social_service_v2.proto b/proto/decentraland/social_service/v2/social_service_v2.proto index 229e9787..5226317d 100644 --- a/proto/decentraland/social_service/v2/social_service_v2.proto +++ b/proto/decentraland/social_service/v2/social_service_v2.proto @@ -210,6 +210,67 @@ message UnblockUserResponse { } } +enum PrivateMessagePrivacySetting { + ALL = 0; + ONLY_FRIENDS = 1; +} + +enum BlockedUsersMessagesVisibilitySetting { + SHOW_MESSAGES = 0; + DO_NOT_SHOW_MESSAGES = 1; +} + +message SocialSettings { + PrivateMessagePrivacySetting private_messages_privacy = 1; + BlockedUsersMessagesVisibilitySetting blocked_users_messages_visibility = 2; +} + +message GetSocialSettingsResponse { + message Ok { + SocialSettings settings = 1; + } + + oneof response { + Ok ok = 1; + InternalServerError internal_server_error = 2; + } +} + +message UpsertSocialSettingsPayload { + optional PrivateMessagePrivacySetting private_messages_privacy = 1; + optional BlockedUsersMessagesVisibilitySetting blocked_users_messages_visibility = 2; +} + +message UpsertSocialSettingsResponse { + oneof response { + SocialSettings ok = 1; + InternalServerError internal_server_error = 2; + InvalidRequest invalid_request = 3; + } +} + +message GetPrivateMessagesSettingsPayload { + repeated User user = 1; +} + +message GetPrivateMessagesSettingsResponse { + message PrivateMessagesSettings { + User user = 1; + PrivateMessagePrivacySetting private_messages_privacy = 2; + } + + message Ok { + repeated PrivateMessagesSettings settings = 1; + } + + oneof response { + Ok ok = 1; + InternalServerError internal_server_error = 2; + InvalidRequest invalid_request = 3; + ProfileNotFound profile_not_found = 4; + } +} + message GetBlockedUsersPayload { optional Pagination pagination = 1; } @@ -266,4 +327,13 @@ service SocialService { rpc GetBlockingStatus(google.protobuf.Empty) returns (GetBlockingStatusResponse) {} rpc SubscribeToBlockUpdates(google.protobuf.Empty) returns (stream BlockUpdate) {} + + // Get all the social settings for the authenticated user + rpc GetSocialSettings(google.protobuf.Empty) returns (GetSocialSettingsResponse) {} + + // Insert or update the social settings for the authenticated user + rpc UpsertSocialSettings(UpsertSocialSettingsPayload) returns (UpsertSocialSettingsResponse) {} + + // Get the private messages privacy settings for the requested users + rpc GetPrivateMessagesSettings(GetPrivateMessagesSettingsPayload) returns (GetPrivateMessagesSettingsResponse) {} } From deb9aa625b5c066f30c029d563784db07862d40b Mon Sep 17 00:00:00 2001 From: Lautaro Petaccio <1120791+LautaroPetaccio@users.noreply.github.com> Date: Tue, 22 Apr 2025 14:52:54 -0300 Subject: [PATCH 08/54] feat: Include is friend when asking for privacy settings (#258) feat: Include isFriend when receiving privacy settings --- proto/decentraland/social_service/v2/social_service_v2.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/proto/decentraland/social_service/v2/social_service_v2.proto b/proto/decentraland/social_service/v2/social_service_v2.proto index 5226317d..99a29d5d 100644 --- a/proto/decentraland/social_service/v2/social_service_v2.proto +++ b/proto/decentraland/social_service/v2/social_service_v2.proto @@ -257,6 +257,7 @@ message GetPrivateMessagesSettingsResponse { message PrivateMessagesSettings { User user = 1; PrivateMessagePrivacySetting private_messages_privacy = 2; + bool is_friend = 3; } message Ok { From 321145fdd73dec67c5c86c93911b5f7760cbd3e5 Mon Sep 17 00:00:00 2001 From: Kevin Szuchet <31735779+kevinszuchet@users.noreply.github.com> Date: Tue, 10 Jun 2025 15:44:24 +0300 Subject: [PATCH 09/54] feat: Add optional from in chat (#270) * feat: Add optional from in Chat * refactor: Rename field + comment * chore: enhance comment * chore: fix typo --- proto/decentraland/kernel/comms/rfc4/comms.proto | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/proto/decentraland/kernel/comms/rfc4/comms.proto b/proto/decentraland/kernel/comms/rfc4/comms.proto index 83f098d3..cf311343 100644 --- a/proto/decentraland/kernel/comms/rfc4/comms.proto +++ b/proto/decentraland/kernel/comms/rfc4/comms.proto @@ -101,6 +101,10 @@ message ProfileResponse { message Chat { string message = 1; double timestamp = 2; + + // Extension: optional forwarded_from to identify the original sender when + // messages are forwarded through an SFU + optional string forwarded_from = 3; } message Scene { From 9a42a0f39f7b753af7a546b436fb74ac7e9281be Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Melucci <163010988+AlejandroAlvarezMelucciDCL@users.noreply.github.com> Date: Tue, 10 Jun 2025 11:11:59 -0300 Subject: [PATCH 10/54] feat: z index and opacity for uielements (#274) feat: add z-index and opacity support for ui elements (#266) * Added z index property to ui transform * Update property id As per protocol-squad's request, I updated the index to match their implementation * Added opacity property --------- Signed-off-by: Alejandro Alvarez Melucci <163010988+AlejandroAlvarezMelucciDCL@users.noreply.github.com> --- proto/decentraland/sdk/components/ui_transform.proto | 3 +++ 1 file changed, 3 insertions(+) diff --git a/proto/decentraland/sdk/components/ui_transform.proto b/proto/decentraland/sdk/components/ui_transform.proto index f7c9de35..0e3c9f0c 100644 --- a/proto/decentraland/sdk/components/ui_transform.proto +++ b/proto/decentraland/sdk/components/ui_transform.proto @@ -172,4 +172,7 @@ message PBUiTransform { optional decentraland.common.Color4 border_bottom_color = 70; optional decentraland.common.Color4 border_left_color = 71; optional decentraland.common.Color4 border_right_color = 72; + + optional float opacity = 73; // default: 1 + optional int32 z_index = 77; // default: 0 — controls render stacking order. Higher values appear in front of lower values. } From ce7a4ebfc094f4764c5182b0e41d3ebc72e7a17d Mon Sep 17 00:00:00 2001 From: Lautaro Petaccio <1120791+LautaroPetaccio@users.noreply.github.com> Date: Tue, 24 Jun 2025 09:12:07 -0300 Subject: [PATCH 11/54] feat: Voice chat protocol (#265) * feat: Add new errors * fix: Update protobuf to a more recent version * fix: Update deps * fix: Outout information about protobufjs * fix: Use fixed protobuf version * feat: Add Start Private Voice chat procedure * feat: Add forbidden error * feat: Add Accept Private Voice Chat messages * feat: Add more properties to the accept response * feat: Add voice chat rejection to the protocol * fix: Remove forbidden from voice chat rejection response * feat: Add private voice subscription updates * fix: Use User as callee * fix: Temporarily remove the subscription RPC call * feat: Add End Call message * fix: Add the not found error to the end private voice chat call * feat: Add get incoming private voice chat request * fix: Override protbuf with older versio * fix: Protobuf types * feat: Add subscription call * fix: Add Yarn or NPM * feat: Add Private Voice Chat credenetials * fix: Change the credentials to return the connection url --- .github/workflows/build-and-publish.yml | 14 +- package.json | 2 +- .../social_service/v2/social_service_v2.proto | 147 ++++++++++++++++++ 3 files changed, 157 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index ecd23366..0f6c6a7a 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -32,7 +32,7 @@ jobs: uses: menduz/oddish-action@master id: publish_dcl_protocol with: - registry-url: 'https://registry.npmjs.org' + registry-url: "https://registry.npmjs.org" access: public ## use action runId instead of current date to generate snapshot numbers deterministic-snapshot: true @@ -42,7 +42,7 @@ jobs: ## publish every package to s3 s3-bucket: ${{ secrets.SDK_TEAM_S3_BUCKET }} - s3-bucket-key-prefix: '@dcl/protocol/branch/${{ steps.myref.outputs.branch }}' + s3-bucket-key-prefix: "@dcl/protocol/branch/${{ steps.myref.outputs.branch }}" s3-bucket-region: ${{ secrets.SDK_TEAM_AWS_REGION }} ## inform gitlab after publishing to proceed with CDN propagation gitlab-token: ${{ secrets.GITLAB_TOKEN }} @@ -67,7 +67,7 @@ jobs: id: fc with: issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' + comment-author: "github-actions[bot]" body-includes: Test this pull request - name: Get the current branch name @@ -87,9 +87,13 @@ jobs: comment-id: ${{ steps.fc.outputs.comment-id }} issue-number: ${{ github.event.pull_request.number }} body: | - # Test this pull request - - The `@dcl/protocol` package can be tested in scenes by running + # Test this pull request on NPM or Yarn + - The `@dcl/protocol` package can be tested in scenes by running the following NPM command: ```bash npm install "${{ steps.url-generator.outputs.body }}" ``` + - The `@dcl/protocol` package can be tested in scenes by running the following YARN command: + ```bash + yarn add "${{ steps.url-generator.outputs.body }}" + ``` edit-mode: replace diff --git a/package.json b/package.json index 18990f05..7e731d25 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "test": "./scripts/check-proto-compabitility.sh" }, "devDependencies": { - "@protobuf-ts/protoc": "^2.8.1", + "@protobuf-ts/protoc": "^2.11.0", "proto-compatibility-tool": "^1.1.2-20220925163655.commit-25bd040", "typescript": "^5.0.4" }, diff --git a/proto/decentraland/social_service/v2/social_service_v2.proto b/proto/decentraland/social_service/v2/social_service_v2.proto index 99a29d5d..d19047b9 100644 --- a/proto/decentraland/social_service/v2/social_service_v2.proto +++ b/proto/decentraland/social_service/v2/social_service_v2.proto @@ -16,6 +16,15 @@ message InvalidRequest { message ProfileNotFound { optional string message = 1; } +message ConflictingError { + optional string message = 1; +} +message ForbiddenError { + optional string message = 1; +} +message NotFoundError { + optional string message = 1; +} // Types message User { string address = 1; } @@ -291,6 +300,126 @@ message BlockUpdate { bool is_blocked = 2; } +// Private voice chats + +// Starting a private voice chat + +message StartPrivateVoiceChatPayload { + User callee = 1; +} + +message StartPrivateVoiceChatResponse { + message Ok { + string call_id = 1; + } + + oneof response { + Ok ok = 1; + InternalServerError internal_server_error = 2; + InvalidRequest invalid_request = 3; + ConflictingError conflicting_error = 4; + ForbiddenError forbidden_error = 5; + } +} + +// Accepting a private voice chat + +message AcceptPrivateVoiceChatPayload { + string call_id = 1; +} + +message AcceptPrivateVoiceChatResponse { + message Ok { + string call_id = 1; + PrivateVoiceChatCredentials credentials = 2; + } + + oneof response { + Ok ok = 1; + InternalServerError internal_server_error = 2; + InvalidRequest invalid_request = 3; + NotFoundError not_found = 4; + ForbiddenError forbidden_error = 5; + } +} + +// Rejecting a private voice chat + +message RejectPrivateVoiceChatPayload { + string call_id = 1; +} + +message RejectPrivateVoiceChatResponse { + message Ok { + string call_id = 1; + } + + oneof response { + Ok ok = 1; + InternalServerError internal_server_error = 2; + InvalidRequest invalid_request = 3; + NotFoundError not_found = 4; + } +} + +// Private voice chat updates + +enum PrivateVoiceChatStatus { + VOICE_CHAT_REQUESTED = 0; + VOICE_CHAT_ACCEPTED = 1; + VOICE_CHAT_REJECTED = 2; + VOICE_CHAT_ENDED = 3; + VOICE_CHAT_EXPIRED = 4; +} + +message PrivateVoiceChatCredentials { + string connection_url = 1; +} + +message PrivateVoiceChatUpdate { + string call_id = 1; + PrivateVoiceChatStatus status = 2; + optional User caller = 3; + optional User callee = 4; + optional PrivateVoiceChatCredentials credentials = 5; +} + +// Ending a private voice chat +// This is sent from the client to the server whenever the user wants to end the voice chat +// This could occur before the voice chat is accepted or after it is accepted + +message EndPrivateVoiceChatPayload { + string call_id = 1; +} + +message EndPrivateVoiceChatResponse { + message Ok { + string call_id = 1; + } + + oneof response { + Ok ok = 1; + InternalServerError internal_server_error = 2; + NotFoundError not_found = 3; + } +} + +// Get incoming private voice chat request + +message GetIncomingPrivateVoiceChatRequestResponse { + message Ok { + User caller = 1; + string call_id = 2; + } + + oneof response { + Ok ok = 1; + NotFoundError not_found = 2; + InternalServerError internal_server_error = 3; + } +} + + service SocialService { // Get the list of friends for the authenticated user rpc GetFriends(GetFriendsPayload) returns (PaginatedFriendsProfilesResponse) {} @@ -337,4 +466,22 @@ service SocialService { // Get the private messages privacy settings for the requested users rpc GetPrivateMessagesSettings(GetPrivateMessagesSettingsPayload) returns (GetPrivateMessagesSettingsResponse) {} + + // Start a private voice chat + rpc StartPrivateVoiceChat(StartPrivateVoiceChatPayload) returns (StartPrivateVoiceChatResponse) {} + + // Accept a private voice chat + rpc AcceptPrivateVoiceChat(AcceptPrivateVoiceChatPayload) returns (AcceptPrivateVoiceChatResponse) {} + + // Reject a private voice chat + rpc RejectPrivateVoiceChat(RejectPrivateVoiceChatPayload) returns (RejectPrivateVoiceChatResponse) {} + + // End a private voice chat + rpc EndPrivateVoiceChat(EndPrivateVoiceChatPayload) returns (EndPrivateVoiceChatResponse) {} + + // Get the incoming private voice chat request + rpc GetIncomingPrivateVoiceChatRequest(google.protobuf.Empty) returns (GetIncomingPrivateVoiceChatRequestResponse) {} + + // Subscribe to private voice chat updates + rpc SubscribeToPrivateVoiceChatUpdates(google.protobuf.Empty) returns (stream PrivateVoiceChatUpdate) {} } From 4d07dcd0f9bc395e3f24061ab2becf0fc06aa4f1 Mon Sep 17 00:00:00 2001 From: Alejo Thomas Ortega Date: Mon, 30 Jun 2025 07:22:46 -0300 Subject: [PATCH 12/54] feat: communities connectivity updates (#276) Co-authored-by: Kevin Szuchet --- .../social_service/v2/social_service_v2.proto | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/proto/decentraland/social_service/v2/social_service_v2.proto b/proto/decentraland/social_service/v2/social_service_v2.proto index d19047b9..09da2acb 100644 --- a/proto/decentraland/social_service/v2/social_service_v2.proto +++ b/proto/decentraland/social_service/v2/social_service_v2.proto @@ -300,6 +300,12 @@ message BlockUpdate { bool is_blocked = 2; } +message CommunityMemberConnectivityUpdate { + string community_id = 1; + User member = 2; + ConnectivityStatus status = 3; +} + // Private voice chats // Starting a private voice chat @@ -467,6 +473,10 @@ service SocialService { // Get the private messages privacy settings for the requested users rpc GetPrivateMessagesSettings(GetPrivateMessagesSettingsPayload) returns (GetPrivateMessagesSettingsResponse) {} + // Subscribe to community member connectivity updates: ONLINE, OFFLINE + rpc SubscribeToCommunityMemberConnectivityUpdates(google.protobuf.Empty) + returns (stream CommunityMemberConnectivityUpdate) {} + // Start a private voice chat rpc StartPrivateVoiceChat(StartPrivateVoiceChatPayload) returns (StartPrivateVoiceChatResponse) {} From db32c311e78bfd7e4b4c4f28ec3391c2619b5d74 Mon Sep 17 00:00:00 2001 From: Pravus Date: Fri, 4 Jul 2025 17:20:54 +0200 Subject: [PATCH 13/54] chore: bring main changes into experimental (#280) From 7b0267f38aa53c160da595ada1452c65a10a375d Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Melucci <163010988+AlejandroAlvarezMelucciDCL@users.noreply.github.com> Date: Fri, 4 Jul 2025 16:10:22 -0300 Subject: [PATCH 14/54] Chore: update experimental with main (#282) * feat: add z-index and opacity support for ui elements (#266) * Added z index property to ui transform * Update property id As per protocol-squad's request, I updated the index to match their implementation Signed-off-by: Alejandro Alvarez Melucci <163010988+AlejandroAlvarezMelucciDCL@users.noreply.github.com> * Added opacity property --------- Signed-off-by: Alejandro Alvarez Melucci <163010988+AlejandroAlvarezMelucciDCL@users.noreply.github.com> * feat: control day/night cycle (#277) * Added PBSkyboxTime. * Fixed typo * Lint pass fix * Added documentation * Fixed implementation to comply with ADR 259 * Removed unintended comment --------- Signed-off-by: Alejandro Alvarez Melucci <163010988+AlejandroAlvarezMelucciDCL@users.noreply.github.com> --- .../sdk/components/skybox_time.proto | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 proto/decentraland/sdk/components/skybox_time.proto diff --git a/proto/decentraland/sdk/components/skybox_time.proto b/proto/decentraland/sdk/components/skybox_time.proto new file mode 100644 index 00000000..60e1ec32 --- /dev/null +++ b/proto/decentraland/sdk/components/skybox_time.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; +package decentraland.sdk.components; + + +import "decentraland/sdk/components/common/id.proto"; + +option (common.ecs_component_id) = 1210; + +// The SkyboxTime component allows controlling the time of day for the skybox, +// affecting the lighting and appearance of the sky in the scene. +message PBSkyboxTime { + uint32 fixed_time = 1; // fixed time of day, represented as a number of seconds since the start of the day, where 0 is 00:00hs, 43200 is 12:00hs and 86400 is 24:00hs + optional TransitionMode transition_mode = 2; // default = TransitionMode.TM_FORWARD, controls the direction of time transitions +} + +// Controls the direction for animated skybox transitions +enum TransitionMode { + TM_FORWARD = 0; // transitions forward (default) + TM_BACKWARD = 1; // transitions backward +} \ No newline at end of file From bca3a64ff933f80f581777aed4beeaddb792960c Mon Sep 17 00:00:00 2001 From: Nicolas Lorusso <56365551+lorux0@users.noreply.github.com> Date: Thu, 10 Jul 2025 12:56:05 -0300 Subject: [PATCH 15/54] fix: emotes interpolation (#279) * add timestamp into emotes & instant into movement * fix retro compatibility * removed incremental id * add is_emoting flag --------- Co-authored-by: Vitaly Popuzin Co-authored-by: Vitaly Popuzin <35366872+popuz@users.noreply.github.com> --- proto/decentraland/kernel/comms/rfc4/comms.proto | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/proto/decentraland/kernel/comms/rfc4/comms.proto b/proto/decentraland/kernel/comms/rfc4/comms.proto index cf311343..7d3cdd47 100644 --- a/proto/decentraland/kernel/comms/rfc4/comms.proto +++ b/proto/decentraland/kernel/comms/rfc4/comms.proto @@ -55,10 +55,11 @@ message Movement { bool is_long_jump = 12; bool is_long_fall = 13; bool is_falling = 14; - bool is_stunned = 15; - float rotation_y = 16; + // interpolation + bool is_instant = 17; + bool is_emoting = 18; } message MovementCompressed { @@ -67,7 +68,7 @@ message MovementCompressed { } message PlayerEmote { - uint32 incremental_id = 1; + float timestamp = 1; string urn = 2; } From 346fe2dd77bbade6dc8bb586e168595181e072ac Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Melucci <163010988+AlejandroAlvarezMelucciDCL@users.noreply.github.com> Date: Tue, 15 Jul 2025 10:37:44 -0300 Subject: [PATCH 16/54] feat: experimental branch - control day night cycle (#278) From 5630df76f6558e779d7819062ffcc98299a3fd86 Mon Sep 17 00:00:00 2001 From: daniele-dcl Date: Mon, 28 Jul 2025 10:48:53 +0200 Subject: [PATCH 17/54] feat: improved light proto (#288) edited light proto --- .../sdk/components/light_source.proto | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/proto/decentraland/sdk/components/light_source.proto b/proto/decentraland/sdk/components/light_source.proto index 67c9a3c9..9a98322d 100644 --- a/proto/decentraland/sdk/components/light_source.proto +++ b/proto/decentraland/sdk/components/light_source.proto @@ -6,30 +6,23 @@ import "decentraland/common/texture.proto"; option (common.ecs_component_id) = 1079; message PBLightSource { - optional bool active = 4; // default = true, whether the lightSource is active or not. - optional decentraland.common.Color3 color = 1; // default = Color.white, the tint of the light, in RGB format where each component is a floating point value with a range from 0 to 1. - optional float brightness = 2; // default = 250, ranges from 1 (dim) to 100,000 (very bright), expressed in Lumens for Point and Spot. - optional float range = 3; // default = 10, how far the light travels, expressed in meters. + optional bool active = 1; // default = true, whether the lightSource is active or not. + optional decentraland.common.Color3 color = 2; // default = Color.white, the tint of the light, in RGB format where each component is a floating point value with a range from 0 to 1. + optional float intensity = 3; // default = 250, ranges from 1 (dim) to 100,000 (very bright), expressed in Lumens for Point and Spot. + optional float range = 4; // default = 0, how far the light travels, expressed in meters. If left at 0 will be computed automatically + optional bool shadow = 5; // default = false, whether the light casts shadows or not. + optional decentraland.common.TextureUnion shadow_mask_texture = 6; // Texture mask through which shadows are cast to simulate caustics, soft shadows, and light shapes such as light entering from a window. oneof type { - Point point = 6; - Spot spot = 7; + Point point = 7; + Spot spot = 8; } message Point { - optional ShadowType shadow = 5; // default = ShadowType.ST_NONE The type of shadow the light source supports. } message Spot { - optional float inner_angle = 1; // default = 21.8. Inner angle can't be higher than outer angle, otherwise will default to same value. Min value is 0. Max value is 179. - optional float outer_angle = 2; // default = 30. Outer angle can't be lower than inner angle, otherwise will inner angle will be set to same value. Max value is 179. - optional ShadowType shadow = 5; // default = ShadowType.ST_NONE The type of shadow the light source supports. - optional decentraland.common.TextureUnion shadow_mask_texture = 8; // Texture mask through which shadows are cast to simulate caustics, soft shadows, and light shapes such as light entering from a window. - } - - enum ShadowType { - ST_NONE = 0; // No shadows are cast from this LightSource. - ST_SOFT = 1; // More realistic type of shadow that reduces block artifacts, noise or pixelation, but requires more processing. - ST_HARD = 2; // Less realistic type of shadow but more performant, uses hard edges. + optional float inner_angle = 9; // default = 21.8. Inner angle can't be higher than outer angle, otherwise will default to same value. Min value is 0. Max value is 179. + optional float outer_angle = 10; // default = 30. Outer angle can't be lower than inner angle, otherwise will inner angle will be set to same value. Max value is 179. } } From 501dda6f24a76b6b040960a81c7eccd65685585e Mon Sep 17 00:00:00 2001 From: Juanma Hidalgo Date: Mon, 28 Jul 2025 10:59:06 +0200 Subject: [PATCH 18/54] feat: add communities voice chat proto objects (#283) * feat: add communities voice chat proto objects * feat: add SubscribeToCommunityVoiceChatUpdates * feat: add eof * feat: remove chat_id * feat: add community voice chat actions (#287) * feat: add ended status to CommunityVoiceChatUpdate (#293) * feat: add ended status to CommunityVoiceChatUpdate * feat: add CommunityVoiceChatStatus * feat: choose another name --------- Co-authored-by: Kevin Szuchet <31735779+kevinszuchet@users.noreply.github.com> --- .../social_service/v2/social_service_v2.proto | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/proto/decentraland/social_service/v2/social_service_v2.proto b/proto/decentraland/social_service/v2/social_service_v2.proto index 09da2acb..49cbfcf4 100644 --- a/proto/decentraland/social_service/v2/social_service_v2.proto +++ b/proto/decentraland/social_service/v2/social_service_v2.proto @@ -425,6 +425,145 @@ message GetIncomingPrivateVoiceChatRequestResponse { } } +// Community Voice Chat messages + +// Community voice chat credentials - specific type for community chats +message CommunityVoiceChatCredentials { + string connection_url = 1; +} + +// Starting a community voice chat +message StartCommunityVoiceChatPayload { + string community_id = 1; +} + +message StartCommunityVoiceChatResponse { + message Ok { + CommunityVoiceChatCredentials credentials = 1; // Moderator gets credentials immediately + } + + oneof response { + Ok ok = 1; + InvalidRequest invalid_request = 2; + ForbiddenError forbidden_error = 3; + ConflictingError conflicting_error = 4; + InternalServerError internal_server_error = 5; + } +} + +// Joining a community voice chat +message JoinCommunityVoiceChatPayload { + string community_id = 1; +} + +message JoinCommunityVoiceChatResponse { + message Ok { + string voice_chat_id = 1; + CommunityVoiceChatCredentials credentials = 2; + } + + oneof response { + Ok ok = 1; + InvalidRequest invalid_request = 2; + ForbiddenError forbidden_error = 3; + NotFoundError not_found_error = 4; + ConflictingError conflicting_error = 5; + InternalServerError internal_server_error = 6; + } +} + +// Request to speak in community voice chat +message RequestToSpeakInCommunityVoiceChatPayload { + string community_id = 1; +} + +message RequestToSpeakInCommunityVoiceChatResponse { + message Ok { + string message = 1; + } + + oneof response { + Ok ok = 1; + InvalidRequest invalid_request = 2; + ForbiddenError forbidden_error = 3; + NotFoundError not_found_error = 4; + InternalServerError internal_server_error = 5; + } +} + +// Promote speaker in community voice chat +message PromoteSpeakerInCommunityVoiceChatPayload { + string community_id = 1; + string user_address = 2; +} + +message PromoteSpeakerInCommunityVoiceChatResponse { + message Ok { + string message = 1; + } + + oneof response { + Ok ok = 1; + InvalidRequest invalid_request = 2; + ForbiddenError forbidden_error = 3; + NotFoundError not_found_error = 4; + InternalServerError internal_server_error = 5; + } +} + +// Demote speaker in community voice chat +message DemoteSpeakerInCommunityVoiceChatPayload { + string community_id = 1; + string user_address = 2; +} + +message DemoteSpeakerInCommunityVoiceChatResponse { + message Ok { + string message = 1; + } + + oneof response { + Ok ok = 1; + InvalidRequest invalid_request = 2; + ForbiddenError forbidden_error = 3; + NotFoundError not_found_error = 4; + InternalServerError internal_server_error = 5; + } +} + +// Kick player from community voice chat +message KickPlayerFromCommunityVoiceChatPayload { + string community_id = 1; + string user_address = 2; +} + +message KickPlayerFromCommunityVoiceChatResponse { + message Ok { + string message = 1; + } + + oneof response { + Ok ok = 1; + InvalidRequest invalid_request = 2; + ForbiddenError forbidden_error = 3; + NotFoundError not_found_error = 4; + InternalServerError internal_server_error = 5; + } +} + +enum CommunityVoiceChatStatus { + COMMUNITY_VOICE_CHAT_STARTED = 0; + COMMUNITY_VOICE_CHAT_ENDED = 1; +} + +// Community voice chat updates/events - 'started' and 'ended' status +message CommunityVoiceChatUpdate { + string community_id = 1; + string voice_chat_id = 2; + int64 created_at = 3; + CommunityVoiceChatStatus status = 4; // 'started' or 'ended' + optional int64 ended_at = 5; // Only present when status is 'ended' +} service SocialService { // Get the list of friends for the authenticated user @@ -494,4 +633,25 @@ service SocialService { // Subscribe to private voice chat updates rpc SubscribeToPrivateVoiceChatUpdates(google.protobuf.Empty) returns (stream PrivateVoiceChatUpdate) {} + + // Start a community voice chat (moderator/owner only) + rpc StartCommunityVoiceChat(StartCommunityVoiceChatPayload) returns (StartCommunityVoiceChatResponse) {} + + // Join a community voice chat + rpc JoinCommunityVoiceChat(JoinCommunityVoiceChatPayload) returns (JoinCommunityVoiceChatResponse) {} + + // Request to speak in community voice chat + rpc RequestToSpeakInCommunityVoiceChat(RequestToSpeakInCommunityVoiceChatPayload) returns (RequestToSpeakInCommunityVoiceChatResponse) {} + + // Promote speaker in community voice chat (moderator only) + rpc PromoteSpeakerInCommunityVoiceChat(PromoteSpeakerInCommunityVoiceChatPayload) returns (PromoteSpeakerInCommunityVoiceChatResponse) {} + + // Demote speaker in community voice chat (moderator only) + rpc DemoteSpeakerInCommunityVoiceChat(DemoteSpeakerInCommunityVoiceChatPayload) returns (DemoteSpeakerInCommunityVoiceChatResponse) {} + + // Kick player from community voice chat (moderator only) + rpc KickPlayerFromCommunityVoiceChat(KickPlayerFromCommunityVoiceChatPayload) returns (KickPlayerFromCommunityVoiceChatResponse) {} + + // Subscribe to community voice chat updates (only 'started' events) + rpc SubscribeToCommunityVoiceChatUpdates(google.protobuf.Empty) returns (stream CommunityVoiceChatUpdate) {} } From 25e60c0fbd7c2197ac4a718ba8576a6dbf3a35d0 Mon Sep 17 00:00:00 2001 From: Kevin Szuchet <31735779+kevinszuchet@users.noreply.github.com> Date: Mon, 28 Jul 2025 18:05:53 +0300 Subject: [PATCH 19/54] feat: Enhance social service errors (#294) * feat: add communities voice chat proto objects * feat: add SubscribeToCommunityVoiceChatUpdates * feat: add eof * feat: remove chat_id * feat: add community voice chat actions (#287) * feat: add ended status to CommunityVoiceChatUpdate (#293) * feat: add ended status to CommunityVoiceChatUpdate * feat: add CommunityVoiceChatStatus * feat: choose another name * feat: experimental branch - control day night cycle (#278) * feat: Enhance errors on Social Service definition * refactor: Try splitting into a new version * refactor: Move errors to new file * feat: Avoid exposing social service v3 * chore: Remove deprecated comment * style: Add spaces between messages + missing eof --------- Co-authored-by: Juanma Hidalgo Co-authored-by: Alejandro Alvarez Melucci <163010988+AlejandroAlvarezMelucciDCL@users.noreply.github.com> --- .../decentraland/social_service/errors.proto | 30 +++++++++++ .../social_service/v2/social_service_v2.proto | 26 ++-------- .../social_service/v3/social_service_v3.proto | 51 +++++++++++++++++++ 3 files changed, 84 insertions(+), 23 deletions(-) create mode 100644 proto/decentraland/social_service/errors.proto create mode 100644 proto/decentraland/social_service/v3/social_service_v3.proto diff --git a/proto/decentraland/social_service/errors.proto b/proto/decentraland/social_service/errors.proto new file mode 100644 index 00000000..cd978fc8 --- /dev/null +++ b/proto/decentraland/social_service/errors.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; +package decentraland.social_service; + +message InvalidFriendshipAction { + optional string message = 1; +} + +message InternalServerError { + optional string message = 1; +} + +message InvalidRequest { + optional string message = 1; +} + +message ProfileNotFound { + optional string message = 1; +} + +message ConflictingError { + optional string message = 1; +} + +message ForbiddenError { + optional string message = 1; +} + +message NotFoundError { + optional string message = 1; +} diff --git a/proto/decentraland/social_service/v2/social_service_v2.proto b/proto/decentraland/social_service/v2/social_service_v2.proto index 49cbfcf4..922dda77 100644 --- a/proto/decentraland/social_service/v2/social_service_v2.proto +++ b/proto/decentraland/social_service/v2/social_service_v2.proto @@ -2,29 +2,7 @@ syntax = "proto3"; package decentraland.social_service.v2; import "google/protobuf/empty.proto"; - -// Errors -message InvalidFriendshipAction { - optional string message = 1; -} -message InternalServerError { - optional string message = 1; -} -message InvalidRequest { - optional string message = 1; -} -message ProfileNotFound { - optional string message = 1; -} -message ConflictingError { - optional string message = 1; -} -message ForbiddenError { - optional string message = 1; -} -message NotFoundError { - optional string message = 1; -} +import "decentraland/social_service/errors.proto"; // Types message User { string address = 1; } @@ -127,6 +105,7 @@ message UpsertFriendshipResponse { Accepted accepted = 1; InvalidFriendshipAction invalid_friendship_action = 2; InternalServerError internal_server_error = 3; + InvalidRequest invalid_request = 4; } } @@ -182,6 +161,7 @@ message GetFriendshipStatusResponse { oneof response { Ok accepted = 1; InternalServerError internal_server_error = 2; + InvalidRequest invalid_request = 3; } } diff --git a/proto/decentraland/social_service/v3/social_service_v3.proto b/proto/decentraland/social_service/v3/social_service_v3.proto new file mode 100644 index 00000000..0315a165 --- /dev/null +++ b/proto/decentraland/social_service/v3/social_service_v3.proto @@ -0,0 +1,51 @@ +syntax = "proto3"; +package decentraland.social_service.v3; + +import "decentraland/social_service/errors.proto"; + +// Common types +message User { string address = 1; } + +message Profile { + string name = 1; + bool has_claimed_name = 2; + string profile_picture_url = 3; +} + +message Friend { + string address = 1; + optional Profile profile = 2; +} + +message Pagination { + int32 limit = 1; + int32 offset = 2; +} + +message PaginatedResponse { + int32 total = 1; + int32 page = 2; +} + +// Get mutual friends +message GetMutualFriendsPayload { + User user = 1; + optional Pagination pagination = 2; +} + +message GetMutualFriendsResponse { + message Ok { + repeated Friend friends = 1; + PaginatedResponse pagination_data = 2; + } + + oneof response { + Ok ok = 1; + InternalServerError internal_server_error = 2; + InvalidRequest invalid_request = 3; + } +} + +service SocialService { + rpc GetMutualFriends(GetMutualFriendsPayload) returns (GetMutualFriendsResponse) {} +} From 554f0f200a6a4ff8021f92e5a8e621f21d6e5bc7 Mon Sep 17 00:00:00 2001 From: Juanma Hidalgo Date: Fri, 1 Aug 2025 10:53:13 +0200 Subject: [PATCH 20/54] feat: add positions array to the CommunityVoiceChatUpdate (#297) * feat: add positions array to the CommunityVoiceChatUpdate * feat: add community fields needed as well * feat: remove the voice_chat_id * feat: add worlds --- .../social_service/v2/social_service_v2.proto | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/proto/decentraland/social_service/v2/social_service_v2.proto b/proto/decentraland/social_service/v2/social_service_v2.proto index 922dda77..85a108c9 100644 --- a/proto/decentraland/social_service/v2/social_service_v2.proto +++ b/proto/decentraland/social_service/v2/social_service_v2.proto @@ -536,13 +536,17 @@ enum CommunityVoiceChatStatus { COMMUNITY_VOICE_CHAT_ENDED = 1; } -// Community voice chat updates/events - 'started' and 'ended' status +// Community voice chat updates/events - 'started' and 'ended' status message CommunityVoiceChatUpdate { string community_id = 1; - string voice_chat_id = 2; - int64 created_at = 3; - CommunityVoiceChatStatus status = 4; // 'started' or 'ended' - optional int64 ended_at = 5; // Only present when status is 'ended' + int64 created_at = 2; + CommunityVoiceChatStatus status = 3; // 'started' or 'ended' + optional int64 ended_at = 4; // Only present when status is 'ended' + repeated string positions = 5; // Positions/coordinates associated with the community (world: false) + bool is_member = 6; // Whether the receiving user is a member of the community + string community_name = 7; // Name of the community + optional string community_image = 8; // Image/picture of the community + repeated string worlds = 9; // World names associated with the community (world: true) } service SocialService { From e03db907636e587c59a2034a48aaf55f2ae3bf8e Mon Sep 17 00:00:00 2001 From: Juanma Hidalgo Date: Mon, 4 Aug 2025 12:46:43 +0200 Subject: [PATCH 21/54] feat: add end community call (#298) --- .../social_service/v2/social_service_v2.proto | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/proto/decentraland/social_service/v2/social_service_v2.proto b/proto/decentraland/social_service/v2/social_service_v2.proto index 85a108c9..be720657 100644 --- a/proto/decentraland/social_service/v2/social_service_v2.proto +++ b/proto/decentraland/social_service/v2/social_service_v2.proto @@ -531,6 +531,25 @@ message KickPlayerFromCommunityVoiceChatResponse { } } +// End community voice chat (moderator/owner only) +message EndCommunityVoiceChatPayload { + string community_id = 1; +} + +message EndCommunityVoiceChatResponse { + message Ok { + string message = 1; + } + + oneof response { + Ok ok = 1; + InvalidRequest invalid_request = 2; + ForbiddenError forbidden_error = 3; + NotFoundError not_found_error = 4; + InternalServerError internal_server_error = 5; + } +} + enum CommunityVoiceChatStatus { COMMUNITY_VOICE_CHAT_STARTED = 0; COMMUNITY_VOICE_CHAT_ENDED = 1; @@ -636,6 +655,9 @@ service SocialService { // Kick player from community voice chat (moderator only) rpc KickPlayerFromCommunityVoiceChat(KickPlayerFromCommunityVoiceChatPayload) returns (KickPlayerFromCommunityVoiceChatResponse) {} + // End community voice chat (moderator/owner only) + rpc EndCommunityVoiceChat(EndCommunityVoiceChatPayload) returns (EndCommunityVoiceChatResponse) {} + // Subscribe to community voice chat updates (only 'started' events) rpc SubscribeToCommunityVoiceChatUpdates(google.protobuf.Empty) returns (stream CommunityVoiceChatUpdate) {} } From 16c6dee28cfd6e6eb5d01c0c5f41986567daa0e9 Mon Sep 17 00:00:00 2001 From: Juanma Hidalgo Date: Tue, 5 Aug 2025 15:48:31 +0200 Subject: [PATCH 22/54] feat: add action to reject request to speak (#299) --- .../social_service/v2/social_service_v2.proto | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/proto/decentraland/social_service/v2/social_service_v2.proto b/proto/decentraland/social_service/v2/social_service_v2.proto index be720657..6afddb56 100644 --- a/proto/decentraland/social_service/v2/social_service_v2.proto +++ b/proto/decentraland/social_service/v2/social_service_v2.proto @@ -531,6 +531,26 @@ message KickPlayerFromCommunityVoiceChatResponse { } } +// Reject speak request in community voice chat +message RejectSpeakRequestInCommunityVoiceChatPayload { + string community_id = 1; + string user_address = 2; +} + +message RejectSpeakRequestInCommunityVoiceChatResponse { + message Ok { + string message = 1; + } + + oneof response { + Ok ok = 1; + InvalidRequest invalid_request = 2; + ForbiddenError forbidden_error = 3; + NotFoundError not_found_error = 4; + InternalServerError internal_server_error = 5; + } +} + // End community voice chat (moderator/owner only) message EndCommunityVoiceChatPayload { string community_id = 1; @@ -655,6 +675,9 @@ service SocialService { // Kick player from community voice chat (moderator only) rpc KickPlayerFromCommunityVoiceChat(KickPlayerFromCommunityVoiceChatPayload) returns (KickPlayerFromCommunityVoiceChatResponse) {} + // Reject speak request in community voice chat (moderator only) + rpc RejectSpeakRequestInCommunityVoiceChat(RejectSpeakRequestInCommunityVoiceChatPayload) returns (RejectSpeakRequestInCommunityVoiceChatResponse) {} + // End community voice chat (moderator/owner only) rpc EndCommunityVoiceChat(EndCommunityVoiceChatPayload) returns (EndCommunityVoiceChatResponse) {} From d8676ddf39d8f24391cd80b0e914cc87a4d4ae4f Mon Sep 17 00:00:00 2001 From: Gon Pombo Date: Thu, 7 Aug 2025 15:42:42 -0300 Subject: [PATCH 23/54] fix experimental validate check (#302) --- .github/workflows/validate.yml | 15 ++++++++++++--- Makefile | 2 +- scripts/check-proto-compabitility.sh | 21 ++++++++++++--------- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 264fbd47..bcd73ab0 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -1,7 +1,9 @@ on: - push: - branches-ignore: + pull_request: + types: [opened, synchronize, reopened] + branches: - "main" + - "experimental" name: validate-compatibility jobs: @@ -13,10 +15,17 @@ jobs: - name: install run: make install - name: buf breaking - run: make buf-breaking + env: + BASE_BRANCH: ${{ github.event.pull_request.base.ref }} + run: | + echo "base_ref=${{ github.event.pull_request.base.ref }}" + echo "BASE_BRANCH=$BASE_BRANCH" + make buf-breaking - name: buf lint run: make buf-lint - name: build and compile test run: make test - name: run the check script + env: + BASE_BRANCH: ${{ github.event.pull_request.base.ref }} run: ./scripts/check-proto-compabitility.sh diff --git a/Makefile b/Makefile index cba0f90c..59cd45af 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ buf-build: node_modules/.bin/buf ./node_modules/.bin/buf build proto/ buf-breaking: node_modules/.bin/buf - ./node_modules/.bin/buf breaking proto/ --against 'https://github.com/decentraland/protocol.git#subdir=proto' + ./node_modules/.bin/buf breaking proto/ --against 'https://github.com/decentraland/protocol.git#branch=$(or $(BASE_BRANCH),main),subdir=proto' test: buf-lint bash scripts/test.sh diff --git a/scripts/check-proto-compabitility.sh b/scripts/check-proto-compabitility.sh index c7f7120a..a067f937 100755 --- a/scripts/check-proto-compabitility.sh +++ b/scripts/check-proto-compabitility.sh @@ -1,21 +1,24 @@ #!/bin/bash set -e -x -# Download the main branch ref zip -protocol_main_zip_url="https://github.com/decentraland/protocol/archive/refs/heads/main.zip" -protocol_main_zip_local="./protocol-main.zip" +# Use BASE_BRANCH environment variable or default to main +BRANCH="${BASE_BRANCH:-main}" + +# Download the reference branch zip +protocol_branch_zip_url="https://github.com/decentraland/protocol/archive/refs/heads/${BRANCH}.zip" +protocol_branch_zip_local="./protocol-${BRANCH}.zip" TMP_ZIP_DIR=$(mktemp -d) -curl -L "$protocol_main_zip_url" -o "$protocol_main_zip_local" -unzip "$protocol_main_zip_local" -d "$TMP_ZIP_DIR" -rm "$protocol_main_zip_local" || true +curl -L "$protocol_branch_zip_url" -o "$protocol_branch_zip_local" +unzip "$protocol_branch_zip_local" -d "$TMP_ZIP_DIR" +rm "$protocol_branch_zip_local" || true -ln -s "$(pwd)/node_modules" "$TMP_ZIP_DIR/protocol-main/node_modules" +ln -s "$(pwd)/node_modules" "$TMP_ZIP_DIR/protocol-${BRANCH}/node_modules" # Run the `proto-compatibility-tool` and exclude the downloaded folder. -echo "Checking the compatibility against $base_url" -./node_modules/.bin/proto-compatibility-tool --recursive "$TMP_ZIP_DIR/protocol-main/proto" "proto" +echo "Checking the compatibility against branch: ${BRANCH}" +./node_modules/.bin/proto-compatibility-tool --recursive "$TMP_ZIP_DIR/protocol-${BRANCH}/proto" "proto" # ../proto-compatibility-tool/dist/bin.js --recursive "$TMP_ZIP_DIR/protocol-main" "." # rm -rf "$TMP_ZIP_DIR" || true \ No newline at end of file From e9319d069b04189d7915e3ec81eb1794016072a1 Mon Sep 17 00:00:00 2001 From: Vitaly Popuzin <35366872+popuz@users.noreply.github.com> Date: Tue, 12 Aug 2025 19:44:03 +0300 Subject: [PATCH 24/54] emote timestamp as not-breaking change (#303) --- proto/decentraland/kernel/comms/rfc4/comms.proto | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proto/decentraland/kernel/comms/rfc4/comms.proto b/proto/decentraland/kernel/comms/rfc4/comms.proto index 7d3cdd47..f6b4e48f 100644 --- a/proto/decentraland/kernel/comms/rfc4/comms.proto +++ b/proto/decentraland/kernel/comms/rfc4/comms.proto @@ -68,8 +68,9 @@ message MovementCompressed { } message PlayerEmote { - float timestamp = 1; + uint32 incremental_id = 1; string urn = 2; + float timestamp = 3; } message SceneEmote { From 7dae801d3af316702ef2ce39f46a3374b0af679a Mon Sep 17 00:00:00 2001 From: Juanma Hidalgo Date: Mon, 18 Aug 2025 10:28:33 +0200 Subject: [PATCH 25/54] feat: add is_raising_hand to RequestToSpeakInCommunityVoiceChatPayload (#304) --- proto/decentraland/social_service/v2/social_service_v2.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/proto/decentraland/social_service/v2/social_service_v2.proto b/proto/decentraland/social_service/v2/social_service_v2.proto index 6afddb56..ccb28148 100644 --- a/proto/decentraland/social_service/v2/social_service_v2.proto +++ b/proto/decentraland/social_service/v2/social_service_v2.proto @@ -455,6 +455,7 @@ message JoinCommunityVoiceChatResponse { // Request to speak in community voice chat message RequestToSpeakInCommunityVoiceChatPayload { string community_id = 1; + bool is_raising_hand = 2; // true to raise hand (request to speak), false to lower hand (withdraw request) } message RequestToSpeakInCommunityVoiceChatResponse { From 905b9630e2c46d34c5a654294ae2fa4fa56ae881 Mon Sep 17 00:00:00 2001 From: Juanma Hidalgo Date: Tue, 30 Sep 2025 20:12:26 +0200 Subject: [PATCH 26/54] feat: add mute speaker chat (#311) * feat: add mute speaker chat * fix: remove messages already defined --- .../social_service/v2/social_service_v2.proto | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/proto/decentraland/social_service/v2/social_service_v2.proto b/proto/decentraland/social_service/v2/social_service_v2.proto index ccb28148..5608eab2 100644 --- a/proto/decentraland/social_service/v2/social_service_v2.proto +++ b/proto/decentraland/social_service/v2/social_service_v2.proto @@ -552,6 +552,27 @@ message RejectSpeakRequestInCommunityVoiceChatResponse { } } +// Community Voice Chat Messages +message MuteSpeakerFromCommunityVoiceChatPayload { + string community_id = 1; + string user_address = 2; + bool muted = 3; +} + +message MuteSpeakerFromCommunityVoiceChatResponse { + message Ok { + bool muted = 1; // The resulting mute state + } + + oneof response { + Ok ok = 1; + ForbiddenError forbidden_error = 2; + NotFoundError not_found_error = 3; + InvalidRequest invalid_request = 4; + InternalServerError internal_server_error = 5; + } +} + // End community voice chat (moderator/owner only) message EndCommunityVoiceChatPayload { string community_id = 1; @@ -684,4 +705,7 @@ service SocialService { // Subscribe to community voice chat updates (only 'started' events) rpc SubscribeToCommunityVoiceChatUpdates(google.protobuf.Empty) returns (stream CommunityVoiceChatUpdate) {} + + // Mute or unmute a speaker in a community voice chat + rpc MuteSpeakerFromCommunityVoiceChat(MuteSpeakerFromCommunityVoiceChatPayload) returns (MuteSpeakerFromCommunityVoiceChatResponse) {} } From b26a1a0e1b86b1dc2fd8f6d58a0d77eeb980f569 Mon Sep 17 00:00:00 2001 From: daniele-dcl Date: Mon, 24 Nov 2025 10:59:17 +0100 Subject: [PATCH 27/54] feat: locomotion settings (#320) new avatar locomotion settings component --- .../avatar_locomotion_settings.proto | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 proto/decentraland/sdk/components/avatar_locomotion_settings.proto diff --git a/proto/decentraland/sdk/components/avatar_locomotion_settings.proto b/proto/decentraland/sdk/components/avatar_locomotion_settings.proto new file mode 100644 index 00000000..a16f2169 --- /dev/null +++ b/proto/decentraland/sdk/components/avatar_locomotion_settings.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; + +package decentraland.sdk.components; + +import "decentraland/sdk/components/common/id.proto"; + +option (common.ecs_component_id) = 1211; + +// The PBAvatarLocomotionSettings component allows scenes to modify locomotion settings defining things such +// as the avatar movement speed, jump height etc. +message PBAvatarLocomotionSettings { + optional float walk_speed = 1; // Maximum speed when walking (in meters per second) + optional float jog_speed = 2; // Maximum speed when jogging (in meters per second) + optional float run_speed = 3; // Maximum speed when running (in meters per second) + optional float jump_height = 4; // Height of a regular jump (in meters) + optional float run_jump_height = 5; // Height of a jump while running (in meters) + optional float hard_landing_cooldown = 6; // Cooldown time after a hard landing before the avatar can move again (in seconds) +} From b4ca911f2f89f2f62a257b2e4c9393d008785142 Mon Sep 17 00:00:00 2001 From: lorenzo-ranciaffi <41125365+lorenzo-ranciaffi@users.noreply.github.com> Date: Tue, 25 Nov 2025 16:58:37 +0100 Subject: [PATCH 28/54] feat: add modifier input action (#316) --- proto/decentraland/sdk/components/common/input_action.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/proto/decentraland/sdk/components/common/input_action.proto b/proto/decentraland/sdk/components/common/input_action.proto index 824d9cf0..38630a09 100644 --- a/proto/decentraland/sdk/components/common/input_action.proto +++ b/proto/decentraland/sdk/components/common/input_action.proto @@ -17,6 +17,7 @@ enum InputAction { IA_ACTION_4 = 11; IA_ACTION_5 = 12; IA_ACTION_6 = 13; + IA_MODIFIER = 14; } // PointerEventType is a kind of interaction that can be detected. From 64a533052b58414f60a1b2ba599cf2207f331d6b Mon Sep 17 00:00:00 2001 From: Nick Khalow <71646502+NickKhalow@users.noreply.github.com> Date: Fri, 5 Dec 2025 17:00:30 +0200 Subject: [PATCH 29/54] feat: audio analysis (#328) * component definition * specify package in the component def * apply lint * omit experimental features * support multiple analysis modes * relax order * optional args --- .../sdk/components/audio_analysis.proto | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 proto/decentraland/sdk/components/audio_analysis.proto diff --git a/proto/decentraland/sdk/components/audio_analysis.proto b/proto/decentraland/sdk/components/audio_analysis.proto new file mode 100644 index 00000000..940f18af --- /dev/null +++ b/proto/decentraland/sdk/components/audio_analysis.proto @@ -0,0 +1,44 @@ +syntax = "proto3"; +package decentraland.sdk.components; + +import "decentraland/sdk/components/common/id.proto"; +option (common.ecs_component_id) = 1212; + +enum PBAudioAnalysisMode { + MODE_RAW = 0; + MODE_LOGARITHMIC = 1; +} + +message PBAudioAnalysis { + + // Parameters section + PBAudioAnalysisMode mode = 1; + + // Used only when mode == MODE_LOGARITHMIC + optional float amplitude_gain = 100; + optional float bands_gain = 101; + // End when mode == MODE_LOGARITHMIC + + // End Parameters section + + // Result section + float amplitude = 200; + + // Protobuf doesn't support fixed arrays -> 8 band fields + float band_0 = 201; + float band_1 = 202; + float band_2 = 203; + float band_3 = 204; + float band_4 = 205; + float band_5 = 206; + float band_6 = 207; + float band_7 = 208; + + // End Result section + + // Future fields + // float spectral_centroid = 13; + // float spectral_flux = 14; + // bool onset = 15; + // float bpm = 16; +} From 1d2c132b3acabf0cbf3cdb2d96674374832e0134 Mon Sep 17 00:00:00 2001 From: Alex Villalba Date: Sat, 6 Dec 2025 20:46:54 +0000 Subject: [PATCH 30/54] Added messages and fields to support Social Emotes (#309) * Added social emote outcome index When the outcome is > -1 it means the avatar started an animation that uses the configuration in one of the outcomes defined in the emote's metadata. * Added is_reacting to know which of the animations in the outcome is being played * Added social emote initiator wallet address * Added target avatar * Added StopEmote message * StopEmote message replaced with field in PlayerEmote * Added is_repeating field * Added interaction_id field * Created LookAtPosition message * Added target avatar wallet address to LookAtPosition message * Added LookAtPosition to oneof list --------- Co-authored-by: Mateo Miccino --- .github/workflows/build-and-publish.yml | 2 ++ .../kernel/comms/rfc4/comms.proto | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 0f6c6a7a..ccf63221 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -22,6 +22,8 @@ jobs: dcl_protocol_s3_bucket_key: ${{ steps.publish_dcl_protocol.outputs.s3-bucket-key }} steps: - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: install run: make install - name: buf build diff --git a/proto/decentraland/kernel/comms/rfc4/comms.proto b/proto/decentraland/kernel/comms/rfc4/comms.proto index f6b4e48f..f480917c 100644 --- a/proto/decentraland/kernel/comms/rfc4/comms.proto +++ b/proto/decentraland/kernel/comms/rfc4/comms.proto @@ -18,6 +18,7 @@ message Packet { PlayerEmote player_emote = 9; SceneEmote scene_emote = 10; MovementCompressed movement_compressed = 12; + LookAtPosition look_at_position = 13; } uint32 protocol_version = 11; } @@ -71,6 +72,24 @@ message PlayerEmote { uint32 incremental_id = 1; string urn = 2; float timestamp = 3; + optional bool is_stopping = 4; // true means the emote has been stopped in the sender's client + optional bool is_repeating = 5; // true when it is not the first time the looping animation plays + optional int32 interaction_id = 6; // identifies an interaction univocaly, established when the start animation is triggered + optional int32 social_emote_outcome = 7; // -1 means it does not use an outcome animation + optional bool is_reacting = 8; // to a social emote started by other user + optional string social_emote_initiator = 9; // wallet address of the user that initiated social emote + optional string target_avatar = 10; // wallet address of the user whose avatar is the target of a directed emote +} + +// Message sent to force an avatar to look at a position +message LookAtPosition +{ + float timestamp = 1; + // world position + float position_x = 2; + float position_y = 3; + float position_z = 4; + string target_avatar_wallet_address = 5; } message SceneEmote { From b2ab885e610f54d772904f91b44e72188b697c2e Mon Sep 17 00:00:00 2001 From: daniele-dcl Date: Wed, 17 Dec 2025 13:01:32 +0100 Subject: [PATCH 31/54] feat: head sync (#326) * added head-sync data to comms proto * added compressed data * added enabled flag * cleaned-up docs and naming of fields * separated yaw and pitch enabled flags --- proto/decentraland/kernel/comms/rfc4/comms.proto | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/proto/decentraland/kernel/comms/rfc4/comms.proto b/proto/decentraland/kernel/comms/rfc4/comms.proto index f480917c..d6f402d0 100644 --- a/proto/decentraland/kernel/comms/rfc4/comms.proto +++ b/proto/decentraland/kernel/comms/rfc4/comms.proto @@ -61,11 +61,17 @@ message Movement { // interpolation bool is_instant = 17; bool is_emoting = 18; + // head-sync (enabled flags + world-space yaw and pitch angles, in degrees) + bool head_ik_yaw_enabled = 19; + bool head_ik_pitch_enabled = 20; + float head_yaw = 21; + float head_pitch = 22; } message MovementCompressed { int32 temporal_data = 1; // bit-compressed: timestamp + animations int64 movement_data = 2; // bit-compressed: position + velocity + int32 head_sync_data = 3; // bit-compressed: enabled flags + yaw + pitch } message PlayerEmote { From d8a2d5b681d7a9799cbc0d1d92305452f74c9ea7 Mon Sep 17 00:00:00 2001 From: Nicolas Lorusso <56365551+lorux0@users.noreply.github.com> Date: Wed, 28 Jan 2026 11:03:35 -0300 Subject: [PATCH 32/54] fix: remove social service v3 (#346) remove social v3 --- .../social_service/v3/social_service_v3.proto | 51 ------------------- 1 file changed, 51 deletions(-) delete mode 100644 proto/decentraland/social_service/v3/social_service_v3.proto diff --git a/proto/decentraland/social_service/v3/social_service_v3.proto b/proto/decentraland/social_service/v3/social_service_v3.proto deleted file mode 100644 index 0315a165..00000000 --- a/proto/decentraland/social_service/v3/social_service_v3.proto +++ /dev/null @@ -1,51 +0,0 @@ -syntax = "proto3"; -package decentraland.social_service.v3; - -import "decentraland/social_service/errors.proto"; - -// Common types -message User { string address = 1; } - -message Profile { - string name = 1; - bool has_claimed_name = 2; - string profile_picture_url = 3; -} - -message Friend { - string address = 1; - optional Profile profile = 2; -} - -message Pagination { - int32 limit = 1; - int32 offset = 2; -} - -message PaginatedResponse { - int32 total = 1; - int32 page = 2; -} - -// Get mutual friends -message GetMutualFriendsPayload { - User user = 1; - optional Pagination pagination = 2; -} - -message GetMutualFriendsResponse { - message Ok { - repeated Friend friends = 1; - PaginatedResponse pagination_data = 2; - } - - oneof response { - Ok ok = 1; - InternalServerError internal_server_error = 2; - InvalidRequest invalid_request = 3; - } -} - -service SocialService { - rpc GetMutualFriends(GetMutualFriendsPayload) returns (GetMutualFriendsResponse) {} -} From 7427639fc9db80fd88af9bdd909b84eff401d88c Mon Sep 17 00:00:00 2001 From: Nicolas Lorusso <56365551+lorux0@users.noreply.github.com> Date: Wed, 28 Jan 2026 12:05:15 -0300 Subject: [PATCH 33/54] feat: pointer max player distance (#344) --- proto/decentraland/sdk/components/pointer_events.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/proto/decentraland/sdk/components/pointer_events.proto b/proto/decentraland/sdk/components/pointer_events.proto index ec90a075..1c0c66b1 100644 --- a/proto/decentraland/sdk/components/pointer_events.proto +++ b/proto/decentraland/sdk/components/pointer_events.proto @@ -27,6 +27,7 @@ message PBPointerEvents { optional float max_distance = 3; // range of interaction (default 10) optional bool show_feedback = 4; // enable or disable hover text and highlight (default true) optional bool show_highlight = 5; // enable or disable hover highlight (default true) + optional float max_player_distance = 6; // range of interaction from the avatar's position (default 0) } message Entry { From 1c3d55b3b8d342af220eb1d7fc7e1ed3ebe25161 Mon Sep 17 00:00:00 2001 From: Maurizio Farmi Date: Fri, 27 Feb 2026 12:09:58 +0100 Subject: [PATCH 34/54] feat: non pointer interaction (#353) * feat: non pointer interaction - add InteractionType enum to input_action.proto * - Added optional priority field to PBPointerEvents for resolution order of overlapping events. - Introduced interaction_type field in Entry message to specify the type of interaction source. * Add PET_PROXIMITY_ENTER and PET_PROXIMITY_LEAVE to PointerEventType enum * Make interaction_type field optional in PBPointerEvents Entry message --- .../decentraland/sdk/components/common/input_action.proto | 7 +++++++ proto/decentraland/sdk/components/pointer_events.proto | 2 ++ 2 files changed, 9 insertions(+) diff --git a/proto/decentraland/sdk/components/common/input_action.proto b/proto/decentraland/sdk/components/common/input_action.proto index 38630a09..2220c08a 100644 --- a/proto/decentraland/sdk/components/common/input_action.proto +++ b/proto/decentraland/sdk/components/common/input_action.proto @@ -26,4 +26,11 @@ enum PointerEventType { PET_DOWN = 1; PET_HOVER_ENTER = 2; PET_HOVER_LEAVE = 3; + PET_PROXIMITY_ENTER = 4; + PET_PROXIMITY_LEAVE = 5; +} + +enum InteractionType { + CURSOR = 0; + PROXIMITY = 1; } \ No newline at end of file diff --git a/proto/decentraland/sdk/components/pointer_events.proto b/proto/decentraland/sdk/components/pointer_events.proto index 6660d078..a000db81 100644 --- a/proto/decentraland/sdk/components/pointer_events.proto +++ b/proto/decentraland/sdk/components/pointer_events.proto @@ -51,11 +51,13 @@ message PBPointerEvents { optional bool show_feedback = 4; // enable or disable hover text and highlight (default true) optional bool show_highlight = 5; // enable or disable hover highlight (default true) optional float max_player_distance = 6; // range of interaction from the avatar's position (default 0) + optional uint32 priority = 7; // resolution order when multiple events overlap, higher wins (default 0) } message Entry { common.PointerEventType event_type = 1; // the kind of interaction to detect Info event_info = 2; // additional configuration for this detection + optional common.InteractionType interaction_type = 3; // the type of interaction source (default 0 == CURSOR) } repeated Entry pointer_events = 1; // the list of relevant events to detect From 203bb4ac5b1da6597f9e361f5ada955a531ada89 Mon Sep 17 00:00:00 2001 From: Pravus Date: Mon, 2 Mar 2026 05:04:38 +0100 Subject: [PATCH 35/54] feat: tween MoveRotateScale mode (#358) --- proto/decentraland/sdk/components/tween.proto | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/proto/decentraland/sdk/components/tween.proto b/proto/decentraland/sdk/components/tween.proto index f95e188e..a5ec516a 100644 --- a/proto/decentraland/sdk/components/tween.proto +++ b/proto/decentraland/sdk/components/tween.proto @@ -19,6 +19,8 @@ message PBTween { RotateContinuous rotate_continuous = 9; MoveContinuous move_continuous = 10; TextureMoveContinuous texture_move_continuous = 11; + MoveRotateScale move_rotate_scale = 12; + MoveRotateScaleContinuous move_rotate_scale_continuous = 13; } optional bool playing = 6; // default true (pause or running) @@ -41,6 +43,15 @@ message Scale { decentraland.common.Vector3 end = 2; } +message MoveRotateScale { + decentraland.common.Vector3 position_start = 1; + decentraland.common.Vector3 position_end = 2; + decentraland.common.Quaternion rotation_start = 3; + decentraland.common.Quaternion rotation_end = 4; + decentraland.common.Vector3 scale_start = 5; + decentraland.common.Vector3 scale_end = 6; +} + // This tween mode allows to move the texture of a PbrMaterial or UnlitMaterial. // You can also specify the movement type (offset or tiling) message TextureMove { @@ -59,6 +70,13 @@ message MoveContinuous { float speed = 2; } +message MoveRotateScaleContinuous { + decentraland.common.Vector3 position_direction = 1; + decentraland.common.Quaternion rotation_direction = 2; + decentraland.common.Vector3 scale_direction = 3; + float speed = 4; +} + message TextureMoveContinuous { decentraland.common.Vector2 direction = 1; float speed = 2; From 276e5b77b63cfe7cb7b22c643e1a3321412e60b5 Mon Sep 17 00:00:00 2001 From: daniele-dcl Date: Mon, 2 Mar 2026 17:01:03 +0100 Subject: [PATCH 36/54] feat: double jump and glide (#340) * replaced jumping flag with jump count integer * added gliding flag to movement message * added double jump and gliding flags to the input modifier component * added double jump height and gliding speed parameters to the avatar locomotion settings component * added gliding falling speed * changed gliding flag to glide state enum * restored is jumping flag for compatibility * marked is jumping field deprecated Co-authored-by: Pravus Signed-off-by: daniele-dcl * fix for deprecated field --------- Signed-off-by: daniele-dcl Co-authored-by: Pravus --- proto/decentraland/kernel/comms/rfc4/comms.proto | 10 +++++++++- .../sdk/components/avatar_locomotion_settings.proto | 3 +++ proto/decentraland/sdk/components/input_modifier.proto | 2 ++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/proto/decentraland/kernel/comms/rfc4/comms.proto b/proto/decentraland/kernel/comms/rfc4/comms.proto index d6f402d0..2f440bef 100644 --- a/proto/decentraland/kernel/comms/rfc4/comms.proto +++ b/proto/decentraland/kernel/comms/rfc4/comms.proto @@ -52,11 +52,13 @@ message Movement { float movement_blend_value = 8; float slide_blend_value = 9; bool is_grounded = 10; - bool is_jumping = 11; + bool is_jumping = 11; // deprecated + int32 jump_count = 24; bool is_long_jump = 12; bool is_long_fall = 13; bool is_falling = 14; bool is_stunned = 15; + GlideState glide_state = 23; float rotation_y = 16; // interpolation bool is_instant = 17; @@ -66,6 +68,12 @@ message Movement { bool head_ik_pitch_enabled = 20; float head_yaw = 21; float head_pitch = 22; + enum GlideState { + PROP_CLOSED = 0; + OPENING_PROP = 1; + GLIDING = 2; + CLOSING_PROP = 3; + } } message MovementCompressed { diff --git a/proto/decentraland/sdk/components/avatar_locomotion_settings.proto b/proto/decentraland/sdk/components/avatar_locomotion_settings.proto index a16f2169..fc1511c1 100644 --- a/proto/decentraland/sdk/components/avatar_locomotion_settings.proto +++ b/proto/decentraland/sdk/components/avatar_locomotion_settings.proto @@ -15,4 +15,7 @@ message PBAvatarLocomotionSettings { optional float jump_height = 4; // Height of a regular jump (in meters) optional float run_jump_height = 5; // Height of a jump while running (in meters) optional float hard_landing_cooldown = 6; // Cooldown time after a hard landing before the avatar can move again (in seconds) + optional float double_jump_height = 7; // Height of the double jump (in meters) + optional float gliding_speed = 8; // Maximum speed when gliding (in meters per second) + optional float gliding_falling_speed = 9; // Maximum falling speed when gliding (in meters per second) } diff --git a/proto/decentraland/sdk/components/input_modifier.proto b/proto/decentraland/sdk/components/input_modifier.proto index 5bd978c6..dd5b94dd 100644 --- a/proto/decentraland/sdk/components/input_modifier.proto +++ b/proto/decentraland/sdk/components/input_modifier.proto @@ -12,6 +12,8 @@ message PBInputModifier { optional bool disable_run = 4; optional bool disable_jump = 5; optional bool disable_emote = 6; + optional bool disable_double_jump = 7; + optional bool disable_gliding = 8; } oneof mode { From 50f6aeb33b7e3c5736606a3a5c634b571ff2c0f0 Mon Sep 17 00:00:00 2001 From: Vitaly Popuzin <35366872+popuz@users.noreply.github.com> Date: Mon, 2 Mar 2026 17:16:22 +0100 Subject: [PATCH 37/54] Chore: sync Experimental with Main (#365) * feat: add name color (#350) Co-authored-by: Kevin Szuchet <31735779+kevinszuchet@users.noreply.github.com> * Feat: physics components for impulse and force (#363) Adds 2 components for Physics Impulse and Force which represents total Impulse/Force applied by the scene to the Entity. Currently limited only to the player entity [js-toolchain PR](https://github.com/decentraland/js-sdk-toolchain/pull/1338) --------- Co-authored-by: MauroCicerchia Co-authored-by: Kevin Szuchet <31735779+kevinszuchet@users.noreply.github.com> --- .../components/physics_combined_force.proto | 21 +++++++++++++++++ .../components/physics_combined_impulse.proto | 23 +++++++++++++++++++ .../social_service/v2/social_service_v2.proto | 3 +++ 3 files changed, 47 insertions(+) create mode 100644 proto/decentraland/sdk/components/physics_combined_force.proto create mode 100644 proto/decentraland/sdk/components/physics_combined_impulse.proto diff --git a/proto/decentraland/sdk/components/physics_combined_force.proto b/proto/decentraland/sdk/components/physics_combined_force.proto new file mode 100644 index 00000000..a78800c4 --- /dev/null +++ b/proto/decentraland/sdk/components/physics_combined_force.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package decentraland.sdk.components; + +import "decentraland/common/vectors.proto"; +import "decentraland/sdk/components/common/id.proto"; + +option (common.ecs_component_id) = 1216; + +/** + * This component applies a continuous physics force. + + * @remarks Low-level component. Use Physics.applyForceToPlayer()/.removeForceToPlayer() instead. + * Direct manipulation will conflict with the force accumulation registry. + * Summary component: stores the accumulated result of all active forces registered by the scene in the current frame. + + * State-like component: the force is applied every physics tick while the component is present on the entity. +*/ +message PBPhysicsCombinedForce { + decentraland.common.Vector3 vector = 1; // Includes force direction and magnitude +} \ No newline at end of file diff --git a/proto/decentraland/sdk/components/physics_combined_impulse.proto b/proto/decentraland/sdk/components/physics_combined_impulse.proto new file mode 100644 index 00000000..a3584add --- /dev/null +++ b/proto/decentraland/sdk/components/physics_combined_impulse.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package decentraland.sdk.components; + +import "decentraland/common/vectors.proto"; +import "decentraland/sdk/components/common/id.proto"; + +option (common.ecs_component_id) = 1215; + +/** + * This component applies a one-shot physics summary impulse. + + * @remarks Low-level component. Use Physics.applyImpulseToPlayer() instead. + * Direct manipulation will conflict with the force accumulation registry. + * Summary component: stores the accumulated result of all impulses registered by the scene in the current frame. + + * Event-like component: each new impulse must increment the eventID to ensure delivery via CRDT, even if the direction is identical to the previous one. + * Renderer processes impulse with the unique ID only once. Increase eventID of the component to apply another impulse. +*/ +message PBPhysicsCombinedImpulse { + decentraland.common.Vector3 vector = 1; // Includes impulse direction and magnitude + uint32 event_id = 2; // Monotonic counter to distinguish different impulses. +} \ No newline at end of file diff --git a/proto/decentraland/social_service/v2/social_service_v2.proto b/proto/decentraland/social_service/v2/social_service_v2.proto index 5608eab2..f0febaf8 100644 --- a/proto/decentraland/social_service/v2/social_service_v2.proto +++ b/proto/decentraland/social_service/v2/social_service_v2.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package decentraland.social_service.v2; import "google/protobuf/empty.proto"; +import "decentraland/common/colors.proto"; import "decentraland/social_service/errors.proto"; // Types @@ -12,6 +13,7 @@ message FriendProfile { string name = 2; bool has_claimed_name = 3; string profile_picture_url = 4; + optional decentraland.common.Color3 name_color = 5; } message BlockedUserProfile { @@ -20,6 +22,7 @@ message BlockedUserProfile { bool has_claimed_name = 3; string profile_picture_url = 4; optional int64 blocked_at = 5; + optional decentraland.common.Color3 name_color = 6; } message Pagination { From 3a0cc035775e4cdc8564f6f714ba5912ac5cf8f8 Mon Sep 17 00:00:00 2001 From: Maurizio Farmi Date: Thu, 9 Apr 2026 16:57:56 +0200 Subject: [PATCH 38/54] feat: avatar masks (#373) * - Introduced AvatarMask enum in avatar_masks.proto for animation bone targeting. - Updated TriggerEmoteRequest and TriggerSceneEmoteRequest messages to include an optional mask field for enhanced emote functionality. * Replaced AvatarMask with AvatarEmoteMask in restricted_actions.proto and add new avatar_emote_mask.proto file * Added UNRECOGNIZED value to AvatarEmoteMask enum. * Removed UNRECOGNIZED value from AvatarEmoteMask enum in avatar_emote_mask.proto. * Refactor TriggerEmoteRequest and TriggerSceneEmoteRequest to replace AvatarEmoteMask with uint32 for mask field, and remove unnecessary import from restricted_actions.proto. * Refactor AvatarEmoteMask: move enum definition from avatar_emote_mask.proto to restricted_actions.proto and delete the obsolete file. * Refactor TriggerEmoteRequest and TriggerSceneEmoteRequest to use AvatarEmoteMask enum from new avatar_emote_mask.proto file, replacing uint32 mask field. * Refactor restricted_actions.proto to import AvatarEmoteMask from avatar_shape.proto, and delete obsolete avatar_emote_mask.proto file. * Refactor TriggerEmoteRequest and TriggerSceneEmoteRequest to replace AvatarEmoteMask with uint32 for mask field, and remove the import of avatar_shape.proto from restricted_actions.proto. * Add optional mask field to PlayerEmote message for animation bone targeting * Add StopEmote RPC to RestrictedActionsService for stopping current emotes * Add StopEmoteRequest message and update StopEmote RPC in RestrictedActionsService --- proto/decentraland/kernel/apis/restricted_actions.proto | 7 +++++++ proto/decentraland/kernel/comms/rfc4/comms.proto | 1 + proto/decentraland/sdk/components/avatar_shape.proto | 5 +++++ 3 files changed, 13 insertions(+) diff --git a/proto/decentraland/kernel/apis/restricted_actions.proto b/proto/decentraland/kernel/apis/restricted_actions.proto index 97d19faf..3bddab3c 100644 --- a/proto/decentraland/kernel/apis/restricted_actions.proto +++ b/proto/decentraland/kernel/apis/restricted_actions.proto @@ -16,6 +16,7 @@ message TeleportToRequest { message TriggerEmoteRequest { string predefined_emote = 1; + optional uint32 mask = 2; } message ChangeRealmRequest { @@ -40,6 +41,7 @@ message CommsAdapterRequest { message TriggerSceneEmoteRequest { string src = 1; optional bool loop = 2; + optional uint32 mask = 3; } message SuccessResponse { @@ -60,6 +62,8 @@ message CopyToClipboardRequest { message EmptyResponse { } +message StopEmoteRequest { } + service RestrictedActionsService { // MovePlayerTo will move the player to a position relative to the current scene. // If 'duration' field is used in the request, the success response depends on the @@ -90,4 +94,7 @@ service RestrictedActionsService { // CopyToClipboard copies the provided text into the clipboard rpc CopyToClipboard(CopyToClipboardRequest) returns (EmptyResponse) {} + + // StopEmote will stop the current emote + rpc StopEmote(StopEmoteRequest) returns (SuccessResponse) {} } diff --git a/proto/decentraland/kernel/comms/rfc4/comms.proto b/proto/decentraland/kernel/comms/rfc4/comms.proto index c3e01619..ad4f4adf 100644 --- a/proto/decentraland/kernel/comms/rfc4/comms.proto +++ b/proto/decentraland/kernel/comms/rfc4/comms.proto @@ -99,6 +99,7 @@ message PlayerEmote { optional bool is_reacting = 8; // to a social emote started by other user optional string social_emote_initiator = 9; // wallet address of the user that initiated social emote optional string target_avatar = 10; // wallet address of the user whose avatar is the target of a directed emote + optional uint32 mask = 11; // mask for which bones an animation applies to. } // Message sent to force an avatar to look at a position diff --git a/proto/decentraland/sdk/components/avatar_shape.proto b/proto/decentraland/sdk/components/avatar_shape.proto index 29f363f8..ce0ba627 100644 --- a/proto/decentraland/sdk/components/avatar_shape.proto +++ b/proto/decentraland/sdk/components/avatar_shape.proto @@ -42,3 +42,8 @@ message PBAvatarShape { optional bool show_only_wearables = 12; // hides the skin + hair + facial features (default: false) } +// Mask for which bones an animation applies to. +enum AvatarEmoteMask { + AEM_FULL_BODY = 0; + AEM_UPPER_BODY = 1; +} \ No newline at end of file From 6915bc89a1c80d5fd9a702b96a2f703841200a28 Mon Sep 17 00:00:00 2001 From: Mirko Jugurdzija Date: Thu, 16 Apr 2026 08:45:59 +0200 Subject: [PATCH 39/54] feat: add Reaction and ChatReaction comms messages for emoji reactions (#376) * feat: add Reaction and ChatReaction comms messages for emoji reactions * fix: add timestamp to reaction message for deduplications * feat: add reaction count for batching * chore: clean up whitespace in comms.proto --- proto/decentraland/kernel/comms/rfc4/comms.proto | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/proto/decentraland/kernel/comms/rfc4/comms.proto b/proto/decentraland/kernel/comms/rfc4/comms.proto index ad4f4adf..8675b5de 100644 --- a/proto/decentraland/kernel/comms/rfc4/comms.proto +++ b/proto/decentraland/kernel/comms/rfc4/comms.proto @@ -18,7 +18,9 @@ message Packet { PlayerEmote player_emote = 9; SceneEmote scene_emote = 10; MovementCompressed movement_compressed = 12; - LookAtPosition look_at_position = 13; + LookAtPosition look_at_position = 13; + Reaction reaction = 14; + ChatReaction chat_reaction = 15; } uint32 protocol_version = 11; } @@ -162,3 +164,15 @@ message Voice { VC_OPUS = 0; } } + +message Reaction { + int32 emoji_index = 1; + float timestamp = 2; + int32 count = 3; +} + +message ChatReaction { + int32 emoji_index = 1; + string message_id = 2; + string address = 3; +} From f092adf0f824b8a96af69f268d6517daf21838ff Mon Sep 17 00:00:00 2001 From: Pravus Date: Thu, 16 Apr 2026 12:00:56 +0200 Subject: [PATCH 40/54] corrected git wrong auto-merging --- proto/decentraland/kernel/comms/rfc4/comms.proto | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/proto/decentraland/kernel/comms/rfc4/comms.proto b/proto/decentraland/kernel/comms/rfc4/comms.proto index 66427042..281a8e29 100644 --- a/proto/decentraland/kernel/comms/rfc4/comms.proto +++ b/proto/decentraland/kernel/comms/rfc4/comms.proto @@ -94,6 +94,14 @@ message PlayerEmote { uint32 incremental_id = 1; string urn = 2; float timestamp = 3; + optional bool is_stopping = 4; // true means the emote has been stopped in the sender's client + optional bool is_repeating = 5; // true when it is not the first time the looping animation plays + optional int32 interaction_id = 6; // identifies an interaction univocaly, established when the start animation is triggered + optional int32 social_emote_outcome = 7; // -1 means it does not use an outcome animation + optional bool is_reacting = 8; // to a social emote started by other user + optional string social_emote_initiator = 9; // wallet address of the user that initiated social emote + optional string target_avatar = 10; // wallet address of the user whose avatar is the target of a directed emote + optional uint32 mask = 11; // mask for which bones an animation applies to. } message SceneEmote { @@ -126,6 +134,9 @@ message ProfileResponse { message Chat { string message = 1; double timestamp = 2; + // Extension: optional forwarded_from to identify the original sender when + // messages are forwarded through an SFU + optional string forwarded_from = 3; } message Scene { From 424957b8790ede502ef37789abeb83115fb90ff0 Mon Sep 17 00:00:00 2001 From: Muna <44584806+decentraland-bot@users.noreply.github.com> Date: Mon, 8 Jun 2026 17:26:37 -0300 Subject: [PATCH 41/54] fix: update oddish-action for npm Trusted Publishing (#416) fix: update oddish-action to support npm Trusted Publishing (OIDC) The old oddish-action version always writes _authToken to .npmrc, which prevents npm from falling through to OIDC-based Trusted Publishing. This caused E404 on publish after npm invalidated granular tokens that bypass 2FA. Update to the latest oddish-action that detects OIDC availability and remove NODE_AUTH_TOKEN so npm uses Trusted Publishing instead. --- .github/workflows/build-and-publish.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 8fac7e33..34e63fea 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -31,7 +31,7 @@ jobs: - name: build and compile test run: make test - name: publish packages - uses: decentraland/oddish-action@914e7c59e93e708152aa703c50196740c0526c1d # @master + uses: decentraland/oddish-action@0074f2d535c43b5c83f136525304a6277166d77f # @master id: publish_dcl_protocol with: registry-url: "https://registry.npmjs.org" @@ -53,7 +53,6 @@ jobs: branch-to-custom-tag: ${{ env.BRANCH_TAG }} env: BRANCH_NAME: ${{ github.ref_name }} - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} AWS_DEFAULT_REGION: us-east-1 AWS_ACCESS_KEY_ID: ${{ secrets.SDK_TEAM_AWS_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.SDK_TEAM_AWS_SECRET }} From a470b131fb9cef8d22b3e5561dd5545e19f951e1 Mon Sep 17 00:00:00 2001 From: Muna <44584806+decentraland-bot@users.noreply.github.com> Date: Mon, 8 Jun 2026 18:22:33 -0300 Subject: [PATCH 42/54] fix: setup-node 24, fix repository.url, and bump actions to v6 (#417) * fix: install npm@11 for Trusted Publishing OIDC auth The GitHub Actions runner ships npm 10.9.8, which does not support tokenless Trusted Publishing (OIDC) auth. npm 11 is required. Without this, `npm publish --provenance` fails with ENEEDAUTH. Matches the working pattern from decentraland/marketplace. Co-Authored-By: Claude Opus 4.6 * fix: add setup-node 24 and fix repository.url for provenance - Add actions/setup-node with node-version 24 (ships npm 11.x for Trusted Publishing OIDC support) - Replace npm@11 corepack workaround with proper Node 24 setup - Fix package.json repository field to full object format required for npm provenance verification Co-Authored-By: Claude Opus 4.6 * chore: bump actions/checkout and actions/setup-node to v6 Both actions now use the node24 runtime, replacing the node20 runtime that is being phased out. Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- .github/workflows/build-and-publish.yml | 6 +++++- package.json | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 34e63fea..12db74f5 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -21,9 +21,13 @@ jobs: outputs: dcl_protocol_s3_bucket_key: ${{ steps.publish_dcl_protocol.outputs.s3-bucket-key }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} + - name: Use Node.js 24 + uses: actions/setup-node@v6 + with: + node-version: 24 - name: install run: make install - name: buf build diff --git a/package.json b/package.json index 7e731d25..91078b9f 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,10 @@ "name": "@dcl/protocol", "version": "1.0.0", "description": "", - "repository": "decentraland/protocol.git", + "repository": { + "type": "git", + "url": "git+https://github.com/decentraland/protocol.git" + }, "homepage": "https://github.com/decentraland/protocol#readme", "bugs": "https://github.com/decentraland/protocol/issues", "keywords": [], From 3c70e8a31967c1171f5470426006675976fcad11 Mon Sep 17 00:00:00 2001 From: Mikhail Agapov <118179774+mikhail-dcl@users.noreply.github.com> Date: Thu, 11 Jun 2026 18:39:02 +0300 Subject: [PATCH 43/54] Feat: Pulse with quantization (#360) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add Quantization as a protobuf plugin * pulse_comms.proto Signed-off-by: Mikhail Agapov * add server message & handshake response * add player state full/delta * add player joined message * add player joined into server message * add player state into client message * Update protobufjs to 7.5.4 to resolve the problem with namespaces * proto/google/ is no longer created or published, so both buf (CI) and protoc (consumers like unity-explorer) resolve descriptor.proto from their own built-in includes — which all have the correct csharp_namespace Signed-off-by: Mikhail Agapov * fix: exclude proto/google from npm package to fix C# codegen namespace Revert protobufjs to 7.2.4 (7.5.4 breaks buf and proto-compatibility-tool with newer reserved syntax) and restore the make install copy step for local tooling. Change "files" from "proto" to "proto/decentraland" so the stripped google/protobuf/descriptor.proto (missing csharp_namespace option) is no longer published. Consumers' protoc resolves descriptor.proto from its own built-in includes which have the correct option csharp_namespace = "Google.Protobuf.Reflection". Co-Authored-By: Claude Opus 4.6 * Adjust states encoding * Add quantization example for PlayerStateDelta Signed-off-by: Mikhail Agapov * Copy protoc-gen-bitwise to the output package * Make Quantize.cs compitable with Unity Signed-off-by: Mikhail Agapov * Isolate PlayerState as a reusable component Signed-off-by: Mikhail Agapov * Add requried quntizationfor delta Signed-off-by: Mikhail Agapov * Add RESYNC_REQUEST Signed-off-by: Mikhail Agapov * ProfileVersionAnnouncement - parcel_index change uint32 -> int32 Signed-off-by: Mikhail Agapov * add emote messages * add duration on EmoteStart for one shot emotes * Fix MovementBlend Range Signed-off-by: Mikhail Agapov * add teleport messages * Add "jump_count" Signed-off-by: Mikhail Agapov * add teleport ClientMessage & ServerMessage * Add BaselineSeq to delta Signed-off-by: Mikhail Agapov * add parcel index & player state in teleport * add head yaw and head pitch to state flags * Add `sequence` to `EmoteStarted` Signed-off-by: Mikhail Agapov * add player state into EmoteStart * Add seq and PlayerState to `EmoteStopped` Signed-off-by: Mikhail Agapov * Add "realm" field to "TeleportRequest" Signed-off-by: Mikhail Agapov * Split pulse_comms into separate files Signed-off-by: Mikhail Agapov * Add Initial State to HandshakeRequest - It's needed to properly authorize reconnection in the middle of the session Signed-off-by: Mikhail Agapov * Add support for "pointing at" Signed-off-by: Mikhail Agapov * Change "Point At" to the absolute value Signed-off-by: Mikhail Agapov * Add "realm" to the initial state Signed-off-by: Mikhail Agapov * Change head_pitch max from 180 to 360 Signed-off-by: Mikhail Agapov * emote mask --------- Signed-off-by: Mikhail Agapov Co-authored-by: Nicolas Lorusso Co-authored-by: Claude Opus 4.6 --- README.md | 198 ++++++++++++++++++ package.json | 5 +- proto/decentraland/common/options.proto | 30 +++ .../common/quantization_example.proto | 132 ++++++++++++ proto/decentraland/pulse/pulse_client.proto | 77 +++++++ proto/decentraland/pulse/pulse_server.proto | 138 ++++++++++++ proto/decentraland/pulse/pulse_shared.proto | 48 +++++ protoc-gen-bitwise/generator_csharp.py | 176 ++++++++++++++++ protoc-gen-bitwise/options_pb2.py | 171 +++++++++++++++ protoc-gen-bitwise/plugin.py | 92 ++++++++ protoc-gen-bitwise/runtime/cs/BitReader.cs | 112 ++++++++++ protoc-gen-bitwise/runtime/cs/BitWriter.cs | 117 +++++++++++ protoc-gen-bitwise/runtime/cs/Quantize.cs | 35 ++++ 13 files changed, 1329 insertions(+), 2 deletions(-) create mode 100644 proto/decentraland/common/options.proto create mode 100644 proto/decentraland/common/quantization_example.proto create mode 100644 proto/decentraland/pulse/pulse_client.proto create mode 100644 proto/decentraland/pulse/pulse_server.proto create mode 100644 proto/decentraland/pulse/pulse_shared.proto create mode 100644 protoc-gen-bitwise/generator_csharp.py create mode 100644 protoc-gen-bitwise/options_pb2.py create mode 100644 protoc-gen-bitwise/plugin.py create mode 100644 protoc-gen-bitwise/runtime/cs/BitReader.cs create mode 100644 protoc-gen-bitwise/runtime/cs/BitWriter.cs create mode 100644 protoc-gen-bitwise/runtime/cs/Quantize.cs diff --git a/README.md b/README.md index 598bdb0b..20c75605 100644 --- a/README.md +++ b/README.md @@ -67,3 +67,201 @@ In this case, there is no problem with when each PR is merged. It's recommendabl ## Comms TODO + +--- + +# Bitwise Serialization Plugin (`protoc-gen-bitwise`) + +A custom protoc plugin that generates C# partial classes with typed float +accessors for quantized `uint32` fields in high-frequency MMO networking +messages (position deltas, player input, etc.). It runs alongside +`--csharp_out` in the same protoc invocation; the two output files coexist +via C# `partial class`. + +## How it works + +Protobuf encodes `uint32` values as varints, which are already compact for +small values: a value up to 2¹⁴−1 costs 2 bytes, up to 2²¹−1 costs 3 bytes. +Rather than a separate binary packing layer, the plugin leverages this: + +1. Declare quantized fields as `uint32` in the `.proto` schema and annotate + them with `[(decentraland.common.quantized)]` to specify the float range + and bit resolution. +2. `--csharp_out` generates the standard protobuf class with the raw `uint32` + property (e.g. `PositionX`). +3. `--bitwise_out` (this plugin) generates a `partial class` extension with a + cached float accessor (e.g. `PositionXQuantized`) that encodes/decodes + transparently via `Quantize.Encode` / `Quantize.Decode`. + +The wire representation is a standard protobuf message — any protobuf-capable +client can read it without knowledge of the plugin. + +## Prerequisites + +| Requirement | Version | +|---|---| +| Python | 3.10+ | +| `protobuf` Python package | 4.x or 3.20+ | +| `protoc` | 3.19+ | + +```bash +pip install protobuf +``` + +## Step 1 — Annotate your `.proto` file + +Declare quantized fields as `uint32` and import `options.proto`: + +```protobuf +syntax = "proto3"; + +import "decentraland/common/options.proto"; + +package decentraland.kernel.comms.v3; + +message PositionDelta { + // Float range [-100, 100] quantized to 16 bits ≈ 0.003-unit precision. + // Stored as uint32 on the wire; protobuf encodes it as a 3-byte varint. + uint32 dx = 1 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dy = 2 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dz = 3 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + + // Unannotated uint32: protobuf varint encodes small values compactly by default. + uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; +} +``` + +### Annotation reference + +| Annotation | Target type | Parameters | Effect | +|---|---|---|---| +| `[(decentraland.common.quantized)]` | `uint32` | `min`, `max`, `bits` | Plugin emits a cached `float {Name}Quantized` accessor | +| `[(decentraland.common.bit_packed)]` | `uint32` | `bits` | Documents the value range; protobuf handles varint compaction automatically | + +### Wire cost at worst-case (all bits set) + +| Quantization bits | Max value | Varint bytes | Tag (field ≤ 15) | Total per field | +|---|---|---|---|---| +| 8 | 255 | 2 | 1 | 3 B | +| 12 | 4 095 | 2 | 1 | 3 B | +| 14 | 16 383 | 2 | 1 | 3 B | +| 16 | 65 535 | 3 | 1 | 4 B | +| 20 | 1 048 575 | 3 | 1 | 4 B | + +Proto3 omits fields equal to their default value (0), so average cost is lower. + +## Step 2 — Run protoc + +```bash +protoc \ + --proto_path=proto \ + --proto_path=/path/to/google/protobuf/include \ + --csharp_out=generated/cs \ + --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.py \ + --bitwise_out=generated/cs \ + proto/decentraland/kernel/comms/v3/comms.proto +``` + +The plugin emits one `*.Bitwise.cs` file (PascalCase, flat in the output +directory) for each `.proto` file that contains at least one `[(quantized)]` +field. + +## Step 3 — Copy the runtime + +Copy `Quantize.cs` into your project: + +``` +Assets/ +└── Scripts/ + └── Networking/ + └── Bitwise/ + └── Quantize.cs ← protoc-gen-bitwise/runtime/cs/Quantize.cs +``` + +`Quantize.cs` lives in the `Decentraland.Networking.Bitwise` namespace and +provides two static methods used by the generated accessors: + +```csharp +public static class Quantize +{ + public static uint Encode(float value, float min, float max, int bits); + public static float Decode(uint encoded, float min, float max, int bits); +} +``` + +## Step 4 — Use the generated code + +The plugin emits a `partial class` that adds float accessors on top of the +standard protobuf-generated `uint32` properties: + +```csharp +using Decentraland.Kernel.Comms.V3; + +// --- Build and send --- +var delta = new PositionDelta(); +delta.DxQuantized = 3.14f; // encodes to uint32, stored in delta.Dx +delta.DyQuantized = 0f; +delta.DzQuantized = -7.5f; +delta.EntityId = 42u; + +byte[] bytes = delta.ToByteArray(); // standard protobuf serialization +SendOnChannel1(bytes); + +// --- Receive and read --- +var received = PositionDelta.Parser.ParseFrom(receivedBytes); +float x = received.DxQuantized; // decoded on first access, cached thereafter +float y = received.DyQuantized; +float z = received.DzQuantized; + +// If raw uint32 fields are mutated directly after construction, invalidate the cache: +received.ResetDecodedCache(); +``` + +## Generated file example + +For the `PositionDelta` message above the plugin emits `PositionDelta.Bitwise.cs`: + +```csharp +// +// Generated by protoc-gen-bitwise. DO NOT EDIT. +// Source: decentraland/kernel/comms/v3/comms.proto +// + +using Decentraland.Networking.Bitwise; + +namespace Decentraland.Kernel.Comms.V3 +{ + public partial class PositionDelta + { + private float? _dx; + public float DxQuantized + { + get => _dx ??= Quantize.Decode(Dx, -100.0f, 100.0f, 16); + set { _dx = value; Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + private float? _dy; + public float DyQuantized + { + get => _dy ??= Quantize.Decode(Dy, -100.0f, 100.0f, 16); + set { _dy = value; Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + private float? _dz; + public float DzQuantized + { + get => _dz ??= Quantize.Decode(Dz, -100.0f, 100.0f, 16); + set { _dz = value; Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. + public void ResetDecodedCache() + { + _dx = null; + _dy = null; + _dz = null; + } + } + +} // namespace Decentraland.Kernel.Comms.V3 +``` diff --git a/package.json b/package.json index 91078b9f..887d6567 100644 --- a/package.json +++ b/package.json @@ -28,9 +28,10 @@ "protobufjs": "7.2.4" }, "files": [ - "proto", + "proto/decentraland", "out-ts", "out-js", - "public" + "public", + "protoc-gen-bitwise" ] } diff --git a/proto/decentraland/common/options.proto b/proto/decentraland/common/options.proto new file mode 100644 index 00000000..8e911547 --- /dev/null +++ b/proto/decentraland/common/options.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package decentraland.common; + +import "google/protobuf/descriptor.proto"; + +// Options for quantizing a float value stored as a uint32 field. +// The float is clamped to [min, max] and uniformly quantized to N bits; +// protobuf encodes the resulting uint32 as a varint on the wire. +// The generator emits a float {Name}Quantized accessor in the partial class. +message QuantizedFloatOptions { + float min = 1; + float max = 2; + uint32 bits = 3; +} + +// Options for bit-packing an integer field into fewer than 32 bits. +message BitPackedOptions { + uint32 bits = 1; +} + +extend google.protobuf.FieldOptions { + // Apply to uint32 fields to enable quantized float encoding. + // Example: uint32 dx = 1 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + QuantizedFloatOptions quantized = 50001; + + // Apply to uint32 fields to pack into fewer bits. + // Example: uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; + BitPackedOptions bit_packed = 50002; +} diff --git a/proto/decentraland/common/quantization_example.proto b/proto/decentraland/common/quantization_example.proto new file mode 100644 index 00000000..a82e430c --- /dev/null +++ b/proto/decentraland/common/quantization_example.proto @@ -0,0 +1,132 @@ +syntax = "proto3"; + +package decentraland.common; + +import "decentraland/common/options.proto"; + +// High-frequency player state messages sent on Channel 1 (unreliable sequenced). +// +// Every message below uses the protoc-gen-bitwise annotations to minimise +// wire size. Wire costs are listed per-message so the trade-offs are clear. +// +// Annotation cheat-sheet: +// [(decentraland.common.quantized) = { min: F, max: F, bits: N }] +// → stores a float as a uint32 quantized to N bits over [min, max]. +// → protobuf encodes the uint32 as a varint: values ≤ 2^14-1 cost 2 bytes, +// values ≤ 2^21-1 cost 3 bytes; the generated partial class adds a +// float {Name}Quantized accessor that encodes/decodes transparently. +// [(decentraland.common.bit_packed) = { bits: N }] +// → documents that this uint32 uses at most N bits; protobuf encodes it +// as a varint (same savings, no generated accessor needed). +// (no annotation) +// → standard protobuf encoding (bool/double/etc. at natural width). +// +// Varint byte costs at worst-case (all bits set): +// ≤ 7 bits → 1 byte (max 127) +// ≤ 14 bits → 2 bytes (max 16 383) +// ≤ 21 bits → 3 bytes (max 2 097 151) +// Proto3 omits fields equal to their default value (0), so average cost is lower. + +// --------------------------------------------------------------------------- +// PositionDelta — Δ position relative to last acknowledged full snapshot. +// +// Sent every client tick (~10 Hz) on Channel 1. +// +// Field | Type | Range | Q bits | Wire worst-case +// ------------|--------|----------------|--------|----------------------- +// dx | uint32 | [-100, 100] | 16 | tag 1B + varint 3B = 4B +// dy | uint32 | [-100, 100] | 16 | tag 1B + varint 3B = 4B +// dz | uint32 | [-100, 100] | 16 | tag 1B + varint 3B = 4B +// entity_id | uint32 | [0, 1 048 575] | 20 | tag 1B + varint 3B = 4B +// sequence | uint32 | [0, 4 095] | 12 | tag 1B + varint 2B = 3B +// --------------------------------------------------------------------------- +// Worst-case: 19 B (vs. 20 B raw: 3×float + 2×uint32) +// Step: dx/dy/dz ≈ 0.003 units +// --------------------------------------------------------------------------- +message PositionDelta { + uint32 dx = 1 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dy = 2 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dz = 3 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; + uint32 sequence = 5 [(decentraland.common.bit_packed) = { bits: 12 }]; +} + +// --------------------------------------------------------------------------- +// PlayerInput — client input snapshot for server-side reconciliation. +// +// Sent every client frame (~30 Hz) on Channel 1. +// +// Field | Type | Range | Q bits | Wire worst-case +// ------------|--------|----------------|--------|----------------------- +// move_x | uint32 | [-1, 1] | 8 | tag 1B + varint 2B = 3B +// move_z | uint32 | [-1, 1] | 8 | tag 1B + varint 2B = 3B +// yaw | uint32 | [-180, 180] | 12 | tag 1B + varint 2B = 3B +// buttons | uint32 | bitmask | 8 | tag 1B + varint 2B = 3B +// sequence | uint32 | [0, 4 095] | 12 | tag 1B + varint 2B = 3B +// --------------------------------------------------------------------------- +// Worst-case: 15 B (vs. 20 B raw: 3×float + 2×uint32) +// Step: move_x/move_z ≈ 0.008; yaw ≈ 0.088° +// --------------------------------------------------------------------------- +message PlayerInput { + // Normalised joystick axes in [-1, 1]. + uint32 move_x = 1 [(decentraland.common.quantized) = { min: -1.0, max: 1.0, bits: 8 }]; + uint32 move_z = 2 [(decentraland.common.quantized) = { min: -1.0, max: 1.0, bits: 8 }]; + + // Horizontal look direction in degrees. + uint32 yaw = 3 [(decentraland.common.quantized) = { min: -180.0, max: 180.0, bits: 12 }]; + + // Bitmask of active buttons (see ButtonFlags below). + uint32 buttons = 4 [(decentraland.common.bit_packed) = { bits: 8 }]; + + // Rolling input sequence number used by the server for reconciliation. + uint32 sequence = 5 [(decentraland.common.bit_packed) = { bits: 12 }]; +} + +// Bitmask values for PlayerInput.buttons. +enum ButtonFlags { + BF_NONE = 0; + BF_JUMP = 1; // bit 0 + BF_SPRINT = 2; // bit 1 + BF_INTERACT = 4; // bit 2 + BF_EMOTE = 8; // bit 3 + BF_CROUCH = 16; // bit 4 + // bits 5-7 reserved +} + +// --------------------------------------------------------------------------- +// AvatarStateSnapshot — full authoritative state, sent on Channel 0 (reliable) +// or on resync requests. Demonstrates wider ranges and mixed encodings. +// +// Field | Type | Range | Q bits | Wire worst-case +// -----------------|--------|------------------|--------|----------------------- +// x | uint32 | [-4096, 4096] | 16 | tag 1B + varint 3B = 4B +// y | uint32 | [-256, 256] | 14 | tag 1B + varint 2B = 3B +// z | uint32 | [-4096, 4096] | 16 | tag 1B + varint 3B = 4B +// pitch | uint32 | [-90, 90] | 10 | tag 1B + varint 2B = 3B +// yaw | uint32 | [-180, 180] | 12 | tag 1B + varint 2B = 3B +// entity_id | uint32 | [0, 1 048 575] | 20 | tag 1B + varint 3B = 4B +// animation_state | uint32 | [0, 63] | 6 | tag 1B + varint 1B = 2B +// is_grounded | bool | — | — | tag 1B + varint 1B = 2B +// timestamp | double | — | — | tag 1B + fixed64 8B = 9B +// --------------------------------------------------------------------------- +// Worst-case: 34 B (vs. 45 B raw: 5×float + 2×uint32 + bool + double) +// Step: x/z ≈ 0.125 units; y ≈ 0.031 units; pitch ≈ 0.176°; yaw ≈ 0.088° +// --------------------------------------------------------------------------- +message AvatarStateSnapshot { + // World-space position. + uint32 x = 1 [(decentraland.common.quantized) = { min: -4096.0, max: 4096.0, bits: 16 }]; + uint32 y = 2 [(decentraland.common.quantized) = { min: -256.0, max: 256.0, bits: 14 }]; + uint32 z = 3 [(decentraland.common.quantized) = { min: -4096.0, max: 4096.0, bits: 16 }]; + + // View angles. + uint32 pitch = 4 [(decentraland.common.quantized) = { min: -90.0, max: 90.0, bits: 10 }]; + uint32 yaw = 5 [(decentraland.common.quantized) = { min: -180.0, max: 180.0, bits: 12 }]; + + // Identity and animation state. + uint32 entity_id = 6 [(decentraland.common.bit_packed) = { bits: 20 }]; + uint32 animation_state = 7 [(decentraland.common.bit_packed) = { bits: 6 }]; + + // Un-annotated fields — encoded at their natural width. + bool is_grounded = 8; + double timestamp = 9; // server epoch milliseconds +} diff --git a/proto/decentraland/pulse/pulse_client.proto b/proto/decentraland/pulse/pulse_client.proto new file mode 100644 index 00000000..6036ec53 --- /dev/null +++ b/proto/decentraland/pulse/pulse_client.proto @@ -0,0 +1,77 @@ +syntax = "proto3"; + +package decentraland.pulse; + +import "decentraland/common/vectors.proto"; +import "decentraland/pulse/pulse_shared.proto"; + +message HandshakeRequest { + bytes auth_chain = 1; + int32 profile_version = 2; + optional PlayerInitialState initial_state = 3; +} + +// Describes the initial state of the player if (re-)connected in the middle of the session +message PlayerInitialState { + PlayerState state = 1; + optional string emote_id = 2; + optional uint32 emote_duration_ms = 3; + // Indicates how many milliseconds ago the emote was started + optional uint32 emote_start_offset_ms = 4; + // Non-empty realm identifier. Rejected if empty. + string realm = 5; + optional int32 emote_mask = 6; +} + +// Similarly to the LiveKit pipeline, a peer announces the version of its profile but +// it only does it when it's changed as it's sent reliably and stored on the server for other peers +message ProfileVersionAnnouncement { + int32 version = 1; +} + +// Since the server doesn't simulate the scenes state, it trusts the values from the client +message PlayerStateInput { + PlayerState state = 1; +} + +// Client sends resync request if it has a gap in the known sequences and, thus, can't apply a delta. +// It's sent reliably to prevent further desynchronization +message ResyncRequest { + uint32 subject_id = 1; + // highest seq the client actually has for this subject + uint32 known_seq = 2; +} + +// Client → Server. Client requests to start an emote. +message EmoteStart { + string emote_id = 1; + optional uint32 duration_ms = 2; + PlayerState player_state = 3; + optional int32 mask = 4; +} + +// Client → Server. Client requests to stop a looping emote. +// One-shot emotes are terminated by the server timer. +message EmoteStop { +} + +// Client → Server. Also announces the peer's realm; peers in different realms never see each +// other. Must be the first gameplay message after handshake. Same-realm re-teleports are valid. +message TeleportRequest { + int32 parcel_index = 1; + decentraland.common.Vector3 position = 2; + // Non-empty realm identifier. Rejected if empty. + string realm = 3; +} + +message ClientMessage { + oneof message { + HandshakeRequest handshake = 1; + PlayerStateInput input = 2; + ResyncRequest resync = 3; + ProfileVersionAnnouncement profile_announcement = 4; + EmoteStart emote_start = 5; + EmoteStop emote_stop = 6; + TeleportRequest teleport = 7; + } +} diff --git a/proto/decentraland/pulse/pulse_server.proto b/proto/decentraland/pulse/pulse_server.proto new file mode 100644 index 00000000..8311e737 --- /dev/null +++ b/proto/decentraland/pulse/pulse_server.proto @@ -0,0 +1,138 @@ +syntax = "proto3"; + +package decentraland.pulse; + +import "decentraland/common/options.proto"; +import "decentraland/pulse/pulse_shared.proto"; + +message HandshakeResponse { + bool success = 1; + optional string error = 2; +} + +message PlayerProfileVersionsAnnounced { + uint32 subject_id = 1; + int32 version = 2; +} + +message PlayerStateDeltaTier0 { + uint32 subject_id = 1; + + // Between two consecutive server simulation ticks, the subject may have sent multiple inputs, each incrementing Seq by 1. So + // the delta's NewSeq will naturally jump by more than 1 compared to the previous delta — even with zero packet loss. + // Without the "BaselineSeq" the client has no way to detect the package loss + uint32 baseline_seq = 2; + uint32 new_seq = 3; + uint32 server_tick = 4; + + // While the player doesn't cross the parcel, this field is omitted from diff + optional int32 parcel_index = 5; + + // X position inside the parcel + optional uint32 position_x = 6 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + + // Y position + optional uint32 position_y = 7 [(decentraland.common.quantized) = { min: 0, max: 200, bits: 13 }]; + + // Z position inside the parcel + optional uint32 position_z = 8 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + + optional uint32 velocity_x = 9 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; + optional uint32 velocity_y = 10 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; + optional uint32 velocity_z = 11 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; + + optional uint32 rotation_y = 12 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + optional uint32 movement_blend = 13 [(decentraland.common.quantized) = { min: 0, max: 3, bits: 5 }]; + optional uint32 slide_blend = 14 [(decentraland.common.quantized) = { min: 0, max: 1, bits: 4 }]; + optional uint32 head_yaw = 15 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + optional uint32 head_pitch = 16 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + + optional uint32 state_flags = 17; + optional GlideState glide_state = 18; + + optional int32 jump_count = 19; + + // Absolute world hit position the player is pointing at. + // Only meaningful when POINTING_AT is set in `state_flags`. + // + // X/Z use 17 bits over the world span (~±3000m, covers GenesisCity + border + + // raycast cutoff) which yields ~0.046m steps — at least as fine as the player's + // own parcel_index + position_x/z combined precision (0.0625m), so no separate + // parcel index is needed for point-at. + // + // Y uses 7 bits over the player altitude range (matches position_y), step ~1.6m. + // Point At is not supposed to change frequently so the wire overhead should be minimal + optional uint32 point_at_x = 20 [(decentraland.common.quantized) = { min: -3000.0, max: 3000.0, bits: 17 }]; + optional uint32 point_at_y = 21 [(decentraland.common.quantized) = { min: 0.0, max: 200.0, bits: 7 }]; + optional uint32 point_at_z = 22 [(decentraland.common.quantized) = { min: -3000.0, max: 3000.0, bits: 17 }]; +} + +// Full State is sent to the client when it is out of sync, and can't recover with a diff only +message PlayerStateFull { + uint32 subject_id = 1; + uint32 sequence = 2; + uint32 server_tick = 3; + PlayerState state = 4; +} + +// Notification to the client, that a peer has joined, it can mean connection or entering the area of interest, it's up to the server to decide +message PlayerJoined { + string user_id = 1; + int32 profile_version = 2; + PlayerStateFull state = 3; +} + +// Notification to the client, that a peer has left, it can mean disconnection or leaving the area of interest +message PlayerLeft { + uint32 subject_id = 1; +} + +enum EmoteStopReason { + COMPLETED = 0; // one-shot timer expired on the server + CANCELLED = 1; // client sent EmoteStop (looping emotes) +} + +// Server → All Observers +// Full player state is piggybacked to ensure the emote is played in the right place. +// Observers use server_tick to scrub animation forward by transit latency. +message EmoteStarted { + uint32 subject_id = 1; + uint32 sequence = 2; + uint32 server_tick = 3; + string emote_id = 4; + PlayerState player_state = 5; + optional int32 mask = 6; +} + +// Server → All Observers +// Client resumes MovementInput only after receiving this. +// Carries full PlayerState so the client can snap to the correct position on resume. +message EmoteStopped { + uint32 subject_id = 1; + uint32 server_tick = 2; + EmoteStopReason reason = 3; + uint32 sequence = 4; + PlayerState player_state = 5; +} + +// Server → All Observers +message TeleportPerformed { + uint32 subject_id = 1; + uint32 sequence = 2; + uint32 server_tick = 3; + PlayerState state = 4; +} + +message ServerMessage { + oneof message { + HandshakeResponse handshake = 1; + PlayerStateFull player_state_full = 2; + PlayerStateDeltaTier0 player_state_delta = 3; + PlayerJoined player_joined = 4; + PlayerLeft player_left = 5; + PlayerProfileVersionsAnnounced player_profile_version_announced = 6; + EmoteStarted emote_started = 7; + EmoteStopped emote_stopped = 8; + TeleportPerformed teleported = 9; + } +} diff --git a/proto/decentraland/pulse/pulse_shared.proto b/proto/decentraland/pulse/pulse_shared.proto new file mode 100644 index 00000000..6f4fba7c --- /dev/null +++ b/proto/decentraland/pulse/pulse_shared.proto @@ -0,0 +1,48 @@ +syntax = "proto3"; + +package decentraland.pulse; + +import "decentraland/common/vectors.proto"; + +enum PlayerAnimationFlags { + NONE = 0; + GROUNDED = 1; + LONG_JUMP = 2; + LONG_FALL = 4; + FALLING = 8; + STUNNED = 16; + HEAD_YAW = 32; + HEAD_PITCH = 64; + POINTING_AT = 128; +} + +enum GlideState { + PROP_CLOSED = 0; + OPENING_PROP = 1; + GLIDING = 2; + CLOSING_PROP = 3; +} + +message PlayerState { + int32 parcel_index = 1; + + decentraland.common.Vector3 position = 2; + decentraland.common.Vector3 velocity = 3; + + float rotation_y = 4; + + float movement_blend = 5; + float slide_blend = 6; + + optional float head_yaw = 7; + optional float head_pitch = 8; + + uint32 state_flags = 9; + GlideState glide_state = 10; + + int32 jump_count = 11; + + // Absolute world hit position the player is pointing at. + // Only meaningful when POINTING_AT is set in `state_flags`. + optional decentraland.common.Vector3 point_at = 12; +} diff --git a/protoc-gen-bitwise/generator_csharp.py b/protoc-gen-bitwise/generator_csharp.py new file mode 100644 index 00000000..376c0dc1 --- /dev/null +++ b/protoc-gen-bitwise/generator_csharp.py @@ -0,0 +1,176 @@ +""" +C# code generator for the protoc-gen-bitwise plugin. + +For every proto message that contains at least one uint32 field annotated with +[(quantized)], this module emits a C# partial class that adds a computed float +property named {FieldName}Quantized. The getter decodes the stored uint32 to +a float; the setter encodes a float back to a uint32. Standard protobuf +handles serialization of the uint32 wire field; this class adds a typed float +accessor on top. + +Protobuf encodes small uint32 values via varint, so a value that fits in +2^20 costs 3 bytes on the wire — cheaper than a raw IEEE 754 float (4 bytes). + +Only uint32 fields are supported. bit_packed and unannotated fields are +passed through without generating any accessor. +""" + +from google.protobuf import descriptor_pb2 + +from options_pb2 import get_field_options + +# FieldDescriptorProto type constants (aliased for readability) +_FT = descriptor_pb2.FieldDescriptorProto + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _snake_to_pascal(name: str) -> str: + """position_x → PositionX""" + return ''.join(word.capitalize() for word in name.split('_')) + + +def _package_to_namespace(package: str) -> str: + """decentraland.kernel.comms.v3 → Decentraland.Kernel.Comms.V3""" + if not package: + return 'Generated' + return '.'.join(part.capitalize() for part in package.split('.')) + + +def _format_float(value: float) -> str: + """Format a Python float as a C# float literal (e.g. -100.0f).""" + text = f'{value:.8g}' + if '.' not in text and 'e' not in text and 'E' not in text: + text += '.0' + return text + 'f' + + +def _format_step(step: float) -> str: + """Format a quantization step size for display in a doc comment (e.g. ≈ 0.003).""" + return f'\u2248 {step:.6g}' + + +# --------------------------------------------------------------------------- +# Per-message code generation +# --------------------------------------------------------------------------- + +def _gen_message(msg_proto, indent: str = ' ') -> list[str] | None: + """ + Generate a C# partial class for a proto message. + + For each uint32 field with a [(quantized)] annotation, emits a cached float + property {FieldName}Quantized backed by the raw uint32 field. + + Returns a list of lines (without trailing newline) or None if the message + has no quantized uint32 fields. + """ + props: list[tuple[str, str, str, int, float]] = [] # (prop_name, mn, mx, bits, step) + + for field in msg_proto.field: + # Repeated/map fields are not supported + if field.label == _FT.LABEL_REPEATED: + continue + + # Only uint32 fields are candidates for quantized accessors + if field.type != _FT.TYPE_UINT32: + continue + + quantized, _ = get_field_options(field.options) + if quantized is None: + continue + + step = (quantized.max - quantized.min) / ((1 << quantized.bits) - 1) + + props.append(( + _snake_to_pascal(field.name), + _format_float(quantized.min), + _format_float(quantized.max), + quantized.bits, + step, + )) + + if not props: + return None + + i = indent + backings: list[str] = [] + lines: list[str] = [] + lines.append(f'public partial class {msg_proto.name}') + lines.append('{') + + for prop_name, mn, mx, bits, step in props: + backing = '_' + prop_name[0].lower() + prop_name[1:] + backings.append(backing) + lines.append(f'{i}private float? {backing};') + lines.append(f'{i}/// Float accessor for . Range [{mn}, {mx}], {bits} bits, step {_format_step(step)}.') + lines.append(f'{i}public float {prop_name}Quantized') + lines.append(f'{i}{{') + lines.append(f'{i}{i}get => {backing} ??= Quantize.Decode({prop_name}, {mn}, {mx}, {bits});') + lines.append(f'{i}{i}set {{ {backing} = value; {prop_name} = Quantize.Encode(value, {mn}, {mx}, {bits}); }}') + lines.append(f'{i}}}') + lines.append('') + + lines.append(f'{i}/// Clears all cached decoded values. Call after mutating raw uint32 fields directly.') + lines.append(f'{i}public void ResetDecodedCache()') + lines.append(f'{i}{{') + for backing in backings: + lines.append(f'{i}{i}{backing} = null;') + lines.append(f'{i}}}') + + lines.append('}') + return lines + + +# --------------------------------------------------------------------------- +# Per-file code generation (public entry point) +# --------------------------------------------------------------------------- + +def generate_csharp(file_proto) -> dict | None: + """ + Generate a C# source file for a FileDescriptorProto. + + Returns a dict with keys 'name' (output path) and 'content' (C# source), + or None if the file contains no quantized uint32 fields. + """ + namespace = _package_to_namespace(file_proto.package) + + # Output is placed flat in the output root, matching --csharp_out convention. + # e.g. "decentraland/kernel/comms/v3/my_message.proto" + # → "MyMessage.Bitwise.cs" + proto_file = file_proto.name.rsplit('/', 1)[-1] + stem = _snake_to_pascal(proto_file.replace('.proto', '')) + out_name = f'{stem}.Bitwise.cs' + + header = [ + '// ', + '// Generated by protoc-gen-bitwise. DO NOT EDIT.', + f'// Source: {file_proto.name}', + '// ', + '', + 'using Decentraland.Networking.Bitwise;', + '', + f'namespace {namespace}', + '{', + ] + footer = [ + '', + f'}} // namespace {namespace}', + ] + + body: list[str] = [] + for msg in file_proto.message_type: + msg_lines = _gen_message(msg) + if msg_lines is None: + continue + # Indent each line by 4 spaces (inside the namespace block) + for line in msg_lines: + body.append((' ' + line) if line else '') + body.append('') + + if not body: + return None # nothing to emit + + content = '\n'.join(header + body + footer) + '\n' + return {'name': out_name, 'content': content} diff --git a/protoc-gen-bitwise/options_pb2.py b/protoc-gen-bitwise/options_pb2.py new file mode 100644 index 00000000..d3e637f2 --- /dev/null +++ b/protoc-gen-bitwise/options_pb2.py @@ -0,0 +1,171 @@ +""" +Manual parser for the custom bitwise field options defined in options.proto. + +Rather than relying on protobuf extension registration (which requires a properly +compiled _pb2 module), this module parses the raw serialized FieldOptions bytes +directly using the protobuf binary wire format. All protobuf runtimes preserve +unknown/unregistered extension bytes when round-tripping, so +field.options.SerializeToString() always contains the extension data even when +the extension is not registered in the Python runtime. + +Wire format reference: + tag = (field_number << 3) | wire_type + wire_type 0 = varint, 1 = 64-bit, 2 = length-delimited, 5 = 32-bit +""" + +import struct + +# Extension field numbers as defined in options.proto +QUANTIZED_FIELD_NUMBER = 50001 +BIT_PACKED_FIELD_NUMBER = 50002 + + +# --------------------------------------------------------------------------- +# Low-level wire-format helpers +# --------------------------------------------------------------------------- + +def _read_varint(data: bytes, pos: int): + """Decode a protobuf varint starting at *pos*. Returns (value, new_pos).""" + result = 0 + shift = 0 + while pos < len(data): + byte = data[pos] + pos += 1 + result |= (byte & 0x7F) << shift + if not (byte & 0x80): + break + shift += 7 + return result, pos + + +def _read_float32(data: bytes, pos: int): + """Decode a little-endian 32-bit float. Returns (value, new_pos).""" + value, = struct.unpack_from(' int: + """Advance *pos* past a field with the given wire_type.""" + if wire_type == 0: + _, pos = _read_varint(data, pos) + elif wire_type == 1: + pos += 8 + elif wire_type == 2: + length, pos = _read_varint(data, pos) + pos += length + elif wire_type == 5: + pos += 4 + # wire types 3 and 4 (start/end group) are deprecated; skip gracefully + return pos + + +# --------------------------------------------------------------------------- +# Option message classes +# --------------------------------------------------------------------------- + +class QuantizedFloatOptions: + """Mirrors the QuantizedFloatOptions proto message.""" + + __slots__ = ('min', 'max', 'bits') + + def __init__(self, min_val: float = 0.0, max_val: float = 0.0, bits: int = 0): + self.min = min_val + self.max = max_val + self.bits = bits + + @classmethod + def from_bytes(cls, data: bytes) -> 'QuantizedFloatOptions': + obj = cls() + pos = 0 + while pos < len(data): + tag, pos = _read_varint(data, pos) + field_num = tag >> 3 + wire_type = tag & 0x7 + if field_num == 1 and wire_type == 5: # min (float) + obj.min, pos = _read_float32(data, pos) + elif field_num == 2 and wire_type == 5: # max (float) + obj.max, pos = _read_float32(data, pos) + elif field_num == 3 and wire_type == 0: # bits (uint32) + obj.bits, pos = _read_varint(data, pos) + else: + pos = _skip_field(data, pos, wire_type) + return obj + + def __repr__(self): + return f'QuantizedFloatOptions(min={self.min}, max={self.max}, bits={self.bits})' + + +class BitPackedOptions: + """Mirrors the BitPackedOptions proto message.""" + + __slots__ = ('bits',) + + def __init__(self, bits: int = 0): + self.bits = bits + + @classmethod + def from_bytes(cls, data: bytes) -> 'BitPackedOptions': + obj = cls() + pos = 0 + while pos < len(data): + tag, pos = _read_varint(data, pos) + field_num = tag >> 3 + wire_type = tag & 0x7 + if field_num == 1 and wire_type == 0: # bits (uint32) + obj.bits, pos = _read_varint(data, pos) + else: + pos = _skip_field(data, pos, wire_type) + return obj + + def __repr__(self): + return f'BitPackedOptions(bits={self.bits})' + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def get_field_options(field_options_proto): + """ + Extract custom bitwise options from a FieldDescriptorProto.options object. + + Serialises the options message to bytes and walks the wire-format stream + looking for extension fields 50001 (quantized) and 50002 (bit_packed). + + Args: + field_options_proto: google.protobuf.descriptor_pb2.FieldOptions instance + (may be a default/empty instance when no options are set). + + Returns: + tuple[QuantizedFloatOptions | None, BitPackedOptions | None] + """ + try: + raw = field_options_proto.SerializeToString() + except Exception: + return None, None + + if not raw: + return None, None + + quantized = None + bit_packed = None + pos = 0 + + while pos < len(raw): + tag, pos = _read_varint(raw, pos) + field_num = tag >> 3 + wire_type = tag & 0x7 + + if wire_type == 2: # length-delimited + length, pos = _read_varint(raw, pos) + value_bytes = raw[pos:pos + length] + pos += length + if field_num == QUANTIZED_FIELD_NUMBER: + quantized = QuantizedFloatOptions.from_bytes(value_bytes) + elif field_num == BIT_PACKED_FIELD_NUMBER: + bit_packed = BitPackedOptions.from_bytes(value_bytes) + # else: unknown length-delimited field — already consumed + else: + pos = _skip_field(raw, pos, wire_type) + + return quantized, bit_packed diff --git a/protoc-gen-bitwise/plugin.py b/protoc-gen-bitwise/plugin.py new file mode 100644 index 00000000..09b5abe4 --- /dev/null +++ b/protoc-gen-bitwise/plugin.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +""" +protoc-gen-bitwise — protoc plugin that generates C# bitwise serialization code. + +Protocol: + 1. protoc writes a serialised CodeGeneratorRequest to this process's stdin. + 2. This plugin reads it, generates C# partial classes with Encode / Decode + methods for every message that carries [(quantized)] or [(bit_packed)] + field annotations. + 3. A serialised CodeGeneratorResponse is written to stdout. + +Usage (from project root): + protoc \\ + --proto_path=proto \\ + --bitwise_out=generated/ \\ + --plugin=protoc-gen-bitwise \\ + proto/my_messages.proto + +Windows invocation (plugin not on PATH as executable): + protoc \\ + --proto_path=proto \\ + --bitwise_out=generated/ \\ + --plugin=protoc-gen-bitwise=python protoc-gen-bitwise/plugin.py \\ + proto/my_messages.proto + +Dependencies: + pip install grpcio-tools # or: pip install protobuf +""" + +import os +import sys + +# Ensure sibling modules (generator_csharp, options_pb2) are importable +# regardless of where protoc invokes this script from. +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +# On Windows, stdin/stdout are opened in text mode by default which corrupts +# the binary protobuf payload. +if sys.platform == 'win32': + import msvcrt + msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) + msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) + +from google.protobuf.compiler import plugin_pb2 + +from generator_csharp import generate_csharp + + +def main() -> None: + request_bytes = sys.stdin.buffer.read() + + request = plugin_pb2.CodeGeneratorRequest() + request.ParseFromString(request_bytes) + + response = plugin_pb2.CodeGeneratorResponse() + # Advertise proto3-optional support so protoc does not reject the plugin. + response.supported_features = ( + plugin_pb2.CodeGeneratorResponse.FEATURE_PROTO3_OPTIONAL + ) + + # Build a lookup map for all file descriptors (needed for imports, though + # the current generator only uses the directly requested files). + file_by_name = {f.name: f for f in request.proto_file} + + for file_name in request.file_to_generate: + # Skip the options definition file itself — it has no messages to generate. + if file_name == 'decentraland/common/options.proto': + continue + + file_proto = file_by_name.get(file_name) + if file_proto is None: + continue + + try: + generated = generate_csharp(file_proto) + except Exception as exc: # noqa: BLE001 + error = response.file.add() + error.name = '' # empty name signals an error to protoc + # Append error text; protoc will print it and fail. + response.error = f'protoc-gen-bitwise: error processing {file_name}: {exc}' + continue + + if generated is not None: + out = response.file.add() + out.name = generated['name'] + out.content = generated['content'] + + sys.stdout.buffer.write(response.SerializeToString()) + + +if __name__ == '__main__': + main() diff --git a/protoc-gen-bitwise/runtime/cs/BitReader.cs b/protoc-gen-bitwise/runtime/cs/BitReader.cs new file mode 100644 index 00000000..51efa39c --- /dev/null +++ b/protoc-gen-bitwise/runtime/cs/BitReader.cs @@ -0,0 +1,112 @@ +// Decentraland.Networking.Bitwise — BitReader +// Copy this file into your Unity project alongside generated *.Bitwise.cs files. + +using System; + +namespace Decentraland.Networking.Bitwise +{ + /// + /// Reads bits from a byte buffer, MSB first within each byte (big-endian bit + /// order). Symmetric counterpart of : every + /// Write… call has a corresponding Read… call with identical arguments that + /// reproduces the original value. + /// + public sealed class BitReader + { + private readonly byte[] _buffer; + private int _bitPos; + + /// Source buffer filled by a . + public BitReader(byte[] buffer) + { + _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer)); + _bitPos = 0; + } + + /// Current read position in bits. + public int BitPosition => _bitPos; + + /// + /// Returns true when all written bits have been consumed + /// (i.e. has reached the end of the buffer). + /// + public bool IsAtEnd => _bitPos >= _buffer.Length * 8; + + // ----------------------------------------------------------------- + // Core primitive + // ----------------------------------------------------------------- + + /// + /// Reads bits and returns them as the + /// least-significant bits of a , MSB first. + /// + public uint ReadBits(int bits) + { + uint value = 0; + for (int i = bits - 1; i >= 0; i--) + { + int byteIdx = _bitPos / 8; + int bitIdx = 7 - (_bitPos % 8); + + if ((_buffer[byteIdx] >> bitIdx & 1) == 1) + value |= 1u << i; + + _bitPos++; + } + return value; + } + + // ----------------------------------------------------------------- + // Quantized float + // ----------------------------------------------------------------- + + /// + /// Reads a quantized float encoded with . + /// Arguments must match those used during encoding exactly. + /// + public float ReadQuantizedFloat(float min, float max, int bits) + { + uint maxQ = (1u << bits) - 1; + uint quantized = ReadBits(bits); + float normalized = (float)quantized / maxQ; + return min + normalized * (max - min); + } + + // ----------------------------------------------------------------- + // Standard IEEE 754 helpers + // ----------------------------------------------------------------- + + /// Reads a 32-bit IEEE 754 float written by . + public float ReadFloat() + { + uint bits = ReadBits(32); + byte[] bytes = + { + (byte)(bits & 0xFF), + (byte)((bits >> 8) & 0xFF), + (byte)((bits >> 16) & 0xFF), + (byte)((bits >> 24) & 0xFF), + }; + return BitConverter.ToSingle(bytes, 0); + } + + /// Reads a 64-bit IEEE 754 double written by . + public double ReadDouble() + { + uint hi = ReadBits(32); + uint lo = ReadBits(32); + byte[] bytes = + { + (byte)(lo & 0xFF), + (byte)((lo >> 8) & 0xFF), + (byte)((lo >> 16) & 0xFF), + (byte)((lo >> 24) & 0xFF), + (byte)(hi & 0xFF), + (byte)((hi >> 8) & 0xFF), + (byte)((hi >> 16) & 0xFF), + (byte)((hi >> 24) & 0xFF), + }; + return BitConverter.ToDouble(bytes, 0); + } + } +} diff --git a/protoc-gen-bitwise/runtime/cs/BitWriter.cs b/protoc-gen-bitwise/runtime/cs/BitWriter.cs new file mode 100644 index 00000000..19b205b8 --- /dev/null +++ b/protoc-gen-bitwise/runtime/cs/BitWriter.cs @@ -0,0 +1,117 @@ +// Decentraland.Networking.Bitwise — BitWriter +// Copy this file into your Unity project alongside generated *.Bitwise.cs files. + +using System; + +namespace Decentraland.Networking.Bitwise +{ + /// + /// Writes bits into a pre-allocated byte buffer, MSB first within each byte + /// (big-endian bit order). This matches the layout expected by + /// so that encode → decode is always a round-trip no-op. + /// + public sealed class BitWriter + { + private readonly byte[] _buffer; + private int _bitPos; + + /// Destination buffer (must be large enough for all writes). + public BitWriter(byte[] buffer) + { + _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer)); + _bitPos = 0; + } + + /// Current write position in bits. + public int BitPosition => _bitPos; + + /// Number of bytes written (rounded up to the nearest byte). + public int ByteLength => (_bitPos + 7) / 8; + + /// Returns a copy of the written bytes (trimmed to ). + public byte[] ToArray() + { + var result = new byte[ByteLength]; + Array.Copy(_buffer, result, ByteLength); + return result; + } + + // ----------------------------------------------------------------- + // Core primitive + // ----------------------------------------------------------------- + + /// + /// Writes the least-significant bits of + /// , MSB first. + /// + public void WriteBits(uint value, int bits) + { + for (int i = bits - 1; i >= 0; i--) + { + int byteIdx = _bitPos / 8; + int bitIdx = 7 - (_bitPos % 8); + + if ((value >> i & 1u) == 1u) + _buffer[byteIdx] |= (byte)(1 << bitIdx); + else + _buffer[byteIdx] &= (byte)~(1 << bitIdx); + + _bitPos++; + } + } + + // ----------------------------------------------------------------- + // Quantized float + // ----------------------------------------------------------------- + + /// + /// Quantizes into bits + /// using the range [, ] and + /// writes it to the buffer. + /// + /// Uses Math.Round (banker's rounding → ties-to-even) to guarantee + /// that encode → decode is a round-trip no-op. + /// + public void WriteQuantizedFloat(float value, float min, float max, int bits) + { + uint maxQ = (1u << bits) - 1; + float clamped = Math.Clamp(value, min, max); + float normalized = (clamped - min) / (max - min); + uint quantized = (uint)Math.Round(normalized * maxQ); + WriteBits(quantized, bits); + } + + // ----------------------------------------------------------------- + // Standard IEEE 754 helpers (used for un-annotated float/double fields) + // ----------------------------------------------------------------- + + /// Writes a 32-bit IEEE 754 float (4 bytes). + public void WriteFloat(float value) + { + byte[] bytes = BitConverter.GetBytes(value); + // GetBytes is little-endian on all platforms; write MSB first. + uint bits = (uint)bytes[0] + | ((uint)bytes[1] << 8) + | ((uint)bytes[2] << 16) + | ((uint)bytes[3] << 24); + WriteBits(bits, 32); + } + + /// Writes a 64-bit IEEE 754 double (8 bytes). + public void WriteDouble(double value) + { + byte[] bytes = BitConverter.GetBytes(value); + uint lo = (uint)bytes[0] + | ((uint)bytes[1] << 8) + | ((uint)bytes[2] << 16) + | ((uint)bytes[3] << 24); + uint hi = (uint)bytes[4] + | ((uint)bytes[5] << 8) + | ((uint)bytes[6] << 16) + | ((uint)bytes[7] << 24); + // Write high 32 bits first so the bit stream is big-endian at word level too. + WriteBits(hi, 32); + WriteBits(lo, 32); + } + } +} diff --git a/protoc-gen-bitwise/runtime/cs/Quantize.cs b/protoc-gen-bitwise/runtime/cs/Quantize.cs new file mode 100644 index 00000000..22c347fe --- /dev/null +++ b/protoc-gen-bitwise/runtime/cs/Quantize.cs @@ -0,0 +1,35 @@ +// Decentraland.Networking.Bitwise — Quantize +// Copy this file into your project alongside generated *.Bitwise.cs files. +// ReSharper disable once RedundantUsingDirective + +using System; + +namespace Decentraland.Networking.Bitwise + // ReSharper disable once ArrangeNamespaceBody +{ + /// + /// Static helpers for quantizing float values to/from unsigned integers. + /// Intended for use with protobuf uint32 fields: the integer is transmitted + /// as a protobuf varint, while the float accessor lives in a generated partial class. + /// + public static class Quantize + { + /// + /// Encodes to a quantized . + /// Values outside [, ] are clamped. + /// + public static uint Encode(float value, float min, float max, int bits) + { + var steps = (1u << bits) - 1; + var t = Math.Clamp((value - min) / (max - min), 0f, 1f); + return (uint)MathF.Round(t * steps); + } + + /// Decodes a quantized back to a float. + public static float Decode(uint encoded, float min, float max, int bits) + { + var steps = (1u << bits) - 1; + return (float)encoded / steps * (max - min) + min; + } + } +} From f0df3e02ab2f989527ee9b45a1dbc85948842aab Mon Sep 17 00:00:00 2001 From: Mikhail Agapov <118179774+mikhail-dcl@users.noreply.github.com> Date: Mon, 15 Jun 2026 23:28:03 +0300 Subject: [PATCH 44/54] chore: port protoc-gen-bitwise to Node (drop Python dependency) (#423) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrite the protoc-gen-bitwise plugin from Python to a dependency-free Node implementation, so protocol generation needs only `node` on PATH — no Python interpreter and no `protobuf` pip package. - plugin.js stdin/stdout protoc plugin entry; advertises FEATURE_PROTO3_OPTIONAL - wire.js self-contained protobuf wire codec for CodeGeneratorRequest/Response (no runtime deps) - options.js parser for the custom quantized / bit_packed options - generator_csharp.js C# partial-class generator with faithful %g float formatting; output is byte-for-byte identical to the Python plugin - test/ + `npm run gen:test` golden-fixture parity test - remove plugin.py, generator_csharp.py, options_pb2.py - package.json ship only the .js + C# runtime (exclude test fixtures) - README.md, CLAUDE.md Python -> Node prerequisites and invocation Verified byte-identical regeneration three ways: protoc end-to-end, a Pulse clean-room rebuild (all 16 generated files), and unity-explorer build-protocol. Consumers invoke the plugin via a tiny `node plugin.js` wrapper (.cmd on Windows, shell script elsewhere). Co-authored-by: Claude Opus 4.8 (1M context) --- CLAUDE.md | 184 ++++++++++++++ README.md | 15 +- package.json | 6 +- protoc-gen-bitwise/generator_csharp.js | 215 ++++++++++++++++ protoc-gen-bitwise/generator_csharp.py | 176 ------------- protoc-gen-bitwise/options.js | 108 ++++++++ protoc-gen-bitwise/options_pb2.py | 171 ------------- protoc-gen-bitwise/plugin.js | 87 +++++++ protoc-gen-bitwise/plugin.py | 92 ------- protoc-gen-bitwise/test/generator.test.js | 180 +++++++++++++ .../test/golden/PulseServer.Bitwise.cs | 145 +++++++++++ .../golden/QuantizationExample.Bitwise.cs | 134 ++++++++++ protoc-gen-bitwise/wire.js | 239 ++++++++++++++++++ 13 files changed, 1305 insertions(+), 447 deletions(-) create mode 100644 CLAUDE.md create mode 100644 protoc-gen-bitwise/generator_csharp.js delete mode 100644 protoc-gen-bitwise/generator_csharp.py create mode 100644 protoc-gen-bitwise/options.js delete mode 100644 protoc-gen-bitwise/options_pb2.py create mode 100755 protoc-gen-bitwise/plugin.js delete mode 100644 protoc-gen-bitwise/plugin.py create mode 100644 protoc-gen-bitwise/test/generator.test.js create mode 100644 protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs create mode 100644 protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs create mode 100644 protoc-gen-bitwise/wire.js diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..130756ee --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,184 @@ +# CLAUDE.md — MMO Networking Stack + +## Project Overview + +High-performance MMO-style multiplayer networking stack. Protocol is **open** (Unity C# client + others). Infrastructure on **AWS**. Goals: high concurrency, aggressive interest management, low-latency state sync. + +| Layer | Choice | +|---|---| +| Transport | ENet (UDP, channel-based) | +| Client | Unity (C#) | +| Server | Custom server | +| Schema source of truth | `.proto` files | +| Serialization | Custom protoc plugin (bitwise encoding) | +| Auth | Decentraland ECDSA chain validation (local, on HANDSHAKE channel 0) | + +--- + +## Serialization: Custom Protoc Plugin + +### What it does +Reads `.proto` files with custom field options and generates **bitwise encode/decode code** in C#, keeping all client implementations bit-for-bit identical. + +### Custom Field Options (`options.proto`) + +```protobuf +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; + +message QuantizedFloatOptions { + float min = 1; + float max = 2; + uint32 bits = 3; +} + +message BitPackedOptions { + uint32 bits = 1; +} + +extend google.protobuf.FieldOptions { + QuantizedFloatOptions quantized = 50001; + BitPackedOptions bit_packed = 50002; +} +``` + +### Usage example + +```protobuf +message PositionDelta { + float dx = 1 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + float dy = 2 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + float dz = 3 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 entity_id = 4 [(bit_packed) = { bits: 20 }]; +} +// Total: 68 bits = 9 bytes on the wire +``` + +### Plugin structure + +``` +protoc-gen-bitwise/ +├── plugin.js # stdin -> CodeGeneratorRequest, stdout -> CodeGeneratorResponse (Node) +├── generator_csharp.js # emits C# for Unity +├── options.js # parses the custom quantized / bit_packed field options +├── wire.js # self-contained protobuf wire codec (zero runtime deps) +└── runtime/cs/ # C# runtime; Quantize.cs is copied into the generated output +``` + +Plugin contract: a protoc plugin that reads a serialized `CodeGeneratorRequest` from stdin and writes a serialized `CodeGeneratorResponse` to stdout. It is a plain Node script — **no `npm install` required, only `node` on PATH**. protoc invokes it through a tiny wrapper that runs `node plugin.js` (`.cmd` on Windows, a shell script elsewhere, since protoc cannot exec a `.js` directly). + +```bash +protoc \ + --proto_path=proto \ + --bitwise_out=generated/ \ + --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.js \ + movement.proto position.proto +``` + +Parity is locked down by `npm run gen:test` (compares generator output against golden C# fixtures in `protoc-gen-bitwise/test/`). + +--- + +## BitWriter / BitReader + +The C# implementation uses the following bit layout: **big-endian within each byte**, MSB written first. Use **`Round`** (not truncate) when quantizing to minimize error. + +### Core math — WriteQuantizedFloat + +``` +normalized = (clamp(value, min, max) - min) / (max - min) // -> [0.0, 1.0] +quantized = Round(normalized * ((1 << bits) - 1)) // -> integer +``` + +### Core math — ReadQuantizedFloat + +``` +normalized = quantized / ((1 << bits) - 1) +value = min + normalized * (max - min) +``` + +### Implementation + +```csharp +public class BitWriter +{ + private byte[] _buffer; + private int _bitPos; + + public BitWriter(byte[] buffer) { _buffer = buffer; _bitPos = 0; } + + public void WriteBits(uint value, int bits) + { + for (int i = bits - 1; i >= 0; i--) + { + int byteIdx = _bitPos / 8; + int bitIdx = 7 - (_bitPos % 8); + if ((value >> i & 1) == 1) _buffer[byteIdx] |= (byte)(1 << bitIdx); + else _buffer[byteIdx] &= (byte)~(1 << bitIdx); + _bitPos++; + } + } + + public void WriteQuantizedFloat(float value, float min, float max, int bits) + { + uint maxQ = (1u << bits) - 1; + float clamped = Math.Clamp(value, min, max); + float normalized = (clamped - min) / (max - min); + uint quantized = (uint)Math.Round(normalized * maxQ); + WriteBits(quantized, bits); + } +} + +public class BitReader +{ + private byte[] _buffer; + private int _bitPos; + + public BitReader(byte[] buffer) { _buffer = buffer; _bitPos = 0; } + + public uint ReadBits(int bits) + { + uint value = 0; + for (int i = bits - 1; i >= 0; i--) + { + int byteIdx = _bitPos / 8; + int bitIdx = 7 - (_bitPos % 8); + if ((_buffer[byteIdx] >> bitIdx & 1) == 1) value |= 1u << i; + _bitPos++; + } + return value; + } + + public float ReadQuantizedFloat(float min, float max, int bits) + { + uint maxQ = (1u << bits) - 1; + uint quantized = ReadBits(bits); + float normalized = (float)quantized / maxQ; + return min + normalized * (max - min); + } +} +``` + +--- + +## Precision Reference + +| Range | Bits | Step size | +|-------------|------|---------------| +| [-100, 100] | 16 | ~0.003 units | +| [-10, 10] | 12 | ~0.005 units | +| [-100, 100] | 12 | ~0.049 units | + +Sub-centimeter precision is achievable at 12-16 bits for position deltas. + +--- + +## Key Design Principles + +- `.proto` files are the **single source of truth** for all message schemas +- The protoc plugin generates **C#** from the schema — never hand-write serialization +- Encode -> decode is a **no-op** (round-trip safe) due to consistent use of `Round` +- Prefer **client-driven resync** over proactive server corrections +- Push complexity to clients where appropriate; server maintains authority +- Channel 0: reliable messages (STATE_FULL snapshots, ACKs, resync requests, HANDSHAKE) +- Channel 1: unreliable sequenced (high-frequency position deltas, client input) \ No newline at end of file diff --git a/README.md b/README.md index 20c75605..b11c3302 100644 --- a/README.md +++ b/README.md @@ -100,13 +100,11 @@ client can read it without knowledge of the plugin. | Requirement | Version | |---|---| -| Python | 3.10+ | -| `protobuf` Python package | 4.x or 3.20+ | +| Node.js | 16+ | | `protoc` | 3.19+ | -```bash -pip install protobuf -``` +The plugin is a dependency-free Node script — no `npm install` or extra +packages are required to run it. ## Step 1 — Annotate your `.proto` file @@ -157,11 +155,16 @@ protoc \ --proto_path=proto \ --proto_path=/path/to/google/protobuf/include \ --csharp_out=generated/cs \ - --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.py \ + --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.js \ --bitwise_out=generated/cs \ proto/decentraland/kernel/comms/v3/comms.proto ``` +> `plugin.js` carries a `#!/usr/bin/env node` shebang. On Windows, protoc +> cannot exec a `.js` directly, so point `--plugin` at a small `.cmd` wrapper +> that runs `node plugin.js`; on unix an equivalent shell script is used. Both +> consumer repos (Pulse, unity-explorer) generate this wrapper automatically. + The plugin emits one `*.Bitwise.cs` file (PascalCase, flat in the output directory) for each `.proto` file that contains at least one `[(quantized)]` field. diff --git a/package.json b/package.json index 887d6567..b0252db0 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "license": "Apache-2.0", "main": "index.js", "scripts": { - "test": "./scripts/check-proto-compabitility.sh" + "test": "./scripts/check-proto-compabitility.sh", + "gen:test": "node protoc-gen-bitwise/test/generator.test.js" }, "devDependencies": { "@protobuf-ts/protoc": "^2.11.0", @@ -32,6 +33,7 @@ "out-ts", "out-js", "public", - "protoc-gen-bitwise" + "protoc-gen-bitwise/*.js", + "protoc-gen-bitwise/runtime" ] } diff --git a/protoc-gen-bitwise/generator_csharp.js b/protoc-gen-bitwise/generator_csharp.js new file mode 100644 index 00000000..dcdb95a0 --- /dev/null +++ b/protoc-gen-bitwise/generator_csharp.js @@ -0,0 +1,215 @@ +'use strict' + +/** + * C# code generator for the protoc-gen-bitwise plugin. + * + * For every proto message that contains at least one uint32 field annotated + * with [(quantized)], this module emits a C# partial class that adds a computed + * float property named {FieldName}Quantized. The getter decodes the stored + * uint32 to a float; the setter encodes a float back to a uint32. Standard + * protobuf handles serialization of the uint32 wire field; this class adds a + * typed float accessor on top. + * + * Only uint32 fields are supported. bit_packed and unannotated fields are + * passed through without generating any accessor. + * + * Port of the original generator_csharp.py — output is intended to be + * byte-for-byte identical. + */ + +const { getFieldOptions } = require('./options') + +// FieldDescriptorProto type/label constants. +const TYPE_UINT32 = 13 +const LABEL_REPEATED = 3 + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** Mirrors Python str.capitalize(): upper-first, lowercase the rest. */ +function capitalize(word) { + if (word.length === 0) return '' + return word[0].toUpperCase() + word.slice(1).toLowerCase() +} + +/** position_x -> PositionX */ +function snakeToPascal(name) { + return name.split('_').map(capitalize).join('') +} + +/** decentraland.kernel.comms.v3 -> Decentraland.Kernel.Comms.V3 */ +function packageToNamespace(pkg) { + if (!pkg) return 'Generated' + return pkg.split('.').map(capitalize).join('.') +} + +/** + * Format a number with C printf %g semantics at `precision` significant digits: + * shortest of fixed/scientific, with trailing zeros and a trailing dot removed. + * Reproduces Python's `f'{value:.{precision}g}'`. + */ +function formatG(value, precision) { + if (precision <= 0) precision = 1 + if (value === 0) return '0' + if (!Number.isFinite(value)) return value > 0 ? 'inf' : 'nan' + + const negative = value < 0 + const v = Math.abs(value) + + // Correctly-rounded scientific form yields the decimal exponent X (handling + // carry such as 9.999 -> 1.0e+1). + const sci = v.toExponential(precision - 1) + const eIdx = sci.indexOf('e') + const X = parseInt(sci.slice(eIdx + 1), 10) + + let result + if (X >= -4 && X < precision) { + // Fixed notation with (precision - 1 - X) fraction digits. + const fractionDigits = precision - 1 - X + result = v.toFixed(fractionDigits >= 0 ? fractionDigits : 0) + if (result.indexOf('.') !== -1) { + result = result.replace(/0+$/, '').replace(/\.$/, '') + } + } else { + // Scientific notation; printf prints at least two exponent digits. + let mantissa = sci.slice(0, eIdx) + if (mantissa.indexOf('.') !== -1) { + mantissa = mantissa.replace(/0+$/, '').replace(/\.$/, '') + } + const expSign = X < 0 ? '-' : '+' + let expDigits = String(Math.abs(X)) + if (expDigits.length < 2) expDigits = '0' + expDigits + result = mantissa + 'e' + expSign + expDigits + } + + return (negative ? '-' : '') + result +} + +/** Format a value as a C# float literal (e.g. -100.0f). */ +function formatFloat(value) { + let text = formatG(value, 8) + if (text.indexOf('.') === -1 && text.indexOf('e') === -1 && text.indexOf('E') === -1) { + text += '.0' + } + return text + 'f' +} + +/** Format a quantization step size for a doc comment (e.g. "≈ 0.003"). */ +function formatStep(step) { + return '≈ ' + formatG(step, 6) +} + +// --------------------------------------------------------------------------- +// Per-message code generation +// --------------------------------------------------------------------------- + +/** + * Generate a C# partial class for a proto message, or null if it has no + * quantized uint32 fields. Returns an array of lines (no trailing newline). + */ +function generateMessage(msgProto, indent) { + const i = indent || ' ' + const props = [] + + for (const field of msgProto.field) { + // Repeated/map fields are not supported. + if (field.label === LABEL_REPEATED) continue + // Only uint32 fields are candidates for quantized accessors. + if (field.type !== TYPE_UINT32) continue + + const { quantized } = getFieldOptions(field.optionsRaw) + if (quantized === null) continue + + const step = (quantized.max - quantized.min) / ((1 << quantized.bits) - 1) + + props.push({ + propName: snakeToPascal(field.name), + mn: formatFloat(quantized.min), + mx: formatFloat(quantized.max), + bits: quantized.bits, + step, + }) + } + + if (props.length === 0) return null + + const backings = [] + const lines = [] + lines.push(`public partial class ${msgProto.name}`) + lines.push('{') + + for (const { propName, mn, mx, bits, step } of props) { + const backing = '_' + propName[0].toLowerCase() + propName.slice(1) + backings.push(backing) + lines.push(`${i}private float? ${backing};`) + lines.push( + `${i}/// Float accessor for . Range [${mn}, ${mx}], ${bits} bits, step ${formatStep(step)}.`, + ) + lines.push(`${i}public float ${propName}Quantized`) + lines.push(`${i}{`) + lines.push(`${i}${i}get => ${backing} ??= Quantize.Decode(${propName}, ${mn}, ${mx}, ${bits});`) + lines.push(`${i}${i}set { ${backing} = value; ${propName} = Quantize.Encode(value, ${mn}, ${mx}, ${bits}); }`) + lines.push(`${i}}`) + lines.push('') + } + + lines.push(`${i}/// Clears all cached decoded values. Call after mutating raw uint32 fields directly.`) + lines.push(`${i}public void ResetDecodedCache()`) + lines.push(`${i}{`) + for (const backing of backings) { + lines.push(`${i}${i}${backing} = null;`) + } + lines.push(`${i}}`) + + lines.push('}') + return lines +} + +// --------------------------------------------------------------------------- +// Per-file code generation (public entry point) +// --------------------------------------------------------------------------- + +/** + * Generate a C# source file for a FileDescriptorProto, or null if the file + * contains no quantized uint32 fields. + * @returns {{name: string, content: string} | null} + */ +function generateCsharp(fileProto) { + const namespace = packageToNamespace(fileProto.package) + + const protoFile = fileProto.name.split('/').pop() + const stem = snakeToPascal(protoFile.replace('.proto', '')) + const outName = `${stem}.Bitwise.cs` + + const header = [ + '// ', + '// Generated by protoc-gen-bitwise. DO NOT EDIT.', + `// Source: ${fileProto.name}`, + '// ', + '', + 'using Decentraland.Networking.Bitwise;', + '', + `namespace ${namespace}`, + '{', + ] + const footer = ['', `} // namespace ${namespace}`] + + const body = [] + for (const msg of fileProto.messageType) { + const msgLines = generateMessage(msg) + if (msgLines === null) continue + // Indent each line by 4 spaces (inside the namespace block). + for (const line of msgLines) { + body.push(line ? ' ' + line : '') + } + body.push('') + } + + if (body.length === 0) return null + + const content = header.concat(body, footer).join('\n') + '\n' + return { name: outName, content } +} + +module.exports = { generateCsharp } diff --git a/protoc-gen-bitwise/generator_csharp.py b/protoc-gen-bitwise/generator_csharp.py deleted file mode 100644 index 376c0dc1..00000000 --- a/protoc-gen-bitwise/generator_csharp.py +++ /dev/null @@ -1,176 +0,0 @@ -""" -C# code generator for the protoc-gen-bitwise plugin. - -For every proto message that contains at least one uint32 field annotated with -[(quantized)], this module emits a C# partial class that adds a computed float -property named {FieldName}Quantized. The getter decodes the stored uint32 to -a float; the setter encodes a float back to a uint32. Standard protobuf -handles serialization of the uint32 wire field; this class adds a typed float -accessor on top. - -Protobuf encodes small uint32 values via varint, so a value that fits in -2^20 costs 3 bytes on the wire — cheaper than a raw IEEE 754 float (4 bytes). - -Only uint32 fields are supported. bit_packed and unannotated fields are -passed through without generating any accessor. -""" - -from google.protobuf import descriptor_pb2 - -from options_pb2 import get_field_options - -# FieldDescriptorProto type constants (aliased for readability) -_FT = descriptor_pb2.FieldDescriptorProto - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _snake_to_pascal(name: str) -> str: - """position_x → PositionX""" - return ''.join(word.capitalize() for word in name.split('_')) - - -def _package_to_namespace(package: str) -> str: - """decentraland.kernel.comms.v3 → Decentraland.Kernel.Comms.V3""" - if not package: - return 'Generated' - return '.'.join(part.capitalize() for part in package.split('.')) - - -def _format_float(value: float) -> str: - """Format a Python float as a C# float literal (e.g. -100.0f).""" - text = f'{value:.8g}' - if '.' not in text and 'e' not in text and 'E' not in text: - text += '.0' - return text + 'f' - - -def _format_step(step: float) -> str: - """Format a quantization step size for display in a doc comment (e.g. ≈ 0.003).""" - return f'\u2248 {step:.6g}' - - -# --------------------------------------------------------------------------- -# Per-message code generation -# --------------------------------------------------------------------------- - -def _gen_message(msg_proto, indent: str = ' ') -> list[str] | None: - """ - Generate a C# partial class for a proto message. - - For each uint32 field with a [(quantized)] annotation, emits a cached float - property {FieldName}Quantized backed by the raw uint32 field. - - Returns a list of lines (without trailing newline) or None if the message - has no quantized uint32 fields. - """ - props: list[tuple[str, str, str, int, float]] = [] # (prop_name, mn, mx, bits, step) - - for field in msg_proto.field: - # Repeated/map fields are not supported - if field.label == _FT.LABEL_REPEATED: - continue - - # Only uint32 fields are candidates for quantized accessors - if field.type != _FT.TYPE_UINT32: - continue - - quantized, _ = get_field_options(field.options) - if quantized is None: - continue - - step = (quantized.max - quantized.min) / ((1 << quantized.bits) - 1) - - props.append(( - _snake_to_pascal(field.name), - _format_float(quantized.min), - _format_float(quantized.max), - quantized.bits, - step, - )) - - if not props: - return None - - i = indent - backings: list[str] = [] - lines: list[str] = [] - lines.append(f'public partial class {msg_proto.name}') - lines.append('{') - - for prop_name, mn, mx, bits, step in props: - backing = '_' + prop_name[0].lower() + prop_name[1:] - backings.append(backing) - lines.append(f'{i}private float? {backing};') - lines.append(f'{i}/// Float accessor for . Range [{mn}, {mx}], {bits} bits, step {_format_step(step)}.') - lines.append(f'{i}public float {prop_name}Quantized') - lines.append(f'{i}{{') - lines.append(f'{i}{i}get => {backing} ??= Quantize.Decode({prop_name}, {mn}, {mx}, {bits});') - lines.append(f'{i}{i}set {{ {backing} = value; {prop_name} = Quantize.Encode(value, {mn}, {mx}, {bits}); }}') - lines.append(f'{i}}}') - lines.append('') - - lines.append(f'{i}/// Clears all cached decoded values. Call after mutating raw uint32 fields directly.') - lines.append(f'{i}public void ResetDecodedCache()') - lines.append(f'{i}{{') - for backing in backings: - lines.append(f'{i}{i}{backing} = null;') - lines.append(f'{i}}}') - - lines.append('}') - return lines - - -# --------------------------------------------------------------------------- -# Per-file code generation (public entry point) -# --------------------------------------------------------------------------- - -def generate_csharp(file_proto) -> dict | None: - """ - Generate a C# source file for a FileDescriptorProto. - - Returns a dict with keys 'name' (output path) and 'content' (C# source), - or None if the file contains no quantized uint32 fields. - """ - namespace = _package_to_namespace(file_proto.package) - - # Output is placed flat in the output root, matching --csharp_out convention. - # e.g. "decentraland/kernel/comms/v3/my_message.proto" - # → "MyMessage.Bitwise.cs" - proto_file = file_proto.name.rsplit('/', 1)[-1] - stem = _snake_to_pascal(proto_file.replace('.proto', '')) - out_name = f'{stem}.Bitwise.cs' - - header = [ - '// ', - '// Generated by protoc-gen-bitwise. DO NOT EDIT.', - f'// Source: {file_proto.name}', - '// ', - '', - 'using Decentraland.Networking.Bitwise;', - '', - f'namespace {namespace}', - '{', - ] - footer = [ - '', - f'}} // namespace {namespace}', - ] - - body: list[str] = [] - for msg in file_proto.message_type: - msg_lines = _gen_message(msg) - if msg_lines is None: - continue - # Indent each line by 4 spaces (inside the namespace block) - for line in msg_lines: - body.append((' ' + line) if line else '') - body.append('') - - if not body: - return None # nothing to emit - - content = '\n'.join(header + body + footer) + '\n' - return {'name': out_name, 'content': content} diff --git a/protoc-gen-bitwise/options.js b/protoc-gen-bitwise/options.js new file mode 100644 index 00000000..150ece0b --- /dev/null +++ b/protoc-gen-bitwise/options.js @@ -0,0 +1,108 @@ +'use strict' + +/** + * Parser for the custom bitwise field options defined in options.proto. + * + * The descriptor decoder hands us the raw serialized FieldOptions bytes (it + * declares `options` as opaque bytes). We walk those bytes looking for the two + * custom extension field numbers — protobuf preserves unknown/unregistered + * extension bytes, so they are always present even though no runtime here knows + * the extension schema. This mirrors the original options_pb2.py. + * + * Wire format: tag = (field_number << 3) | wire_type + * wire_type 0 = varint, 1 = 64-bit, 2 = length-delimited, 5 = 32-bit + */ + +const { readVarint, skipField } = require('./wire') + +// Extension field numbers as defined in options.proto. +const QUANTIZED_FIELD_NUMBER = 50001 +const BIT_PACKED_FIELD_NUMBER = 50002 + +/** Parse a serialized QuantizedFloatOptions message: { min, max, bits }. */ +function parseQuantized(data) { + const opts = { min: 0.0, max: 0.0, bits: 0 } + let pos = 0 + while (pos < data.length) { + let tag + ;[tag, pos] = readVarint(data, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 5) { + // min (float) + opts.min = data.readFloatLE(pos) + pos += 4 + } else if (fieldNum === 2 && wireType === 5) { + // max (float) + opts.max = data.readFloatLE(pos) + pos += 4 + } else if (fieldNum === 3 && wireType === 0) { + // bits (uint32) + ;[opts.bits, pos] = readVarint(data, pos) + } else { + pos = skipField(data, pos, wireType) + } + } + return opts +} + +/** Parse a serialized BitPackedOptions message: { bits }. */ +function parseBitPacked(data) { + const opts = { bits: 0 } + let pos = 0 + while (pos < data.length) { + let tag + ;[tag, pos] = readVarint(data, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 0) { + // bits (uint32) + ;[opts.bits, pos] = readVarint(data, pos) + } else { + pos = skipField(data, pos, wireType) + } + } + return opts +} + +/** + * Extract custom bitwise options from raw FieldOptions bytes. + * + * @param {Buffer|null} optionsRaw serialized FieldOptions, or null when unset. + * @returns {{quantized: object|null, bitPacked: object|null}} + */ +function getFieldOptions(optionsRaw) { + if (!optionsRaw || optionsRaw.length === 0) { + return { quantized: null, bitPacked: null } + } + + let quantized = null + let bitPacked = null + let pos = 0 + + while (pos < optionsRaw.length) { + let tag + ;[tag, pos] = readVarint(optionsRaw, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + + if (wireType === 2) { + let len + ;[len, pos] = readVarint(optionsRaw, pos) + const valueBytes = optionsRaw.subarray(pos, pos + len) + pos += len + if (fieldNum === QUANTIZED_FIELD_NUMBER) { + quantized = parseQuantized(valueBytes) + } else if (fieldNum === BIT_PACKED_FIELD_NUMBER) { + bitPacked = parseBitPacked(valueBytes) + } + // else: unknown length-delimited field — already consumed. + } else { + pos = skipField(optionsRaw, pos, wireType) + } + } + + return { quantized, bitPacked } +} + +module.exports = { getFieldOptions } diff --git a/protoc-gen-bitwise/options_pb2.py b/protoc-gen-bitwise/options_pb2.py deleted file mode 100644 index d3e637f2..00000000 --- a/protoc-gen-bitwise/options_pb2.py +++ /dev/null @@ -1,171 +0,0 @@ -""" -Manual parser for the custom bitwise field options defined in options.proto. - -Rather than relying on protobuf extension registration (which requires a properly -compiled _pb2 module), this module parses the raw serialized FieldOptions bytes -directly using the protobuf binary wire format. All protobuf runtimes preserve -unknown/unregistered extension bytes when round-tripping, so -field.options.SerializeToString() always contains the extension data even when -the extension is not registered in the Python runtime. - -Wire format reference: - tag = (field_number << 3) | wire_type - wire_type 0 = varint, 1 = 64-bit, 2 = length-delimited, 5 = 32-bit -""" - -import struct - -# Extension field numbers as defined in options.proto -QUANTIZED_FIELD_NUMBER = 50001 -BIT_PACKED_FIELD_NUMBER = 50002 - - -# --------------------------------------------------------------------------- -# Low-level wire-format helpers -# --------------------------------------------------------------------------- - -def _read_varint(data: bytes, pos: int): - """Decode a protobuf varint starting at *pos*. Returns (value, new_pos).""" - result = 0 - shift = 0 - while pos < len(data): - byte = data[pos] - pos += 1 - result |= (byte & 0x7F) << shift - if not (byte & 0x80): - break - shift += 7 - return result, pos - - -def _read_float32(data: bytes, pos: int): - """Decode a little-endian 32-bit float. Returns (value, new_pos).""" - value, = struct.unpack_from(' int: - """Advance *pos* past a field with the given wire_type.""" - if wire_type == 0: - _, pos = _read_varint(data, pos) - elif wire_type == 1: - pos += 8 - elif wire_type == 2: - length, pos = _read_varint(data, pos) - pos += length - elif wire_type == 5: - pos += 4 - # wire types 3 and 4 (start/end group) are deprecated; skip gracefully - return pos - - -# --------------------------------------------------------------------------- -# Option message classes -# --------------------------------------------------------------------------- - -class QuantizedFloatOptions: - """Mirrors the QuantizedFloatOptions proto message.""" - - __slots__ = ('min', 'max', 'bits') - - def __init__(self, min_val: float = 0.0, max_val: float = 0.0, bits: int = 0): - self.min = min_val - self.max = max_val - self.bits = bits - - @classmethod - def from_bytes(cls, data: bytes) -> 'QuantizedFloatOptions': - obj = cls() - pos = 0 - while pos < len(data): - tag, pos = _read_varint(data, pos) - field_num = tag >> 3 - wire_type = tag & 0x7 - if field_num == 1 and wire_type == 5: # min (float) - obj.min, pos = _read_float32(data, pos) - elif field_num == 2 and wire_type == 5: # max (float) - obj.max, pos = _read_float32(data, pos) - elif field_num == 3 and wire_type == 0: # bits (uint32) - obj.bits, pos = _read_varint(data, pos) - else: - pos = _skip_field(data, pos, wire_type) - return obj - - def __repr__(self): - return f'QuantizedFloatOptions(min={self.min}, max={self.max}, bits={self.bits})' - - -class BitPackedOptions: - """Mirrors the BitPackedOptions proto message.""" - - __slots__ = ('bits',) - - def __init__(self, bits: int = 0): - self.bits = bits - - @classmethod - def from_bytes(cls, data: bytes) -> 'BitPackedOptions': - obj = cls() - pos = 0 - while pos < len(data): - tag, pos = _read_varint(data, pos) - field_num = tag >> 3 - wire_type = tag & 0x7 - if field_num == 1 and wire_type == 0: # bits (uint32) - obj.bits, pos = _read_varint(data, pos) - else: - pos = _skip_field(data, pos, wire_type) - return obj - - def __repr__(self): - return f'BitPackedOptions(bits={self.bits})' - - -# --------------------------------------------------------------------------- -# Public API -# --------------------------------------------------------------------------- - -def get_field_options(field_options_proto): - """ - Extract custom bitwise options from a FieldDescriptorProto.options object. - - Serialises the options message to bytes and walks the wire-format stream - looking for extension fields 50001 (quantized) and 50002 (bit_packed). - - Args: - field_options_proto: google.protobuf.descriptor_pb2.FieldOptions instance - (may be a default/empty instance when no options are set). - - Returns: - tuple[QuantizedFloatOptions | None, BitPackedOptions | None] - """ - try: - raw = field_options_proto.SerializeToString() - except Exception: - return None, None - - if not raw: - return None, None - - quantized = None - bit_packed = None - pos = 0 - - while pos < len(raw): - tag, pos = _read_varint(raw, pos) - field_num = tag >> 3 - wire_type = tag & 0x7 - - if wire_type == 2: # length-delimited - length, pos = _read_varint(raw, pos) - value_bytes = raw[pos:pos + length] - pos += length - if field_num == QUANTIZED_FIELD_NUMBER: - quantized = QuantizedFloatOptions.from_bytes(value_bytes) - elif field_num == BIT_PACKED_FIELD_NUMBER: - bit_packed = BitPackedOptions.from_bytes(value_bytes) - # else: unknown length-delimited field — already consumed - else: - pos = _skip_field(raw, pos, wire_type) - - return quantized, bit_packed diff --git a/protoc-gen-bitwise/plugin.js b/protoc-gen-bitwise/plugin.js new file mode 100755 index 00000000..b08c6a12 --- /dev/null +++ b/protoc-gen-bitwise/plugin.js @@ -0,0 +1,87 @@ +#!/usr/bin/env node +'use strict' + +/** + * protoc-gen-bitwise — protoc plugin that generates C# bitwise serialization code. + * + * Protocol: + * 1. protoc writes a serialised CodeGeneratorRequest to this process's stdin. + * 2. This plugin reads it, generates C# partial classes with float accessor + * properties for every message that carries [(quantized)] field annotations. + * 3. A serialised CodeGeneratorResponse is written to stdout. + * + * Implemented in plain Node with a self-contained protobuf wire codec (see + * wire.js) so it runs with only `node` on PATH — no npm install required, even + * when invoked directly from a sibling checkout. + * + * Usage (from project root): + * protoc \ + * --proto_path=proto \ + * --bitwise_out=generated/ \ + * --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.js \ + * proto/my_messages.proto + * + * On Windows, protoc needs an executable wrapper, e.g. a .cmd that runs: + * node "\protoc-gen-bitwise\plugin.js" %* + */ + +const { decodeRequest, encodeResponse } = require('./wire') +const { generateCsharp } = require('./generator_csharp') + +// CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL — advertised so protoc +// does not reject the plugin when the schema uses proto3 `optional` fields. +const FEATURE_PROTO3_OPTIONAL = 1 + +function readStdin() { + return new Promise((resolve, reject) => { + const chunks = [] + process.stdin.on('data', (chunk) => chunks.push(chunk)) + process.stdin.on('end', () => resolve(Buffer.concat(chunks))) + process.stdin.on('error', reject) + }) +} + +async function main() { + const requestBytes = await readStdin() + const request = decodeRequest(requestBytes) + + // Lookup map for all file descriptors (parity with the Python plugin). + const fileByName = new Map() + for (const file of request.protoFile) fileByName.set(file.name, file) + + const files = [] + let error = null + + for (const fileName of request.fileToGenerate) { + // Skip the options definition file itself — it has no messages to generate. + if (fileName === 'decentraland/common/options.proto') continue + + const fileProto = fileByName.get(fileName) + if (!fileProto) continue + + try { + const generated = generateCsharp(fileProto) + if (generated) files.push(generated) + } catch (exc) { + error = `protoc-gen-bitwise: error processing ${fileName}: ${exc && exc.message ? exc.message : exc}` + } + } + + const response = encodeResponse({ + error, + supportedFeatures: FEATURE_PROTO3_OPTIONAL, + files, + }) + + // Let the write flush before the process exits (do not call process.exit()). + process.stdout.write(response) +} + +main().catch((exc) => { + const response = encodeResponse({ + error: `protoc-gen-bitwise: ${exc && exc.stack ? exc.stack : exc}`, + supportedFeatures: FEATURE_PROTO3_OPTIONAL, + files: [], + }) + process.stdout.write(response) +}) diff --git a/protoc-gen-bitwise/plugin.py b/protoc-gen-bitwise/plugin.py deleted file mode 100644 index 09b5abe4..00000000 --- a/protoc-gen-bitwise/plugin.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python3 -""" -protoc-gen-bitwise — protoc plugin that generates C# bitwise serialization code. - -Protocol: - 1. protoc writes a serialised CodeGeneratorRequest to this process's stdin. - 2. This plugin reads it, generates C# partial classes with Encode / Decode - methods for every message that carries [(quantized)] or [(bit_packed)] - field annotations. - 3. A serialised CodeGeneratorResponse is written to stdout. - -Usage (from project root): - protoc \\ - --proto_path=proto \\ - --bitwise_out=generated/ \\ - --plugin=protoc-gen-bitwise \\ - proto/my_messages.proto - -Windows invocation (plugin not on PATH as executable): - protoc \\ - --proto_path=proto \\ - --bitwise_out=generated/ \\ - --plugin=protoc-gen-bitwise=python protoc-gen-bitwise/plugin.py \\ - proto/my_messages.proto - -Dependencies: - pip install grpcio-tools # or: pip install protobuf -""" - -import os -import sys - -# Ensure sibling modules (generator_csharp, options_pb2) are importable -# regardless of where protoc invokes this script from. -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -# On Windows, stdin/stdout are opened in text mode by default which corrupts -# the binary protobuf payload. -if sys.platform == 'win32': - import msvcrt - msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) - msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) - -from google.protobuf.compiler import plugin_pb2 - -from generator_csharp import generate_csharp - - -def main() -> None: - request_bytes = sys.stdin.buffer.read() - - request = plugin_pb2.CodeGeneratorRequest() - request.ParseFromString(request_bytes) - - response = plugin_pb2.CodeGeneratorResponse() - # Advertise proto3-optional support so protoc does not reject the plugin. - response.supported_features = ( - plugin_pb2.CodeGeneratorResponse.FEATURE_PROTO3_OPTIONAL - ) - - # Build a lookup map for all file descriptors (needed for imports, though - # the current generator only uses the directly requested files). - file_by_name = {f.name: f for f in request.proto_file} - - for file_name in request.file_to_generate: - # Skip the options definition file itself — it has no messages to generate. - if file_name == 'decentraland/common/options.proto': - continue - - file_proto = file_by_name.get(file_name) - if file_proto is None: - continue - - try: - generated = generate_csharp(file_proto) - except Exception as exc: # noqa: BLE001 - error = response.file.add() - error.name = '' # empty name signals an error to protoc - # Append error text; protoc will print it and fail. - response.error = f'protoc-gen-bitwise: error processing {file_name}: {exc}' - continue - - if generated is not None: - out = response.file.add() - out.name = generated['name'] - out.content = generated['content'] - - sys.stdout.buffer.write(response.SerializeToString()) - - -if __name__ == '__main__': - main() diff --git a/protoc-gen-bitwise/test/generator.test.js b/protoc-gen-bitwise/test/generator.test.js new file mode 100644 index 00000000..95512154 --- /dev/null +++ b/protoc-gen-bitwise/test/generator.test.js @@ -0,0 +1,180 @@ +'use strict' + +/** + * Byte-for-byte parity test for the C# generator. + * + * Builds in-memory FieldDescriptorProto structures (with real serialized + * FieldOptions extension bytes), runs them through the same options parser and + * generator the plugin uses, and asserts the output equals committed golden + * files. This locks down the %g float formatting and the emitted layout without + * needing protoc. + * + * Run with: node protoc-gen-bitwise/test/generator.test.js (or `npm run gen:test`) + */ + +const assert = require('assert') +const fs = require('fs') +const path = require('path') + +const { generateCsharp } = require('../generator_csharp') + +// FieldDescriptorProto.Type +const TYPE_DOUBLE = 1 +const TYPE_BOOL = 8 +const TYPE_UINT32 = 13 +// FieldDescriptorProto.Label +const LABEL_OPTIONAL = 1 + +// --- minimal wire encoders to build serialized FieldOptions extension bytes --- + +function encodeVarint(value) { + const bytes = [] + let v = value + while (v > 0x7f) { + bytes.push((v & 0x7f) | 0x80) + v = Math.floor(v / 128) + } + bytes.push(v & 0x7f) + return Buffer.from(bytes) +} + +function tag(fieldNum, wireType) { + return encodeVarint((fieldNum << 3) | wireType) +} + +function floatLE(value) { + const b = Buffer.alloc(4) + b.writeFloatLE(value, 0) + return b +} + +function lengthDelimited(fieldNum, payload) { + return Buffer.concat([tag(fieldNum, 2), encodeVarint(payload.length), payload]) +} + +// FieldOptions bytes carrying ext 50001 (quantized) = { min, max, bits } +function quantizedOptions(min, max, bits) { + const inner = Buffer.concat([ + tag(1, 5), floatLE(min), + tag(2, 5), floatLE(max), + tag(3, 0), encodeVarint(bits), + ]) + return lengthDelimited(50001, inner) +} + +// FieldOptions bytes carrying ext 50002 (bit_packed) = { bits } +function bitPackedOptions(bits) { + const inner = Buffer.concat([tag(1, 0), encodeVarint(bits)]) + return lengthDelimited(50002, inner) +} + +function field(name, type, optionsRaw) { + return { name, label: LABEL_OPTIONAL, type, optionsRaw: optionsRaw || null } +} + +function readGolden(name) { + return fs.readFileSync(path.join(__dirname, 'golden', name), 'utf8') +} + +// -------------------------------------------------------------------------- +// Case 1: quantization_example.proto +// -------------------------------------------------------------------------- + +const quantizationExample = { + name: 'decentraland/common/quantization_example.proto', + package: 'decentraland.common', + messageType: [ + { + name: 'PositionDelta', + field: [ + field('dx', TYPE_UINT32, quantizedOptions(-100, 100, 16)), + field('dy', TYPE_UINT32, quantizedOptions(-100, 100, 16)), + field('dz', TYPE_UINT32, quantizedOptions(-100, 100, 16)), + field('entity_id', TYPE_UINT32, bitPackedOptions(20)), + field('sequence', TYPE_UINT32, bitPackedOptions(12)), + ], + }, + { + name: 'PlayerInput', + field: [ + field('move_x', TYPE_UINT32, quantizedOptions(-1, 1, 8)), + field('move_z', TYPE_UINT32, quantizedOptions(-1, 1, 8)), + field('yaw', TYPE_UINT32, quantizedOptions(-180, 180, 12)), + field('buttons', TYPE_UINT32, bitPackedOptions(8)), + field('sequence', TYPE_UINT32, bitPackedOptions(12)), + ], + }, + { + name: 'AvatarStateSnapshot', + field: [ + field('x', TYPE_UINT32, quantizedOptions(-4096, 4096, 16)), + field('y', TYPE_UINT32, quantizedOptions(-256, 256, 14)), + field('z', TYPE_UINT32, quantizedOptions(-4096, 4096, 16)), + field('pitch', TYPE_UINT32, quantizedOptions(-90, 90, 10)), + field('yaw', TYPE_UINT32, quantizedOptions(-180, 180, 12)), + field('entity_id', TYPE_UINT32, bitPackedOptions(20)), + field('animation_state', TYPE_UINT32, bitPackedOptions(6)), + field('is_grounded', TYPE_BOOL, null), + field('timestamp', TYPE_DOUBLE, null), + ], + }, + ], +} + +// -------------------------------------------------------------------------- +// Case 2: pulse_server.proto (PlayerStateDeltaTier0) — wider bit/range coverage +// -------------------------------------------------------------------------- + +const pulseServer = { + name: 'decentraland/pulse/pulse_server.proto', + package: 'decentraland.pulse', + messageType: [ + { + name: 'PlayerStateDeltaTier0', + field: [ + field('position_x', TYPE_UINT32, quantizedOptions(0, 16, 8)), + field('position_y', TYPE_UINT32, quantizedOptions(0, 200, 13)), + field('position_z', TYPE_UINT32, quantizedOptions(0, 16, 8)), + field('velocity_x', TYPE_UINT32, quantizedOptions(-50, 50, 8)), + field('velocity_y', TYPE_UINT32, quantizedOptions(-50, 50, 8)), + field('velocity_z', TYPE_UINT32, quantizedOptions(-50, 50, 8)), + field('rotation_y', TYPE_UINT32, quantizedOptions(0, 360, 7)), + field('movement_blend', TYPE_UINT32, quantizedOptions(0, 3, 5)), + field('slide_blend', TYPE_UINT32, quantizedOptions(0, 1, 4)), + field('head_yaw', TYPE_UINT32, quantizedOptions(0, 360, 7)), + field('head_pitch', TYPE_UINT32, quantizedOptions(0, 360, 7)), + field('point_at_x', TYPE_UINT32, quantizedOptions(-3000, 3000, 17)), + field('point_at_y', TYPE_UINT32, quantizedOptions(0, 200, 7)), + field('point_at_z', TYPE_UINT32, quantizedOptions(-3000, 3000, 17)), + ], + }, + ], +} + +// -------------------------------------------------------------------------- + +const cases = [ + { proto: quantizationExample, golden: 'QuantizationExample.Bitwise.cs' }, + { proto: pulseServer, golden: 'PulseServer.Bitwise.cs' }, +] + +let failed = 0 +for (const { proto, golden } of cases) { + const result = generateCsharp(proto) + assert.ok(result, `expected output for ${golden}`) + assert.strictEqual(result.name, golden, `output filename for ${golden}`) + try { + assert.strictEqual(result.content, readGolden(golden)) + console.log(`ok - ${golden} matches golden`) + } catch (e) { + failed++ + console.error(`FAIL - ${golden} differs from golden`) + console.error(e.message) + } +} + +if (failed > 0) { + console.error(`\n${failed} golden mismatch(es)`) + process.exit(1) +} +console.log('\nAll bitwise generator golden tests passed.') diff --git a/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs b/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs new file mode 100644 index 00000000..255ba7bf --- /dev/null +++ b/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs @@ -0,0 +1,145 @@ +// +// Generated by protoc-gen-bitwise. DO NOT EDIT. +// Source: decentraland/pulse/pulse_server.proto +// + +using Decentraland.Networking.Bitwise; + +namespace Decentraland.Pulse +{ + public partial class PlayerStateDeltaTier0 + { + private float? _positionX; + /// Float accessor for . Range [0.0f, 16.0f], 8 bits, step ≈ 0.0627451. + public float PositionXQuantized + { + get => _positionX ??= Quantize.Decode(PositionX, 0.0f, 16.0f, 8); + set { _positionX = value; PositionX = Quantize.Encode(value, 0.0f, 16.0f, 8); } + } + + private float? _positionY; + /// Float accessor for . Range [0.0f, 200.0f], 13 bits, step ≈ 0.024417. + public float PositionYQuantized + { + get => _positionY ??= Quantize.Decode(PositionY, 0.0f, 200.0f, 13); + set { _positionY = value; PositionY = Quantize.Encode(value, 0.0f, 200.0f, 13); } + } + + private float? _positionZ; + /// Float accessor for . Range [0.0f, 16.0f], 8 bits, step ≈ 0.0627451. + public float PositionZQuantized + { + get => _positionZ ??= Quantize.Decode(PositionZ, 0.0f, 16.0f, 8); + set { _positionZ = value; PositionZ = Quantize.Encode(value, 0.0f, 16.0f, 8); } + } + + private float? _velocityX; + /// Float accessor for . Range [-50.0f, 50.0f], 8 bits, step ≈ 0.392157. + public float VelocityXQuantized + { + get => _velocityX ??= Quantize.Decode(VelocityX, -50.0f, 50.0f, 8); + set { _velocityX = value; VelocityX = Quantize.Encode(value, -50.0f, 50.0f, 8); } + } + + private float? _velocityY; + /// Float accessor for . Range [-50.0f, 50.0f], 8 bits, step ≈ 0.392157. + public float VelocityYQuantized + { + get => _velocityY ??= Quantize.Decode(VelocityY, -50.0f, 50.0f, 8); + set { _velocityY = value; VelocityY = Quantize.Encode(value, -50.0f, 50.0f, 8); } + } + + private float? _velocityZ; + /// Float accessor for . Range [-50.0f, 50.0f], 8 bits, step ≈ 0.392157. + public float VelocityZQuantized + { + get => _velocityZ ??= Quantize.Decode(VelocityZ, -50.0f, 50.0f, 8); + set { _velocityZ = value; VelocityZ = Quantize.Encode(value, -50.0f, 50.0f, 8); } + } + + private float? _rotationY; + /// Float accessor for . Range [0.0f, 360.0f], 7 bits, step ≈ 2.83465. + public float RotationYQuantized + { + get => _rotationY ??= Quantize.Decode(RotationY, 0.0f, 360.0f, 7); + set { _rotationY = value; RotationY = Quantize.Encode(value, 0.0f, 360.0f, 7); } + } + + private float? _movementBlend; + /// Float accessor for . Range [0.0f, 3.0f], 5 bits, step ≈ 0.0967742. + public float MovementBlendQuantized + { + get => _movementBlend ??= Quantize.Decode(MovementBlend, 0.0f, 3.0f, 5); + set { _movementBlend = value; MovementBlend = Quantize.Encode(value, 0.0f, 3.0f, 5); } + } + + private float? _slideBlend; + /// Float accessor for . Range [0.0f, 1.0f], 4 bits, step ≈ 0.0666667. + public float SlideBlendQuantized + { + get => _slideBlend ??= Quantize.Decode(SlideBlend, 0.0f, 1.0f, 4); + set { _slideBlend = value; SlideBlend = Quantize.Encode(value, 0.0f, 1.0f, 4); } + } + + private float? _headYaw; + /// Float accessor for . Range [0.0f, 360.0f], 7 bits, step ≈ 2.83465. + public float HeadYawQuantized + { + get => _headYaw ??= Quantize.Decode(HeadYaw, 0.0f, 360.0f, 7); + set { _headYaw = value; HeadYaw = Quantize.Encode(value, 0.0f, 360.0f, 7); } + } + + private float? _headPitch; + /// Float accessor for . Range [0.0f, 360.0f], 7 bits, step ≈ 2.83465. + public float HeadPitchQuantized + { + get => _headPitch ??= Quantize.Decode(HeadPitch, 0.0f, 360.0f, 7); + set { _headPitch = value; HeadPitch = Quantize.Encode(value, 0.0f, 360.0f, 7); } + } + + private float? _pointAtX; + /// Float accessor for . Range [-3000.0f, 3000.0f], 17 bits, step ≈ 0.0457767. + public float PointAtXQuantized + { + get => _pointAtX ??= Quantize.Decode(PointAtX, -3000.0f, 3000.0f, 17); + set { _pointAtX = value; PointAtX = Quantize.Encode(value, -3000.0f, 3000.0f, 17); } + } + + private float? _pointAtY; + /// Float accessor for . Range [0.0f, 200.0f], 7 bits, step ≈ 1.5748. + public float PointAtYQuantized + { + get => _pointAtY ??= Quantize.Decode(PointAtY, 0.0f, 200.0f, 7); + set { _pointAtY = value; PointAtY = Quantize.Encode(value, 0.0f, 200.0f, 7); } + } + + private float? _pointAtZ; + /// Float accessor for . Range [-3000.0f, 3000.0f], 17 bits, step ≈ 0.0457767. + public float PointAtZQuantized + { + get => _pointAtZ ??= Quantize.Decode(PointAtZ, -3000.0f, 3000.0f, 17); + set { _pointAtZ = value; PointAtZ = Quantize.Encode(value, -3000.0f, 3000.0f, 17); } + } + + /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. + public void ResetDecodedCache() + { + _positionX = null; + _positionY = null; + _positionZ = null; + _velocityX = null; + _velocityY = null; + _velocityZ = null; + _rotationY = null; + _movementBlend = null; + _slideBlend = null; + _headYaw = null; + _headPitch = null; + _pointAtX = null; + _pointAtY = null; + _pointAtZ = null; + } + } + + +} // namespace Decentraland.Pulse diff --git a/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs b/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs new file mode 100644 index 00000000..c0456936 --- /dev/null +++ b/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs @@ -0,0 +1,134 @@ +// +// Generated by protoc-gen-bitwise. DO NOT EDIT. +// Source: decentraland/common/quantization_example.proto +// + +using Decentraland.Networking.Bitwise; + +namespace Decentraland.Common +{ + public partial class PositionDelta + { + private float? _dx; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. + public float DxQuantized + { + get => _dx ??= Quantize.Decode(Dx, -100.0f, 100.0f, 16); + set { _dx = value; Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + private float? _dy; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. + public float DyQuantized + { + get => _dy ??= Quantize.Decode(Dy, -100.0f, 100.0f, 16); + set { _dy = value; Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + private float? _dz; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. + public float DzQuantized + { + get => _dz ??= Quantize.Decode(Dz, -100.0f, 100.0f, 16); + set { _dz = value; Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. + public void ResetDecodedCache() + { + _dx = null; + _dy = null; + _dz = null; + } + } + + public partial class PlayerInput + { + private float? _moveX; + /// Float accessor for . Range [-1.0f, 1.0f], 8 bits, step ≈ 0.00784314. + public float MoveXQuantized + { + get => _moveX ??= Quantize.Decode(MoveX, -1.0f, 1.0f, 8); + set { _moveX = value; MoveX = Quantize.Encode(value, -1.0f, 1.0f, 8); } + } + + private float? _moveZ; + /// Float accessor for . Range [-1.0f, 1.0f], 8 bits, step ≈ 0.00784314. + public float MoveZQuantized + { + get => _moveZ ??= Quantize.Decode(MoveZ, -1.0f, 1.0f, 8); + set { _moveZ = value; MoveZ = Quantize.Encode(value, -1.0f, 1.0f, 8); } + } + + private float? _yaw; + /// Float accessor for . Range [-180.0f, 180.0f], 12 bits, step ≈ 0.0879121. + public float YawQuantized + { + get => _yaw ??= Quantize.Decode(Yaw, -180.0f, 180.0f, 12); + set { _yaw = value; Yaw = Quantize.Encode(value, -180.0f, 180.0f, 12); } + } + + /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. + public void ResetDecodedCache() + { + _moveX = null; + _moveZ = null; + _yaw = null; + } + } + + public partial class AvatarStateSnapshot + { + private float? _x; + /// Float accessor for . Range [-4096.0f, 4096.0f], 16 bits, step ≈ 0.125002. + public float XQuantized + { + get => _x ??= Quantize.Decode(X, -4096.0f, 4096.0f, 16); + set { _x = value; X = Quantize.Encode(value, -4096.0f, 4096.0f, 16); } + } + + private float? _y; + /// Float accessor for . Range [-256.0f, 256.0f], 14 bits, step ≈ 0.0312519. + public float YQuantized + { + get => _y ??= Quantize.Decode(Y, -256.0f, 256.0f, 14); + set { _y = value; Y = Quantize.Encode(value, -256.0f, 256.0f, 14); } + } + + private float? _z; + /// Float accessor for . Range [-4096.0f, 4096.0f], 16 bits, step ≈ 0.125002. + public float ZQuantized + { + get => _z ??= Quantize.Decode(Z, -4096.0f, 4096.0f, 16); + set { _z = value; Z = Quantize.Encode(value, -4096.0f, 4096.0f, 16); } + } + + private float? _pitch; + /// Float accessor for . Range [-90.0f, 90.0f], 10 bits, step ≈ 0.175953. + public float PitchQuantized + { + get => _pitch ??= Quantize.Decode(Pitch, -90.0f, 90.0f, 10); + set { _pitch = value; Pitch = Quantize.Encode(value, -90.0f, 90.0f, 10); } + } + + private float? _yaw; + /// Float accessor for . Range [-180.0f, 180.0f], 12 bits, step ≈ 0.0879121. + public float YawQuantized + { + get => _yaw ??= Quantize.Decode(Yaw, -180.0f, 180.0f, 12); + set { _yaw = value; Yaw = Quantize.Encode(value, -180.0f, 180.0f, 12); } + } + + /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. + public void ResetDecodedCache() + { + _x = null; + _y = null; + _z = null; + _pitch = null; + _yaw = null; + } + } + + +} // namespace Decentraland.Common diff --git a/protoc-gen-bitwise/wire.js b/protoc-gen-bitwise/wire.js new file mode 100644 index 00000000..e2bd2d57 --- /dev/null +++ b/protoc-gen-bitwise/wire.js @@ -0,0 +1,239 @@ +'use strict' + +/** + * Self-contained protobuf wire-format codec for protoc-gen-bitwise. + * + * The plugin only needs a tiny slice of the descriptor/plugin schemas, so + * rather than pull in a protobuf runtime (which would force every consumer — + * including the sibling Pulse checkout that runs this file directly off disk — + * to `npm install` the protocol repo) we decode the handful of fields we care + * about by walking the wire format directly. This mirrors the original Python + * plugin, which already hand-parsed FieldOptions for the same reason. + * + * Decoded subset: + * CodeGeneratorRequest { file_to_generate = 1; proto_file = 15; } + * FileDescriptorProto { name = 1; package = 2; message_type = 4; } + * DescriptorProto { name = 1; field = 2; } + * FieldDescriptorProto { name = 1; label = 4; type = 5; options = 8 (raw bytes); } + * + * Encoded subset: + * CodeGeneratorResponse { error = 1; supported_features = 2; file = 15; } + * CodeGeneratorResponse.File { name = 1; content = 15; } + * + * Wire types: 0 = varint, 1 = 64-bit, 2 = length-delimited, 5 = 32-bit. + */ + +// --------------------------------------------------------------------------- +// Low-level readers +// --------------------------------------------------------------------------- + +/** Decode a protobuf varint at `pos`. Returns [value, newPos]. */ +function readVarint(buf, pos) { + let result = 0 + let shift = 0 + let byte + do { + byte = buf[pos++] + // Multiplication (not <<) keeps values correct past 32 bits. + result += (byte & 0x7f) * Math.pow(2, shift) + shift += 7 + } while (byte & 0x80) + return [result, pos] +} + +/** Advance `pos` past a field of the given wire type. Returns newPos. */ +function skipField(buf, pos, wireType) { + switch (wireType) { + case 0: // varint + return readVarint(buf, pos)[1] + case 1: // 64-bit + return pos + 8 + case 2: { + // length-delimited + const [length, next] = readVarint(buf, pos) + return next + length + } + case 5: // 32-bit + return pos + 4 + default: + // Groups (3/4) are deprecated and never appear in descriptors. + return pos + } +} + +// --------------------------------------------------------------------------- +// Request decoding +// --------------------------------------------------------------------------- + +function decodeFieldDescriptor(buf) { + const field = { name: '', label: 0, type: 0, optionsRaw: null } + let pos = 0 + while (pos < buf.length) { + let tag + ;[tag, pos] = readVarint(buf, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + field.name = buf.toString('utf8', pos, pos + len) + pos += len + } else if (fieldNum === 4 && wireType === 0) { + ;[field.label, pos] = readVarint(buf, pos) + } else if (fieldNum === 5 && wireType === 0) { + ;[field.type, pos] = readVarint(buf, pos) + } else if (fieldNum === 8 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + // Keep the raw FieldOptions bytes so custom extensions survive (the + // descriptor schema doesn't know about ext 50001/50002). + field.optionsRaw = buf.subarray(pos, pos + len) + pos += len + } else { + pos = skipField(buf, pos, wireType) + } + } + return field +} + +function decodeDescriptor(buf) { + const message = { name: '', field: [] } + let pos = 0 + while (pos < buf.length) { + let tag + ;[tag, pos] = readVarint(buf, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + message.name = buf.toString('utf8', pos, pos + len) + pos += len + } else if (fieldNum === 2 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + message.field.push(decodeFieldDescriptor(buf.subarray(pos, pos + len))) + pos += len + } else { + // nested_type / enum_type / etc. are intentionally ignored — the + // generator only walks top-level messages, matching the Python plugin. + pos = skipField(buf, pos, wireType) + } + } + return message +} + +function decodeFileDescriptor(buf) { + const file = { name: '', package: '', messageType: [] } + let pos = 0 + while (pos < buf.length) { + let tag + ;[tag, pos] = readVarint(buf, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + file.name = buf.toString('utf8', pos, pos + len) + pos += len + } else if (fieldNum === 2 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + file.package = buf.toString('utf8', pos, pos + len) + pos += len + } else if (fieldNum === 4 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + file.messageType.push(decodeDescriptor(buf.subarray(pos, pos + len))) + pos += len + } else { + pos = skipField(buf, pos, wireType) + } + } + return file +} + +/** Decode a serialized CodeGeneratorRequest. */ +function decodeRequest(buf) { + const request = { fileToGenerate: [], protoFile: [] } + let pos = 0 + while (pos < buf.length) { + let tag + ;[tag, pos] = readVarint(buf, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + request.fileToGenerate.push(buf.toString('utf8', pos, pos + len)) + pos += len + } else if (fieldNum === 15 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + request.protoFile.push(decodeFileDescriptor(buf.subarray(pos, pos + len))) + pos += len + } else { + pos = skipField(buf, pos, wireType) + } + } + return request +} + +// --------------------------------------------------------------------------- +// Response encoding +// --------------------------------------------------------------------------- + +/** Encode an unsigned integer as a protobuf varint Buffer. */ +function encodeVarint(value) { + const bytes = [] + let v = value + while (v > 0x7f) { + bytes.push((v & 0x7f) | 0x80) + v = Math.floor(v / 128) + } + bytes.push(v & 0x7f) + return Buffer.from(bytes) +} + +function encodeTag(fieldNum, wireType) { + return encodeVarint((fieldNum << 3) | wireType) +} + +/** Encode a length-delimited (string/bytes) field: tag + length + payload. */ +function encodeLengthDelimited(fieldNum, payload) { + const buf = Buffer.isBuffer(payload) ? payload : Buffer.from(payload, 'utf8') + return Buffer.concat([encodeTag(fieldNum, 2), encodeVarint(buf.length), buf]) +} + +function encodeFile(file) { + return Buffer.concat([ + encodeLengthDelimited(1, file.name), // name = 1 + encodeLengthDelimited(15, file.content), // content = 15 + ]) +} + +/** + * Encode a CodeGeneratorResponse. + * @param {{error?: string|null, supportedFeatures?: number, files?: Array<{name:string,content:string}>}} response + */ +function encodeResponse(response) { + const parts = [] + if (response.error != null) { + parts.push(encodeLengthDelimited(1, response.error)) // error = 1 + } + if (response.supportedFeatures != null) { + parts.push(encodeTag(2, 0)) // supported_features = 2 (varint) + parts.push(encodeVarint(response.supportedFeatures)) + } + for (const file of response.files || []) { + parts.push(encodeLengthDelimited(15, encodeFile(file))) // file = 15 + } + return Buffer.concat(parts) +} + +module.exports = { + readVarint, + skipField, + decodeRequest, + encodeResponse, +} From c2f6832f0d96aa567c97c019fc4959aa23f7c0e1 Mon Sep 17 00:00:00 2001 From: Gon Pombo Date: Thu, 25 Jun 2026 15:17:31 +0200 Subject: [PATCH 45/54] feat: add AMT_HIDE_NAMETAGS modifier (#411) Co-authored-by: gon boedo --- proto/decentraland/sdk/components/avatar_modifier_area.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/proto/decentraland/sdk/components/avatar_modifier_area.proto b/proto/decentraland/sdk/components/avatar_modifier_area.proto index aea88858..c8936d94 100644 --- a/proto/decentraland/sdk/components/avatar_modifier_area.proto +++ b/proto/decentraland/sdk/components/avatar_modifier_area.proto @@ -29,4 +29,5 @@ message PBAvatarModifierArea { enum AvatarModifierType { AMT_HIDE_AVATARS = 0; // avatars are invisible AMT_DISABLE_PASSPORTS = 1; // selecting (e.g. clicking) an avatar will not bring up their profile. + AMT_HIDE_NAMETAGS = 2; // the name tag displayed above an avatar is hidden. } From c09a897de91cce7f14fc7e4c39045d274f547c2f Mon Sep 17 00:00:00 2001 From: Mikhail Agapov <118179774+mikhail-dcl@users.noreply.github.com> Date: Thu, 11 Jun 2026 18:39:02 +0300 Subject: [PATCH 46/54] Feat: Pulse with quantization (#360) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add Quantization as a protobuf plugin * pulse_comms.proto Signed-off-by: Mikhail Agapov * add server message & handshake response * add player state full/delta * add player joined message * add player joined into server message * add player state into client message * Update protobufjs to 7.5.4 to resolve the problem with namespaces * proto/google/ is no longer created or published, so both buf (CI) and protoc (consumers like unity-explorer) resolve descriptor.proto from their own built-in includes — which all have the correct csharp_namespace Signed-off-by: Mikhail Agapov * fix: exclude proto/google from npm package to fix C# codegen namespace Revert protobufjs to 7.2.4 (7.5.4 breaks buf and proto-compatibility-tool with newer reserved syntax) and restore the make install copy step for local tooling. Change "files" from "proto" to "proto/decentraland" so the stripped google/protobuf/descriptor.proto (missing csharp_namespace option) is no longer published. Consumers' protoc resolves descriptor.proto from its own built-in includes which have the correct option csharp_namespace = "Google.Protobuf.Reflection". Co-Authored-By: Claude Opus 4.6 * Adjust states encoding * Add quantization example for PlayerStateDelta Signed-off-by: Mikhail Agapov * Copy protoc-gen-bitwise to the output package * Make Quantize.cs compitable with Unity Signed-off-by: Mikhail Agapov * Isolate PlayerState as a reusable component Signed-off-by: Mikhail Agapov * Add requried quntizationfor delta Signed-off-by: Mikhail Agapov * Add RESYNC_REQUEST Signed-off-by: Mikhail Agapov * ProfileVersionAnnouncement - parcel_index change uint32 -> int32 Signed-off-by: Mikhail Agapov * add emote messages * add duration on EmoteStart for one shot emotes * Fix MovementBlend Range Signed-off-by: Mikhail Agapov * add teleport messages * Add "jump_count" Signed-off-by: Mikhail Agapov * add teleport ClientMessage & ServerMessage * Add BaselineSeq to delta Signed-off-by: Mikhail Agapov * add parcel index & player state in teleport * add head yaw and head pitch to state flags * Add `sequence` to `EmoteStarted` Signed-off-by: Mikhail Agapov * add player state into EmoteStart * Add seq and PlayerState to `EmoteStopped` Signed-off-by: Mikhail Agapov * Add "realm" field to "TeleportRequest" Signed-off-by: Mikhail Agapov * Split pulse_comms into separate files Signed-off-by: Mikhail Agapov * Add Initial State to HandshakeRequest - It's needed to properly authorize reconnection in the middle of the session Signed-off-by: Mikhail Agapov * Add support for "pointing at" Signed-off-by: Mikhail Agapov * Change "Point At" to the absolute value Signed-off-by: Mikhail Agapov * Add "realm" to the initial state Signed-off-by: Mikhail Agapov * Change head_pitch max from 180 to 360 Signed-off-by: Mikhail Agapov * emote mask --------- Signed-off-by: Mikhail Agapov Co-authored-by: Nicolas Lorusso Co-authored-by: Claude Opus 4.6 --- README.md | 198 ++++++++++++++++++ package.json | 5 +- proto/decentraland/common/options.proto | 30 +++ .../common/quantization_example.proto | 132 ++++++++++++ proto/decentraland/pulse/pulse_client.proto | 77 +++++++ proto/decentraland/pulse/pulse_server.proto | 138 ++++++++++++ proto/decentraland/pulse/pulse_shared.proto | 48 +++++ protoc-gen-bitwise/generator_csharp.py | 176 ++++++++++++++++ protoc-gen-bitwise/options_pb2.py | 171 +++++++++++++++ protoc-gen-bitwise/plugin.py | 92 ++++++++ protoc-gen-bitwise/runtime/cs/BitReader.cs | 112 ++++++++++ protoc-gen-bitwise/runtime/cs/BitWriter.cs | 117 +++++++++++ protoc-gen-bitwise/runtime/cs/Quantize.cs | 35 ++++ 13 files changed, 1329 insertions(+), 2 deletions(-) create mode 100644 proto/decentraland/common/options.proto create mode 100644 proto/decentraland/common/quantization_example.proto create mode 100644 proto/decentraland/pulse/pulse_client.proto create mode 100644 proto/decentraland/pulse/pulse_server.proto create mode 100644 proto/decentraland/pulse/pulse_shared.proto create mode 100644 protoc-gen-bitwise/generator_csharp.py create mode 100644 protoc-gen-bitwise/options_pb2.py create mode 100644 protoc-gen-bitwise/plugin.py create mode 100644 protoc-gen-bitwise/runtime/cs/BitReader.cs create mode 100644 protoc-gen-bitwise/runtime/cs/BitWriter.cs create mode 100644 protoc-gen-bitwise/runtime/cs/Quantize.cs diff --git a/README.md b/README.md index 598bdb0b..20c75605 100644 --- a/README.md +++ b/README.md @@ -67,3 +67,201 @@ In this case, there is no problem with when each PR is merged. It's recommendabl ## Comms TODO + +--- + +# Bitwise Serialization Plugin (`protoc-gen-bitwise`) + +A custom protoc plugin that generates C# partial classes with typed float +accessors for quantized `uint32` fields in high-frequency MMO networking +messages (position deltas, player input, etc.). It runs alongside +`--csharp_out` in the same protoc invocation; the two output files coexist +via C# `partial class`. + +## How it works + +Protobuf encodes `uint32` values as varints, which are already compact for +small values: a value up to 2¹⁴−1 costs 2 bytes, up to 2²¹−1 costs 3 bytes. +Rather than a separate binary packing layer, the plugin leverages this: + +1. Declare quantized fields as `uint32` in the `.proto` schema and annotate + them with `[(decentraland.common.quantized)]` to specify the float range + and bit resolution. +2. `--csharp_out` generates the standard protobuf class with the raw `uint32` + property (e.g. `PositionX`). +3. `--bitwise_out` (this plugin) generates a `partial class` extension with a + cached float accessor (e.g. `PositionXQuantized`) that encodes/decodes + transparently via `Quantize.Encode` / `Quantize.Decode`. + +The wire representation is a standard protobuf message — any protobuf-capable +client can read it without knowledge of the plugin. + +## Prerequisites + +| Requirement | Version | +|---|---| +| Python | 3.10+ | +| `protobuf` Python package | 4.x or 3.20+ | +| `protoc` | 3.19+ | + +```bash +pip install protobuf +``` + +## Step 1 — Annotate your `.proto` file + +Declare quantized fields as `uint32` and import `options.proto`: + +```protobuf +syntax = "proto3"; + +import "decentraland/common/options.proto"; + +package decentraland.kernel.comms.v3; + +message PositionDelta { + // Float range [-100, 100] quantized to 16 bits ≈ 0.003-unit precision. + // Stored as uint32 on the wire; protobuf encodes it as a 3-byte varint. + uint32 dx = 1 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dy = 2 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dz = 3 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + + // Unannotated uint32: protobuf varint encodes small values compactly by default. + uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; +} +``` + +### Annotation reference + +| Annotation | Target type | Parameters | Effect | +|---|---|---|---| +| `[(decentraland.common.quantized)]` | `uint32` | `min`, `max`, `bits` | Plugin emits a cached `float {Name}Quantized` accessor | +| `[(decentraland.common.bit_packed)]` | `uint32` | `bits` | Documents the value range; protobuf handles varint compaction automatically | + +### Wire cost at worst-case (all bits set) + +| Quantization bits | Max value | Varint bytes | Tag (field ≤ 15) | Total per field | +|---|---|---|---|---| +| 8 | 255 | 2 | 1 | 3 B | +| 12 | 4 095 | 2 | 1 | 3 B | +| 14 | 16 383 | 2 | 1 | 3 B | +| 16 | 65 535 | 3 | 1 | 4 B | +| 20 | 1 048 575 | 3 | 1 | 4 B | + +Proto3 omits fields equal to their default value (0), so average cost is lower. + +## Step 2 — Run protoc + +```bash +protoc \ + --proto_path=proto \ + --proto_path=/path/to/google/protobuf/include \ + --csharp_out=generated/cs \ + --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.py \ + --bitwise_out=generated/cs \ + proto/decentraland/kernel/comms/v3/comms.proto +``` + +The plugin emits one `*.Bitwise.cs` file (PascalCase, flat in the output +directory) for each `.proto` file that contains at least one `[(quantized)]` +field. + +## Step 3 — Copy the runtime + +Copy `Quantize.cs` into your project: + +``` +Assets/ +└── Scripts/ + └── Networking/ + └── Bitwise/ + └── Quantize.cs ← protoc-gen-bitwise/runtime/cs/Quantize.cs +``` + +`Quantize.cs` lives in the `Decentraland.Networking.Bitwise` namespace and +provides two static methods used by the generated accessors: + +```csharp +public static class Quantize +{ + public static uint Encode(float value, float min, float max, int bits); + public static float Decode(uint encoded, float min, float max, int bits); +} +``` + +## Step 4 — Use the generated code + +The plugin emits a `partial class` that adds float accessors on top of the +standard protobuf-generated `uint32` properties: + +```csharp +using Decentraland.Kernel.Comms.V3; + +// --- Build and send --- +var delta = new PositionDelta(); +delta.DxQuantized = 3.14f; // encodes to uint32, stored in delta.Dx +delta.DyQuantized = 0f; +delta.DzQuantized = -7.5f; +delta.EntityId = 42u; + +byte[] bytes = delta.ToByteArray(); // standard protobuf serialization +SendOnChannel1(bytes); + +// --- Receive and read --- +var received = PositionDelta.Parser.ParseFrom(receivedBytes); +float x = received.DxQuantized; // decoded on first access, cached thereafter +float y = received.DyQuantized; +float z = received.DzQuantized; + +// If raw uint32 fields are mutated directly after construction, invalidate the cache: +received.ResetDecodedCache(); +``` + +## Generated file example + +For the `PositionDelta` message above the plugin emits `PositionDelta.Bitwise.cs`: + +```csharp +// +// Generated by protoc-gen-bitwise. DO NOT EDIT. +// Source: decentraland/kernel/comms/v3/comms.proto +// + +using Decentraland.Networking.Bitwise; + +namespace Decentraland.Kernel.Comms.V3 +{ + public partial class PositionDelta + { + private float? _dx; + public float DxQuantized + { + get => _dx ??= Quantize.Decode(Dx, -100.0f, 100.0f, 16); + set { _dx = value; Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + private float? _dy; + public float DyQuantized + { + get => _dy ??= Quantize.Decode(Dy, -100.0f, 100.0f, 16); + set { _dy = value; Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + private float? _dz; + public float DzQuantized + { + get => _dz ??= Quantize.Decode(Dz, -100.0f, 100.0f, 16); + set { _dz = value; Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. + public void ResetDecodedCache() + { + _dx = null; + _dy = null; + _dz = null; + } + } + +} // namespace Decentraland.Kernel.Comms.V3 +``` diff --git a/package.json b/package.json index 1e5294e5..2312a78a 100644 --- a/package.json +++ b/package.json @@ -28,9 +28,10 @@ "protobufjs": "7.2.4" }, "files": [ - "proto", + "proto/decentraland", "out-ts", "out-js", - "public" + "public", + "protoc-gen-bitwise" ] } diff --git a/proto/decentraland/common/options.proto b/proto/decentraland/common/options.proto new file mode 100644 index 00000000..8e911547 --- /dev/null +++ b/proto/decentraland/common/options.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package decentraland.common; + +import "google/protobuf/descriptor.proto"; + +// Options for quantizing a float value stored as a uint32 field. +// The float is clamped to [min, max] and uniformly quantized to N bits; +// protobuf encodes the resulting uint32 as a varint on the wire. +// The generator emits a float {Name}Quantized accessor in the partial class. +message QuantizedFloatOptions { + float min = 1; + float max = 2; + uint32 bits = 3; +} + +// Options for bit-packing an integer field into fewer than 32 bits. +message BitPackedOptions { + uint32 bits = 1; +} + +extend google.protobuf.FieldOptions { + // Apply to uint32 fields to enable quantized float encoding. + // Example: uint32 dx = 1 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + QuantizedFloatOptions quantized = 50001; + + // Apply to uint32 fields to pack into fewer bits. + // Example: uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; + BitPackedOptions bit_packed = 50002; +} diff --git a/proto/decentraland/common/quantization_example.proto b/proto/decentraland/common/quantization_example.proto new file mode 100644 index 00000000..a82e430c --- /dev/null +++ b/proto/decentraland/common/quantization_example.proto @@ -0,0 +1,132 @@ +syntax = "proto3"; + +package decentraland.common; + +import "decentraland/common/options.proto"; + +// High-frequency player state messages sent on Channel 1 (unreliable sequenced). +// +// Every message below uses the protoc-gen-bitwise annotations to minimise +// wire size. Wire costs are listed per-message so the trade-offs are clear. +// +// Annotation cheat-sheet: +// [(decentraland.common.quantized) = { min: F, max: F, bits: N }] +// → stores a float as a uint32 quantized to N bits over [min, max]. +// → protobuf encodes the uint32 as a varint: values ≤ 2^14-1 cost 2 bytes, +// values ≤ 2^21-1 cost 3 bytes; the generated partial class adds a +// float {Name}Quantized accessor that encodes/decodes transparently. +// [(decentraland.common.bit_packed) = { bits: N }] +// → documents that this uint32 uses at most N bits; protobuf encodes it +// as a varint (same savings, no generated accessor needed). +// (no annotation) +// → standard protobuf encoding (bool/double/etc. at natural width). +// +// Varint byte costs at worst-case (all bits set): +// ≤ 7 bits → 1 byte (max 127) +// ≤ 14 bits → 2 bytes (max 16 383) +// ≤ 21 bits → 3 bytes (max 2 097 151) +// Proto3 omits fields equal to their default value (0), so average cost is lower. + +// --------------------------------------------------------------------------- +// PositionDelta — Δ position relative to last acknowledged full snapshot. +// +// Sent every client tick (~10 Hz) on Channel 1. +// +// Field | Type | Range | Q bits | Wire worst-case +// ------------|--------|----------------|--------|----------------------- +// dx | uint32 | [-100, 100] | 16 | tag 1B + varint 3B = 4B +// dy | uint32 | [-100, 100] | 16 | tag 1B + varint 3B = 4B +// dz | uint32 | [-100, 100] | 16 | tag 1B + varint 3B = 4B +// entity_id | uint32 | [0, 1 048 575] | 20 | tag 1B + varint 3B = 4B +// sequence | uint32 | [0, 4 095] | 12 | tag 1B + varint 2B = 3B +// --------------------------------------------------------------------------- +// Worst-case: 19 B (vs. 20 B raw: 3×float + 2×uint32) +// Step: dx/dy/dz ≈ 0.003 units +// --------------------------------------------------------------------------- +message PositionDelta { + uint32 dx = 1 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dy = 2 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dz = 3 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; + uint32 sequence = 5 [(decentraland.common.bit_packed) = { bits: 12 }]; +} + +// --------------------------------------------------------------------------- +// PlayerInput — client input snapshot for server-side reconciliation. +// +// Sent every client frame (~30 Hz) on Channel 1. +// +// Field | Type | Range | Q bits | Wire worst-case +// ------------|--------|----------------|--------|----------------------- +// move_x | uint32 | [-1, 1] | 8 | tag 1B + varint 2B = 3B +// move_z | uint32 | [-1, 1] | 8 | tag 1B + varint 2B = 3B +// yaw | uint32 | [-180, 180] | 12 | tag 1B + varint 2B = 3B +// buttons | uint32 | bitmask | 8 | tag 1B + varint 2B = 3B +// sequence | uint32 | [0, 4 095] | 12 | tag 1B + varint 2B = 3B +// --------------------------------------------------------------------------- +// Worst-case: 15 B (vs. 20 B raw: 3×float + 2×uint32) +// Step: move_x/move_z ≈ 0.008; yaw ≈ 0.088° +// --------------------------------------------------------------------------- +message PlayerInput { + // Normalised joystick axes in [-1, 1]. + uint32 move_x = 1 [(decentraland.common.quantized) = { min: -1.0, max: 1.0, bits: 8 }]; + uint32 move_z = 2 [(decentraland.common.quantized) = { min: -1.0, max: 1.0, bits: 8 }]; + + // Horizontal look direction in degrees. + uint32 yaw = 3 [(decentraland.common.quantized) = { min: -180.0, max: 180.0, bits: 12 }]; + + // Bitmask of active buttons (see ButtonFlags below). + uint32 buttons = 4 [(decentraland.common.bit_packed) = { bits: 8 }]; + + // Rolling input sequence number used by the server for reconciliation. + uint32 sequence = 5 [(decentraland.common.bit_packed) = { bits: 12 }]; +} + +// Bitmask values for PlayerInput.buttons. +enum ButtonFlags { + BF_NONE = 0; + BF_JUMP = 1; // bit 0 + BF_SPRINT = 2; // bit 1 + BF_INTERACT = 4; // bit 2 + BF_EMOTE = 8; // bit 3 + BF_CROUCH = 16; // bit 4 + // bits 5-7 reserved +} + +// --------------------------------------------------------------------------- +// AvatarStateSnapshot — full authoritative state, sent on Channel 0 (reliable) +// or on resync requests. Demonstrates wider ranges and mixed encodings. +// +// Field | Type | Range | Q bits | Wire worst-case +// -----------------|--------|------------------|--------|----------------------- +// x | uint32 | [-4096, 4096] | 16 | tag 1B + varint 3B = 4B +// y | uint32 | [-256, 256] | 14 | tag 1B + varint 2B = 3B +// z | uint32 | [-4096, 4096] | 16 | tag 1B + varint 3B = 4B +// pitch | uint32 | [-90, 90] | 10 | tag 1B + varint 2B = 3B +// yaw | uint32 | [-180, 180] | 12 | tag 1B + varint 2B = 3B +// entity_id | uint32 | [0, 1 048 575] | 20 | tag 1B + varint 3B = 4B +// animation_state | uint32 | [0, 63] | 6 | tag 1B + varint 1B = 2B +// is_grounded | bool | — | — | tag 1B + varint 1B = 2B +// timestamp | double | — | — | tag 1B + fixed64 8B = 9B +// --------------------------------------------------------------------------- +// Worst-case: 34 B (vs. 45 B raw: 5×float + 2×uint32 + bool + double) +// Step: x/z ≈ 0.125 units; y ≈ 0.031 units; pitch ≈ 0.176°; yaw ≈ 0.088° +// --------------------------------------------------------------------------- +message AvatarStateSnapshot { + // World-space position. + uint32 x = 1 [(decentraland.common.quantized) = { min: -4096.0, max: 4096.0, bits: 16 }]; + uint32 y = 2 [(decentraland.common.quantized) = { min: -256.0, max: 256.0, bits: 14 }]; + uint32 z = 3 [(decentraland.common.quantized) = { min: -4096.0, max: 4096.0, bits: 16 }]; + + // View angles. + uint32 pitch = 4 [(decentraland.common.quantized) = { min: -90.0, max: 90.0, bits: 10 }]; + uint32 yaw = 5 [(decentraland.common.quantized) = { min: -180.0, max: 180.0, bits: 12 }]; + + // Identity and animation state. + uint32 entity_id = 6 [(decentraland.common.bit_packed) = { bits: 20 }]; + uint32 animation_state = 7 [(decentraland.common.bit_packed) = { bits: 6 }]; + + // Un-annotated fields — encoded at their natural width. + bool is_grounded = 8; + double timestamp = 9; // server epoch milliseconds +} diff --git a/proto/decentraland/pulse/pulse_client.proto b/proto/decentraland/pulse/pulse_client.proto new file mode 100644 index 00000000..6036ec53 --- /dev/null +++ b/proto/decentraland/pulse/pulse_client.proto @@ -0,0 +1,77 @@ +syntax = "proto3"; + +package decentraland.pulse; + +import "decentraland/common/vectors.proto"; +import "decentraland/pulse/pulse_shared.proto"; + +message HandshakeRequest { + bytes auth_chain = 1; + int32 profile_version = 2; + optional PlayerInitialState initial_state = 3; +} + +// Describes the initial state of the player if (re-)connected in the middle of the session +message PlayerInitialState { + PlayerState state = 1; + optional string emote_id = 2; + optional uint32 emote_duration_ms = 3; + // Indicates how many milliseconds ago the emote was started + optional uint32 emote_start_offset_ms = 4; + // Non-empty realm identifier. Rejected if empty. + string realm = 5; + optional int32 emote_mask = 6; +} + +// Similarly to the LiveKit pipeline, a peer announces the version of its profile but +// it only does it when it's changed as it's sent reliably and stored on the server for other peers +message ProfileVersionAnnouncement { + int32 version = 1; +} + +// Since the server doesn't simulate the scenes state, it trusts the values from the client +message PlayerStateInput { + PlayerState state = 1; +} + +// Client sends resync request if it has a gap in the known sequences and, thus, can't apply a delta. +// It's sent reliably to prevent further desynchronization +message ResyncRequest { + uint32 subject_id = 1; + // highest seq the client actually has for this subject + uint32 known_seq = 2; +} + +// Client → Server. Client requests to start an emote. +message EmoteStart { + string emote_id = 1; + optional uint32 duration_ms = 2; + PlayerState player_state = 3; + optional int32 mask = 4; +} + +// Client → Server. Client requests to stop a looping emote. +// One-shot emotes are terminated by the server timer. +message EmoteStop { +} + +// Client → Server. Also announces the peer's realm; peers in different realms never see each +// other. Must be the first gameplay message after handshake. Same-realm re-teleports are valid. +message TeleportRequest { + int32 parcel_index = 1; + decentraland.common.Vector3 position = 2; + // Non-empty realm identifier. Rejected if empty. + string realm = 3; +} + +message ClientMessage { + oneof message { + HandshakeRequest handshake = 1; + PlayerStateInput input = 2; + ResyncRequest resync = 3; + ProfileVersionAnnouncement profile_announcement = 4; + EmoteStart emote_start = 5; + EmoteStop emote_stop = 6; + TeleportRequest teleport = 7; + } +} diff --git a/proto/decentraland/pulse/pulse_server.proto b/proto/decentraland/pulse/pulse_server.proto new file mode 100644 index 00000000..8311e737 --- /dev/null +++ b/proto/decentraland/pulse/pulse_server.proto @@ -0,0 +1,138 @@ +syntax = "proto3"; + +package decentraland.pulse; + +import "decentraland/common/options.proto"; +import "decentraland/pulse/pulse_shared.proto"; + +message HandshakeResponse { + bool success = 1; + optional string error = 2; +} + +message PlayerProfileVersionsAnnounced { + uint32 subject_id = 1; + int32 version = 2; +} + +message PlayerStateDeltaTier0 { + uint32 subject_id = 1; + + // Between two consecutive server simulation ticks, the subject may have sent multiple inputs, each incrementing Seq by 1. So + // the delta's NewSeq will naturally jump by more than 1 compared to the previous delta — even with zero packet loss. + // Without the "BaselineSeq" the client has no way to detect the package loss + uint32 baseline_seq = 2; + uint32 new_seq = 3; + uint32 server_tick = 4; + + // While the player doesn't cross the parcel, this field is omitted from diff + optional int32 parcel_index = 5; + + // X position inside the parcel + optional uint32 position_x = 6 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + + // Y position + optional uint32 position_y = 7 [(decentraland.common.quantized) = { min: 0, max: 200, bits: 13 }]; + + // Z position inside the parcel + optional uint32 position_z = 8 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + + optional uint32 velocity_x = 9 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; + optional uint32 velocity_y = 10 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; + optional uint32 velocity_z = 11 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; + + optional uint32 rotation_y = 12 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + optional uint32 movement_blend = 13 [(decentraland.common.quantized) = { min: 0, max: 3, bits: 5 }]; + optional uint32 slide_blend = 14 [(decentraland.common.quantized) = { min: 0, max: 1, bits: 4 }]; + optional uint32 head_yaw = 15 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + optional uint32 head_pitch = 16 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + + optional uint32 state_flags = 17; + optional GlideState glide_state = 18; + + optional int32 jump_count = 19; + + // Absolute world hit position the player is pointing at. + // Only meaningful when POINTING_AT is set in `state_flags`. + // + // X/Z use 17 bits over the world span (~±3000m, covers GenesisCity + border + + // raycast cutoff) which yields ~0.046m steps — at least as fine as the player's + // own parcel_index + position_x/z combined precision (0.0625m), so no separate + // parcel index is needed for point-at. + // + // Y uses 7 bits over the player altitude range (matches position_y), step ~1.6m. + // Point At is not supposed to change frequently so the wire overhead should be minimal + optional uint32 point_at_x = 20 [(decentraland.common.quantized) = { min: -3000.0, max: 3000.0, bits: 17 }]; + optional uint32 point_at_y = 21 [(decentraland.common.quantized) = { min: 0.0, max: 200.0, bits: 7 }]; + optional uint32 point_at_z = 22 [(decentraland.common.quantized) = { min: -3000.0, max: 3000.0, bits: 17 }]; +} + +// Full State is sent to the client when it is out of sync, and can't recover with a diff only +message PlayerStateFull { + uint32 subject_id = 1; + uint32 sequence = 2; + uint32 server_tick = 3; + PlayerState state = 4; +} + +// Notification to the client, that a peer has joined, it can mean connection or entering the area of interest, it's up to the server to decide +message PlayerJoined { + string user_id = 1; + int32 profile_version = 2; + PlayerStateFull state = 3; +} + +// Notification to the client, that a peer has left, it can mean disconnection or leaving the area of interest +message PlayerLeft { + uint32 subject_id = 1; +} + +enum EmoteStopReason { + COMPLETED = 0; // one-shot timer expired on the server + CANCELLED = 1; // client sent EmoteStop (looping emotes) +} + +// Server → All Observers +// Full player state is piggybacked to ensure the emote is played in the right place. +// Observers use server_tick to scrub animation forward by transit latency. +message EmoteStarted { + uint32 subject_id = 1; + uint32 sequence = 2; + uint32 server_tick = 3; + string emote_id = 4; + PlayerState player_state = 5; + optional int32 mask = 6; +} + +// Server → All Observers +// Client resumes MovementInput only after receiving this. +// Carries full PlayerState so the client can snap to the correct position on resume. +message EmoteStopped { + uint32 subject_id = 1; + uint32 server_tick = 2; + EmoteStopReason reason = 3; + uint32 sequence = 4; + PlayerState player_state = 5; +} + +// Server → All Observers +message TeleportPerformed { + uint32 subject_id = 1; + uint32 sequence = 2; + uint32 server_tick = 3; + PlayerState state = 4; +} + +message ServerMessage { + oneof message { + HandshakeResponse handshake = 1; + PlayerStateFull player_state_full = 2; + PlayerStateDeltaTier0 player_state_delta = 3; + PlayerJoined player_joined = 4; + PlayerLeft player_left = 5; + PlayerProfileVersionsAnnounced player_profile_version_announced = 6; + EmoteStarted emote_started = 7; + EmoteStopped emote_stopped = 8; + TeleportPerformed teleported = 9; + } +} diff --git a/proto/decentraland/pulse/pulse_shared.proto b/proto/decentraland/pulse/pulse_shared.proto new file mode 100644 index 00000000..6f4fba7c --- /dev/null +++ b/proto/decentraland/pulse/pulse_shared.proto @@ -0,0 +1,48 @@ +syntax = "proto3"; + +package decentraland.pulse; + +import "decentraland/common/vectors.proto"; + +enum PlayerAnimationFlags { + NONE = 0; + GROUNDED = 1; + LONG_JUMP = 2; + LONG_FALL = 4; + FALLING = 8; + STUNNED = 16; + HEAD_YAW = 32; + HEAD_PITCH = 64; + POINTING_AT = 128; +} + +enum GlideState { + PROP_CLOSED = 0; + OPENING_PROP = 1; + GLIDING = 2; + CLOSING_PROP = 3; +} + +message PlayerState { + int32 parcel_index = 1; + + decentraland.common.Vector3 position = 2; + decentraland.common.Vector3 velocity = 3; + + float rotation_y = 4; + + float movement_blend = 5; + float slide_blend = 6; + + optional float head_yaw = 7; + optional float head_pitch = 8; + + uint32 state_flags = 9; + GlideState glide_state = 10; + + int32 jump_count = 11; + + // Absolute world hit position the player is pointing at. + // Only meaningful when POINTING_AT is set in `state_flags`. + optional decentraland.common.Vector3 point_at = 12; +} diff --git a/protoc-gen-bitwise/generator_csharp.py b/protoc-gen-bitwise/generator_csharp.py new file mode 100644 index 00000000..376c0dc1 --- /dev/null +++ b/protoc-gen-bitwise/generator_csharp.py @@ -0,0 +1,176 @@ +""" +C# code generator for the protoc-gen-bitwise plugin. + +For every proto message that contains at least one uint32 field annotated with +[(quantized)], this module emits a C# partial class that adds a computed float +property named {FieldName}Quantized. The getter decodes the stored uint32 to +a float; the setter encodes a float back to a uint32. Standard protobuf +handles serialization of the uint32 wire field; this class adds a typed float +accessor on top. + +Protobuf encodes small uint32 values via varint, so a value that fits in +2^20 costs 3 bytes on the wire — cheaper than a raw IEEE 754 float (4 bytes). + +Only uint32 fields are supported. bit_packed and unannotated fields are +passed through without generating any accessor. +""" + +from google.protobuf import descriptor_pb2 + +from options_pb2 import get_field_options + +# FieldDescriptorProto type constants (aliased for readability) +_FT = descriptor_pb2.FieldDescriptorProto + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _snake_to_pascal(name: str) -> str: + """position_x → PositionX""" + return ''.join(word.capitalize() for word in name.split('_')) + + +def _package_to_namespace(package: str) -> str: + """decentraland.kernel.comms.v3 → Decentraland.Kernel.Comms.V3""" + if not package: + return 'Generated' + return '.'.join(part.capitalize() for part in package.split('.')) + + +def _format_float(value: float) -> str: + """Format a Python float as a C# float literal (e.g. -100.0f).""" + text = f'{value:.8g}' + if '.' not in text and 'e' not in text and 'E' not in text: + text += '.0' + return text + 'f' + + +def _format_step(step: float) -> str: + """Format a quantization step size for display in a doc comment (e.g. ≈ 0.003).""" + return f'\u2248 {step:.6g}' + + +# --------------------------------------------------------------------------- +# Per-message code generation +# --------------------------------------------------------------------------- + +def _gen_message(msg_proto, indent: str = ' ') -> list[str] | None: + """ + Generate a C# partial class for a proto message. + + For each uint32 field with a [(quantized)] annotation, emits a cached float + property {FieldName}Quantized backed by the raw uint32 field. + + Returns a list of lines (without trailing newline) or None if the message + has no quantized uint32 fields. + """ + props: list[tuple[str, str, str, int, float]] = [] # (prop_name, mn, mx, bits, step) + + for field in msg_proto.field: + # Repeated/map fields are not supported + if field.label == _FT.LABEL_REPEATED: + continue + + # Only uint32 fields are candidates for quantized accessors + if field.type != _FT.TYPE_UINT32: + continue + + quantized, _ = get_field_options(field.options) + if quantized is None: + continue + + step = (quantized.max - quantized.min) / ((1 << quantized.bits) - 1) + + props.append(( + _snake_to_pascal(field.name), + _format_float(quantized.min), + _format_float(quantized.max), + quantized.bits, + step, + )) + + if not props: + return None + + i = indent + backings: list[str] = [] + lines: list[str] = [] + lines.append(f'public partial class {msg_proto.name}') + lines.append('{') + + for prop_name, mn, mx, bits, step in props: + backing = '_' + prop_name[0].lower() + prop_name[1:] + backings.append(backing) + lines.append(f'{i}private float? {backing};') + lines.append(f'{i}/// Float accessor for . Range [{mn}, {mx}], {bits} bits, step {_format_step(step)}.') + lines.append(f'{i}public float {prop_name}Quantized') + lines.append(f'{i}{{') + lines.append(f'{i}{i}get => {backing} ??= Quantize.Decode({prop_name}, {mn}, {mx}, {bits});') + lines.append(f'{i}{i}set {{ {backing} = value; {prop_name} = Quantize.Encode(value, {mn}, {mx}, {bits}); }}') + lines.append(f'{i}}}') + lines.append('') + + lines.append(f'{i}/// Clears all cached decoded values. Call after mutating raw uint32 fields directly.') + lines.append(f'{i}public void ResetDecodedCache()') + lines.append(f'{i}{{') + for backing in backings: + lines.append(f'{i}{i}{backing} = null;') + lines.append(f'{i}}}') + + lines.append('}') + return lines + + +# --------------------------------------------------------------------------- +# Per-file code generation (public entry point) +# --------------------------------------------------------------------------- + +def generate_csharp(file_proto) -> dict | None: + """ + Generate a C# source file for a FileDescriptorProto. + + Returns a dict with keys 'name' (output path) and 'content' (C# source), + or None if the file contains no quantized uint32 fields. + """ + namespace = _package_to_namespace(file_proto.package) + + # Output is placed flat in the output root, matching --csharp_out convention. + # e.g. "decentraland/kernel/comms/v3/my_message.proto" + # → "MyMessage.Bitwise.cs" + proto_file = file_proto.name.rsplit('/', 1)[-1] + stem = _snake_to_pascal(proto_file.replace('.proto', '')) + out_name = f'{stem}.Bitwise.cs' + + header = [ + '// ', + '// Generated by protoc-gen-bitwise. DO NOT EDIT.', + f'// Source: {file_proto.name}', + '// ', + '', + 'using Decentraland.Networking.Bitwise;', + '', + f'namespace {namespace}', + '{', + ] + footer = [ + '', + f'}} // namespace {namespace}', + ] + + body: list[str] = [] + for msg in file_proto.message_type: + msg_lines = _gen_message(msg) + if msg_lines is None: + continue + # Indent each line by 4 spaces (inside the namespace block) + for line in msg_lines: + body.append((' ' + line) if line else '') + body.append('') + + if not body: + return None # nothing to emit + + content = '\n'.join(header + body + footer) + '\n' + return {'name': out_name, 'content': content} diff --git a/protoc-gen-bitwise/options_pb2.py b/protoc-gen-bitwise/options_pb2.py new file mode 100644 index 00000000..d3e637f2 --- /dev/null +++ b/protoc-gen-bitwise/options_pb2.py @@ -0,0 +1,171 @@ +""" +Manual parser for the custom bitwise field options defined in options.proto. + +Rather than relying on protobuf extension registration (which requires a properly +compiled _pb2 module), this module parses the raw serialized FieldOptions bytes +directly using the protobuf binary wire format. All protobuf runtimes preserve +unknown/unregistered extension bytes when round-tripping, so +field.options.SerializeToString() always contains the extension data even when +the extension is not registered in the Python runtime. + +Wire format reference: + tag = (field_number << 3) | wire_type + wire_type 0 = varint, 1 = 64-bit, 2 = length-delimited, 5 = 32-bit +""" + +import struct + +# Extension field numbers as defined in options.proto +QUANTIZED_FIELD_NUMBER = 50001 +BIT_PACKED_FIELD_NUMBER = 50002 + + +# --------------------------------------------------------------------------- +# Low-level wire-format helpers +# --------------------------------------------------------------------------- + +def _read_varint(data: bytes, pos: int): + """Decode a protobuf varint starting at *pos*. Returns (value, new_pos).""" + result = 0 + shift = 0 + while pos < len(data): + byte = data[pos] + pos += 1 + result |= (byte & 0x7F) << shift + if not (byte & 0x80): + break + shift += 7 + return result, pos + + +def _read_float32(data: bytes, pos: int): + """Decode a little-endian 32-bit float. Returns (value, new_pos).""" + value, = struct.unpack_from(' int: + """Advance *pos* past a field with the given wire_type.""" + if wire_type == 0: + _, pos = _read_varint(data, pos) + elif wire_type == 1: + pos += 8 + elif wire_type == 2: + length, pos = _read_varint(data, pos) + pos += length + elif wire_type == 5: + pos += 4 + # wire types 3 and 4 (start/end group) are deprecated; skip gracefully + return pos + + +# --------------------------------------------------------------------------- +# Option message classes +# --------------------------------------------------------------------------- + +class QuantizedFloatOptions: + """Mirrors the QuantizedFloatOptions proto message.""" + + __slots__ = ('min', 'max', 'bits') + + def __init__(self, min_val: float = 0.0, max_val: float = 0.0, bits: int = 0): + self.min = min_val + self.max = max_val + self.bits = bits + + @classmethod + def from_bytes(cls, data: bytes) -> 'QuantizedFloatOptions': + obj = cls() + pos = 0 + while pos < len(data): + tag, pos = _read_varint(data, pos) + field_num = tag >> 3 + wire_type = tag & 0x7 + if field_num == 1 and wire_type == 5: # min (float) + obj.min, pos = _read_float32(data, pos) + elif field_num == 2 and wire_type == 5: # max (float) + obj.max, pos = _read_float32(data, pos) + elif field_num == 3 and wire_type == 0: # bits (uint32) + obj.bits, pos = _read_varint(data, pos) + else: + pos = _skip_field(data, pos, wire_type) + return obj + + def __repr__(self): + return f'QuantizedFloatOptions(min={self.min}, max={self.max}, bits={self.bits})' + + +class BitPackedOptions: + """Mirrors the BitPackedOptions proto message.""" + + __slots__ = ('bits',) + + def __init__(self, bits: int = 0): + self.bits = bits + + @classmethod + def from_bytes(cls, data: bytes) -> 'BitPackedOptions': + obj = cls() + pos = 0 + while pos < len(data): + tag, pos = _read_varint(data, pos) + field_num = tag >> 3 + wire_type = tag & 0x7 + if field_num == 1 and wire_type == 0: # bits (uint32) + obj.bits, pos = _read_varint(data, pos) + else: + pos = _skip_field(data, pos, wire_type) + return obj + + def __repr__(self): + return f'BitPackedOptions(bits={self.bits})' + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def get_field_options(field_options_proto): + """ + Extract custom bitwise options from a FieldDescriptorProto.options object. + + Serialises the options message to bytes and walks the wire-format stream + looking for extension fields 50001 (quantized) and 50002 (bit_packed). + + Args: + field_options_proto: google.protobuf.descriptor_pb2.FieldOptions instance + (may be a default/empty instance when no options are set). + + Returns: + tuple[QuantizedFloatOptions | None, BitPackedOptions | None] + """ + try: + raw = field_options_proto.SerializeToString() + except Exception: + return None, None + + if not raw: + return None, None + + quantized = None + bit_packed = None + pos = 0 + + while pos < len(raw): + tag, pos = _read_varint(raw, pos) + field_num = tag >> 3 + wire_type = tag & 0x7 + + if wire_type == 2: # length-delimited + length, pos = _read_varint(raw, pos) + value_bytes = raw[pos:pos + length] + pos += length + if field_num == QUANTIZED_FIELD_NUMBER: + quantized = QuantizedFloatOptions.from_bytes(value_bytes) + elif field_num == BIT_PACKED_FIELD_NUMBER: + bit_packed = BitPackedOptions.from_bytes(value_bytes) + # else: unknown length-delimited field — already consumed + else: + pos = _skip_field(raw, pos, wire_type) + + return quantized, bit_packed diff --git a/protoc-gen-bitwise/plugin.py b/protoc-gen-bitwise/plugin.py new file mode 100644 index 00000000..09b5abe4 --- /dev/null +++ b/protoc-gen-bitwise/plugin.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +""" +protoc-gen-bitwise — protoc plugin that generates C# bitwise serialization code. + +Protocol: + 1. protoc writes a serialised CodeGeneratorRequest to this process's stdin. + 2. This plugin reads it, generates C# partial classes with Encode / Decode + methods for every message that carries [(quantized)] or [(bit_packed)] + field annotations. + 3. A serialised CodeGeneratorResponse is written to stdout. + +Usage (from project root): + protoc \\ + --proto_path=proto \\ + --bitwise_out=generated/ \\ + --plugin=protoc-gen-bitwise \\ + proto/my_messages.proto + +Windows invocation (plugin not on PATH as executable): + protoc \\ + --proto_path=proto \\ + --bitwise_out=generated/ \\ + --plugin=protoc-gen-bitwise=python protoc-gen-bitwise/plugin.py \\ + proto/my_messages.proto + +Dependencies: + pip install grpcio-tools # or: pip install protobuf +""" + +import os +import sys + +# Ensure sibling modules (generator_csharp, options_pb2) are importable +# regardless of where protoc invokes this script from. +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +# On Windows, stdin/stdout are opened in text mode by default which corrupts +# the binary protobuf payload. +if sys.platform == 'win32': + import msvcrt + msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) + msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) + +from google.protobuf.compiler import plugin_pb2 + +from generator_csharp import generate_csharp + + +def main() -> None: + request_bytes = sys.stdin.buffer.read() + + request = plugin_pb2.CodeGeneratorRequest() + request.ParseFromString(request_bytes) + + response = plugin_pb2.CodeGeneratorResponse() + # Advertise proto3-optional support so protoc does not reject the plugin. + response.supported_features = ( + plugin_pb2.CodeGeneratorResponse.FEATURE_PROTO3_OPTIONAL + ) + + # Build a lookup map for all file descriptors (needed for imports, though + # the current generator only uses the directly requested files). + file_by_name = {f.name: f for f in request.proto_file} + + for file_name in request.file_to_generate: + # Skip the options definition file itself — it has no messages to generate. + if file_name == 'decentraland/common/options.proto': + continue + + file_proto = file_by_name.get(file_name) + if file_proto is None: + continue + + try: + generated = generate_csharp(file_proto) + except Exception as exc: # noqa: BLE001 + error = response.file.add() + error.name = '' # empty name signals an error to protoc + # Append error text; protoc will print it and fail. + response.error = f'protoc-gen-bitwise: error processing {file_name}: {exc}' + continue + + if generated is not None: + out = response.file.add() + out.name = generated['name'] + out.content = generated['content'] + + sys.stdout.buffer.write(response.SerializeToString()) + + +if __name__ == '__main__': + main() diff --git a/protoc-gen-bitwise/runtime/cs/BitReader.cs b/protoc-gen-bitwise/runtime/cs/BitReader.cs new file mode 100644 index 00000000..51efa39c --- /dev/null +++ b/protoc-gen-bitwise/runtime/cs/BitReader.cs @@ -0,0 +1,112 @@ +// Decentraland.Networking.Bitwise — BitReader +// Copy this file into your Unity project alongside generated *.Bitwise.cs files. + +using System; + +namespace Decentraland.Networking.Bitwise +{ + /// + /// Reads bits from a byte buffer, MSB first within each byte (big-endian bit + /// order). Symmetric counterpart of : every + /// Write… call has a corresponding Read… call with identical arguments that + /// reproduces the original value. + /// + public sealed class BitReader + { + private readonly byte[] _buffer; + private int _bitPos; + + /// Source buffer filled by a . + public BitReader(byte[] buffer) + { + _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer)); + _bitPos = 0; + } + + /// Current read position in bits. + public int BitPosition => _bitPos; + + /// + /// Returns true when all written bits have been consumed + /// (i.e. has reached the end of the buffer). + /// + public bool IsAtEnd => _bitPos >= _buffer.Length * 8; + + // ----------------------------------------------------------------- + // Core primitive + // ----------------------------------------------------------------- + + /// + /// Reads bits and returns them as the + /// least-significant bits of a , MSB first. + /// + public uint ReadBits(int bits) + { + uint value = 0; + for (int i = bits - 1; i >= 0; i--) + { + int byteIdx = _bitPos / 8; + int bitIdx = 7 - (_bitPos % 8); + + if ((_buffer[byteIdx] >> bitIdx & 1) == 1) + value |= 1u << i; + + _bitPos++; + } + return value; + } + + // ----------------------------------------------------------------- + // Quantized float + // ----------------------------------------------------------------- + + /// + /// Reads a quantized float encoded with . + /// Arguments must match those used during encoding exactly. + /// + public float ReadQuantizedFloat(float min, float max, int bits) + { + uint maxQ = (1u << bits) - 1; + uint quantized = ReadBits(bits); + float normalized = (float)quantized / maxQ; + return min + normalized * (max - min); + } + + // ----------------------------------------------------------------- + // Standard IEEE 754 helpers + // ----------------------------------------------------------------- + + /// Reads a 32-bit IEEE 754 float written by . + public float ReadFloat() + { + uint bits = ReadBits(32); + byte[] bytes = + { + (byte)(bits & 0xFF), + (byte)((bits >> 8) & 0xFF), + (byte)((bits >> 16) & 0xFF), + (byte)((bits >> 24) & 0xFF), + }; + return BitConverter.ToSingle(bytes, 0); + } + + /// Reads a 64-bit IEEE 754 double written by . + public double ReadDouble() + { + uint hi = ReadBits(32); + uint lo = ReadBits(32); + byte[] bytes = + { + (byte)(lo & 0xFF), + (byte)((lo >> 8) & 0xFF), + (byte)((lo >> 16) & 0xFF), + (byte)((lo >> 24) & 0xFF), + (byte)(hi & 0xFF), + (byte)((hi >> 8) & 0xFF), + (byte)((hi >> 16) & 0xFF), + (byte)((hi >> 24) & 0xFF), + }; + return BitConverter.ToDouble(bytes, 0); + } + } +} diff --git a/protoc-gen-bitwise/runtime/cs/BitWriter.cs b/protoc-gen-bitwise/runtime/cs/BitWriter.cs new file mode 100644 index 00000000..19b205b8 --- /dev/null +++ b/protoc-gen-bitwise/runtime/cs/BitWriter.cs @@ -0,0 +1,117 @@ +// Decentraland.Networking.Bitwise — BitWriter +// Copy this file into your Unity project alongside generated *.Bitwise.cs files. + +using System; + +namespace Decentraland.Networking.Bitwise +{ + /// + /// Writes bits into a pre-allocated byte buffer, MSB first within each byte + /// (big-endian bit order). This matches the layout expected by + /// so that encode → decode is always a round-trip no-op. + /// + public sealed class BitWriter + { + private readonly byte[] _buffer; + private int _bitPos; + + /// Destination buffer (must be large enough for all writes). + public BitWriter(byte[] buffer) + { + _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer)); + _bitPos = 0; + } + + /// Current write position in bits. + public int BitPosition => _bitPos; + + /// Number of bytes written (rounded up to the nearest byte). + public int ByteLength => (_bitPos + 7) / 8; + + /// Returns a copy of the written bytes (trimmed to ). + public byte[] ToArray() + { + var result = new byte[ByteLength]; + Array.Copy(_buffer, result, ByteLength); + return result; + } + + // ----------------------------------------------------------------- + // Core primitive + // ----------------------------------------------------------------- + + /// + /// Writes the least-significant bits of + /// , MSB first. + /// + public void WriteBits(uint value, int bits) + { + for (int i = bits - 1; i >= 0; i--) + { + int byteIdx = _bitPos / 8; + int bitIdx = 7 - (_bitPos % 8); + + if ((value >> i & 1u) == 1u) + _buffer[byteIdx] |= (byte)(1 << bitIdx); + else + _buffer[byteIdx] &= (byte)~(1 << bitIdx); + + _bitPos++; + } + } + + // ----------------------------------------------------------------- + // Quantized float + // ----------------------------------------------------------------- + + /// + /// Quantizes into bits + /// using the range [, ] and + /// writes it to the buffer. + /// + /// Uses Math.Round (banker's rounding → ties-to-even) to guarantee + /// that encode → decode is a round-trip no-op. + /// + public void WriteQuantizedFloat(float value, float min, float max, int bits) + { + uint maxQ = (1u << bits) - 1; + float clamped = Math.Clamp(value, min, max); + float normalized = (clamped - min) / (max - min); + uint quantized = (uint)Math.Round(normalized * maxQ); + WriteBits(quantized, bits); + } + + // ----------------------------------------------------------------- + // Standard IEEE 754 helpers (used for un-annotated float/double fields) + // ----------------------------------------------------------------- + + /// Writes a 32-bit IEEE 754 float (4 bytes). + public void WriteFloat(float value) + { + byte[] bytes = BitConverter.GetBytes(value); + // GetBytes is little-endian on all platforms; write MSB first. + uint bits = (uint)bytes[0] + | ((uint)bytes[1] << 8) + | ((uint)bytes[2] << 16) + | ((uint)bytes[3] << 24); + WriteBits(bits, 32); + } + + /// Writes a 64-bit IEEE 754 double (8 bytes). + public void WriteDouble(double value) + { + byte[] bytes = BitConverter.GetBytes(value); + uint lo = (uint)bytes[0] + | ((uint)bytes[1] << 8) + | ((uint)bytes[2] << 16) + | ((uint)bytes[3] << 24); + uint hi = (uint)bytes[4] + | ((uint)bytes[5] << 8) + | ((uint)bytes[6] << 16) + | ((uint)bytes[7] << 24); + // Write high 32 bits first so the bit stream is big-endian at word level too. + WriteBits(hi, 32); + WriteBits(lo, 32); + } + } +} diff --git a/protoc-gen-bitwise/runtime/cs/Quantize.cs b/protoc-gen-bitwise/runtime/cs/Quantize.cs new file mode 100644 index 00000000..22c347fe --- /dev/null +++ b/protoc-gen-bitwise/runtime/cs/Quantize.cs @@ -0,0 +1,35 @@ +// Decentraland.Networking.Bitwise — Quantize +// Copy this file into your project alongside generated *.Bitwise.cs files. +// ReSharper disable once RedundantUsingDirective + +using System; + +namespace Decentraland.Networking.Bitwise + // ReSharper disable once ArrangeNamespaceBody +{ + /// + /// Static helpers for quantizing float values to/from unsigned integers. + /// Intended for use with protobuf uint32 fields: the integer is transmitted + /// as a protobuf varint, while the float accessor lives in a generated partial class. + /// + public static class Quantize + { + /// + /// Encodes to a quantized . + /// Values outside [, ] are clamped. + /// + public static uint Encode(float value, float min, float max, int bits) + { + var steps = (1u << bits) - 1; + var t = Math.Clamp((value - min) / (max - min), 0f, 1f); + return (uint)MathF.Round(t * steps); + } + + /// Decodes a quantized back to a float. + public static float Decode(uint encoded, float min, float max, int bits) + { + var steps = (1u << bits) - 1; + return (float)encoded / steps * (max - min) + min; + } + } +} From 58b8a929e5482c11b98376b99eb1c1124e6fdc98 Mon Sep 17 00:00:00 2001 From: Mikhail Agapov <118179774+mikhail-dcl@users.noreply.github.com> Date: Mon, 15 Jun 2026 23:28:03 +0300 Subject: [PATCH 47/54] chore: port protoc-gen-bitwise to Node (drop Python dependency) (#423) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrite the protoc-gen-bitwise plugin from Python to a dependency-free Node implementation, so protocol generation needs only `node` on PATH — no Python interpreter and no `protobuf` pip package. - plugin.js stdin/stdout protoc plugin entry; advertises FEATURE_PROTO3_OPTIONAL - wire.js self-contained protobuf wire codec for CodeGeneratorRequest/Response (no runtime deps) - options.js parser for the custom quantized / bit_packed options - generator_csharp.js C# partial-class generator with faithful %g float formatting; output is byte-for-byte identical to the Python plugin - test/ + `npm run gen:test` golden-fixture parity test - remove plugin.py, generator_csharp.py, options_pb2.py - package.json ship only the .js + C# runtime (exclude test fixtures) - README.md, CLAUDE.md Python -> Node prerequisites and invocation Verified byte-identical regeneration three ways: protoc end-to-end, a Pulse clean-room rebuild (all 16 generated files), and unity-explorer build-protocol. Consumers invoke the plugin via a tiny `node plugin.js` wrapper (.cmd on Windows, shell script elsewhere). Co-authored-by: Claude Opus 4.8 (1M context) --- CLAUDE.md | 184 ++++++++++++++ README.md | 15 +- package.json | 6 +- protoc-gen-bitwise/generator_csharp.js | 215 ++++++++++++++++ protoc-gen-bitwise/generator_csharp.py | 176 ------------- protoc-gen-bitwise/options.js | 108 ++++++++ protoc-gen-bitwise/options_pb2.py | 171 ------------- protoc-gen-bitwise/plugin.js | 87 +++++++ protoc-gen-bitwise/plugin.py | 92 ------- protoc-gen-bitwise/test/generator.test.js | 180 +++++++++++++ .../test/golden/PulseServer.Bitwise.cs | 145 +++++++++++ .../golden/QuantizationExample.Bitwise.cs | 134 ++++++++++ protoc-gen-bitwise/wire.js | 239 ++++++++++++++++++ 13 files changed, 1305 insertions(+), 447 deletions(-) create mode 100644 CLAUDE.md create mode 100644 protoc-gen-bitwise/generator_csharp.js delete mode 100644 protoc-gen-bitwise/generator_csharp.py create mode 100644 protoc-gen-bitwise/options.js delete mode 100644 protoc-gen-bitwise/options_pb2.py create mode 100755 protoc-gen-bitwise/plugin.js delete mode 100644 protoc-gen-bitwise/plugin.py create mode 100644 protoc-gen-bitwise/test/generator.test.js create mode 100644 protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs create mode 100644 protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs create mode 100644 protoc-gen-bitwise/wire.js diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..130756ee --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,184 @@ +# CLAUDE.md — MMO Networking Stack + +## Project Overview + +High-performance MMO-style multiplayer networking stack. Protocol is **open** (Unity C# client + others). Infrastructure on **AWS**. Goals: high concurrency, aggressive interest management, low-latency state sync. + +| Layer | Choice | +|---|---| +| Transport | ENet (UDP, channel-based) | +| Client | Unity (C#) | +| Server | Custom server | +| Schema source of truth | `.proto` files | +| Serialization | Custom protoc plugin (bitwise encoding) | +| Auth | Decentraland ECDSA chain validation (local, on HANDSHAKE channel 0) | + +--- + +## Serialization: Custom Protoc Plugin + +### What it does +Reads `.proto` files with custom field options and generates **bitwise encode/decode code** in C#, keeping all client implementations bit-for-bit identical. + +### Custom Field Options (`options.proto`) + +```protobuf +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; + +message QuantizedFloatOptions { + float min = 1; + float max = 2; + uint32 bits = 3; +} + +message BitPackedOptions { + uint32 bits = 1; +} + +extend google.protobuf.FieldOptions { + QuantizedFloatOptions quantized = 50001; + BitPackedOptions bit_packed = 50002; +} +``` + +### Usage example + +```protobuf +message PositionDelta { + float dx = 1 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + float dy = 2 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + float dz = 3 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 entity_id = 4 [(bit_packed) = { bits: 20 }]; +} +// Total: 68 bits = 9 bytes on the wire +``` + +### Plugin structure + +``` +protoc-gen-bitwise/ +├── plugin.js # stdin -> CodeGeneratorRequest, stdout -> CodeGeneratorResponse (Node) +├── generator_csharp.js # emits C# for Unity +├── options.js # parses the custom quantized / bit_packed field options +├── wire.js # self-contained protobuf wire codec (zero runtime deps) +└── runtime/cs/ # C# runtime; Quantize.cs is copied into the generated output +``` + +Plugin contract: a protoc plugin that reads a serialized `CodeGeneratorRequest` from stdin and writes a serialized `CodeGeneratorResponse` to stdout. It is a plain Node script — **no `npm install` required, only `node` on PATH**. protoc invokes it through a tiny wrapper that runs `node plugin.js` (`.cmd` on Windows, a shell script elsewhere, since protoc cannot exec a `.js` directly). + +```bash +protoc \ + --proto_path=proto \ + --bitwise_out=generated/ \ + --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.js \ + movement.proto position.proto +``` + +Parity is locked down by `npm run gen:test` (compares generator output against golden C# fixtures in `protoc-gen-bitwise/test/`). + +--- + +## BitWriter / BitReader + +The C# implementation uses the following bit layout: **big-endian within each byte**, MSB written first. Use **`Round`** (not truncate) when quantizing to minimize error. + +### Core math — WriteQuantizedFloat + +``` +normalized = (clamp(value, min, max) - min) / (max - min) // -> [0.0, 1.0] +quantized = Round(normalized * ((1 << bits) - 1)) // -> integer +``` + +### Core math — ReadQuantizedFloat + +``` +normalized = quantized / ((1 << bits) - 1) +value = min + normalized * (max - min) +``` + +### Implementation + +```csharp +public class BitWriter +{ + private byte[] _buffer; + private int _bitPos; + + public BitWriter(byte[] buffer) { _buffer = buffer; _bitPos = 0; } + + public void WriteBits(uint value, int bits) + { + for (int i = bits - 1; i >= 0; i--) + { + int byteIdx = _bitPos / 8; + int bitIdx = 7 - (_bitPos % 8); + if ((value >> i & 1) == 1) _buffer[byteIdx] |= (byte)(1 << bitIdx); + else _buffer[byteIdx] &= (byte)~(1 << bitIdx); + _bitPos++; + } + } + + public void WriteQuantizedFloat(float value, float min, float max, int bits) + { + uint maxQ = (1u << bits) - 1; + float clamped = Math.Clamp(value, min, max); + float normalized = (clamped - min) / (max - min); + uint quantized = (uint)Math.Round(normalized * maxQ); + WriteBits(quantized, bits); + } +} + +public class BitReader +{ + private byte[] _buffer; + private int _bitPos; + + public BitReader(byte[] buffer) { _buffer = buffer; _bitPos = 0; } + + public uint ReadBits(int bits) + { + uint value = 0; + for (int i = bits - 1; i >= 0; i--) + { + int byteIdx = _bitPos / 8; + int bitIdx = 7 - (_bitPos % 8); + if ((_buffer[byteIdx] >> bitIdx & 1) == 1) value |= 1u << i; + _bitPos++; + } + return value; + } + + public float ReadQuantizedFloat(float min, float max, int bits) + { + uint maxQ = (1u << bits) - 1; + uint quantized = ReadBits(bits); + float normalized = (float)quantized / maxQ; + return min + normalized * (max - min); + } +} +``` + +--- + +## Precision Reference + +| Range | Bits | Step size | +|-------------|------|---------------| +| [-100, 100] | 16 | ~0.003 units | +| [-10, 10] | 12 | ~0.005 units | +| [-100, 100] | 12 | ~0.049 units | + +Sub-centimeter precision is achievable at 12-16 bits for position deltas. + +--- + +## Key Design Principles + +- `.proto` files are the **single source of truth** for all message schemas +- The protoc plugin generates **C#** from the schema — never hand-write serialization +- Encode -> decode is a **no-op** (round-trip safe) due to consistent use of `Round` +- Prefer **client-driven resync** over proactive server corrections +- Push complexity to clients where appropriate; server maintains authority +- Channel 0: reliable messages (STATE_FULL snapshots, ACKs, resync requests, HANDSHAKE) +- Channel 1: unreliable sequenced (high-frequency position deltas, client input) \ No newline at end of file diff --git a/README.md b/README.md index 20c75605..b11c3302 100644 --- a/README.md +++ b/README.md @@ -100,13 +100,11 @@ client can read it without knowledge of the plugin. | Requirement | Version | |---|---| -| Python | 3.10+ | -| `protobuf` Python package | 4.x or 3.20+ | +| Node.js | 16+ | | `protoc` | 3.19+ | -```bash -pip install protobuf -``` +The plugin is a dependency-free Node script — no `npm install` or extra +packages are required to run it. ## Step 1 — Annotate your `.proto` file @@ -157,11 +155,16 @@ protoc \ --proto_path=proto \ --proto_path=/path/to/google/protobuf/include \ --csharp_out=generated/cs \ - --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.py \ + --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.js \ --bitwise_out=generated/cs \ proto/decentraland/kernel/comms/v3/comms.proto ``` +> `plugin.js` carries a `#!/usr/bin/env node` shebang. On Windows, protoc +> cannot exec a `.js` directly, so point `--plugin` at a small `.cmd` wrapper +> that runs `node plugin.js`; on unix an equivalent shell script is used. Both +> consumer repos (Pulse, unity-explorer) generate this wrapper automatically. + The plugin emits one `*.Bitwise.cs` file (PascalCase, flat in the output directory) for each `.proto` file that contains at least one `[(quantized)]` field. diff --git a/package.json b/package.json index 2312a78a..bb3533f8 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "license": "Apache-2.0", "main": "index.js", "scripts": { - "test": "./scripts/check-proto-compabitility.sh" + "test": "./scripts/check-proto-compabitility.sh", + "gen:test": "node protoc-gen-bitwise/test/generator.test.js" }, "devDependencies": { "@protobuf-ts/protoc": "^2.8.1", @@ -32,6 +33,7 @@ "out-ts", "out-js", "public", - "protoc-gen-bitwise" + "protoc-gen-bitwise/*.js", + "protoc-gen-bitwise/runtime" ] } diff --git a/protoc-gen-bitwise/generator_csharp.js b/protoc-gen-bitwise/generator_csharp.js new file mode 100644 index 00000000..dcdb95a0 --- /dev/null +++ b/protoc-gen-bitwise/generator_csharp.js @@ -0,0 +1,215 @@ +'use strict' + +/** + * C# code generator for the protoc-gen-bitwise plugin. + * + * For every proto message that contains at least one uint32 field annotated + * with [(quantized)], this module emits a C# partial class that adds a computed + * float property named {FieldName}Quantized. The getter decodes the stored + * uint32 to a float; the setter encodes a float back to a uint32. Standard + * protobuf handles serialization of the uint32 wire field; this class adds a + * typed float accessor on top. + * + * Only uint32 fields are supported. bit_packed and unannotated fields are + * passed through without generating any accessor. + * + * Port of the original generator_csharp.py — output is intended to be + * byte-for-byte identical. + */ + +const { getFieldOptions } = require('./options') + +// FieldDescriptorProto type/label constants. +const TYPE_UINT32 = 13 +const LABEL_REPEATED = 3 + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** Mirrors Python str.capitalize(): upper-first, lowercase the rest. */ +function capitalize(word) { + if (word.length === 0) return '' + return word[0].toUpperCase() + word.slice(1).toLowerCase() +} + +/** position_x -> PositionX */ +function snakeToPascal(name) { + return name.split('_').map(capitalize).join('') +} + +/** decentraland.kernel.comms.v3 -> Decentraland.Kernel.Comms.V3 */ +function packageToNamespace(pkg) { + if (!pkg) return 'Generated' + return pkg.split('.').map(capitalize).join('.') +} + +/** + * Format a number with C printf %g semantics at `precision` significant digits: + * shortest of fixed/scientific, with trailing zeros and a trailing dot removed. + * Reproduces Python's `f'{value:.{precision}g}'`. + */ +function formatG(value, precision) { + if (precision <= 0) precision = 1 + if (value === 0) return '0' + if (!Number.isFinite(value)) return value > 0 ? 'inf' : 'nan' + + const negative = value < 0 + const v = Math.abs(value) + + // Correctly-rounded scientific form yields the decimal exponent X (handling + // carry such as 9.999 -> 1.0e+1). + const sci = v.toExponential(precision - 1) + const eIdx = sci.indexOf('e') + const X = parseInt(sci.slice(eIdx + 1), 10) + + let result + if (X >= -4 && X < precision) { + // Fixed notation with (precision - 1 - X) fraction digits. + const fractionDigits = precision - 1 - X + result = v.toFixed(fractionDigits >= 0 ? fractionDigits : 0) + if (result.indexOf('.') !== -1) { + result = result.replace(/0+$/, '').replace(/\.$/, '') + } + } else { + // Scientific notation; printf prints at least two exponent digits. + let mantissa = sci.slice(0, eIdx) + if (mantissa.indexOf('.') !== -1) { + mantissa = mantissa.replace(/0+$/, '').replace(/\.$/, '') + } + const expSign = X < 0 ? '-' : '+' + let expDigits = String(Math.abs(X)) + if (expDigits.length < 2) expDigits = '0' + expDigits + result = mantissa + 'e' + expSign + expDigits + } + + return (negative ? '-' : '') + result +} + +/** Format a value as a C# float literal (e.g. -100.0f). */ +function formatFloat(value) { + let text = formatG(value, 8) + if (text.indexOf('.') === -1 && text.indexOf('e') === -1 && text.indexOf('E') === -1) { + text += '.0' + } + return text + 'f' +} + +/** Format a quantization step size for a doc comment (e.g. "≈ 0.003"). */ +function formatStep(step) { + return '≈ ' + formatG(step, 6) +} + +// --------------------------------------------------------------------------- +// Per-message code generation +// --------------------------------------------------------------------------- + +/** + * Generate a C# partial class for a proto message, or null if it has no + * quantized uint32 fields. Returns an array of lines (no trailing newline). + */ +function generateMessage(msgProto, indent) { + const i = indent || ' ' + const props = [] + + for (const field of msgProto.field) { + // Repeated/map fields are not supported. + if (field.label === LABEL_REPEATED) continue + // Only uint32 fields are candidates for quantized accessors. + if (field.type !== TYPE_UINT32) continue + + const { quantized } = getFieldOptions(field.optionsRaw) + if (quantized === null) continue + + const step = (quantized.max - quantized.min) / ((1 << quantized.bits) - 1) + + props.push({ + propName: snakeToPascal(field.name), + mn: formatFloat(quantized.min), + mx: formatFloat(quantized.max), + bits: quantized.bits, + step, + }) + } + + if (props.length === 0) return null + + const backings = [] + const lines = [] + lines.push(`public partial class ${msgProto.name}`) + lines.push('{') + + for (const { propName, mn, mx, bits, step } of props) { + const backing = '_' + propName[0].toLowerCase() + propName.slice(1) + backings.push(backing) + lines.push(`${i}private float? ${backing};`) + lines.push( + `${i}/// Float accessor for . Range [${mn}, ${mx}], ${bits} bits, step ${formatStep(step)}.`, + ) + lines.push(`${i}public float ${propName}Quantized`) + lines.push(`${i}{`) + lines.push(`${i}${i}get => ${backing} ??= Quantize.Decode(${propName}, ${mn}, ${mx}, ${bits});`) + lines.push(`${i}${i}set { ${backing} = value; ${propName} = Quantize.Encode(value, ${mn}, ${mx}, ${bits}); }`) + lines.push(`${i}}`) + lines.push('') + } + + lines.push(`${i}/// Clears all cached decoded values. Call after mutating raw uint32 fields directly.`) + lines.push(`${i}public void ResetDecodedCache()`) + lines.push(`${i}{`) + for (const backing of backings) { + lines.push(`${i}${i}${backing} = null;`) + } + lines.push(`${i}}`) + + lines.push('}') + return lines +} + +// --------------------------------------------------------------------------- +// Per-file code generation (public entry point) +// --------------------------------------------------------------------------- + +/** + * Generate a C# source file for a FileDescriptorProto, or null if the file + * contains no quantized uint32 fields. + * @returns {{name: string, content: string} | null} + */ +function generateCsharp(fileProto) { + const namespace = packageToNamespace(fileProto.package) + + const protoFile = fileProto.name.split('/').pop() + const stem = snakeToPascal(protoFile.replace('.proto', '')) + const outName = `${stem}.Bitwise.cs` + + const header = [ + '// ', + '// Generated by protoc-gen-bitwise. DO NOT EDIT.', + `// Source: ${fileProto.name}`, + '// ', + '', + 'using Decentraland.Networking.Bitwise;', + '', + `namespace ${namespace}`, + '{', + ] + const footer = ['', `} // namespace ${namespace}`] + + const body = [] + for (const msg of fileProto.messageType) { + const msgLines = generateMessage(msg) + if (msgLines === null) continue + // Indent each line by 4 spaces (inside the namespace block). + for (const line of msgLines) { + body.push(line ? ' ' + line : '') + } + body.push('') + } + + if (body.length === 0) return null + + const content = header.concat(body, footer).join('\n') + '\n' + return { name: outName, content } +} + +module.exports = { generateCsharp } diff --git a/protoc-gen-bitwise/generator_csharp.py b/protoc-gen-bitwise/generator_csharp.py deleted file mode 100644 index 376c0dc1..00000000 --- a/protoc-gen-bitwise/generator_csharp.py +++ /dev/null @@ -1,176 +0,0 @@ -""" -C# code generator for the protoc-gen-bitwise plugin. - -For every proto message that contains at least one uint32 field annotated with -[(quantized)], this module emits a C# partial class that adds a computed float -property named {FieldName}Quantized. The getter decodes the stored uint32 to -a float; the setter encodes a float back to a uint32. Standard protobuf -handles serialization of the uint32 wire field; this class adds a typed float -accessor on top. - -Protobuf encodes small uint32 values via varint, so a value that fits in -2^20 costs 3 bytes on the wire — cheaper than a raw IEEE 754 float (4 bytes). - -Only uint32 fields are supported. bit_packed and unannotated fields are -passed through without generating any accessor. -""" - -from google.protobuf import descriptor_pb2 - -from options_pb2 import get_field_options - -# FieldDescriptorProto type constants (aliased for readability) -_FT = descriptor_pb2.FieldDescriptorProto - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _snake_to_pascal(name: str) -> str: - """position_x → PositionX""" - return ''.join(word.capitalize() for word in name.split('_')) - - -def _package_to_namespace(package: str) -> str: - """decentraland.kernel.comms.v3 → Decentraland.Kernel.Comms.V3""" - if not package: - return 'Generated' - return '.'.join(part.capitalize() for part in package.split('.')) - - -def _format_float(value: float) -> str: - """Format a Python float as a C# float literal (e.g. -100.0f).""" - text = f'{value:.8g}' - if '.' not in text and 'e' not in text and 'E' not in text: - text += '.0' - return text + 'f' - - -def _format_step(step: float) -> str: - """Format a quantization step size for display in a doc comment (e.g. ≈ 0.003).""" - return f'\u2248 {step:.6g}' - - -# --------------------------------------------------------------------------- -# Per-message code generation -# --------------------------------------------------------------------------- - -def _gen_message(msg_proto, indent: str = ' ') -> list[str] | None: - """ - Generate a C# partial class for a proto message. - - For each uint32 field with a [(quantized)] annotation, emits a cached float - property {FieldName}Quantized backed by the raw uint32 field. - - Returns a list of lines (without trailing newline) or None if the message - has no quantized uint32 fields. - """ - props: list[tuple[str, str, str, int, float]] = [] # (prop_name, mn, mx, bits, step) - - for field in msg_proto.field: - # Repeated/map fields are not supported - if field.label == _FT.LABEL_REPEATED: - continue - - # Only uint32 fields are candidates for quantized accessors - if field.type != _FT.TYPE_UINT32: - continue - - quantized, _ = get_field_options(field.options) - if quantized is None: - continue - - step = (quantized.max - quantized.min) / ((1 << quantized.bits) - 1) - - props.append(( - _snake_to_pascal(field.name), - _format_float(quantized.min), - _format_float(quantized.max), - quantized.bits, - step, - )) - - if not props: - return None - - i = indent - backings: list[str] = [] - lines: list[str] = [] - lines.append(f'public partial class {msg_proto.name}') - lines.append('{') - - for prop_name, mn, mx, bits, step in props: - backing = '_' + prop_name[0].lower() + prop_name[1:] - backings.append(backing) - lines.append(f'{i}private float? {backing};') - lines.append(f'{i}/// Float accessor for . Range [{mn}, {mx}], {bits} bits, step {_format_step(step)}.') - lines.append(f'{i}public float {prop_name}Quantized') - lines.append(f'{i}{{') - lines.append(f'{i}{i}get => {backing} ??= Quantize.Decode({prop_name}, {mn}, {mx}, {bits});') - lines.append(f'{i}{i}set {{ {backing} = value; {prop_name} = Quantize.Encode(value, {mn}, {mx}, {bits}); }}') - lines.append(f'{i}}}') - lines.append('') - - lines.append(f'{i}/// Clears all cached decoded values. Call after mutating raw uint32 fields directly.') - lines.append(f'{i}public void ResetDecodedCache()') - lines.append(f'{i}{{') - for backing in backings: - lines.append(f'{i}{i}{backing} = null;') - lines.append(f'{i}}}') - - lines.append('}') - return lines - - -# --------------------------------------------------------------------------- -# Per-file code generation (public entry point) -# --------------------------------------------------------------------------- - -def generate_csharp(file_proto) -> dict | None: - """ - Generate a C# source file for a FileDescriptorProto. - - Returns a dict with keys 'name' (output path) and 'content' (C# source), - or None if the file contains no quantized uint32 fields. - """ - namespace = _package_to_namespace(file_proto.package) - - # Output is placed flat in the output root, matching --csharp_out convention. - # e.g. "decentraland/kernel/comms/v3/my_message.proto" - # → "MyMessage.Bitwise.cs" - proto_file = file_proto.name.rsplit('/', 1)[-1] - stem = _snake_to_pascal(proto_file.replace('.proto', '')) - out_name = f'{stem}.Bitwise.cs' - - header = [ - '// ', - '// Generated by protoc-gen-bitwise. DO NOT EDIT.', - f'// Source: {file_proto.name}', - '// ', - '', - 'using Decentraland.Networking.Bitwise;', - '', - f'namespace {namespace}', - '{', - ] - footer = [ - '', - f'}} // namespace {namespace}', - ] - - body: list[str] = [] - for msg in file_proto.message_type: - msg_lines = _gen_message(msg) - if msg_lines is None: - continue - # Indent each line by 4 spaces (inside the namespace block) - for line in msg_lines: - body.append((' ' + line) if line else '') - body.append('') - - if not body: - return None # nothing to emit - - content = '\n'.join(header + body + footer) + '\n' - return {'name': out_name, 'content': content} diff --git a/protoc-gen-bitwise/options.js b/protoc-gen-bitwise/options.js new file mode 100644 index 00000000..150ece0b --- /dev/null +++ b/protoc-gen-bitwise/options.js @@ -0,0 +1,108 @@ +'use strict' + +/** + * Parser for the custom bitwise field options defined in options.proto. + * + * The descriptor decoder hands us the raw serialized FieldOptions bytes (it + * declares `options` as opaque bytes). We walk those bytes looking for the two + * custom extension field numbers — protobuf preserves unknown/unregistered + * extension bytes, so they are always present even though no runtime here knows + * the extension schema. This mirrors the original options_pb2.py. + * + * Wire format: tag = (field_number << 3) | wire_type + * wire_type 0 = varint, 1 = 64-bit, 2 = length-delimited, 5 = 32-bit + */ + +const { readVarint, skipField } = require('./wire') + +// Extension field numbers as defined in options.proto. +const QUANTIZED_FIELD_NUMBER = 50001 +const BIT_PACKED_FIELD_NUMBER = 50002 + +/** Parse a serialized QuantizedFloatOptions message: { min, max, bits }. */ +function parseQuantized(data) { + const opts = { min: 0.0, max: 0.0, bits: 0 } + let pos = 0 + while (pos < data.length) { + let tag + ;[tag, pos] = readVarint(data, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 5) { + // min (float) + opts.min = data.readFloatLE(pos) + pos += 4 + } else if (fieldNum === 2 && wireType === 5) { + // max (float) + opts.max = data.readFloatLE(pos) + pos += 4 + } else if (fieldNum === 3 && wireType === 0) { + // bits (uint32) + ;[opts.bits, pos] = readVarint(data, pos) + } else { + pos = skipField(data, pos, wireType) + } + } + return opts +} + +/** Parse a serialized BitPackedOptions message: { bits }. */ +function parseBitPacked(data) { + const opts = { bits: 0 } + let pos = 0 + while (pos < data.length) { + let tag + ;[tag, pos] = readVarint(data, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 0) { + // bits (uint32) + ;[opts.bits, pos] = readVarint(data, pos) + } else { + pos = skipField(data, pos, wireType) + } + } + return opts +} + +/** + * Extract custom bitwise options from raw FieldOptions bytes. + * + * @param {Buffer|null} optionsRaw serialized FieldOptions, or null when unset. + * @returns {{quantized: object|null, bitPacked: object|null}} + */ +function getFieldOptions(optionsRaw) { + if (!optionsRaw || optionsRaw.length === 0) { + return { quantized: null, bitPacked: null } + } + + let quantized = null + let bitPacked = null + let pos = 0 + + while (pos < optionsRaw.length) { + let tag + ;[tag, pos] = readVarint(optionsRaw, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + + if (wireType === 2) { + let len + ;[len, pos] = readVarint(optionsRaw, pos) + const valueBytes = optionsRaw.subarray(pos, pos + len) + pos += len + if (fieldNum === QUANTIZED_FIELD_NUMBER) { + quantized = parseQuantized(valueBytes) + } else if (fieldNum === BIT_PACKED_FIELD_NUMBER) { + bitPacked = parseBitPacked(valueBytes) + } + // else: unknown length-delimited field — already consumed. + } else { + pos = skipField(optionsRaw, pos, wireType) + } + } + + return { quantized, bitPacked } +} + +module.exports = { getFieldOptions } diff --git a/protoc-gen-bitwise/options_pb2.py b/protoc-gen-bitwise/options_pb2.py deleted file mode 100644 index d3e637f2..00000000 --- a/protoc-gen-bitwise/options_pb2.py +++ /dev/null @@ -1,171 +0,0 @@ -""" -Manual parser for the custom bitwise field options defined in options.proto. - -Rather than relying on protobuf extension registration (which requires a properly -compiled _pb2 module), this module parses the raw serialized FieldOptions bytes -directly using the protobuf binary wire format. All protobuf runtimes preserve -unknown/unregistered extension bytes when round-tripping, so -field.options.SerializeToString() always contains the extension data even when -the extension is not registered in the Python runtime. - -Wire format reference: - tag = (field_number << 3) | wire_type - wire_type 0 = varint, 1 = 64-bit, 2 = length-delimited, 5 = 32-bit -""" - -import struct - -# Extension field numbers as defined in options.proto -QUANTIZED_FIELD_NUMBER = 50001 -BIT_PACKED_FIELD_NUMBER = 50002 - - -# --------------------------------------------------------------------------- -# Low-level wire-format helpers -# --------------------------------------------------------------------------- - -def _read_varint(data: bytes, pos: int): - """Decode a protobuf varint starting at *pos*. Returns (value, new_pos).""" - result = 0 - shift = 0 - while pos < len(data): - byte = data[pos] - pos += 1 - result |= (byte & 0x7F) << shift - if not (byte & 0x80): - break - shift += 7 - return result, pos - - -def _read_float32(data: bytes, pos: int): - """Decode a little-endian 32-bit float. Returns (value, new_pos).""" - value, = struct.unpack_from(' int: - """Advance *pos* past a field with the given wire_type.""" - if wire_type == 0: - _, pos = _read_varint(data, pos) - elif wire_type == 1: - pos += 8 - elif wire_type == 2: - length, pos = _read_varint(data, pos) - pos += length - elif wire_type == 5: - pos += 4 - # wire types 3 and 4 (start/end group) are deprecated; skip gracefully - return pos - - -# --------------------------------------------------------------------------- -# Option message classes -# --------------------------------------------------------------------------- - -class QuantizedFloatOptions: - """Mirrors the QuantizedFloatOptions proto message.""" - - __slots__ = ('min', 'max', 'bits') - - def __init__(self, min_val: float = 0.0, max_val: float = 0.0, bits: int = 0): - self.min = min_val - self.max = max_val - self.bits = bits - - @classmethod - def from_bytes(cls, data: bytes) -> 'QuantizedFloatOptions': - obj = cls() - pos = 0 - while pos < len(data): - tag, pos = _read_varint(data, pos) - field_num = tag >> 3 - wire_type = tag & 0x7 - if field_num == 1 and wire_type == 5: # min (float) - obj.min, pos = _read_float32(data, pos) - elif field_num == 2 and wire_type == 5: # max (float) - obj.max, pos = _read_float32(data, pos) - elif field_num == 3 and wire_type == 0: # bits (uint32) - obj.bits, pos = _read_varint(data, pos) - else: - pos = _skip_field(data, pos, wire_type) - return obj - - def __repr__(self): - return f'QuantizedFloatOptions(min={self.min}, max={self.max}, bits={self.bits})' - - -class BitPackedOptions: - """Mirrors the BitPackedOptions proto message.""" - - __slots__ = ('bits',) - - def __init__(self, bits: int = 0): - self.bits = bits - - @classmethod - def from_bytes(cls, data: bytes) -> 'BitPackedOptions': - obj = cls() - pos = 0 - while pos < len(data): - tag, pos = _read_varint(data, pos) - field_num = tag >> 3 - wire_type = tag & 0x7 - if field_num == 1 and wire_type == 0: # bits (uint32) - obj.bits, pos = _read_varint(data, pos) - else: - pos = _skip_field(data, pos, wire_type) - return obj - - def __repr__(self): - return f'BitPackedOptions(bits={self.bits})' - - -# --------------------------------------------------------------------------- -# Public API -# --------------------------------------------------------------------------- - -def get_field_options(field_options_proto): - """ - Extract custom bitwise options from a FieldDescriptorProto.options object. - - Serialises the options message to bytes and walks the wire-format stream - looking for extension fields 50001 (quantized) and 50002 (bit_packed). - - Args: - field_options_proto: google.protobuf.descriptor_pb2.FieldOptions instance - (may be a default/empty instance when no options are set). - - Returns: - tuple[QuantizedFloatOptions | None, BitPackedOptions | None] - """ - try: - raw = field_options_proto.SerializeToString() - except Exception: - return None, None - - if not raw: - return None, None - - quantized = None - bit_packed = None - pos = 0 - - while pos < len(raw): - tag, pos = _read_varint(raw, pos) - field_num = tag >> 3 - wire_type = tag & 0x7 - - if wire_type == 2: # length-delimited - length, pos = _read_varint(raw, pos) - value_bytes = raw[pos:pos + length] - pos += length - if field_num == QUANTIZED_FIELD_NUMBER: - quantized = QuantizedFloatOptions.from_bytes(value_bytes) - elif field_num == BIT_PACKED_FIELD_NUMBER: - bit_packed = BitPackedOptions.from_bytes(value_bytes) - # else: unknown length-delimited field — already consumed - else: - pos = _skip_field(raw, pos, wire_type) - - return quantized, bit_packed diff --git a/protoc-gen-bitwise/plugin.js b/protoc-gen-bitwise/plugin.js new file mode 100755 index 00000000..b08c6a12 --- /dev/null +++ b/protoc-gen-bitwise/plugin.js @@ -0,0 +1,87 @@ +#!/usr/bin/env node +'use strict' + +/** + * protoc-gen-bitwise — protoc plugin that generates C# bitwise serialization code. + * + * Protocol: + * 1. protoc writes a serialised CodeGeneratorRequest to this process's stdin. + * 2. This plugin reads it, generates C# partial classes with float accessor + * properties for every message that carries [(quantized)] field annotations. + * 3. A serialised CodeGeneratorResponse is written to stdout. + * + * Implemented in plain Node with a self-contained protobuf wire codec (see + * wire.js) so it runs with only `node` on PATH — no npm install required, even + * when invoked directly from a sibling checkout. + * + * Usage (from project root): + * protoc \ + * --proto_path=proto \ + * --bitwise_out=generated/ \ + * --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.js \ + * proto/my_messages.proto + * + * On Windows, protoc needs an executable wrapper, e.g. a .cmd that runs: + * node "\protoc-gen-bitwise\plugin.js" %* + */ + +const { decodeRequest, encodeResponse } = require('./wire') +const { generateCsharp } = require('./generator_csharp') + +// CodeGeneratorResponse.Feature.FEATURE_PROTO3_OPTIONAL — advertised so protoc +// does not reject the plugin when the schema uses proto3 `optional` fields. +const FEATURE_PROTO3_OPTIONAL = 1 + +function readStdin() { + return new Promise((resolve, reject) => { + const chunks = [] + process.stdin.on('data', (chunk) => chunks.push(chunk)) + process.stdin.on('end', () => resolve(Buffer.concat(chunks))) + process.stdin.on('error', reject) + }) +} + +async function main() { + const requestBytes = await readStdin() + const request = decodeRequest(requestBytes) + + // Lookup map for all file descriptors (parity with the Python plugin). + const fileByName = new Map() + for (const file of request.protoFile) fileByName.set(file.name, file) + + const files = [] + let error = null + + for (const fileName of request.fileToGenerate) { + // Skip the options definition file itself — it has no messages to generate. + if (fileName === 'decentraland/common/options.proto') continue + + const fileProto = fileByName.get(fileName) + if (!fileProto) continue + + try { + const generated = generateCsharp(fileProto) + if (generated) files.push(generated) + } catch (exc) { + error = `protoc-gen-bitwise: error processing ${fileName}: ${exc && exc.message ? exc.message : exc}` + } + } + + const response = encodeResponse({ + error, + supportedFeatures: FEATURE_PROTO3_OPTIONAL, + files, + }) + + // Let the write flush before the process exits (do not call process.exit()). + process.stdout.write(response) +} + +main().catch((exc) => { + const response = encodeResponse({ + error: `protoc-gen-bitwise: ${exc && exc.stack ? exc.stack : exc}`, + supportedFeatures: FEATURE_PROTO3_OPTIONAL, + files: [], + }) + process.stdout.write(response) +}) diff --git a/protoc-gen-bitwise/plugin.py b/protoc-gen-bitwise/plugin.py deleted file mode 100644 index 09b5abe4..00000000 --- a/protoc-gen-bitwise/plugin.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python3 -""" -protoc-gen-bitwise — protoc plugin that generates C# bitwise serialization code. - -Protocol: - 1. protoc writes a serialised CodeGeneratorRequest to this process's stdin. - 2. This plugin reads it, generates C# partial classes with Encode / Decode - methods for every message that carries [(quantized)] or [(bit_packed)] - field annotations. - 3. A serialised CodeGeneratorResponse is written to stdout. - -Usage (from project root): - protoc \\ - --proto_path=proto \\ - --bitwise_out=generated/ \\ - --plugin=protoc-gen-bitwise \\ - proto/my_messages.proto - -Windows invocation (plugin not on PATH as executable): - protoc \\ - --proto_path=proto \\ - --bitwise_out=generated/ \\ - --plugin=protoc-gen-bitwise=python protoc-gen-bitwise/plugin.py \\ - proto/my_messages.proto - -Dependencies: - pip install grpcio-tools # or: pip install protobuf -""" - -import os -import sys - -# Ensure sibling modules (generator_csharp, options_pb2) are importable -# regardless of where protoc invokes this script from. -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -# On Windows, stdin/stdout are opened in text mode by default which corrupts -# the binary protobuf payload. -if sys.platform == 'win32': - import msvcrt - msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) - msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) - -from google.protobuf.compiler import plugin_pb2 - -from generator_csharp import generate_csharp - - -def main() -> None: - request_bytes = sys.stdin.buffer.read() - - request = plugin_pb2.CodeGeneratorRequest() - request.ParseFromString(request_bytes) - - response = plugin_pb2.CodeGeneratorResponse() - # Advertise proto3-optional support so protoc does not reject the plugin. - response.supported_features = ( - plugin_pb2.CodeGeneratorResponse.FEATURE_PROTO3_OPTIONAL - ) - - # Build a lookup map for all file descriptors (needed for imports, though - # the current generator only uses the directly requested files). - file_by_name = {f.name: f for f in request.proto_file} - - for file_name in request.file_to_generate: - # Skip the options definition file itself — it has no messages to generate. - if file_name == 'decentraland/common/options.proto': - continue - - file_proto = file_by_name.get(file_name) - if file_proto is None: - continue - - try: - generated = generate_csharp(file_proto) - except Exception as exc: # noqa: BLE001 - error = response.file.add() - error.name = '' # empty name signals an error to protoc - # Append error text; protoc will print it and fail. - response.error = f'protoc-gen-bitwise: error processing {file_name}: {exc}' - continue - - if generated is not None: - out = response.file.add() - out.name = generated['name'] - out.content = generated['content'] - - sys.stdout.buffer.write(response.SerializeToString()) - - -if __name__ == '__main__': - main() diff --git a/protoc-gen-bitwise/test/generator.test.js b/protoc-gen-bitwise/test/generator.test.js new file mode 100644 index 00000000..95512154 --- /dev/null +++ b/protoc-gen-bitwise/test/generator.test.js @@ -0,0 +1,180 @@ +'use strict' + +/** + * Byte-for-byte parity test for the C# generator. + * + * Builds in-memory FieldDescriptorProto structures (with real serialized + * FieldOptions extension bytes), runs them through the same options parser and + * generator the plugin uses, and asserts the output equals committed golden + * files. This locks down the %g float formatting and the emitted layout without + * needing protoc. + * + * Run with: node protoc-gen-bitwise/test/generator.test.js (or `npm run gen:test`) + */ + +const assert = require('assert') +const fs = require('fs') +const path = require('path') + +const { generateCsharp } = require('../generator_csharp') + +// FieldDescriptorProto.Type +const TYPE_DOUBLE = 1 +const TYPE_BOOL = 8 +const TYPE_UINT32 = 13 +// FieldDescriptorProto.Label +const LABEL_OPTIONAL = 1 + +// --- minimal wire encoders to build serialized FieldOptions extension bytes --- + +function encodeVarint(value) { + const bytes = [] + let v = value + while (v > 0x7f) { + bytes.push((v & 0x7f) | 0x80) + v = Math.floor(v / 128) + } + bytes.push(v & 0x7f) + return Buffer.from(bytes) +} + +function tag(fieldNum, wireType) { + return encodeVarint((fieldNum << 3) | wireType) +} + +function floatLE(value) { + const b = Buffer.alloc(4) + b.writeFloatLE(value, 0) + return b +} + +function lengthDelimited(fieldNum, payload) { + return Buffer.concat([tag(fieldNum, 2), encodeVarint(payload.length), payload]) +} + +// FieldOptions bytes carrying ext 50001 (quantized) = { min, max, bits } +function quantizedOptions(min, max, bits) { + const inner = Buffer.concat([ + tag(1, 5), floatLE(min), + tag(2, 5), floatLE(max), + tag(3, 0), encodeVarint(bits), + ]) + return lengthDelimited(50001, inner) +} + +// FieldOptions bytes carrying ext 50002 (bit_packed) = { bits } +function bitPackedOptions(bits) { + const inner = Buffer.concat([tag(1, 0), encodeVarint(bits)]) + return lengthDelimited(50002, inner) +} + +function field(name, type, optionsRaw) { + return { name, label: LABEL_OPTIONAL, type, optionsRaw: optionsRaw || null } +} + +function readGolden(name) { + return fs.readFileSync(path.join(__dirname, 'golden', name), 'utf8') +} + +// -------------------------------------------------------------------------- +// Case 1: quantization_example.proto +// -------------------------------------------------------------------------- + +const quantizationExample = { + name: 'decentraland/common/quantization_example.proto', + package: 'decentraland.common', + messageType: [ + { + name: 'PositionDelta', + field: [ + field('dx', TYPE_UINT32, quantizedOptions(-100, 100, 16)), + field('dy', TYPE_UINT32, quantizedOptions(-100, 100, 16)), + field('dz', TYPE_UINT32, quantizedOptions(-100, 100, 16)), + field('entity_id', TYPE_UINT32, bitPackedOptions(20)), + field('sequence', TYPE_UINT32, bitPackedOptions(12)), + ], + }, + { + name: 'PlayerInput', + field: [ + field('move_x', TYPE_UINT32, quantizedOptions(-1, 1, 8)), + field('move_z', TYPE_UINT32, quantizedOptions(-1, 1, 8)), + field('yaw', TYPE_UINT32, quantizedOptions(-180, 180, 12)), + field('buttons', TYPE_UINT32, bitPackedOptions(8)), + field('sequence', TYPE_UINT32, bitPackedOptions(12)), + ], + }, + { + name: 'AvatarStateSnapshot', + field: [ + field('x', TYPE_UINT32, quantizedOptions(-4096, 4096, 16)), + field('y', TYPE_UINT32, quantizedOptions(-256, 256, 14)), + field('z', TYPE_UINT32, quantizedOptions(-4096, 4096, 16)), + field('pitch', TYPE_UINT32, quantizedOptions(-90, 90, 10)), + field('yaw', TYPE_UINT32, quantizedOptions(-180, 180, 12)), + field('entity_id', TYPE_UINT32, bitPackedOptions(20)), + field('animation_state', TYPE_UINT32, bitPackedOptions(6)), + field('is_grounded', TYPE_BOOL, null), + field('timestamp', TYPE_DOUBLE, null), + ], + }, + ], +} + +// -------------------------------------------------------------------------- +// Case 2: pulse_server.proto (PlayerStateDeltaTier0) — wider bit/range coverage +// -------------------------------------------------------------------------- + +const pulseServer = { + name: 'decentraland/pulse/pulse_server.proto', + package: 'decentraland.pulse', + messageType: [ + { + name: 'PlayerStateDeltaTier0', + field: [ + field('position_x', TYPE_UINT32, quantizedOptions(0, 16, 8)), + field('position_y', TYPE_UINT32, quantizedOptions(0, 200, 13)), + field('position_z', TYPE_UINT32, quantizedOptions(0, 16, 8)), + field('velocity_x', TYPE_UINT32, quantizedOptions(-50, 50, 8)), + field('velocity_y', TYPE_UINT32, quantizedOptions(-50, 50, 8)), + field('velocity_z', TYPE_UINT32, quantizedOptions(-50, 50, 8)), + field('rotation_y', TYPE_UINT32, quantizedOptions(0, 360, 7)), + field('movement_blend', TYPE_UINT32, quantizedOptions(0, 3, 5)), + field('slide_blend', TYPE_UINT32, quantizedOptions(0, 1, 4)), + field('head_yaw', TYPE_UINT32, quantizedOptions(0, 360, 7)), + field('head_pitch', TYPE_UINT32, quantizedOptions(0, 360, 7)), + field('point_at_x', TYPE_UINT32, quantizedOptions(-3000, 3000, 17)), + field('point_at_y', TYPE_UINT32, quantizedOptions(0, 200, 7)), + field('point_at_z', TYPE_UINT32, quantizedOptions(-3000, 3000, 17)), + ], + }, + ], +} + +// -------------------------------------------------------------------------- + +const cases = [ + { proto: quantizationExample, golden: 'QuantizationExample.Bitwise.cs' }, + { proto: pulseServer, golden: 'PulseServer.Bitwise.cs' }, +] + +let failed = 0 +for (const { proto, golden } of cases) { + const result = generateCsharp(proto) + assert.ok(result, `expected output for ${golden}`) + assert.strictEqual(result.name, golden, `output filename for ${golden}`) + try { + assert.strictEqual(result.content, readGolden(golden)) + console.log(`ok - ${golden} matches golden`) + } catch (e) { + failed++ + console.error(`FAIL - ${golden} differs from golden`) + console.error(e.message) + } +} + +if (failed > 0) { + console.error(`\n${failed} golden mismatch(es)`) + process.exit(1) +} +console.log('\nAll bitwise generator golden tests passed.') diff --git a/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs b/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs new file mode 100644 index 00000000..255ba7bf --- /dev/null +++ b/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs @@ -0,0 +1,145 @@ +// +// Generated by protoc-gen-bitwise. DO NOT EDIT. +// Source: decentraland/pulse/pulse_server.proto +// + +using Decentraland.Networking.Bitwise; + +namespace Decentraland.Pulse +{ + public partial class PlayerStateDeltaTier0 + { + private float? _positionX; + /// Float accessor for . Range [0.0f, 16.0f], 8 bits, step ≈ 0.0627451. + public float PositionXQuantized + { + get => _positionX ??= Quantize.Decode(PositionX, 0.0f, 16.0f, 8); + set { _positionX = value; PositionX = Quantize.Encode(value, 0.0f, 16.0f, 8); } + } + + private float? _positionY; + /// Float accessor for . Range [0.0f, 200.0f], 13 bits, step ≈ 0.024417. + public float PositionYQuantized + { + get => _positionY ??= Quantize.Decode(PositionY, 0.0f, 200.0f, 13); + set { _positionY = value; PositionY = Quantize.Encode(value, 0.0f, 200.0f, 13); } + } + + private float? _positionZ; + /// Float accessor for . Range [0.0f, 16.0f], 8 bits, step ≈ 0.0627451. + public float PositionZQuantized + { + get => _positionZ ??= Quantize.Decode(PositionZ, 0.0f, 16.0f, 8); + set { _positionZ = value; PositionZ = Quantize.Encode(value, 0.0f, 16.0f, 8); } + } + + private float? _velocityX; + /// Float accessor for . Range [-50.0f, 50.0f], 8 bits, step ≈ 0.392157. + public float VelocityXQuantized + { + get => _velocityX ??= Quantize.Decode(VelocityX, -50.0f, 50.0f, 8); + set { _velocityX = value; VelocityX = Quantize.Encode(value, -50.0f, 50.0f, 8); } + } + + private float? _velocityY; + /// Float accessor for . Range [-50.0f, 50.0f], 8 bits, step ≈ 0.392157. + public float VelocityYQuantized + { + get => _velocityY ??= Quantize.Decode(VelocityY, -50.0f, 50.0f, 8); + set { _velocityY = value; VelocityY = Quantize.Encode(value, -50.0f, 50.0f, 8); } + } + + private float? _velocityZ; + /// Float accessor for . Range [-50.0f, 50.0f], 8 bits, step ≈ 0.392157. + public float VelocityZQuantized + { + get => _velocityZ ??= Quantize.Decode(VelocityZ, -50.0f, 50.0f, 8); + set { _velocityZ = value; VelocityZ = Quantize.Encode(value, -50.0f, 50.0f, 8); } + } + + private float? _rotationY; + /// Float accessor for . Range [0.0f, 360.0f], 7 bits, step ≈ 2.83465. + public float RotationYQuantized + { + get => _rotationY ??= Quantize.Decode(RotationY, 0.0f, 360.0f, 7); + set { _rotationY = value; RotationY = Quantize.Encode(value, 0.0f, 360.0f, 7); } + } + + private float? _movementBlend; + /// Float accessor for . Range [0.0f, 3.0f], 5 bits, step ≈ 0.0967742. + public float MovementBlendQuantized + { + get => _movementBlend ??= Quantize.Decode(MovementBlend, 0.0f, 3.0f, 5); + set { _movementBlend = value; MovementBlend = Quantize.Encode(value, 0.0f, 3.0f, 5); } + } + + private float? _slideBlend; + /// Float accessor for . Range [0.0f, 1.0f], 4 bits, step ≈ 0.0666667. + public float SlideBlendQuantized + { + get => _slideBlend ??= Quantize.Decode(SlideBlend, 0.0f, 1.0f, 4); + set { _slideBlend = value; SlideBlend = Quantize.Encode(value, 0.0f, 1.0f, 4); } + } + + private float? _headYaw; + /// Float accessor for . Range [0.0f, 360.0f], 7 bits, step ≈ 2.83465. + public float HeadYawQuantized + { + get => _headYaw ??= Quantize.Decode(HeadYaw, 0.0f, 360.0f, 7); + set { _headYaw = value; HeadYaw = Quantize.Encode(value, 0.0f, 360.0f, 7); } + } + + private float? _headPitch; + /// Float accessor for . Range [0.0f, 360.0f], 7 bits, step ≈ 2.83465. + public float HeadPitchQuantized + { + get => _headPitch ??= Quantize.Decode(HeadPitch, 0.0f, 360.0f, 7); + set { _headPitch = value; HeadPitch = Quantize.Encode(value, 0.0f, 360.0f, 7); } + } + + private float? _pointAtX; + /// Float accessor for . Range [-3000.0f, 3000.0f], 17 bits, step ≈ 0.0457767. + public float PointAtXQuantized + { + get => _pointAtX ??= Quantize.Decode(PointAtX, -3000.0f, 3000.0f, 17); + set { _pointAtX = value; PointAtX = Quantize.Encode(value, -3000.0f, 3000.0f, 17); } + } + + private float? _pointAtY; + /// Float accessor for . Range [0.0f, 200.0f], 7 bits, step ≈ 1.5748. + public float PointAtYQuantized + { + get => _pointAtY ??= Quantize.Decode(PointAtY, 0.0f, 200.0f, 7); + set { _pointAtY = value; PointAtY = Quantize.Encode(value, 0.0f, 200.0f, 7); } + } + + private float? _pointAtZ; + /// Float accessor for . Range [-3000.0f, 3000.0f], 17 bits, step ≈ 0.0457767. + public float PointAtZQuantized + { + get => _pointAtZ ??= Quantize.Decode(PointAtZ, -3000.0f, 3000.0f, 17); + set { _pointAtZ = value; PointAtZ = Quantize.Encode(value, -3000.0f, 3000.0f, 17); } + } + + /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. + public void ResetDecodedCache() + { + _positionX = null; + _positionY = null; + _positionZ = null; + _velocityX = null; + _velocityY = null; + _velocityZ = null; + _rotationY = null; + _movementBlend = null; + _slideBlend = null; + _headYaw = null; + _headPitch = null; + _pointAtX = null; + _pointAtY = null; + _pointAtZ = null; + } + } + + +} // namespace Decentraland.Pulse diff --git a/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs b/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs new file mode 100644 index 00000000..c0456936 --- /dev/null +++ b/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs @@ -0,0 +1,134 @@ +// +// Generated by protoc-gen-bitwise. DO NOT EDIT. +// Source: decentraland/common/quantization_example.proto +// + +using Decentraland.Networking.Bitwise; + +namespace Decentraland.Common +{ + public partial class PositionDelta + { + private float? _dx; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. + public float DxQuantized + { + get => _dx ??= Quantize.Decode(Dx, -100.0f, 100.0f, 16); + set { _dx = value; Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + private float? _dy; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. + public float DyQuantized + { + get => _dy ??= Quantize.Decode(Dy, -100.0f, 100.0f, 16); + set { _dy = value; Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + private float? _dz; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. + public float DzQuantized + { + get => _dz ??= Quantize.Decode(Dz, -100.0f, 100.0f, 16); + set { _dz = value; Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. + public void ResetDecodedCache() + { + _dx = null; + _dy = null; + _dz = null; + } + } + + public partial class PlayerInput + { + private float? _moveX; + /// Float accessor for . Range [-1.0f, 1.0f], 8 bits, step ≈ 0.00784314. + public float MoveXQuantized + { + get => _moveX ??= Quantize.Decode(MoveX, -1.0f, 1.0f, 8); + set { _moveX = value; MoveX = Quantize.Encode(value, -1.0f, 1.0f, 8); } + } + + private float? _moveZ; + /// Float accessor for . Range [-1.0f, 1.0f], 8 bits, step ≈ 0.00784314. + public float MoveZQuantized + { + get => _moveZ ??= Quantize.Decode(MoveZ, -1.0f, 1.0f, 8); + set { _moveZ = value; MoveZ = Quantize.Encode(value, -1.0f, 1.0f, 8); } + } + + private float? _yaw; + /// Float accessor for . Range [-180.0f, 180.0f], 12 bits, step ≈ 0.0879121. + public float YawQuantized + { + get => _yaw ??= Quantize.Decode(Yaw, -180.0f, 180.0f, 12); + set { _yaw = value; Yaw = Quantize.Encode(value, -180.0f, 180.0f, 12); } + } + + /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. + public void ResetDecodedCache() + { + _moveX = null; + _moveZ = null; + _yaw = null; + } + } + + public partial class AvatarStateSnapshot + { + private float? _x; + /// Float accessor for . Range [-4096.0f, 4096.0f], 16 bits, step ≈ 0.125002. + public float XQuantized + { + get => _x ??= Quantize.Decode(X, -4096.0f, 4096.0f, 16); + set { _x = value; X = Quantize.Encode(value, -4096.0f, 4096.0f, 16); } + } + + private float? _y; + /// Float accessor for . Range [-256.0f, 256.0f], 14 bits, step ≈ 0.0312519. + public float YQuantized + { + get => _y ??= Quantize.Decode(Y, -256.0f, 256.0f, 14); + set { _y = value; Y = Quantize.Encode(value, -256.0f, 256.0f, 14); } + } + + private float? _z; + /// Float accessor for . Range [-4096.0f, 4096.0f], 16 bits, step ≈ 0.125002. + public float ZQuantized + { + get => _z ??= Quantize.Decode(Z, -4096.0f, 4096.0f, 16); + set { _z = value; Z = Quantize.Encode(value, -4096.0f, 4096.0f, 16); } + } + + private float? _pitch; + /// Float accessor for . Range [-90.0f, 90.0f], 10 bits, step ≈ 0.175953. + public float PitchQuantized + { + get => _pitch ??= Quantize.Decode(Pitch, -90.0f, 90.0f, 10); + set { _pitch = value; Pitch = Quantize.Encode(value, -90.0f, 90.0f, 10); } + } + + private float? _yaw; + /// Float accessor for . Range [-180.0f, 180.0f], 12 bits, step ≈ 0.0879121. + public float YawQuantized + { + get => _yaw ??= Quantize.Decode(Yaw, -180.0f, 180.0f, 12); + set { _yaw = value; Yaw = Quantize.Encode(value, -180.0f, 180.0f, 12); } + } + + /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. + public void ResetDecodedCache() + { + _x = null; + _y = null; + _z = null; + _pitch = null; + _yaw = null; + } + } + + +} // namespace Decentraland.Common diff --git a/protoc-gen-bitwise/wire.js b/protoc-gen-bitwise/wire.js new file mode 100644 index 00000000..e2bd2d57 --- /dev/null +++ b/protoc-gen-bitwise/wire.js @@ -0,0 +1,239 @@ +'use strict' + +/** + * Self-contained protobuf wire-format codec for protoc-gen-bitwise. + * + * The plugin only needs a tiny slice of the descriptor/plugin schemas, so + * rather than pull in a protobuf runtime (which would force every consumer — + * including the sibling Pulse checkout that runs this file directly off disk — + * to `npm install` the protocol repo) we decode the handful of fields we care + * about by walking the wire format directly. This mirrors the original Python + * plugin, which already hand-parsed FieldOptions for the same reason. + * + * Decoded subset: + * CodeGeneratorRequest { file_to_generate = 1; proto_file = 15; } + * FileDescriptorProto { name = 1; package = 2; message_type = 4; } + * DescriptorProto { name = 1; field = 2; } + * FieldDescriptorProto { name = 1; label = 4; type = 5; options = 8 (raw bytes); } + * + * Encoded subset: + * CodeGeneratorResponse { error = 1; supported_features = 2; file = 15; } + * CodeGeneratorResponse.File { name = 1; content = 15; } + * + * Wire types: 0 = varint, 1 = 64-bit, 2 = length-delimited, 5 = 32-bit. + */ + +// --------------------------------------------------------------------------- +// Low-level readers +// --------------------------------------------------------------------------- + +/** Decode a protobuf varint at `pos`. Returns [value, newPos]. */ +function readVarint(buf, pos) { + let result = 0 + let shift = 0 + let byte + do { + byte = buf[pos++] + // Multiplication (not <<) keeps values correct past 32 bits. + result += (byte & 0x7f) * Math.pow(2, shift) + shift += 7 + } while (byte & 0x80) + return [result, pos] +} + +/** Advance `pos` past a field of the given wire type. Returns newPos. */ +function skipField(buf, pos, wireType) { + switch (wireType) { + case 0: // varint + return readVarint(buf, pos)[1] + case 1: // 64-bit + return pos + 8 + case 2: { + // length-delimited + const [length, next] = readVarint(buf, pos) + return next + length + } + case 5: // 32-bit + return pos + 4 + default: + // Groups (3/4) are deprecated and never appear in descriptors. + return pos + } +} + +// --------------------------------------------------------------------------- +// Request decoding +// --------------------------------------------------------------------------- + +function decodeFieldDescriptor(buf) { + const field = { name: '', label: 0, type: 0, optionsRaw: null } + let pos = 0 + while (pos < buf.length) { + let tag + ;[tag, pos] = readVarint(buf, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + field.name = buf.toString('utf8', pos, pos + len) + pos += len + } else if (fieldNum === 4 && wireType === 0) { + ;[field.label, pos] = readVarint(buf, pos) + } else if (fieldNum === 5 && wireType === 0) { + ;[field.type, pos] = readVarint(buf, pos) + } else if (fieldNum === 8 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + // Keep the raw FieldOptions bytes so custom extensions survive (the + // descriptor schema doesn't know about ext 50001/50002). + field.optionsRaw = buf.subarray(pos, pos + len) + pos += len + } else { + pos = skipField(buf, pos, wireType) + } + } + return field +} + +function decodeDescriptor(buf) { + const message = { name: '', field: [] } + let pos = 0 + while (pos < buf.length) { + let tag + ;[tag, pos] = readVarint(buf, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + message.name = buf.toString('utf8', pos, pos + len) + pos += len + } else if (fieldNum === 2 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + message.field.push(decodeFieldDescriptor(buf.subarray(pos, pos + len))) + pos += len + } else { + // nested_type / enum_type / etc. are intentionally ignored — the + // generator only walks top-level messages, matching the Python plugin. + pos = skipField(buf, pos, wireType) + } + } + return message +} + +function decodeFileDescriptor(buf) { + const file = { name: '', package: '', messageType: [] } + let pos = 0 + while (pos < buf.length) { + let tag + ;[tag, pos] = readVarint(buf, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + file.name = buf.toString('utf8', pos, pos + len) + pos += len + } else if (fieldNum === 2 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + file.package = buf.toString('utf8', pos, pos + len) + pos += len + } else if (fieldNum === 4 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + file.messageType.push(decodeDescriptor(buf.subarray(pos, pos + len))) + pos += len + } else { + pos = skipField(buf, pos, wireType) + } + } + return file +} + +/** Decode a serialized CodeGeneratorRequest. */ +function decodeRequest(buf) { + const request = { fileToGenerate: [], protoFile: [] } + let pos = 0 + while (pos < buf.length) { + let tag + ;[tag, pos] = readVarint(buf, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + request.fileToGenerate.push(buf.toString('utf8', pos, pos + len)) + pos += len + } else if (fieldNum === 15 && wireType === 2) { + let len + ;[len, pos] = readVarint(buf, pos) + request.protoFile.push(decodeFileDescriptor(buf.subarray(pos, pos + len))) + pos += len + } else { + pos = skipField(buf, pos, wireType) + } + } + return request +} + +// --------------------------------------------------------------------------- +// Response encoding +// --------------------------------------------------------------------------- + +/** Encode an unsigned integer as a protobuf varint Buffer. */ +function encodeVarint(value) { + const bytes = [] + let v = value + while (v > 0x7f) { + bytes.push((v & 0x7f) | 0x80) + v = Math.floor(v / 128) + } + bytes.push(v & 0x7f) + return Buffer.from(bytes) +} + +function encodeTag(fieldNum, wireType) { + return encodeVarint((fieldNum << 3) | wireType) +} + +/** Encode a length-delimited (string/bytes) field: tag + length + payload. */ +function encodeLengthDelimited(fieldNum, payload) { + const buf = Buffer.isBuffer(payload) ? payload : Buffer.from(payload, 'utf8') + return Buffer.concat([encodeTag(fieldNum, 2), encodeVarint(buf.length), buf]) +} + +function encodeFile(file) { + return Buffer.concat([ + encodeLengthDelimited(1, file.name), // name = 1 + encodeLengthDelimited(15, file.content), // content = 15 + ]) +} + +/** + * Encode a CodeGeneratorResponse. + * @param {{error?: string|null, supportedFeatures?: number, files?: Array<{name:string,content:string}>}} response + */ +function encodeResponse(response) { + const parts = [] + if (response.error != null) { + parts.push(encodeLengthDelimited(1, response.error)) // error = 1 + } + if (response.supportedFeatures != null) { + parts.push(encodeTag(2, 0)) // supported_features = 2 (varint) + parts.push(encodeVarint(response.supportedFeatures)) + } + for (const file of response.files || []) { + parts.push(encodeLengthDelimited(15, encodeFile(file))) // file = 15 + } + return Buffer.concat(parts) +} + +module.exports = { + readVarint, + skipField, + decodeRequest, + encodeResponse, +} From 74c59083854347e390048cff596fe52c7388694e Mon Sep 17 00:00:00 2001 From: robtfm <50659922+robtfm@users.noreply.github.com> Date: Fri, 3 Jul 2026 14:14:50 +0100 Subject: [PATCH 48/54] feat: power-law quantized float option; apply to Pulse velocity deltas (#431) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: power-law quantized float option; apply to Pulse velocity deltas Add a new `(decentraland.common.quantized_power)` field option for uint32 fields: an `(bits-1)`-bit linear unorm magnitude `u` (high bits) plus a sign (LSB), decoded as `sign * max * u^pow`. Unlike the linear `quantized` option, zero is exactly representable (`u=0` -> `0`) and `pow > 1` concentrates resolution near zero. Putting the sign in the LSB (`(magnitude << 1) | sign`) makes the varint cost track magnitude rather than direction — a small `|value|` of either sign stays in one varint byte, and a stopped axis canonicalizes to 0 (omitted by proto3). `PlayerStateDeltaTier0.velocity_x/y/z` switch from `quantized{min:-50,max:50,bits:8}` to `quantized_power{max:50,pow:2,bits:8}`. The linear quantizer could not represent an exact 0 over [-50,50] in 8 bits (±0.196 residual), which drove a steady drift on stopped foreign avatars; the power curve fixes that and spends its bits on the low speeds where avatars actually move. Implemented end to end against the Node protoc-gen-bitwise plugin: options.proto message + extension (50003), the options.js parser, generator_csharp.js dispatch, the C# runtime (Quantize.EncodePower/DecodePower), golden-fixture coverage (test helper + regenerated PulseServer/QuantizationExample goldens), plus the README, CLAUDE.md and quantization_example.proto docs. Breaking wire change for velocity_x/y/z: server (pulse), Unity, godot and bevy must regenerate and deploy in lockstep. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: robtfm <50659922+robtfm@users.noreply.github.com> * Quantize PlayerState identically to PlayerStateDeltaTier0 Signed-off-by: Mikhail Agapov * Add "step" const to the quantization gen Signed-off-by: Mikhail Agapov --------- Signed-off-by: robtfm <50659922+robtfm@users.noreply.github.com> Signed-off-by: Mikhail Agapov Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: Mikhail Agapov --- CLAUDE.md | 14 ++++- README.md | 1 + proto/decentraland/common/options.proto | 21 +++++++ .../common/quantization_example.proto | 32 +++++++++++ proto/decentraland/pulse/pulse_server.proto | 10 +++- proto/decentraland/pulse/pulse_shared.proto | 33 +++++++---- protoc-gen-bitwise/generator_csharp.js | 57 +++++++++++++------ protoc-gen-bitwise/options.js | 39 +++++++++++-- protoc-gen-bitwise/runtime/cs/Quantize.cs | 35 ++++++++++++ protoc-gen-bitwise/test/generator.test.js | 24 +++++++- .../test/golden/PulseServer.Bitwise.cs | 18 +++--- .../golden/QuantizationExample.Bitwise.cs | 35 ++++++++++++ 12 files changed, 268 insertions(+), 51 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 130756ee..03a2a374 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,13 +32,23 @@ message QuantizedFloatOptions { uint32 bits = 3; } +// Signed power-law quantizer: an (bits-1)-bit magnitude (high bits) plus a sign +// (LSB), decoded as sign * max * u^pow. Exact zero; pow>1 concentrates resolution +// near zero; sign in the LSB keeps small magnitudes in one varint byte. +message QuantizedPowerFloatOptions { + float max = 1; + float pow = 2; + uint32 bits = 3; +} + message BitPackedOptions { uint32 bits = 1; } extend google.protobuf.FieldOptions { - QuantizedFloatOptions quantized = 50001; - BitPackedOptions bit_packed = 50002; + QuantizedFloatOptions quantized = 50001; + BitPackedOptions bit_packed = 50002; + QuantizedPowerFloatOptions quantized_power = 50003; } ``` diff --git a/README.md b/README.md index b11c3302..be89895f 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,7 @@ message PositionDelta { | Annotation | Target type | Parameters | Effect | |---|---|---|---| | `[(decentraland.common.quantized)]` | `uint32` | `min`, `max`, `bits` | Plugin emits a cached `float {Name}Quantized` accessor | +| `[(decentraland.common.quantized_power)]` | `uint32` | `max`, `pow`, `bits` | Power-law quantizer over `[-max, max]`: `(bits-1)`-bit magnitude (high bits) + sign (LSB), decoded as `sign·max·u^pow`. Exact zero; `pow>1` gives fine resolution near zero, coarse near `±max`; sign in the LSB keeps small magnitudes one varint byte. Cached `float {Name}Quantized` accessor (`Quantize.EncodePower`/`DecodePower`) | | `[(decentraland.common.bit_packed)]` | `uint32` | `bits` | Documents the value range; protobuf handles varint compaction automatically | ### Wire cost at worst-case (all bits set) diff --git a/proto/decentraland/common/options.proto b/proto/decentraland/common/options.proto index 8e911547..8e4a3517 100644 --- a/proto/decentraland/common/options.proto +++ b/proto/decentraland/common/options.proto @@ -14,6 +14,22 @@ message QuantizedFloatOptions { uint32 bits = 3; } +// Options for quantizing a signed float with a power-law curve, stored as a uint32 field. +// The value is split into an (bits-1)-bit linear unorm magnitude `u in [0, 1]` plus a sign, +// and reconstructed as `value = sign * max * u^pow`. Because `u = 0` maps to exactly `0`, zero is +// representable (unlike the linear quantizer, whose codes straddle zero), and `pow > 1` concentrates +// resolution near zero — useful for fields like velocity that need fine detail at low magnitudes and +// only coarse detail near the extremes. The range is symmetric ([-max, max]); there is no `min`. +// The encoded layout puts the magnitude in the high bits and the sign in the LSB +// (`(magnitude << 1) | sign`), so the varint cost tracks magnitude, not direction: a small `|value|` +// of either sign stays in one varint byte. Zero canonicalizes to `0`, so proto3 omits a stopped field. +// The generator emits a float {Name}Quantized accessor, same as the linear option. +message QuantizedPowerFloatOptions { + float max = 1; + float pow = 2; + uint32 bits = 3; +} + // Options for bit-packing an integer field into fewer than 32 bits. message BitPackedOptions { uint32 bits = 1; @@ -27,4 +43,9 @@ extend google.protobuf.FieldOptions { // Apply to uint32 fields to pack into fewer bits. // Example: uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; BitPackedOptions bit_packed = 50002; + + // Apply to uint32 fields to enable power-law quantized float encoding (sign + magnitude curve). + // A field carries either `quantized` or `quantized_power`, not both. + // Example: uint32 velocity_x = 9 [(decentraland.common.quantized_power) = { max: 50.0, pow: 2.0, bits: 8 }]; + QuantizedPowerFloatOptions quantized_power = 50003; } diff --git a/proto/decentraland/common/quantization_example.proto b/proto/decentraland/common/quantization_example.proto index a82e430c..e8092f04 100644 --- a/proto/decentraland/common/quantization_example.proto +++ b/proto/decentraland/common/quantization_example.proto @@ -15,6 +15,13 @@ import "decentraland/common/options.proto"; // → protobuf encodes the uint32 as a varint: values ≤ 2^14-1 cost 2 bytes, // values ≤ 2^21-1 cost 3 bytes; the generated partial class adds a // float {Name}Quantized accessor that encodes/decodes transparently. +// [(decentraland.common.quantized_power) = { max: F, pow: F, bits: N }] +// → stores a signed float over [-max, max] as an (N-1)-bit linear unorm +// magnitude (high bits) plus a sign (LSB), reconstructed as sign·max·u^pow. +// Zero is exact, and pow>1 buys fine resolution near zero at the cost of +// coarse near ±max. Putting the sign in the LSB keeps a small |value| of +// either direction in one varint byte. Same generated float {Name}Quantized +// accessor as the linear option. // [(decentraland.common.bit_packed) = { bits: N }] // → documents that this uint32 uses at most N bits; protobuf encodes it // as a varint (same savings, no generated accessor needed). @@ -82,6 +89,31 @@ message PlayerInput { uint32 sequence = 5 [(decentraland.common.bit_packed) = { bits: 12 }]; } +// --------------------------------------------------------------------------- +// VelocityState — per-axis velocity, demonstrating the power-law quantizer. +// +// Velocity needs an exact zero (a stopped avatar must read 0, not a quantization +// residual, or it drifts) and fine resolution at locomotion speeds, but only +// coarse resolution near the ±50 m/s extremes. The linear quantizer can give +// none of these from 8 bits: its codes straddle zero (±0.196) and spend uniform +// resolution on speeds that never occur. quantized_power solves all three. +// +// Field | Type | Range | pow | bits | Wire worst-case +// -------|--------|------------|-----|------|----------------------- +// vx | uint32 | [-50, 50] | 2 | 8 | tag 1B + varint 1–2B = 2–3B +// vy | uint32 | [-50, 50] | 2 | 8 | tag 1B + varint 1–2B = 2–3B +// vz | uint32 | [-50, 50] | 2 | 8 | tag 1B + varint 1–2B = 2–3B +// --------------------------------------------------------------------------- +// Step: ≈ 0.003 m/s near zero, ≈ 0.78 m/s near ±50 (vs. 0.392 uniform linear) +// Wire: sign in the LSB, so |v| ≲ 12.5 m/s (magnitude code ≤ 63) costs a single +// varint byte regardless of direction; only faster axes spill to 2 bytes. +// --------------------------------------------------------------------------- +message VelocityState { + uint32 vx = 1 [(decentraland.common.quantized_power) = { max: 50.0, pow: 2.0, bits: 8 }]; + uint32 vy = 2 [(decentraland.common.quantized_power) = { max: 50.0, pow: 2.0, bits: 8 }]; + uint32 vz = 3 [(decentraland.common.quantized_power) = { max: 50.0, pow: 2.0, bits: 8 }]; +} + // Bitmask values for PlayerInput.buttons. enum ButtonFlags { BF_NONE = 0; diff --git a/proto/decentraland/pulse/pulse_server.proto b/proto/decentraland/pulse/pulse_server.proto index 8311e737..1a9ca80e 100644 --- a/proto/decentraland/pulse/pulse_server.proto +++ b/proto/decentraland/pulse/pulse_server.proto @@ -37,9 +37,13 @@ message PlayerStateDeltaTier0 { // Z position inside the parcel optional uint32 position_z = 8 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; - optional uint32 velocity_x = 9 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; - optional uint32 velocity_y = 10 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; - optional uint32 velocity_z = 11 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; + // Power-law quantized: a sign bit + 7-bit magnitude curve over [-50, 50]. Zero is exactly + // representable (a stopped peer reports an exact 0 instead of the linear quantizer's ±0.196 + // residual that drove foreign-avatar drift), and pow=2 concentrates resolution at the low speeds + // where avatars actually move, leaving only the visually-irrelevant extremes coarse. + optional uint32 velocity_x = 9 [(decentraland.common.quantized_power) = { max: 50, pow: 2, bits: 8 }]; + optional uint32 velocity_y = 10 [(decentraland.common.quantized_power) = { max: 50, pow: 2, bits: 8 }]; + optional uint32 velocity_z = 11 [(decentraland.common.quantized_power) = { max: 50, pow: 2, bits: 8 }]; optional uint32 rotation_y = 12 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; optional uint32 movement_blend = 13 [(decentraland.common.quantized) = { min: 0, max: 3, bits: 5 }]; diff --git a/proto/decentraland/pulse/pulse_shared.proto b/proto/decentraland/pulse/pulse_shared.proto index 6f4fba7c..cc027f3c 100644 --- a/proto/decentraland/pulse/pulse_shared.proto +++ b/proto/decentraland/pulse/pulse_shared.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package decentraland.pulse; -import "decentraland/common/vectors.proto"; +import "decentraland/common/options.proto"; enum PlayerAnimationFlags { NONE = 0; @@ -23,26 +23,35 @@ enum GlideState { CLOSING_PROP = 3; } +// Quantized identically to PlayerStateDeltaTier0 so a full state and a stream of deltas land on the same grid. message PlayerState { int32 parcel_index = 1; - decentraland.common.Vector3 position = 2; - decentraland.common.Vector3 velocity = 3; + // Position inside the parcel (X/Z) and world altitude (Y). + uint32 position_x = 2 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + uint32 position_y = 3 [(decentraland.common.quantized) = { min: 0, max: 200, bits: 13 }]; + uint32 position_z = 4 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; - float rotation_y = 4; + uint32 velocity_x = 5 [(decentraland.common.quantized_power) = { max: 50, pow: 2, bits: 8 }]; + uint32 velocity_y = 6 [(decentraland.common.quantized_power) = { max: 50, pow: 2, bits: 8 }]; + uint32 velocity_z = 7 [(decentraland.common.quantized_power) = { max: 50, pow: 2, bits: 8 }]; - float movement_blend = 5; - float slide_blend = 6; + uint32 rotation_y = 8 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; - optional float head_yaw = 7; - optional float head_pitch = 8; + uint32 movement_blend = 9 [(decentraland.common.quantized) = { min: 0, max: 3, bits: 5 }]; + uint32 slide_blend = 10 [(decentraland.common.quantized) = { min: 0, max: 1, bits: 4 }]; - uint32 state_flags = 9; - GlideState glide_state = 10; + optional uint32 head_yaw = 11 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + optional uint32 head_pitch = 12 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; - int32 jump_count = 11; + uint32 state_flags = 13; + GlideState glide_state = 14; + + int32 jump_count = 15; // Absolute world hit position the player is pointing at. // Only meaningful when POINTING_AT is set in `state_flags`. - optional decentraland.common.Vector3 point_at = 12; + optional uint32 point_at_x = 16 [(decentraland.common.quantized) = { min: -3000.0, max: 3000.0, bits: 17 }]; + optional uint32 point_at_y = 17 [(decentraland.common.quantized) = { min: 0.0, max: 200.0, bits: 7 }]; + optional uint32 point_at_z = 18 [(decentraland.common.quantized) = { min: -3000.0, max: 3000.0, bits: 17 }]; } diff --git a/protoc-gen-bitwise/generator_csharp.js b/protoc-gen-bitwise/generator_csharp.js index dcdb95a0..93b23c26 100644 --- a/protoc-gen-bitwise/generator_csharp.js +++ b/protoc-gen-bitwise/generator_csharp.js @@ -118,18 +118,39 @@ function generateMessage(msgProto, indent) { // Only uint32 fields are candidates for quantized accessors. if (field.type !== TYPE_UINT32) continue - const { quantized } = getFieldOptions(field.optionsRaw) - if (quantized === null) continue - - const step = (quantized.max - quantized.min) / ((1 << quantized.bits) - 1) - - props.push({ - propName: snakeToPascal(field.name), - mn: formatFloat(quantized.min), - mx: formatFloat(quantized.max), - bits: quantized.bits, - step, - }) + const { quantized, quantizedPower } = getFieldOptions(field.optionsRaw) + const propName = snakeToPascal(field.name) + + let doc, getExpr, setExpr, step + if (quantized !== null) { + const mn = formatFloat(quantized.min) + const mx = formatFloat(quantized.max) + const bits = quantized.bits + // Uniform quantizer — the step is constant across the whole range. + step = (quantized.max - quantized.min) / ((1 << bits) - 1) + doc = `Range [${mn}, ${mx}], ${bits} bits, step ${formatStep(step)}.` + getExpr = `Quantize.Decode(${propName}, ${mn}, ${mx}, ${bits})` + setExpr = `Quantize.Encode(value, ${mn}, ${mx}, ${bits})` + } else if (quantizedPower !== null) { + const mx = formatFloat(quantizedPower.max) + const pw = formatFloat(quantizedPower.pow) + const bits = quantizedPower.bits + // Power curve is non-uniform: the finest step sits next to zero (first magnitude code), + // the COARSEST at the top of the range. The coarsest step upper-bounds the error for any + // value, so that's what the exposed {Name}QuantizedStep const carries (safe as a tolerance). + const magSteps = (1 << (bits - 1)) - 1 + const nearZeroStep = quantizedPower.max * Math.pow(1 / magSteps, quantizedPower.pow) + step = quantizedPower.max * (1 - Math.pow((magSteps - 1) / magSteps, quantizedPower.pow)) + doc = + `Range [-${mx}, ${mx}], power ${pw}, ${bits} bits ` + + `(sign + ${bits - 1}-bit magnitude), near-zero step ${formatStep(nearZeroStep)}.` + getExpr = `Quantize.DecodePower(${propName}, ${mx}, ${pw}, ${bits})` + setExpr = `Quantize.EncodePower(value, ${mx}, ${pw}, ${bits})` + } else { + continue + } + + props.push({ propName, doc, getExpr, setExpr, step }) } if (props.length === 0) return null @@ -139,17 +160,17 @@ function generateMessage(msgProto, indent) { lines.push(`public partial class ${msgProto.name}`) lines.push('{') - for (const { propName, mn, mx, bits, step } of props) { + for (const { propName, doc, getExpr, setExpr, step } of props) { const backing = '_' + propName[0].toLowerCase() + propName.slice(1) backings.push(backing) lines.push(`${i}private float? ${backing};`) - lines.push( - `${i}/// Float accessor for . Range [${mn}, ${mx}], ${bits} bits, step ${formatStep(step)}.`, - ) + lines.push(`${i}/// Coarsest quantization step of . Safe as an equality tolerance.`) + lines.push(`${i}public const float ${propName}QuantizedStep = ${formatFloat(step)};`) + lines.push(`${i}/// Float accessor for . ${doc}`) lines.push(`${i}public float ${propName}Quantized`) lines.push(`${i}{`) - lines.push(`${i}${i}get => ${backing} ??= Quantize.Decode(${propName}, ${mn}, ${mx}, ${bits});`) - lines.push(`${i}${i}set { ${backing} = value; ${propName} = Quantize.Encode(value, ${mn}, ${mx}, ${bits}); }`) + lines.push(`${i}${i}get => ${backing} ??= ${getExpr};`) + lines.push(`${i}${i}set { ${backing} = value; ${propName} = ${setExpr}; }`) lines.push(`${i}}`) lines.push('') } diff --git a/protoc-gen-bitwise/options.js b/protoc-gen-bitwise/options.js index 150ece0b..e4a788cc 100644 --- a/protoc-gen-bitwise/options.js +++ b/protoc-gen-bitwise/options.js @@ -4,7 +4,7 @@ * Parser for the custom bitwise field options defined in options.proto. * * The descriptor decoder hands us the raw serialized FieldOptions bytes (it - * declares `options` as opaque bytes). We walk those bytes looking for the two + * declares `options` as opaque bytes). We walk those bytes looking for the * custom extension field numbers — protobuf preserves unknown/unregistered * extension bytes, so they are always present even though no runtime here knows * the extension schema. This mirrors the original options_pb2.py. @@ -18,6 +18,7 @@ const { readVarint, skipField } = require('./wire') // Extension field numbers as defined in options.proto. const QUANTIZED_FIELD_NUMBER = 50001 const BIT_PACKED_FIELD_NUMBER = 50002 +const QUANTIZED_POWER_FIELD_NUMBER = 50003 /** Parse a serialized QuantizedFloatOptions message: { min, max, bits }. */ function parseQuantized(data) { @@ -46,6 +47,33 @@ function parseQuantized(data) { return opts } +/** Parse a serialized QuantizedPowerFloatOptions message: { max, pow, bits }. */ +function parseQuantizedPower(data) { + const opts = { max: 0.0, pow: 0.0, bits: 0 } + let pos = 0 + while (pos < data.length) { + let tag + ;[tag, pos] = readVarint(data, pos) + const fieldNum = tag >>> 3 + const wireType = tag & 0x7 + if (fieldNum === 1 && wireType === 5) { + // max (float) + opts.max = data.readFloatLE(pos) + pos += 4 + } else if (fieldNum === 2 && wireType === 5) { + // pow (float) + opts.pow = data.readFloatLE(pos) + pos += 4 + } else if (fieldNum === 3 && wireType === 0) { + // bits (uint32) + ;[opts.bits, pos] = readVarint(data, pos) + } else { + pos = skipField(data, pos, wireType) + } + } + return opts +} + /** Parse a serialized BitPackedOptions message: { bits }. */ function parseBitPacked(data) { const opts = { bits: 0 } @@ -69,15 +97,16 @@ function parseBitPacked(data) { * Extract custom bitwise options from raw FieldOptions bytes. * * @param {Buffer|null} optionsRaw serialized FieldOptions, or null when unset. - * @returns {{quantized: object|null, bitPacked: object|null}} + * @returns {{quantized: object|null, bitPacked: object|null, quantizedPower: object|null}} */ function getFieldOptions(optionsRaw) { if (!optionsRaw || optionsRaw.length === 0) { - return { quantized: null, bitPacked: null } + return { quantized: null, bitPacked: null, quantizedPower: null } } let quantized = null let bitPacked = null + let quantizedPower = null let pos = 0 while (pos < optionsRaw.length) { @@ -95,6 +124,8 @@ function getFieldOptions(optionsRaw) { quantized = parseQuantized(valueBytes) } else if (fieldNum === BIT_PACKED_FIELD_NUMBER) { bitPacked = parseBitPacked(valueBytes) + } else if (fieldNum === QUANTIZED_POWER_FIELD_NUMBER) { + quantizedPower = parseQuantizedPower(valueBytes) } // else: unknown length-delimited field — already consumed. } else { @@ -102,7 +133,7 @@ function getFieldOptions(optionsRaw) { } } - return { quantized, bitPacked } + return { quantized, bitPacked, quantizedPower } } module.exports = { getFieldOptions } diff --git a/protoc-gen-bitwise/runtime/cs/Quantize.cs b/protoc-gen-bitwise/runtime/cs/Quantize.cs index 22c347fe..80323615 100644 --- a/protoc-gen-bitwise/runtime/cs/Quantize.cs +++ b/protoc-gen-bitwise/runtime/cs/Quantize.cs @@ -31,5 +31,40 @@ public static float Decode(uint encoded, float min, float max, int bits) var steps = (1u << bits) - 1; return (float)encoded / steps * (max - min) + min; } + + /// + /// Encodes with a power-law curve into an + /// ( - 1)-bit linear unorm magnitude plus a sign bit. The magnitude is + /// (|value| / max)^(1/pow), so the inverse reconstructs + /// sign * max * u^pow. Zero is representable exactly; > 1 + /// concentrates resolution near zero. Magnitudes outside [0, ] are + /// clamped. + /// + /// The encoded layout puts the magnitude in the high bits and the sign in the LSB + /// ((magnitude << 1) | sign). This keeps the varint cost a function of + /// magnitude rather than direction — a small |value| of either sign stays in a + /// single varint byte. Zero canonicalizes to 0 (a zero magnitude never sets the + /// sign bit), so proto3 still omits a stopped field entirely. + /// + /// + public static uint EncodePower(float value, float max, float pow, int bits) + { + var magnitudeSteps = (1u << (bits - 1)) - 1; + var t = Math.Clamp(MathF.Abs(value) / max, 0f, 1f); + var u = MathF.Pow(t, 1f / pow); + var magnitude = (uint)MathF.Round(u * magnitudeSteps); + var sign = value < 0f && magnitude != 0u ? 1u : 0u; + return (magnitude << 1) | sign; + } + + /// Decodes a power-law quantized back to a float (inverse of + /// ). + public static float DecodePower(uint encoded, float max, float pow, int bits) + { + var magnitudeSteps = (1u << (bits - 1)) - 1; + var u = (float)(encoded >> 1) / magnitudeSteps; + var magnitude = max * MathF.Pow(u, pow); + return (encoded & 1u) != 0 ? -magnitude : magnitude; + } } } diff --git a/protoc-gen-bitwise/test/generator.test.js b/protoc-gen-bitwise/test/generator.test.js index 95512154..563aaca5 100644 --- a/protoc-gen-bitwise/test/generator.test.js +++ b/protoc-gen-bitwise/test/generator.test.js @@ -68,6 +68,16 @@ function bitPackedOptions(bits) { return lengthDelimited(50002, inner) } +// FieldOptions bytes carrying ext 50003 (quantized_power) = { max, pow, bits } +function quantizedPowerOptions(max, pow, bits) { + const inner = Buffer.concat([ + tag(1, 5), floatLE(max), + tag(2, 5), floatLE(pow), + tag(3, 0), encodeVarint(bits), + ]) + return lengthDelimited(50003, inner) +} + function field(name, type, optionsRaw) { return { name, label: LABEL_OPTIONAL, type, optionsRaw: optionsRaw || null } } @@ -104,6 +114,14 @@ const quantizationExample = { field('sequence', TYPE_UINT32, bitPackedOptions(12)), ], }, + { + name: 'VelocityState', + field: [ + field('vx', TYPE_UINT32, quantizedPowerOptions(50, 2, 8)), + field('vy', TYPE_UINT32, quantizedPowerOptions(50, 2, 8)), + field('vz', TYPE_UINT32, quantizedPowerOptions(50, 2, 8)), + ], + }, { name: 'AvatarStateSnapshot', field: [ @@ -135,9 +153,9 @@ const pulseServer = { field('position_x', TYPE_UINT32, quantizedOptions(0, 16, 8)), field('position_y', TYPE_UINT32, quantizedOptions(0, 200, 13)), field('position_z', TYPE_UINT32, quantizedOptions(0, 16, 8)), - field('velocity_x', TYPE_UINT32, quantizedOptions(-50, 50, 8)), - field('velocity_y', TYPE_UINT32, quantizedOptions(-50, 50, 8)), - field('velocity_z', TYPE_UINT32, quantizedOptions(-50, 50, 8)), + field('velocity_x', TYPE_UINT32, quantizedPowerOptions(50, 2, 8)), + field('velocity_y', TYPE_UINT32, quantizedPowerOptions(50, 2, 8)), + field('velocity_z', TYPE_UINT32, quantizedPowerOptions(50, 2, 8)), field('rotation_y', TYPE_UINT32, quantizedOptions(0, 360, 7)), field('movement_blend', TYPE_UINT32, quantizedOptions(0, 3, 5)), field('slide_blend', TYPE_UINT32, quantizedOptions(0, 1, 4)), diff --git a/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs b/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs index 255ba7bf..f121577e 100644 --- a/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs +++ b/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs @@ -34,27 +34,27 @@ public float PositionZQuantized } private float? _velocityX; - /// Float accessor for . Range [-50.0f, 50.0f], 8 bits, step ≈ 0.392157. + /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. public float VelocityXQuantized { - get => _velocityX ??= Quantize.Decode(VelocityX, -50.0f, 50.0f, 8); - set { _velocityX = value; VelocityX = Quantize.Encode(value, -50.0f, 50.0f, 8); } + get => _velocityX ??= Quantize.DecodePower(VelocityX, 50.0f, 2.0f, 8); + set { _velocityX = value; VelocityX = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } } private float? _velocityY; - /// Float accessor for . Range [-50.0f, 50.0f], 8 bits, step ≈ 0.392157. + /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. public float VelocityYQuantized { - get => _velocityY ??= Quantize.Decode(VelocityY, -50.0f, 50.0f, 8); - set { _velocityY = value; VelocityY = Quantize.Encode(value, -50.0f, 50.0f, 8); } + get => _velocityY ??= Quantize.DecodePower(VelocityY, 50.0f, 2.0f, 8); + set { _velocityY = value; VelocityY = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } } private float? _velocityZ; - /// Float accessor for . Range [-50.0f, 50.0f], 8 bits, step ≈ 0.392157. + /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. public float VelocityZQuantized { - get => _velocityZ ??= Quantize.Decode(VelocityZ, -50.0f, 50.0f, 8); - set { _velocityZ = value; VelocityZ = Quantize.Encode(value, -50.0f, 50.0f, 8); } + get => _velocityZ ??= Quantize.DecodePower(VelocityZ, 50.0f, 2.0f, 8); + set { _velocityZ = value; VelocityZ = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } } private float? _rotationY; diff --git a/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs b/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs index c0456936..596301b6 100644 --- a/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs +++ b/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs @@ -77,6 +77,41 @@ public void ResetDecodedCache() } } + public partial class VelocityState + { + private float? _vx; + /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. + public float VxQuantized + { + get => _vx ??= Quantize.DecodePower(Vx, 50.0f, 2.0f, 8); + set { _vx = value; Vx = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } + } + + private float? _vy; + /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. + public float VyQuantized + { + get => _vy ??= Quantize.DecodePower(Vy, 50.0f, 2.0f, 8); + set { _vy = value; Vy = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } + } + + private float? _vz; + /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. + public float VzQuantized + { + get => _vz ??= Quantize.DecodePower(Vz, 50.0f, 2.0f, 8); + set { _vz = value; Vz = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } + } + + /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. + public void ResetDecodedCache() + { + _vx = null; + _vy = null; + _vz = null; + } + } + public partial class AvatarStateSnapshot { private float? _x; From 9faec2f0c039532441d15fff409c5f754273f863 Mon Sep 17 00:00:00 2001 From: Mikhail Agapov <118179774+mikhail-dcl@users.noreply.github.com> Date: Mon, 6 Jul 2026 19:12:53 +0300 Subject: [PATCH 49/54] fix: additional quantization improvements (#435) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Quantize TeleportRequest * Add Quantized Values Validation Signed-off-by: Mikhail Agapov * Import "decentraland/common/vectors.proto" is unused Signed-off-by: Mikhail Agapov * protoc-gen-bitwise: add range check, drop decode cache Emit AreQuantizedFieldsInRange() (per-field integer bounds check, used by the server to reject out-of-range client codes) and remove the per-field decoded- float cache — decode on get, encode on set. Regenerate goldens; add UPDATE_GOLDEN mode to the golden test so future generator changes are one command. Co-Authored-By: Claude Fable 5 --------- Signed-off-by: Mikhail Agapov Co-authored-by: Claude Fable 5 --- README.md | 28 +-- proto/decentraland/pulse/pulse_client.proto | 8 +- protoc-gen-bitwise/generator_csharp.js | 46 +++-- protoc-gen-bitwise/test/generator.test.js | 16 ++ .../test/golden/PulseServer.Bitwise.cs | 138 +++++++------ .../golden/QuantizationExample.Bitwise.cs | 190 ++++++++++-------- 6 files changed, 245 insertions(+), 181 deletions(-) diff --git a/README.md b/README.md index be89895f..a968d863 100644 --- a/README.md +++ b/README.md @@ -213,12 +213,9 @@ SendOnChannel1(bytes); // --- Receive and read --- var received = PositionDelta.Parser.ParseFrom(receivedBytes); -float x = received.DxQuantized; // decoded on first access, cached thereafter +float x = received.DxQuantized; // decoded from the stored uint32 on each access float y = received.DyQuantized; float z = received.DzQuantized; - -// If raw uint32 fields are mutated directly after construction, invalidate the cache: -received.ResetDecodedCache(); ``` ## Generated file example @@ -237,33 +234,22 @@ namespace Decentraland.Kernel.Comms.V3 { public partial class PositionDelta { - private float? _dx; public float DxQuantized { - get => _dx ??= Quantize.Decode(Dx, -100.0f, 100.0f, 16); - set { _dx = value; Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); } + get => Quantize.Decode(Dx, -100.0f, 100.0f, 16); + set => Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); } - private float? _dy; public float DyQuantized { - get => _dy ??= Quantize.Decode(Dy, -100.0f, 100.0f, 16); - set { _dy = value; Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); } + get => Quantize.Decode(Dy, -100.0f, 100.0f, 16); + set => Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); } - private float? _dz; public float DzQuantized { - get => _dz ??= Quantize.Decode(Dz, -100.0f, 100.0f, 16); - set { _dz = value; Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); } - } - - /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. - public void ResetDecodedCache() - { - _dx = null; - _dy = null; - _dz = null; + get => Quantize.Decode(Dz, -100.0f, 100.0f, 16); + set => Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); } } diff --git a/proto/decentraland/pulse/pulse_client.proto b/proto/decentraland/pulse/pulse_client.proto index 6036ec53..09760dab 100644 --- a/proto/decentraland/pulse/pulse_client.proto +++ b/proto/decentraland/pulse/pulse_client.proto @@ -2,8 +2,8 @@ syntax = "proto3"; package decentraland.pulse; -import "decentraland/common/vectors.proto"; import "decentraland/pulse/pulse_shared.proto"; +import "decentraland/common/options.proto"; message HandshakeRequest { bytes auth_chain = 1; @@ -59,9 +59,11 @@ message EmoteStop { // other. Must be the first gameplay message after handshake. Same-realm re-teleports are valid. message TeleportRequest { int32 parcel_index = 1; - decentraland.common.Vector3 position = 2; + uint32 position_x = 2 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + uint32 position_y = 3 [(decentraland.common.quantized) = { min: 0, max: 200, bits: 13 }]; + uint32 position_z = 4 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; // Non-empty realm identifier. Rejected if empty. - string realm = 3; + string realm = 5; } message ClientMessage { diff --git a/protoc-gen-bitwise/generator_csharp.js b/protoc-gen-bitwise/generator_csharp.js index 93b23c26..4cb3a2ee 100644 --- a/protoc-gen-bitwise/generator_csharp.js +++ b/protoc-gen-bitwise/generator_csharp.js @@ -121,11 +121,11 @@ function generateMessage(msgProto, indent) { const { quantized, quantizedPower } = getFieldOptions(field.optionsRaw) const propName = snakeToPascal(field.name) - let doc, getExpr, setExpr, step + let doc, getExpr, setExpr, step, bits if (quantized !== null) { const mn = formatFloat(quantized.min) const mx = formatFloat(quantized.max) - const bits = quantized.bits + bits = quantized.bits // Uniform quantizer — the step is constant across the whole range. step = (quantized.max - quantized.min) / ((1 << bits) - 1) doc = `Range [${mn}, ${mx}], ${bits} bits, step ${formatStep(step)}.` @@ -134,7 +134,7 @@ function generateMessage(msgProto, indent) { } else if (quantizedPower !== null) { const mx = formatFloat(quantizedPower.max) const pw = formatFloat(quantizedPower.pow) - const bits = quantizedPower.bits + bits = quantizedPower.bits // Power curve is non-uniform: the finest step sits next to zero (first magnitude code), // the COARSEST at the top of the range. The coarsest step upper-bounds the error for any // value, so that's what the exposed {Name}QuantizedStep const carries (safe as a tolerance). @@ -150,38 +150,50 @@ function generateMessage(msgProto, indent) { continue } - props.push({ propName, doc, getExpr, setExpr, step }) + // Highest code the encoder can emit for this field. Both the linear quantizer (top code + // `2^bits - 1`) and the power quantizer (`(magnitude << 1) | sign` with an `bits-1`-bit + // magnitude, so top code `((2^(bits-1)-1) << 1) | 1 == 2^bits - 1`) share this bound. + const maxCode = 2 ** bits - 1 + + props.push({ propName, doc, getExpr, setExpr, step, maxCode }) } if (props.length === 0) return null - const backings = [] const lines = [] lines.push(`public partial class ${msgProto.name}`) lines.push('{') + // Decode on every get, encode on every set — no backing cache. The raw uint32 property is the + // single source of truth, so the float accessor can never disagree with the code on the wire + // (get-after-set returns the on-grid value a receiver decodes) and there is no stale-cache + // hazard when the raw field is mutated directly. Decode is a multiply-add; only power fields + // pay a MathF.Pow. for (const { propName, doc, getExpr, setExpr, step } of props) { - const backing = '_' + propName[0].toLowerCase() + propName.slice(1) - backings.push(backing) - lines.push(`${i}private float? ${backing};`) lines.push(`${i}/// Coarsest quantization step of . Safe as an equality tolerance.`) lines.push(`${i}public const float ${propName}QuantizedStep = ${formatFloat(step)};`) lines.push(`${i}/// Float accessor for . ${doc}`) lines.push(`${i}public float ${propName}Quantized`) lines.push(`${i}{`) - lines.push(`${i}${i}get => ${backing} ??= ${getExpr};`) - lines.push(`${i}${i}set { ${backing} = value; ${propName} = ${setExpr}; }`) + lines.push(`${i}${i}get => ${getExpr};`) + lines.push(`${i}${i}set => ${propName} = ${setExpr};`) lines.push(`${i}}`) lines.push('') } - lines.push(`${i}/// Clears all cached decoded values. Call after mutating raw uint32 fields directly.`) - lines.push(`${i}public void ResetDecodedCache()`) - lines.push(`${i}{`) - for (const backing of backings) { - lines.push(`${i}${i}${backing} = null;`) - } - lines.push(`${i}}`) + lines.push(`${i}/// `) + lines.push(`${i}/// True when every quantized field holds a wire code within its declared bit width`) + lines.push(`${i}/// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger`) + lines.push(`${i}/// value is a malformed/hostile message: decoding it would land far outside the field's`) + lines.push(`${i}/// [min, max] and, since the server relays raw codes verbatim, poison every observer.`) + lines.push(`${i}/// Reject before storing or relaying. Pure integer comparison — no decode.`) + lines.push(`${i}/// `) + lines.push(`${i}public bool AreQuantizedFieldsInRange() =>`) + props.forEach(({ propName, maxCode }, idx) => { + const prefix = idx === 0 ? '' : '&& ' + const suffix = idx === props.length - 1 ? ';' : '' + lines.push(`${i}${i}${prefix}${propName} <= ${maxCode}u${suffix}`) + }) lines.push('}') return lines diff --git a/protoc-gen-bitwise/test/generator.test.js b/protoc-gen-bitwise/test/generator.test.js index 563aaca5..dc20b39e 100644 --- a/protoc-gen-bitwise/test/generator.test.js +++ b/protoc-gen-bitwise/test/generator.test.js @@ -176,11 +176,22 @@ const cases = [ { proto: pulseServer, golden: 'PulseServer.Bitwise.cs' }, ] +// Set UPDATE_GOLDEN=1 to rewrite the golden files from the current generator output +// (after an intentional generator change), then re-run without it to verify. +const update = process.env.UPDATE_GOLDEN === '1' + let failed = 0 for (const { proto, golden } of cases) { const result = generateCsharp(proto) assert.ok(result, `expected output for ${golden}`) assert.strictEqual(result.name, golden, `output filename for ${golden}`) + + if (update) { + fs.writeFileSync(path.join(__dirname, 'golden', golden), result.content) + console.log(`updated - ${golden}`) + continue + } + try { assert.strictEqual(result.content, readGolden(golden)) console.log(`ok - ${golden} matches golden`) @@ -191,6 +202,11 @@ for (const { proto, golden } of cases) { } } +if (update) { + console.log('\nGolden files updated. Re-run `npm run gen:test` to verify.') + process.exit(0) +} + if (failed > 0) { console.error(`\n${failed} golden mismatch(es)`) process.exit(1) diff --git a/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs b/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs index f121577e..ea3c3d1a 100644 --- a/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs +++ b/protoc-gen-bitwise/test/golden/PulseServer.Bitwise.cs @@ -9,136 +9,154 @@ namespace Decentraland.Pulse { public partial class PlayerStateDeltaTier0 { - private float? _positionX; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PositionXQuantizedStep = 0.062745098f; /// Float accessor for . Range [0.0f, 16.0f], 8 bits, step ≈ 0.0627451. public float PositionXQuantized { - get => _positionX ??= Quantize.Decode(PositionX, 0.0f, 16.0f, 8); - set { _positionX = value; PositionX = Quantize.Encode(value, 0.0f, 16.0f, 8); } + get => Quantize.Decode(PositionX, 0.0f, 16.0f, 8); + set => PositionX = Quantize.Encode(value, 0.0f, 16.0f, 8); } - private float? _positionY; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PositionYQuantizedStep = 0.024417043f; /// Float accessor for . Range [0.0f, 200.0f], 13 bits, step ≈ 0.024417. public float PositionYQuantized { - get => _positionY ??= Quantize.Decode(PositionY, 0.0f, 200.0f, 13); - set { _positionY = value; PositionY = Quantize.Encode(value, 0.0f, 200.0f, 13); } + get => Quantize.Decode(PositionY, 0.0f, 200.0f, 13); + set => PositionY = Quantize.Encode(value, 0.0f, 200.0f, 13); } - private float? _positionZ; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PositionZQuantizedStep = 0.062745098f; /// Float accessor for . Range [0.0f, 16.0f], 8 bits, step ≈ 0.0627451. public float PositionZQuantized { - get => _positionZ ??= Quantize.Decode(PositionZ, 0.0f, 16.0f, 8); - set { _positionZ = value; PositionZ = Quantize.Encode(value, 0.0f, 16.0f, 8); } + get => Quantize.Decode(PositionZ, 0.0f, 16.0f, 8); + set => PositionZ = Quantize.Encode(value, 0.0f, 16.0f, 8); } - private float? _velocityX; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float VelocityXQuantizedStep = 0.78430157f; /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. public float VelocityXQuantized { - get => _velocityX ??= Quantize.DecodePower(VelocityX, 50.0f, 2.0f, 8); - set { _velocityX = value; VelocityX = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } + get => Quantize.DecodePower(VelocityX, 50.0f, 2.0f, 8); + set => VelocityX = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } - private float? _velocityY; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float VelocityYQuantizedStep = 0.78430157f; /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. public float VelocityYQuantized { - get => _velocityY ??= Quantize.DecodePower(VelocityY, 50.0f, 2.0f, 8); - set { _velocityY = value; VelocityY = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } + get => Quantize.DecodePower(VelocityY, 50.0f, 2.0f, 8); + set => VelocityY = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } - private float? _velocityZ; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float VelocityZQuantizedStep = 0.78430157f; /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. public float VelocityZQuantized { - get => _velocityZ ??= Quantize.DecodePower(VelocityZ, 50.0f, 2.0f, 8); - set { _velocityZ = value; VelocityZ = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } + get => Quantize.DecodePower(VelocityZ, 50.0f, 2.0f, 8); + set => VelocityZ = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } - private float? _rotationY; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float RotationYQuantizedStep = 2.8346457f; /// Float accessor for . Range [0.0f, 360.0f], 7 bits, step ≈ 2.83465. public float RotationYQuantized { - get => _rotationY ??= Quantize.Decode(RotationY, 0.0f, 360.0f, 7); - set { _rotationY = value; RotationY = Quantize.Encode(value, 0.0f, 360.0f, 7); } + get => Quantize.Decode(RotationY, 0.0f, 360.0f, 7); + set => RotationY = Quantize.Encode(value, 0.0f, 360.0f, 7); } - private float? _movementBlend; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float MovementBlendQuantizedStep = 0.096774194f; /// Float accessor for . Range [0.0f, 3.0f], 5 bits, step ≈ 0.0967742. public float MovementBlendQuantized { - get => _movementBlend ??= Quantize.Decode(MovementBlend, 0.0f, 3.0f, 5); - set { _movementBlend = value; MovementBlend = Quantize.Encode(value, 0.0f, 3.0f, 5); } + get => Quantize.Decode(MovementBlend, 0.0f, 3.0f, 5); + set => MovementBlend = Quantize.Encode(value, 0.0f, 3.0f, 5); } - private float? _slideBlend; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float SlideBlendQuantizedStep = 0.066666667f; /// Float accessor for . Range [0.0f, 1.0f], 4 bits, step ≈ 0.0666667. public float SlideBlendQuantized { - get => _slideBlend ??= Quantize.Decode(SlideBlend, 0.0f, 1.0f, 4); - set { _slideBlend = value; SlideBlend = Quantize.Encode(value, 0.0f, 1.0f, 4); } + get => Quantize.Decode(SlideBlend, 0.0f, 1.0f, 4); + set => SlideBlend = Quantize.Encode(value, 0.0f, 1.0f, 4); } - private float? _headYaw; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float HeadYawQuantizedStep = 2.8346457f; /// Float accessor for . Range [0.0f, 360.0f], 7 bits, step ≈ 2.83465. public float HeadYawQuantized { - get => _headYaw ??= Quantize.Decode(HeadYaw, 0.0f, 360.0f, 7); - set { _headYaw = value; HeadYaw = Quantize.Encode(value, 0.0f, 360.0f, 7); } + get => Quantize.Decode(HeadYaw, 0.0f, 360.0f, 7); + set => HeadYaw = Quantize.Encode(value, 0.0f, 360.0f, 7); } - private float? _headPitch; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float HeadPitchQuantizedStep = 2.8346457f; /// Float accessor for . Range [0.0f, 360.0f], 7 bits, step ≈ 2.83465. public float HeadPitchQuantized { - get => _headPitch ??= Quantize.Decode(HeadPitch, 0.0f, 360.0f, 7); - set { _headPitch = value; HeadPitch = Quantize.Encode(value, 0.0f, 360.0f, 7); } + get => Quantize.Decode(HeadPitch, 0.0f, 360.0f, 7); + set => HeadPitch = Quantize.Encode(value, 0.0f, 360.0f, 7); } - private float? _pointAtX; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PointAtXQuantizedStep = 0.045776716f; /// Float accessor for . Range [-3000.0f, 3000.0f], 17 bits, step ≈ 0.0457767. public float PointAtXQuantized { - get => _pointAtX ??= Quantize.Decode(PointAtX, -3000.0f, 3000.0f, 17); - set { _pointAtX = value; PointAtX = Quantize.Encode(value, -3000.0f, 3000.0f, 17); } + get => Quantize.Decode(PointAtX, -3000.0f, 3000.0f, 17); + set => PointAtX = Quantize.Encode(value, -3000.0f, 3000.0f, 17); } - private float? _pointAtY; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PointAtYQuantizedStep = 1.5748031f; /// Float accessor for . Range [0.0f, 200.0f], 7 bits, step ≈ 1.5748. public float PointAtYQuantized { - get => _pointAtY ??= Quantize.Decode(PointAtY, 0.0f, 200.0f, 7); - set { _pointAtY = value; PointAtY = Quantize.Encode(value, 0.0f, 200.0f, 7); } + get => Quantize.Decode(PointAtY, 0.0f, 200.0f, 7); + set => PointAtY = Quantize.Encode(value, 0.0f, 200.0f, 7); } - private float? _pointAtZ; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PointAtZQuantizedStep = 0.045776716f; /// Float accessor for . Range [-3000.0f, 3000.0f], 17 bits, step ≈ 0.0457767. public float PointAtZQuantized { - get => _pointAtZ ??= Quantize.Decode(PointAtZ, -3000.0f, 3000.0f, 17); - set { _pointAtZ = value; PointAtZ = Quantize.Encode(value, -3000.0f, 3000.0f, 17); } + get => Quantize.Decode(PointAtZ, -3000.0f, 3000.0f, 17); + set => PointAtZ = Quantize.Encode(value, -3000.0f, 3000.0f, 17); } - /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. - public void ResetDecodedCache() - { - _positionX = null; - _positionY = null; - _positionZ = null; - _velocityX = null; - _velocityY = null; - _velocityZ = null; - _rotationY = null; - _movementBlend = null; - _slideBlend = null; - _headYaw = null; - _headPitch = null; - _pointAtX = null; - _pointAtY = null; - _pointAtZ = null; - } + /// + /// True when every quantized field holds a wire code within its declared bit width + /// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger + /// value is a malformed/hostile message: decoding it would land far outside the field's + /// [min, max] and, since the server relays raw codes verbatim, poison every observer. + /// Reject before storing or relaying. Pure integer comparison — no decode. + /// + public bool AreQuantizedFieldsInRange() => + PositionX <= 255u + && PositionY <= 8191u + && PositionZ <= 255u + && VelocityX <= 255u + && VelocityY <= 255u + && VelocityZ <= 255u + && RotationY <= 127u + && MovementBlend <= 31u + && SlideBlend <= 15u + && HeadYaw <= 127u + && HeadPitch <= 127u + && PointAtX <= 131071u + && PointAtY <= 127u + && PointAtZ <= 131071u; } diff --git a/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs b/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs index 596301b6..62f157e6 100644 --- a/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs +++ b/protoc-gen-bitwise/test/golden/QuantizationExample.Bitwise.cs @@ -9,160 +9,190 @@ namespace Decentraland.Common { public partial class PositionDelta { - private float? _dx; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float DxQuantizedStep = 0.0030518044f; /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. public float DxQuantized { - get => _dx ??= Quantize.Decode(Dx, -100.0f, 100.0f, 16); - set { _dx = value; Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); } + get => Quantize.Decode(Dx, -100.0f, 100.0f, 16); + set => Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); } - private float? _dy; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float DyQuantizedStep = 0.0030518044f; /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. public float DyQuantized { - get => _dy ??= Quantize.Decode(Dy, -100.0f, 100.0f, 16); - set { _dy = value; Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); } + get => Quantize.Decode(Dy, -100.0f, 100.0f, 16); + set => Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); } - private float? _dz; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float DzQuantizedStep = 0.0030518044f; /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. public float DzQuantized { - get => _dz ??= Quantize.Decode(Dz, -100.0f, 100.0f, 16); - set { _dz = value; Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); } - } - - /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. - public void ResetDecodedCache() - { - _dx = null; - _dy = null; - _dz = null; - } + get => Quantize.Decode(Dz, -100.0f, 100.0f, 16); + set => Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); + } + + /// + /// True when every quantized field holds a wire code within its declared bit width + /// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger + /// value is a malformed/hostile message: decoding it would land far outside the field's + /// [min, max] and, since the server relays raw codes verbatim, poison every observer. + /// Reject before storing or relaying. Pure integer comparison — no decode. + /// + public bool AreQuantizedFieldsInRange() => + Dx <= 65535u + && Dy <= 65535u + && Dz <= 65535u; } public partial class PlayerInput { - private float? _moveX; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float MoveXQuantizedStep = 0.0078431373f; /// Float accessor for . Range [-1.0f, 1.0f], 8 bits, step ≈ 0.00784314. public float MoveXQuantized { - get => _moveX ??= Quantize.Decode(MoveX, -1.0f, 1.0f, 8); - set { _moveX = value; MoveX = Quantize.Encode(value, -1.0f, 1.0f, 8); } + get => Quantize.Decode(MoveX, -1.0f, 1.0f, 8); + set => MoveX = Quantize.Encode(value, -1.0f, 1.0f, 8); } - private float? _moveZ; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float MoveZQuantizedStep = 0.0078431373f; /// Float accessor for . Range [-1.0f, 1.0f], 8 bits, step ≈ 0.00784314. public float MoveZQuantized { - get => _moveZ ??= Quantize.Decode(MoveZ, -1.0f, 1.0f, 8); - set { _moveZ = value; MoveZ = Quantize.Encode(value, -1.0f, 1.0f, 8); } + get => Quantize.Decode(MoveZ, -1.0f, 1.0f, 8); + set => MoveZ = Quantize.Encode(value, -1.0f, 1.0f, 8); } - private float? _yaw; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float YawQuantizedStep = 0.087912088f; /// Float accessor for . Range [-180.0f, 180.0f], 12 bits, step ≈ 0.0879121. public float YawQuantized { - get => _yaw ??= Quantize.Decode(Yaw, -180.0f, 180.0f, 12); - set { _yaw = value; Yaw = Quantize.Encode(value, -180.0f, 180.0f, 12); } - } - - /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. - public void ResetDecodedCache() - { - _moveX = null; - _moveZ = null; - _yaw = null; - } + get => Quantize.Decode(Yaw, -180.0f, 180.0f, 12); + set => Yaw = Quantize.Encode(value, -180.0f, 180.0f, 12); + } + + /// + /// True when every quantized field holds a wire code within its declared bit width + /// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger + /// value is a malformed/hostile message: decoding it would land far outside the field's + /// [min, max] and, since the server relays raw codes verbatim, poison every observer. + /// Reject before storing or relaying. Pure integer comparison — no decode. + /// + public bool AreQuantizedFieldsInRange() => + MoveX <= 255u + && MoveZ <= 255u + && Yaw <= 4095u; } public partial class VelocityState { - private float? _vx; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float VxQuantizedStep = 0.78430157f; /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. public float VxQuantized { - get => _vx ??= Quantize.DecodePower(Vx, 50.0f, 2.0f, 8); - set { _vx = value; Vx = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } + get => Quantize.DecodePower(Vx, 50.0f, 2.0f, 8); + set => Vx = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } - private float? _vy; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float VyQuantizedStep = 0.78430157f; /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. public float VyQuantized { - get => _vy ??= Quantize.DecodePower(Vy, 50.0f, 2.0f, 8); - set { _vy = value; Vy = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } + get => Quantize.DecodePower(Vy, 50.0f, 2.0f, 8); + set => Vy = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } - private float? _vz; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float VzQuantizedStep = 0.78430157f; /// Float accessor for . Range [-50.0f, 50.0f], power 2.0f, 8 bits (sign + 7-bit magnitude), near-zero step ≈ 0.00310001. public float VzQuantized { - get => _vz ??= Quantize.DecodePower(Vz, 50.0f, 2.0f, 8); - set { _vz = value; Vz = Quantize.EncodePower(value, 50.0f, 2.0f, 8); } - } - - /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. - public void ResetDecodedCache() - { - _vx = null; - _vy = null; - _vz = null; - } + get => Quantize.DecodePower(Vz, 50.0f, 2.0f, 8); + set => Vz = Quantize.EncodePower(value, 50.0f, 2.0f, 8); + } + + /// + /// True when every quantized field holds a wire code within its declared bit width + /// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger + /// value is a malformed/hostile message: decoding it would land far outside the field's + /// [min, max] and, since the server relays raw codes verbatim, poison every observer. + /// Reject before storing or relaying. Pure integer comparison — no decode. + /// + public bool AreQuantizedFieldsInRange() => + Vx <= 255u + && Vy <= 255u + && Vz <= 255u; } public partial class AvatarStateSnapshot { - private float? _x; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float XQuantizedStep = 0.12500191f; /// Float accessor for . Range [-4096.0f, 4096.0f], 16 bits, step ≈ 0.125002. public float XQuantized { - get => _x ??= Quantize.Decode(X, -4096.0f, 4096.0f, 16); - set { _x = value; X = Quantize.Encode(value, -4096.0f, 4096.0f, 16); } + get => Quantize.Decode(X, -4096.0f, 4096.0f, 16); + set => X = Quantize.Encode(value, -4096.0f, 4096.0f, 16); } - private float? _y; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float YQuantizedStep = 0.031251907f; /// Float accessor for . Range [-256.0f, 256.0f], 14 bits, step ≈ 0.0312519. public float YQuantized { - get => _y ??= Quantize.Decode(Y, -256.0f, 256.0f, 14); - set { _y = value; Y = Quantize.Encode(value, -256.0f, 256.0f, 14); } + get => Quantize.Decode(Y, -256.0f, 256.0f, 14); + set => Y = Quantize.Encode(value, -256.0f, 256.0f, 14); } - private float? _z; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float ZQuantizedStep = 0.12500191f; /// Float accessor for . Range [-4096.0f, 4096.0f], 16 bits, step ≈ 0.125002. public float ZQuantized { - get => _z ??= Quantize.Decode(Z, -4096.0f, 4096.0f, 16); - set { _z = value; Z = Quantize.Encode(value, -4096.0f, 4096.0f, 16); } + get => Quantize.Decode(Z, -4096.0f, 4096.0f, 16); + set => Z = Quantize.Encode(value, -4096.0f, 4096.0f, 16); } - private float? _pitch; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float PitchQuantizedStep = 0.17595308f; /// Float accessor for . Range [-90.0f, 90.0f], 10 bits, step ≈ 0.175953. public float PitchQuantized { - get => _pitch ??= Quantize.Decode(Pitch, -90.0f, 90.0f, 10); - set { _pitch = value; Pitch = Quantize.Encode(value, -90.0f, 90.0f, 10); } + get => Quantize.Decode(Pitch, -90.0f, 90.0f, 10); + set => Pitch = Quantize.Encode(value, -90.0f, 90.0f, 10); } - private float? _yaw; + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float YawQuantizedStep = 0.087912088f; /// Float accessor for . Range [-180.0f, 180.0f], 12 bits, step ≈ 0.0879121. public float YawQuantized { - get => _yaw ??= Quantize.Decode(Yaw, -180.0f, 180.0f, 12); - set { _yaw = value; Yaw = Quantize.Encode(value, -180.0f, 180.0f, 12); } - } - - /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. - public void ResetDecodedCache() - { - _x = null; - _y = null; - _z = null; - _pitch = null; - _yaw = null; - } + get => Quantize.Decode(Yaw, -180.0f, 180.0f, 12); + set => Yaw = Quantize.Encode(value, -180.0f, 180.0f, 12); + } + + /// + /// True when every quantized field holds a wire code within its declared bit width + /// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger + /// value is a malformed/hostile message: decoding it would land far outside the field's + /// [min, max] and, since the server relays raw codes verbatim, poison every observer. + /// Reject before storing or relaying. Pure integer comparison — no decode. + /// + public bool AreQuantizedFieldsInRange() => + X <= 65535u + && Y <= 16383u + && Z <= 65535u + && Pitch <= 1023u + && Yaw <= 4095u; } From f355d6c7a31809950fc9e524b2e9babc4b70e60a Mon Sep 17 00:00:00 2001 From: Mikhail Agapov Date: Tue, 7 Jul 2026 16:46:26 +0300 Subject: [PATCH 50/54] docs: describe protoc-gen-bitwise as it actually works; drop dead BitReader/BitWriter CLAUDE.md still documented the original bit-stream design (BitWriter/ BitReader classes, float fields, "68 bits = 9 bytes on the wire"). The plugin actually emits quantized-accessor partials (*.Bitwise.cs) over plain uint32 fields sent as standard protobuf varints, backed only by Quantize.cs; runtime BitReader.cs/BitWriter.cs were referenced by nothing and are removed. Also sync README with current generator output (QuantizedStep consts, AreQuantizedFieldsInRange, EncodePower/ DecodePower, per-proto-file output naming) and drop the incorrect "cached" accessor wording. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 128 ++++++++------------- README.md | 41 ++++++- protoc-gen-bitwise/runtime/cs/BitReader.cs | 112 ------------------ protoc-gen-bitwise/runtime/cs/BitWriter.cs | 117 ------------------- 4 files changed, 84 insertions(+), 314 deletions(-) delete mode 100644 protoc-gen-bitwise/runtime/cs/BitReader.cs delete mode 100644 protoc-gen-bitwise/runtime/cs/BitWriter.cs diff --git a/CLAUDE.md b/CLAUDE.md index 03a2a374..8810c93f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,7 +10,7 @@ High-performance MMO-style multiplayer networking stack. Protocol is **open** (U | Client | Unity (C#) | | Server | Custom server | | Schema source of truth | `.proto` files | -| Serialization | Custom protoc plugin (bitwise encoding) | +| Serialization | Standard protobuf wire format + custom protoc plugin (quantized float accessors) | | Auth | Decentraland ECDSA chain validation (local, on HANDSHAKE channel 0) | --- @@ -18,7 +18,15 @@ High-performance MMO-style multiplayer networking stack. Protocol is **open** (U ## Serialization: Custom Protoc Plugin ### What it does -Reads `.proto` files with custom field options and generates **bitwise encode/decode code** in C#, keeping all client implementations bit-for-bit identical. +Reads `.proto` files with custom field options and generates C# **partial classes** (`*.Bitwise.cs`) that add typed float accessors on top of quantized `uint32` fields, keeping the quantization math bit-for-bit identical across all client implementations. + +The wire format is **standard protobuf** — a quantized value lives in a plain `uint32` field and travels as an ordinary varint. There is no custom bit stream; any protobuf-capable client can parse the messages without this plugin. Per annotated field the plugin emits: + +- `float {Field}Quantized` — computed accessor (no backing cache): the getter decodes the stored `uint32`, the setter encodes a float back into it, via the static `Quantize` helpers +- `const float {Field}QuantizedStep` — the coarsest quantization step of the field, safe as an equality tolerance +- per message: `bool AreQuantizedFieldsInRange()` — pure-integer check that every stored code fits its declared bit width (`0 .. 2^bits-1`); reject malformed/hostile messages before storing or relaying + +Only non-repeated `uint32` fields get accessors; `bit_packed` and unannotated fields pass through with no generated code. ### Custom Field Options (`options.proto`) @@ -54,25 +62,31 @@ extend google.protobuf.FieldOptions { ### Usage example +Quantized fields are declared **`uint32`** (not `float`) — the float type exists only in the generated accessor: + ```protobuf message PositionDelta { - float dx = 1 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; - float dy = 2 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; - float dz = 3 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dx = 1 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dy = 2 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dz = 3 [(quantized) = { min: -100.0, max: 100.0, bits: 16 }]; uint32 entity_id = 4 [(bit_packed) = { bits: 20 }]; + uint32 sequence = 5 [(bit_packed) = { bits: 12 }]; } -// Total: 68 bits = 9 bytes on the wire +// Varint wire cost: dx/dy/dz/entity_id ≤ 4 B each (1 B tag + ≤ 3 B varint), +// sequence ≤ 3 B — worst-case 19 B, less when proto3 omits zero-valued fields. ``` +`proto/decentraland/common/quantization_example.proto` is the fully worked reference: per-field wire costs for the linear, power-law, and bit-packed annotations. + ### Plugin structure ``` protoc-gen-bitwise/ ├── plugin.js # stdin -> CodeGeneratorRequest, stdout -> CodeGeneratorResponse (Node) -├── generator_csharp.js # emits C# for Unity -├── options.js # parses the custom quantized / bit_packed field options +├── generator_csharp.js # emits the *.Bitwise.cs accessor partials for Unity +├── options.js # parses the custom quantized / quantized_power / bit_packed field options ├── wire.js # self-contained protobuf wire codec (zero runtime deps) -└── runtime/cs/ # C# runtime; Quantize.cs is copied into the generated output +└── runtime/cs/ # C# runtime: Quantize.cs — consumers copy it next to the generated files ``` Plugin contract: a protoc plugin that reads a serialized `CodeGeneratorRequest` from stdin and writes a serialized `CodeGeneratorResponse` to stdout. It is a plain Node script — **no `npm install` required, only `node` on PATH**. protoc invokes it through a tiny wrapper that runs `node plugin.js` (`.cmd` on Windows, a shell script elsewhere, since protoc cannot exec a `.js` directly). @@ -89,85 +103,30 @@ Parity is locked down by `npm run gen:test` (compares generator output against g --- -## BitWriter / BitReader +## Quantize Runtime -The C# implementation uses the following bit layout: **big-endian within each byte**, MSB written first. Use **`Round`** (not truncate) when quantizing to minimize error. +Generated accessors call the static `Quantize` class (`protoc-gen-bitwise/runtime/cs/Quantize.cs`, namespace `Decentraland.Networking.Bitwise`) — the only C# runtime file; consumers copy it next to the generated `*.Bitwise.cs` partials. Quantization uses **`Round`** (not truncate) to minimize error; identical rounding on both sides makes encode -> decode a round-trip no-op. -### Core math — WriteQuantizedFloat +### Core math — linear (`Quantize.Encode` / `Quantize.Decode`) ``` -normalized = (clamp(value, min, max) - min) / (max - min) // -> [0.0, 1.0] -quantized = Round(normalized * ((1 << bits) - 1)) // -> integer +encoded = Round(clamp01((value - min) / (max - min)) * (2^bits - 1)) +decoded = encoded / (2^bits - 1) * (max - min) + min ``` -### Core math — ReadQuantizedFloat +### Core math — power-law (`Quantize.EncodePower` / `Quantize.DecodePower`) + +For signed fields like velocity that need an exact zero and fine resolution near zero: ``` -normalized = quantized / ((1 << bits) - 1) -value = min + normalized * (max - min) +u = clamp01(|value| / max) ^ (1 / pow) +encoded = (Round(u * (2^(bits-1) - 1)) << 1) | sign // magnitude in high bits, sign in LSB +decoded = sign * max * ((encoded >> 1) / (2^(bits-1) - 1)) ^ pow ``` -### Implementation - -```csharp -public class BitWriter -{ - private byte[] _buffer; - private int _bitPos; - - public BitWriter(byte[] buffer) { _buffer = buffer; _bitPos = 0; } - - public void WriteBits(uint value, int bits) - { - for (int i = bits - 1; i >= 0; i--) - { - int byteIdx = _bitPos / 8; - int bitIdx = 7 - (_bitPos % 8); - if ((value >> i & 1) == 1) _buffer[byteIdx] |= (byte)(1 << bitIdx); - else _buffer[byteIdx] &= (byte)~(1 << bitIdx); - _bitPos++; - } - } - - public void WriteQuantizedFloat(float value, float min, float max, int bits) - { - uint maxQ = (1u << bits) - 1; - float clamped = Math.Clamp(value, min, max); - float normalized = (clamped - min) / (max - min); - uint quantized = (uint)Math.Round(normalized * maxQ); - WriteBits(quantized, bits); - } -} - -public class BitReader -{ - private byte[] _buffer; - private int _bitPos; - - public BitReader(byte[] buffer) { _buffer = buffer; _bitPos = 0; } - - public uint ReadBits(int bits) - { - uint value = 0; - for (int i = bits - 1; i >= 0; i--) - { - int byteIdx = _bitPos / 8; - int bitIdx = 7 - (_bitPos % 8); - if ((_buffer[byteIdx] >> bitIdx & 1) == 1) value |= 1u << i; - _bitPos++; - } - return value; - } - - public float ReadQuantizedFloat(float min, float max, int bits) - { - uint maxQ = (1u << bits) - 1; - uint quantized = ReadBits(bits); - float normalized = (float)quantized / maxQ; - return min + normalized * (max - min); - } -} -``` +- Zero encodes exactly to code `0` (a zero magnitude never sets the sign bit), so proto3 omits a stopped field entirely +- `pow > 1` concentrates resolution near zero, coarse near `±max` +- Sign in the LSB makes the varint cost track magnitude, not direction — a small `|value|` of either sign stays in one varint byte --- @@ -181,13 +140,24 @@ public class BitReader Sub-centimeter precision is achievable at 12-16 bits for position deltas. +Wire cost is varint-based — 1 tag byte per present field (field numbers ≤ 15) plus: + +| Code bits | Worst-case varint | Worst-case field total | +|-----------|-------------------|------------------------| +| ≤ 7 | 1 B | 2 B | +| ≤ 14 | 2 B | 3 B | +| ≤ 21 | 3 B | 4 B | + +Proto3 omits fields equal to 0, so typical cost is lower than worst-case. + --- ## Key Design Principles - `.proto` files are the **single source of truth** for all message schemas -- The protoc plugin generates **C#** from the schema — never hand-write serialization +- The protoc plugin generates the **C# quantized accessors** from the schema — never hand-write quantization math; standard protobuf handles the wire encoding - Encode -> decode is a **no-op** (round-trip safe) due to consistent use of `Round` +- Validate inbound quantized messages with `AreQuantizedFieldsInRange()` before storing or relaying — the server relays raw codes verbatim - Prefer **client-driven resync** over proactive server corrections - Push complexity to clients where appropriate; server maintains authority - Channel 0: reliable messages (STATE_FULL snapshots, ACKs, resync requests, HANDSHAKE) diff --git a/README.md b/README.md index a968d863..a846d9a2 100644 --- a/README.md +++ b/README.md @@ -90,8 +90,10 @@ Rather than a separate binary packing layer, the plugin leverages this: 2. `--csharp_out` generates the standard protobuf class with the raw `uint32` property (e.g. `PositionX`). 3. `--bitwise_out` (this plugin) generates a `partial class` extension with a - cached float accessor (e.g. `PositionXQuantized`) that encodes/decodes - transparently via `Quantize.Encode` / `Quantize.Decode`. + computed float accessor (e.g. `PositionXQuantized`) that encodes/decodes + transparently via `Quantize.Encode` / `Quantize.Decode` on every access — + the raw `uint32` property remains the single source of truth (no cache to + go stale when the raw field is mutated directly). The wire representation is a standard protobuf message — any protobuf-capable client can read it without knowledge of the plugin. @@ -133,8 +135,8 @@ message PositionDelta { | Annotation | Target type | Parameters | Effect | |---|---|---|---| -| `[(decentraland.common.quantized)]` | `uint32` | `min`, `max`, `bits` | Plugin emits a cached `float {Name}Quantized` accessor | -| `[(decentraland.common.quantized_power)]` | `uint32` | `max`, `pow`, `bits` | Power-law quantizer over `[-max, max]`: `(bits-1)`-bit magnitude (high bits) + sign (LSB), decoded as `sign·max·u^pow`. Exact zero; `pow>1` gives fine resolution near zero, coarse near `±max`; sign in the LSB keeps small magnitudes one varint byte. Cached `float {Name}Quantized` accessor (`Quantize.EncodePower`/`DecodePower`) | +| `[(decentraland.common.quantized)]` | `uint32` | `min`, `max`, `bits` | Plugin emits a `float {Name}Quantized` accessor and a `{Name}QuantizedStep` const | +| `[(decentraland.common.quantized_power)]` | `uint32` | `max`, `pow`, `bits` | Power-law quantizer over `[-max, max]`: `(bits-1)`-bit magnitude (high bits) + sign (LSB), decoded as `sign·max·u^pow`. Exact zero; `pow>1` gives fine resolution near zero, coarse near `±max`; sign in the LSB keeps small magnitudes one varint byte. `float {Name}Quantized` accessor (`Quantize.EncodePower`/`DecodePower`) | | `[(decentraland.common.bit_packed)]` | `uint32` | `bits` | Documents the value range; protobuf handles varint compaction automatically | ### Wire cost at worst-case (all bits set) @@ -183,13 +185,15 @@ Assets/ ``` `Quantize.cs` lives in the `Decentraland.Networking.Bitwise` namespace and -provides two static methods used by the generated accessors: +provides the static encode/decode methods used by the generated accessors: ```csharp public static class Quantize { public static uint Encode(float value, float min, float max, int bits); public static float Decode(uint encoded, float min, float max, int bits); + public static uint EncodePower(float value, float max, float pow, int bits); + public static float DecodePower(uint encoded, float max, float pow, int bits); } ``` @@ -213,6 +217,7 @@ SendOnChannel1(bytes); // --- Receive and read --- var received = PositionDelta.Parser.ParseFrom(receivedBytes); +if (!received.AreQuantizedFieldsInRange()) return; // reject malformed/hostile codes float x = received.DxQuantized; // decoded from the stored uint32 on each access float y = received.DyQuantized; float z = received.DzQuantized; @@ -220,7 +225,10 @@ float z = received.DzQuantized; ## Generated file example -For the `PositionDelta` message above the plugin emits `PositionDelta.Bitwise.cs`: +For the `comms.proto` file above the plugin emits `Comms.Bitwise.cs` (one file +per `.proto`, named after the proto file). Each quantized field gets a +`{Name}QuantizedStep` const and a float accessor; each message gets an +`AreQuantizedFieldsInRange()` guard for validating inbound wire codes: ```csharp // @@ -234,23 +242,44 @@ namespace Decentraland.Kernel.Comms.V3 { public partial class PositionDelta { + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float DxQuantizedStep = 0.0030518044f; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. public float DxQuantized { get => Quantize.Decode(Dx, -100.0f, 100.0f, 16); set => Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); } + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float DyQuantizedStep = 0.0030518044f; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. public float DyQuantized { get => Quantize.Decode(Dy, -100.0f, 100.0f, 16); set => Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); } + /// Coarsest quantization step of . Safe as an equality tolerance. + public const float DzQuantizedStep = 0.0030518044f; + /// Float accessor for . Range [-100.0f, 100.0f], 16 bits, step ≈ 0.0030518. public float DzQuantized { get => Quantize.Decode(Dz, -100.0f, 100.0f, 16); set => Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); } + + /// + /// True when every quantized field holds a wire code within its declared bit width + /// (0 .. 2^bits-1). The encoder never emits a code above this bound, so a larger + /// value is a malformed/hostile message: decoding it would land far outside the field's + /// [min, max] and, since the server relays raw codes verbatim, poison every observer. + /// Reject before storing or relaying. Pure integer comparison — no decode. + /// + public bool AreQuantizedFieldsInRange() => + Dx <= 65535u + && Dy <= 65535u + && Dz <= 65535u; } } // namespace Decentraland.Kernel.Comms.V3 diff --git a/protoc-gen-bitwise/runtime/cs/BitReader.cs b/protoc-gen-bitwise/runtime/cs/BitReader.cs deleted file mode 100644 index 51efa39c..00000000 --- a/protoc-gen-bitwise/runtime/cs/BitReader.cs +++ /dev/null @@ -1,112 +0,0 @@ -// Decentraland.Networking.Bitwise — BitReader -// Copy this file into your Unity project alongside generated *.Bitwise.cs files. - -using System; - -namespace Decentraland.Networking.Bitwise -{ - /// - /// Reads bits from a byte buffer, MSB first within each byte (big-endian bit - /// order). Symmetric counterpart of : every - /// Write… call has a corresponding Read… call with identical arguments that - /// reproduces the original value. - /// - public sealed class BitReader - { - private readonly byte[] _buffer; - private int _bitPos; - - /// Source buffer filled by a . - public BitReader(byte[] buffer) - { - _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer)); - _bitPos = 0; - } - - /// Current read position in bits. - public int BitPosition => _bitPos; - - /// - /// Returns true when all written bits have been consumed - /// (i.e. has reached the end of the buffer). - /// - public bool IsAtEnd => _bitPos >= _buffer.Length * 8; - - // ----------------------------------------------------------------- - // Core primitive - // ----------------------------------------------------------------- - - /// - /// Reads bits and returns them as the - /// least-significant bits of a , MSB first. - /// - public uint ReadBits(int bits) - { - uint value = 0; - for (int i = bits - 1; i >= 0; i--) - { - int byteIdx = _bitPos / 8; - int bitIdx = 7 - (_bitPos % 8); - - if ((_buffer[byteIdx] >> bitIdx & 1) == 1) - value |= 1u << i; - - _bitPos++; - } - return value; - } - - // ----------------------------------------------------------------- - // Quantized float - // ----------------------------------------------------------------- - - /// - /// Reads a quantized float encoded with . - /// Arguments must match those used during encoding exactly. - /// - public float ReadQuantizedFloat(float min, float max, int bits) - { - uint maxQ = (1u << bits) - 1; - uint quantized = ReadBits(bits); - float normalized = (float)quantized / maxQ; - return min + normalized * (max - min); - } - - // ----------------------------------------------------------------- - // Standard IEEE 754 helpers - // ----------------------------------------------------------------- - - /// Reads a 32-bit IEEE 754 float written by . - public float ReadFloat() - { - uint bits = ReadBits(32); - byte[] bytes = - { - (byte)(bits & 0xFF), - (byte)((bits >> 8) & 0xFF), - (byte)((bits >> 16) & 0xFF), - (byte)((bits >> 24) & 0xFF), - }; - return BitConverter.ToSingle(bytes, 0); - } - - /// Reads a 64-bit IEEE 754 double written by . - public double ReadDouble() - { - uint hi = ReadBits(32); - uint lo = ReadBits(32); - byte[] bytes = - { - (byte)(lo & 0xFF), - (byte)((lo >> 8) & 0xFF), - (byte)((lo >> 16) & 0xFF), - (byte)((lo >> 24) & 0xFF), - (byte)(hi & 0xFF), - (byte)((hi >> 8) & 0xFF), - (byte)((hi >> 16) & 0xFF), - (byte)((hi >> 24) & 0xFF), - }; - return BitConverter.ToDouble(bytes, 0); - } - } -} diff --git a/protoc-gen-bitwise/runtime/cs/BitWriter.cs b/protoc-gen-bitwise/runtime/cs/BitWriter.cs deleted file mode 100644 index 19b205b8..00000000 --- a/protoc-gen-bitwise/runtime/cs/BitWriter.cs +++ /dev/null @@ -1,117 +0,0 @@ -// Decentraland.Networking.Bitwise — BitWriter -// Copy this file into your Unity project alongside generated *.Bitwise.cs files. - -using System; - -namespace Decentraland.Networking.Bitwise -{ - /// - /// Writes bits into a pre-allocated byte buffer, MSB first within each byte - /// (big-endian bit order). This matches the layout expected by - /// so that encode → decode is always a round-trip no-op. - /// - public sealed class BitWriter - { - private readonly byte[] _buffer; - private int _bitPos; - - /// Destination buffer (must be large enough for all writes). - public BitWriter(byte[] buffer) - { - _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer)); - _bitPos = 0; - } - - /// Current write position in bits. - public int BitPosition => _bitPos; - - /// Number of bytes written (rounded up to the nearest byte). - public int ByteLength => (_bitPos + 7) / 8; - - /// Returns a copy of the written bytes (trimmed to ). - public byte[] ToArray() - { - var result = new byte[ByteLength]; - Array.Copy(_buffer, result, ByteLength); - return result; - } - - // ----------------------------------------------------------------- - // Core primitive - // ----------------------------------------------------------------- - - /// - /// Writes the least-significant bits of - /// , MSB first. - /// - public void WriteBits(uint value, int bits) - { - for (int i = bits - 1; i >= 0; i--) - { - int byteIdx = _bitPos / 8; - int bitIdx = 7 - (_bitPos % 8); - - if ((value >> i & 1u) == 1u) - _buffer[byteIdx] |= (byte)(1 << bitIdx); - else - _buffer[byteIdx] &= (byte)~(1 << bitIdx); - - _bitPos++; - } - } - - // ----------------------------------------------------------------- - // Quantized float - // ----------------------------------------------------------------- - - /// - /// Quantizes into bits - /// using the range [, ] and - /// writes it to the buffer. - /// - /// Uses Math.Round (banker's rounding → ties-to-even) to guarantee - /// that encode → decode is a round-trip no-op. - /// - public void WriteQuantizedFloat(float value, float min, float max, int bits) - { - uint maxQ = (1u << bits) - 1; - float clamped = Math.Clamp(value, min, max); - float normalized = (clamped - min) / (max - min); - uint quantized = (uint)Math.Round(normalized * maxQ); - WriteBits(quantized, bits); - } - - // ----------------------------------------------------------------- - // Standard IEEE 754 helpers (used for un-annotated float/double fields) - // ----------------------------------------------------------------- - - /// Writes a 32-bit IEEE 754 float (4 bytes). - public void WriteFloat(float value) - { - byte[] bytes = BitConverter.GetBytes(value); - // GetBytes is little-endian on all platforms; write MSB first. - uint bits = (uint)bytes[0] - | ((uint)bytes[1] << 8) - | ((uint)bytes[2] << 16) - | ((uint)bytes[3] << 24); - WriteBits(bits, 32); - } - - /// Writes a 64-bit IEEE 754 double (8 bytes). - public void WriteDouble(double value) - { - byte[] bytes = BitConverter.GetBytes(value); - uint lo = (uint)bytes[0] - | ((uint)bytes[1] << 8) - | ((uint)bytes[2] << 16) - | ((uint)bytes[3] << 24); - uint hi = (uint)bytes[4] - | ((uint)bytes[5] << 8) - | ((uint)bytes[6] << 16) - | ((uint)bytes[7] << 24); - // Write high 32 bits first so the bit stream is big-endian at word level too. - WriteBits(hi, 32); - WriteBits(lo, 32); - } - } -} From 818f24d08b077d58026adc1774c03dd35c5a8a3c Mon Sep 17 00:00:00 2001 From: Mikhail Agapov Date: Tue, 7 Jul 2026 16:46:58 +0300 Subject: [PATCH 51/54] fix: normalize CRLF when comparing gen:test golden fixtures With core.autocrlf=true (and no .gitattributes) git materializes the golden .cs fixtures with CRLF on Windows while the generator always emits LF, so the strict byte comparison failed on any fresh Windows checkout. Normalize line endings when reading the goldens. Co-Authored-By: Claude Fable 5 --- protoc-gen-bitwise/test/generator.test.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/protoc-gen-bitwise/test/generator.test.js b/protoc-gen-bitwise/test/generator.test.js index dc20b39e..d4b26b4b 100644 --- a/protoc-gen-bitwise/test/generator.test.js +++ b/protoc-gen-bitwise/test/generator.test.js @@ -83,7 +83,9 @@ function field(name, type, optionsRaw) { } function readGolden(name) { - return fs.readFileSync(path.join(__dirname, 'golden', name), 'utf8') + // Git may materialize the goldens with CRLF on Windows (core.autocrlf=true, + // no .gitattributes); the generator always emits LF, so normalize first. + return fs.readFileSync(path.join(__dirname, 'golden', name), 'utf8').replace(/\r\n/g, '\n') } // -------------------------------------------------------------------------- From 5965969face6e4c991ce9cffccdc76e85dc69716 Mon Sep 17 00:00:00 2001 From: Lorenzo Ranciaffi Date: Thu, 9 Jul 2026 09:24:22 +0200 Subject: [PATCH 52/54] fix: add realm in pulse PlayerJoined message --- proto/decentraland/pulse/pulse_server.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/proto/decentraland/pulse/pulse_server.proto b/proto/decentraland/pulse/pulse_server.proto index 1a9ca80e..4a054ccd 100644 --- a/proto/decentraland/pulse/pulse_server.proto +++ b/proto/decentraland/pulse/pulse_server.proto @@ -84,6 +84,7 @@ message PlayerJoined { string user_id = 1; int32 profile_version = 2; PlayerStateFull state = 3; + string realm = 4; } // Notification to the client, that a peer has left, it can mean disconnection or leaving the area of interest From 1c58939deac4a858f1f2a54d0fb9f24cd57be212 Mon Sep 17 00:00:00 2001 From: Lorenzo Ranciaffi Date: Fri, 10 Jul 2026 14:27:26 +0200 Subject: [PATCH 53/54] added realm to teleport performed message --- proto/decentraland/pulse/pulse_server.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/proto/decentraland/pulse/pulse_server.proto b/proto/decentraland/pulse/pulse_server.proto index 4a054ccd..9957ac32 100644 --- a/proto/decentraland/pulse/pulse_server.proto +++ b/proto/decentraland/pulse/pulse_server.proto @@ -126,6 +126,7 @@ message TeleportPerformed { uint32 sequence = 2; uint32 server_tick = 3; PlayerState state = 4; + string realm = 5; } message ServerMessage { From a442c7190a1e26b378f930d5b336eee81044bfa3 Mon Sep 17 00:00:00 2001 From: Mikhail Agapov <118179774+mikhail-dcl@users.noreply.github.com> Date: Wed, 22 Jul 2026 17:16:00 +0300 Subject: [PATCH 54/54] feat: Pulse pure scenes listeners (#437) * feat: add SceneListenerHandshakeRequest to ClientMessage Scene-listener connect message: signed-fetch auth chain, realm, and an immutable parcel-index AoI. New ClientMessage oneof case (= 8). Co-Authored-By: Claude Fable 5 * feat: scene-listener AoI as parcel rects Replace SceneListenerHandshakeRequest.parcel_indices with inclusive parcel-coordinate rects (sint32) so big scenes announce their AoI in a few bytes. The sum of rect areas is capped server-side. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- proto/decentraland/pulse/pulse_client.proto | 22 +++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/proto/decentraland/pulse/pulse_client.proto b/proto/decentraland/pulse/pulse_client.proto index 09760dab..aca97cbd 100644 --- a/proto/decentraland/pulse/pulse_client.proto +++ b/proto/decentraland/pulse/pulse_client.proto @@ -66,6 +66,27 @@ message TeleportRequest { string realm = 5; } +// Inclusive rectangle in parcel coordinates. A single parcel is min == max. +message ParcelRect { + sint32 min_x = 1; + sint32 min_z = 2; + sint32 max_x = 3; + sint32 max_z = 4; +} + +// Scene-listener connect: same signed-fetch auth chain as HandshakeRequest, plus an +// immutable parcel-set area of interest. No initial state — a listener is never a subject. +message SceneListenerHandshakeRequest { + // Signed-fetch headers JSON — identical shape to HandshakeRequest.auth_chain. + bytes auth_chain = 1; + // AoI realm partition — same rules as TeleportRequest.realm. + string realm = 2; + // Announced AoI as inclusive parcel-coordinate rects; fixed for the connection + // lifetime. The sum of rect areas is capped server-side (SceneListener:MaxParcels) — + // overlaps count per rect, so announce disjoint rects. + repeated ParcelRect parcel_rects = 3; +} + message ClientMessage { oneof message { HandshakeRequest handshake = 1; @@ -75,5 +96,6 @@ message ClientMessage { EmoteStart emote_start = 5; EmoteStop emote_stop = 6; TeleportRequest teleport = 7; + SceneListenerHandshakeRequest scene_listener_handshake = 8; } }