refactor: migrate all encode/decode to stream-based APIs

examples/example_decode_block.py

@@ -11,13 +11,15 @@
 """
 
 import asyncio
+import io
 import random
 
 from pactus.block import Block
 from pactus_jsonrpc.client import PactusOpenRPCClient
 
 TESTNET_RPC = "https://testnet1.pactus.org/jsonrpc"
 
+
 async def fetch_and_decode_block():
     client = PactusOpenRPCClient(
         headers={},
@@ -34,7 +36,7 @@ async def fetch_and_decode_block():
     print(f"\nFetching block at height {height}...")
 
     res = await client.pactus.blockchain.get_block(height=height, verbosity=0)
-    block = Block.decode(bytes.fromhex(res["data"]))
+    block = Block.decode(io.BytesIO(bytes.fromhex(res["data"])))
 
     print(f"  - Hash:       {block.id}")
     print(f"  - Version:    {block.header.version}")
@@ -48,10 +50,10 @@ async def fetch_and_decode_block():
     print(f"    - Absentees:  {block.prev_cert.absentees}")
     print(f"    - Signature:  {block.prev_cert.signature.string()}")
 
-
     print("  - Transactions:")
     for i, tx in enumerate(block.transactions):
         print(f"     - {i}: {tx.id()}")
 
+
 if __name__ == "__main__":
     asyncio.run(fetch_and_decode_block())

pactus/block/block.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import hashlib
+import io
 import struct
 
 from pactus.crypto.hash import Hash
@@ -23,22 +24,22 @@ def __init__(
         self.transactions = transactions
 
     @classmethod
-    def decode(cls, data: bytes) -> Block:
-        """Decode a Block from raw bytes."""
-        header, data = Header.decode(data)
+    def decode(cls, buf: io.BufferedReader) -> Block:
+        """Decode a Block from the stream."""
+        header = Header.decode(buf)
 
         # Genesis block has no certificate
         is_genesis = header.prev_block_hash.is_undef()
 
         prev_cert = None
         if not is_genesis:
-            prev_cert, data = Certificate.decode(data)
+            prev_cert = Certificate.decode(buf)
 
-        num_txs, data = encoding.read_var_int(data)
+        num_txs = encoding.read_var_int(buf)
 
         transactions = []
         for _ in range(num_txs):
-            tx, data = Transaction.decode(data)
+            tx = Transaction.decode(buf)
             transactions.append(tx)
 
         return cls(header, prev_cert, transactions)
@@ -54,7 +55,9 @@ def id(self) -> Hash:
         return Hash(hashlib.blake2b(buf, digest_size=32).digest())
 
     def _header_bytes(self) -> bytes:
-        return self.header.encode(b"")
+        buf = io.BytesIO()
+        self.header.encode(buf)
+        return buf.getvalue()
 
     def _txs_root(self) -> bytes:
         return Block._merkle_root([tx.id().data for tx in self.transactions])

pactus/block/certificate.py

@@ -1,4 +1,7 @@
+from __future__ import annotations
+
 import hashlib
+import io
 
 from pactus.crypto.bls.signature import Signature
 from pactus.encoding import encoding
@@ -22,42 +25,40 @@ def __init__(
         self.signature = signature
 
     @classmethod
-    def decode(cls, buf: bytes) -> tuple:
-        """
-        Decode a Certificate from bytes.
-        Returns (Certificate, remaining_buf).
-        """
-        height, buf = Height.decode(buf)
-        round_, buf = Round.decode(buf)
-
-        num_committers, buf = encoding.read_var_int(buf)
+    def decode(cls, buf: io.BufferedReader) -> Certificate:
+        """Decode a Certificate from the stream."""
+        height = Height.decode(buf)
+        round_ = Round.decode(buf)
+
+        num_committers = encoding.read_var_int(buf)
         committers = []
         for _ in range(num_committers):
-            n, buf = encoding.read_var_int(buf)
+            n = encoding.read_var_int(buf)
             committers.append(n)
 
-        num_absentees, buf = encoding.read_var_int(buf)
+        num_absentees = encoding.read_var_int(buf)
         absentees = []
         for _ in range(num_absentees):
-            n, buf = encoding.read_var_int(buf)
+            n = encoding.read_var_int(buf)
             absentees.append(n)
 
-        signature, buf = Signature.decode(buf)
+        signature = Signature.decode(buf)
 
-        cert = cls(height, round_, committers, absentees, signature)
-        return cert, buf
+        return cls(height, round_, committers, absentees, signature)
 
-    def encode(self, buf: bytes) -> bytes:
-        buf = self.height.encode(buf)
-        buf = self.round.encode(buf)
-        buf = encoding.append_var_int(buf, len(self.committers))
+    def encode(self, buf: io.Writer) -> None:
+        self.height.encode(buf)
+        self.round.encode(buf)
+        encoding.append_var_int(buf, len(self.committers))
         for n in self.committers:
-            buf = encoding.append_var_int(buf, n)
-        buf = encoding.append_var_int(buf, len(self.absentees))
+            encoding.append_var_int(buf, n)
+        encoding.append_var_int(buf, len(self.absentees))
         for n in self.absentees:
-            buf = encoding.append_var_int(buf, n)
-        return self.signature.encode(buf)
+            encoding.append_var_int(buf, n)
+        self.signature.encode(buf)
 
     def hash(self) -> bytes:
         """Return the certificate hash (blake2b-256 of encoded bytes)."""
-        return hashlib.blake2b(self.encode(b""), digest_size=32).digest()
+        buf = io.BytesIO()
+        self.encode(buf)
+        return hashlib.blake2b(buf.getvalue(), digest_size=32).digest()

pactus/block/header.py

@@ -1,3 +1,10 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import io
+
 from pactus.crypto.address import Address
 from pactus.crypto.hash import Hash
 from pactus.encoding import encoding
@@ -21,32 +28,28 @@ def __init__(
         self.proposer_address = proposer_address
 
     @classmethod
-    def decode(cls, buf: bytes) -> tuple:
-        """
-        Decode a Header from bytes.
-        Returns (Header, remaining_buf).
-        """
-        version, buf = encoding.read_uint8(buf)
-        unix_time, buf = encoding.read_uint32(buf)
-        prev_block_hash, buf = Hash.decode(buf)
-        state_root, buf = Hash.decode(buf)
-        sortition_seed, buf = encoding.read_fixed_bytes(buf, 48)
-        proposer_address, buf = Address.decode(buf)
-
-        header = cls(
+    def decode(cls, buf: io.BufferedReader) -> Header:
+        """Decode a Header from the stream."""
+        version = encoding.read_uint8(buf)
+        unix_time = encoding.read_uint32(buf)
+        prev_block_hash = Hash.decode(buf)
+        state_root = Hash.decode(buf)
+        sortition_seed = encoding.read_fixed_bytes(buf, 48)
+        proposer_address = Address.decode(buf)
+
+        return cls(
             version,
             unix_time,
             prev_block_hash,
             state_root,
             sortition_seed,
             proposer_address,
         )
-        return header, buf
-
-    def encode(self, buf: bytes) -> bytes:
-        buf = encoding.append_uint8(buf, self.version)
-        buf = encoding.append_uint32(buf, self.unix_time)
-        buf = self.prev_block_hash.encode(buf)
-        buf = self.state_root.encode(buf)
-        buf = encoding.append_fixed_bytes(buf, self.sortition_seed)
-        return self.proposer_address.encode(buf)
+
+    def encode(self, buf: io.Writer) -> None:
+        encoding.append_uint8(buf, self.version)
+        encoding.append_uint32(buf, self.unix_time)
+        self.prev_block_hash.encode(buf)
+        self.state_root.encode(buf)
+        encoding.append_fixed_bytes(buf, self.sortition_seed)
+        self.proposer_address.encode(buf)

pactus/crypto/address.py

@@ -1,11 +1,15 @@
 from __future__ import annotations
 
 from enum import Enum
+from typing import TYPE_CHECKING
 
 from pactus.crypto.hrp import HRP
 from pactus.encoding import encoding
 from pactus.utils import utils
 
+if TYPE_CHECKING:
+    import io
+
 # Address format: hrp + `1` + type + data + checksum
 
 ADDRESS_SIZE = 21
@@ -86,20 +90,18 @@ def is_account_address(self) -> bool:
     def is_validator_address(self) -> bool:
         return self.address_type() == AddressType.VALIDATOR
 
-    def encode(self, buf: bytes) -> bytes:
+    def encode(self, buf: io.Writer) -> None:
         if self.is_treasury_address():
-            return encoding.append_uint8(buf, AddressType.TREASURY.value)
-        return encoding.append_fixed_bytes(buf, self.raw_bytes())
+            encoding.append_uint8(buf, AddressType.TREASURY.value)
+        else:
+            encoding.append_fixed_bytes(buf, self.raw_bytes())
 
     @classmethod
-    def decode(cls, buf: bytes) -> tuple:
-        """
-        Decode an Address from bytes.
-        Returns (Address, remaining_buf).
-        """
-        addr_type, buf = encoding.read_uint8(buf)
+    def decode(cls, buf: io.BufferedReader) -> Address:
+        """Decode an Address from the stream."""
+        addr_type = encoding.read_uint8(buf)
         if addr_type == AddressType.TREASURY.value:
-            return cls(AddressType.TREASURY, bytes(20)), buf
+            return cls(AddressType.TREASURY, bytes(20))
 
-        data, buf = encoding.read_fixed_bytes(buf, 20)
-        return cls(AddressType(addr_type), data), buf
+        data = encoding.read_fixed_bytes(buf, 20)
+        return cls(AddressType(addr_type), data)

pactus/crypto/bls/public_key.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import hashlib
+from typing import TYPE_CHECKING
 
 from ripemd.ripemd160 import ripemd160
 
@@ -13,6 +14,9 @@
 from .bls12_381.serdesZ import deserialize, serialize
 from .signature import DST, SIGNATURE_TYPE_BLS, Signature
 
+if TYPE_CHECKING:
+    import io
+
 PUBLIC_KEY_SIZE = 96
 
 
@@ -61,13 +65,13 @@ def string(self) -> str:
             self.raw_bytes(),
         )
 
-    def encode(self, buf: bytes) -> bytes:
-        return encoding.append_fixed_bytes(buf, self.raw_bytes())
+    def encode(self, buf: io.Writer) -> None:
+        encoding.append_fixed_bytes(buf, self.raw_bytes())
 
     @classmethod
-    def decode(cls, buf: bytes) -> tuple:
-        data, buf = encoding.read_fixed_bytes(buf, PUBLIC_KEY_SIZE)
-        return cls.from_bytes(data), buf
+    def decode(cls, buf: io.BufferedReader) -> PublicKey:
+        data = encoding.read_fixed_bytes(buf, PUBLIC_KEY_SIZE)
+        return cls.from_bytes(data)
 
     def account_address(self) -> Address:
         return self._make_address(AddressType.BLS_ACCOUNT)

pactus/crypto/bls/signature.py

@@ -1,10 +1,15 @@
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
 from pactus.encoding import encoding
 
 from .bls12_381.bls_sig_g1 import aggregate_sigs
 from .bls12_381.serdesZ import deserialize, serialize
 
+if TYPE_CHECKING:
+    import io
+
 SIGNATURE_SIZE = 48
 SIGNATURE_TYPE_BLS = 1
 DST = b"BLS_SIG_BLS12381G1_XMD:SHA-256_SSWU_RO_NUL_"
@@ -39,11 +44,11 @@ def raw_bytes(self) -> bytes:
     def string(self) -> str:
         return self.raw_bytes().hex()
 
-    def encode(self, buf: bytes) -> bytes:
-        return encoding.append_fixed_bytes(buf, self.raw_bytes())
+    def encode(self, buf: io.Writer) -> None:
+        encoding.append_fixed_bytes(buf, self.raw_bytes())
 
     @classmethod
-    def decode(cls, buf: bytes) -> tuple:
-        data, buf = encoding.read_fixed_bytes(buf, SIGNATURE_SIZE)
+    def decode(cls, buf: io.BufferedReader) -> Signature:
+        data = encoding.read_fixed_bytes(buf, SIGNATURE_SIZE)
         point_g1 = deserialize(data, is_ell2=False)
-        return cls(point_g1), buf
+        return cls(point_g1)

pactus/crypto/ed25519/public_key.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import hashlib
+from typing import TYPE_CHECKING
 
 from cryptography.exceptions import InvalidSignature
 from cryptography.hazmat.primitives.asymmetric import ed25519
@@ -13,6 +14,9 @@
 
 from .signature import SIGNATURE_TYPE_ED25519, Signature
 
+if TYPE_CHECKING:
+    import io
+
 PUBLIC_KEY_SIZE = 32
 
 
@@ -50,13 +54,13 @@ def string(self) -> str:
             self.raw_bytes(),
         )
 
-    def encode(self, buf: bytes) -> bytes:
-        return encoding.append_fixed_bytes(buf, self.raw_bytes())
+    def encode(self, buf: io.Writer) -> None:
+        encoding.append_fixed_bytes(buf, self.raw_bytes())
 
     @classmethod
-    def decode(cls, buf: bytes) -> tuple:
-        data, buf = encoding.read_fixed_bytes(buf, PUBLIC_KEY_SIZE)
-        return cls(ed25519.Ed25519PublicKey.from_public_bytes(data)), buf
+    def decode(cls, buf: io.BufferedReader) -> PublicKey:
+        data = encoding.read_fixed_bytes(buf, PUBLIC_KEY_SIZE)
+        return cls(ed25519.Ed25519PublicKey.from_public_bytes(data))
 
     def account_address(self) -> Address:
         blake2b = hashlib.blake2b(digest_size=32)