Replace remerkleable with eth-ssz-specs

pyproject.toml

@@ -12,7 +12,7 @@ readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.12, <3.15"
 urls = { homepage = "https://github.com/ethereum/consensus-specs" }
 dependencies = [
-  "eth-remerkleable==0.1.31",
+  "eth-ssz-specs==0.0.1.dev2",
   "eth-utils==6.0.0",
   "frozendict==2.4.7",
   "lru-dict==1.4.1",

pysetup/generate_specs.py

@@ -1,4 +1,5 @@
 import argparse
+import ast
 import copy
 import sys
 from collections import OrderedDict
@@ -17,7 +18,7 @@
     objects_to_spec,
     parse_config_vars,
 )
-from pysetup.md_doc_paths import get_md_doc_paths
+from pysetup.md_doc_paths import get_md_doc_paths, PREVIOUS_FORK_OF
 from pysetup.md_to_spec import MarkdownToSpec
 from pysetup.spec_builders import spec_builders
 from pysetup.typing import BuildTarget, SpecObject  # type: ignore[attr-defined]
@@ -99,7 +100,59 @@ def build_spec(
         new_objects = copy.deepcopy(class_objects)
         dependency_order_class_objects(class_objects)
 
-    return objects_to_spec(preset_name, spec_object, fork, class_objects)
+    # Names this fork's own documents declare, as opposed to the ones it inherits.
+    redefined: set[str] = set()
+    for source_file, parsed in zip(source_files, all_specs, strict=True):
+        if f"/{fork}/" not in source_file.as_posix():
+            continue
+        redefined |= set(parsed.custom_types)
+        redefined |= set(parsed.ssz_objects)
+        redefined |= set(parsed.dataclasses)
+
+    shared_types = collect_shared_types(fork, redefined, spec_object, class_objects)
+
+    return objects_to_spec(preset_name, spec_object, fork, class_objects, shared_types)
+
+
+def collect_shared_types(
+    fork: str,
+    redefined: set[str],
+    spec_object: SpecObject,
+    class_objects: dict[str, str],
+) -> dict[str, str]:
+    """
+    Find the types this fork inherits unchanged, mapped to the module they come from.
+
+    The SSZ type system compares by exact type, so a ``Slot`` built under one fork
+    must be the same class as the next fork's ``Slot``, and a container handed to
+    ``upgrade_to_<fork>`` must be an instance of the field's declared class. Binding
+    an unchanged type to the previous fork's class is what makes that hold.
+
+    A type counts as unchanged only when this fork's own documents neither declare
+    it nor declare anything it is built from -- a container holding a redefined
+    field type is a different shape, and has to be declared again here.
+    """
+    previous = PREVIOUS_FORK_OF[fork]
+    if previous is None:
+        return {}
+
+    definitions = {**spec_object.custom_types, **class_objects}
+    # What each definition is built from, read as code rather than as text: a
+    # docstring that says which container a branch proves against does not make
+    # the branch depend on that container.
+    built_from = {
+        name: {node.id for node in ast.walk(ast.parse(text)) if isinstance(node, ast.Name)}
+        for name, text in definitions.items()
+    }
+
+    redefined = set(redefined)
+    candidates = set(definitions) - redefined
+    # Whatever is built from a redefined type is itself redefined. Settle it.
+    while newly_redefined := {n for n in candidates if built_from[n] & redefined}:
+        candidates -= newly_redefined
+        redefined |= newly_redefined
+
+    return dict.fromkeys(sorted(candidates), previous)
 
 
 def parse_build_targets(targets_str: str) -> list[BuildTarget]:

pysetup/helpers.py

@@ -21,20 +21,8 @@ def collect_prev_forks(fork: str) -> list[str]:
         forks.append(fork)
 
 
-def requires_mypy_type_ignore(value: str) -> bool:
-    return (
-        value.startswith(("BitList", "ByteVector"))
-        or (value.startswith("List") and not re.match(r"^List\[\w+,\s*\w+\]$", value))
-        or (value.startswith("Vector") and any(k in value for k in ["ceillog2", "floorlog2"]))
-    )
-
-
 def gen_new_type_definition(name: str, value: str) -> str:
-    return (
-        f"class {name}({value}):\n    pass"
-        if not requires_mypy_type_ignore(value)
-        else f"class {name}(\n    {value}  # type: ignore\n):\n    pass"
-    )
+    return f"class {name}({value}):\n    pass"
 
 
 def make_function_abstract(protocol_def: ProtocolDefinition, key: str):
@@ -43,15 +31,31 @@ def make_function_abstract(protocol_def: ProtocolDefinition, key: str):
 
 
 def objects_to_spec(
-    preset_name: str, spec_object: SpecObject, fork: str, ordered_class_objects: dict[str, str]
+    preset_name: str,
+    spec_object: SpecObject,
+    fork: str,
+    ordered_class_objects: dict[str, str],
+    shared_types: dict[str, str] | None = None,
 ) -> str:
     """
     Given all the objects that constitute a spec, combine them into a single pyfile.
+
+    ``shared_types`` maps the name of a type this fork inherits unchanged to the
+    module it is inherited from. Such a type is bound to the previous fork's class
+    instead of being declared again, so that ``phase0.Slot`` and ``bellatrix.Slot``
+    are one class. The SSZ type system compares by exact type, so a value built
+    under one fork has to stay usable under the next.
     """
+    shared_types = shared_types or {}
 
     def gen_new_type_definitions(custom_types: dict[str, str]) -> str:
         return "\n\n\n".join(
-            [gen_new_type_definition(key, value) for key, value in custom_types.items()]
+            [
+                f"{key}: TypeAlias = {shared_types[key]}.{key}"
+                if key in shared_types
+                else gen_new_type_definition(key, value)
+                for key, value in custom_types.items()
+            ]
         )
 
     new_type_definitions = gen_new_type_definitions(spec_object.custom_types)
@@ -97,7 +101,10 @@ def format_protocol(protocol_name: str, protocol_def: ProtocolDefinition) -> str
     ordered_class_objects = {
         k: v for k, v in ordered_class_objects.items() if k not in deprecate_containers
     }
-    ordered_class_objects_spec = "\n\n\n".join(ordered_class_objects.values())
+    ordered_class_objects_spec = "\n\n\n".join(
+        f"{k}: TypeAlias = {shared_types[k]}.{k}" if k in shared_types else v
+        for k, v in ordered_class_objects.items()
+    )
 
     # Access global dict of config vars for runtime configurables
     # Ignore variable between quotes and doubles quotes

pysetup/md_to_spec.py

@@ -22,7 +22,6 @@
     "ByteVector",
     "List",
     "ProgressiveBitList",
-    "ProgressiveByteList",
     "ProgressiveList",
     "Vector",
 )
@@ -46,6 +45,10 @@
     "Uint256",
 )
 
+# Calls a collection's bound may contain. Anything else is a spec helper, which
+# the generated specification defines after its types.
+BOUND_SAFE_CALLS = frozenset({"active_fields", "ceillog2", "floorlog2", *SCALAR_BASE_CLASSES})
+
 
 class MarkdownToSpec:
     def __init__(
@@ -214,44 +217,21 @@ def _process_code_class(self, source: str, cls: ast.ClassDef) -> None:
             # before them in the generated specification.
             self.spec["custom_types"][class_name] = parent_class
             return
-        if parent_class == "ProgressiveContainer":
-            source = re.sub(
-                r"^(.*ProgressiveContainer.*)$", r"\1  # type: ignore", source, flags=re.MULTILINE
-            )
-        elif parent_class in COLLECTION_BASE_CLASSES:
-            base = cls.bases[0]
-            args = []
-            if isinstance(base, ast.Subscript):
-                args = base.slice.elts if isinstance(base.slice, ast.Tuple) else [base.slice]
-            # Types whose length is given by a helper function only appear in
-            # networking schemas. They cannot be compiled, since helpers are
-            # defined after types in the generated specification. Math helpers
-            # like ceillog2 and floorlog2 are excluded, as they are defined
-            # before types.
+        if parent_class in COLLECTION_BASE_CLASSES or parent_class == "ProgressiveContainer":
+            # A collection declares its bound in the class body, as `LIMIT`,
+            # `LENGTH`, or `ACTIVE_FIELDS`. Types whose bound comes from a
+            # helper function only appear in networking schemas. They cannot be
+            # compiled, since helpers are defined after types in the generated
+            # specification. Everything available before types is allowed:
+            # scalar constructors, the builtin int, and the math helpers.
             if any(
                 isinstance(node, ast.Call)
-                and not (
-                    isinstance(node.func, ast.Name) and node.func.id in ("ceillog2", "floorlog2")
-                )
-                for arg in args
-                for node in ast.walk(arg)
+                and not (isinstance(node.func, ast.Name) and node.func.id in BOUND_SAFE_CALLS)
+                for statement in cls.body
+                if isinstance(statement, ast.Assign)
+                for node in ast.walk(statement)
             ):
                 return
-            # mypy accepts a subscripted base class only when all of its
-            # arguments are plain names or literals. Expressions like `A + 1`
-            # or `A * B` make the base class invalid and require an ignore
-            # comment. Configuration variables also require one, since they
-            # are rewritten to `config.X` attribute expressions.
-            if not all(
-                isinstance(arg, ast.Constant)
-                or (isinstance(arg, ast.Name) and arg.id not in self.config)
-                for arg in args
-            ):
-                # The comment must go on the line where the base class
-                # expression ends, as that is where mypy reports the error.
-                source_lines = source.split("\n")
-                source_lines[base.end_lineno - cls.lineno] += "  # type: ignore"
-                source = "\n".join(source_lines)
         else:
             assert parent_class is None or parent_class == "Container"
         self.spec["ssz_objects"][class_name] = source
@@ -283,7 +263,6 @@ def _process_table(self, table: Table) -> None:
                         "Bytes",
                         "List",
                         "ProgressiveBitList",
-                        "ProgressiveByteList",
                         "ProgressiveList",
                         "Union",
                         "Vector",

pysetup/spec_builders/altair.py

@@ -12,8 +12,7 @@ def imports(cls, preset_name: str) -> str:
 from typing import NewType, Union as PyUnion
 
 from eth_consensus_specs.phase0 import {preset_name} as phase0
-from eth_consensus_specs.test.helpers.merkle import build_proof
-from eth_consensus_specs.utils.ssz.ssz_typing import Path
+from eth_consensus_specs.test.helpers.merkle import build_proof, get_generalized_index
 """
 
     @classmethod
@@ -26,16 +25,9 @@ def preparations(cls):
     @classmethod
     def sundry_functions(cls) -> str:
         return """
-def get_generalized_index(ssz_class: Any, *path: PyUnion[int, SSZVariableName]) -> GeneralizedIndex:
-    ssz_path = Path(ssz_class)
-    for item in path:
-        ssz_path = ssz_path / item
-    return GeneralizedIndex(ssz_path.gindex())
-
-
 def compute_merkle_proof(object: SSZObject,
                          index: GeneralizedIndex) -> list[Bytes32]:
-    return build_proof(object.get_backing(), index)"""
+    return build_proof(object, index)"""
 
     @classmethod
     def hardcoded_ssz_dep_constants(cls) -> dict[str, str]:

pysetup/spec_builders/bellatrix.py

@@ -11,7 +11,8 @@ def imports(cls, preset_name: str):
         return f"""
 from typing import Protocol
 from eth_consensus_specs.altair import {preset_name} as altair
-from eth_consensus_specs.utils.ssz.ssz_typing import Bytes8, ByteList, ByteVector
+from ssz.byte_arrays import ByteList, ByteVector
+from eth_consensus_specs.utils.ssz.bytes import Bytes8
 """
 
     @classmethod

pysetup/spec_builders/gloas.py

@@ -9,7 +9,9 @@ class GloasSpecBuilder(BaseSpecBuilder):
     @classmethod
     def imports(cls, preset_name: str):
         return f"""
-from eth_consensus_specs.utils.ssz.ssz_typing import ProgressiveBitList, ProgressiveByteList, ProgressiveContainer, ProgressiveList
+from ssz.bitfields import ProgressiveBitList
+from ssz.collections import ProgressiveList
+from ssz.container import active_fields, ProgressiveContainer
 
 from eth_consensus_specs.fulu import {preset_name} as fulu
 """

pysetup/spec_builders/phase0.py

@@ -29,19 +29,23 @@ def imports(cls, preset_name: str) -> str:
     Any, Callable, Dict, DefaultDict, Set, Sequence, Tuple, Optional, TypeAlias, TypeVar, NamedTuple, Final
 )
 
+from ssz.bitfields import BitList, BitVector
+from ssz.boolean import Boolean
+from ssz.collections import List, Vector
+from ssz.container import Container
+from ssz.ssz_base import SSZType
+from ssz.uint import Byte, Uint8, Uint32, Uint64, Uint256
+from eth_consensus_specs.utils.ssz.bytes import (
+    Bytes1, Bytes4, Bytes20, Bytes32, Bytes48, Bytes96)
 from eth_consensus_specs.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes
-from eth_consensus_specs.utils.ssz.ssz_typing import (
-    View, Boolean, Byte, Container, List, Vector, Uint8, Uint32, Uint64, Uint256,
-    Bytes1, Bytes4, Bytes20, Bytes32, Bytes48, Bytes96, BitList)
-from eth_consensus_specs.utils.ssz.ssz_typing import BitVector  # noqa: F401
 from eth_consensus_specs.utils import bls
 from eth_consensus_specs.utils.hash_function import hash
 """
 
     @classmethod
     def preparations(cls) -> str:
         return """
-SSZObject = TypeVar('SSZObject', bound=View)
+SSZObject = TypeVar('SSZObject', bound=SSZType)
 """
 
     @classmethod

specs/_features/eip8025/beacon-chain.md

@@ -37,7 +37,7 @@ and imports proof types from [proof-engine.md](./proof-engine.md).
 ### New `ProofData`
 
 ```python
-class ProofData(ProgressiveByteList):
+class ProofData(ProgressiveList[Byte]):
     """
     The opaque proof bytes of an execution proof.
     """

specs/_features/eip8025/p2p-interface.md

@@ -57,30 +57,36 @@ and imports proof types from [proof-engine.md](./proof-engine.md).
 ### New `ProofByRootIdentifiers`
 
 ```python
-class ProofByRootIdentifiers(List[ProofByRootIdentifier, MAX_REQUEST_BLOCKS_DENEB]):
+class ProofByRootIdentifiers(List[ProofByRootIdentifier]):
     """
     The identifiers of the execution proofs requested in an
     ``ExecutionProofsByRoot`` request.
     """
+
+    LIMIT = MAX_REQUEST_BLOCKS_DENEB
 ```
 
 ### New `ProofTypes`
 
 ```python
-class ProofTypes(List[ProofType, MAX_EXECUTION_PROOFS_PER_PAYLOAD]):
+class ProofTypes(List[ProofType]):
     """
     A selection of execution proof types.
     """
+
+    LIMIT = MAX_EXECUTION_PROOFS_PER_PAYLOAD
 ```
 
 ### New `SignedExecutionProofs`
 
 ```python
-class SignedExecutionProofs(List[SignedExecutionProof, compute_max_request_execution_proofs()]):
+class SignedExecutionProofs(List[SignedExecutionProof]):
     """
     Signed execution proofs returned in an ``ExecutionProofsByRange`` or
     ``ExecutionProofsByRoot`` response.
     """
+
+    LIMIT = compute_max_request_execution_proofs()
 ```
 
 ## Containers

specs/_features/eip8148/beacon-chain.md

@@ -102,7 +102,9 @@ class SweepThresholds(ProgressiveList[Gwei]):
 #### `BeaconState`
 
 ```python
-class BeaconState(ProgressiveContainer(active_fields=[1] * 47)):
+class BeaconState(ProgressiveContainer):
+    ACTIVE_FIELDS = active_fields(width=47)
+
     genesis_time: Uint64
     genesis_validators_root: Root
     slot: Slot
@@ -156,7 +158,9 @@ class BeaconState(ProgressiveContainer(active_fields=[1] * 47)):
 #### `ExecutionRequests`
 
 ```python
-class ExecutionRequests(ProgressiveContainer(active_fields=[1] * 6)):
+class ExecutionRequests(ProgressiveContainer):
+    ACTIVE_FIELDS = active_fields(width=6)
+
     deposits: DepositRequests
     withdrawals: WithdrawalRequests
     consolidations: ConsolidationRequests