From eaba80adef88a6743484d180cb40c570d91a0090 Mon Sep 17 00:00:00 2001 From: cyc60 Date: Tue, 28 May 2024 17:41:41 +0300 Subject: [PATCH] Use py-ssz package --- poetry.lock | 93 ++++--- pyproject.toml | 4 +- sw_utils/signing.py | 4 +- sw_utils/ssz/__init__.py | 25 -- sw_utils/ssz/abc.py | 156 ----------- sw_utils/ssz/cache/__init__.py | 1 - sw_utils/ssz/cache/cache.py | 48 ---- sw_utils/ssz/cache/utils.py | 43 --- sw_utils/ssz/constants.py | 25 -- sw_utils/ssz/exceptions.py | 22 -- sw_utils/ssz/hash.py | 14 - sw_utils/ssz/hash_tree.py | 385 --------------------------- sw_utils/ssz/hashable_structure.py | 406 ----------------------------- sw_utils/ssz/hashable_vector.py | 26 -- sw_utils/ssz/sedes/__init__.py | 8 - sw_utils/ssz/sedes/base.py | 98 ------- sw_utils/ssz/sedes/basic.py | 177 ------------- sw_utils/ssz/sedes/byte.py | 37 --- sw_utils/ssz/sedes/byte_vector.py | 60 ----- sw_utils/ssz/sedes/container.py | 182 ------------- sw_utils/ssz/sedes/serializable.py | 395 ---------------------------- sw_utils/ssz/sedes/uint.py | 47 ---- sw_utils/ssz/sedes/vector.py | 146 ----------- sw_utils/ssz/typing.py | 10 - sw_utils/ssz/utils.py | 221 ---------------- 25 files changed, 49 insertions(+), 2584 deletions(-) delete mode 100644 sw_utils/ssz/__init__.py delete mode 100644 sw_utils/ssz/abc.py delete mode 100644 sw_utils/ssz/cache/__init__.py delete mode 100644 sw_utils/ssz/cache/cache.py delete mode 100644 sw_utils/ssz/cache/utils.py delete mode 100644 sw_utils/ssz/constants.py delete mode 100644 sw_utils/ssz/exceptions.py delete mode 100644 sw_utils/ssz/hash.py delete mode 100644 sw_utils/ssz/hash_tree.py delete mode 100644 sw_utils/ssz/hashable_structure.py delete mode 100644 sw_utils/ssz/hashable_vector.py delete mode 100644 sw_utils/ssz/sedes/__init__.py delete mode 100644 sw_utils/ssz/sedes/base.py delete mode 100644 sw_utils/ssz/sedes/basic.py delete mode 100644 sw_utils/ssz/sedes/byte.py delete mode 100644 sw_utils/ssz/sedes/byte_vector.py delete mode 100644 sw_utils/ssz/sedes/container.py delete mode 100644 sw_utils/ssz/sedes/serializable.py delete mode 100644 sw_utils/ssz/sedes/uint.py delete mode 100644 sw_utils/ssz/sedes/vector.py delete mode 100644 sw_utils/ssz/typing.py delete mode 100644 sw_utils/ssz/utils.py diff --git a/poetry.lock b/poetry.lock index 15a06b8..1251d40 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "aiohttp" @@ -1692,22 +1692,22 @@ virtualenv = ">=20.10.0" [[package]] name = "protobuf" -version = "5.26.1" +version = "5.27.0" description = "" optional = false python-versions = ">=3.8" files = [ - {file = "protobuf-5.26.1-cp310-abi3-win32.whl", hash = "sha256:3c388ea6ddfe735f8cf69e3f7dc7611e73107b60bdfcf5d0f024c3ccd3794e23"}, - {file = "protobuf-5.26.1-cp310-abi3-win_amd64.whl", hash = "sha256:e6039957449cb918f331d32ffafa8eb9255769c96aa0560d9a5bf0b4e00a2a33"}, - {file = "protobuf-5.26.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:38aa5f535721d5bb99861166c445c4105c4e285c765fbb2ac10f116e32dcd46d"}, - {file = "protobuf-5.26.1-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:fbfe61e7ee8c1860855696e3ac6cfd1b01af5498facc6834fcc345c9684fb2ca"}, - {file = "protobuf-5.26.1-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:f7417703f841167e5a27d48be13389d52ad705ec09eade63dfc3180a959215d7"}, - {file = "protobuf-5.26.1-cp38-cp38-win32.whl", hash = "sha256:d693d2504ca96750d92d9de8a103102dd648fda04540495535f0fec7577ed8fc"}, - {file = "protobuf-5.26.1-cp38-cp38-win_amd64.whl", hash = "sha256:9b557c317ebe6836835ec4ef74ec3e994ad0894ea424314ad3552bc6e8835b4e"}, - {file = "protobuf-5.26.1-cp39-cp39-win32.whl", hash = "sha256:b9ba3ca83c2e31219ffbeb9d76b63aad35a3eb1544170c55336993d7a18ae72c"}, - {file = "protobuf-5.26.1-cp39-cp39-win_amd64.whl", hash = "sha256:7ee014c2c87582e101d6b54260af03b6596728505c79f17c8586e7523aaa8f8c"}, - {file = "protobuf-5.26.1-py3-none-any.whl", hash = "sha256:da612f2720c0183417194eeaa2523215c4fcc1a1949772dc65f05047e08d5932"}, - {file = "protobuf-5.26.1.tar.gz", hash = "sha256:8ca2a1d97c290ec7b16e4e5dff2e5ae150cc1582f55b5ab300d45cb0dfa90e51"}, + {file = "protobuf-5.27.0-cp310-abi3-win32.whl", hash = "sha256:2f83bf341d925650d550b8932b71763321d782529ac0eaf278f5242f513cc04e"}, + {file = "protobuf-5.27.0-cp310-abi3-win_amd64.whl", hash = "sha256:b276e3f477ea1eebff3c2e1515136cfcff5ac14519c45f9b4aa2f6a87ea627c4"}, + {file = "protobuf-5.27.0-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:744489f77c29174328d32f8921566fb0f7080a2f064c5137b9d6f4b790f9e0c1"}, + {file = "protobuf-5.27.0-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:f51f33d305e18646f03acfdb343aac15b8115235af98bc9f844bf9446573827b"}, + {file = "protobuf-5.27.0-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:56937f97ae0dcf4e220ff2abb1456c51a334144c9960b23597f044ce99c29c89"}, + {file = "protobuf-5.27.0-cp38-cp38-win32.whl", hash = "sha256:a17f4d664ea868102feaa30a674542255f9f4bf835d943d588440d1f49a3ed15"}, + {file = "protobuf-5.27.0-cp38-cp38-win_amd64.whl", hash = "sha256:aabbbcf794fbb4c692ff14ce06780a66d04758435717107c387f12fb477bf0d8"}, + {file = "protobuf-5.27.0-cp39-cp39-win32.whl", hash = "sha256:587be23f1212da7a14a6c65fd61995f8ef35779d4aea9e36aad81f5f3b80aec5"}, + {file = "protobuf-5.27.0-cp39-cp39-win_amd64.whl", hash = "sha256:7cb65fc8fba680b27cf7a07678084c6e68ee13cab7cace734954c25a43da6d0f"}, + {file = "protobuf-5.27.0-py3-none-any.whl", hash = "sha256:673ad60f1536b394b4fa0bcd3146a4130fcad85bfe3b60eaa86d6a0ace0fa374"}, + {file = "protobuf-5.27.0.tar.gz", hash = "sha256:07f2b9a15255e3cf3f137d884af7972407b556a7a220912b252f26dc3121e6bf"}, ] [[package]] @@ -1842,8 +1842,8 @@ astroid = ">=3.2.2,<=3.3.0-dev0" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = [ {version = ">=0.2", markers = "python_version < \"3.11\""}, + {version = ">=0.3.6", markers = "python_version >= \"3.11\""}, {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, - {version = ">=0.3.6", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, ] isort = ">=4.2.5,<5.13.0 || >5.13.0,<6" mccabe = ">=0.6,<0.8" @@ -1857,40 +1857,17 @@ testutils = ["gitpython (>3)"] [[package]] name = "pyrsistent" -version = "0.19.3" +version = "0.16.1" description = "Persistent/Functional/Immutable data structures" optional = false -python-versions = ">=3.7" +python-versions = ">=2.7" files = [ - {file = "pyrsistent-0.19.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:20460ac0ea439a3e79caa1dbd560344b64ed75e85d8703943e0b66c2a6150e4a"}, - {file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c18264cb84b5e68e7085a43723f9e4c1fd1d935ab240ce02c0324a8e01ccb64"}, - {file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b774f9288dda8d425adb6544e5903f1fb6c273ab3128a355c6b972b7df39dcf"}, - {file = "pyrsistent-0.19.3-cp310-cp310-win32.whl", hash = "sha256:5a474fb80f5e0d6c9394d8db0fc19e90fa540b82ee52dba7d246a7791712f74a"}, - {file = "pyrsistent-0.19.3-cp310-cp310-win_amd64.whl", hash = "sha256:49c32f216c17148695ca0e02a5c521e28a4ee6c5089f97e34fe24163113722da"}, - {file = "pyrsistent-0.19.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f0774bf48631f3a20471dd7c5989657b639fd2d285b861237ea9e82c36a415a9"}, - {file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ab2204234c0ecd8b9368dbd6a53e83c3d4f3cab10ecaf6d0e772f456c442393"}, - {file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e42296a09e83028b3476f7073fcb69ffebac0e66dbbfd1bd847d61f74db30f19"}, - {file = "pyrsistent-0.19.3-cp311-cp311-win32.whl", hash = "sha256:64220c429e42a7150f4bfd280f6f4bb2850f95956bde93c6fda1b70507af6ef3"}, - {file = "pyrsistent-0.19.3-cp311-cp311-win_amd64.whl", hash = "sha256:016ad1afadf318eb7911baa24b049909f7f3bb2c5b1ed7b6a8f21db21ea3faa8"}, - {file = "pyrsistent-0.19.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4db1bd596fefd66b296a3d5d943c94f4fac5bcd13e99bffe2ba6a759d959a28"}, - {file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aeda827381f5e5d65cced3024126529ddc4289d944f75e090572c77ceb19adbf"}, - {file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:42ac0b2f44607eb92ae88609eda931a4f0dfa03038c44c772e07f43e738bcac9"}, - {file = "pyrsistent-0.19.3-cp37-cp37m-win32.whl", hash = "sha256:e8f2b814a3dc6225964fa03d8582c6e0b6650d68a232df41e3cc1b66a5d2f8d1"}, - {file = "pyrsistent-0.19.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c9bb60a40a0ab9aba40a59f68214eed5a29c6274c83b2cc206a359c4a89fa41b"}, - {file = "pyrsistent-0.19.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a2471f3f8693101975b1ff85ffd19bb7ca7dd7c38f8a81701f67d6b4f97b87d8"}, - {file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc5d149f31706762c1f8bda2e8c4f8fead6e80312e3692619a75301d3dbb819a"}, - {file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3311cb4237a341aa52ab8448c27e3a9931e2ee09561ad150ba94e4cfd3fc888c"}, - {file = "pyrsistent-0.19.3-cp38-cp38-win32.whl", hash = "sha256:f0e7c4b2f77593871e918be000b96c8107da48444d57005b6a6bc61fb4331b2c"}, - {file = "pyrsistent-0.19.3-cp38-cp38-win_amd64.whl", hash = "sha256:c147257a92374fde8498491f53ffa8f4822cd70c0d85037e09028e478cababb7"}, - {file = "pyrsistent-0.19.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b735e538f74ec31378f5a1e3886a26d2ca6351106b4dfde376a26fc32a044edc"}, - {file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99abb85579e2165bd8522f0c0138864da97847875ecbd45f3e7e2af569bfc6f2"}, - {file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a8cb235fa6d3fd7aae6a4f1429bbb1fec1577d978098da1252f0489937786f3"}, - {file = "pyrsistent-0.19.3-cp39-cp39-win32.whl", hash = "sha256:c74bed51f9b41c48366a286395c67f4e894374306b197e62810e0fdaf2364da2"}, - {file = "pyrsistent-0.19.3-cp39-cp39-win_amd64.whl", hash = "sha256:878433581fc23e906d947a6814336eee031a00e6defba224234169ae3d3d6a98"}, - {file = "pyrsistent-0.19.3-py3-none-any.whl", hash = "sha256:ccf0d6bd208f8111179f0c26fdf84ed7c3891982f2edaeae7422575f47e66b64"}, - {file = "pyrsistent-0.19.3.tar.gz", hash = "sha256:1a2994773706bbb4995c31a97bc94f1418314923bd1048c6d964837040376440"}, + {file = "pyrsistent-0.16.1.tar.gz", hash = "sha256:aa2ae1c2e496f4d6777f869ea5de7166a8ccb9c2e06ebcf6c7ff1b670c98c5ef"}, ] +[package.dependencies] +six = "*" + [[package]] name = "pytest" version = "7.4.4" @@ -2323,6 +2300,28 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +[[package]] +name = "ssz" +version = "0.5.0" +description = "ssz: Python implementation of the Simple Serialization encoding and decoding" +optional = false +python-versions = "<4,>=3.8" +files = [ + {file = "ssz-0.5.0-py3-none-any.whl", hash = "sha256:111dec40bee7cb7c3d52fe621a2247d5420ab999ffc0cbecb470782c28b56414"}, + {file = "ssz-0.5.0.tar.gz", hash = "sha256:5e0fe0483c5c9bdf3ea0fa957986b48ca34c784af57e39257863dd3743e20861"}, +] + +[package.dependencies] +eth-utils = ">=2" +lru-dict = ">=1.1.6" +pyrsistent = ">=0.16.0,<0.17" + +[package.extras] +dev = ["build (>=0.9.0)", "bumpversion (>=0.5.3)", "hypothesis (==4.54.0)", "ipython", "pre-commit (>=3.4.0)", "pytest (>=7.0.0)", "pytest-xdist (>=2.4.0)", "ruamel.yaml (>=0.17.0)", "sphinx (>=6.0.0)", "sphinx-rtd-theme (>=1.0.0)", "towncrier (>=21,<22)", "tox (>=4.0.0)", "twine", "wheel"] +docs = ["sphinx (>=6.0.0)", "sphinx-rtd-theme (>=1.0.0)", "towncrier (>=21,<22)"] +test = ["hypothesis (==4.54.0)", "pytest (>=7.0.0)", "pytest-xdist (>=2.4.0)"] +yaml = ["ruamel.yaml (>=0.17.0)"] + [[package]] name = "stevedore" version = "5.2.0" @@ -2413,13 +2412,13 @@ urllib3 = ">=2" [[package]] name = "typing-extensions" -version = "4.11.0" +version = "4.12.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, - {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, + {file = "typing_extensions-4.12.0-py3-none-any.whl", hash = "sha256:b349c66bea9016ac22978d800cfff206d5f9816951f12a7d0ec5578b0a819594"}, + {file = "typing_extensions-4.12.0.tar.gz", hash = "sha256:8cbcdc8606ebcb0d95453ad7dc5065e6237b6aa230a31e81d0f440c30fed5fd8"}, ] [[package]] @@ -2691,4 +2690,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "500c358be3ce666c65848eaa4ac8d1020b6df30c949c00ab2eceab4b81a7a01d" +content-hash = "3087088f3c1be66fae94c97238f71dc21b57265314d0565dfc9e286704aa3c33" diff --git a/pyproject.toml b/pyproject.toml index 8d198f8..8f06f56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "sw-utils" -version = "0.6.13" +version = "0.6.14" description = "StakeWise Python utils" authors = ["StakeWise Labs "] license = "GPL-3.0-or-later" @@ -13,8 +13,8 @@ py-ecc = "^6.0.0" ipfshttpclient = "^0.8.0a2" web3 = "==6.15.1" tenacity = "==8.2.3" -pyrsistent = "0.19.3" pyjwt = "==2.8.0" +ssz = "==0.5.0" [tool.poetry.group.dev.dependencies] pylint = "^3.0.1" diff --git a/sw_utils/signing.py b/sw_utils/signing.py index 8a98002..a2e0a6d 100644 --- a/sw_utils/signing.py +++ b/sw_utils/signing.py @@ -1,11 +1,9 @@ -# pylint: disable=W0511 -# TODO: remove once https://github.com/ethereum/py-ssz/issues/127 fixed import milagro_bls_binding as bls from eth_typing import BLSPubkey, BLSSignature, HexAddress from eth_utils import to_canonical_address from py_ecc.bls import G2ProofOfPossession +from ssz import Serializable, bytes4, bytes32, bytes48, bytes96, uint64 -from .ssz import Serializable, bytes4, bytes32, bytes48, bytes96, uint64 from .typings import Bytes32, ConsensusFork ETH1_ADDRESS_WITHDRAWAL_PREFIX = bytes.fromhex('01') diff --git a/sw_utils/ssz/__init__.py b/sw_utils/ssz/__init__.py deleted file mode 100644 index a0234c9..0000000 --- a/sw_utils/ssz/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -from .cache import SSZCache # noqa: F401 -from .exceptions import DeserializationError # noqa: F401 -from .exceptions import SerializationError, SSZException -from .sedes import ByteVector # noqa: F401 -from .sedes import ( - BaseSedes, - BasicSedes, - Byte, - Container, - ProperCompositeSedes, - Serializable, - UInt, - Vector, - byte_vector, - bytes4, - bytes32, - bytes48, - bytes96, - uint8, - uint16, - uint32, - uint64, - uint128, - uint256, -) diff --git a/sw_utils/ssz/abc.py b/sw_utils/ssz/abc.py deleted file mode 100644 index 91e61fe..0000000 --- a/sw_utils/ssz/abc.py +++ /dev/null @@ -1,156 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any, Generic, Iterable, Iterator, Optional, TypeVar, Union - -from eth_typing import Hash32 -from pyrsistent.typing import PVector - -from sw_utils.ssz.hash_tree import HashTree -from sw_utils.ssz.sedes.base import BaseProperCompositeSedes - -TStructure = TypeVar('TStructure') -TElement = TypeVar('TElement') - - -class HashableStructureAPI(ABC, Generic[TElement]): - @classmethod - @abstractmethod - def from_iterable_and_sedes( - cls, - iterable: Iterable[TElement], - sedes: BaseProperCompositeSedes, - max_length: Optional[int], - ): - ... - - # - # Element and hash tree access - # - @property - @abstractmethod - def elements(self) -> PVector[TElement]: - ... - - @property - @abstractmethod - def chunks(self) -> PVector[Hash32]: - ... - - @property - @abstractmethod - def hash_tree(self) -> HashTree: - ... - - @property - @abstractmethod - def raw_root(self) -> Hash32: - ... - - @property - @abstractmethod - def hash_tree_root(self) -> Hash32: - ... - - # - # Partial PVector interface - # - @abstractmethod - def __len__(self) -> int: - ... - - @abstractmethod - def __getitem__(self, index: int) -> TElement: - ... - - @abstractmethod - def __iter__(self) -> Iterator[TElement]: - ... - - @abstractmethod - def __hash__(self) -> int: - ... - - @abstractmethod - def __eq__(self, other: Any) -> bool: - ... - - @abstractmethod - def transform(self, *transformations): - ... - - @abstractmethod - def mset(self: TStructure, *args: Union[int, TElement]) -> TStructure: - ... - - @abstractmethod - def set(self: TStructure, index: int, value: TElement) -> TStructure: - ... - - @abstractmethod - def evolver( - self: TStructure, - ) -> 'HashableStructureEvolverAPI[TStructure, TElement]': - ... - - -class ResizableHashableStructureAPI(HashableStructureAPI[TElement]): - @abstractmethod - def append(self: TStructure, value: TElement) -> TStructure: - ... - - @abstractmethod - def extend(self: TStructure, values: Iterable[TElement]) -> TStructure: - ... - - @abstractmethod - def __add__(self: TStructure, values: Iterable[TElement]) -> TStructure: - ... - - @abstractmethod - def __mul__(self: TStructure, times: int) -> TStructure: - ... - - @abstractmethod - def evolver( - self: TStructure, - ) -> 'ResizableHashableStructureEvolverAPI[TStructure, TElement]': - ... - - -class HashableStructureEvolverAPI(ABC, Generic[TStructure, TElement]): - @abstractmethod - def __init__(self, hashable_structure: TStructure) -> None: - ... - - @abstractmethod - def __getitem__(self, index: int) -> TElement: - ... - - @abstractmethod - def set(self, index: int, element: TElement) -> None: - ... - - @abstractmethod - def __setitem__(self, index: int, element: TElement) -> None: - ... - - @abstractmethod - def __len__(self) -> int: - ... - - @abstractmethod - def is_dirty(self) -> bool: - ... - - @abstractmethod - def persistent(self) -> TStructure: - ... - - -class ResizableHashableStructureEvolverAPI(HashableStructureEvolverAPI[TStructure, TElement]): - @abstractmethod - def append(self, element: TElement) -> None: - ... - - @abstractmethod - def extend(self, iterable: Iterable[TElement]) -> None: - ... diff --git a/sw_utils/ssz/cache/__init__.py b/sw_utils/ssz/cache/__init__.py deleted file mode 100644 index 1810d67..0000000 --- a/sw_utils/ssz/cache/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .cache import SSZCache # noqa: F401 diff --git a/sw_utils/ssz/cache/cache.py b/sw_utils/ssz/cache/cache.py deleted file mode 100644 index d22d2dd..0000000 --- a/sw_utils/ssz/cache/cache.py +++ /dev/null @@ -1,48 +0,0 @@ -from collections.abc import MutableMapping -from typing import TYPE_CHECKING, Iterator - -from lru import LRU - -if TYPE_CHECKING: - MM = MutableMapping[bytes, bytes] -else: - MM = MutableMapping - -DEFAULT_CACHE_SIZE = 2**10 - - -class SSZCache(MM): - def __init__(self, cache_size: int = DEFAULT_CACHE_SIZE) -> None: - self._cache_size = cache_size - self.clear() - - def clear(self) -> None: - self._cached_values = LRU(self._cache_size) - - def _exists(self, key: bytes) -> bool: - return key in self._cached_values - - def __contains__(self, key: bytes) -> bool: - return self._exists(key) - - def __getitem__(self, key: bytes) -> bytes: - return self._cached_values[key] - - def __setitem__(self, key: bytes, value: bytes) -> None: - self._cached_values[key] = value - - def __delitem__(self, key: bytes) -> None: - if key in self._cached_values: - del self._cached_values[key] - else: - raise KeyError(f'key: {key} not found') - - def __iter__(self) -> Iterator[bytes]: - raise NotImplementedError('By default, DB classes cannot be iterated.') - - def __len__(self) -> int: - raise NotImplementedError('By default, classes cannot return the total number of keys.') - - @property - def cache_size(self) -> int: - return self._cache_size diff --git a/sw_utils/ssz/cache/utils.py b/sw_utils/ssz/cache/utils.py deleted file mode 100644 index e1bf2f0..0000000 --- a/sw_utils/ssz/cache/utils.py +++ /dev/null @@ -1,43 +0,0 @@ -import functools -from typing import Any, Iterable - -from eth_typing import Hash32 -from eth_utils import to_tuple - -from sw_utils.ssz.sedes.base import TSedes -from sw_utils.ssz.typing import CacheObj - - -def get_key(sedes, value: Any) -> str: - key = get_base_key(sedes, value).hex() - if len(key) == 0: - key = '' - return f'{sedes.get_sedes_id()}{key}' - - -@functools.lru_cache(maxsize=2**12) -def get_base_key(sedes: TSedes, value: Any) -> bytes: - return sedes.serialize(value) - - -@to_tuple -def get_merkle_leaves_without_cache(value: Any, element_sedes: TSedes) -> Iterable[Hash32]: - for element in value: - yield element_sedes.get_hash_tree_root(element) - - -@to_tuple -def get_merkle_leaves_with_cache( - value: Any, element_sedes: TSedes, cache: CacheObj -) -> Iterable[Hash32]: - """ - Generate the merkle leaves for every element in `value`, from the cache. - - NOTE: cache will be mutated when any new merkle leaves are generated. - """ - for element in value: - key = element_sedes.get_key(element) - if key not in cache: - root, cache = element_sedes.get_hash_tree_root_and_leaves(element, cache) - cache[key] = root - yield cache[key] diff --git a/sw_utils/ssz/constants.py b/sw_utils/ssz/constants.py deleted file mode 100644 index a788b29..0000000 --- a/sw_utils/ssz/constants.py +++ /dev/null @@ -1,25 +0,0 @@ -from eth_typing import Hash32 -from eth_utils.toolz import iterate, take - -from sw_utils.ssz.hash import hash_eth2 - -CHUNK_SIZE = 32 # named BYTES_PER_CHUNK in the spec -EMPTY_CHUNK = Hash32(b'\x00' * CHUNK_SIZE) - -SIGNATURE_FIELD_NAME = 'signature' - -# number of bytes for a serialized offset -OFFSET_SIZE = 4 - -FIELDS_META_ATTR = 'fields' - -ZERO_BYTES32 = Hash32(b'\x00' * 32) -MAX_ZERO_HASHES_LAYER = 100 -ZERO_HASHES = tuple( - take( - MAX_ZERO_HASHES_LAYER, - iterate(lambda child: hash_eth2(child + child), ZERO_BYTES32), - ) -) - -BASE_TYPES = (int, bytes, bool) diff --git a/sw_utils/ssz/exceptions.py b/sw_utils/ssz/exceptions.py deleted file mode 100644 index 453e87b..0000000 --- a/sw_utils/ssz/exceptions.py +++ /dev/null @@ -1,22 +0,0 @@ -class SSZException(Exception): - """ - Base class for exceptions raised by this package. - """ - - pass - - -class SerializationError(SSZException): - """ - Exception raised if serialization fails. - """ - - pass - - -class DeserializationError(SSZException): - """ - Exception raised if deserialization fails. - """ - - pass diff --git a/sw_utils/ssz/hash.py b/sw_utils/ssz/hash.py deleted file mode 100644 index d0250c8..0000000 --- a/sw_utils/ssz/hash.py +++ /dev/null @@ -1,14 +0,0 @@ -import functools -import hashlib - -from eth_typing import Hash32 - - -@functools.lru_cache(maxsize=2**12) -def hash_eth2(data: bytes) -> Hash32: - """ - Return SHA-256 hashed result. - Note: it's a placeholder and we aim to migrate to a S[T/N]ARK-friendly hash function in - a future Ethereum 2.0 deployment phase. - """ - return Hash32(hashlib.sha256(data).digest()) diff --git a/sw_utils/ssz/hash_tree.py b/sw_utils/ssz/hash_tree.py deleted file mode 100644 index a5f3efc..0000000 --- a/sw_utils/ssz/hash_tree.py +++ /dev/null @@ -1,385 +0,0 @@ -import itertools -from functools import partial -from numbers import Integral -from typing import Any, Generator, Iterable, Optional, Union - -# `transform` comes from a non-public API which is considered stable, but future changes can not be -# ruled out completely. Therefore, the implementation should be reviewed whenever pyrsistent is -# updated. See https://github.com/tobgu/pyrsistent/issues/180 for more information. -from eth_typing import Hash32 -from eth_utils.toolz import drop, iterate, partition, pipe, take -from pyrsistent import pmap, pvector -from pyrsistent._transformations import transform -from pyrsistent.typing import PMap, PVector - -from sw_utils.ssz.constants import ZERO_HASHES -from sw_utils.ssz.hash import hash_eth2 -from sw_utils.ssz.utils import get_next_power_of_two - -RawHashTreeLayer = PVector[Hash32] -RawHashTree = PVector[RawHashTreeLayer] - - -def validate_chunk_count(chunk_count: Optional[int]) -> None: - if chunk_count is not None: - if chunk_count <= 0: - raise ValueError(f'Chunk count is not positive: {chunk_count}') - - -def validate_raw_hash_tree(raw_hash_tree: RawHashTree, chunk_count: Optional[int] = None) -> None: - if len(raw_hash_tree) == 0: - raise ValueError('Hash tree is empty') - - if len(raw_hash_tree[0]) == 0: - raise ValueError('Hash tree contains zero chunks') - - if chunk_count is not None and len(raw_hash_tree[0]) > chunk_count: - raise ValueError( - f'Hash tree contains {len(raw_hash_tree[0])} chunks which exceeds chunk count ' - f'{chunk_count}' - ) - - if len(raw_hash_tree[-1]) != 1: - raise ValueError( - f'Hash tree root layer contains {len(raw_hash_tree[-1])} items instead of 1' - ) - - -class HashTree(PVector[Hash32]): - def __init__(self, raw_hash_tree: RawHashTree, chunk_count: Optional[int] = None) -> None: - validate_chunk_count(chunk_count) - validate_raw_hash_tree(raw_hash_tree, chunk_count) - - self.chunk_count = chunk_count - self.raw_hash_tree = raw_hash_tree - - @classmethod - def compute(cls, chunks: Iterable[Hash32], chunk_count: Optional[int] = None) -> 'HashTree': - raw_hash_tree = compute_hash_tree(chunks, chunk_count) - return cls(raw_hash_tree, chunk_count) - - @property - def chunks(self) -> RawHashTreeLayer: - return self.raw_hash_tree[0] - - @property - def root(self) -> Hash32: - return self.raw_hash_tree[-1][0] - - def transform(self, *transformations): - return transform(self, transformations) - - def evolver(self): - return HashTreeEvolver(self) - - # - # Comparison - # - def __hash__(self) -> int: - return hash((self.root, self.chunk_count)) - - def __eq__(self, other: Any) -> bool: - return ( - isinstance(other, HashTree) - and self.root == other.root - and self.chunk_count == other.chunk_count - ) - - # - # Chunk access - # - def __len__(self) -> int: - return len(self.chunks) - - def __getitem__(self, index: Union[int, slice]) -> Hash32: - return self.chunks[index] - - def index(self, value: Hash32, *args, **kwargs) -> Hash32: - return self.chunks.index(value, *args, **kwargs) - - def count(self, value: Hash32) -> int: - return self.chunks.count(value) - - # - # Tree modifications via evolver - # - def append(self, value: Hash32) -> 'HashTree': - evolver = self.evolver() - evolver.append(value) - return evolver.persistent() - - def extend(self, value: Iterable[Hash32]) -> 'HashTree': - evolver = self.evolver() - evolver.extend(value) - return evolver.persistent() - - def __add__(self, other: Iterable[Hash32]) -> 'HashTree': - return self.extend(other) - - def __mul__(self, times: int) -> 'HashTree': - if times <= 0: - raise ValueError(f'Multiplier must be greater or equal to 1, got {times}') - - evolver = self.evolver() - - for _ in range(times - 1): - evolver.extend(self) - - return evolver.persistent() - - def mset(self, *args: Union[int, Hash32]) -> 'HashTree': - if len(args) % 2 != 0: - raise TypeError( - f'mset must be called with an even number of arguments, got {len(args)}' - ) - - evolver = self.evolver() - for index, value in partition(2, args): - evolver[index] = value - return evolver.persistent() - - def set(self, index: int, value: Hash32) -> 'HashTree': - return self.mset(index, value) - - # - # Removal of chunks - # - def delete(self, index: int, stop: Optional[int] = None) -> 'HashTree': - if stop is None: - stop = index + 1 - chunks = self.chunks.delete(index, stop) - return self.__class__.compute(chunks, self.chunk_count) - - def remove(self, value: Hash32) -> 'HashTree': - chunks = self.chunks.remove(value) - return self.__class__.compute(chunks, self.chunk_count) - - -class HashTreeEvolver: - def __init__(self, hash_tree: 'HashTree') -> None: - self.original_hash_tree = hash_tree - self.updated_chunks: PMap[int, Hash32] = pmap() - self.appended_chunks: PVector[Hash32] = pvector() - - # - # Getters - # - def __getitem__(self, index: int) -> Hash32: - if index < 0: - index += len(self) - - if index in self.updated_chunks: - return self.updated_chunks[index] - else: - if 0 <= index < len(self.original_hash_tree): - return self.original_hash_tree[index] - elif index < len(self): - index_in_appendix = index - len(self.original_hash_tree) - return self.appended_chunks[index_in_appendix] - else: - raise IndexError(f'Index out of bounds: {index}') - - def __len__(self) -> int: - return len(self.original_hash_tree) + len(self.appended_chunks) - - def is_dirty(self) -> bool: - return self.updated_chunks or self.appended_chunks - - # - # Setters - # - def set(self, index: Integral, value: Hash32) -> None: - self[index] = value - - def __setitem__(self, index: Integral, value: Hash32) -> None: - if index < 0: - index += len(self) - - if 0 <= index < len(self.original_hash_tree): - self.updated_chunks = self.updated_chunks.set(index, value) - elif index < len(self): - index_in_appendix = index - len(self.original_hash_tree) - self.appended_chunks = self.appended_chunks.set(index_in_appendix, value) - else: - raise IndexError(f'Index out of bounds: {index}') - - # - # Length changing modifiers - # - def append(self, value: Hash32) -> None: - self.appended_chunks = self.appended_chunks.append(value) - self._check_chunk_count() - - def extend(self, values: Iterable[Hash32]) -> None: - self.appended_chunks = self.appended_chunks.extend(values) - self._check_chunk_count() - - def _check_chunk_count(self) -> None: - chunk_count = self.original_hash_tree.chunk_count - if chunk_count is not None and len(self) > chunk_count: - raise ValueError(f'Hash tree exceeds size chunk count {chunk_count}') - - # - # Not implemented - # - def delete(self, index, stop=None): - raise NotImplementedError() - - def __delitem__(self, index): - raise NotImplementedError() - - def remove(self, value): - raise NotImplementedError() - - # - # Persist - # - def persistent(self) -> 'HashTree': - if not self.is_dirty(): - return self.original_hash_tree - else: - setters = ( - partial(set_chunk_in_tree, index=index, chunk=chunk) - for index, chunk in self.updated_chunks.items() - ) - appenders = ( - partial(append_chunk_to_tree, chunk=chunk) for chunk in self.appended_chunks - ) - raw_hash_tree = pipe(self.original_hash_tree.raw_hash_tree, *setters, *appenders) - return self.original_hash_tree.__class__( - raw_hash_tree, self.original_hash_tree.chunk_count - ) - - -def hash_layer(child_layer: RawHashTreeLayer, layer_index: int) -> RawHashTreeLayer: - if len(child_layer) % 2 == 0: - padded_child_layer = child_layer - else: - padded_child_layer = child_layer.append(ZERO_HASHES[layer_index]) - - child_pairs = partition(2, padded_child_layer) - parent_layer = pvector( - hash_eth2(left_child + right_child) for left_child, right_child in child_pairs - ) - return parent_layer - - -def generate_hash_tree_layers( - chunks: RawHashTreeLayer, -) -> Generator[RawHashTreeLayer, None, None]: - yield chunks - previous_layer = chunks - - for previous_layer_index in itertools.count(): - # stop if the root has been reached - if len(previous_layer) <= 1: - break - if previous_layer_index > 1000: - raise Exception('Invariant: Root is reached quickly') - - next_layer = hash_layer(previous_layer, previous_layer_index) - - yield next_layer - previous_layer = next_layer - - -def get_num_layers(num_chunks: int, chunk_count: Optional[int]) -> int: - if chunk_count is not None: - virtual_size = chunk_count - else: - virtual_size = num_chunks - - return get_next_power_of_two(virtual_size).bit_length() - - -def generate_chunk_tree_padding( - unpadded_chunk_tree: PVector[Hash32], chunk_count: Optional[int] -) -> Generator[Hash32, None, None]: - if chunk_count is None: - return - - num_chunks = len(unpadded_chunk_tree[0]) - if num_chunks > chunk_count: - raise ValueError( - f'Number of chunks in tree ({num_chunks}) exceeds chunk count {chunk_count}' - ) - - num_existing_layers = len(unpadded_chunk_tree) - num_target_layers = get_next_power_of_two(chunk_count).bit_length() - - previous_root = unpadded_chunk_tree[-1][0] - for previous_layer_index in range(num_existing_layers - 1, num_target_layers - 1): - next_root = hash_eth2(previous_root + ZERO_HASHES[previous_layer_index]) - yield pvector([next_root]) - previous_root = next_root - - -def pad_hash_tree( - unpadded_chunk_tree: RawHashTree, chunk_count: Optional[int] = None -) -> RawHashTree: - padding = pvector(generate_chunk_tree_padding(unpadded_chunk_tree, chunk_count)) - return unpadded_chunk_tree + padding - - -def compute_hash_tree(chunks: Iterable[Hash32], chunk_count: Optional[int] = None) -> RawHashTree: - validate_chunk_count(chunk_count) - - chunks = pvector(chunks) - - if not chunks: - raise ValueError('Number of chunks is 0') - if chunk_count is not None and len(chunks) > chunk_count: - raise ValueError(f'Number of chunks ({len(chunks)}) exceeds chunk_count ({chunk_count})') - - unpadded_chunk_tree = pvector(generate_hash_tree_layers(chunks)) - return pad_hash_tree(unpadded_chunk_tree, chunk_count) - - -def recompute_hash_in_tree( - hash_tree: RawHashTree, layer_index: int, hash_index: int -) -> RawHashTree: - if layer_index == 0: - raise ValueError('Cannot recompute hash in leaf layer as it consists of chunks not hashes') - - child_layer_index = layer_index - 1 - left_child_hash_index = hash_index * 2 - right_child_hash_index = left_child_hash_index + 1 - - child_layer = hash_tree[child_layer_index] - left_child_hash = child_layer[left_child_hash_index] - try: - right_child_hash = child_layer[right_child_hash_index] - except IndexError: - right_child_hash = ZERO_HASHES[child_layer_index] - - # create the layer if it doesn't exist yet (otherwise, pyrsistent would create a PMap) - if layer_index == len(hash_tree): - hash_tree = hash_tree.append(pvector()) - - parent_hash = hash_eth2(left_child_hash + right_child_hash) - return hash_tree.transform((layer_index, hash_index), parent_hash) - - -def set_chunk_in_tree(hash_tree: RawHashTree, index: int, chunk: Hash32) -> RawHashTree: - hash_tree_with_updated_chunk = hash_tree.transform((0, index), chunk) - - parent_layer_indices = drop(1, range(len(hash_tree))) - parent_hash_indices = drop(1, take(len(hash_tree), iterate(lambda index: index // 2, index))) - - update_functions = ( - partial(recompute_hash_in_tree, layer_index=layer_index, hash_index=hash_index) - for layer_index, hash_index in zip(parent_layer_indices, parent_hash_indices) - ) - - hash_tree_with_updated_branch = pipe(hash_tree_with_updated_chunk, *update_functions) - - if len(hash_tree_with_updated_branch[-1]) == 1: - return hash_tree_with_updated_branch - elif len(hash_tree_with_updated_branch[-1]) == 2: - return recompute_hash_in_tree(hash_tree_with_updated_branch, len(hash_tree), 0) - else: - raise Exception('Unreachable') - - -def append_chunk_to_tree(hash_tree: RawHashTree, chunk: Hash32) -> RawHashTree: - return set_chunk_in_tree(hash_tree, len(hash_tree[0]), chunk) diff --git a/sw_utils/ssz/hashable_structure.py b/sw_utils/ssz/hashable_structure.py deleted file mode 100644 index 09edf27..0000000 --- a/sw_utils/ssz/hashable_structure.py +++ /dev/null @@ -1,406 +0,0 @@ -import functools -import itertools -from typing import ( - Any, - Dict, - Generator, - Iterable, - Iterator, - List, - Optional, - Sequence, - Tuple, - TypeVar, - Union, -) - -from eth_typing import Hash32 -from eth_utils import to_dict, to_tuple -from eth_utils.toolz import groupby, partition, pipe -from pyrsistent import pvector -from pyrsistent._transformations import transform -from pyrsistent.typing import PVector - -from sw_utils.ssz.abc import ( - HashableStructureAPI, - HashableStructureEvolverAPI, - ResizableHashableStructureAPI, - ResizableHashableStructureEvolverAPI, -) -from sw_utils.ssz.constants import CHUNK_SIZE, ZERO_BYTES32 -from sw_utils.ssz.hash_tree import HashTree -from sw_utils.ssz.sedes.base import BaseProperCompositeSedes - -TStructure = TypeVar('TStructure', bound='BaseHashableStructure') -TResizableStructure = TypeVar('TResizableStructure', bound='BaseResizableHashableStructure') -TElement = TypeVar('TElement') - - -def update_element_in_chunk(original_chunk: Hash32, index: int, element: bytes) -> Hash32: - """ - Replace part of a chunk with a given element. - - The chunk is interpreted as a concatenated sequence of equally sized elements. This function - replaces the element given by its index in the chunk with the given data. - - If the length of the element is zero or not a divisor of the chunk size, a `ValueError` is - raised. If the index is out of range, an `IndexError` is raised. - """ - element_size = len(element) - chunk_size = len(original_chunk) - - if element_size == 0: - raise ValueError('Element size is zero') - if chunk_size % element_size != 0: - raise ValueError(f'Element size is not a divisor of chunk size: {element_size}') - if not 0 <= index < chunk_size // element_size: - raise IndexError(f'Index out of range for element size {element_size}: {index}') - - first_byte_index = index * element_size - last_byte_index = first_byte_index + element_size - - prefix = original_chunk[:first_byte_index] - suffix = original_chunk[last_byte_index:] - return Hash32(prefix + element + suffix) - - -def update_elements_in_chunk(original_chunk: Hash32, updated_elements: Dict[int, bytes]) -> Hash32: - """ - Update multiple elements in a chunk. - - The set of updates is given by a dictionary mapping indices to elements. The items of the - dictionary will be passed one by one to `update_element_in_chunk`. - """ - return pipe( - original_chunk, - *( - functools.partial(update_element_in_chunk, index=index, element=element) - for index, element in updated_elements.items() - ), - ) - - -def get_num_padding_elements( - *, num_original_elements: int, num_original_chunks: int, element_size: int -) -> int: - """Compute the number of elements that would still fit in the empty space of the last chunk.""" - total_size = num_original_chunks * CHUNK_SIZE - used_size = num_original_elements * element_size - padding_size = total_size - used_size - num_elements_in_padding = padding_size // element_size - return num_elements_in_padding - - -@to_dict -def get_updated_chunks( - *, - updated_elements: Dict[int, bytes], - appended_elements: Sequence[bytes], - original_chunks: Sequence[Hash32], - element_size: int, - num_original_elements: int, - num_padding_elements: int, -) -> Generator[Tuple[int, Hash32], None, None]: - """ - For an element changeset, compute the updates that have to be applied to the existing chunks. - - The changeset is given as a dictionary of element indices to updated elements and a sequence of - appended elements. Note that appended elements that do not affect existing chunks are ignored. - - The pre-existing state is given by the sequence of original chunks and the number of elements - represented by these chunks. - - The return value is a dictionary mapping chunk indices to chunks. - """ - effective_appended_elements = appended_elements[:num_padding_elements] - elements_per_chunk = CHUNK_SIZE // element_size - - padding_elements_with_indices = dict( - enumerate(effective_appended_elements, start=num_original_elements) - ) - effective_updated_elements = {**updated_elements, **padding_elements_with_indices} - - element_indices = effective_updated_elements.keys() - element_indices_by_chunk = groupby( - lambda element_index: element_index // elements_per_chunk, element_indices - ) - - for chunk_index, element_indices in element_indices_by_chunk.items(): - chunk_updates = { - element_index % elements_per_chunk: effective_updated_elements[element_index] - for element_index in element_indices - } - updated_chunk = update_elements_in_chunk(original_chunks[chunk_index], chunk_updates) - yield chunk_index, updated_chunk - - -@to_tuple -def get_appended_chunks( - *, appended_elements: Sequence[bytes], element_size: int, num_padding_elements: int -) -> Generator[Hash32, None, None]: - """Get the sequence of appended chunks.""" - if len(appended_elements) <= num_padding_elements: - return - - elements_per_chunk = CHUNK_SIZE // element_size - - chunk_partitioned_elements = partition( - elements_per_chunk, - appended_elements[num_padding_elements:], - pad=b'\x00' * element_size, - ) - for elements_in_chunk in chunk_partitioned_elements: - yield Hash32(b''.join(elements_in_chunk)) - - -class BaseHashableStructure(HashableStructureAPI[TElement]): - def __init__( - self, - elements: PVector[TElement], - hash_tree: HashTree, - sedes: BaseProperCompositeSedes, - max_length: Optional[int] = None, - ) -> None: - self._elements = elements - self._hash_tree = hash_tree - self._sedes = sedes - self._max_length = max_length - - @classmethod - def from_iterable_and_sedes( - cls, - iterable: Iterable[TElement], - sedes: BaseProperCompositeSedes, - max_length: Optional[int] = None, - ): - elements = pvector(iterable) - if max_length and len(elements) > max_length: - raise ValueError( - f'Number of elements {len(elements)} exceeds maximum length {max_length}' - ) - - serialized_elements = [ - sedes.serialize_element_for_tree(index, element) - for index, element in enumerate(elements) - ] - appended_chunks = get_appended_chunks( - appended_elements=serialized_elements, - element_size=sedes.element_size_in_tree, - num_padding_elements=0, - ) - hash_tree = HashTree.compute(appended_chunks or [ZERO_BYTES32], sedes.chunk_count) - return cls(elements, hash_tree, sedes, max_length) - - @property - def elements(self) -> PVector[TElement]: - return self._elements - - @property - def hash_tree(self) -> HashTree: - return self._hash_tree - - @property - def chunks(self) -> PVector[Hash32]: - return self.hash_tree.chunks - - @property - def max_length(self) -> Optional[int]: - return self._max_length - - @property - def raw_root(self) -> Hash32: - return self.hash_tree.root - - @property - def sedes(self) -> BaseProperCompositeSedes: - return self._sedes - - # - # Hash and equality - # - def __hash__(self) -> int: - # hashable structures have the same hash if they share both sedes and root - return hash((self.sedes, self.hash_tree_root)) - - def __eq__(self, other: Any) -> bool: - # hashable structures are equal if they use the same sedes and have the same root - if isinstance(other, BaseHashableStructure): - sedes_equal = self.sedes == other.sedes - roots_equal = self.hash_tree_root == other.hash_tree_root - return sedes_equal and roots_equal - else: - return False - - # - # PVector interface - # - def __len__(self) -> int: - return len(self.elements) - - def __getitem__(self, index: int) -> TElement: - return self.elements[index] - - def __iter__(self) -> Iterator[TElement]: - return iter(self.elements) - - def transform(self, *transformations): - return transform(self, transformations) - - def mset(self: TStructure, *args: Union[int, TElement]) -> TStructure: - if len(args) % 2 != 0: - raise TypeError( - f'mset must be called with an even number of arguments, got {len(args)}' - ) - - evolver = self.evolver() - for index, value in partition(2, args): - evolver[index] = value - return evolver.persistent() - - def set(self: TStructure, index: int, value: TElement) -> TStructure: - return self.mset(index, value) - - def evolver( - self: TStructure, - ) -> 'HashableStructureEvolverAPI[TStructure, TElement]': - return HashableStructureEvolver(self) - - -class HashableStructureEvolver(HashableStructureEvolverAPI[TStructure, TElement]): - def __init__(self, hashable_structure: TStructure) -> None: - self._original_structure = hashable_structure - self._updated_elements: Dict[int, TElement] = {} - # `self._appended_elements` is only used in the subclass ResizableHashableStructureEvolver, - # but the implementation of `persistent` already processes it so that it does not have to - # be implemented twice. - self._appended_elements: List[TElement] = [] - - def __getitem__(self, index: int) -> TElement: - if index < 0: - index += len(self) - - if index in self._updated_elements: - return self._updated_elements[index] - elif 0 <= index < len(self._original_structure): - return self._original_structure[index] - elif 0 <= index < len(self): - return self._appended_elements[index - len(self._original_structure)] - else: - raise IndexError(f'Index out of bounds: {index}') - - def set(self, index: int, element: TElement) -> None: - self[index] = element - - def __setitem__(self, index: int, element: TElement) -> None: - if index < 0: - index += len(self) - - if 0 <= index < len(self._original_structure): - self._updated_elements[index] = element - elif 0 <= index < len(self): - self._appended_elements[index - len(self._original_structure)] = element - else: - raise IndexError(f'Index out of bounds: {index}') - - def __len__(self) -> int: - return len(self._original_structure) + len(self._appended_elements) - - def is_dirty(self) -> bool: - return bool(self._updated_elements or self._appended_elements) - - def persistent(self) -> TStructure: - if not self.is_dirty(): - return self._original_structure - - sedes = self._original_structure.sedes - - num_original_elements = len(self._original_structure) - num_original_chunks = len(self._original_structure.chunks) - num_padding_elements = get_num_padding_elements( - num_original_elements=num_original_elements, - num_original_chunks=num_original_chunks, - element_size=sedes.element_size_in_tree, - ) - - updated_elements = { - index: sedes.serialize_element_for_tree(index, element) - for index, element in self._updated_elements.items() - } - appended_elements = [ - sedes.serialize_element_for_tree(index, element) - for index, element in enumerate(self._appended_elements, start=num_original_elements) - ] - - updated_chunks = get_updated_chunks( - updated_elements=updated_elements, - appended_elements=appended_elements, - original_chunks=self._original_structure.chunks, - num_original_elements=num_original_elements, - num_padding_elements=num_padding_elements, - element_size=sedes.element_size_in_tree, - ) - appended_chunks = get_appended_chunks( - appended_elements=appended_elements, - element_size=sedes.element_size_in_tree, - num_padding_elements=num_padding_elements, - ) - - elements = self._original_structure.elements.mset( - *itertools.chain.from_iterable(self._updated_elements.items()) # type: ignore - ).extend(self._appended_elements) - hash_tree = self._original_structure.hash_tree.mset( - *itertools.chain.from_iterable(updated_chunks.items()) # type: ignore - ).extend(appended_chunks) - - return self._original_structure.__class__( - elements, hash_tree, self._original_structure.sedes - ) - - -class BaseResizableHashableStructure( - BaseHashableStructure, ResizableHashableStructureAPI[TElement] -): - def append(self: TResizableStructure, value: TElement) -> TResizableStructure: - evolver = self.evolver() - evolver.append(value) - return evolver.persistent() - - def extend(self: TResizableStructure, values: Iterable[TElement]) -> TResizableStructure: - evolver = self.evolver() - evolver.extend(values) - return evolver.persistent() - - def __add__(self: TResizableStructure, values: Iterable[TElement]) -> TResizableStructure: - return self.extend(values) - - def __mul__(self: TResizableStructure, times: int) -> TResizableStructure: - if times <= 0: - raise ValueError(f'Multiplication factor must be positive: {times}') - elif times == 1: - return self - else: - return (self + self) * (times - 1) - - def evolver( - self: TResizableStructure, - ) -> 'ResizableHashableStructureEvolverAPI[TResizableStructure, TElement]': - return ResizableHashableStructureEvolver(self) - - -class ResizableHashableStructureEvolver( - HashableStructureEvolver, ResizableHashableStructureEvolverAPI[TStructure, TElement] -): - def append(self, element: TElement) -> None: - max_length = self._original_structure.max_length - if max_length is not None and len(self) + 1 > max_length: - raise ValueError(f'Structure would exceed maximum length {max_length}') - self._appended_elements.append(element) - - def extend(self, elements: Iterable[TElement]) -> None: - extension = list(elements) - - max_length = self._original_structure.max_length - if max_length is not None and len(self) + len(extension) > max_length: - raise ValueError(f'Structure would exceed maximum length {max_length}') - - self._appended_elements.extend(extension) diff --git a/sw_utils/ssz/hashable_vector.py b/sw_utils/ssz/hashable_vector.py deleted file mode 100644 index a251a49..0000000 --- a/sw_utils/ssz/hashable_vector.py +++ /dev/null @@ -1,26 +0,0 @@ -from typing import TYPE_CHECKING, Iterable, Sequence, TypeVar - -from eth_typing import Hash32 -from pyrsistent import pvector - -from sw_utils.ssz.hashable_structure import BaseHashableStructure - -if TYPE_CHECKING: - from sw_utils.ssz.sedes import Vector - -TElement = TypeVar('TElement') - - -class HashableVector(BaseHashableStructure[TElement], Sequence[TElement]): - @classmethod - def from_iterable(cls, iterable: Iterable[TElement], sedes: 'Vector[TElement, TElement]'): - elements = pvector(iterable) - if len(elements) != sedes.length: - raise ValueError( - f'Vector has length {sedes.length}, but {len(elements)} elements are given' - ) - return super().from_iterable_and_sedes(elements, sedes, max_length=None) - - @property - def hash_tree_root(self) -> Hash32: - return self.raw_root diff --git a/sw_utils/ssz/sedes/__init__.py b/sw_utils/ssz/sedes/__init__.py deleted file mode 100644 index ff45d22..0000000 --- a/sw_utils/ssz/sedes/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from .base import BaseSedes -from .basic import BasicSedes, ProperCompositeSedes -from .byte import Byte, byte -from .byte_vector import ByteVector, bytes1, bytes4, bytes32, bytes48, bytes96 -from .container import Container -from .serializable import Serializable -from .uint import UInt, uint8, uint16, uint32, uint64, uint128, uint256 -from .vector import Vector diff --git a/sw_utils/ssz/sedes/base.py b/sw_utils/ssz/sedes/base.py deleted file mode 100644 index ec3b726..0000000 --- a/sw_utils/ssz/sedes/base.py +++ /dev/null @@ -1,98 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any, Generic, Optional, Tuple - -from eth_typing import Hash32 - -from sw_utils.ssz.typing import CacheObj, TDeserialized, TSerializable - - -class BaseSedes(ABC, Generic[TSerializable, TDeserialized]): - # - # Size - # - @property - @abstractmethod - def is_fixed_sized(self) -> bool: - ... - - @abstractmethod - def get_fixed_size(self) -> int: - ... - - # - # Serialization - # - @abstractmethod - def serialize(self, value: TSerializable) -> bytes: - ... - - # - # Deserialization - # - @abstractmethod - def deserialize(self, data: bytes) -> TDeserialized: - ... - - # - # Tree hashing - # - @abstractmethod - def get_hash_tree_root(self, value: TSerializable) -> Hash32: - ... - - @abstractmethod - def get_hash_tree_root_and_leaves( - self, value: TSerializable, cache: CacheObj - ) -> Tuple[Hash32, CacheObj]: - ... - - @abstractmethod - def get_sedes_id(self) -> str: - ... - - @abstractmethod - def get_key(self, value: Any) -> str: - ... - - # - # Equality and hashing - # - @abstractmethod - def __hash__(self) -> int: - ... - - @abstractmethod - def __eq__(self, other: Any) -> bool: - ... - - -TSedes = BaseSedes[Any, Any] - - -class BaseProperCompositeSedes(BaseSedes[TSerializable, TDeserialized]): - @property - @abstractmethod - def is_packing(self) -> bool: - ... - - @abstractmethod - def get_element_sedes(self, index: int) -> BaseSedes: - ... - - @property - @abstractmethod - def element_size_in_tree(self) -> int: - ... - - @abstractmethod - def serialize_element_for_tree(self, index: int, element: TSerializable) -> bytes: - ... - - @property - @abstractmethod - def chunk_count(self) -> Optional[int]: - ... - - -class BaseBitfieldCompositeSedes(BaseSedes[TSerializable, TDeserialized]): - ... diff --git a/sw_utils/ssz/sedes/basic.py b/sw_utils/ssz/sedes/basic.py deleted file mode 100644 index 896d6b2..0000000 --- a/sw_utils/ssz/sedes/basic.py +++ /dev/null @@ -1,177 +0,0 @@ -import io -import operator -from abc import abstractmethod -from typing import IO, Any, Generator, Iterable, Sequence, Tuple - -from eth_typing import Hash32 -from eth_utils import to_tuple -from eth_utils.toolz import accumulate, concatv - -from sw_utils.ssz import constants -from sw_utils.ssz.cache.utils import get_key -from sw_utils.ssz.constants import CHUNK_SIZE -from sw_utils.ssz.exceptions import DeserializationError -from sw_utils.ssz.sedes.base import ( - BaseBitfieldCompositeSedes, - BaseProperCompositeSedes, - BaseSedes, - TSedes, -) -from sw_utils.ssz.typing import CacheObj, TDeserialized, TSerializable -from sw_utils.ssz.utils import encode_offset, merkleize, merkleize_with_cache, pack - - -class BasicSedes(BaseSedes[TSerializable, TDeserialized]): - def __init__(self, size: int): - if size <= 0: - raise ValueError('Length must be greater than 0') - - self.size = size - - # - # Size - # - is_fixed_sized = True - - def get_fixed_size(self): - return self.size - - # - # Tree hashing - # - def get_hash_tree_root(self, value: TSerializable) -> Hash32: - serialized_value = self.serialize(value) - return merkleize(pack((serialized_value,))) - - def get_hash_tree_root_and_leaves( - self, value: TSerializable, cache: CacheObj - ) -> Tuple[Hash32, CacheObj]: - serialized_value = self.serialize(value) - return merkleize_with_cache(pack((serialized_value,)), cache=cache) - - def get_key(self, value: Any) -> str: - return get_key(self, value) - - -def _compute_fixed_size_section_length(element_sedes: Iterable[TSedes]) -> int: - return sum( - sedes.get_fixed_size() if sedes.is_fixed_sized else constants.OFFSET_SIZE - for sedes in element_sedes - ) - - -class BitfieldCompositeSedes(BaseBitfieldCompositeSedes[TSerializable, TDeserialized]): - def get_key(self, value: Any) -> str: - return get_key(self, value) - - -class ProperCompositeSedes(BaseProperCompositeSedes[TSerializable, TDeserialized]): - @to_tuple - def _get_item_sedes_pairs( - self, value: Sequence[TSerializable] - ) -> Generator[Tuple[TSerializable, TSedes], None, None]: - for index, element in enumerate(value): - yield element, self.get_element_sedes(index) - - def _validate_serializable(self, value: Any) -> None: - ... - - def serialize(self, value: TSerializable) -> bytes: - self._validate_serializable(value) - - if not len(value): - return b'' - - pairs = self._get_item_sedes_pairs(value) # slow - element_sedes = tuple(sedes for element, sedes in pairs) - - has_fixed_size_section_length_cache = hasattr(value, '_fixed_size_section_length_cache') - if has_fixed_size_section_length_cache: - if value._fixed_size_section_length_cache is None: - fixed_size_section_length = _compute_fixed_size_section_length(element_sedes) - value._fixed_size_section_length_cache = fixed_size_section_length - else: - fixed_size_section_length = value._fixed_size_section_length_cache - else: - fixed_size_section_length = _compute_fixed_size_section_length(element_sedes) - - variable_size_section_parts = tuple( - sedes.serialize(item) for item, sedes in pairs if not sedes.is_fixed_sized # slow - ) - - if variable_size_section_parts: - offsets = tuple( - accumulate( - operator.add, - map(len, variable_size_section_parts[:-1]), - fixed_size_section_length, - ) - ) - else: - offsets = () - - offsets_iter = iter(offsets) - - fixed_size_section_parts = tuple( - sedes.serialize(item) # slow - if sedes.is_fixed_sized - else encode_offset(next(offsets_iter)) - for item, sedes in pairs - ) - - try: - next(offsets_iter) - except StopIteration: - pass - else: - raise DeserializationError('Did not consume all offsets while decoding value') - - return b''.join(concatv(fixed_size_section_parts, variable_size_section_parts)) - - def serialize_element_for_tree(self, index: int, element: TSerializable) -> bytes: - sedes = self.get_element_sedes(index) - if self.is_packing: - return sedes.serialize(element) - else: - return sedes.get_hash_tree_root(element) - - @property - def element_size_in_tree(self) -> int: - if self.is_packing: - # note that only homogenous composites with fixed sized elements are packed - return self.get_element_sedes(0).get_fixed_size() - else: - return CHUNK_SIZE - - def deserialize(self, data: bytes) -> TDeserialized: - stream = io.BytesIO(data) - value = self._deserialize_stream(stream) - extra_data = stream.read() - if extra_data: - raise DeserializationError(f'Got {len(extra_data)} superfluous bytes') - return value - - @abstractmethod - def _deserialize_stream(self, stream: IO[bytes]) -> TDeserialized: - ... - - def get_key(self, value: Any) -> str: - return get_key(self, value) - - -class HomogeneousProperCompositeSedes(ProperCompositeSedes[TSerializable, TDeserialized]): - def get_sedes_id(self) -> str: - sedes_name = self.__class__.__name__ - return f'{sedes_name}({self.element_sedes.get_sedes_id()},{self.max_length})' - - @property - def is_packing(self) -> bool: - return isinstance(self.element_sedes, BasicSedes) - - @property - def chunk_count(self) -> int: - if self.is_packing: - element_size = self.element_sedes.get_fixed_size() - return (element_size * self.max_length + CHUNK_SIZE - 1) // CHUNK_SIZE - else: - return self.max_length diff --git a/sw_utils/ssz/sedes/byte.py b/sw_utils/ssz/sedes/byte.py deleted file mode 100644 index c90e8dd..0000000 --- a/sw_utils/ssz/sedes/byte.py +++ /dev/null @@ -1,37 +0,0 @@ -from typing import Any - -from sw_utils.ssz.exceptions import DeserializationError, SerializationError -from sw_utils.ssz.sedes.basic import BasicSedes - - -class Byte(BasicSedes[bytes, bytes]): - size = 1 - - def __init__(self) -> None: - super().__init__(1) - - def serialize(self, value: bytes) -> bytes: - if len(value) != 1: - raise SerializationError( - f'The `Byte` sedes can only serialize single bytes. Got: {value!r}' - ) - return value - - def deserialize(self, data: bytes) -> bytes: - if len(data) != 1: - raise DeserializationError( - f'The `Byte` sedes can only deserialize single bytes. Got: {data!r}' - ) - return data - - def get_sedes_id(self) -> str: - return self.__class__.__name__ - - def __hash__(self) -> int: - return hash((Byte,)) - - def __eq__(self, other: Any) -> bool: - return isinstance(other, Byte) - - -byte = Byte() diff --git a/sw_utils/ssz/sedes/byte_vector.py b/sw_utils/ssz/sedes/byte_vector.py deleted file mode 100644 index 7030a6e..0000000 --- a/sw_utils/ssz/sedes/byte_vector.py +++ /dev/null @@ -1,60 +0,0 @@ -from typing import Tuple, Union - -from eth_typing import Hash32 - -from sw_utils.ssz.exceptions import DeserializationError, SerializationError -from sw_utils.ssz.sedes.byte import byte -from sw_utils.ssz.sedes.vector import Vector -from sw_utils.ssz.typing import CacheObj -from sw_utils.ssz.utils import merkleize, merkleize_with_cache, pack_bytes - -BytesOrByteArray = Union[bytes, bytearray] - - -class ByteVector(Vector[BytesOrByteArray, bytes]): - """Equivalent to `Vector(byte, size)` but more efficient.""" - - def __init__(self, size: int) -> None: - super().__init__(element_sedes=byte, length=size) - - def serialize(self, value: BytesOrByteArray) -> bytes: - if len(value) != self.length: - raise SerializationError( - f'Cannot serialize length {len(value)} byte-string as bytes{self.length}' - ) - - return value - - def serialize_element_for_tree(self, index: int, byte_value: int) -> bytes: - if not 0 <= byte_value < 256: - raise SerializationError(f'Cannot serialize byte as its value {byte_value} is invalid') - - return bytes((byte_value,)) - - def deserialize(self, data: bytes) -> bytes: - if len(data) != self.length: - raise DeserializationError( - f'Cannot deserialize length {len(data)} data as bytes{self.length}' - ) - return data - - def get_hash_tree_root(self, value: bytes) -> bytes: - serialized_value = self.serialize(value) - return merkleize(pack_bytes(serialized_value)) - - def get_hash_tree_root_and_leaves( - self, value: bytes, cache: CacheObj - ) -> Tuple[Hash32, CacheObj]: - serialized_value = self.serialize(value) - return merkleize_with_cache(pack_bytes(serialized_value), cache=cache) - - def get_sedes_id(self) -> str: - return f'{self.__class__.__name__}{self.length}' - - -bytes1 = ByteVector(1) -bytes4 = ByteVector(4) -bytes8 = ByteVector(8) -bytes32 = ByteVector(32) -bytes48 = ByteVector(48) -bytes96 = ByteVector(96) diff --git a/sw_utils/ssz/sedes/container.py b/sw_utils/ssz/sedes/container.py deleted file mode 100644 index 3ee99ce..0000000 --- a/sw_utils/ssz/sedes/container.py +++ /dev/null @@ -1,182 +0,0 @@ -from typing import IO, Any, Iterable, Sequence, Tuple - -from eth_typing import Hash32 -from eth_utils import ValidationError, to_tuple -from eth_utils.toolz import sliding_window - -from sw_utils.ssz.exceptions import DeserializationError, SerializationError -from sw_utils.ssz.hashable_structure import BaseHashableStructure -from sw_utils.ssz.sedes.base import BaseSedes, TSedes -from sw_utils.ssz.sedes.basic import ProperCompositeSedes -from sw_utils.ssz.typing import CacheObj -from sw_utils.ssz.utils import merkleize, read_exact, s_decode_offset - - -@to_tuple -def _deserialize_fixed_size_items_and_offsets(stream, field_sedes): - for sedes in field_sedes: - if sedes.is_fixed_sized: - field_size = sedes.get_fixed_size() - field_data = read_exact(field_size, stream) - yield (sedes.deserialize(field_data), sedes) - else: - yield (s_decode_offset(stream), sedes) - - -class Container(ProperCompositeSedes[Sequence[Any], Tuple[Any, ...]]): - def __init__(self, field_sedes: Sequence[TSedes]) -> None: - if len(field_sedes) == 0: - raise ValidationError('Cannot define container without any fields') - self.field_sedes = tuple(field_sedes) - - # - # Size - # - @property - def is_fixed_sized(self): - return all(field.is_fixed_sized for field in self.field_sedes) - - def get_fixed_size(self): - if not self.is_fixed_sized: - raise ValueError('Container contains dynamically sized elements') - - return sum(field.get_fixed_size() for field in self.field_sedes) - - # - # Serialization - # - def get_element_sedes(self, index: int) -> BaseSedes: - return self.field_sedes[index] - - @property - def is_packing(self) -> bool: - return False - - @property - def chunk_count(self) -> int: - return len(self.field_sedes) - - def _validate_serializable(self, value: Sequence[Any]) -> None: - if len(value) != len(self.field_sedes): - raise SerializationError( - f'Incorrect element count: Expected: {len(self.field_sedes)} / Got: {len(value)}' - ) - - # - # Deserialization - # - def deserialize_fixed_size_parts( - self, stream: IO[bytes] - ) -> Iterable[Tuple[Tuple[Any], Tuple[int, TSedes]]]: - fixed_items_and_offets = _deserialize_fixed_size_items_and_offsets(stream, self.field_sedes) - fixed_size_values = tuple( - item for item, sedes in fixed_items_and_offets if sedes.is_fixed_sized - ) - offset_pairs = tuple( - (item, sedes) for item, sedes in fixed_items_and_offets if not sedes.is_fixed_sized - ) - return fixed_size_values, offset_pairs - - @to_tuple - def deserialize_variable_size_parts( - self, offset_pairs: Tuple[Tuple[int, TSedes], ...], stream: IO[bytes] - ) -> Iterable[Any]: - offsets, fields = zip(*offset_pairs) - - *head_fields, last_field = fields - for sedes, (left_offset, right_offset) in zip(head_fields, sliding_window(2, offsets)): - field_length = right_offset - left_offset - field_data = read_exact(field_length, stream) - yield sedes.deserialize(field_data) - - # simply reading to the end of the current stream gives us all of the final element data - final_field_data = stream.read() - yield last_field.deserialize(final_field_data) - - def _deserialize_stream(self, stream: IO[bytes]) -> Tuple[Any, ...]: - if not self.field_sedes: - # TODO: likely remove once - # https://github.com/ethereum/eth2.0-specs/issues/854 is resolved - return tuple() - - fixed_size_values, offset_pairs = self.deserialize_fixed_size_parts(stream) - - if not offset_pairs: - return fixed_size_values - - variable_size_values = self.deserialize_variable_size_parts(offset_pairs, stream) - - fixed_size_parts_iter = iter(fixed_size_values) - variable_size_parts_iter = iter(variable_size_values) - - value = tuple( - next(fixed_size_parts_iter) if sedes.is_fixed_sized else next(variable_size_parts_iter) - for sedes in self.field_sedes - ) - - # Verify that both iterables have been fully consumed. - try: - next(fixed_size_parts_iter) - except StopIteration: - pass - else: - raise DeserializationError('Did not consume all fixed size values') - - try: - next(variable_size_parts_iter) - except StopIteration: - pass - else: - raise DeserializationError('Did not consume all variable size values') - - return value - - # - # Tree hashing - # - def get_hash_tree_root(self, value: Tuple[Any, ...]) -> bytes: - if isinstance(value, BaseHashableStructure) and value.sedes == self: - return value.hash_tree_root - - merkle_leaves = tuple( - sedes.get_hash_tree_root(element) for element, sedes in zip(value, self.field_sedes) - ) - return merkleize(merkle_leaves) - - def get_hash_tree_root_and_leaves( - self, value: Tuple[Any, ...], cache: CacheObj - ) -> Tuple[Hash32, CacheObj]: - merkle_leaves = () - for element, sedes in zip(value, self.field_sedes): - key = sedes.get_key(element) - if key not in cache: - if hasattr(sedes, 'get_hash_tree_root_and_leaves'): - root, cache = sedes.get_hash_tree_root_and_leaves(element, cache) - cache[key] = root - else: - cache[key] = sedes.get_hash_tree_root(element) - - merkle_leaves += (cache[key],) - - return merkleize(merkle_leaves), cache - - def serialize(self, value) -> bytes: - if hasattr(value, '_serialize_cache') and value._serialize_cache is not None: - return value._serialize_cache - elif hasattr(value, '_serialize_cache') and value._serialize_cache is None: - value._serialize_cache = super().serialize(value) - return value._serialize_cache - else: - return super().serialize(value) - - def get_sedes_id(self) -> str: - return ','.join(field.get_sedes_id() for field in self.field_sedes) - - # - # Equality and hashing - # - def __hash__(self) -> int: - return hash((hash(Container), hash(self.field_sedes))) - - def __eq__(self, other: Any) -> bool: - return isinstance(other, Container) and other.field_sedes == self.field_sedes diff --git a/sw_utils/ssz/sedes/serializable.py b/sw_utils/ssz/sedes/serializable.py deleted file mode 100644 index 6ddbb6e..0000000 --- a/sw_utils/ssz/sedes/serializable.py +++ /dev/null @@ -1,395 +0,0 @@ -import abc -import collections.abc -import copy -import operator -import re -from typing import NamedTuple, Optional, Sequence, Tuple, Type - -from eth_utils import ValidationError, to_dict, to_set, to_tuple -from eth_utils.toolz import assoc, merge -from lru import LRU - -from sw_utils.ssz.cache.cache import DEFAULT_CACHE_SIZE -from sw_utils.ssz.cache.utils import get_base_key -from sw_utils.ssz.constants import FIELDS_META_ATTR -from sw_utils.ssz.sedes.base import BaseSedes -from sw_utils.ssz.sedes.container import Container -from sw_utils.ssz.typing import TSerializable -from sw_utils.ssz.utils import get_duplicates, is_immutable_field_value - - -class Meta(NamedTuple): - has_fields: bool - fields: Optional[Tuple[Tuple[str, BaseSedes], ...]] - container_sedes: Optional[Container] - field_names: Optional[Tuple[str, ...]] - field_attrs: Optional[Tuple[str, ...]] - - -def validate_args_and_kwargs(args, kwargs, arg_names): - duplicate_arg_names = get_duplicates(arg_names) - if duplicate_arg_names: - raise ValueError('Duplicate argument names: {0}'.format(sorted(duplicate_arg_names))) - - needed_arg_names = set(arg_names[len(args) :]) - used_arg_names = set(arg_names[: len(args)]) - - duplicate_arg_names = used_arg_names.intersection(kwargs.keys()) - if duplicate_arg_names: - raise TypeError('Duplicate kwargs: {0}'.format(sorted(duplicate_arg_names))) - - unknown_arg_names = set(kwargs.keys()).difference(arg_names) - if unknown_arg_names: - raise TypeError('Unknown kwargs: {0}'.format(sorted(unknown_arg_names))) - - missing_arg_names = set(needed_arg_names).difference(kwargs.keys()) - if missing_arg_names: - raise TypeError('Missing kwargs: {0}'.format(sorted(missing_arg_names))) - - -@to_tuple -def merge_kwargs_to_args(args, kwargs, arg_names): - validate_args_and_kwargs(args, kwargs, arg_names) - - needed_arg_names = arg_names[len(args) :] - - yield from args - for arg_name in needed_arg_names: - yield kwargs[arg_name] - - -@to_dict -def merge_args_to_kwargs(args, kwargs, arg_names): - yield from kwargs.items() - for value, name in zip(args, arg_names): - yield name, value - - -class BaseSerializable(collections.abc.Sequence): - cache = None - - def __init__(self, *args, cache=None, **kwargs): - arg_names = self._meta.field_names or () - validate_args_and_kwargs(args, kwargs, arg_names) - field_values = merge_kwargs_to_args(args, kwargs, arg_names) - - # Ensure that all the fields have been given values in initialization - if len(field_values) != len(arg_names): - raise TypeError( - 'Argument count mismatch. expected {0} - got {1} - missing {2}'.format( - len(arg_names), - len(field_values), - ','.join(arg_names[len(field_values) :]), - ) - ) - - for value, attr in zip(field_values, self._meta.field_attrs or ()): - setattr(self, attr, make_immutable(value)) - - self.cache = LRU(DEFAULT_CACHE_SIZE) if cache is None else cache - - def as_dict(self): - return dict((field, value) for field, value in zip(self._meta.field_names, self)) - - def __iter__(self): - for attr in self._meta.field_attrs: - yield getattr(self, attr) - - def __getitem__(self, index): - if isinstance(index, int): - attr = self._meta.field_attrs[index] - return getattr(self, attr) - elif isinstance(index, slice): - field_slice = self._meta.field_attrs[index] - return tuple(getattr(self, field) for field in field_slice) - elif isinstance(index, str): - return getattr(self, index) - else: - raise IndexError('Unsupported type for __getitem__: {0}'.format(type(index))) - - def __len__(self): - return len(self._meta.fields) - - def __eq__(self, other): - satisfies_class_relationship = issubclass(self.__class__, other.__class__) or issubclass( - other.__class__, self.__class__ - ) - - if not satisfies_class_relationship: - return False - else: - return self.hash_tree_root == other.hash_tree_root - - def __getstate__(self): - state = self.__dict__.copy() - # The hash() builtin is not stable across processes - # (https://docs.python.org/3/reference/datamodel.html#object.__hash__), so we do this here - # to ensure pickled instances don't carry the cached hash() as that may cause issues like - # https://github.com/ethereum/py-evm/issues/1318 - state['_hash_cache'] = None - return state - - _hash_cache = None - - def __hash__(self): - if self._hash_cache is None: - self._hash_cache = hash(self.__class__) * int.from_bytes(self.hash_tree_root, 'little') - return self._hash_cache - - def copy(self, *args, **kwargs): - missing_overrides = ( - set(self._meta.field_names) - .difference(kwargs.keys()) - .difference(self._meta.field_names[: len(args)]) - ) - unchanged_kwargs = { - key: value if is_immutable_field_value(value) else copy.deepcopy(value) - for key, value in self.as_dict().items() - if key in missing_overrides - } - - combined_kwargs = dict(**unchanged_kwargs, **kwargs) - all_kwargs = merge_args_to_kwargs(args, combined_kwargs, self._meta.field_names) - - result = type(self)(**all_kwargs) - result.cache = self.cache - - return result - - def reset_cache(self): - self.cache.clear() - self._fixed_size_section_length_cache = None - self._serialize_cache = None - - def __copy__(self): - return self.copy() - - def __deepcopy__(self, memodict=None): - if memodict is None: - memodict = {} - - cls = self.__class__ - result = cls.__new__(cls) - memodict[id(self)] = result - - for k, v in self.__dict__.items(): - if k != 'cache': - setattr(result, k, copy.deepcopy(v, memodict)) - - result.cache = self.cache - result._fixed_size_section_length_cache = self._fixed_size_section_length_cache - - return result - - _fixed_size_section_length_cache = None - _serialize_cache = None - - @property - def hash_tree_root(self): - return self.__class__.get_hash_tree_root(self, cache=True) - - @classmethod - def get_sedes_id(cls) -> str: - # Serializable implementation name should be unique - return cls.__name__ - - def get_key(self) -> bytes: - # Serilaize with self._meta.container_sedes - key = get_base_key(self._meta.container_sedes, self).hex() - - if len(key) == 0: - key = '' - - return f'{self.__class__.get_sedes_id()}{key}' - - -def make_immutable(value): - if isinstance(value, list): - return tuple(make_immutable(item) for item in value) - else: - return value - - -@to_tuple -def _mk_field_attrs(field_names, extra_namespace): - namespace = set(field_names).union(extra_namespace) - for field in field_names: - while True: - field = '_' + field - if field not in namespace: - namespace.add(field) - yield field - break - - -@to_dict -def _mk_field_props(field_names, field_attrs): - for field, attr in zip(field_names, field_attrs): - getter = operator.attrgetter(attr) - yield field, property(getter) - - -def _validate_field_names(field_names: Sequence[str]) -> None: - # check that field names are unique - duplicate_field_names = get_duplicates(field_names) - if duplicate_field_names: - raise TypeError( - 'The following fields are duplicated in the `fields` ' - 'declaration: ' - '{0}'.format(','.join(sorted(duplicate_field_names))) - ) - - # check that field names are valid identifiers - invalid_field_names = { - field_name for field_name in field_names if not _is_valid_identifier(field_name) - } - if invalid_field_names: - raise TypeError( - 'The following field names are not valid python identifiers: {0}'.format( - ','.join('`{0}`'.format(item) for item in sorted(invalid_field_names)) - ) - ) - - -IDENTIFIER_REGEX = re.compile(r'^[^\d\W]\w*\Z', re.UNICODE) - - -def _is_valid_identifier(value): - # Source: https://stackoverflow.com/questions/5474008/regular-expression-to-confirm-whether-a-string-is-a-valid-identifier-in-python # noqa: E501 - if not isinstance(value, str): - return False - return bool(IDENTIFIER_REGEX.match(value)) - - -@to_set -def _get_class_namespace(cls): - if hasattr(cls, '__dict__'): - yield from cls.__dict__.keys() - if hasattr(cls, '__slots__'): - yield from cls.__slots__ - - -class MetaSerializable(abc.ABCMeta): - def __new__(mcls, name, bases, namespace): - declares_fields = FIELDS_META_ATTR in namespace - - if declares_fields: - has_fields = True - fields = namespace.pop(FIELDS_META_ATTR) - field_sedes = tuple(sedes for field_name, sedes in fields) - try: - sedes = Container(field_sedes) - except ValidationError as exception: - # catch empty or duplicate fields and reraise as a TypeError as this would be an - # invalid class definition - raise TypeError(str(exception)) from exception - - else: - serializable_bases = tuple(base for base in bases if isinstance(base, MetaSerializable)) - bases_with_fields = tuple(base for base in serializable_bases if base._meta.has_fields) - - if len(bases_with_fields) == 0: - has_fields = False - fields = None - sedes = None - elif len(bases_with_fields) == 1: - has_fields = True - fields = bases_with_fields[0]._meta.fields - sedes = bases_with_fields[0]._meta.container_sedes - else: - raise TypeError( - 'Fields need to be declared explicitly as class has multiple `Serializable` ' - 'parents with fields themselves' - ) - - # create the class without any fields as neither the class itself nor any of its ancestors - # have defined fields - if not has_fields: - meta = Meta( - has_fields=False, - fields=None, - container_sedes=None, - field_names=None, - field_attrs=None, - ) - return super().__new__(mcls, name, bases, assoc(namespace, '_meta', meta)) - - # from here on, we can assume that we've got fields and a sedes object - if sedes is None: - raise Exception('Invariant: sedes has been initialized earlier') - if len(fields) == 0: - raise Exception('Invariant: number of fields has been checked at initializion of sedes') - - field_names, _ = zip(*fields) - _validate_field_names(field_names) - - # the actual field values are stored in separate *private* attributes. - # This computes attribute names that don't conflict with other - # attributes already present on the class. - reserved_namespace = set(namespace.keys()).union( - attr - for base in bases - for parent_cls in base.__mro__ - for attr in _get_class_namespace(parent_cls) - ) - field_attrs = _mk_field_attrs(field_names, reserved_namespace) - field_props = _mk_field_props(field_names, field_attrs) - - # construct the Meta object to store field information for the class - meta = Meta( - has_fields=True, - fields=fields, - container_sedes=sedes, - field_names=field_names, - field_attrs=field_attrs, - ) - return super().__new__(mcls, name, bases, merge(namespace, field_props, {'_meta': meta})) - - # - # Implement BaseSedes methods as pass-throughs to the container sedes - # - def serialize(cls: Type[TSerializable], value: TSerializable) -> bytes: - # return cls._meta.container_sedes.serialize(value) - if value._serialize_cache is None: - value._serialize_cache = cls._meta.container_sedes.serialize(value) - return value._serialize_cache - - def deserialize(cls: Type[TSerializable], data: bytes) -> TSerializable: - deserialized_fields = cls._meta.container_sedes.deserialize(data) - deserialized_field_dict = dict(zip(cls._meta.field_names, deserialized_fields)) - return cls(**deserialized_field_dict) - - def get_hash_tree_root( - cls: Type[TSerializable], value: TSerializable, cache: bool = True - ) -> bytes: - if cache: - root, cache = cls._meta.container_sedes.get_hash_tree_root_and_leaves( - value, value.cache - ) - value.cache = cache - return root - else: - return cls._meta.container_sedes.get_hash_tree_root(value) - - @property - def is_fixed_sized(cls): - return cls._meta.container_sedes.is_fixed_sized - - def get_fixed_size(cls): - return cls._meta.container_sedes.get_fixed_size() - - -# Make any class created with MetaSerializable an instance of BaseSedes -BaseSedes.register(MetaSerializable) - - -class Serializable(BaseSerializable, metaclass=MetaSerializable): - """ - The base class for serializable objects. - """ - - def __str__(self) -> str: - return f"[{', '.join((str(v) for v in self))}]" - - def __repr__(self) -> str: - return f'<{self.__class__.__name__}: {str(self)}>' diff --git a/sw_utils/ssz/sedes/uint.py b/sw_utils/ssz/sedes/uint.py deleted file mode 100644 index 538167b..0000000 --- a/sw_utils/ssz/sedes/uint.py +++ /dev/null @@ -1,47 +0,0 @@ -from typing import Any - -from sw_utils.ssz.exceptions import DeserializationError, SerializationError -from sw_utils.ssz.sedes.basic import BasicSedes - - -class UInt(BasicSedes[int, int]): - def __init__(self, num_bits: int) -> None: - if num_bits % 8 != 0: - raise ValueError('Number of bits must be a multiple of 8') - self.num_bits = num_bits - super().__init__(num_bits // 8) - - def serialize(self, value: int) -> bytes: - if value < 0: - raise SerializationError(f'Can only serialize non-negative integers, got {value}') - - try: - return value.to_bytes(self.size, 'little') - except OverflowError: - raise SerializationError( - f'{value} is too large to be serialized in {self.size * 8} bits' - ) - - def deserialize(self, data: bytes) -> int: - if len(data) != self.size: - raise DeserializationError( - f'Cannot deserialize length {len(data)} byte-string as uint{self.size*8}' - ) - return int.from_bytes(data, 'little') - - def get_sedes_id(self) -> str: - return f'{self.__class__.__name__}{self.num_bits}' - - def __hash__(self) -> int: - return hash((hash(UInt), self.num_bits)) - - def __eq__(self, other: Any) -> bool: - return isinstance(other, UInt) and other.num_bits == self.num_bits - - -uint8 = UInt(8) -uint16 = UInt(16) -uint32 = UInt(32) -uint64 = UInt(64) -uint128 = UInt(128) -uint256 = UInt(256) diff --git a/sw_utils/ssz/sedes/vector.py b/sw_utils/ssz/sedes/vector.py deleted file mode 100644 index 327ff25..0000000 --- a/sw_utils/ssz/sedes/vector.py +++ /dev/null @@ -1,146 +0,0 @@ -from typing import IO, Any, Iterable, Sequence, Tuple - -from eth_typing import Hash32 -from eth_utils import to_tuple -from eth_utils.toolz import sliding_window - -from sw_utils.ssz.cache.utils import ( - get_merkle_leaves_with_cache, - get_merkle_leaves_without_cache, -) -from sw_utils.ssz.exceptions import SerializationError -from sw_utils.ssz.hashable_structure import BaseHashableStructure -from sw_utils.ssz.hashable_vector import HashableVector -from sw_utils.ssz.sedes.base import BaseSedes, TSedes -from sw_utils.ssz.sedes.basic import BasicSedes, HomogeneousProperCompositeSedes -from sw_utils.ssz.typing import CacheObj, TDeserializedElement, TSerializableElement -from sw_utils.ssz.utils import ( - merkleize, - merkleize_with_cache, - pack, - read_exact, - s_decode_offset, -) - -TSedesPairs = Tuple[ - Tuple[BaseSedes[TSerializableElement, TDeserializedElement], TSerializableElement], - ..., -] - - -class Vector( - HomogeneousProperCompositeSedes[ - Sequence[TSerializableElement], Tuple[TDeserializedElement, ...] - ] -): - def __init__(self, element_sedes: TSedes, length: int) -> None: - if length < 1: - raise ValueError(f'Vectors must have a size of 1 or greater, got {length}') - self.element_sedes = element_sedes - self.max_length = length - - @property - def length(self) -> int: - return self.max_length - - def get_element_sedes(self, index) -> BaseSedes[TSerializableElement, TDeserializedElement]: - return self.element_sedes - - # - # Size - # - @property - def is_fixed_sized(self) -> bool: - return self.element_sedes.is_fixed_sized - - def get_fixed_size(self) -> int: - if not self.is_fixed_sized: - raise ValueError('Vector is not fixed size.') - - if self.length == 0: - return 0 - else: - return self.length * self.element_sedes.get_fixed_size() - - # - # Serialization - # - def _validate_serializable(self, value: Any) -> None: - if len(value) != self.length: - raise SerializationError( - f'Length mismatch. Cannot serialize value with length ' - f'{len(value)} as {self.length}-tuple' - ) - - # - # Deserialization - # - def _deserialize_stream(self, stream: IO[bytes]) -> Iterable[TDeserializedElement]: - elements = self._deserialize_stream_to_tuple(stream) - return HashableVector.from_iterable(elements, sedes=self) - - @to_tuple - def _deserialize_stream_to_tuple(self, stream: IO[bytes]) -> Iterable[TDeserializedElement]: - if self.element_sedes.is_fixed_sized: - element_size = self.element_sedes.get_fixed_size() - for _ in range(self.length): - element_data = read_exact(element_size, stream) - yield self.element_sedes.deserialize(element_data) - else: - offsets = tuple(s_decode_offset(stream) for _ in range(self.length)) - - for left_offset, right_offset in sliding_window(2, offsets): - element_length = right_offset - left_offset - element_data = read_exact(element_length, stream) - yield self.element_sedes.deserialize(element_data) - - # simply reading to the end of the current stream gives us all of the final element data - final_element_data = stream.read() - yield self.element_sedes.deserialize(final_element_data) - - # - # Tree hashing - # - def get_hash_tree_root(self, value: Sequence[Any]) -> bytes: - if isinstance(value, BaseHashableStructure) and value.sedes == self: - return value.hash_tree_root - - if isinstance(self.element_sedes, BasicSedes): - serialized_elements = tuple(self.element_sedes.serialize(element) for element in value) - return merkleize(pack(serialized_elements)) - else: - element_tree_hashes = tuple( - self.element_sedes.get_hash_tree_root(element) for element in value - ) - return merkleize(element_tree_hashes) - - def get_hash_tree_root_and_leaves( - self, value: Sequence[Any], cache: CacheObj - ) -> Tuple[Hash32, CacheObj]: - merkle_leaves = () - if isinstance(self.element_sedes, BasicSedes): - serialized_elements = tuple(self.element_sedes.serialize(element) for element in value) - merkle_leaves = pack(serialized_elements) - else: - has_get_hash_tree_root_and_leaves = hasattr( - self.element_sedes, 'get_hash_tree_root_and_leaves' - ) - if has_get_hash_tree_root_and_leaves: - merkle_leaves = get_merkle_leaves_with_cache(value, self.element_sedes, cache) - else: - merkle_leaves = get_merkle_leaves_without_cache(value, self.element_sedes) - - return merkleize_with_cache(merkle_leaves, cache=cache, limit=self.chunk_count) - - # - # Equality and hashing - # - def __hash__(self) -> int: - return hash((hash(Vector), hash(self.element_sedes), self.length)) - - def __eq__(self, other: Any) -> bool: - return ( - isinstance(other, Vector) - and other.element_sedes == self.element_sedes - and other.length == self.length - ) diff --git a/sw_utils/ssz/typing.py b/sw_utils/ssz/typing.py deleted file mode 100644 index c185797..0000000 --- a/sw_utils/ssz/typing.py +++ /dev/null @@ -1,10 +0,0 @@ -from collections.abc import MutableMapping -from typing import NewType, TypeVar, Union - -TSerializable = TypeVar('TSerializable') -TDeserialized = TypeVar('TDeserialized') - -TSerializableElement = TypeVar('TSerializable') -TDeserializedElement = TypeVar('TDeserialized') - -CacheObj = NewType('CacheObj', Union[MutableMapping, dict]) diff --git a/sw_utils/ssz/utils.py b/sw_utils/ssz/utils.py deleted file mode 100644 index faaee35..0000000 --- a/sw_utils/ssz/utils.py +++ /dev/null @@ -1,221 +0,0 @@ -import collections -import functools -from typing import IO, Any, Sequence, Tuple - -from eth_typing import Hash32 - -from sw_utils.ssz.constants import ( - BASE_TYPES, - CHUNK_SIZE, - EMPTY_CHUNK, - OFFSET_SIZE, - ZERO_HASHES, -) -from sw_utils.ssz.exceptions import DeserializationError -from sw_utils.ssz.hash import hash_eth2 -from sw_utils.ssz.typing import CacheObj - - -def get_duplicates(values): - counts = collections.Counter(values) - return tuple(item for item, num in counts.items() if num > 1) - - -def read_exact(num_bytes: int, stream: IO[bytes]) -> bytes: - data = stream.read(num_bytes) - if len(data) != num_bytes: - raise DeserializationError(f'Tried to read {num_bytes}. Only got {len(data)} bytes') - return data - - -def encode_offset(offset: int) -> bytes: - return offset.to_bytes(OFFSET_SIZE, 'little') - - -def decode_offset(data: bytes) -> int: - return int.from_bytes(data, 'little') - - -def s_decode_offset(stream: IO[bytes]) -> int: - data = read_exact(OFFSET_SIZE, stream) - return decode_offset(data) - - -def get_items_per_chunk(item_size: int) -> int: - if item_size < 0: - raise ValueError('Item size must be positive integer') - elif item_size == 0: - return 1 - elif CHUNK_SIZE % item_size != 0: - raise ValueError('Item size must be a divisor of chunk size') - elif item_size <= CHUNK_SIZE: - return CHUNK_SIZE // item_size - else: - raise Exception('Invariant: unreachable') - - -def pad_zeros(value: bytes) -> bytes: - if len(value) >= CHUNK_SIZE: - raise ValueError( - f'The length of given value {len(value)} should be less than CHUNK_SIZE ({CHUNK_SIZE})' - ) - return value.ljust(CHUNK_SIZE, b'\x00') - - -@functools.lru_cache(maxsize=2**12) -def to_chunks(packed_data: bytes) -> Tuple[bytes, ...]: - size = len(packed_data) - number_of_full_chunks = size // CHUNK_SIZE - last_chunk_is_full = size % CHUNK_SIZE == 0 - - full_chunks = tuple( - packed_data[chunk_index * CHUNK_SIZE : (chunk_index + 1) * CHUNK_SIZE] - for chunk_index in range(number_of_full_chunks) - ) - if last_chunk_is_full: - return full_chunks - else: - last_chunk = pad_zeros(packed_data[number_of_full_chunks * CHUNK_SIZE :]) - return full_chunks + (last_chunk,) - - -@functools.lru_cache(maxsize=2**12) -def pack(serialized_values: Sequence[bytes]) -> Tuple[Hash32, ...]: - if len(serialized_values) == 0: - return (EMPTY_CHUNK,) - - data = b''.join(serialized_values) - return to_chunks(data) - - -@functools.lru_cache(maxsize=2**12) -def pack_bytes(byte_string: bytes) -> Tuple[bytes, ...]: - if len(byte_string) == 0: - return (EMPTY_CHUNK,) - - return to_chunks(byte_string) - - -@functools.lru_cache(maxsize=2**12) -def pack_bits(values: Sequence[bool]) -> Tuple[Hash32]: - as_bytearray = get_serialized_bytearray(values, len(values), extra_byte=False) - packed = bytes(as_bytearray) - return to_chunks(packed) - - -def get_next_power_of_two(value: int) -> int: - if value <= 0: - return 1 - else: - return 2 ** (value - 1).bit_length() - - -def _get_chunk_and_max_depth( - chunks: Sequence[Hash32], limit: int, chunk_len: int -) -> Tuple[int, int]: - chunk_depth = max(chunk_len - 1, 0).bit_length() - max_depth = max(chunk_depth, (limit - 1).bit_length()) - if max_depth > len(ZERO_HASHES): - raise ValueError(f'The number of layers is greater than {len(ZERO_HASHES)}') - - return chunk_depth, max_depth - - -def _get_merkleized_result( - chunks: Sequence[Hash32], - chunk_len: int, - chunk_depth: int, - max_depth: int, - cache: CacheObj, -) -> Tuple[Hash32, CacheObj]: - merkleized_result_per_layers = [None for _ in range(max_depth + 1)] - - def merge(leaf: bytes, leaf_index: int) -> None: - node = leaf - layer = 0 - while True: - if leaf_index & (1 << layer) == 0: - if leaf_index == chunk_len and layer < chunk_depth: - # Keep going if we are complementing the void to the next power of 2 - key = node + ZERO_HASHES[layer] - if key not in cache: - cache[key] = hash_eth2(key) - node = cache[key] - else: - break - else: - key = merkleized_result_per_layers[layer] + node - if key not in cache: - cache[key] = hash_eth2(key) - node = cache[key] - layer += 1 - - merkleized_result_per_layers[layer] = node - - # Merge in leaf by leaf. - for leaf_index in range(chunk_len): - merge(chunks[leaf_index], leaf_index) - - # Complement with 0 if empty, or if not the right power of 2 - if 1 << chunk_depth != chunk_len: - merge(ZERO_HASHES[0], chunk_len) - - # The next power of two may be smaller than the ultimate virtual size, - # complement with zero-hashes at each depth. - for layer in range(chunk_depth, max_depth): - key = merkleized_result_per_layers[layer] + ZERO_HASHES[layer] - if key not in cache: - cache[key] = hash_eth2(merkleized_result_per_layers[layer] + ZERO_HASHES[layer]) - merkleized_result_per_layers[layer + 1] = cache[key] - - root = merkleized_result_per_layers[max_depth] - - return root, cache - - -def merkleize_with_cache( - chunks: Sequence[Hash32], cache: CacheObj, limit: int = None -) -> Tuple[Hash32, CacheObj]: - chunk_len = len(chunks) - if limit is None: - limit = chunk_len - chunk_depth, max_depth = _get_chunk_and_max_depth(chunks, limit, chunk_len) - - if limit == 0: - return ZERO_HASHES[0], cache - - return _get_merkleized_result( - chunks=chunks, - chunk_len=chunk_len, - chunk_depth=chunk_depth, - max_depth=max_depth, - cache=cache, - ) - - -def merkleize(chunks: Sequence[Hash32], limit: int = None) -> Hash32: - root, _ = merkleize_with_cache(chunks, {}, limit) - return root - - -def mix_in_length(root: Hash32, length: int) -> Hash32: - return hash_eth2(root + length.to_bytes(CHUNK_SIZE, 'little')) - - -def get_serialized_bytearray(value: Sequence[bool], bit_count: int, extra_byte: bool) -> bytearray: - if extra_byte: - # Serialize Bitlist - as_bytearray = bytearray(bit_count // 8 + 1) - else: - # Serialize Bitvector - as_bytearray = bytearray((bit_count + 7) // 8) - - for i in range(bit_count): - as_bytearray[i // 8] |= value[i] << (i % 8) - return as_bytearray - - -def is_immutable_field_value(value: Any) -> bool: - return type(value) in BASE_TYPES or ( - isinstance(value, tuple) and (len(value) == 0 or is_immutable_field_value(value[0])) - )