diff --git a/eth/tools/_utils/git.py b/eth/tools/_utils/git.py index e8bda31d93..d45eb91340 100644 --- a/eth/tools/_utils/git.py +++ b/eth/tools/_utils/git.py @@ -3,6 +3,6 @@ from eth_utils import to_text -def get_version_from_git(): +def get_version_from_git() -> str: version = subprocess.check_output(["git", "describe"]).strip() return to_text(version) diff --git a/eth/tools/_utils/hashing.py b/eth/tools/_utils/hashing.py index 79d5296af1..f5172ccd0d 100644 --- a/eth/tools/_utils/hashing.py +++ b/eth/tools/_utils/hashing.py @@ -2,10 +2,19 @@ import rlp +from typing import ( + Iterable, + Tuple, +) + +from eth_typing import ( + Hash32, +) + from eth.rlp.logs import Log -def hash_log_entries(log_entries): +def hash_log_entries(log_entries: Iterable[Tuple[bytes, bytes, bytes]]) -> Hash32: """ Helper function for computing the RLP hash of the logs from transaction execution. diff --git a/eth/tools/_utils/mappings.py b/eth/tools/_utils/mappings.py index 236845c1e8..8732ab5726 100644 --- a/eth/tools/_utils/mappings.py +++ b/eth/tools/_utils/mappings.py @@ -1,21 +1,27 @@ from collections.abc import Mapping import itertools +from typing import ( + Any, + Dict, + Sequence, +) + from cytoolz import merge_with -def merge_if_dicts(values): +def merge_if_dicts(values: Sequence[Dict[Any, Any]]) -> Any: if all(isinstance(item, Mapping) for item in values): return merge_with(merge_if_dicts, *values) else: return values[-1] -def deep_merge(*dicts): +def deep_merge(*dicts: Dict[Any, Any]) -> Dict[Any, Any]: return merge_with(merge_if_dicts, *dicts) -def is_cleanly_mergable(*dicts): +def is_cleanly_mergable(*dicts: Dict[Any, Any]) -> bool: """Check that nothing will be overwritten when dictionaries are merged using `deep_merge`. Examples: diff --git a/eth/tools/_utils/normalization.py b/eth/tools/_utils/normalization.py index 5553d4a916..d5b44a42c2 100644 --- a/eth/tools/_utils/normalization.py +++ b/eth/tools/_utils/normalization.py @@ -1,9 +1,19 @@ import binascii -from collections.abc import ( +import functools + +from typing import ( + Any, + AnyStr, + Callable, + cast, + Dict, Iterable, + List, Mapping, + Sequence, + Tuple, + Union, ) -import functools from cytoolz import ( assoc_in, @@ -15,6 +25,10 @@ ) import cytoolz.curried +from eth_typing import ( + Address, +) + from eth_utils import ( apply_formatters_to_dict, big_endian_to_int, @@ -41,21 +55,31 @@ is_cleanly_mergable, ) +from eth.typing import ( + AccountState, + GeneralState, + IntConvertible, + Normalizer, + TransactionDict, + TransactionNormalizer, +) + # # Primitives # @functools.lru_cache(maxsize=1024) -def normalize_int(value): +def normalize_int(value: IntConvertible) -> int: """ Robust to integer conversion, handling hex values, string representations, and special cases like `0x`. """ if is_integer(value): - return value + return cast(int, value) elif is_bytes(value): return big_endian_to_int(value) elif is_hex(value) and is_0x_prefixed(value): + value = cast(str, value) if len(value) == 2: return 0 else: @@ -66,9 +90,9 @@ def normalize_int(value): raise TypeError("Unsupported type: Got `{0}`".format(type(value))) -def normalize_bytes(value): +def normalize_bytes(value: Union[bytes, str]) -> bytes: if is_bytes(value): - return value + return cast(bytes, value) elif is_text(value) and is_hex(value): return decode_hex(value) elif is_text(value): @@ -78,7 +102,7 @@ def normalize_bytes(value): @functools.lru_cache(maxsize=1024) -def to_int(value): +def to_int(value: str) -> int: """ Robust to integer conversion, handling hex values, string representations, and special cases like `0x`. @@ -93,7 +117,7 @@ def to_int(value): @functools.lru_cache(maxsize=128) -def normalize_to_address(value): +def normalize_to_address(value: AnyStr) -> Address: if value: return to_canonical_address(value) else: @@ -106,21 +130,24 @@ def normalize_to_address(value): # # Containers # -def dict_normalizer(formatters, required=None, optional=None): +def dict_normalizer(formatters: Dict[Any, Callable[..., Any]], + required: Iterable[Any]=None, + optional: Iterable[Any]=None) -> Normalizer: + all_keys = set(formatters.keys()) if required is None and optional is None: - required = all_keys + required_set_form = all_keys + elif required is not None and optional is not None: + raise ValueError("Both required and optional keys specified") elif required is not None: - required = set(required) + required_set_form = set(required) elif optional is not None: - required = all_keys - set(optional) - else: - raise ValueError("Both required and optional keys specified") + required_set_form = all_keys - set(optional) - def normalizer(d): + def normalizer(d: Dict[Any, Any]) -> Dict[str, Any]: keys = set(d.keys()) - missing_keys = required - keys + missing_keys = required_set_form - keys superfluous_keys = keys - all_keys if missing_keys: raise KeyError("Missing required keys: {}".format(", ".join(missing_keys))) @@ -132,9 +159,9 @@ def normalizer(d): return normalizer -def dict_options_normalizer(normalizers): +def dict_options_normalizer(normalizers: Iterable[Normalizer]) -> Normalizer: - def normalize(d): + def normalize(d: Dict[Any, Any]) -> Dict[str, Any]: first_exception = None for normalizer in normalizers: try: @@ -153,7 +180,7 @@ def normalize(d): # # Composition # -def state_definition_to_dict(state_definition): +def state_definition_to_dict(state_definition: GeneralState) -> AccountState: """Convert a state definition to the canonical dict form. State can either be defined in the canonical form, or as a list of sub states that are then @@ -225,7 +252,7 @@ def state_definition_to_dict(state_definition): ) -normalize_main_transaction = dict_normalizer({ +normalize_main_transaction = cast(Normalizer, dict_normalizer({ "data": normalize_bytes, "gasLimit": normalize_int, "gasPrice": normalize_int, @@ -233,15 +260,15 @@ def state_definition_to_dict(state_definition): "secretKey": normalize_bytes, "to": normalize_to_address, "value": normalize_int, -}) +})) -normalize_transaction = dict_options_normalizer([ +normalize_transaction = cast(TransactionNormalizer, dict_options_normalizer([ normalize_main_transaction, -]) +])) -normalize_main_transaction_group = dict_normalizer({ +normalize_main_transaction_group = cast(Normalizer, dict_normalizer({ "data": eth_utils.curried.apply_formatter_to_array(normalize_bytes), "gasLimit": eth_utils.curried.apply_formatter_to_array(normalize_int), "gasPrice": normalize_int, @@ -249,12 +276,12 @@ def state_definition_to_dict(state_definition): "secretKey": normalize_bytes, "to": normalize_to_address, "value": eth_utils.curried.apply_formatter_to_array(normalize_int), -}) +})) -normalize_transaction_group = dict_options_normalizer([ +normalize_transaction_group = cast(TransactionNormalizer, dict_options_normalizer([ normalize_main_transaction_group, -]) +])) normalize_execution = dict_normalizer({ @@ -305,10 +332,14 @@ def state_definition_to_dict(state_definition): # # Fixture Normalizers # -def normalize_unsigned_transaction(transaction, indexes): +def normalize_unsigned_transaction(transaction: TransactionDict, + indexes: Dict[str, Any]) -> TransactionDict: + normalized = normalize_transaction_group(transaction) return merge(normalized, { - transaction_key: normalized[transaction_key][indexes[index_key]] + # Dynamic key access not yet allowed with TypedDict + # https://github.com/python/mypy/issues/5359 + transaction_key: normalized[transaction_key][indexes[index_key]] # type: ignore for transaction_key, index_key in [ ("gasLimit", "gas"), ("value", "value"), @@ -318,7 +349,7 @@ def normalize_unsigned_transaction(transaction, indexes): }) -def normalize_account_state(account_state): +def normalize_account_state(account_state: AccountState) -> AccountState: return { to_canonical_address(address): { 'balance': to_int(state['balance']), @@ -333,14 +364,17 @@ def normalize_account_state(account_state): @to_dict -def normalize_post_state(post_state): +def normalize_post_state(post_state: Dict[str, Any]) -> Iterable[Tuple[str, bytes]]: yield 'hash', decode_hex(post_state['hash']) if 'logs' in post_state: yield 'logs', decode_hex(post_state['logs']) @curry -def normalize_statetest_fixture(fixture, fork, post_state_index): +def normalize_statetest_fixture(fixture: Dict[str, Any], + fork: str, + post_state_index: int) -> Dict[str, Any]: + post_state = fixture['post'][fork][post_state_index] normalized_fixture = { @@ -356,7 +390,7 @@ def normalize_statetest_fixture(fixture, fork, post_state_index): return normalized_fixture -def normalize_exec(exec_params): +def normalize_exec(exec_params: Dict[str, Any]) -> Dict[str, Any]: return { 'origin': to_canonical_address(exec_params['origin']), 'address': to_canonical_address(exec_params['address']), @@ -368,7 +402,7 @@ def normalize_exec(exec_params): } -def normalize_callcreates(callcreates): +def normalize_callcreates(callcreates: Sequence[Dict[str, Any]]) -> List[Dict[str, Any]]: return [ { 'data': decode_hex(created_call['data']), @@ -384,7 +418,7 @@ def normalize_callcreates(callcreates): @to_dict -def normalize_vmtest_fixture(fixture): +def normalize_vmtest_fixture(fixture: Dict[str, Any]) -> Iterable[Tuple[str, Any]]: yield 'env', normalize_environment(fixture['env']) yield 'exec', normalize_exec(fixture['exec']) yield 'pre', normalize_account_state(fixture['pre']) @@ -405,7 +439,7 @@ def normalize_vmtest_fixture(fixture): yield 'logs', decode_hex(fixture['logs']) -def normalize_signed_transaction(transaction): +def normalize_signed_transaction(transaction: Dict[str, Any]) -> Dict[str, Any]: return { 'data': robust_decode_hex(transaction['data']), 'gasLimit': to_int(transaction['gasLimit']), @@ -420,7 +454,7 @@ def normalize_signed_transaction(transaction): @curry -def normalize_transactiontest_fixture(fixture, fork): +def normalize_transactiontest_fixture(fixture: Dict[str, Any], fork: str) -> Dict[str, Any]: normalized_fixture = {} @@ -440,7 +474,7 @@ def normalize_transactiontest_fixture(fixture, fork): return normalized_fixture -def normalize_block_header(header): +def normalize_block_header(header: Dict[str, Any]) -> Dict[str, Any]: normalized_header = { 'bloom': big_endian_to_int(decode_hex(header['bloom'])), 'coinbase': to_canonical_address(header['coinbase']), @@ -468,7 +502,7 @@ def normalize_block_header(header): return normalized_header -def normalize_block(block): +def normalize_block(block: Dict[str, Any]) -> Dict[str, Any]: normalized_block = {} try: @@ -487,7 +521,7 @@ def normalize_block(block): return normalized_block -def normalize_blockchain_fixtures(fixture): +def normalize_blockchain_fixtures(fixture: Dict[str, Any]) -> Dict[str, Any]: normalized_fixture = { 'blocks': [normalize_block(block_fixture) for block_fixture in fixture['blocks']], 'genesisBlockHeader': normalize_block_header(fixture['genesisBlockHeader']), diff --git a/eth/tools/_utils/vyper.py b/eth/tools/_utils/vyper.py index 1a8df17169..b1717e32fd 100644 --- a/eth/tools/_utils/vyper.py +++ b/eth/tools/_utils/vyper.py @@ -1,5 +1,13 @@ import functools +from typing import ( + Any, + Callable, + Dict, + List, + Tuple, +) + try: from vyper.compile_lll import ( compile_to_assembly, @@ -12,9 +20,9 @@ vyper_available = True -def require_vyper(fn): +def require_vyper(fn: Callable[..., Any]) -> Callable[..., Any]: @functools.wraps(fn) - def inner(*args, **kwargs): + def inner(*args: Any, **kwargs: Any) -> Any: if vyper_available: return fn(*args, **kwargs) else: @@ -23,7 +31,7 @@ def inner(*args, **kwargs): @require_vyper -def compile_vyper_lll(vyper_code): +def compile_vyper_lll(vyper_code: List[Any]) -> Tuple[bytes, Dict[str, Any]]: lll_node = LLLnode.from_list(vyper_code) assembly = compile_to_assembly(lll_node) code = assembly_to_evm(assembly) diff --git a/eth/tools/builder/chain/builders.py b/eth/tools/builder/chain/builders.py index 9d7b4f6098..93b37faede 100644 --- a/eth/tools/builder/chain/builders.py +++ b/eth/tools/builder/chain/builders.py @@ -92,7 +92,7 @@ def build(obj: Any, *applicators: Callable[..., Any]) -> Any: # Constructors (creation of chain classes) # @curry -def name(class_name: str, chain_class: BaseChain) -> type: +def name(class_name: str, chain_class: Type[BaseChain]) -> Type[BaseChain]: """ Assign the given name to the chain class. """ @@ -100,7 +100,7 @@ def name(class_name: str, chain_class: BaseChain) -> type: @curry -def chain_id(chain_id: int, chain_class: BaseChain) -> type: +def chain_id(chain_id: int, chain_class: Type[BaseChain]) -> Type[BaseChain]: """ Set the ``chain_id`` for the chain class. """ @@ -108,7 +108,7 @@ def chain_id(chain_id: int, chain_class: BaseChain) -> type: @curry -def fork_at(vm_class: Type[BaseVM], at_block: int, chain_class: BaseChain) -> type: +def fork_at(vm_class: Type[BaseVM], at_block: int, chain_class: Type[BaseChain]) -> Type[BaseChain]: """ Adds the ``vm_class`` to the chain's ``vm_configuration``. @@ -168,7 +168,7 @@ def _set_vm_dao_support_false(vm_configuration: VMConfiguration) -> VMConfigurat @curry -def disable_dao_fork(chain_class: BaseChain) -> type: +def disable_dao_fork(chain_class: Type[BaseChain]) -> Type[BaseChain]: """ Set the ``support_dao_fork`` flag to ``False`` on the :class:`~eth.vm.forks.homestead.HomesteadVM`. Requires that presence of @@ -199,7 +199,8 @@ def _set_vm_dao_fork_block_number(dao_fork_block_number: BlockNumber, @curry -def dao_fork_at(dao_fork_block_number: BlockNumber, chain_class: BaseChain) -> type: +def dao_fork_at(dao_fork_block_number: BlockNumber, + chain_class: Type[BaseChain]) -> Type[BaseChain]: """ Set the block number on which the DAO fork will happen. Requires that a version of the :class:`~eth.vm.forks.homestead.HomesteadVM` is present in @@ -263,7 +264,7 @@ def _mix_in_pow_mining(vm_configuration: VMConfiguration) -> VMConfiguration: @curry -def enable_pow_mining(chain_class: BaseChain) -> type: +def enable_pow_mining(chain_class: Type[BaseChain]) -> Type[BaseChain]: """ Inject on demand generation of the proof of work mining seal on newly mined blocks into each of the chain's vms. @@ -299,7 +300,7 @@ def _mix_in_disable_seal_validation(vm_configuration: VMConfiguration) -> VMConf @curry -def disable_pow_check(chain_class: Type[BaseChain]) -> type: +def disable_pow_check(chain_class: Type[BaseChain]) -> Type[BaseChain]: """ Disable the proof of work validation check for each of the chain's vms. This allows for block mining without generation of the proof of work seal. diff --git a/eth/tools/fixtures/_utils.py b/eth/tools/fixtures/_utils.py index a0ec8bc52b..fe9e85e107 100644 --- a/eth/tools/fixtures/_utils.py +++ b/eth/tools/fixtures/_utils.py @@ -2,20 +2,26 @@ import functools import os +from typing import ( + Any, + Callable, + Iterable, +) + from eth_utils import to_tuple @to_tuple -def recursive_find_files(base_dir, pattern): +def recursive_find_files(base_dir: str, pattern: str) -> Iterable[str]: for dirpath, _, filenames in os.walk(base_dir): for filename in filenames: if fnmatch.fnmatch(filename, pattern): yield os.path.join(dirpath, filename) -def require_pytest(fn): +def require_pytest(fn: Callable[..., Any]) -> Callable[..., Any]: @functools.wraps(fn) - def inner(*args, **kwargs): + def inner(*args: Any, **kwargs: Any) -> Callable[..., Any]: try: import pytest # noqa: F401 except ImportError: diff --git a/eth/tools/fixtures/fillers/_utils.py b/eth/tools/fixtures/fillers/_utils.py index 25eee5dfbd..018fa9c885 100644 --- a/eth/tools/fixtures/fillers/_utils.py +++ b/eth/tools/fixtures/fillers/_utils.py @@ -1,11 +1,29 @@ import copy import random +from typing import ( + Any, + Dict, + List, + Tuple, + Type, +) + +from eth_typing import ( + Address, +) + from eth_utils import ( int_to_big_endian, ) from eth.db.backends.memory import MemoryDB +from eth.db.account import BaseAccountDB + +from eth.typing import ( + AccountState, + TransactionDict, +) from eth.utils.db import ( apply_state_dict, @@ -17,13 +35,15 @@ from eth_keys import keys -def wrap_in_list(item): +def wrap_in_list(item: Any) -> List[Any]: return [item] -def add_transaction_to_group(group, transaction): +def add_transaction_to_group(group: Dict[str, Any], + transaction: TransactionDict) -> Tuple[Dict[str, Any], Dict[str, int]]: + for key in ["gasPrice", "nonce", "secretKey", "to"]: - if key in transaction and transaction[key] != group[key]: + if key in transaction and transaction[key] != group[key]: # type: ignore # https://github.com/python/mypy/issues/5359 # noqa: 501 raise ValueError("Can't add transaction as it differs in {}".format(key)) new_group = copy.deepcopy(group) @@ -35,26 +55,26 @@ def add_transaction_to_group(group, transaction): raise ValueError("Can't add transaction as {} is ambiguous".format(key)) index = 0 else: - if transaction[key] not in new_group[key]: - new_group[key].append(transaction[key]) - index = new_group[key].index(transaction[key]) + if transaction[key] not in new_group[key]: # type: ignore # https://github.com/python/mypy/issues/5359 # noqa: 501 + new_group[key].append(transaction[key]) # type: ignore # https://github.com/python/mypy/issues/5359 # noqa: 501 + index = new_group[key].index(transaction[key]) # type: ignore # https://github.com/python/mypy/issues/5359 # noqa: 501 indexes[index_key] = index else: assert key not in transaction return new_group, indexes -def calc_state_root(state, account_db_class): +def calc_state_root(state: AccountState, account_db_class: Type[BaseAccountDB]) -> bytes: account_db = account_db_class(MemoryDB()) apply_state_dict(account_db, state) return account_db.state_root -def generate_random_keypair(): +def generate_random_keypair() -> Tuple[bytes, Address]: key_object = keys.PrivateKey(pad32(int_to_big_endian(random.getrandbits(8 * 32)))) return key_object.to_bytes(), key_object.public_key.to_canonical_address() -def generate_random_address(): +def generate_random_address() -> Address: _, address = generate_random_keypair() return address diff --git a/eth/tools/fixtures/fillers/common.py b/eth/tools/fixtures/fillers/common.py index 9c3f7d996b..0d2dc33493 100644 --- a/eth/tools/fixtures/fillers/common.py +++ b/eth/tools/fixtures/fillers/common.py @@ -7,8 +7,10 @@ ) from typing import ( # noqa: F401 Any, + Callable, Dict, List, + Sequence, ) from cytoolz import ( assoc, @@ -39,6 +41,11 @@ compile_vyper_lll, ) +from eth.typing import ( + GeneralState, + TransactionDict, +) + from ._utils import ( add_transaction_to_group, wrap_in_list, @@ -69,10 +76,10 @@ "secretKey": decode_hex("0x45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8"), "to": to_canonical_address("0x0f572e5295c57f15886f9b263e2f6d2d6c7b5ec6"), "value": 0 -} +} # type: TransactionDict -def get_default_transaction(networks): +def get_default_transaction(networks: Any) -> TransactionDict: return DEFAULT_MAIN_TRANSACTION @@ -96,7 +103,7 @@ def get_default_transaction(networks): # Filler Generation # -def setup_filler(name, environment=None): +def setup_filler(name: str, environment: Dict[Any, Any]=None) -> Dict[str, Dict[str, Any]]: environment = normalize_environment(environment or {}) return {name: { "env": environment, @@ -104,7 +111,7 @@ def setup_filler(name, environment=None): }} -def setup_main_filler(name, environment=None): +def setup_main_filler(name: str, environment: Dict[Any, Any]=None) -> Dict[str, Dict[str, Any]]: """ Kick off the filler generation process by creating the general filler scaffold with a test name and general information about the testing environment. @@ -131,7 +138,7 @@ def setup_main_filler(name, environment=None): return setup_filler(name, merge(DEFAULT_MAIN_ENVIRONMENT, environment or {})) -def pre_state(*raw_state, filler): +def pre_state(*raw_state: GeneralState, filler: Dict[str, Any]) -> None: """ Specify the state prior to the test execution. Multiple invocations don't override the state but extend it instead. @@ -162,7 +169,7 @@ def pre_state(*raw_state, filler): (address, {"balance", }) """ @wraps(pre_state) - def _pre_state(filler): + def _pre_state(filler: Dict[str, Any]) -> Dict[str, Any]: test_name = get_test_name(filler) old_pre_state = filler[test_name].get("pre_state", {}) @@ -178,7 +185,11 @@ def _pre_state(filler): return assoc_in(filler, [test_name, "pre"], new_pre_state) -def _expect(post_state, networks, transaction, filler): +def _expect(post_state: Dict[str, Any], + networks: Any, + transaction: TransactionDict, + filler: Dict[str, Any]) -> Dict[str, Any]: + test_name = get_test_name(filler) test = filler[test_name] test_update = {test_name: {}} # type: Dict[str, Dict[Any, Any]] @@ -231,7 +242,10 @@ def _expect(post_state, networks, transaction, filler): return deep_merge(filler, test_update) -def expect(post_state=None, networks=None, transaction=None): +def expect(post_state: Dict[str, Any]=None, + networks: Any=None, + transaction: TransactionDict=None) -> Callable[..., Dict[str, Any]]: + """ Specify the expected result for the test. @@ -276,7 +290,7 @@ def expect(post_state=None, networks=None, transaction=None): @curry -def execution(execution, filler): +def execution(execution: Dict[str, Any], filler: Dict[str, Any]) -> Dict[str, Any]: """ For VM tests, specify the code that is being run as well as the current state of the EVM. State tests don't support this object. The parameter is a dictionary specifying some diff --git a/eth/tools/fixtures/fillers/main.py b/eth/tools/fixtures/fillers/main.py index 9f93a738e6..b006c9be69 100644 --- a/eth/tools/fixtures/fillers/main.py +++ b/eth/tools/fixtures/fillers/main.py @@ -1,3 +1,8 @@ +from typing import ( + Any, + Dict, +) + from cytoolz import ( assoc_in, merge, @@ -22,7 +27,11 @@ # # Primary test filler # -def fill_test(filler, info=None, apply_formatter=True, **kwargs): +def fill_test(filler: Dict[str, Any], + info: Dict[str, Any]=None, + apply_formatter: bool=True, + **kwargs: Any) -> Dict[str, Any]: + test_name = get_test_name(filler) test = filler[test_name] diff --git a/eth/tools/fixtures/fillers/state.py b/eth/tools/fixtures/fillers/state.py index e621b669c0..2b5f8f7235 100644 --- a/eth/tools/fixtures/fillers/state.py +++ b/eth/tools/fixtures/fillers/state.py @@ -1,7 +1,9 @@ from collections import defaultdict from typing import ( # noqa: F401 + Any, Dict, List, + Sequence, ) from eth_utils import encode_hex @@ -45,7 +47,7 @@ assert all(network in ACCOUNT_STATE_DB_CLASSES for network in ALL_NETWORKS) -def fill_state_test(filler): +def fill_state_test(filler: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: """ Filler function for filling state tests. """ diff --git a/eth/tools/fixtures/fillers/vm.py b/eth/tools/fixtures/fillers/vm.py index 513fb08348..8dd8fcc11f 100644 --- a/eth/tools/fixtures/fillers/vm.py +++ b/eth/tools/fixtures/fillers/vm.py @@ -1,3 +1,11 @@ +from typing import ( + Any, + Dict, + Iterable, + Tuple, + Union, +) + from eth.tools.fixtures.helpers import ( get_test_name, ) @@ -15,14 +23,14 @@ def fill_vm_test( - filler, - *, - call_creates=None, - gas_price=None, - gas_remaining=0, - logs=None, - output=b"" -): + filler: Dict[str, Any], + *, + call_creates: Any=None, + gas_price: Union[int, str]=None, + gas_remaining: Union[int, str]=0, + logs: Iterable[Tuple[bytes, bytes, bytes]]=None, + output: bytes=b"") -> Dict[str, Dict[str, Any]]: + test_name = get_test_name(filler) test = filler[test_name] diff --git a/eth/tools/fixtures/generation.py b/eth/tools/fixtures/generation.py index 5d92db5d21..a63b2b718f 100644 --- a/eth/tools/fixtures/generation.py +++ b/eth/tools/fixtures/generation.py @@ -1,6 +1,12 @@ import hashlib import os +from typing import ( + Any, + Callable, + Iterable, +) + from cytoolz import ( curry, identity, @@ -15,14 +21,14 @@ # # Pytest fixture generation # -def idfn(fixture_params): +def idfn(fixture_params: Iterable[Any]) -> str: """ Function for pytest to produce uniform names for fixtures. """ return ":".join((str(item) for item in fixture_params)) -def get_fixtures_file_hash(all_fixture_paths): +def get_fixtures_file_hash(all_fixture_paths: Iterable[str]) -> str: """ Returns the MD5 hash of the fixture files. Used for cache busting. """ @@ -34,10 +40,10 @@ def get_fixtures_file_hash(all_fixture_paths): @curry -def generate_fixture_tests(metafunc, - base_fixture_path, - filter_fn=identity, - preprocess_fn=identity): +def generate_fixture_tests(metafunc: Any, + base_fixture_path: str, + filter_fn: Callable[..., Any]=identity, + preprocess_fn: Callable[..., Any]=identity) -> None: """ Helper function for use with `pytest_generate_tests` which will use the pytest caching facilities to reduce the load time for fixture tests. diff --git a/eth/tools/fixtures/helpers.py b/eth/tools/fixtures/helpers.py index bf398d033d..06c0c15b3d 100644 --- a/eth/tools/fixtures/helpers.py +++ b/eth/tools/fixtures/helpers.py @@ -2,6 +2,14 @@ import rlp +from typing import ( + Any, + Dict, + Iterable, + Tuple, + Type, +) + from cytoolz import first from eth_utils import ( @@ -10,12 +18,27 @@ from eth import MainnetChain from eth.db.atomic import AtomicDB +from eth.rlp.blocks import ( + BaseBlock, +) +from eth.chains.base import ( + BaseChain, +) +from eth.db.account import ( + BaseAccountDB, +) from eth.tools.builder.chain import ( disable_pow_check, ) +from eth.typing import ( + AccountState, +) from eth.utils.state import ( diff_account_db, ) +from eth.vm.base import ( + BaseVM, +) from eth.vm.forks import ( ByzantiumVM, TangerineWhistleVM, @@ -28,7 +51,7 @@ # # State Setup # -def setup_account_db(desired_state, account_db): +def setup_account_db(desired_state: AccountState, account_db: BaseAccountDB) -> None: for account, account_data in desired_state.items(): for slot, value in account_data['storage'].items(): account_db.set_storage(account, slot, value) @@ -43,7 +66,7 @@ def setup_account_db(desired_state, account_db): account_db.persist() -def verify_account_db(expected_state, account_db): +def verify_account_db(expected_state: AccountState, account_db: BaseAccountDB) -> None: diff = diff_account_db(expected_state, account_db) if diff: error_messages = [] @@ -76,7 +99,7 @@ def verify_account_db(expected_state, account_db): ) -def chain_vm_configuration(fixture): +def chain_vm_configuration(fixture: Dict[str, Any]) -> Iterable[Tuple[int, Type[BaseVM]]]: network = fixture['network'] if network == 'Frontier': @@ -129,7 +152,7 @@ def chain_vm_configuration(fixture): raise ValueError("Network {0} does not match any known VM rules".format(network)) -def genesis_params_from_fixture(fixture): +def genesis_params_from_fixture(fixture: Dict[str, Any]) -> Dict[str, Any]: return { 'parent_hash': fixture['genesisBlockHeader']['parentHash'], 'uncles_hash': fixture['genesisBlockHeader']['uncleHash'], @@ -149,7 +172,8 @@ def genesis_params_from_fixture(fixture): } -def new_chain_from_fixture(fixture, chain_cls=MainnetChain): +def new_chain_from_fixture(fixture: Dict[str, Any], + chain_cls: Type[BaseChain]=MainnetChain) -> BaseChain: base_db = AtomicDB() vm_config = chain_vm_configuration(fixture) @@ -169,7 +193,8 @@ def new_chain_from_fixture(fixture, chain_cls=MainnetChain): ) -def apply_fixture_block_to_chain(block_fixture, chain): +def apply_fixture_block_to_chain(block_fixture: Dict[str, Any], + chain: BaseChain) -> Tuple[BaseBlock, BaseBlock, BaseBlock]: ''' :return: (premined_block, mined_block, rlp_encoded_mined_block) ''' @@ -191,12 +216,12 @@ def apply_fixture_block_to_chain(block_fixture, chain): return (block, mined_block, rlp_encoded_mined_block) -def should_run_slow_tests(): +def should_run_slow_tests() -> bool: if os.environ.get('TRAVIS_EVENT_TYPE') == 'cron': return True return False -def get_test_name(filler): +def get_test_name(filler: Dict[str, Any]) -> str: assert len(filler) == 1 return first(filler) diff --git a/eth/tools/fixtures/loading.py b/eth/tools/fixtures/loading.py index 2d8a513549..02cd96419d 100644 --- a/eth/tools/fixtures/loading.py +++ b/eth/tools/fixtures/loading.py @@ -2,6 +2,14 @@ import json import os +from typing import ( + Any, + Callable, + Dict, + Iterable, + Tuple, +) + from cytoolz import ( curry, identity, @@ -18,13 +26,13 @@ # # Filesystem fixture loading. # -def find_fixture_files(fixtures_base_dir): +def find_fixture_files(fixtures_base_dir: str) -> Iterable[str]: all_fixture_paths = recursive_find_files(fixtures_base_dir, "*.json") return all_fixture_paths @to_tuple -def find_fixtures(fixtures_base_dir): +def find_fixtures(fixtures_base_dir: str) -> Iterable[Tuple[str, str]]: """ Finds all of the (fixture_path, fixture_key) pairs for a given path under the JSON test fixtures directory. @@ -43,7 +51,7 @@ def find_fixtures(fixtures_base_dir): # all fixtures from the same file are executed sequentially allowing us to keep # a small rolling cache of the loaded fixture files. @functools.lru_cache(maxsize=16) -def load_json_fixture(fixture_path): +def load_json_fixture(fixture_path: str) -> Dict[str, Any]: """ Loads a fixture file, caching the most recent files it loaded. """ @@ -52,7 +60,9 @@ def load_json_fixture(fixture_path): return file_fixtures -def load_fixture(fixture_path, fixture_key, normalize_fn=identity): +def load_fixture(fixture_path: str, + fixture_key: str, + normalize_fn: Callable[..., Any]=identity) -> Dict[str, Any]: """ Loads a specific fixture from a fixture file, optionally passing it through a normalization function. @@ -64,7 +74,10 @@ def load_fixture(fixture_path, fixture_key, normalize_fn=identity): @require_pytest @curry -def filter_fixtures(all_fixtures, fixtures_base_dir, mark_fn=None, ignore_fn=None): +def filter_fixtures(all_fixtures: Iterable[Any], + fixtures_base_dir: str, + mark_fn: Callable[[str, str], bool]=None, + ignore_fn: Callable[[str, str], bool]=None) -> Any: """ Helper function for filtering test fixtures. diff --git a/eth/tools/fixtures/normalization.py b/eth/tools/fixtures/normalization.py index 89b78657ae..9448a2df97 100644 --- a/eth/tools/fixtures/normalization.py +++ b/eth/tools/fixtures/normalization.py @@ -1,9 +1,17 @@ import binascii -from collections.abc import ( +import functools + +from typing import ( + Any, + AnyStr, + Callable, + Dict, Iterable, + List, Mapping, + Sequence, + Tuple, ) -import functools from cytoolz import ( assoc_in, @@ -15,6 +23,10 @@ ) import cytoolz.curried +from eth_typing import ( + Address, +) + from eth_utils import ( apply_formatters_to_dict, big_endian_to_int, @@ -40,13 +52,23 @@ deep_merge, is_cleanly_mergable, ) +from eth.tools._utils.normalization import ( + normalize_transaction_group +) + +from eth.typing import ( + AccountState, + GeneralState, + Normalizer, + TransactionDict, +) # # Primitives # @functools.lru_cache(maxsize=1024) -def normalize_int(value): +def normalize_int(value: Any) -> int: """ Robust to integer conversion, handling hex values, string representations, and special cases like `0x`. @@ -66,7 +88,7 @@ def normalize_int(value): raise TypeError("Unsupported type: Got `{0}`".format(type(value))) -def normalize_bytes(value): +def normalize_bytes(value: Any) -> bytes: if is_bytes(value): return value elif is_text(value) and is_hex(value): @@ -76,7 +98,7 @@ def normalize_bytes(value): @functools.lru_cache(maxsize=1024) -def to_int(value): +def to_int(value: str) -> int: """ Robust to integer conversion, handling hex values, string representations, and special cases like `0x`. @@ -91,7 +113,7 @@ def to_int(value): @functools.lru_cache(maxsize=128) -def normalize_to_address(value): +def normalize_to_address(value: AnyStr) -> Address: if value: return to_canonical_address(value) else: @@ -104,21 +126,24 @@ def normalize_to_address(value): # # Containers # -def dict_normalizer(formatters, required=None, optional=None): +def dict_normalizer(formatters: Dict[Any, Callable[..., Any]], + required: Iterable[Any]=None, + optional: Iterable[Any]=None) -> Normalizer: + all_keys = set(formatters.keys()) if required is None and optional is None: - required = all_keys + required_set_form = all_keys + elif required is not None and optional is not None: + raise ValueError("Both required and optional keys specified") elif required is not None: - required = set(required) + required_set_form = set(required) elif optional is not None: - required = all_keys - set(optional) - else: - raise ValueError("Both required and optional keys specified") + required_set_form = all_keys - set(optional) - def normalizer(d): + def normalizer(d: Dict[Any, Any]) -> Dict[str, Any]: keys = set(d.keys()) - missing_keys = required - keys + missing_keys = required_set_form - keys superfluous_keys = keys - all_keys if missing_keys: raise KeyError("Missing required keys: {}".format(", ".join(missing_keys))) @@ -130,9 +155,9 @@ def normalizer(d): return normalizer -def dict_options_normalizer(normalizers): +def dict_options_normalizer(normalizers: Iterable[Normalizer]) -> Normalizer: - def normalize(d): + def normalize(d: Dict[Any, Any]) -> Dict[str, Any]: first_exception = None for normalizer in normalizers: try: @@ -151,7 +176,7 @@ def normalize(d): # # Composition # -def state_definition_to_dict(state_definition): +def state_definition_to_dict(state_definition: GeneralState) -> AccountState: """Convert a state definition to the canonical dict form. State can either be defined in the canonical form, or as a list of sub states that are then @@ -223,38 +248,6 @@ def state_definition_to_dict(state_definition): ) -normalize_main_transaction = dict_normalizer({ - "data": normalize_bytes, - "gasLimit": normalize_int, - "gasPrice": normalize_int, - "nonce": normalize_int, - "secretKey": normalize_bytes, - "to": normalize_to_address, - "value": normalize_int, -}) - - -normalize_transaction = dict_options_normalizer([ - normalize_main_transaction, -]) - - -normalize_main_transaction_group = dict_normalizer({ - "data": eth_utils.curried.apply_formatter_to_array(normalize_bytes), - "gasLimit": eth_utils.curried.apply_formatter_to_array(normalize_int), - "gasPrice": normalize_int, - "nonce": normalize_int, - "secretKey": normalize_bytes, - "to": normalize_to_address, - "value": eth_utils.curried.apply_formatter_to_array(normalize_int), -}) - - -normalize_transaction_group = dict_options_normalizer([ - normalize_main_transaction_group, -]) - - normalize_execution = dict_normalizer({ "address": to_canonical_address, "origin": to_canonical_address, @@ -303,10 +296,14 @@ def state_definition_to_dict(state_definition): # # Fixture Normalizers # -def normalize_unsigned_transaction(transaction, indexes): +def normalize_unsigned_transaction(transaction: TransactionDict, + indexes: Dict[str, Any]) -> TransactionDict: + normalized = normalize_transaction_group(transaction) return merge(normalized, { - transaction_key: normalized[transaction_key][indexes[index_key]] + # Dynamic key access not yet allowed with TypedDict + # https://github.com/python/mypy/issues/5359 + transaction_key: normalized[transaction_key][indexes[index_key]] # type: ignore for transaction_key, index_key in [ ("gasLimit", "gas"), ("value", "value"), @@ -316,7 +313,7 @@ def normalize_unsigned_transaction(transaction, indexes): }) -def normalize_account_state(account_state): +def normalize_account_state(account_state: AccountState) -> AccountState: return { to_canonical_address(address): { 'balance': to_int(state['balance']), @@ -331,14 +328,17 @@ def normalize_account_state(account_state): @to_dict -def normalize_post_state(post_state): +def normalize_post_state(post_state: Dict[str, Any]) -> Iterable[Tuple[str, bytes]]: yield 'hash', decode_hex(post_state['hash']) if 'logs' in post_state: yield 'logs', decode_hex(post_state['logs']) @curry -def normalize_statetest_fixture(fixture, fork, post_state_index): +def normalize_statetest_fixture(fixture: Dict[str, Any], + fork: str, + post_state_index: int) -> Dict[str, Any]: + post_state = fixture['post'][fork][post_state_index] normalized_fixture = { @@ -354,7 +354,7 @@ def normalize_statetest_fixture(fixture, fork, post_state_index): return normalized_fixture -def normalize_exec(exec_params): +def normalize_exec(exec_params: Dict[str, Any]) -> Dict[str, Any]: return { 'origin': to_canonical_address(exec_params['origin']), 'address': to_canonical_address(exec_params['address']), @@ -366,7 +366,7 @@ def normalize_exec(exec_params): } -def normalize_callcreates(callcreates): +def normalize_callcreates(callcreates: Sequence[Dict[str, Any]]) -> List[Dict[str, Any]]: return [ { 'data': decode_hex(created_call['data']), @@ -382,7 +382,7 @@ def normalize_callcreates(callcreates): @to_dict -def normalize_vmtest_fixture(fixture): +def normalize_vmtest_fixture(fixture: Dict[str, Any]) -> Iterable[Tuple[str, Any]]: yield 'env', normalize_environment(fixture['env']) yield 'exec', normalize_exec(fixture['exec']) yield 'pre', normalize_account_state(fixture['pre']) @@ -403,7 +403,7 @@ def normalize_vmtest_fixture(fixture): yield 'logs', decode_hex(fixture['logs']) -def normalize_signed_transaction(transaction): +def normalize_signed_transaction(transaction: Dict[str, Any]) -> Dict[str, Any]: return { 'data': robust_decode_hex(transaction['data']), 'gasLimit': to_int(transaction['gasLimit']), @@ -418,7 +418,7 @@ def normalize_signed_transaction(transaction): @curry -def normalize_transactiontest_fixture(fixture, fork): +def normalize_transactiontest_fixture(fixture: Dict[str, Any], fork: str) -> Dict[str, Any]: normalized_fixture = {} @@ -438,7 +438,7 @@ def normalize_transactiontest_fixture(fixture, fork): return normalized_fixture -def normalize_block_header(header): +def normalize_block_header(header: Dict[str, Any]) -> Dict[str, Any]: normalized_header = { 'bloom': big_endian_to_int(decode_hex(header['bloom'])), 'coinbase': to_canonical_address(header['coinbase']), @@ -466,7 +466,7 @@ def normalize_block_header(header): return normalized_header -def normalize_block(block): +def normalize_block(block: Dict[str, Any]) -> Dict[str, Any]: normalized_block = {} try: @@ -485,7 +485,7 @@ def normalize_block(block): return normalized_block -def normalize_blockchain_fixtures(fixture): +def normalize_blockchain_fixtures(fixture: Dict[str, Any]) -> Dict[str, Any]: normalized_fixture = { 'blocks': [normalize_block(block_fixture) for block_fixture in fixture['blocks']], 'genesisBlockHeader': normalize_block_header(fixture['genesisBlockHeader']), diff --git a/eth/typing.py b/eth/typing.py index c0aa6577f5..4ec64f866f 100644 --- a/eth/typing.py +++ b/eth/typing.py @@ -1,4 +1,6 @@ from typing import ( + Any, + Callable, Dict, Iterable, List, @@ -9,6 +11,7 @@ from eth_typing import ( Address, + HexStr, ) from mypy_extensions import ( TypedDict, @@ -32,4 +35,20 @@ List[Tuple[Address, Dict[str, Union[int, bytes, Dict[int, int]]]]] ] +TransactionDict = TypedDict('TransactionDict', + {'nonce': int, + 'gasLimit': int, + 'gasPrice': int, + 'to': Address, + 'value': int, + 'data': bytes, + 'secretKey': bytes, + }) + +Normalizer = Callable[[Dict[Any, Any]], Dict[str, Any]] + +TransactionNormalizer = Callable[[TransactionDict], TransactionDict] + VRS = NewType("VRS", Tuple[int, int, int]) + +IntConvertible = Union[int, bytes, HexStr, str] diff --git a/eth/utils/datatypes.py b/eth/utils/datatypes.py index 49b70dc5a6..91e04950f3 100644 --- a/eth/utils/datatypes.py +++ b/eth/utils/datatypes.py @@ -9,7 +9,7 @@ ) -from typing import Any, Dict, Tuple, Iterator, List +from typing import Any, Dict, Tuple, Type, TypeVar, Iterator, List def _is_local_prop(prop: str) -> bool: @@ -61,14 +61,17 @@ def _get_top_level_keys(overrides: Dict[str, Any]) -> Iterator[str]: # dynamic subclasses where generated through this method +T = TypeVar('T') + + class Configurable(object): """ Base class for simple inline subclassing """ @classmethod - def configure(cls, + def configure(cls: Type[T], __name__: str=None, - **overrides: Any) -> type: + **overrides: Any) -> Type[T]: if __name__ is None: __name__ = cls.__name__ diff --git a/tox.ini b/tox.ini index 91faa92e3a..0cd44700d4 100644 --- a/tox.ini +++ b/tox.ini @@ -106,7 +106,7 @@ commands= # TODO: Drop --ignore-missing-imports once we have type annotations for eth_utils, coincurve and cytoolz mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --no-strict-optional --check-untyped-defs --disallow-incomplete-defs -p eth mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --no-strict-optional --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs --disallow-any-generics -p eth.utils - mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --no-strict-optional --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs --disallow-any-generics -p eth.tools.builder + mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --no-strict-optional --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs --disallow-any-generics -p eth.tools [testenv:py36-lint] diff --git a/trinity/rpc/modules/evm.py b/trinity/rpc/modules/evm.py index 5a08160923..9aa503423c 100644 --- a/trinity/rpc/modules/evm.py +++ b/trinity/rpc/modules/evm.py @@ -6,7 +6,7 @@ ) from eth.chains.base import ( - Chain + BaseChain ) from eth.tools.fixtures import ( apply_fixture_block_to_chain, @@ -25,7 +25,7 @@ class EVM(RPCModule): @format_params(normalize_blockchain_fixtures) - async def resetToGenesisFixture(self, chain_info: Any) -> Chain: + async def resetToGenesisFixture(self, chain_info: Any) -> BaseChain: ''' This method is a special case. It returns a new chain object which is then replaced inside :class:`~trinity.rpc.main.RPCServer`