From f7c300c5a276afce6012cfcfcba0fc62b2c467a5 Mon Sep 17 00:00:00 2001 From: Bhargavasomu Date: Sat, 20 Oct 2018 18:51:53 +0530 Subject: [PATCH 1/6] Enable Type Hinting for eth.tools._utils submodule --- eth/tools/_utils/git.py | 2 +- eth/tools/_utils/hashing.py | 7 ++- eth/tools/_utils/mappings.py | 12 +++-- eth/tools/_utils/normalization.py | 81 +++++++++++++++++++++---------- eth/tools/_utils/vyper.py | 13 +++-- 5 files changed, 82 insertions(+), 33 deletions(-) diff --git a/eth/tools/_utils/git.py b/eth/tools/_utils/git.py index e8bda31d93..d45eb91340 100644 --- a/eth/tools/_utils/git.py +++ b/eth/tools/_utils/git.py @@ -3,6 +3,6 @@ from eth_utils import to_text -def get_version_from_git(): +def get_version_from_git() -> str: version = subprocess.check_output(["git", "describe"]).strip() return to_text(version) diff --git a/eth/tools/_utils/hashing.py b/eth/tools/_utils/hashing.py index 79d5296af1..456e2703c3 100644 --- a/eth/tools/_utils/hashing.py +++ b/eth/tools/_utils/hashing.py @@ -2,10 +2,15 @@ import rlp +from typing import ( + Iterable, + Tuple, +) + from eth.rlp.logs import Log -def hash_log_entries(log_entries): +def hash_log_entries(log_entries: Iterable[Tuple[bytes, bytes, bytes]]) -> bytes: """ Helper function for computing the RLP hash of the logs from transaction execution. diff --git a/eth/tools/_utils/mappings.py b/eth/tools/_utils/mappings.py index 236845c1e8..dc8f490dbe 100644 --- a/eth/tools/_utils/mappings.py +++ b/eth/tools/_utils/mappings.py @@ -1,21 +1,27 @@ from collections.abc import Mapping import itertools +from typing import ( + Any, + Dict, + Sequence, +) + from cytoolz import merge_with -def merge_if_dicts(values): +def merge_if_dicts(values: Sequence[Any]) -> Any: if all(isinstance(item, Mapping) for item in values): return merge_with(merge_if_dicts, *values) else: return values[-1] -def deep_merge(*dicts): +def deep_merge(*dicts: Dict[Any, Any]) -> Dict[Any, Any]: return merge_with(merge_if_dicts, *dicts) -def is_cleanly_mergable(*dicts): +def is_cleanly_mergable(*dicts: Dict[Any, Any]) -> bool: """Check that nothing will be overwritten when dictionaries are merged using `deep_merge`. Examples: diff --git a/eth/tools/_utils/normalization.py b/eth/tools/_utils/normalization.py index 5553d4a916..271e65dc44 100644 --- a/eth/tools/_utils/normalization.py +++ b/eth/tools/_utils/normalization.py @@ -5,6 +5,16 @@ ) import functools +from typing import ( + Any, + AnyStr, + Callable, + Dict, + List, + Sequence, + Tuple, +) + from cytoolz import ( assoc_in, compose, @@ -15,6 +25,10 @@ ) import cytoolz.curried +from eth_typing import ( + Address, +) + from eth_utils import ( apply_formatters_to_dict, big_endian_to_int, @@ -41,12 +55,17 @@ is_cleanly_mergable, ) +from eth.typing import ( + AccountState, + GeneralState, +) + # # Primitives # @functools.lru_cache(maxsize=1024) -def normalize_int(value): +def normalize_int(value: Any) -> int: """ Robust to integer conversion, handling hex values, string representations, and special cases like `0x`. @@ -66,7 +85,7 @@ def normalize_int(value): raise TypeError("Unsupported type: Got `{0}`".format(type(value))) -def normalize_bytes(value): +def normalize_bytes(value: Any) -> bytes: if is_bytes(value): return value elif is_text(value) and is_hex(value): @@ -78,7 +97,7 @@ def normalize_bytes(value): @functools.lru_cache(maxsize=1024) -def to_int(value): +def to_int(value: Any) -> int: """ Robust to integer conversion, handling hex values, string representations, and special cases like `0x`. @@ -93,7 +112,7 @@ def to_int(value): @functools.lru_cache(maxsize=128) -def normalize_to_address(value): +def normalize_to_address(value: AnyStr) -> Address: if value: return to_canonical_address(value) else: @@ -106,21 +125,28 @@ def normalize_to_address(value): # # Containers # -def dict_normalizer(formatters, required=None, optional=None): + +NormalizerType = Callable[[Dict[Any, Any]], Iterable[Tuple[Any, Any]]] + + +def dict_normalizer(formatters: Dict[Any, Any], + required: Iterable[Any]=None, + optional: Iterable[Any]=None) -> NormalizerType: + all_keys = set(formatters.keys()) if required is None and optional is None: - required = all_keys + required_set_form = all_keys elif required is not None: - required = set(required) + required_set_form = set(required) elif optional is not None: - required = all_keys - set(optional) + required_set_form = all_keys - set(optional) else: raise ValueError("Both required and optional keys specified") - def normalizer(d): + def normalizer(d: Dict[Any, Any]) -> Iterable[Tuple[Any, Any]]: keys = set(d.keys()) - missing_keys = required - keys + missing_keys = required_set_form - keys superfluous_keys = keys - all_keys if missing_keys: raise KeyError("Missing required keys: {}".format(", ".join(missing_keys))) @@ -132,9 +158,9 @@ def normalizer(d): return normalizer -def dict_options_normalizer(normalizers): +def dict_options_normalizer(normalizers: Iterable[Callable[..., Any]]) -> Callable[..., Any]: - def normalize(d): + def normalize(d: Dict[Any, Any]) -> Callable[..., Any]: first_exception = None for normalizer in normalizers: try: @@ -153,7 +179,7 @@ def normalize(d): # # Composition # -def state_definition_to_dict(state_definition): +def state_definition_to_dict(state_definition: GeneralState) -> AccountState: """Convert a state definition to the canonical dict form. State can either be defined in the canonical form, or as a list of sub states that are then @@ -305,7 +331,9 @@ def state_definition_to_dict(state_definition): # # Fixture Normalizers # -def normalize_unsigned_transaction(transaction, indexes): +def normalize_unsigned_transaction(transaction: Dict[str, Any], + indexes: Dict[str, Any]) -> Dict[str, Any]: + normalized = normalize_transaction_group(transaction) return merge(normalized, { transaction_key: normalized[transaction_key][indexes[index_key]] @@ -318,7 +346,7 @@ def normalize_unsigned_transaction(transaction, indexes): }) -def normalize_account_state(account_state): +def normalize_account_state(account_state: AccountState) -> AccountState: return { to_canonical_address(address): { 'balance': to_int(state['balance']), @@ -333,14 +361,17 @@ def normalize_account_state(account_state): @to_dict -def normalize_post_state(post_state): +def normalize_post_state(post_state: Dict[str, Any]) -> Iterable[Tuple[str, bytes]]: yield 'hash', decode_hex(post_state['hash']) if 'logs' in post_state: yield 'logs', decode_hex(post_state['logs']) @curry -def normalize_statetest_fixture(fixture, fork, post_state_index): +def normalize_statetest_fixture(fixture: Dict[str, Any], + fork: str, + post_state_index: int) -> Dict[str, Any]: + post_state = fixture['post'][fork][post_state_index] normalized_fixture = { @@ -356,7 +387,7 @@ def normalize_statetest_fixture(fixture, fork, post_state_index): return normalized_fixture -def normalize_exec(exec_params): +def normalize_exec(exec_params: Dict[str, Any]) -> Dict[str, Any]: return { 'origin': to_canonical_address(exec_params['origin']), 'address': to_canonical_address(exec_params['address']), @@ -368,7 +399,7 @@ def normalize_exec(exec_params): } -def normalize_callcreates(callcreates): +def normalize_callcreates(callcreates: Sequence[Dict[str, Any]]) -> List[Dict[str, Any]]: return [ { 'data': decode_hex(created_call['data']), @@ -384,7 +415,7 @@ def normalize_callcreates(callcreates): @to_dict -def normalize_vmtest_fixture(fixture): +def normalize_vmtest_fixture(fixture: Dict[str, Any]) -> Iterable[Tuple[str, Any]]: yield 'env', normalize_environment(fixture['env']) yield 'exec', normalize_exec(fixture['exec']) yield 'pre', normalize_account_state(fixture['pre']) @@ -405,7 +436,7 @@ def normalize_vmtest_fixture(fixture): yield 'logs', decode_hex(fixture['logs']) -def normalize_signed_transaction(transaction): +def normalize_signed_transaction(transaction: Dict[str, Any]) -> Dict[str, Any]: return { 'data': robust_decode_hex(transaction['data']), 'gasLimit': to_int(transaction['gasLimit']), @@ -420,7 +451,7 @@ def normalize_signed_transaction(transaction): @curry -def normalize_transactiontest_fixture(fixture, fork): +def normalize_transactiontest_fixture(fixture: Dict[str, Any], fork: str) -> Dict[str, Any]: normalized_fixture = {} @@ -440,7 +471,7 @@ def normalize_transactiontest_fixture(fixture, fork): return normalized_fixture -def normalize_block_header(header): +def normalize_block_header(header: Dict[str, Any]) -> Dict[str, Any]: normalized_header = { 'bloom': big_endian_to_int(decode_hex(header['bloom'])), 'coinbase': to_canonical_address(header['coinbase']), @@ -468,7 +499,7 @@ def normalize_block_header(header): return normalized_header -def normalize_block(block): +def normalize_block(block: Dict[str, Any]) -> Dict[str, Any]: normalized_block = {} try: @@ -487,7 +518,7 @@ def normalize_block(block): return normalized_block -def normalize_blockchain_fixtures(fixture): +def normalize_blockchain_fixtures(fixture: Dict[str, Any]) -> Dict[str, Any]: normalized_fixture = { 'blocks': [normalize_block(block_fixture) for block_fixture in fixture['blocks']], 'genesisBlockHeader': normalize_block_header(fixture['genesisBlockHeader']), diff --git a/eth/tools/_utils/vyper.py b/eth/tools/_utils/vyper.py index 1a8df17169..48f98172fc 100644 --- a/eth/tools/_utils/vyper.py +++ b/eth/tools/_utils/vyper.py @@ -1,5 +1,12 @@ import functools +from typing import ( + Any, + Callable, + Dict, + Tuple, +) + try: from vyper.compile_lll import ( compile_to_assembly, @@ -12,9 +19,9 @@ vyper_available = True -def require_vyper(fn): +def require_vyper(fn: Callable[..., Any]) -> Callable[..., Any]: @functools.wraps(fn) - def inner(*args, **kwargs): + def inner(*args: Any, **kwargs: Any) -> Any: if vyper_available: return fn(*args, **kwargs) else: @@ -23,7 +30,7 @@ def inner(*args, **kwargs): @require_vyper -def compile_vyper_lll(vyper_code): +def compile_vyper_lll(vyper_code: Any) -> Tuple[bytes, Dict[str, Any]]: lll_node = LLLnode.from_list(vyper_code) assembly = compile_to_assembly(lll_node) code = assembly_to_evm(assembly) From b7e60a6a461ada57903bee851191699b0bc28dd5 Mon Sep 17 00:00:00 2001 From: Bhargavasomu Date: Sun, 21 Oct 2018 18:30:52 +0530 Subject: [PATCH 2/6] Enable Type Hinting for eth.tools completely --- eth/tools/_utils/hashing.py | 6 +- eth/tools/_utils/mappings.py | 2 +- eth/tools/_utils/normalization.py | 50 ++++++------- eth/tools/_utils/vyper.py | 3 +- eth/tools/builder/chain/builders.py | 15 ++-- eth/tools/fixtures/_utils.py | 12 +++- eth/tools/fixtures/fillers/_utils.py | 38 +++++++--- eth/tools/fixtures/fillers/common.py | 32 ++++++--- eth/tools/fixtures/fillers/main.py | 11 ++- eth/tools/fixtures/fillers/state.py | 4 +- eth/tools/fixtures/fillers/vm.py | 24 ++++--- eth/tools/fixtures/generation.py | 18 +++-- eth/tools/fixtures/helpers.py | 44 +++++++++--- eth/tools/fixtures/loading.py | 23 ++++-- eth/tools/fixtures/normalization.py | 103 +++++++++++++++++---------- eth/typing.py | 16 +++++ eth/utils/datatypes.py | 9 ++- tox.ini | 2 +- trinity/rpc/modules/evm.py | 4 +- 19 files changed, 285 insertions(+), 131 deletions(-) diff --git a/eth/tools/_utils/hashing.py b/eth/tools/_utils/hashing.py index 456e2703c3..f5172ccd0d 100644 --- a/eth/tools/_utils/hashing.py +++ b/eth/tools/_utils/hashing.py @@ -7,10 +7,14 @@ Tuple, ) +from eth_typing import ( + Hash32, +) + from eth.rlp.logs import Log -def hash_log_entries(log_entries: Iterable[Tuple[bytes, bytes, bytes]]) -> bytes: +def hash_log_entries(log_entries: Iterable[Tuple[bytes, bytes, bytes]]) -> Hash32: """ Helper function for computing the RLP hash of the logs from transaction execution. diff --git a/eth/tools/_utils/mappings.py b/eth/tools/_utils/mappings.py index dc8f490dbe..8732ab5726 100644 --- a/eth/tools/_utils/mappings.py +++ b/eth/tools/_utils/mappings.py @@ -10,7 +10,7 @@ from cytoolz import merge_with -def merge_if_dicts(values: Sequence[Any]) -> Any: +def merge_if_dicts(values: Sequence[Dict[Any, Any]]) -> Any: if all(isinstance(item, Mapping) for item in values): return merge_with(merge_if_dicts, *values) else: diff --git a/eth/tools/_utils/normalization.py b/eth/tools/_utils/normalization.py index 271e65dc44..51a9f82030 100644 --- a/eth/tools/_utils/normalization.py +++ b/eth/tools/_utils/normalization.py @@ -1,8 +1,4 @@ import binascii -from collections.abc import ( - Iterable, - Mapping, -) import functools from typing import ( @@ -10,7 +6,9 @@ AnyStr, Callable, Dict, + Iterable, List, + Mapping, Sequence, Tuple, ) @@ -58,6 +56,8 @@ from eth.typing import ( AccountState, GeneralState, + NormalizerType, + TransactionDict, ) @@ -97,7 +97,7 @@ def normalize_bytes(value: Any) -> bytes: @functools.lru_cache(maxsize=1024) -def to_int(value: Any) -> int: +def to_int(value: str) -> int: """ Robust to integer conversion, handling hex values, string representations, and special cases like `0x`. @@ -125,11 +125,7 @@ def normalize_to_address(value: AnyStr) -> Address: # # Containers # - -NormalizerType = Callable[[Dict[Any, Any]], Iterable[Tuple[Any, Any]]] - - -def dict_normalizer(formatters: Dict[Any, Any], +def dict_normalizer(formatters: Dict[Any, Callable[..., Any]], required: Iterable[Any]=None, optional: Iterable[Any]=None) -> NormalizerType: @@ -137,14 +133,14 @@ def dict_normalizer(formatters: Dict[Any, Any], if required is None and optional is None: required_set_form = all_keys + elif required is not None and optional is not None: + raise ValueError("Both required and optional keys specified") elif required is not None: required_set_form = set(required) elif optional is not None: required_set_form = all_keys - set(optional) - else: - raise ValueError("Both required and optional keys specified") - def normalizer(d: Dict[Any, Any]) -> Iterable[Tuple[Any, Any]]: + def normalizer(d: Dict[Any, Any]) -> Dict[str, Any]: keys = set(d.keys()) missing_keys = required_set_form - keys superfluous_keys = keys - all_keys @@ -158,9 +154,9 @@ def normalizer(d: Dict[Any, Any]) -> Iterable[Tuple[Any, Any]]: return normalizer -def dict_options_normalizer(normalizers: Iterable[Callable[..., Any]]) -> Callable[..., Any]: +def dict_options_normalizer(normalizers: Iterable[NormalizerType]) -> NormalizerType: - def normalize(d: Dict[Any, Any]) -> Callable[..., Any]: + def normalize(d: Dict[Any, Any]) -> Dict[str, Any]: first_exception = None for normalizer in normalizers: try: @@ -251,7 +247,7 @@ def state_definition_to_dict(state_definition: GeneralState) -> AccountState: ) -normalize_main_transaction = dict_normalizer({ +normalize_main_transaction = dict_normalizer({ # type: ignore # Overwrite type hint not yet supported # noqa: 501 "data": normalize_bytes, "gasLimit": normalize_int, "gasPrice": normalize_int, @@ -259,15 +255,15 @@ def state_definition_to_dict(state_definition: GeneralState) -> AccountState: "secretKey": normalize_bytes, "to": normalize_to_address, "value": normalize_int, -}) +}) # type: TransactionNormalizer -normalize_transaction = dict_options_normalizer([ +normalize_transaction = dict_options_normalizer([ # type: ignore # Overwrite type hint not yet supported # noqa: 501 normalize_main_transaction, -]) +]) # type: TransactionNormalizer -normalize_main_transaction_group = dict_normalizer({ +normalize_main_transaction_group = dict_normalizer({ # type: ignore # Overwrite type hint not yet supported # noqa: 501 "data": eth_utils.curried.apply_formatter_to_array(normalize_bytes), "gasLimit": eth_utils.curried.apply_formatter_to_array(normalize_int), "gasPrice": normalize_int, @@ -275,12 +271,12 @@ def state_definition_to_dict(state_definition: GeneralState) -> AccountState: "secretKey": normalize_bytes, "to": normalize_to_address, "value": eth_utils.curried.apply_formatter_to_array(normalize_int), -}) +}) # type: TransactionNormalizer -normalize_transaction_group = dict_options_normalizer([ +normalize_transaction_group = dict_options_normalizer([ # type: ignore # Overwrite type hint not yet supported # noqa: 501 normalize_main_transaction_group, -]) +]) # type: TransactionNormalizer normalize_execution = dict_normalizer({ @@ -331,12 +327,12 @@ def state_definition_to_dict(state_definition: GeneralState) -> AccountState: # # Fixture Normalizers # -def normalize_unsigned_transaction(transaction: Dict[str, Any], - indexes: Dict[str, Any]) -> Dict[str, Any]: +def normalize_unsigned_transaction(transaction: TransactionDict, + indexes: Dict[str, Any]) -> TransactionDict: - normalized = normalize_transaction_group(transaction) + normalized = normalize_transaction_group(transaction) # type: TransactionDict return merge(normalized, { - transaction_key: normalized[transaction_key][indexes[index_key]] + transaction_key: normalized[transaction_key][indexes[index_key]] # type: ignore # https://github.com/python/mypy/issues/5359 # noqa: 501 for transaction_key, index_key in [ ("gasLimit", "gas"), ("value", "value"), diff --git a/eth/tools/_utils/vyper.py b/eth/tools/_utils/vyper.py index 48f98172fc..b1717e32fd 100644 --- a/eth/tools/_utils/vyper.py +++ b/eth/tools/_utils/vyper.py @@ -4,6 +4,7 @@ Any, Callable, Dict, + List, Tuple, ) @@ -30,7 +31,7 @@ def inner(*args: Any, **kwargs: Any) -> Any: @require_vyper -def compile_vyper_lll(vyper_code: Any) -> Tuple[bytes, Dict[str, Any]]: +def compile_vyper_lll(vyper_code: List[Any]) -> Tuple[bytes, Dict[str, Any]]: lll_node = LLLnode.from_list(vyper_code) assembly = compile_to_assembly(lll_node) code = assembly_to_evm(assembly) diff --git a/eth/tools/builder/chain/builders.py b/eth/tools/builder/chain/builders.py index 9d7b4f6098..93b37faede 100644 --- a/eth/tools/builder/chain/builders.py +++ b/eth/tools/builder/chain/builders.py @@ -92,7 +92,7 @@ def build(obj: Any, *applicators: Callable[..., Any]) -> Any: # Constructors (creation of chain classes) # @curry -def name(class_name: str, chain_class: BaseChain) -> type: +def name(class_name: str, chain_class: Type[BaseChain]) -> Type[BaseChain]: """ Assign the given name to the chain class. """ @@ -100,7 +100,7 @@ def name(class_name: str, chain_class: BaseChain) -> type: @curry -def chain_id(chain_id: int, chain_class: BaseChain) -> type: +def chain_id(chain_id: int, chain_class: Type[BaseChain]) -> Type[BaseChain]: """ Set the ``chain_id`` for the chain class. """ @@ -108,7 +108,7 @@ def chain_id(chain_id: int, chain_class: BaseChain) -> type: @curry -def fork_at(vm_class: Type[BaseVM], at_block: int, chain_class: BaseChain) -> type: +def fork_at(vm_class: Type[BaseVM], at_block: int, chain_class: Type[BaseChain]) -> Type[BaseChain]: """ Adds the ``vm_class`` to the chain's ``vm_configuration``. @@ -168,7 +168,7 @@ def _set_vm_dao_support_false(vm_configuration: VMConfiguration) -> VMConfigurat @curry -def disable_dao_fork(chain_class: BaseChain) -> type: +def disable_dao_fork(chain_class: Type[BaseChain]) -> Type[BaseChain]: """ Set the ``support_dao_fork`` flag to ``False`` on the :class:`~eth.vm.forks.homestead.HomesteadVM`. Requires that presence of @@ -199,7 +199,8 @@ def _set_vm_dao_fork_block_number(dao_fork_block_number: BlockNumber, @curry -def dao_fork_at(dao_fork_block_number: BlockNumber, chain_class: BaseChain) -> type: +def dao_fork_at(dao_fork_block_number: BlockNumber, + chain_class: Type[BaseChain]) -> Type[BaseChain]: """ Set the block number on which the DAO fork will happen. Requires that a version of the :class:`~eth.vm.forks.homestead.HomesteadVM` is present in @@ -263,7 +264,7 @@ def _mix_in_pow_mining(vm_configuration: VMConfiguration) -> VMConfiguration: @curry -def enable_pow_mining(chain_class: BaseChain) -> type: +def enable_pow_mining(chain_class: Type[BaseChain]) -> Type[BaseChain]: """ Inject on demand generation of the proof of work mining seal on newly mined blocks into each of the chain's vms. @@ -299,7 +300,7 @@ def _mix_in_disable_seal_validation(vm_configuration: VMConfiguration) -> VMConf @curry -def disable_pow_check(chain_class: Type[BaseChain]) -> type: +def disable_pow_check(chain_class: Type[BaseChain]) -> Type[BaseChain]: """ Disable the proof of work validation check for each of the chain's vms. This allows for block mining without generation of the proof of work seal. diff --git a/eth/tools/fixtures/_utils.py b/eth/tools/fixtures/_utils.py index a0ec8bc52b..fe9e85e107 100644 --- a/eth/tools/fixtures/_utils.py +++ b/eth/tools/fixtures/_utils.py @@ -2,20 +2,26 @@ import functools import os +from typing import ( + Any, + Callable, + Iterable, +) + from eth_utils import to_tuple @to_tuple -def recursive_find_files(base_dir, pattern): +def recursive_find_files(base_dir: str, pattern: str) -> Iterable[str]: for dirpath, _, filenames in os.walk(base_dir): for filename in filenames: if fnmatch.fnmatch(filename, pattern): yield os.path.join(dirpath, filename) -def require_pytest(fn): +def require_pytest(fn: Callable[..., Any]) -> Callable[..., Any]: @functools.wraps(fn) - def inner(*args, **kwargs): + def inner(*args: Any, **kwargs: Any) -> Callable[..., Any]: try: import pytest # noqa: F401 except ImportError: diff --git a/eth/tools/fixtures/fillers/_utils.py b/eth/tools/fixtures/fillers/_utils.py index 25eee5dfbd..018fa9c885 100644 --- a/eth/tools/fixtures/fillers/_utils.py +++ b/eth/tools/fixtures/fillers/_utils.py @@ -1,11 +1,29 @@ import copy import random +from typing import ( + Any, + Dict, + List, + Tuple, + Type, +) + +from eth_typing import ( + Address, +) + from eth_utils import ( int_to_big_endian, ) from eth.db.backends.memory import MemoryDB +from eth.db.account import BaseAccountDB + +from eth.typing import ( + AccountState, + TransactionDict, +) from eth.utils.db import ( apply_state_dict, @@ -17,13 +35,15 @@ from eth_keys import keys -def wrap_in_list(item): +def wrap_in_list(item: Any) -> List[Any]: return [item] -def add_transaction_to_group(group, transaction): +def add_transaction_to_group(group: Dict[str, Any], + transaction: TransactionDict) -> Tuple[Dict[str, Any], Dict[str, int]]: + for key in ["gasPrice", "nonce", "secretKey", "to"]: - if key in transaction and transaction[key] != group[key]: + if key in transaction and transaction[key] != group[key]: # type: ignore # https://github.com/python/mypy/issues/5359 # noqa: 501 raise ValueError("Can't add transaction as it differs in {}".format(key)) new_group = copy.deepcopy(group) @@ -35,26 +55,26 @@ def add_transaction_to_group(group, transaction): raise ValueError("Can't add transaction as {} is ambiguous".format(key)) index = 0 else: - if transaction[key] not in new_group[key]: - new_group[key].append(transaction[key]) - index = new_group[key].index(transaction[key]) + if transaction[key] not in new_group[key]: # type: ignore # https://github.com/python/mypy/issues/5359 # noqa: 501 + new_group[key].append(transaction[key]) # type: ignore # https://github.com/python/mypy/issues/5359 # noqa: 501 + index = new_group[key].index(transaction[key]) # type: ignore # https://github.com/python/mypy/issues/5359 # noqa: 501 indexes[index_key] = index else: assert key not in transaction return new_group, indexes -def calc_state_root(state, account_db_class): +def calc_state_root(state: AccountState, account_db_class: Type[BaseAccountDB]) -> bytes: account_db = account_db_class(MemoryDB()) apply_state_dict(account_db, state) return account_db.state_root -def generate_random_keypair(): +def generate_random_keypair() -> Tuple[bytes, Address]: key_object = keys.PrivateKey(pad32(int_to_big_endian(random.getrandbits(8 * 32)))) return key_object.to_bytes(), key_object.public_key.to_canonical_address() -def generate_random_address(): +def generate_random_address() -> Address: _, address = generate_random_keypair() return address diff --git a/eth/tools/fixtures/fillers/common.py b/eth/tools/fixtures/fillers/common.py index 9c3f7d996b..0d2dc33493 100644 --- a/eth/tools/fixtures/fillers/common.py +++ b/eth/tools/fixtures/fillers/common.py @@ -7,8 +7,10 @@ ) from typing import ( # noqa: F401 Any, + Callable, Dict, List, + Sequence, ) from cytoolz import ( assoc, @@ -39,6 +41,11 @@ compile_vyper_lll, ) +from eth.typing import ( + GeneralState, + TransactionDict, +) + from ._utils import ( add_transaction_to_group, wrap_in_list, @@ -69,10 +76,10 @@ "secretKey": decode_hex("0x45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8"), "to": to_canonical_address("0x0f572e5295c57f15886f9b263e2f6d2d6c7b5ec6"), "value": 0 -} +} # type: TransactionDict -def get_default_transaction(networks): +def get_default_transaction(networks: Any) -> TransactionDict: return DEFAULT_MAIN_TRANSACTION @@ -96,7 +103,7 @@ def get_default_transaction(networks): # Filler Generation # -def setup_filler(name, environment=None): +def setup_filler(name: str, environment: Dict[Any, Any]=None) -> Dict[str, Dict[str, Any]]: environment = normalize_environment(environment or {}) return {name: { "env": environment, @@ -104,7 +111,7 @@ def setup_filler(name, environment=None): }} -def setup_main_filler(name, environment=None): +def setup_main_filler(name: str, environment: Dict[Any, Any]=None) -> Dict[str, Dict[str, Any]]: """ Kick off the filler generation process by creating the general filler scaffold with a test name and general information about the testing environment. @@ -131,7 +138,7 @@ def setup_main_filler(name, environment=None): return setup_filler(name, merge(DEFAULT_MAIN_ENVIRONMENT, environment or {})) -def pre_state(*raw_state, filler): +def pre_state(*raw_state: GeneralState, filler: Dict[str, Any]) -> None: """ Specify the state prior to the test execution. Multiple invocations don't override the state but extend it instead. @@ -162,7 +169,7 @@ def pre_state(*raw_state, filler): (address, {"balance", }) """ @wraps(pre_state) - def _pre_state(filler): + def _pre_state(filler: Dict[str, Any]) -> Dict[str, Any]: test_name = get_test_name(filler) old_pre_state = filler[test_name].get("pre_state", {}) @@ -178,7 +185,11 @@ def _pre_state(filler): return assoc_in(filler, [test_name, "pre"], new_pre_state) -def _expect(post_state, networks, transaction, filler): +def _expect(post_state: Dict[str, Any], + networks: Any, + transaction: TransactionDict, + filler: Dict[str, Any]) -> Dict[str, Any]: + test_name = get_test_name(filler) test = filler[test_name] test_update = {test_name: {}} # type: Dict[str, Dict[Any, Any]] @@ -231,7 +242,10 @@ def _expect(post_state, networks, transaction, filler): return deep_merge(filler, test_update) -def expect(post_state=None, networks=None, transaction=None): +def expect(post_state: Dict[str, Any]=None, + networks: Any=None, + transaction: TransactionDict=None) -> Callable[..., Dict[str, Any]]: + """ Specify the expected result for the test. @@ -276,7 +290,7 @@ def expect(post_state=None, networks=None, transaction=None): @curry -def execution(execution, filler): +def execution(execution: Dict[str, Any], filler: Dict[str, Any]) -> Dict[str, Any]: """ For VM tests, specify the code that is being run as well as the current state of the EVM. State tests don't support this object. The parameter is a dictionary specifying some diff --git a/eth/tools/fixtures/fillers/main.py b/eth/tools/fixtures/fillers/main.py index 9f93a738e6..b006c9be69 100644 --- a/eth/tools/fixtures/fillers/main.py +++ b/eth/tools/fixtures/fillers/main.py @@ -1,3 +1,8 @@ +from typing import ( + Any, + Dict, +) + from cytoolz import ( assoc_in, merge, @@ -22,7 +27,11 @@ # # Primary test filler # -def fill_test(filler, info=None, apply_formatter=True, **kwargs): +def fill_test(filler: Dict[str, Any], + info: Dict[str, Any]=None, + apply_formatter: bool=True, + **kwargs: Any) -> Dict[str, Any]: + test_name = get_test_name(filler) test = filler[test_name] diff --git a/eth/tools/fixtures/fillers/state.py b/eth/tools/fixtures/fillers/state.py index e621b669c0..2b5f8f7235 100644 --- a/eth/tools/fixtures/fillers/state.py +++ b/eth/tools/fixtures/fillers/state.py @@ -1,7 +1,9 @@ from collections import defaultdict from typing import ( # noqa: F401 + Any, Dict, List, + Sequence, ) from eth_utils import encode_hex @@ -45,7 +47,7 @@ assert all(network in ACCOUNT_STATE_DB_CLASSES for network in ALL_NETWORKS) -def fill_state_test(filler): +def fill_state_test(filler: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: """ Filler function for filling state tests. """ diff --git a/eth/tools/fixtures/fillers/vm.py b/eth/tools/fixtures/fillers/vm.py index 513fb08348..8dd8fcc11f 100644 --- a/eth/tools/fixtures/fillers/vm.py +++ b/eth/tools/fixtures/fillers/vm.py @@ -1,3 +1,11 @@ +from typing import ( + Any, + Dict, + Iterable, + Tuple, + Union, +) + from eth.tools.fixtures.helpers import ( get_test_name, ) @@ -15,14 +23,14 @@ def fill_vm_test( - filler, - *, - call_creates=None, - gas_price=None, - gas_remaining=0, - logs=None, - output=b"" -): + filler: Dict[str, Any], + *, + call_creates: Any=None, + gas_price: Union[int, str]=None, + gas_remaining: Union[int, str]=0, + logs: Iterable[Tuple[bytes, bytes, bytes]]=None, + output: bytes=b"") -> Dict[str, Dict[str, Any]]: + test_name = get_test_name(filler) test = filler[test_name] diff --git a/eth/tools/fixtures/generation.py b/eth/tools/fixtures/generation.py index 5d92db5d21..a63b2b718f 100644 --- a/eth/tools/fixtures/generation.py +++ b/eth/tools/fixtures/generation.py @@ -1,6 +1,12 @@ import hashlib import os +from typing import ( + Any, + Callable, + Iterable, +) + from cytoolz import ( curry, identity, @@ -15,14 +21,14 @@ # # Pytest fixture generation # -def idfn(fixture_params): +def idfn(fixture_params: Iterable[Any]) -> str: """ Function for pytest to produce uniform names for fixtures. """ return ":".join((str(item) for item in fixture_params)) -def get_fixtures_file_hash(all_fixture_paths): +def get_fixtures_file_hash(all_fixture_paths: Iterable[str]) -> str: """ Returns the MD5 hash of the fixture files. Used for cache busting. """ @@ -34,10 +40,10 @@ def get_fixtures_file_hash(all_fixture_paths): @curry -def generate_fixture_tests(metafunc, - base_fixture_path, - filter_fn=identity, - preprocess_fn=identity): +def generate_fixture_tests(metafunc: Any, + base_fixture_path: str, + filter_fn: Callable[..., Any]=identity, + preprocess_fn: Callable[..., Any]=identity) -> None: """ Helper function for use with `pytest_generate_tests` which will use the pytest caching facilities to reduce the load time for fixture tests. diff --git a/eth/tools/fixtures/helpers.py b/eth/tools/fixtures/helpers.py index bf398d033d..ed7dc5f080 100644 --- a/eth/tools/fixtures/helpers.py +++ b/eth/tools/fixtures/helpers.py @@ -2,6 +2,15 @@ import rlp +from typing import ( + Any, + cast, + Dict, + Iterable, + Tuple, + Type, +) + from cytoolz import first from eth_utils import ( @@ -10,12 +19,27 @@ from eth import MainnetChain from eth.db.atomic import AtomicDB +from eth.rlp.blocks import ( + BaseBlock, +) +from eth.chains.base import ( + BaseChain, +) +from eth.db.account import ( + BaseAccountDB, +) from eth.tools.builder.chain import ( disable_pow_check, ) +from eth.typing import ( + AccountState, +) from eth.utils.state import ( diff_account_db, ) +from eth.vm.base import ( + BaseVM, +) from eth.vm.forks import ( ByzantiumVM, TangerineWhistleVM, @@ -28,7 +52,7 @@ # # State Setup # -def setup_account_db(desired_state, account_db): +def setup_account_db(desired_state: AccountState, account_db: BaseAccountDB) -> None: for account, account_data in desired_state.items(): for slot, value in account_data['storage'].items(): account_db.set_storage(account, slot, value) @@ -43,7 +67,7 @@ def setup_account_db(desired_state, account_db): account_db.persist() -def verify_account_db(expected_state, account_db): +def verify_account_db(expected_state: AccountState, account_db: BaseAccountDB) -> None: diff = diff_account_db(expected_state, account_db) if diff: error_messages = [] @@ -76,7 +100,7 @@ def verify_account_db(expected_state, account_db): ) -def chain_vm_configuration(fixture): +def chain_vm_configuration(fixture: Dict[str, Any]) -> Iterable[Tuple[int, Type[BaseVM]]]: network = fixture['network'] if network == 'Frontier': @@ -129,7 +153,7 @@ def chain_vm_configuration(fixture): raise ValueError("Network {0} does not match any known VM rules".format(network)) -def genesis_params_from_fixture(fixture): +def genesis_params_from_fixture(fixture: Dict[str, Any]) -> Dict[str, Any]: return { 'parent_hash': fixture['genesisBlockHeader']['parentHash'], 'uncles_hash': fixture['genesisBlockHeader']['uncleHash'], @@ -149,7 +173,8 @@ def genesis_params_from_fixture(fixture): } -def new_chain_from_fixture(fixture, chain_cls=MainnetChain): +def new_chain_from_fixture(fixture: Dict[str, Any], + chain_cls: Type[BaseChain]=MainnetChain) -> BaseChain: base_db = AtomicDB() vm_config = chain_vm_configuration(fixture) @@ -162,6 +187,8 @@ def new_chain_from_fixture(fixture, chain_cls=MainnetChain): if 'sealEngine' in fixture and fixture['sealEngine'] == 'NoProof': ChainFromFixture = disable_pow_check(ChainFromFixture) + cast(Type[BaseChain], ChainFromFixture) + return ChainFromFixture.from_genesis( base_db, genesis_params=genesis_params_from_fixture(fixture), @@ -169,7 +196,8 @@ def new_chain_from_fixture(fixture, chain_cls=MainnetChain): ) -def apply_fixture_block_to_chain(block_fixture, chain): +def apply_fixture_block_to_chain(block_fixture: Dict[str, Any], + chain: BaseChain) -> Tuple[BaseBlock, BaseBlock, BaseBlock]: ''' :return: (premined_block, mined_block, rlp_encoded_mined_block) ''' @@ -191,12 +219,12 @@ def apply_fixture_block_to_chain(block_fixture, chain): return (block, mined_block, rlp_encoded_mined_block) -def should_run_slow_tests(): +def should_run_slow_tests() -> bool: if os.environ.get('TRAVIS_EVENT_TYPE') == 'cron': return True return False -def get_test_name(filler): +def get_test_name(filler: Dict[str, Any]) -> str: assert len(filler) == 1 return first(filler) diff --git a/eth/tools/fixtures/loading.py b/eth/tools/fixtures/loading.py index 2d8a513549..02cd96419d 100644 --- a/eth/tools/fixtures/loading.py +++ b/eth/tools/fixtures/loading.py @@ -2,6 +2,14 @@ import json import os +from typing import ( + Any, + Callable, + Dict, + Iterable, + Tuple, +) + from cytoolz import ( curry, identity, @@ -18,13 +26,13 @@ # # Filesystem fixture loading. # -def find_fixture_files(fixtures_base_dir): +def find_fixture_files(fixtures_base_dir: str) -> Iterable[str]: all_fixture_paths = recursive_find_files(fixtures_base_dir, "*.json") return all_fixture_paths @to_tuple -def find_fixtures(fixtures_base_dir): +def find_fixtures(fixtures_base_dir: str) -> Iterable[Tuple[str, str]]: """ Finds all of the (fixture_path, fixture_key) pairs for a given path under the JSON test fixtures directory. @@ -43,7 +51,7 @@ def find_fixtures(fixtures_base_dir): # all fixtures from the same file are executed sequentially allowing us to keep # a small rolling cache of the loaded fixture files. @functools.lru_cache(maxsize=16) -def load_json_fixture(fixture_path): +def load_json_fixture(fixture_path: str) -> Dict[str, Any]: """ Loads a fixture file, caching the most recent files it loaded. """ @@ -52,7 +60,9 @@ def load_json_fixture(fixture_path): return file_fixtures -def load_fixture(fixture_path, fixture_key, normalize_fn=identity): +def load_fixture(fixture_path: str, + fixture_key: str, + normalize_fn: Callable[..., Any]=identity) -> Dict[str, Any]: """ Loads a specific fixture from a fixture file, optionally passing it through a normalization function. @@ -64,7 +74,10 @@ def load_fixture(fixture_path, fixture_key, normalize_fn=identity): @require_pytest @curry -def filter_fixtures(all_fixtures, fixtures_base_dir, mark_fn=None, ignore_fn=None): +def filter_fixtures(all_fixtures: Iterable[Any], + fixtures_base_dir: str, + mark_fn: Callable[[str, str], bool]=None, + ignore_fn: Callable[[str, str], bool]=None) -> Any: """ Helper function for filtering test fixtures. diff --git a/eth/tools/fixtures/normalization.py b/eth/tools/fixtures/normalization.py index 89b78657ae..4dc53273dc 100644 --- a/eth/tools/fixtures/normalization.py +++ b/eth/tools/fixtures/normalization.py @@ -1,9 +1,17 @@ import binascii -from collections.abc import ( +import functools + +from typing import ( + Any, + AnyStr, + Callable, + Dict, Iterable, + List, Mapping, + Sequence, + Tuple, ) -import functools from cytoolz import ( assoc_in, @@ -15,6 +23,10 @@ ) import cytoolz.curried +from eth_typing import ( + Address, +) + from eth_utils import ( apply_formatters_to_dict, big_endian_to_int, @@ -41,12 +53,19 @@ is_cleanly_mergable, ) +from eth.typing import ( + AccountState, + GeneralState, + NormalizerType, + TransactionDict, +) + # # Primitives # @functools.lru_cache(maxsize=1024) -def normalize_int(value): +def normalize_int(value: Any) -> int: """ Robust to integer conversion, handling hex values, string representations, and special cases like `0x`. @@ -66,7 +85,7 @@ def normalize_int(value): raise TypeError("Unsupported type: Got `{0}`".format(type(value))) -def normalize_bytes(value): +def normalize_bytes(value: Any) -> bytes: if is_bytes(value): return value elif is_text(value) and is_hex(value): @@ -76,7 +95,7 @@ def normalize_bytes(value): @functools.lru_cache(maxsize=1024) -def to_int(value): +def to_int(value: str) -> int: """ Robust to integer conversion, handling hex values, string representations, and special cases like `0x`. @@ -91,7 +110,7 @@ def to_int(value): @functools.lru_cache(maxsize=128) -def normalize_to_address(value): +def normalize_to_address(value: AnyStr) -> Address: if value: return to_canonical_address(value) else: @@ -104,21 +123,24 @@ def normalize_to_address(value): # # Containers # -def dict_normalizer(formatters, required=None, optional=None): +def dict_normalizer(formatters: Dict[Any, Callable[..., Any]], + required: Iterable[Any]=None, + optional: Iterable[Any]=None) -> NormalizerType: + all_keys = set(formatters.keys()) if required is None and optional is None: - required = all_keys + required_set_form = all_keys + elif required is not None and optional is not None: + raise ValueError("Both required and optional keys specified") elif required is not None: - required = set(required) + required_set_form = set(required) elif optional is not None: - required = all_keys - set(optional) - else: - raise ValueError("Both required and optional keys specified") + required_set_form = all_keys - set(optional) - def normalizer(d): + def normalizer(d: Dict[Any, Any]) -> Dict[str, Any]: keys = set(d.keys()) - missing_keys = required - keys + missing_keys = required_set_form - keys superfluous_keys = keys - all_keys if missing_keys: raise KeyError("Missing required keys: {}".format(", ".join(missing_keys))) @@ -130,9 +152,9 @@ def normalizer(d): return normalizer -def dict_options_normalizer(normalizers): +def dict_options_normalizer(normalizers: Iterable[NormalizerType]) -> NormalizerType: - def normalize(d): + def normalize(d: Dict[Any, Any]) -> Dict[str, Any]: first_exception = None for normalizer in normalizers: try: @@ -151,7 +173,7 @@ def normalize(d): # # Composition # -def state_definition_to_dict(state_definition): +def state_definition_to_dict(state_definition: GeneralState) -> AccountState: """Convert a state definition to the canonical dict form. State can either be defined in the canonical form, or as a list of sub states that are then @@ -223,7 +245,7 @@ def state_definition_to_dict(state_definition): ) -normalize_main_transaction = dict_normalizer({ +normalize_main_transaction = dict_normalizer({ # type: ignore # Overwrite type hint not yet supported by mypy # noqa: 501 "data": normalize_bytes, "gasLimit": normalize_int, "gasPrice": normalize_int, @@ -231,15 +253,15 @@ def state_definition_to_dict(state_definition): "secretKey": normalize_bytes, "to": normalize_to_address, "value": normalize_int, -}) +}) # type: TransactionNormalizer -normalize_transaction = dict_options_normalizer([ +normalize_transaction = dict_options_normalizer([ # type: ignore # Overwrite type hint not yet supported by mypy # noqa: 501 normalize_main_transaction, -]) +]) # type: TransactionNormalizer -normalize_main_transaction_group = dict_normalizer({ +normalize_main_transaction_group = dict_normalizer({ # type: ignore # Overwrite type hint not yet supported by mypy # noqa: 501 "data": eth_utils.curried.apply_formatter_to_array(normalize_bytes), "gasLimit": eth_utils.curried.apply_formatter_to_array(normalize_int), "gasPrice": normalize_int, @@ -247,12 +269,12 @@ def state_definition_to_dict(state_definition): "secretKey": normalize_bytes, "to": normalize_to_address, "value": eth_utils.curried.apply_formatter_to_array(normalize_int), -}) +}) # type: TransactionNormalizer -normalize_transaction_group = dict_options_normalizer([ +normalize_transaction_group = dict_options_normalizer([ # type: ignore # Overwrite type hint not yet supported by mypy # noqa: 501 normalize_main_transaction_group, -]) +]) # type: TransactionNormalizer normalize_execution = dict_normalizer({ @@ -303,10 +325,12 @@ def state_definition_to_dict(state_definition): # # Fixture Normalizers # -def normalize_unsigned_transaction(transaction, indexes): +def normalize_unsigned_transaction(transaction: TransactionDict, + indexes: Dict[str, Any]) -> TransactionDict: + normalized = normalize_transaction_group(transaction) return merge(normalized, { - transaction_key: normalized[transaction_key][indexes[index_key]] + transaction_key: normalized[transaction_key][indexes[index_key]] # https://github.com/python/mypy/issues/5359 # noqa: 501 for transaction_key, index_key in [ ("gasLimit", "gas"), ("value", "value"), @@ -316,7 +340,7 @@ def normalize_unsigned_transaction(transaction, indexes): }) -def normalize_account_state(account_state): +def normalize_account_state(account_state: AccountState) -> AccountState: return { to_canonical_address(address): { 'balance': to_int(state['balance']), @@ -331,14 +355,17 @@ def normalize_account_state(account_state): @to_dict -def normalize_post_state(post_state): +def normalize_post_state(post_state: Dict[str, Any]) -> Iterable[Tuple[str, bytes]]: yield 'hash', decode_hex(post_state['hash']) if 'logs' in post_state: yield 'logs', decode_hex(post_state['logs']) @curry -def normalize_statetest_fixture(fixture, fork, post_state_index): +def normalize_statetest_fixture(fixture: Dict[str, Any], + fork: str, + post_state_index: int) -> Dict[str, Any]: + post_state = fixture['post'][fork][post_state_index] normalized_fixture = { @@ -354,7 +381,7 @@ def normalize_statetest_fixture(fixture, fork, post_state_index): return normalized_fixture -def normalize_exec(exec_params): +def normalize_exec(exec_params: Dict[str, Any]) -> Dict[str, Any]: return { 'origin': to_canonical_address(exec_params['origin']), 'address': to_canonical_address(exec_params['address']), @@ -366,7 +393,7 @@ def normalize_exec(exec_params): } -def normalize_callcreates(callcreates): +def normalize_callcreates(callcreates: Sequence[Dict[str, Any]]) -> List[Dict[str, Any]]: return [ { 'data': decode_hex(created_call['data']), @@ -382,7 +409,7 @@ def normalize_callcreates(callcreates): @to_dict -def normalize_vmtest_fixture(fixture): +def normalize_vmtest_fixture(fixture: Dict[str, Any]) -> Iterable[Tuple[str, Any]]: yield 'env', normalize_environment(fixture['env']) yield 'exec', normalize_exec(fixture['exec']) yield 'pre', normalize_account_state(fixture['pre']) @@ -403,7 +430,7 @@ def normalize_vmtest_fixture(fixture): yield 'logs', decode_hex(fixture['logs']) -def normalize_signed_transaction(transaction): +def normalize_signed_transaction(transaction: Dict[str, Any]) -> Dict[str, Any]: return { 'data': robust_decode_hex(transaction['data']), 'gasLimit': to_int(transaction['gasLimit']), @@ -418,7 +445,7 @@ def normalize_signed_transaction(transaction): @curry -def normalize_transactiontest_fixture(fixture, fork): +def normalize_transactiontest_fixture(fixture: Dict[str, Any], fork: str) -> Dict[str, Any]: normalized_fixture = {} @@ -438,7 +465,7 @@ def normalize_transactiontest_fixture(fixture, fork): return normalized_fixture -def normalize_block_header(header): +def normalize_block_header(header: Dict[str, Any]) -> Dict[str, Any]: normalized_header = { 'bloom': big_endian_to_int(decode_hex(header['bloom'])), 'coinbase': to_canonical_address(header['coinbase']), @@ -466,7 +493,7 @@ def normalize_block_header(header): return normalized_header -def normalize_block(block): +def normalize_block(block: Dict[str, Any]) -> Dict[str, Any]: normalized_block = {} try: @@ -485,7 +512,7 @@ def normalize_block(block): return normalized_block -def normalize_blockchain_fixtures(fixture): +def normalize_blockchain_fixtures(fixture: Dict[str, Any]) -> Dict[str, Any]: normalized_fixture = { 'blocks': [normalize_block(block_fixture) for block_fixture in fixture['blocks']], 'genesisBlockHeader': normalize_block_header(fixture['genesisBlockHeader']), diff --git a/eth/typing.py b/eth/typing.py index c0aa6577f5..b17af8982a 100644 --- a/eth/typing.py +++ b/eth/typing.py @@ -1,4 +1,6 @@ from typing import ( + Any, + Callable, Dict, Iterable, List, @@ -32,4 +34,18 @@ List[Tuple[Address, Dict[str, Union[int, bytes, Dict[int, int]]]]] ] +TransactionDict = TypedDict('TransactionDict', + {'nonce': int, + 'gasLimit': int, + 'gasPrice': int, + 'to': Address, + 'value': int, + 'data': bytes, + 'secretKey': bytes, + }) + +NormalizerType = Callable[[Dict[Any, Any]], Dict[str, Any]] + +TransactionNormalizer = Callable[[TransactionDict], TransactionDict] + VRS = NewType("VRS", Tuple[int, int, int]) diff --git a/eth/utils/datatypes.py b/eth/utils/datatypes.py index 49b70dc5a6..91e04950f3 100644 --- a/eth/utils/datatypes.py +++ b/eth/utils/datatypes.py @@ -9,7 +9,7 @@ ) -from typing import Any, Dict, Tuple, Iterator, List +from typing import Any, Dict, Tuple, Type, TypeVar, Iterator, List def _is_local_prop(prop: str) -> bool: @@ -61,14 +61,17 @@ def _get_top_level_keys(overrides: Dict[str, Any]) -> Iterator[str]: # dynamic subclasses where generated through this method +T = TypeVar('T') + + class Configurable(object): """ Base class for simple inline subclassing """ @classmethod - def configure(cls, + def configure(cls: Type[T], __name__: str=None, - **overrides: Any) -> type: + **overrides: Any) -> Type[T]: if __name__ is None: __name__ = cls.__name__ diff --git a/tox.ini b/tox.ini index 91faa92e3a..0cd44700d4 100644 --- a/tox.ini +++ b/tox.ini @@ -106,7 +106,7 @@ commands= # TODO: Drop --ignore-missing-imports once we have type annotations for eth_utils, coincurve and cytoolz mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --no-strict-optional --check-untyped-defs --disallow-incomplete-defs -p eth mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --no-strict-optional --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs --disallow-any-generics -p eth.utils - mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --no-strict-optional --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs --disallow-any-generics -p eth.tools.builder + mypy --follow-imports=silent --warn-unused-ignores --ignore-missing-imports --no-strict-optional --check-untyped-defs --disallow-incomplete-defs --disallow-untyped-defs --disallow-any-generics -p eth.tools [testenv:py36-lint] diff --git a/trinity/rpc/modules/evm.py b/trinity/rpc/modules/evm.py index 5a08160923..9aa503423c 100644 --- a/trinity/rpc/modules/evm.py +++ b/trinity/rpc/modules/evm.py @@ -6,7 +6,7 @@ ) from eth.chains.base import ( - Chain + BaseChain ) from eth.tools.fixtures import ( apply_fixture_block_to_chain, @@ -25,7 +25,7 @@ class EVM(RPCModule): @format_params(normalize_blockchain_fixtures) - async def resetToGenesisFixture(self, chain_info: Any) -> Chain: + async def resetToGenesisFixture(self, chain_info: Any) -> BaseChain: ''' This method is a special case. It returns a new chain object which is then replaced inside :class:`~trinity.rpc.main.RPCServer` From 0f1ca1b599fce8a9dfbb6028571110c192f64244 Mon Sep 17 00:00:00 2001 From: Christoph Burgdorf Date: Thu, 25 Oct 2018 18:26:18 +0200 Subject: [PATCH 3/6] Cleanup Normalizer type hints --- eth/tools/_utils/normalization.py | 30 ++++++++++--------- eth/tools/fixtures/normalization.py | 45 ++++++----------------------- eth/typing.py | 2 +- 3 files changed, 27 insertions(+), 50 deletions(-) diff --git a/eth/tools/_utils/normalization.py b/eth/tools/_utils/normalization.py index 51a9f82030..3a8f80365f 100644 --- a/eth/tools/_utils/normalization.py +++ b/eth/tools/_utils/normalization.py @@ -5,6 +5,7 @@ Any, AnyStr, Callable, + cast, Dict, Iterable, List, @@ -56,8 +57,9 @@ from eth.typing import ( AccountState, GeneralState, - NormalizerType, + Normalizer, TransactionDict, + TransactionNormalizer, ) @@ -127,7 +129,7 @@ def normalize_to_address(value: AnyStr) -> Address: # def dict_normalizer(formatters: Dict[Any, Callable[..., Any]], required: Iterable[Any]=None, - optional: Iterable[Any]=None) -> NormalizerType: + optional: Iterable[Any]=None) -> Normalizer: all_keys = set(formatters.keys()) @@ -154,7 +156,7 @@ def normalizer(d: Dict[Any, Any]) -> Dict[str, Any]: return normalizer -def dict_options_normalizer(normalizers: Iterable[NormalizerType]) -> NormalizerType: +def dict_options_normalizer(normalizers: Iterable[Normalizer]) -> Normalizer: def normalize(d: Dict[Any, Any]) -> Dict[str, Any]: first_exception = None @@ -247,7 +249,7 @@ def state_definition_to_dict(state_definition: GeneralState) -> AccountState: ) -normalize_main_transaction = dict_normalizer({ # type: ignore # Overwrite type hint not yet supported # noqa: 501 +normalize_main_transaction = cast(Normalizer, dict_normalizer({ "data": normalize_bytes, "gasLimit": normalize_int, "gasPrice": normalize_int, @@ -255,15 +257,15 @@ def state_definition_to_dict(state_definition: GeneralState) -> AccountState: "secretKey": normalize_bytes, "to": normalize_to_address, "value": normalize_int, -}) # type: TransactionNormalizer +})) -normalize_transaction = dict_options_normalizer([ # type: ignore # Overwrite type hint not yet supported # noqa: 501 +normalize_transaction = cast(TransactionNormalizer, dict_options_normalizer([ normalize_main_transaction, -]) # type: TransactionNormalizer +])) -normalize_main_transaction_group = dict_normalizer({ # type: ignore # Overwrite type hint not yet supported # noqa: 501 +normalize_main_transaction_group = cast(Normalizer, dict_normalizer({ "data": eth_utils.curried.apply_formatter_to_array(normalize_bytes), "gasLimit": eth_utils.curried.apply_formatter_to_array(normalize_int), "gasPrice": normalize_int, @@ -271,12 +273,12 @@ def state_definition_to_dict(state_definition: GeneralState) -> AccountState: "secretKey": normalize_bytes, "to": normalize_to_address, "value": eth_utils.curried.apply_formatter_to_array(normalize_int), -}) # type: TransactionNormalizer +})) -normalize_transaction_group = dict_options_normalizer([ # type: ignore # Overwrite type hint not yet supported # noqa: 501 +normalize_transaction_group = cast(TransactionNormalizer, dict_options_normalizer([ normalize_main_transaction_group, -]) # type: TransactionNormalizer +])) normalize_execution = dict_normalizer({ @@ -330,9 +332,11 @@ def state_definition_to_dict(state_definition: GeneralState) -> AccountState: def normalize_unsigned_transaction(transaction: TransactionDict, indexes: Dict[str, Any]) -> TransactionDict: - normalized = normalize_transaction_group(transaction) # type: TransactionDict + normalized = normalize_transaction_group(transaction) return merge(normalized, { - transaction_key: normalized[transaction_key][indexes[index_key]] # type: ignore # https://github.com/python/mypy/issues/5359 # noqa: 501 + # Dynamic key access not yet allowed with TypedDict + # https://github.com/python/mypy/issues/5359 + transaction_key: normalized[transaction_key][indexes[index_key]] # type: ignore for transaction_key, index_key in [ ("gasLimit", "gas"), ("value", "value"), diff --git a/eth/tools/fixtures/normalization.py b/eth/tools/fixtures/normalization.py index 4dc53273dc..9448a2df97 100644 --- a/eth/tools/fixtures/normalization.py +++ b/eth/tools/fixtures/normalization.py @@ -52,11 +52,14 @@ deep_merge, is_cleanly_mergable, ) +from eth.tools._utils.normalization import ( + normalize_transaction_group +) from eth.typing import ( AccountState, GeneralState, - NormalizerType, + Normalizer, TransactionDict, ) @@ -125,7 +128,7 @@ def normalize_to_address(value: AnyStr) -> Address: # def dict_normalizer(formatters: Dict[Any, Callable[..., Any]], required: Iterable[Any]=None, - optional: Iterable[Any]=None) -> NormalizerType: + optional: Iterable[Any]=None) -> Normalizer: all_keys = set(formatters.keys()) @@ -152,7 +155,7 @@ def normalizer(d: Dict[Any, Any]) -> Dict[str, Any]: return normalizer -def dict_options_normalizer(normalizers: Iterable[NormalizerType]) -> NormalizerType: +def dict_options_normalizer(normalizers: Iterable[Normalizer]) -> Normalizer: def normalize(d: Dict[Any, Any]) -> Dict[str, Any]: first_exception = None @@ -245,38 +248,6 @@ def state_definition_to_dict(state_definition: GeneralState) -> AccountState: ) -normalize_main_transaction = dict_normalizer({ # type: ignore # Overwrite type hint not yet supported by mypy # noqa: 501 - "data": normalize_bytes, - "gasLimit": normalize_int, - "gasPrice": normalize_int, - "nonce": normalize_int, - "secretKey": normalize_bytes, - "to": normalize_to_address, - "value": normalize_int, -}) # type: TransactionNormalizer - - -normalize_transaction = dict_options_normalizer([ # type: ignore # Overwrite type hint not yet supported by mypy # noqa: 501 - normalize_main_transaction, -]) # type: TransactionNormalizer - - -normalize_main_transaction_group = dict_normalizer({ # type: ignore # Overwrite type hint not yet supported by mypy # noqa: 501 - "data": eth_utils.curried.apply_formatter_to_array(normalize_bytes), - "gasLimit": eth_utils.curried.apply_formatter_to_array(normalize_int), - "gasPrice": normalize_int, - "nonce": normalize_int, - "secretKey": normalize_bytes, - "to": normalize_to_address, - "value": eth_utils.curried.apply_formatter_to_array(normalize_int), -}) # type: TransactionNormalizer - - -normalize_transaction_group = dict_options_normalizer([ # type: ignore # Overwrite type hint not yet supported by mypy # noqa: 501 - normalize_main_transaction_group, -]) # type: TransactionNormalizer - - normalize_execution = dict_normalizer({ "address": to_canonical_address, "origin": to_canonical_address, @@ -330,7 +301,9 @@ def normalize_unsigned_transaction(transaction: TransactionDict, normalized = normalize_transaction_group(transaction) return merge(normalized, { - transaction_key: normalized[transaction_key][indexes[index_key]] # https://github.com/python/mypy/issues/5359 # noqa: 501 + # Dynamic key access not yet allowed with TypedDict + # https://github.com/python/mypy/issues/5359 + transaction_key: normalized[transaction_key][indexes[index_key]] # type: ignore for transaction_key, index_key in [ ("gasLimit", "gas"), ("value", "value"), diff --git a/eth/typing.py b/eth/typing.py index b17af8982a..d5a1246e66 100644 --- a/eth/typing.py +++ b/eth/typing.py @@ -44,7 +44,7 @@ 'secretKey': bytes, }) -NormalizerType = Callable[[Dict[Any, Any]], Dict[str, Any]] +Normalizer = Callable[[Dict[Any, Any]], Dict[str, Any]] TransactionNormalizer = Callable[[TransactionDict], TransactionDict] From 614b22156a8115e2ad348ac5abc0668a45b30db5 Mon Sep 17 00:00:00 2001 From: Christoph Burgdorf Date: Thu, 25 Oct 2018 18:44:03 +0200 Subject: [PATCH 4/6] Remove superfluous cast --- eth/tools/fixtures/helpers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/eth/tools/fixtures/helpers.py b/eth/tools/fixtures/helpers.py index ed7dc5f080..483d0b0607 100644 --- a/eth/tools/fixtures/helpers.py +++ b/eth/tools/fixtures/helpers.py @@ -187,8 +187,6 @@ def new_chain_from_fixture(fixture: Dict[str, Any], if 'sealEngine' in fixture and fixture['sealEngine'] == 'NoProof': ChainFromFixture = disable_pow_check(ChainFromFixture) - cast(Type[BaseChain], ChainFromFixture) - return ChainFromFixture.from_genesis( base_db, genesis_params=genesis_params_from_fixture(fixture), From 060f793023d43c0c0bf967fed2ed985e3130fc60 Mon Sep 17 00:00:00 2001 From: Christoph Burgdorf Date: Thu, 25 Oct 2018 18:52:49 +0200 Subject: [PATCH 5/6] Tighten input of normalize_int --- eth/tools/_utils/normalization.py | 6 ++++-- eth/tools/fixtures/helpers.py | 1 - eth/typing.py | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/eth/tools/_utils/normalization.py b/eth/tools/_utils/normalization.py index 3a8f80365f..0c9abe7b39 100644 --- a/eth/tools/_utils/normalization.py +++ b/eth/tools/_utils/normalization.py @@ -57,6 +57,7 @@ from eth.typing import ( AccountState, GeneralState, + IntConvertible, Normalizer, TransactionDict, TransactionNormalizer, @@ -67,16 +68,17 @@ # Primitives # @functools.lru_cache(maxsize=1024) -def normalize_int(value: Any) -> int: +def normalize_int(value: IntConvertible) -> int: """ Robust to integer conversion, handling hex values, string representations, and special cases like `0x`. """ if is_integer(value): - return value + return cast(int, value) elif is_bytes(value): return big_endian_to_int(value) elif is_hex(value) and is_0x_prefixed(value): + value = cast(str, value) if len(value) == 2: return 0 else: diff --git a/eth/tools/fixtures/helpers.py b/eth/tools/fixtures/helpers.py index 483d0b0607..06c0c15b3d 100644 --- a/eth/tools/fixtures/helpers.py +++ b/eth/tools/fixtures/helpers.py @@ -4,7 +4,6 @@ from typing import ( Any, - cast, Dict, Iterable, Tuple, diff --git a/eth/typing.py b/eth/typing.py index d5a1246e66..4ec64f866f 100644 --- a/eth/typing.py +++ b/eth/typing.py @@ -11,6 +11,7 @@ from eth_typing import ( Address, + HexStr, ) from mypy_extensions import ( TypedDict, @@ -49,3 +50,5 @@ TransactionNormalizer = Callable[[TransactionDict], TransactionDict] VRS = NewType("VRS", Tuple[int, int, int]) + +IntConvertible = Union[int, bytes, HexStr, str] From 7101306652b6375e8186f725ccf89a5eeb65ad03 Mon Sep 17 00:00:00 2001 From: Christoph Burgdorf Date: Thu, 25 Oct 2018 18:55:58 +0200 Subject: [PATCH 6/6] Tighten input of normalize_bytes --- eth/tools/_utils/normalization.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eth/tools/_utils/normalization.py b/eth/tools/_utils/normalization.py index 0c9abe7b39..d5b44a42c2 100644 --- a/eth/tools/_utils/normalization.py +++ b/eth/tools/_utils/normalization.py @@ -12,6 +12,7 @@ Mapping, Sequence, Tuple, + Union, ) from cytoolz import ( @@ -89,9 +90,9 @@ def normalize_int(value: IntConvertible) -> int: raise TypeError("Unsupported type: Got `{0}`".format(type(value))) -def normalize_bytes(value: Any) -> bytes: +def normalize_bytes(value: Union[bytes, str]) -> bytes: if is_bytes(value): - return value + return cast(bytes, value) elif is_text(value) and is_hex(value): return decode_hex(value) elif is_text(value):