diff --git a/Makefile b/Makefile index e56eb4bb266..d11ea72779f 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ test: poetry run pytest --cache-clear tests/performance coverage-html: - poetry run pytest --cov-report html + poetry run pytest -m "not perf" --cov-report=html pr: lint test security-baseline complexity-baseline diff --git a/aws_lambda_powertools/tracing/extensions.py b/aws_lambda_powertools/tracing/extensions.py index 2bb0125e841..6c641238c98 100644 --- a/aws_lambda_powertools/tracing/extensions.py +++ b/aws_lambda_powertools/tracing/extensions.py @@ -8,8 +8,8 @@ def aiohttp_trace_config(): TraceConfig aiohttp trace config """ - from aws_xray_sdk.ext.aiohttp.client import aws_xray_trace_config + from aws_xray_sdk.ext.aiohttp.client import aws_xray_trace_config # pragma: no cover - aws_xray_trace_config.__doc__ = "aiohttp extension for X-Ray (aws_xray_trace_config)" + aws_xray_trace_config.__doc__ = "aiohttp extension for X-Ray (aws_xray_trace_config)" # pragma: no cover - return aws_xray_trace_config() + return aws_xray_trace_config() # pragma: no cover diff --git a/aws_lambda_powertools/utilities/idempotency/__init__.py b/aws_lambda_powertools/utilities/idempotency/__init__.py index 98e2be15415..b46d0855a93 100644 --- a/aws_lambda_powertools/utilities/idempotency/__init__.py +++ b/aws_lambda_powertools/utilities/idempotency/__init__.py @@ -5,6 +5,6 @@ from aws_lambda_powertools.utilities.idempotency.persistence.base import BasePersistenceLayer from aws_lambda_powertools.utilities.idempotency.persistence.dynamodb import DynamoDBPersistenceLayer -from .idempotency import idempotent +from .idempotency import IdempotencyConfig, idempotent -__all__ = ("DynamoDBPersistenceLayer", "BasePersistenceLayer", "idempotent") +__all__ = ("DynamoDBPersistenceLayer", "BasePersistenceLayer", "idempotent", "IdempotencyConfig") diff --git a/aws_lambda_powertools/utilities/idempotency/config.py b/aws_lambda_powertools/utilities/idempotency/config.py new file mode 100644 index 00000000000..52afb3bad8c --- /dev/null +++ b/aws_lambda_powertools/utilities/idempotency/config.py @@ -0,0 +1,43 @@ +from typing import Dict + + +class IdempotencyConfig: + def __init__( + self, + event_key_jmespath: str = "", + payload_validation_jmespath: str = "", + jmespath_options: Dict = None, + raise_on_no_idempotency_key: bool = False, + expires_after_seconds: int = 60 * 60, # 1 hour default + use_local_cache: bool = False, + local_cache_max_items: int = 256, + hash_function: str = "md5", + ): + """ + Initialize the base persistence layer + + Parameters + ---------- + event_key_jmespath: str + A jmespath expression to extract the idempotency key from the event record + payload_validation_jmespath: str + A jmespath expression to extract the payload to be validated from the event record + raise_on_no_idempotency_key: bool, optional + Raise exception if no idempotency key was found in the request, by default False + expires_after_seconds: int + The number of seconds to wait before a record is expired + use_local_cache: bool, optional + Whether to locally cache idempotency results, by default False + local_cache_max_items: int, optional + Max number of items to store in local cache, by default 1024 + hash_function: str, optional + Function to use for calculating hashes, by default md5. + """ + self.event_key_jmespath = event_key_jmespath + self.payload_validation_jmespath = payload_validation_jmespath + self.jmespath_options = jmespath_options + self.raise_on_no_idempotency_key = raise_on_no_idempotency_key + self.expires_after_seconds = expires_after_seconds + self.use_local_cache = use_local_cache + self.local_cache_max_items = local_cache_max_items + self.hash_function = hash_function diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index bc556f49912..235e5c884d6 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -5,6 +5,7 @@ from typing import Any, Callable, Dict, Optional from aws_lambda_powertools.middleware_factory import lambda_handler_decorator +from aws_lambda_powertools.utilities.idempotency.config import IdempotencyConfig from aws_lambda_powertools.utilities.idempotency.exceptions import ( IdempotencyAlreadyInProgressError, IdempotencyInconsistentStateError, @@ -29,6 +30,7 @@ def idempotent( event: Dict[str, Any], context: LambdaContext, persistence_store: BasePersistenceLayer, + config: IdempotencyConfig = None, ) -> Any: """ Middleware to handle idempotency @@ -43,20 +45,25 @@ def idempotent( Lambda's Context persistence_store: BasePersistenceLayer Instance of BasePersistenceLayer to store data + config: IdempotencyConfig + Configutation Examples -------- **Processes Lambda's event in an idempotent manner** - >>> from aws_lambda_powertools.utilities.idempotency import idempotent, DynamoDBPersistenceLayer + >>> from aws_lambda_powertools.utilities.idempotency import ( + >>> idempotent, DynamoDBPersistenceLayer, IdempotencyConfig + >>> ) >>> - >>> persistence_store = DynamoDBPersistenceLayer(event_key_jmespath="body", table_name="idempotency_store") + >>> idem_config=IdempotencyConfig(event_key_jmespath="body") + >>> persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency_store") >>> - >>> @idempotent(persistence_store=persistence_store) + >>> @idempotent(config=idem_config, persistence_store=persistence_layer) >>> def handler(event, context): >>> return {"StatusCode": 200} """ - idempotency_handler = IdempotencyHandler(handler, event, context, persistence_store) + idempotency_handler = IdempotencyHandler(handler, event, context, config or IdempotencyConfig(), persistence_store) # IdempotencyInconsistentStateError can happen under rare but expected cases when persistent state changes in the # small time between put & get requests. In most cases we can retry successfully on this exception. @@ -82,6 +89,7 @@ def __init__( lambda_handler: Callable[[Any, LambdaContext], Any], event: Dict[str, Any], context: LambdaContext, + config: IdempotencyConfig, persistence_store: BasePersistenceLayer, ): """ @@ -98,6 +106,7 @@ def __init__( persistence_store : BasePersistenceLayer Instance of persistence layer to store idempotency records """ + persistence_store.configure(config) self.persistence_store = persistence_store self.context = context self.event = event diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index 352ba40b5f6..58f67a292e7 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -9,13 +9,14 @@ import warnings from abc import ABC, abstractmethod from types import MappingProxyType -from typing import Any, Dict +from typing import Any, Dict, Optional import jmespath from aws_lambda_powertools.shared.cache_dict import LRUDict from aws_lambda_powertools.shared.jmespath_functions import PowertoolsFunctions from aws_lambda_powertools.shared.json_encoder import Encoder +from aws_lambda_powertools.utilities.idempotency.config import IdempotencyConfig from aws_lambda_powertools.utilities.idempotency.exceptions import ( IdempotencyInvalidStatusError, IdempotencyItemAlreadyExistsError, @@ -107,55 +108,49 @@ class BasePersistenceLayer(ABC): Abstract Base Class for Idempotency persistence layer. """ - def __init__( - self, - event_key_jmespath: str = "", - payload_validation_jmespath: str = "", - expires_after_seconds: int = 60 * 60, # 1 hour default - use_local_cache: bool = False, - local_cache_max_items: int = 256, - hash_function: str = "md5", - raise_on_no_idempotency_key: bool = False, - jmespath_options: Dict = None, - ) -> None: + def __init__(self): + """Initialize the defaults """ + self.configured = False + self.event_key_jmespath: Optional[str] = None + self.event_key_compiled_jmespath = None + self.jmespath_options: Optional[dict] = None + self.payload_validation_enabled = False + self.validation_key_jmespath = None + self.raise_on_no_idempotency_key = False + self.expires_after_seconds: int = 60 * 60 # 1 hour default + self.use_local_cache = False + self._cache: Optional[LRUDict] = None + self.hash_function = None + + def configure(self, config: IdempotencyConfig) -> None: """ - Initialize the base persistence layer + Initialize the base persistence layer from the configuration settings Parameters ---------- - event_key_jmespath: str - A jmespath expression to extract the idempotency key from the event record - payload_validation_jmespath: str - A jmespath expression to extract the payload to be validated from the event record - expires_after_seconds: int - The number of seconds to wait before a record is expired - use_local_cache: bool, optional - Whether to locally cache idempotency results, by default False - local_cache_max_items: int, optional - Max number of items to store in local cache, by default 1024 - hash_function: str, optional - Function to use for calculating hashes, by default md5. - raise_on_no_idempotency_key: bool, optional - Raise exception if no idempotency key was found in the request, by default False - jmespath_options : Dict - Alternative JMESPath options to be included when filtering expr - """ - self.event_key_jmespath = event_key_jmespath - if self.event_key_jmespath: - self.event_key_compiled_jmespath = jmespath.compile(event_key_jmespath) - self.expires_after_seconds = expires_after_seconds - self.use_local_cache = use_local_cache - if self.use_local_cache: - self._cache = LRUDict(max_items=local_cache_max_items) - self.payload_validation_enabled = False - if payload_validation_jmespath: - self.validation_key_jmespath = jmespath.compile(payload_validation_jmespath) + config: IdempotencyConfig + Idempotency configuration settings + """ + if self.configured: + # Prevent being reconfigured multiple times + return + self.configured = True + + self.event_key_jmespath = config.event_key_jmespath + if config.event_key_jmespath: + self.event_key_compiled_jmespath = jmespath.compile(config.event_key_jmespath) + self.jmespath_options = config.jmespath_options + if not self.jmespath_options: + self.jmespath_options = {"custom_functions": PowertoolsFunctions()} + if config.payload_validation_jmespath: + self.validation_key_jmespath = jmespath.compile(config.payload_validation_jmespath) self.payload_validation_enabled = True - self.hash_function = getattr(hashlib, hash_function) - self.raise_on_no_idempotency_key = raise_on_no_idempotency_key - if not jmespath_options: - jmespath_options = {"custom_functions": PowertoolsFunctions()} - self.jmespath_options = jmespath_options + self.raise_on_no_idempotency_key = config.raise_on_no_idempotency_key + self.expires_after_seconds = config.expires_after_seconds + self.use_local_cache = config.use_local_cache + if self.use_local_cache: + self._cache = LRUDict(max_items=config.local_cache_max_items) + self.hash_function = getattr(hashlib, config.hash_function) def _get_hashed_idempotency_key(self, lambda_event: Dict[str, Any]) -> str: """ @@ -180,9 +175,9 @@ def _get_hashed_idempotency_key(self, lambda_event: Dict[str, Any]) -> str: ) if self.is_missing_idempotency_key(data): - warnings.warn(f"No value found for idempotency_key. jmespath: {self.event_key_jmespath}") if self.raise_on_no_idempotency_key: raise IdempotencyKeyError("No data found to create a hashed idempotency_key") + warnings.warn(f"No value found for idempotency_key. jmespath: {self.event_key_jmespath}") return self._generate_hash(data) diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py index 4d66448755d..d87cd71ff4e 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py @@ -26,8 +26,6 @@ def __init__( validation_key_attr: str = "validation", boto_config: Optional[Config] = None, boto3_session: Optional[boto3.session.Session] = None, - *args, - **kwargs, ): """ Initialize the DynamoDB client @@ -57,9 +55,9 @@ def __init__( **Create a DynamoDB persistence layer with custom settings** >>> from aws_lambda_powertools.utilities.idempotency import idempotent, DynamoDBPersistenceLayer >>> - >>> persistence_store = DynamoDBPersistenceLayer(event_key="body", table_name="idempotency_store") + >>> persistence_store = DynamoDBPersistenceLayer(table_name="idempotency_store") >>> - >>> @idempotent(persistence_store=persistence_store) + >>> @idempotent(persistence_store=persistence_store, event_key="body") >>> def handler(event, context): >>> return {"StatusCode": 200} """ @@ -74,7 +72,7 @@ def __init__( self.status_attr = status_attr self.data_attr = data_attr self.validation_key_attr = validation_key_attr - super(DynamoDBPersistenceLayer, self).__init__(*args, **kwargs) + super(DynamoDBPersistenceLayer, self).__init__() def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord: """ diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 8ea23bde5ae..784f6597a23 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -75,7 +75,7 @@ storage layer, so you'll need to create a table first. You can quickly start by initializing the `DynamoDBPersistenceLayer` class outside the Lambda handler, and using it with the `idempotent` decorator on your lambda handler. The only required parameter is `table_name`, but you likely -want to specify `event_key_jmespath` as well. +want to specify `event_key_jmespath` via `IdempotencyConfig` class. `event_key_jmespath`: A JMESpath expression which will be used to extract the payload from the event your Lambda handler is called with. This payload will be used as the key to decide if future invocations are duplicates. If you don't pass @@ -83,24 +83,24 @@ this parameter, the entire event will be used as the key. === "app.py" - ```python hl_lines="2 6-9 11" + ```python hl_lines="2-4 8-9 11" import json - from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, idempotent + from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, DynamoDBPersistenceLayer, idempotent + ) # Treat everything under the "body" key in # the event json object as our payload - persistence_layer = DynamoDBPersistenceLayer( - table_name="IdempotencyTable", - event_key_jmespath="body", - ) + config = IdempotencyConfig(event_key_jmespath="body") + persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") - @idempotent(persistence_store=persistence_layer) + @idempotent(config=config, persistence_store=persistence_layer) def handler(event, context): body = json.loads(event['body']) payment = create_subscription_payment( user=body['user'], product=body['product_id'] - ) + ) ... return {"message": "success", "statusCode": 200, "payment_id": payment.id} ``` @@ -174,9 +174,8 @@ change this window with the `expires_after_seconds` parameter: === "app.py" - ```python hl_lines="4" - DynamoDBPersistenceLayer( - table_name="IdempotencyTable", + ```python hl_lines="3" + IdempotencyConfig( event_key_jmespath="body", expires_after_seconds=5*60, # 5 minutes ) @@ -203,9 +202,8 @@ execution environment. You can change this with the `local_cache_max_items` para === "app.py" - ```python hl_lines="4 5" - DynamoDBPersistenceLayer( - table_name="IdempotencyTable", + ```python hl_lines="3 4" + IdempotencyConfig( event_key_jmespath="body", use_local_cache=True, local_cache_max_items=1000 @@ -224,16 +222,18 @@ idempotent invocations. === "app.py" - ```python hl_lines="6" - from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, idempotent + ```python hl_lines="7" + from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, DynamoDBPersistenceLayer, idempotent + ) - persistence_layer = DynamoDBPersistenceLayer( - table_name="IdempotencyTable", + config = IdempotencyConfig( event_key_jmespath="[userDetail, productId]", payload_validation_jmespath="amount" ) + persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") - @idempotent(persistence_store=persistence_layer) + @idempotent(config=config, persistence_store=persistence_layer) def handler(event, context): # Creating a subscription payment is a side # effect of calling this function! @@ -273,12 +273,42 @@ and we will raise `IdempotencyKeyError` if none was found. === "app.py" - ```python hl_lines="4" - DynamoDBPersistenceLayer( - table_name="IdempotencyTable", - event_key_jmespath="body", + ```python hl_lines="8" + from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, DynamoDBPersistenceLayer, idempotent + ) + + # Requires "user"."uid" and from the "body" json parsed "order_id" to be present + config = IdempotencyConfig( + event_key_jmespath="[user.uid, powertools_json(body).order_id]", raise_on_no_idempotency_key=True, ) + persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") + + @idempotent(config=config, persistence_store=persistence_layer) + def handler(event, context): + pass + ``` +=== "Success Event" + + ```json + { + "user": { + "uid": "BB0D045C-8878-40C8-889E-38B3CB0A61B1", + "name": "Foo" + }, + "body": "{\"order_id\": 10000}" + } + ``` +=== "Failure Event" + + ```json + { + "user": { + "name": "Joe Bloggs" + }, + "body": "{\"total_amount\": 10000}" + } ``` ### Changing dynamoDB attribute names @@ -298,10 +328,9 @@ This example demonstrates changing the attribute names to custom values: === "app.py" - ```python hl_lines="4-8" + ```python hl_lines="3-7" persistence_layer = DynamoDBPersistenceLayer( table_name="IdempotencyTable", - event_key_jmespath="[userDetail, productId]", key_attr="idempotency_key", expiry_attr="expires_at", status_attr="current_status", @@ -316,35 +345,39 @@ or `boto3_session` parameters when constructing the persistence store. === "Custom session" - ```python hl_lines="1 4 8" + ```python hl_lines="1 7 10" import boto3 - from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, idempotent + from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, DynamoDBPersistenceLayer, idempotent + ) + config = IdempotencyConfig(event_key_jmespath="body") boto3_session = boto3.session.Session() persistence_layer = DynamoDBPersistenceLayer( table_name="IdempotencyTable", - event_key_jmespath="body", boto3_session=boto3_session ) - @idempotent(persistence_store=persistence_layer) + @idempotent(config=config, persistence_store=persistence_layer) def handler(event, context): ... ``` === "Custom config" - ```python hl_lines="1 4 8" + ```python hl_lines="1 7 10" from botocore.config import Config - from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, idempotent + from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, DynamoDBPersistenceLayer, idempotent + ) + config = IdempotencyConfig(event_key_jmespath="body") boto_config = Config() persistence_layer = DynamoDBPersistenceLayer( table_name="IdempotencyTable", - event_key_jmespath="body", boto_config=boto_config ) - @idempotent(persistence_store=persistence_layer) + @idempotent(config=config, persistence_store=persistence_layer) def handler(event, context): ... ``` @@ -372,15 +405,15 @@ The idempotency utility can be used with the `validator` decorator. Ensure that ```python hl_lines="9 10" from aws_lambda_powertools.utilities.validation import validator, envelopes - from aws_lambda_powertools.utilities.idempotency.idempotency import idempotent - - persistence_layer = DynamoDBPersistenceLayer( - table_name="IdempotencyTable", - event_key_jmespath="[message, username]", + from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, DynamoDBPersistenceLayer, idempotent ) + config = IdempotencyConfig(event_key_jmespath="[message, username]") + persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") + @validator(envelope=envelopes.API_GATEWAY_HTTP) - @idempotent(persistence_store=persistence_layer) + @idempotent(config=config, persistence_store=persistence_layer) def lambda_handler(event, context): cause_some_side_effects(event['username') return {"message": event['message'], "statusCode": 200} diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/functional/idempotency/conftest.py b/tests/functional/idempotency/conftest.py index 9ae030f02d1..532d551ef40 100644 --- a/tests/functional/idempotency/conftest.py +++ b/tests/functional/idempotency/conftest.py @@ -13,6 +13,7 @@ from aws_lambda_powertools.shared.json_encoder import Encoder from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer +from aws_lambda_powertools.utilities.idempotency.idempotency import IdempotencyConfig from aws_lambda_powertools.utilities.validation import envelopes from aws_lambda_powertools.utilities.validation.base import unwrap_event_from_envelope @@ -151,51 +152,44 @@ def hashed_validation_key(lambda_apigw_event): @pytest.fixture -def persistence_store(config, request, default_jmespath): - persistence_store = DynamoDBPersistenceLayer( +def persistence_store(config): + return DynamoDBPersistenceLayer(table_name=TABLE_NAME, boto_config=config) + + +@pytest.fixture +def idempotency_config(config, request, default_jmespath): + return IdempotencyConfig( event_key_jmespath=request.param.get("event_key_jmespath") or default_jmespath, - table_name=TABLE_NAME, - boto_config=config, use_local_cache=request.param["use_local_cache"], ) - return persistence_store @pytest.fixture -def persistence_store_without_jmespath(config, request): - persistence_store = DynamoDBPersistenceLayer( - table_name=TABLE_NAME, boto_config=config, use_local_cache=request.param["use_local_cache"], - ) - return persistence_store +def config_without_jmespath(config, request): + return IdempotencyConfig(use_local_cache=request.param["use_local_cache"]) @pytest.fixture -def persistence_store_with_validation(config, request, default_jmespath): - persistence_store = DynamoDBPersistenceLayer( +def config_with_validation(config, request, default_jmespath): + return IdempotencyConfig( event_key_jmespath=default_jmespath, - table_name=TABLE_NAME, - boto_config=config, use_local_cache=request.param, payload_validation_jmespath="requestContext", ) - return persistence_store @pytest.fixture -def persistence_store_with_jmespath_options(config, request): +def config_with_jmespath_options(config, request): class CustomFunctions(functions.Functions): @functions.signature({"types": ["string"]}) def _func_echo_decoder(self, value): return value - persistence_store = DynamoDBPersistenceLayer( - table_name=TABLE_NAME, - boto_config=config, + return IdempotencyConfig( use_local_cache=False, event_key_jmespath=request.param, jmespath_options={"custom_functions": CustomFunctions()}, ) - return persistence_store @pytest.fixture diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 872d9f39365..999b34fc8f6 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -7,6 +7,7 @@ import pytest from botocore import stub +from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig from aws_lambda_powertools.utilities.idempotency.exceptions import ( IdempotencyAlreadyInProgressError, IdempotencyInconsistentStateError, @@ -24,9 +25,10 @@ # Using parametrize to run test twice, with two separate instances of persistence store. One instance with caching # enabled, and one without. -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_already_completed( - persistence_store, + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, hashed_idempotency_key, @@ -56,7 +58,7 @@ def test_idempotent_lambda_already_completed( stubber.add_response("get_item", ddb_response, expected_params) stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): raise Exception @@ -67,9 +69,14 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_in_progress( - persistence_store, lambda_apigw_event, lambda_response, timestamp_future, hashed_idempotency_key + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, + lambda_apigw_event, + lambda_response, + timestamp_future, + hashed_idempotency_key, ): """ Test idempotent decorator where lambda_handler is already processing an event with matching event key @@ -94,7 +101,7 @@ def test_idempotent_lambda_in_progress( stubber.add_response("get_item", ddb_response, expected_params) stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -110,9 +117,15 @@ def lambda_handler(event, context): @pytest.mark.skipif(sys.version_info < (3, 8), reason="issue with pytest mock lib for < 3.8") -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_idempotent_lambda_in_progress_with_cache( - persistence_store, lambda_apigw_event, lambda_response, timestamp_future, hashed_idempotency_key, mocker + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, + lambda_apigw_event, + lambda_response, + timestamp_future, + hashed_idempotency_key, + mocker, ): """ Test idempotent decorator where lambda_handler is already processing an event with matching event key, cache @@ -145,7 +158,7 @@ def test_idempotent_lambda_in_progress_with_cache( stubber.add_response("get_item", copy.deepcopy(ddb_response), copy.deepcopy(expected_params)) stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -168,9 +181,10 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_first_execution( - persistence_store, + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, expected_params_update_item, expected_params_put_item, @@ -190,7 +204,7 @@ def test_idempotent_lambda_first_execution( stubber.add_response("update_item", ddb_response, expected_params_update_item) stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -201,10 +215,11 @@ def lambda_handler(event, context): @pytest.mark.skipif(sys.version_info < (3, 8), reason="issue with pytest mock lib for < 3.8") -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) def test_idempotent_lambda_first_execution_cached( - persistence_store, - lambda_apigw_event, + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, + lambda_apigw_event: DynamoDBPersistenceLayer, expected_params_update_item, expected_params_put_item, lambda_response, @@ -224,7 +239,7 @@ def test_idempotent_lambda_first_execution_cached( stubber.add_response("update_item", ddb_response, expected_params_update_item) stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -245,9 +260,10 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_expired( - persistence_store, + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_expired, lambda_response, @@ -268,7 +284,7 @@ def test_idempotent_lambda_expired( stubber.add_response("update_item", ddb_response, expected_params_update_item) stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -278,9 +294,10 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_exception( - persistence_store, + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, lambda_response, @@ -304,7 +321,7 @@ def test_idempotent_lambda_exception( stubber.add_response("delete_item", ddb_response, expected_params_delete_item) stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): raise Exception("Something went wrong!") @@ -316,10 +333,11 @@ def lambda_handler(event, context): @pytest.mark.parametrize( - "persistence_store_with_validation", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True + "config_with_validation", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True ) def test_idempotent_lambda_already_completed_with_validation_bad_payload( - persistence_store_with_validation, + config_with_validation: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, lambda_response, @@ -330,7 +348,7 @@ def test_idempotent_lambda_already_completed_with_validation_bad_payload( Test idempotent decorator where event with matching event key has already been succesfully processed """ - stubber = stub.Stubber(persistence_store_with_validation.table.meta.client) + stubber = stub.Stubber(persistence_store.table.meta.client) ddb_response = { "Item": { "id": {"S": hashed_idempotency_key}, @@ -347,7 +365,7 @@ def test_idempotent_lambda_already_completed_with_validation_bad_payload( stubber.add_response("get_item", ddb_response, expected_params) stubber.activate() - @idempotent(persistence_store=persistence_store_with_validation) + @idempotent(config=config_with_validation, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -359,9 +377,10 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_lambda_expired_during_request( - persistence_store, + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_expired, lambda_response, @@ -402,7 +421,7 @@ def test_idempotent_lambda_expired_during_request( stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return lambda_response @@ -414,9 +433,10 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_persistence_exception_deleting( - persistence_store, + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, lambda_response, @@ -435,7 +455,7 @@ def test_idempotent_persistence_exception_deleting( stubber.add_client_error("delete_item", "UnrecoverableError") stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): raise Exception("Something went wrong!") @@ -447,9 +467,10 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_persistence_exception_updating( - persistence_store, + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, lambda_response, @@ -468,7 +489,7 @@ def test_idempotent_persistence_exception_updating( stubber.add_client_error("update_item", "UnrecoverableError") stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return {"message": "success!"} @@ -480,9 +501,10 @@ def lambda_handler(event, context): stubber.deactivate() -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True) def test_idempotent_persistence_exception_getting( - persistence_store, + idempotency_config: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, lambda_response, @@ -499,7 +521,7 @@ def test_idempotent_persistence_exception_getting( stubber.add_client_error("get_item", "UnexpectedException") stubber.activate() - @idempotent(persistence_store=persistence_store) + @idempotent(config=idempotency_config, persistence_store=persistence_store) def lambda_handler(event, context): return {"message": "success!"} @@ -512,10 +534,11 @@ def lambda_handler(event, context): @pytest.mark.parametrize( - "persistence_store_with_validation", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True + "config_with_validation", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True ) def test_idempotent_lambda_first_execution_with_validation( - persistence_store_with_validation, + config_with_validation: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, expected_params_update_item_with_validation, expected_params_put_item_with_validation, @@ -526,15 +549,15 @@ def test_idempotent_lambda_first_execution_with_validation( """ Test idempotent decorator when lambda is executed with an event with a previously unknown event key """ - stubber = stub.Stubber(persistence_store_with_validation.table.meta.client) + stubber = stub.Stubber(persistence_store.table.meta.client) ddb_response = {} stubber.add_response("put_item", ddb_response, expected_params_put_item_with_validation) stubber.add_response("update_item", ddb_response, expected_params_update_item_with_validation) stubber.activate() - @idempotent(persistence_store=persistence_store_with_validation) - def lambda_handler(lambda_apigw_event, context): + @idempotent(config=config_with_validation, persistence_store=persistence_store) + def lambda_handler(event, context): return lambda_response lambda_handler(lambda_apigw_event, {}) @@ -544,10 +567,11 @@ def lambda_handler(lambda_apigw_event, context): @pytest.mark.parametrize( - "persistence_store_without_jmespath", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True + "config_without_jmespath", [{"use_local_cache": False}, {"use_local_cache": True}], indirect=True ) def test_idempotent_lambda_with_validator_util( - persistence_store_without_jmespath, + config_without_jmespath: IdempotencyConfig, + persistence_store: DynamoDBPersistenceLayer, lambda_apigw_event, timestamp_future, serialized_lambda_response, @@ -560,7 +584,7 @@ def test_idempotent_lambda_with_validator_util( validator utility to unwrap the event """ - stubber = stub.Stubber(persistence_store_without_jmespath.table.meta.client) + stubber = stub.Stubber(persistence_store.table.meta.client) ddb_response = { "Item": { "id": {"S": hashed_idempotency_key_with_envelope}, @@ -580,7 +604,7 @@ def test_idempotent_lambda_with_validator_util( stubber.activate() @validator(envelope=envelopes.API_GATEWAY_HTTP) - @idempotent(persistence_store=persistence_store_without_jmespath) + @idempotent(config=config_without_jmespath, persistence_store=persistence_store) def lambda_handler(event, context): mock_function() return "shouldn't get here!" @@ -601,10 +625,13 @@ def test_data_record_invalid_status_value(): assert e.value.args[0] == "UNSUPPORTED_STATUS" -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": True}], indirect=True) -def test_in_progress_never_saved_to_cache(persistence_store): +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) +def test_in_progress_never_saved_to_cache( + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer +): # GIVEN a data record with status "INPROGRESS" # and persistence_store has use_local_cache = True + persistence_store.configure(idempotency_config) data_record = DataRecord("key", status="INPROGRESS") # WHEN saving to local cache @@ -614,9 +641,10 @@ def test_in_progress_never_saved_to_cache(persistence_store): assert persistence_store._cache.get("key") is None -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False}], indirect=True) -def test_user_local_disabled(persistence_store): +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}], indirect=True) +def test_user_local_disabled(idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer): # GIVEN a persistence_store with use_local_cache = False + persistence_store.configure(idempotency_config) # WHEN calling any local cache options data_record = DataRecord("key", status="COMPLETED") @@ -633,9 +661,13 @@ def test_user_local_disabled(persistence_store): assert not hasattr("persistence_store", "_cache") -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": True}], indirect=True) -def test_delete_from_cache_when_empty(persistence_store): +@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": True}], indirect=True) +def test_delete_from_cache_when_empty( + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer +): # GIVEN use_local_cache is True AND the local cache is empty + persistence_store.configure(idempotency_config) + try: # WHEN we _delete_from_cache persistence_store._delete_from_cache("key_does_not_exist") @@ -663,9 +695,14 @@ def test_is_missing_idempotency_key(): assert BasePersistenceLayer.is_missing_idempotency_key("") -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": False, "event_key_jmespath": "body"}], indirect=True) -def test_default_no_raise_on_missing_idempotency_key(persistence_store): +@pytest.mark.parametrize( + "idempotency_config", [{"use_local_cache": False, "event_key_jmespath": "body"}], indirect=True +) +def test_default_no_raise_on_missing_idempotency_key( + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer +): # GIVEN a persistence_store with use_local_cache = False and event_key_jmespath = "body" + persistence_store.configure(idempotency_config) assert persistence_store.use_local_cache is False assert "body" in persistence_store.event_key_jmespath @@ -677,10 +714,13 @@ def test_default_no_raise_on_missing_idempotency_key(persistence_store): @pytest.mark.parametrize( - "persistence_store", [{"use_local_cache": False, "event_key_jmespath": "[body, x]"}], indirect=True + "idempotency_config", [{"use_local_cache": False, "event_key_jmespath": "[body, x]"}], indirect=True ) -def test_raise_on_no_idempotency_key(persistence_store): +def test_raise_on_no_idempotency_key( + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer +): # GIVEN a persistence_store with raise_on_no_idempotency_key and no idempotency key in the request + persistence_store.configure(idempotency_config) persistence_store.raise_on_no_idempotency_key = True assert persistence_store.use_local_cache is False assert "body" in persistence_store.event_key_jmespath @@ -693,11 +733,21 @@ def test_raise_on_no_idempotency_key(persistence_store): assert "No data found to create a hashed idempotency_key" in str(excinfo.value) -@pytest.mark.parametrize("persistence_store", [{"use_local_cache": True}], indirect=True) -def test_jmespath_with_powertools_json(persistence_store): +@pytest.mark.parametrize( + "idempotency_config", + [ + { + "use_local_cache": False, + "event_key_jmespath": "[requestContext.authorizer.claims.sub, powertools_json(body).id]", + } + ], + indirect=True, +) +def test_jmespath_with_powertools_json( + idempotency_config: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer +): # GIVEN an event_key_jmespath with powertools_json custom function - persistence_store.event_key_jmespath = "[requestContext.authorizer.claims.sub, powertools_json(body).id]" - persistence_store.event_key_compiled_jmespath = jmespath.compile(persistence_store.event_key_jmespath) + persistence_store.configure(idempotency_config) sub_attr_value = "cognito_user" key_attr_value = "some_key" expected_value = [sub_attr_value, key_attr_value] @@ -713,11 +763,15 @@ def test_jmespath_with_powertools_json(persistence_store): assert result == persistence_store._generate_hash(expected_value) -@pytest.mark.parametrize("persistence_store_with_jmespath_options", ["powertools_json(data).payload"], indirect=True) -def test_custom_jmespath_function_overrides_builtin_functions(persistence_store_with_jmespath_options): +@pytest.mark.parametrize("config_with_jmespath_options", ["powertools_json(data).payload"], indirect=True) +def test_custom_jmespath_function_overrides_builtin_functions( + config_with_jmespath_options: IdempotencyConfig, persistence_store: DynamoDBPersistenceLayer +): # GIVEN an persistence store with a custom jmespath_options # AND use a builtin powertools custom function + persistence_store.configure(config_with_jmespath_options) + with pytest.raises(jmespath.exceptions.UnknownFunctionError, match="Unknown function: powertools_json()"): # WHEN calling _get_hashed_idempotency_key # THEN raise unknown function - persistence_store_with_jmespath_options._get_hashed_idempotency_key({}) + persistence_store._get_hashed_idempotency_key({})