From 473f24c19ac45614e8805141a1d0f507c7f0cd8b Mon Sep 17 00:00:00 2001 From: Ran Isenberg Date: Sat, 10 Jun 2023 09:57:31 +0300 Subject: [PATCH 1/5] docs(idempotency): add cdk table example --- docs/utilities/idempotency.md | 104 +++++++++++++++++++++------------- 1 file changed, 66 insertions(+), 38 deletions(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 37b34bf9750..3322452f3a6 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -77,31 +77,55 @@ If you're not [changing the default configuration for the DynamoDB persistence l ???+ tip "Tip: You can share a single state table for all functions" You can reuse the same DynamoDB table to store idempotency state. We add `module_name` and [qualified name for classes and functions](https://peps.python.org/pep-3155/){target="_blank"} in addition to the idempotency key as a hash key. -```yaml hl_lines="5-13 21-23" title="AWS Serverless Application Model (SAM) example" -Resources: - IdempotencyTable: - Type: AWS::DynamoDB::Table - Properties: - AttributeDefinitions: - - AttributeName: id - AttributeType: S - KeySchema: - - AttributeName: id - KeyType: HASH - TimeToLiveSpecification: - AttributeName: expiration - Enabled: true - BillingMode: PAY_PER_REQUEST - - HelloWorldFunction: - Type: AWS::Serverless::Function - Properties: - Runtime: python3.9 - ... - Policies: - - DynamoDBCrudPolicy: - TableName: !Ref IdempotencyTable -``` +=== "sam.yaml" + + ```yaml hl_lines="5-13 21-23" title="AWS Serverless Application Model (SAM) example" + Resources: + IdempotencyTable: + Type: AWS::DynamoDB::Table + Properties: + AttributeDefinitions: + - AttributeName: id + AttributeType: S + KeySchema: + - AttributeName: id + KeyType: HASH + TimeToLiveSpecification: + AttributeName: expiration + Enabled: true + BillingMode: PAY_PER_REQUEST + + HelloWorldFunction: + Type: AWS::Serverless::Function + Properties: + Runtime: python3.9 + ... + Policies: + - DynamoDBCrudPolicy: + TableName: !Ref IdempotencyTable + ``` +=== "cdk.py" + + ```python title="AWS Cloud Development Kit (CDK) Construct example" + from aws_cdk import RemovalPolicy + from aws_cdk import aws_dynamodb as dynamodb + from constructs import Construct + + + class IdempotencyConstruct(Construct): + + def __init__(self, scope: Construct, id_: str) -> None: + super().__init__(scope, id_) + self.idempotency_table = dynamodb.Table( + self, + 'IdempotencyTable', + partition_key=dynamodb.Attribute(name='id', type=dynamodb.AttributeType.STRING), + billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST, + removal_policy=RemovalPolicy.DESTROY, + time_to_live_attribute='expiration', + point_in_time_recovery=True, + ) + ``` ???+ warning "Warning: Large responses with DynamoDB persistence layer" When using this utility with DynamoDB, your function's responses must be [smaller than 400KB](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html#limits-items){target="_blank"}. @@ -148,7 +172,7 @@ You can quickly start by initializing the `DynamoDBPersistenceLayer` class and u === "Example event" - ```json + ```json { "username": "xyz", "product_id": "123456789" @@ -334,10 +358,12 @@ In this example, we have a Lambda handler that creates a payment for a user subs Imagine the function executes successfully, but the client never receives the response due to a connection issue. It is safe to retry in this instance, as the idempotent decorator will return a previously saved response. -**What we want here** is to instruct Idempotency to use `user` and `product_id` fields from our incoming payload as our idempotency key. If we were to treat the entire request as our idempotency key, a simple HTTP header change would cause our customer to be charged twice. +**What we want here** is to instruct Idempotency to use `user` and `product_id` fields from our incoming payload as our idempotency key. +If we were to treat the entire request as our idempotency key, a simple HTTP header change would cause our customer to be charged twice. ???+ tip "Deserializing JSON strings in payloads for increased accuracy." - The payload extracted by the `event_key_jmespath` is treated as a string by default. This means there could be differences in whitespace even when the JSON payload itself is identical. + The payload extracted by the `event_key_jmespath` is treated as a string by default. + This means there could be differences in whitespace even when the JSON payload itself is identical. To alter this behaviour, we can use the [JMESPath built-in function](jmespath_functions.md#powertools_json-function){target="_blank"} `powertools_json()` to treat the payload as a JSON object (dict) rather than a string. @@ -410,7 +436,8 @@ Imagine the function executes successfully, but the client never receives the re ???+ note This is automatically done when you decorate your Lambda handler with [@idempotent decorator](#idempotent-decorator). -To prevent against extended failed retries when a [Lambda function times out](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/){target="_blank"}, Powertools for AWS Lambda (Python) calculates and includes the remaining invocation available time as part of the idempotency record. +To prevent against extended failed retries when a [Lambda function times out](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/){target="_blank"}, +Powertools for AWS Lambda (Python) calculates and includes the remaining invocation available time as part of the idempotency record. ???+ example If a second invocation happens **after** this timestamp, and the record is marked as `INPROGRESS`, we will execute the invocation again as if it was in the `EXPIRED` state (e.g, `expire_seconds` field elapsed). @@ -418,7 +445,8 @@ To prevent against extended failed retries when a [Lambda function times out](ht This means that if an invocation expired during execution, it will be quickly executed again on the next retry. ???+ important - If you are only using the [@idempotent_function decorator](#idempotent_function-decorator) to guard isolated parts of your code, you must use `register_lambda_context` available in the [idempotency config object](#customizing-the-default-behavior) to benefit from this protection. + If you are only using the [@idempotent_function decorator](#idempotent_function-decorator) to guard isolated parts of your code, + you must use `register_lambda_context` available in the [idempotency config object](#customizing-the-default-behavior) to benefit from this protection. Here is an example on how you register the Lambda context in your handler: @@ -698,14 +726,14 @@ When using DynamoDB as a persistence layer, you can alter the attribute names by Idempotent decorator can be further configured with **`IdempotencyConfig`** as seen in the previous example. These are the available options for further configuration -| Parameter | Default | Description | -| ------------------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | -| **event_key_jmespath** | `""` | JMESPath expression to extract the idempotency key from the event record using [built-in functions](/utilities/jmespath_functions){target="_blank"} | -| **payload_validation_jmespath** | `""` | JMESPath expression to validate whether certain parameters have changed in the event while the event payload | -| **raise_on_no_idempotency_key** | `False` | Raise exception if no idempotency key was found in the request | -| **expires_after_seconds** | 3600 | The number of seconds to wait before a record is expired | -| **use_local_cache** | `False` | Whether to locally cache idempotency results | -| **local_cache_max_items** | 256 | Max number of items to store in local cache | +| Parameter | Default | Description | +| ------------------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **event_key_jmespath** | `""` | JMESPath expression to extract the idempotency key from the event record using [built-in functions](/utilities/jmespath_functions){target="_blank"} | +| **payload_validation_jmespath** | `""` | JMESPath expression to validate whether certain parameters have changed in the event while the event payload | +| **raise_on_no_idempotency_key** | `False` | Raise exception if no idempotency key was found in the request | +| **expires_after_seconds** | 3600 | The number of seconds to wait before a record is expired | +| **use_local_cache** | `False` | Whether to locally cache idempotency results | +| **local_cache_max_items** | 256 | Max number of items to store in local cache | | **hash_function** | `md5` | Function to use for calculating hashes, as provided by [hashlib](https://docs.python.org/3/library/hashlib.html){target="_blank"} in the standard library. | ### Handling concurrent executions with the same payload From dbe3cb944d48f5ee71c801b43580052fa09500a9 Mon Sep 17 00:00:00 2001 From: Ran Isenberg Date: Sat, 10 Jun 2023 22:09:57 +0300 Subject: [PATCH 2/5] add role policy --- docs/utilities/idempotency.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 3322452f3a6..f9f94a9b6f1 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -109,12 +109,13 @@ If you're not [changing the default configuration for the DynamoDB persistence l ```python title="AWS Cloud Development Kit (CDK) Construct example" from aws_cdk import RemovalPolicy from aws_cdk import aws_dynamodb as dynamodb + from aws_cdk import aws_iam as iam from constructs import Construct class IdempotencyConstruct(Construct): - def __init__(self, scope: Construct, id_: str) -> None: + def __init__(self, scope: Construct, id_: str, lambda_role: iam.Role) -> None: super().__init__(scope, id_) self.idempotency_table = dynamodb.Table( self, @@ -125,6 +126,18 @@ If you're not [changing the default configuration for the DynamoDB persistence l time_to_live_attribute='expiration', point_in_time_recovery=True, ) + lambda_role.attach_inline_policy( + iam.Policy( + self, + 'idempotency-policy', + statements=[ + iam.PolicyStatement( + actions=['dynamodb:PutItem', 'dynamodb:GetItem', 'dynamodb:UpdateItem', 'dynamodb:DeleteItem'], + resources=[self.idempotency_table.table_arn], + effect=iam.Effect.ALLOW, + ) + ] + )) ``` ???+ warning "Warning: Large responses with DynamoDB persistence layer" From 7dd5c44f554637cbb694fdcc3f6c3e07e158439c Mon Sep 17 00:00:00 2001 From: Ran Isenberg Date: Sat, 10 Jun 2023 22:13:51 +0300 Subject: [PATCH 3/5] add line hl --- docs/utilities/idempotency.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index f9f94a9b6f1..6ded5a37f0a 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -106,7 +106,7 @@ If you're not [changing the default configuration for the DynamoDB persistence l ``` === "cdk.py" - ```python title="AWS Cloud Development Kit (CDK) Construct example" + ```python hl_lines="14 17 26-29" title="AWS Cloud Development Kit (CDK) Construct example" from aws_cdk import RemovalPolicy from aws_cdk import aws_dynamodb as dynamodb from aws_cdk import aws_iam as iam From 5901e6dad1197d20ece26a66a700d72d3e4c5c20 Mon Sep 17 00:00:00 2001 From: Ran Isenberg Date: Wed, 14 Jun 2023 20:26:21 +0300 Subject: [PATCH 4/5] cr fixes --- docs/utilities/idempotency.md | 58 ++--------------------------------- examples/idempotency/cdk.py | 22 +++++++++++++ examples/idempotency/sam.yaml | 0 3 files changed, 25 insertions(+), 55 deletions(-) create mode 100644 examples/idempotency/cdk.py create mode 100644 examples/idempotency/sam.yaml diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 6ded5a37f0a..7335860e430 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -80,64 +80,12 @@ If you're not [changing the default configuration for the DynamoDB persistence l === "sam.yaml" ```yaml hl_lines="5-13 21-23" title="AWS Serverless Application Model (SAM) example" - Resources: - IdempotencyTable: - Type: AWS::DynamoDB::Table - Properties: - AttributeDefinitions: - - AttributeName: id - AttributeType: S - KeySchema: - - AttributeName: id - KeyType: HASH - TimeToLiveSpecification: - AttributeName: expiration - Enabled: true - BillingMode: PAY_PER_REQUEST - - HelloWorldFunction: - Type: AWS::Serverless::Function - Properties: - Runtime: python3.9 - ... - Policies: - - DynamoDBCrudPolicy: - TableName: !Ref IdempotencyTable + --8<-- "examples/idempotency/sam.yaml" ``` === "cdk.py" - ```python hl_lines="14 17 26-29" title="AWS Cloud Development Kit (CDK) Construct example" - from aws_cdk import RemovalPolicy - from aws_cdk import aws_dynamodb as dynamodb - from aws_cdk import aws_iam as iam - from constructs import Construct - - - class IdempotencyConstruct(Construct): - - def __init__(self, scope: Construct, id_: str, lambda_role: iam.Role) -> None: - super().__init__(scope, id_) - self.idempotency_table = dynamodb.Table( - self, - 'IdempotencyTable', - partition_key=dynamodb.Attribute(name='id', type=dynamodb.AttributeType.STRING), - billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST, - removal_policy=RemovalPolicy.DESTROY, - time_to_live_attribute='expiration', - point_in_time_recovery=True, - ) - lambda_role.attach_inline_policy( - iam.Policy( - self, - 'idempotency-policy', - statements=[ - iam.PolicyStatement( - actions=['dynamodb:PutItem', 'dynamodb:GetItem', 'dynamodb:UpdateItem', 'dynamodb:DeleteItem'], - resources=[self.idempotency_table.table_arn], - effect=iam.Effect.ALLOW, - ) - ] - )) + ```python hl_lines="14 17 20-23" title="AWS Cloud Development Kit (CDK) Construct example" + --8<-- "examples/idempotency/cdk.py" ``` ???+ warning "Warning: Large responses with DynamoDB persistence layer" diff --git a/examples/idempotency/cdk.py b/examples/idempotency/cdk.py new file mode 100644 index 00000000000..2b856ce384b --- /dev/null +++ b/examples/idempotency/cdk.py @@ -0,0 +1,22 @@ +from aws_cdk import RemovalPolicy +from aws_cdk import aws_dynamodb as dynamodb +from aws_cdk import aws_iam as iam +from constructs import Construct + + +class IdempotencyConstruct(Construct): + def __init__(self, scope: Construct, id_: str, lambda_role: iam.Role) -> None: + super().__init__(scope, id_) + self.idempotency_table = dynamodb.Table( + self, + "IdempotencyTable", + partition_key=dynamodb.Attribute(name="id", type=dynamodb.AttributeType.STRING), + billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST, + removal_policy=RemovalPolicy.DESTROY, + time_to_live_attribute="expiration", + point_in_time_recovery=True, + ) + self.idempotency_table.grant(lambda_role, "dynamodb:PutItem") + self.idempotency_table.grant(lambda_role, "dynamodb:GetItem") + self.idempotency_table.grant(lambda_role, "dynamodb:UpdateItem") + self.idempotency_table.grant(lambda_role, "dynamodb:DeleteItem") diff --git a/examples/idempotency/sam.yaml b/examples/idempotency/sam.yaml new file mode 100644 index 00000000000..e69de29bb2d From c4bb07b329ada99299bf52057402da1bbbc080ac Mon Sep 17 00:00:00 2001 From: Ruben Fonseca Date: Thu, 15 Jun 2023 15:48:15 +0200 Subject: [PATCH 5/5] fix: cdk and sam samples --- docs/utilities/idempotency.md | 4 ++-- examples/idempotency/cdk.py | 11 +++++------ examples/idempotency/sam.yaml | 31 +++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 7335860e430..a0f42b4f304 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -79,12 +79,12 @@ If you're not [changing the default configuration for the DynamoDB persistence l === "sam.yaml" - ```yaml hl_lines="5-13 21-23" title="AWS Serverless Application Model (SAM) example" + ```yaml hl_lines="6-14 24-31" title="AWS Serverless Application Model (SAM) example" --8<-- "examples/idempotency/sam.yaml" ``` === "cdk.py" - ```python hl_lines="14 17 20-23" title="AWS Cloud Development Kit (CDK) Construct example" + ```python hl_lines="10 13 16 19-21" title="AWS Cloud Development Kit (CDK) Construct example" --8<-- "examples/idempotency/cdk.py" ``` diff --git a/examples/idempotency/cdk.py b/examples/idempotency/cdk.py index 2b856ce384b..4b22656defe 100644 --- a/examples/idempotency/cdk.py +++ b/examples/idempotency/cdk.py @@ -5,8 +5,8 @@ class IdempotencyConstruct(Construct): - def __init__(self, scope: Construct, id_: str, lambda_role: iam.Role) -> None: - super().__init__(scope, id_) + def __init__(self, scope: Construct, name: str, lambda_role: iam.Role) -> None: + super().__init__(scope, name) self.idempotency_table = dynamodb.Table( self, "IdempotencyTable", @@ -16,7 +16,6 @@ def __init__(self, scope: Construct, id_: str, lambda_role: iam.Role) -> None: time_to_live_attribute="expiration", point_in_time_recovery=True, ) - self.idempotency_table.grant(lambda_role, "dynamodb:PutItem") - self.idempotency_table.grant(lambda_role, "dynamodb:GetItem") - self.idempotency_table.grant(lambda_role, "dynamodb:UpdateItem") - self.idempotency_table.grant(lambda_role, "dynamodb:DeleteItem") + self.idempotency_table.grant( + lambda_role, "dynamodb:PutItem", "dynamodb:GetItem", "dynamodb:UpdateItem", "dynamodb:DeleteItem" + ) diff --git a/examples/idempotency/sam.yaml b/examples/idempotency/sam.yaml index e69de29bb2d..ee9b7540de9 100644 --- a/examples/idempotency/sam.yaml +++ b/examples/idempotency/sam.yaml @@ -0,0 +1,31 @@ +Transform: AWS::Serverless-2016-10-31 +Resources: + IdempotencyTable: + Type: AWS::DynamoDB::Table + Properties: + AttributeDefinitions: + - AttributeName: id + AttributeType: S + KeySchema: + - AttributeName: id + KeyType: HASH + TimeToLiveSpecification: + AttributeName: expiration + Enabled: true + BillingMode: PAY_PER_REQUEST + + HelloWorldFunction: + Type: AWS::Serverless::Function + Properties: + Runtime: python3.9 + Handler: app.py + Policies: + - Statement: + - Sid: AllowDynamodbReadWrite + Effect: Allow + Action: + - dynamodb:PutItem + - dynamodb:GetItem + - dynamodb:UpdateItem + - dynamodb:DeleteItem + Resource: !GetAtt IdempotencyTable.Arn