From 9657467f0d9825d8cac4d42fdde4ed270a7c81c5 Mon Sep 17 00:00:00 2001 From: Simon Thulbourn Date: Wed, 31 Jul 2024 22:37:08 +0200 Subject: [PATCH 1/5] fix(utilities): DDB Large numbers Signed-off-by: Simon Thulbourn --- .../utilities/data_classes/dynamo_db_stream_event.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py b/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py index 7339ed33fce..f00e156bf95 100644 --- a/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py +++ b/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py @@ -73,6 +73,10 @@ def _deserialize_bool(self, value: bool) -> bool: return value def _deserialize_n(self, value: str) -> Decimal: + if len(value) > 38: + l = len(value[38:]) - len(value[38:].rstrip('0')) + value = value[:-l] + return DYNAMODB_CONTEXT.create_decimal(value) def _deserialize_s(self, value: str) -> str: From 04b4cd23c00248aafcf6e9c9a53f267539364a98 Mon Sep 17 00:00:00 2001 From: Simon Thulbourn Date: Wed, 31 Jul 2024 20:46:52 +0000 Subject: [PATCH 2/5] rename var --- .../utilities/data_classes/dynamo_db_stream_event.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py b/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py index f00e156bf95..3e264c302d7 100644 --- a/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py +++ b/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py @@ -74,8 +74,8 @@ def _deserialize_bool(self, value: bool) -> bool: def _deserialize_n(self, value: str) -> Decimal: if len(value) > 38: - l = len(value[38:]) - len(value[38:].rstrip('0')) - value = value[:-l] + tail = len(value[38:]) - len(value[38:].rstrip("0")) + value = value[:-tail] return DYNAMODB_CONTEXT.create_decimal(value) From 84cc78a42fa8b40cf0e7b7f05be7a7a7760f7ffa Mon Sep 17 00:00:00 2001 From: Simon Thulbourn Date: Wed, 31 Jul 2024 20:51:02 +0000 Subject: [PATCH 3/5] add unit test for large numbers --- .../test_dynamo_db_stream_event.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/unit/data_classes/required_dependencies/test_dynamo_db_stream_event.py b/tests/unit/data_classes/required_dependencies/test_dynamo_db_stream_event.py index f7672abd69b..8f2682444c5 100644 --- a/tests/unit/data_classes/required_dependencies/test_dynamo_db_stream_event.py +++ b/tests/unit/data_classes/required_dependencies/test_dynamo_db_stream_event.py @@ -44,6 +44,27 @@ def test_dynamodb_stream_trigger_event(): assert dynamodb.stream_view_type == StreamViewType.NEW_AND_OLD_IMAGES +def test_dynamodb_stream_record_deserialization_large_int(): + decimal_context = Context( + Emin=-128, + Emax=126, + prec=38, + traps=[Clamped, Overflow, Inexact, Rounded, Underflow], + ) + data = { + "Keys": {"key1": {"attr1": "value1"}}, + "NewImage": { + "Name": {"S": "Joe"}, + "Age": {"N": "11011111111111111000000000000000000000000000000"}, + }, + } + record = StreamRecord(data) + assert record.new_image == { + "Name": "Joe", + "Age": decimal_context.create_decimal("11011111111111111000000000000000000000"), + } + + def test_dynamodb_stream_record_deserialization(): byte_list = [s.encode("utf-8") for s in ["item1", "item2"]] decimal_context = Context( From 7a6bb97915742951e7ec6312edff14c47e15546f Mon Sep 17 00:00:00 2001 From: Simon Thulbourn Date: Thu, 1 Aug 2024 08:58:47 +0000 Subject: [PATCH 4/5] remove leading 0s too --- .../utilities/data_classes/dynamo_db_stream_event.py | 1 + .../required_dependencies/test_dynamo_db_stream_event.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py b/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py index 3e264c302d7..d41c7a24309 100644 --- a/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py +++ b/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py @@ -73,6 +73,7 @@ def _deserialize_bool(self, value: bool) -> bool: return value def _deserialize_n(self, value: str) -> Decimal: + value = value.lstrip("0") if len(value) > 38: tail = len(value[38:]) - len(value[38:].rstrip("0")) value = value[:-tail] diff --git a/tests/unit/data_classes/required_dependencies/test_dynamo_db_stream_event.py b/tests/unit/data_classes/required_dependencies/test_dynamo_db_stream_event.py index 8f2682444c5..9bc3473501d 100644 --- a/tests/unit/data_classes/required_dependencies/test_dynamo_db_stream_event.py +++ b/tests/unit/data_classes/required_dependencies/test_dynamo_db_stream_event.py @@ -55,7 +55,7 @@ def test_dynamodb_stream_record_deserialization_large_int(): "Keys": {"key1": {"attr1": "value1"}}, "NewImage": { "Name": {"S": "Joe"}, - "Age": {"N": "11011111111111111000000000000000000000000000000"}, + "Age": {"N": "000000011011111111111111000000000000000000000000000000"}, }, } record = StreamRecord(data) From 20fcdfe51993df70f01a23dca1b3a1af5631945a Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Fri, 2 Aug 2024 11:04:56 +0100 Subject: [PATCH 5/5] Small refactor --- .../data_classes/dynamo_db_stream_event.py | 5 ++- .../test_dynamo_db_stream_event.py | 38 ++++++++++++------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py b/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py index d41c7a24309..0fedf55dc25 100644 --- a/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py +++ b/aws_lambda_powertools/utilities/data_classes/dynamo_db_stream_event.py @@ -75,8 +75,11 @@ def _deserialize_bool(self, value: bool) -> bool: def _deserialize_n(self, value: str) -> Decimal: value = value.lstrip("0") if len(value) > 38: + # See: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.NamingRulesDataTypes.html#HowItWorks.DataTypes.Number + # Calculate the number of trailing zeros after the 38th character tail = len(value[38:]) - len(value[38:].rstrip("0")) - value = value[:-tail] + # Trim the value: remove trailing zeros if any, or just take the first 38 characters + value = value[:-tail] if tail > 0 else value[:38] return DYNAMODB_CONTEXT.create_decimal(value) diff --git a/tests/unit/data_classes/required_dependencies/test_dynamo_db_stream_event.py b/tests/unit/data_classes/required_dependencies/test_dynamo_db_stream_event.py index 9bc3473501d..464d9a0eba1 100644 --- a/tests/unit/data_classes/required_dependencies/test_dynamo_db_stream_event.py +++ b/tests/unit/data_classes/required_dependencies/test_dynamo_db_stream_event.py @@ -8,14 +8,15 @@ ) from tests.functional.utils import load_event +DECIMAL_CONTEXT = Context( + Emin=-128, + Emax=126, + prec=38, + traps=[Clamped, Overflow, Inexact, Rounded, Underflow], +) + def test_dynamodb_stream_trigger_event(): - decimal_context = Context( - Emin=-128, - Emax=126, - prec=38, - traps=[Clamped, Overflow, Inexact, Rounded, Underflow], - ) raw_event = load_event("dynamoStreamEvent.json") parsed_event = DynamoDBStreamEvent(raw_event) @@ -36,7 +37,7 @@ def test_dynamodb_stream_trigger_event(): assert dynamodb.approximate_creation_date_time == record_raw["dynamodb"]["ApproximateCreationDateTime"] keys = dynamodb.keys assert keys is not None - assert keys["Id"] == decimal_context.create_decimal(101) + assert keys["Id"] == DECIMAL_CONTEXT.create_decimal(101) assert dynamodb.new_image.get("Message") == record_raw["dynamodb"]["NewImage"]["Message"]["S"] assert dynamodb.old_image is None assert dynamodb.sequence_number == record_raw["dynamodb"]["SequenceNumber"] @@ -45,12 +46,6 @@ def test_dynamodb_stream_trigger_event(): def test_dynamodb_stream_record_deserialization_large_int(): - decimal_context = Context( - Emin=-128, - Emax=126, - prec=38, - traps=[Clamped, Overflow, Inexact, Rounded, Underflow], - ) data = { "Keys": {"key1": {"attr1": "value1"}}, "NewImage": { @@ -61,7 +56,22 @@ def test_dynamodb_stream_record_deserialization_large_int(): record = StreamRecord(data) assert record.new_image == { "Name": "Joe", - "Age": decimal_context.create_decimal("11011111111111111000000000000000000000"), + "Age": DECIMAL_CONTEXT.create_decimal("11011111111111111000000000000000000000"), + } + + +def test_dynamodb_stream_record_deserialization_large_int_without_trailing_zeros(): + data = { + "Keys": {"key1": {"attr1": "value1"}}, + "NewImage": { + "Name": {"S": "Joe"}, + "Age": {"N": "000000011011111111111112222222222221111111111111111111111"}, + }, + } + record = StreamRecord(data) + assert record.new_image == { + "Name": "Joe", + "Age": DECIMAL_CONTEXT.create_decimal("11011111111111112222222222221111111111"), }