From 8539002610138288acba71b8ec1dd65ac670f44d Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sun, 4 Oct 2020 18:12:35 +0200 Subject: [PATCH 01/22] fix: comment out validators #118 --- .../advanced_parser/schemas/dynamodb.py | 17 ++++---- .../utilities/advanced_parser/schemas/sqs.py | 42 +++++++++---------- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/aws_lambda_powertools/utilities/advanced_parser/schemas/dynamodb.py b/aws_lambda_powertools/utilities/advanced_parser/schemas/dynamodb.py index 0c4e95fc9bc..3dc8b021fa9 100644 --- a/aws_lambda_powertools/utilities/advanced_parser/schemas/dynamodb.py +++ b/aws_lambda_powertools/utilities/advanced_parser/schemas/dynamodb.py @@ -1,7 +1,7 @@ from datetime import date from typing import Any, Dict, List, Optional -from pydantic import BaseModel, root_validator +from pydantic import BaseModel from typing_extensions import Literal @@ -14,15 +14,16 @@ class DynamoScheme(BaseModel): SizeBytes: int StreamViewType: Literal["NEW_AND_OLD_IMAGES", "KEYS_ONLY", "NEW_IMAGE", "OLD_IMAGE"] + # context on why it's commented: https://github.com/awslabs/aws-lambda-powertools-python/pull/118 # since both images are optional, they can both be None. However, at least one must # exist in a legal schema of NEW_AND_OLD_IMAGES type - @root_validator - def check_one_image_exists(cls, values): - new_img, old_img = values.get("NewImage"), values.get("OldImage") - stream_type = values.get("StreamViewType") - if stream_type == "NEW_AND_OLD_IMAGES" and not new_img and not old_img: - raise TypeError("DynamoDB streams schema failed validation, missing both new & old stream images") - return values + # @root_validator + # def check_one_image_exists(cls, values): # noqa: E800 + # new_img, old_img = values.get("NewImage"), values.get("OldImage") # noqa: E800 + # stream_type = values.get("StreamViewType") # noqa: E800 + # if stream_type == "NEW_AND_OLD_IMAGES" and not new_img and not old_img: # noqa: E800 + # raise TypeError("DynamoDB streams schema failed validation, missing both new & old stream images") # noqa: E800,E501 + # return values # noqa: E800 class UserIdentity(BaseModel): diff --git a/aws_lambda_powertools/utilities/advanced_parser/schemas/sqs.py b/aws_lambda_powertools/utilities/advanced_parser/schemas/sqs.py index 862236281f2..efb92ad5345 100644 --- a/aws_lambda_powertools/utilities/advanced_parser/schemas/sqs.py +++ b/aws_lambda_powertools/utilities/advanced_parser/schemas/sqs.py @@ -1,8 +1,7 @@ -import re from datetime import datetime from typing import Dict, List, Optional -from pydantic import BaseModel, root_validator, validator +from pydantic import BaseModel from typing_extensions import Literal @@ -24,28 +23,29 @@ class SqsMsgAttributeSchema(BaseModel): binaryListValues: List[str] = [] dataType: str + # context on why it's commented: https://github.com/awslabs/aws-lambda-powertools-python/pull/118 # Amazon SQS supports the logical data types String, Number, and Binary with optional custom data type # labels with the format .custom-data-type. # https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html#sqs-message-attributes - @validator("dataType") - def valid_type(cls, v): # noqa: VNE001 - pattern = re.compile("Number.*|String.*|Binary.*") - if not pattern.match(v): - raise TypeError("data type is invalid") - return v - - # validate that dataType and value are not None and match - @root_validator - def check_str_and_binary_values(cls, values): - binary_val, str_val = values.get("binaryValue", ""), values.get("stringValue", "") - data_type = values.get("dataType") - if not str_val and not binary_val: - raise TypeError("both binaryValue and stringValue are missing") - if data_type.startswith("Binary") and not binary_val: - raise TypeError("binaryValue is missing") - if (data_type.startswith("String") or data_type.startswith("Number")) and not str_val: - raise TypeError("stringValue is missing") - return values + # @validator("dataType") + # def valid_type(cls, v): # noqa: VNE001,E800 # noqa: E800 + # pattern = re.compile("Number.*|String.*|Binary.*") # noqa: E800 + # if not pattern.match(v): # noqa: E800 + # raise TypeError("data type is invalid") # noqa: E800 + # return v # noqa: E800 + # + # # validate that dataType and value are not None and match + # @root_validator + # def check_str_and_binary_values(cls, values): # noqa: E800 + # binary_val, str_val = values.get("binaryValue", ""), values.get("stringValue", "") # noqa: E800 + # data_type = values.get("dataType") # noqa: E800 + # if not str_val and not binary_val: # noqa: E800 + # raise TypeError("both binaryValue and stringValue are missing") # noqa: E800 + # if data_type.startswith("Binary") and not binary_val: # noqa: E800 + # raise TypeError("binaryValue is missing") # noqa: E800 + # if (data_type.startswith("String") or data_type.startswith("Number")) and not str_val: # noqa: E800 + # raise TypeError("stringValue is missing") # noqa: E800 + # return values # noqa: E800 class SqsRecordSchema(BaseModel): From df854d315d79d29a9f2622e129a0d134aaf67243 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sun, 4 Oct 2020 18:45:25 +0200 Subject: [PATCH 02/22] improv: rename advanced_parser to parser --- .../utilities/{advanced_parser => parser}/__init__.py | 0 .../{advanced_parser => parser}/envelopes/__init__.py | 0 .../{advanced_parser => parser}/envelopes/base.py | 0 .../{advanced_parser => parser}/envelopes/dynamodb.py | 4 ++-- .../{advanced_parser => parser}/envelopes/envelopes.py | 8 ++++---- .../envelopes/event_bridge.py | 4 ++-- .../{advanced_parser => parser}/envelopes/sqs.py | 4 ++-- .../utilities/{advanced_parser => parser}/parser.py | 10 +++++----- .../{advanced_parser => parser}/schemas/__init__.py | 0 .../{advanced_parser => parser}/schemas/dynamodb.py | 0 .../schemas/event_bridge.py | 0 .../{advanced_parser => parser}/schemas/sqs.py | 0 tests/functional/parser/schemas.py | 2 +- tests/functional/parser/test_dynamodb.py | 4 ++-- tests/functional/parser/test_eventbridge.py | 4 ++-- tests/functional/parser/test_sqs.py | 4 ++-- 16 files changed, 22 insertions(+), 22 deletions(-) rename aws_lambda_powertools/utilities/{advanced_parser => parser}/__init__.py (100%) rename aws_lambda_powertools/utilities/{advanced_parser => parser}/envelopes/__init__.py (100%) rename aws_lambda_powertools/utilities/{advanced_parser => parser}/envelopes/base.py (100%) rename aws_lambda_powertools/utilities/{advanced_parser => parser}/envelopes/dynamodb.py (87%) rename aws_lambda_powertools/utilities/{advanced_parser => parser}/envelopes/envelopes.py (74%) rename aws_lambda_powertools/utilities/{advanced_parser => parser}/envelopes/event_bridge.py (77%) rename aws_lambda_powertools/utilities/{advanced_parser => parser}/envelopes/sqs.py (86%) rename aws_lambda_powertools/utilities/{advanced_parser => parser}/parser.py (90%) rename aws_lambda_powertools/utilities/{advanced_parser => parser}/schemas/__init__.py (100%) rename aws_lambda_powertools/utilities/{advanced_parser => parser}/schemas/dynamodb.py (100%) rename aws_lambda_powertools/utilities/{advanced_parser => parser}/schemas/event_bridge.py (100%) rename aws_lambda_powertools/utilities/{advanced_parser => parser}/schemas/sqs.py (100%) diff --git a/aws_lambda_powertools/utilities/advanced_parser/__init__.py b/aws_lambda_powertools/utilities/parser/__init__.py similarity index 100% rename from aws_lambda_powertools/utilities/advanced_parser/__init__.py rename to aws_lambda_powertools/utilities/parser/__init__.py diff --git a/aws_lambda_powertools/utilities/advanced_parser/envelopes/__init__.py b/aws_lambda_powertools/utilities/parser/envelopes/__init__.py similarity index 100% rename from aws_lambda_powertools/utilities/advanced_parser/envelopes/__init__.py rename to aws_lambda_powertools/utilities/parser/envelopes/__init__.py diff --git a/aws_lambda_powertools/utilities/advanced_parser/envelopes/base.py b/aws_lambda_powertools/utilities/parser/envelopes/base.py similarity index 100% rename from aws_lambda_powertools/utilities/advanced_parser/envelopes/base.py rename to aws_lambda_powertools/utilities/parser/envelopes/base.py diff --git a/aws_lambda_powertools/utilities/advanced_parser/envelopes/dynamodb.py b/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py similarity index 87% rename from aws_lambda_powertools/utilities/advanced_parser/envelopes/dynamodb.py rename to aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py index 81b9de02315..961ad4b4a53 100644 --- a/aws_lambda_powertools/utilities/advanced_parser/envelopes/dynamodb.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py @@ -4,8 +4,8 @@ from pydantic import BaseModel, ValidationError from typing_extensions import Literal -from aws_lambda_powertools.utilities.advanced_parser.envelopes.base import BaseEnvelope -from aws_lambda_powertools.utilities.advanced_parser.schemas import DynamoDBSchema +from aws_lambda_powertools.utilities.parser.envelopes.base import BaseEnvelope +from aws_lambda_powertools.utilities.parser.schemas import DynamoDBSchema logger = logging.getLogger(__name__) diff --git a/aws_lambda_powertools/utilities/advanced_parser/envelopes/envelopes.py b/aws_lambda_powertools/utilities/parser/envelopes/envelopes.py similarity index 74% rename from aws_lambda_powertools/utilities/advanced_parser/envelopes/envelopes.py rename to aws_lambda_powertools/utilities/parser/envelopes/envelopes.py index 332c1eadef0..95771a4db16 100644 --- a/aws_lambda_powertools/utilities/advanced_parser/envelopes/envelopes.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/envelopes.py @@ -4,10 +4,10 @@ from pydantic import BaseModel -from aws_lambda_powertools.utilities.advanced_parser.envelopes.base import BaseEnvelope -from aws_lambda_powertools.utilities.advanced_parser.envelopes.dynamodb import DynamoDBEnvelope -from aws_lambda_powertools.utilities.advanced_parser.envelopes.event_bridge import EventBridgeEnvelope -from aws_lambda_powertools.utilities.advanced_parser.envelopes.sqs import SqsEnvelope +from aws_lambda_powertools.utilities.parser.envelopes.base import BaseEnvelope +from aws_lambda_powertools.utilities.parser.envelopes.dynamodb import DynamoDBEnvelope +from aws_lambda_powertools.utilities.parser.envelopes.event_bridge import EventBridgeEnvelope +from aws_lambda_powertools.utilities.parser.envelopes.sqs import SqsEnvelope logger = logging.getLogger(__name__) diff --git a/aws_lambda_powertools/utilities/advanced_parser/envelopes/event_bridge.py b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py similarity index 77% rename from aws_lambda_powertools/utilities/advanced_parser/envelopes/event_bridge.py rename to aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py index 00052d41da0..41e192a8140 100644 --- a/aws_lambda_powertools/utilities/advanced_parser/envelopes/event_bridge.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py @@ -3,8 +3,8 @@ from pydantic import BaseModel, ValidationError -from aws_lambda_powertools.utilities.advanced_parser.envelopes.base import BaseEnvelope -from aws_lambda_powertools.utilities.advanced_parser.schemas import EventBridgeSchema +from aws_lambda_powertools.utilities.parser.envelopes.base import BaseEnvelope +from aws_lambda_powertools.utilities.parser.schemas import EventBridgeSchema logger = logging.getLogger(__name__) diff --git a/aws_lambda_powertools/utilities/advanced_parser/envelopes/sqs.py b/aws_lambda_powertools/utilities/parser/envelopes/sqs.py similarity index 86% rename from aws_lambda_powertools/utilities/advanced_parser/envelopes/sqs.py rename to aws_lambda_powertools/utilities/parser/envelopes/sqs.py index 8ef2e685c4f..e2ca9f9b039 100644 --- a/aws_lambda_powertools/utilities/advanced_parser/envelopes/sqs.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/sqs.py @@ -3,8 +3,8 @@ from pydantic import BaseModel, ValidationError -from aws_lambda_powertools.utilities.advanced_parser.envelopes.base import BaseEnvelope -from aws_lambda_powertools.utilities.advanced_parser.schemas import SqsSchema +from aws_lambda_powertools.utilities.parser.envelopes.base import BaseEnvelope +from aws_lambda_powertools.utilities.parser.schemas import SqsSchema logger = logging.getLogger(__name__) diff --git a/aws_lambda_powertools/utilities/advanced_parser/parser.py b/aws_lambda_powertools/utilities/parser/parser.py similarity index 90% rename from aws_lambda_powertools/utilities/advanced_parser/parser.py rename to aws_lambda_powertools/utilities/parser/parser.py index b501d0a5146..3683b116647 100644 --- a/aws_lambda_powertools/utilities/advanced_parser/parser.py +++ b/aws_lambda_powertools/utilities/parser/parser.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, ValidationError from aws_lambda_powertools.middleware_factory import lambda_handler_decorator -from aws_lambda_powertools.utilities.advanced_parser.envelopes import Envelope, parse_envelope +from aws_lambda_powertools.utilities.parser.envelopes import Envelope, parse_envelope logger = logging.getLogger(__name__) @@ -21,11 +21,11 @@ def parser( As Lambda follows (event, context) signature we can remove some of the boilerplate and also capture any exception any Lambda function throws as metadata. - event will be the parsed and passed as a BaseModel pydantic class of the input type "schema" + event will be the parsed and passed as a BaseModel pydantic class of the input type "schema" to the lambda handler. event will be extracted from the envelope in case envelope is not None. In case envelope is None, the complete event is parsed to match the schema parameter BaseModel definition. - In case envelope is not None, first the event is parsed as the envelope's schema definition, and the user + In case envelope is not None, first the event is parsed as the envelope's schema definition, and the user message is extracted and parsed again as the schema parameter's definition. Example @@ -41,9 +41,9 @@ def handler(event: MyBusiness , context: LambdaContext): handler: input for lambda_handler_decorator, wraps the handler lambda event: AWS event dictionary context: AWS lambda context - schema: pydantic BaseModel class. This is the user data schema that will replace the event. + schema: pydantic BaseModel class. This is the user data schema that will replace the event. event parameter will be parsed and a new schema object will be created from it. - envelope: what envelope to extract the schema from, can be any AWS service that is currently + envelope: what envelope to extract the schema from, can be any AWS service that is currently supported in the envelopes module. Can be None. Raises diff --git a/aws_lambda_powertools/utilities/advanced_parser/schemas/__init__.py b/aws_lambda_powertools/utilities/parser/schemas/__init__.py similarity index 100% rename from aws_lambda_powertools/utilities/advanced_parser/schemas/__init__.py rename to aws_lambda_powertools/utilities/parser/schemas/__init__.py diff --git a/aws_lambda_powertools/utilities/advanced_parser/schemas/dynamodb.py b/aws_lambda_powertools/utilities/parser/schemas/dynamodb.py similarity index 100% rename from aws_lambda_powertools/utilities/advanced_parser/schemas/dynamodb.py rename to aws_lambda_powertools/utilities/parser/schemas/dynamodb.py diff --git a/aws_lambda_powertools/utilities/advanced_parser/schemas/event_bridge.py b/aws_lambda_powertools/utilities/parser/schemas/event_bridge.py similarity index 100% rename from aws_lambda_powertools/utilities/advanced_parser/schemas/event_bridge.py rename to aws_lambda_powertools/utilities/parser/schemas/event_bridge.py diff --git a/aws_lambda_powertools/utilities/advanced_parser/schemas/sqs.py b/aws_lambda_powertools/utilities/parser/schemas/sqs.py similarity index 100% rename from aws_lambda_powertools/utilities/advanced_parser/schemas/sqs.py rename to aws_lambda_powertools/utilities/parser/schemas/sqs.py diff --git a/tests/functional/parser/schemas.py b/tests/functional/parser/schemas.py index 3667601e630..b0b61fe61d4 100644 --- a/tests/functional/parser/schemas.py +++ b/tests/functional/parser/schemas.py @@ -3,7 +3,7 @@ from pydantic import BaseModel from typing_extensions import Literal -from aws_lambda_powertools.utilities.advanced_parser.schemas import ( +from aws_lambda_powertools.utilities.parser.schemas import ( DynamoDBSchema, DynamoRecordSchema, DynamoScheme, diff --git a/tests/functional/parser/test_dynamodb.py b/tests/functional/parser/test_dynamodb.py index 42e22cb45e7..b79ba98cece 100644 --- a/tests/functional/parser/test_dynamodb.py +++ b/tests/functional/parser/test_dynamodb.py @@ -3,8 +3,8 @@ import pytest from pydantic.error_wrappers import ValidationError -from aws_lambda_powertools.utilities.advanced_parser.envelopes.envelopes import Envelope -from aws_lambda_powertools.utilities.advanced_parser.parser import parser +from aws_lambda_powertools.utilities.parser.envelopes.envelopes import Envelope +from aws_lambda_powertools.utilities.parser.parser import parser from aws_lambda_powertools.utilities.typing import LambdaContext from tests.functional.parser.schemas import MyAdvancedDynamoBusiness, MyDynamoBusiness from tests.functional.parser.utils import load_event diff --git a/tests/functional/parser/test_eventbridge.py b/tests/functional/parser/test_eventbridge.py index 92122605886..693244c7604 100644 --- a/tests/functional/parser/test_eventbridge.py +++ b/tests/functional/parser/test_eventbridge.py @@ -3,8 +3,8 @@ import pytest from pydantic import ValidationError -from aws_lambda_powertools.utilities.advanced_parser.envelopes.envelopes import Envelope -from aws_lambda_powertools.utilities.advanced_parser.parser import parser +from aws_lambda_powertools.utilities.parser.envelopes.envelopes import Envelope +from aws_lambda_powertools.utilities.parser.parser import parser from aws_lambda_powertools.utilities.typing import LambdaContext from tests.functional.parser.schemas import MyAdvancedEventbridgeBusiness, MyEventbridgeBusiness from tests.functional.parser.utils import load_event diff --git a/tests/functional/parser/test_sqs.py b/tests/functional/parser/test_sqs.py index da1363f758a..e7f59b8609a 100644 --- a/tests/functional/parser/test_sqs.py +++ b/tests/functional/parser/test_sqs.py @@ -3,8 +3,8 @@ import pytest from pydantic import ValidationError -from aws_lambda_powertools.utilities.advanced_parser.envelopes.envelopes import Envelope -from aws_lambda_powertools.utilities.advanced_parser.parser import parser +from aws_lambda_powertools.utilities.parser.envelopes.envelopes import Envelope +from aws_lambda_powertools.utilities.parser.parser import parser from aws_lambda_powertools.utilities.typing import LambdaContext from tests.functional.parser.schemas import MyAdvancedSqsBusiness, MySqsBusiness from tests.functional.parser.utils import load_event From 1e4a49c84d37f124c86600bf5c138c602d0b669c Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sun, 4 Oct 2020 18:47:42 +0200 Subject: [PATCH 03/22] chore: remove test for commented code --- tests/functional/parser/test_dynamodb.py | 28 ------------------------ 1 file changed, 28 deletions(-) diff --git a/tests/functional/parser/test_dynamodb.py b/tests/functional/parser/test_dynamodb.py index b79ba98cece..943ae0a310e 100644 --- a/tests/functional/parser/test_dynamodb.py +++ b/tests/functional/parser/test_dynamodb.py @@ -69,31 +69,3 @@ def test_validate_event_does_not_conform_with_schema(): event_dict: Any = {"hello": "s"} with pytest.raises(ValidationError): handle_dynamodb(event_dict, LambdaContext()) - - -def test_validate_event_neither_image_exists_with_schema(): - event_dict: Any = { - "Records": [ - { - "eventID": "1", - "eventName": "INSERT", - "eventVersion": "1.0", - "eventSourceARN": "eventsource_arn", - "awsRegion": "us-west-2", - "eventSource": "aws:dynamodb", - "dynamodb": { - "StreamViewType": "NEW_AND_OLD_IMAGES", - "SequenceNumber": "111", - "SizeBytes": 26, - "Keys": {"Id": {"N": "101"}}, - }, - } - ] - } - with pytest.raises(ValidationError) as exc_info: - handle_dynamodb(event_dict, LambdaContext()) - - validation_error: ValidationError = exc_info.value - assert len(validation_error.errors()) == 1 - error = validation_error.errors()[0] - assert error["msg"] == "DynamoDB streams schema failed validation, missing both new & old stream images" From 991bec7223d6f48296cf3440ec4958c0c56686a4 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sun, 4 Oct 2020 19:57:57 +0200 Subject: [PATCH 04/22] improv: envelope structure & import --- .../utilities/parser/__init__.py | 4 +- .../utilities/parser/envelopes/__init__.py | 8 +++- .../utilities/parser/envelopes/base.py | 10 +++++ .../utilities/parser/envelopes/dynamodb.py | 27 +++++++++--- .../utilities/parser/envelopes/envelopes.py | 42 ------------------- .../parser/envelopes/event_bridge.py | 17 +++++++- .../utilities/parser/envelopes/sqs.py | 32 ++++++++++---- .../utilities/parser/exceptions.py | 2 + .../utilities/parser/parser.py | 5 ++- tests/functional/parser/test_dynamodb.py | 4 +- tests/functional/parser/test_eventbridge.py | 5 +-- tests/functional/parser/test_sqs.py | 6 +-- 12 files changed, 93 insertions(+), 69 deletions(-) delete mode 100644 aws_lambda_powertools/utilities/parser/envelopes/envelopes.py create mode 100644 aws_lambda_powertools/utilities/parser/exceptions.py diff --git a/aws_lambda_powertools/utilities/parser/__init__.py b/aws_lambda_powertools/utilities/parser/__init__.py index 017b5086bb0..be959623ae9 100644 --- a/aws_lambda_powertools/utilities/parser/__init__.py +++ b/aws_lambda_powertools/utilities/parser/__init__.py @@ -1,6 +1,6 @@ """Advanced parser utility """ -from .envelopes import Envelope, InvalidEnvelopeError, parse_envelope +from . import envelopes from .parser import parser -__all__ = ["InvalidEnvelopeError", "Envelope", "parse_envelope", "parser"] +__all__ = ["envelopes", "parser"] diff --git a/aws_lambda_powertools/utilities/parser/envelopes/__init__.py b/aws_lambda_powertools/utilities/parser/envelopes/__init__.py index 5fa4c396ba1..bb7e4761aa9 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/__init__.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/__init__.py @@ -1,3 +1,7 @@ -from .envelopes import Envelope, InvalidEnvelopeError, parse_envelope +from .dynamodb import DynamoDBEnvelope +from .event_bridge import EventBridgeEnvelope +from .sqs import SqsEnvelope -__all__ = ["InvalidEnvelopeError", "Envelope", "parse_envelope"] +SQS = SqsEnvelope +DYNAMODB_STREAM = DynamoDBEnvelope +EVENTBRIDGE = EventBridgeEnvelope diff --git a/aws_lambda_powertools/utilities/parser/envelopes/base.py b/aws_lambda_powertools/utilities/parser/envelopes/base.py index d972bfb3872..12dc0555da2 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/base.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/base.py @@ -4,6 +4,8 @@ from pydantic import BaseModel, ValidationError +from ..exceptions import InvalidEnvelopeError + logger = logging.getLogger(__name__) @@ -38,3 +40,11 @@ def _parse_user_json_string_schema(user_event: str, schema: BaseModel) -> Any: @abstractmethod def parse(self, event: Dict[str, Any], schema: BaseModel): return NotImplemented + + +def parse_envelope(event: Dict[str, Any], envelope: BaseEnvelope, schema: BaseModel): + if not callable(envelope) and not isinstance(BaseEnvelope): + logger.exception("envelope must be a callable and instance of BaseEnvelope") + raise InvalidEnvelopeError("envelope must be a callable and instance of BaseEnvelope") + logger.debug(f"Parsing and validating event schema, envelope={envelope}") + return envelope().parse(event=event, schema=schema) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py b/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py index 961ad4b4a53..1ceed8d37b6 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py @@ -4,17 +4,34 @@ from pydantic import BaseModel, ValidationError from typing_extensions import Literal -from aws_lambda_powertools.utilities.parser.envelopes.base import BaseEnvelope -from aws_lambda_powertools.utilities.parser.schemas import DynamoDBSchema +from ..schemas import DynamoDBSchema +from .base import BaseEnvelope logger = logging.getLogger(__name__) -# returns a List of dictionaries which each contains two keys, "NewImage" and "OldImage". -# The values are the parsed schema models. The images' values can also be None. -# Length of the list is the record's amount in the original event. class DynamoDBEnvelope(BaseEnvelope): + """ DynamoDB Stream Envelope to extract data within NewImage/OldImage + + Note: Values are the parsed schema models. Images' values can also be None, and + length of the list is the record's amount in the original event. + """ + def parse(self, event: Dict[str, Any], schema: BaseModel) -> List[Dict[Literal["NewImage", "OldImage"], BaseModel]]: + """Parses DynamoDB Stream records found in either NewImage and OldImage with schema provided + + Parameters + ---------- + event : Dict + Lambda event to be parsed + schema : BaseModel + User schema provided to parse after extracting data using envelope + + Returns + ------- + List + List of records parsed with schema provided + """ try: parsed_envelope = DynamoDBSchema(**event) except (ValidationError, TypeError): diff --git a/aws_lambda_powertools/utilities/parser/envelopes/envelopes.py b/aws_lambda_powertools/utilities/parser/envelopes/envelopes.py deleted file mode 100644 index 95771a4db16..00000000000 --- a/aws_lambda_powertools/utilities/parser/envelopes/envelopes.py +++ /dev/null @@ -1,42 +0,0 @@ -import logging -from enum import Enum -from typing import Any, Dict - -from pydantic import BaseModel - -from aws_lambda_powertools.utilities.parser.envelopes.base import BaseEnvelope -from aws_lambda_powertools.utilities.parser.envelopes.dynamodb import DynamoDBEnvelope -from aws_lambda_powertools.utilities.parser.envelopes.event_bridge import EventBridgeEnvelope -from aws_lambda_powertools.utilities.parser.envelopes.sqs import SqsEnvelope - -logger = logging.getLogger(__name__) - - -"""Built-in envelopes""" - - -class Envelope(str, Enum): - SQS = "sqs" - EVENTBRIDGE = "eventbridge" - DYNAMODB_STREAM = "dynamodb_stream" - - -class InvalidEnvelopeError(Exception): - """Input envelope is not one of the Envelope enum values""" - - -# enum to BaseEnvelope handler class -__ENVELOPE_MAPPING = { - Envelope.SQS: SqsEnvelope, - Envelope.DYNAMODB_STREAM: DynamoDBEnvelope, - Envelope.EVENTBRIDGE: EventBridgeEnvelope, -} - - -def parse_envelope(event: Dict[str, Any], envelope: Envelope, schema: BaseModel): - envelope_handler: BaseEnvelope = __ENVELOPE_MAPPING.get(envelope) - if envelope_handler is None: - logger.exception("envelope must be an instance of Envelope enum") - raise InvalidEnvelopeError("envelope must be an instance of Envelope enum") - logger.debug(f"Parsing and validating event schema, envelope={str(envelope.value)}") - return envelope_handler().parse(event=event, schema=schema) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py index 41e192a8140..8e3c0e68c31 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py @@ -9,9 +9,24 @@ logger = logging.getLogger(__name__) -# returns a parsed BaseModel object according to schema type class EventBridgeEnvelope(BaseEnvelope): + """EventBridge envelope to extract data in detail key""" + def parse(self, event: Dict[str, Any], schema: BaseModel) -> BaseModel: + """Parses data found with schema provided + + Parameters + ---------- + event : Dict + Lambda event to be parsed + schema : BaseModel + User schema provided to parse after extracting data using envelope + + Returns + ------- + Any + Parsed detail payload with schema provided + """ try: parsed_envelope = EventBridgeSchema(**event) except (ValidationError, TypeError): diff --git a/aws_lambda_powertools/utilities/parser/envelopes/sqs.py b/aws_lambda_powertools/utilities/parser/envelopes/sqs.py index e2ca9f9b039..951426f860d 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/sqs.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/sqs.py @@ -3,19 +3,37 @@ from pydantic import BaseModel, ValidationError -from aws_lambda_powertools.utilities.parser.envelopes.base import BaseEnvelope -from aws_lambda_powertools.utilities.parser.schemas import SqsSchema +from ..schemas import SqsSchema +from .base import BaseEnvelope logger = logging.getLogger(__name__) -# returns a list of parsed schemas of type BaseModel or plain string. -# The record's body parameter is a string. However, it can also be a JSON encoded string which -# can then be parsed into a BaseModel object. -# Note that all records will be parsed the same way so if schema is str, -# all the items in the list will be parsed as str and npt as JSON (and vice versa). class SqsEnvelope(BaseEnvelope): + """SQS Envelope to extract array of Records + + The record's body parameter is a string, though it can also be a JSON encoded string. + Regardless of it's type it'll be parsed into a BaseModel object. + + Note: Records will be parsed the same way so if schema is str, + all items in the list will be parsed as str and npt as JSON (and vice versa) + """ + def parse(self, event: Dict[str, Any], schema: Union[BaseModel, str]) -> List[Union[BaseModel, str]]: + """Parses records found with schema provided + + Parameters + ---------- + event : Dict + Lambda event to be parsed + schema : BaseModel + User schema provided to parse after extracting data using envelope + + Returns + ------- + List + List of records parsed with schema provided + """ try: parsed_envelope = SqsSchema(**event) except (ValidationError, TypeError): diff --git a/aws_lambda_powertools/utilities/parser/exceptions.py b/aws_lambda_powertools/utilities/parser/exceptions.py new file mode 100644 index 00000000000..8a08ce23154 --- /dev/null +++ b/aws_lambda_powertools/utilities/parser/exceptions.py @@ -0,0 +1,2 @@ +class InvalidEnvelopeError(Exception): + """Input envelope is not one of the Envelope enum values""" diff --git a/aws_lambda_powertools/utilities/parser/parser.py b/aws_lambda_powertools/utilities/parser/parser.py index 3683b116647..6a29692df6d 100644 --- a/aws_lambda_powertools/utilities/parser/parser.py +++ b/aws_lambda_powertools/utilities/parser/parser.py @@ -4,7 +4,8 @@ from pydantic import BaseModel, ValidationError from aws_lambda_powertools.middleware_factory import lambda_handler_decorator -from aws_lambda_powertools.utilities.parser.envelopes import Envelope, parse_envelope + +from .envelopes.base import BaseEnvelope, parse_envelope logger = logging.getLogger(__name__) @@ -15,7 +16,7 @@ def parser( event: Dict[str, Any], context: Dict[str, Any], schema: BaseModel, - envelope: Optional[Envelope] = None, + envelope: Optional[BaseEnvelope] = None, ) -> Any: """Decorator to conduct advanced parsing & validation for lambda handlers events diff --git a/tests/functional/parser/test_dynamodb.py b/tests/functional/parser/test_dynamodb.py index 943ae0a310e..4d20dab12a0 100644 --- a/tests/functional/parser/test_dynamodb.py +++ b/tests/functional/parser/test_dynamodb.py @@ -3,14 +3,14 @@ import pytest from pydantic.error_wrappers import ValidationError -from aws_lambda_powertools.utilities.parser.envelopes.envelopes import Envelope +from aws_lambda_powertools.utilities.parser import envelopes from aws_lambda_powertools.utilities.parser.parser import parser from aws_lambda_powertools.utilities.typing import LambdaContext from tests.functional.parser.schemas import MyAdvancedDynamoBusiness, MyDynamoBusiness from tests.functional.parser.utils import load_event -@parser(schema=MyDynamoBusiness, envelope=Envelope.DYNAMODB_STREAM) +@parser(schema=MyDynamoBusiness, envelope=envelopes.DYNAMODB_STREAM) def handle_dynamodb(event: List[Dict[str, MyDynamoBusiness]], _: LambdaContext): assert len(event) == 2 assert event[0]["OldImage"] is None diff --git a/tests/functional/parser/test_eventbridge.py b/tests/functional/parser/test_eventbridge.py index 693244c7604..0de57a0de44 100644 --- a/tests/functional/parser/test_eventbridge.py +++ b/tests/functional/parser/test_eventbridge.py @@ -3,14 +3,13 @@ import pytest from pydantic import ValidationError -from aws_lambda_powertools.utilities.parser.envelopes.envelopes import Envelope -from aws_lambda_powertools.utilities.parser.parser import parser +from aws_lambda_powertools.utilities.parser import envelopes, parser from aws_lambda_powertools.utilities.typing import LambdaContext from tests.functional.parser.schemas import MyAdvancedEventbridgeBusiness, MyEventbridgeBusiness from tests.functional.parser.utils import load_event -@parser(schema=MyEventbridgeBusiness, envelope=Envelope.EVENTBRIDGE) +@parser(schema=MyEventbridgeBusiness, envelope=envelopes.EVENTBRIDGE) def handle_eventbridge(event: MyEventbridgeBusiness, _: LambdaContext): assert event.instance_id == "i-1234567890abcdef0" assert event.state == "terminated" diff --git a/tests/functional/parser/test_sqs.py b/tests/functional/parser/test_sqs.py index e7f59b8609a..e8fccfbff04 100644 --- a/tests/functional/parser/test_sqs.py +++ b/tests/functional/parser/test_sqs.py @@ -3,7 +3,7 @@ import pytest from pydantic import ValidationError -from aws_lambda_powertools.utilities.parser.envelopes.envelopes import Envelope +from aws_lambda_powertools.utilities.parser import envelopes from aws_lambda_powertools.utilities.parser.parser import parser from aws_lambda_powertools.utilities.typing import LambdaContext from tests.functional.parser.schemas import MyAdvancedSqsBusiness, MySqsBusiness @@ -11,7 +11,7 @@ from tests.functional.validator.conftest import sqs_event # noqa: F401 -@parser(schema=str, envelope=Envelope.SQS) +@parser(schema=str, envelope=envelopes.SQS) def handle_sqs_str_body(event: List[str], _: LambdaContext): assert len(event) == 2 assert event[0] == "Test message." @@ -23,7 +23,7 @@ def test_handle_sqs_trigger_event_str_body(): handle_sqs_str_body(event_dict, LambdaContext()) -@parser(schema=MySqsBusiness, envelope=Envelope.SQS) +@parser(schema=MySqsBusiness, envelope=envelopes.SQS) def handle_sqs_json_body(event: List[MySqsBusiness], _: LambdaContext): assert len(event) == 1 assert event[0].message == "hello world" From c7f242a5d0b52600ad8336450c61da644296e588 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sun, 4 Oct 2020 20:04:41 +0200 Subject: [PATCH 05/22] fix: snake_case --- aws_lambda_powertools/utilities/parser/schemas/event_bridge.py | 2 +- tests/functional/parser/test_eventbridge.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_lambda_powertools/utilities/parser/schemas/event_bridge.py b/aws_lambda_powertools/utilities/parser/schemas/event_bridge.py index c5e319ac28e..40b00848047 100644 --- a/aws_lambda_powertools/utilities/parser/schemas/event_bridge.py +++ b/aws_lambda_powertools/utilities/parser/schemas/event_bridge.py @@ -12,5 +12,5 @@ class EventBridgeSchema(BaseModel): time: datetime region: str resources: List[str] - detailtype: str = Field(None, alias="detail-type") + detail_type: str = Field(None, alias="detail-type") detail: Dict[str, Any] diff --git a/tests/functional/parser/test_eventbridge.py b/tests/functional/parser/test_eventbridge.py index 0de57a0de44..2b165839ed8 100644 --- a/tests/functional/parser/test_eventbridge.py +++ b/tests/functional/parser/test_eventbridge.py @@ -27,7 +27,7 @@ def handle_eventbridge_no_envelope(event: MyAdvancedEventbridgeBusiness, _: Lamb assert event.region == "us-west-1" assert event.resources == ["arn:aws:ec2:us-west-1:123456789012:instance/i-1234567890abcdef0"] assert event.source == "aws.ec2" - assert event.detailtype == "EC2 Instance State-change Notification" + assert event.detail_type == "EC2 Instance State-change Notification" def test_handle_eventbridge_trigger_event(): From 64f3261bca102ecf270838c5ccd9c49f24f67b64 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 5 Oct 2020 10:47:05 +0200 Subject: [PATCH 06/22] improv: raise own exception; remove duplicates --- .../utilities/parser/envelopes/base.py | 25 ++++++++++--------- .../utilities/parser/envelopes/dynamodb.py | 12 ++++++--- .../parser/envelopes/event_bridge.py | 10 ++++++-- .../utilities/parser/envelopes/sqs.py | 11 +++++--- .../utilities/parser/exceptions.py | 6 ++++- .../utilities/parser/parser.py | 14 +++++------ tests/functional/parser/test_dynamodb.py | 8 +++--- tests/functional/parser/test_eventbridge.py | 5 ++-- tests/functional/parser/test_sqs.py | 8 +++--- 9 files changed, 57 insertions(+), 42 deletions(-) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/base.py b/aws_lambda_powertools/utilities/parser/envelopes/base.py index 12dc0555da2..41842623989 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/base.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/base.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, ValidationError -from ..exceptions import InvalidEnvelopeError +from ..exceptions import InvalidEnvelopeError, SchemaValidationError logger = logging.getLogger(__name__) @@ -14,28 +14,29 @@ class BaseEnvelope(ABC): def _parse_user_dict_schema(user_event: Dict[str, Any], schema: BaseModel) -> Any: if user_event is None: return None - logger.debug("parsing user dictionary schema") try: + logger.debug("parsing user dictionary schema") return schema(**user_event) - except (ValidationError, TypeError): + except (ValidationError, TypeError) as e: logger.exception("Validation exception while extracting user custom schema") - raise + raise SchemaValidationError("Failed to extract custom schema") from e @staticmethod def _parse_user_json_string_schema(user_event: str, schema: BaseModel) -> Any: if user_event is None: return None + # this is used in cases where the underlying schema is not a Dict that can be parsed as baseModel - # but a plain string i.e SQS has plain string payload - if schema == str: + # but a plain string as payload i.e. SQS: "body": "Test message." + if schema is str: logger.debug("input is string, returning") return user_event - logger.debug("trying to parse as json encoded string") + try: + logger.debug("trying to parse as json encoded string") return schema.parse_raw(user_event) - except (ValidationError, TypeError): - logger.exception("Validation exception while extracting user custom schema") - raise + except (ValidationError, TypeError) as e: + raise SchemaValidationError("Failed to extract custom schema from JSON string") from e @abstractmethod def parse(self, event: Dict[str, Any], schema: BaseModel): @@ -44,7 +45,7 @@ def parse(self, event: Dict[str, Any], schema: BaseModel): def parse_envelope(event: Dict[str, Any], envelope: BaseEnvelope, schema: BaseModel): if not callable(envelope) and not isinstance(BaseEnvelope): - logger.exception("envelope must be a callable and instance of BaseEnvelope") - raise InvalidEnvelopeError("envelope must be a callable and instance of BaseEnvelope") + raise InvalidEnvelopeError(f"envelope must be a callable and instance of BaseEnvelope, envelope={envelope}") + logger.debug(f"Parsing and validating event schema, envelope={envelope}") return envelope().parse(event=event, schema=schema) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py b/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py index 1ceed8d37b6..00af540831f 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py @@ -4,6 +4,7 @@ from pydantic import BaseModel, ValidationError from typing_extensions import Literal +from ..exceptions import SchemaValidationError from ..schemas import DynamoDBSchema from .base import BaseEnvelope @@ -31,12 +32,17 @@ def parse(self, event: Dict[str, Any], schema: BaseModel) -> List[Dict[Literal[" ------- List List of records parsed with schema provided + + Raises + ------ + SchemaValidationError + When input event doesn't conform with schema provided """ try: parsed_envelope = DynamoDBSchema(**event) - except (ValidationError, TypeError): - logger.exception("Validation exception received from input dynamodb stream event") - raise + except (ValidationError, TypeError) as e: + raise SchemaValidationError("DynamoDB input doesn't conform with schema") from e + output = [] for record in parsed_envelope.Records: output.append( diff --git a/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py index 8e3c0e68c31..021bbdf9171 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py @@ -6,6 +6,8 @@ from aws_lambda_powertools.utilities.parser.envelopes.base import BaseEnvelope from aws_lambda_powertools.utilities.parser.schemas import EventBridgeSchema +from ..exceptions import SchemaValidationError + logger = logging.getLogger(__name__) @@ -26,10 +28,14 @@ def parse(self, event: Dict[str, Any], schema: BaseModel) -> BaseModel: ------- Any Parsed detail payload with schema provided + + Raises + ------ + SchemaValidationError + When input event doesn't conform with schema provided """ try: parsed_envelope = EventBridgeSchema(**event) except (ValidationError, TypeError): - logger.exception("Validation exception received from input eventbridge event") - raise + raise SchemaValidationError("EventBridge input doesn't conform with schema") from ValidationError return self._parse_user_dict_schema(parsed_envelope.detail, schema) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/sqs.py b/aws_lambda_powertools/utilities/parser/envelopes/sqs.py index 951426f860d..98b94e1580c 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/sqs.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/sqs.py @@ -3,6 +3,7 @@ from pydantic import BaseModel, ValidationError +from ..exceptions import SchemaValidationError from ..schemas import SqsSchema from .base import BaseEnvelope @@ -33,12 +34,16 @@ def parse(self, event: Dict[str, Any], schema: Union[BaseModel, str]) -> List[Un ------- List List of records parsed with schema provided + + Raises + ------ + SchemaValidationError + When input event doesn't conform with schema provided """ try: parsed_envelope = SqsSchema(**event) - except (ValidationError, TypeError): - logger.exception("Validation exception received from input sqs event") - raise + except (ValidationError, TypeError) as e: + raise SchemaValidationError("SQS input doesn't conform with schema") from e output = [] for record in parsed_envelope.Records: output.append(self._parse_user_json_string_schema(record.body, schema)) diff --git a/aws_lambda_powertools/utilities/parser/exceptions.py b/aws_lambda_powertools/utilities/parser/exceptions.py index 8a08ce23154..f70b65b1bb0 100644 --- a/aws_lambda_powertools/utilities/parser/exceptions.py +++ b/aws_lambda_powertools/utilities/parser/exceptions.py @@ -1,2 +1,6 @@ class InvalidEnvelopeError(Exception): - """Input envelope is not one of the Envelope enum values""" + """Input envelope is not callable and instance of BaseEnvelope""" + + +class SchemaValidationError(Exception): + """Input data does not conform with schema""" diff --git a/aws_lambda_powertools/utilities/parser/parser.py b/aws_lambda_powertools/utilities/parser/parser.py index 6a29692df6d..9a46905e4c6 100644 --- a/aws_lambda_powertools/utilities/parser/parser.py +++ b/aws_lambda_powertools/utilities/parser/parser.py @@ -6,6 +6,7 @@ from aws_lambda_powertools.middleware_factory import lambda_handler_decorator from .envelopes.base import BaseEnvelope, parse_envelope +from .exceptions import SchemaValidationError logger = logging.getLogger(__name__) @@ -49,19 +50,16 @@ def handler(event: MyBusiness , context: LambdaContext): Raises ------ - err - TypeError - in case event is None - pydantic.ValidationError - event fails validation, either of the envelope + SchemaValidationError + When input event doesn't conform with schema provided """ lambda_handler_name = handler.__name__ - parsed_event = None if envelope is None: try: - logger.debug("Parsing and validating event schema, no envelope is used") + logger.debug("Parsing and validating event schema; no envelope used") parsed_event = schema(**event) - except (ValidationError, TypeError): - logger.exception("Validation exception received from input event") - raise + except (ValidationError, TypeError) as e: + raise SchemaValidationError("Input event doesn't conform with schema") from e else: parsed_event = parse_envelope(event, envelope, schema) diff --git a/tests/functional/parser/test_dynamodb.py b/tests/functional/parser/test_dynamodb.py index 4d20dab12a0..3710ff211f9 100644 --- a/tests/functional/parser/test_dynamodb.py +++ b/tests/functional/parser/test_dynamodb.py @@ -1,10 +1,8 @@ from typing import Any, Dict, List import pytest -from pydantic.error_wrappers import ValidationError -from aws_lambda_powertools.utilities.parser import envelopes -from aws_lambda_powertools.utilities.parser.parser import parser +from aws_lambda_powertools.utilities.parser import envelopes, exceptions, parser from aws_lambda_powertools.utilities.typing import LambdaContext from tests.functional.parser.schemas import MyAdvancedDynamoBusiness, MyDynamoBusiness from tests.functional.parser.utils import load_event @@ -61,11 +59,11 @@ def test_dynamo_db_stream_trigger_event_no_envelope(): def test_validate_event_does_not_conform_with_schema_no_envelope(): event_dict: Any = {"hello": "s"} - with pytest.raises(ValidationError): + with pytest.raises(exceptions.SchemaValidationError): handle_dynamodb_no_envelope(event_dict, LambdaContext()) def test_validate_event_does_not_conform_with_schema(): event_dict: Any = {"hello": "s"} - with pytest.raises(ValidationError): + with pytest.raises(exceptions.SchemaValidationError): handle_dynamodb(event_dict, LambdaContext()) diff --git a/tests/functional/parser/test_eventbridge.py b/tests/functional/parser/test_eventbridge.py index 2b165839ed8..7e7e66e2de2 100644 --- a/tests/functional/parser/test_eventbridge.py +++ b/tests/functional/parser/test_eventbridge.py @@ -1,9 +1,8 @@ from typing import Any import pytest -from pydantic import ValidationError -from aws_lambda_powertools.utilities.parser import envelopes, parser +from aws_lambda_powertools.utilities.parser import envelopes, exceptions, parser from aws_lambda_powertools.utilities.typing import LambdaContext from tests.functional.parser.schemas import MyAdvancedEventbridgeBusiness, MyEventbridgeBusiness from tests.functional.parser.utils import load_event @@ -47,7 +46,7 @@ def test_validate_event_does_not_conform_with_user_dict_schema(): "resources": ["arn:aws:ec2:us-west-1:123456789012:instance/i-1234567890abcdef0"], "detail": {}, } - with pytest.raises(ValidationError) as e: + with pytest.raises(exceptions.SchemaValidationError) as e: handle_eventbridge(event_dict, LambdaContext()) print(e.exconly()) diff --git a/tests/functional/parser/test_sqs.py b/tests/functional/parser/test_sqs.py index e8fccfbff04..6117f63717b 100644 --- a/tests/functional/parser/test_sqs.py +++ b/tests/functional/parser/test_sqs.py @@ -1,10 +1,8 @@ from typing import Any, List import pytest -from pydantic import ValidationError -from aws_lambda_powertools.utilities.parser import envelopes -from aws_lambda_powertools.utilities.parser.parser import parser +from aws_lambda_powertools.utilities.parser import envelopes, exceptions, parser from aws_lambda_powertools.utilities.typing import LambdaContext from tests.functional.parser.schemas import MyAdvancedSqsBusiness, MySqsBusiness from tests.functional.parser.utils import load_event @@ -37,7 +35,7 @@ def test_handle_sqs_trigger_event_json_body(sqs_event): # noqa: F811 def test_validate_event_does_not_conform_with_schema(): event: Any = {"invalid": "event"} - with pytest.raises(ValidationError): + with pytest.raises(exceptions.SchemaValidationError): handle_sqs_json_body(event, LambdaContext()) @@ -65,7 +63,7 @@ def test_validate_event_does_not_conform_user_json_string_with_schema(): ] } - with pytest.raises(ValidationError): + with pytest.raises(exceptions.SchemaValidationError): handle_sqs_json_body(event, LambdaContext()) From ad00cb042293f1a4328d0ed2d9f3dc0844c27496 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 9 Oct 2020 13:57:51 +0200 Subject: [PATCH 07/22] fix: unnecessary return; better error handling --- aws_lambda_powertools/utilities/parser/envelopes/base.py | 5 +---- .../utilities/parser/envelopes/event_bridge.py | 4 ++-- aws_lambda_powertools/utilities/parser/parser.py | 5 +++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/base.py b/aws_lambda_powertools/utilities/parser/envelopes/base.py index 41842623989..302d37fbff9 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/base.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/base.py @@ -23,9 +23,6 @@ def _parse_user_dict_schema(user_event: Dict[str, Any], schema: BaseModel) -> An @staticmethod def _parse_user_json_string_schema(user_event: str, schema: BaseModel) -> Any: - if user_event is None: - return None - # this is used in cases where the underlying schema is not a Dict that can be parsed as baseModel # but a plain string as payload i.e. SQS: "body": "Test message." if schema is str: @@ -40,7 +37,7 @@ def _parse_user_json_string_schema(user_event: str, schema: BaseModel) -> Any: @abstractmethod def parse(self, event: Dict[str, Any], schema: BaseModel): - return NotImplemented + return NotImplemented # pragma: no cover def parse_envelope(event: Dict[str, Any], envelope: BaseEnvelope, schema: BaseModel): diff --git a/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py index 021bbdf9171..8e00b3f2b25 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py @@ -36,6 +36,6 @@ def parse(self, event: Dict[str, Any], schema: BaseModel) -> BaseModel: """ try: parsed_envelope = EventBridgeSchema(**event) - except (ValidationError, TypeError): - raise SchemaValidationError("EventBridge input doesn't conform with schema") from ValidationError + except ValidationError as e: + raise SchemaValidationError("EventBridge input doesn't conform with schema") from e return self._parse_user_dict_schema(parsed_envelope.detail, schema) diff --git a/aws_lambda_powertools/utilities/parser/parser.py b/aws_lambda_powertools/utilities/parser/parser.py index 9a46905e4c6..6e663863bd0 100644 --- a/aws_lambda_powertools/utilities/parser/parser.py +++ b/aws_lambda_powertools/utilities/parser/parser.py @@ -53,15 +53,16 @@ def handler(event: MyBusiness , context: LambdaContext): SchemaValidationError When input event doesn't conform with schema provided """ - lambda_handler_name = handler.__name__ if envelope is None: try: logger.debug("Parsing and validating event schema; no envelope used") parsed_event = schema(**event) - except (ValidationError, TypeError) as e: + except ValidationError as e: raise SchemaValidationError("Input event doesn't conform with schema") from e + else: parsed_event = parse_envelope(event, envelope, schema) + lambda_handler_name = handler.__name__ logger.debug(f"Calling handler {lambda_handler_name}") return handler(parsed_event, context) From 292c0458dd8def9e05f075b1dc4aabd4297d6d60 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 9 Oct 2020 16:14:01 +0200 Subject: [PATCH 08/22] improv: test parser --- .../utilities/parser/envelopes/base.py | 10 ++-- tests/functional/parser/conftest.py | 47 +++++++++++++++++++ tests/functional/parser/test_eventbridge.py | 5 ++ tests/functional/parser/test_parser.py | 42 +++++++++++++++++ 4 files changed, 99 insertions(+), 5 deletions(-) create mode 100644 tests/functional/parser/conftest.py create mode 100644 tests/functional/parser/test_parser.py diff --git a/aws_lambda_powertools/utilities/parser/envelopes/base.py b/aws_lambda_powertools/utilities/parser/envelopes/base.py index 302d37fbff9..6135c0f183f 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/base.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/base.py @@ -14,11 +14,11 @@ class BaseEnvelope(ABC): def _parse_user_dict_schema(user_event: Dict[str, Any], schema: BaseModel) -> Any: if user_event is None: return None + try: logger.debug("parsing user dictionary schema") return schema(**user_event) except (ValidationError, TypeError) as e: - logger.exception("Validation exception while extracting user custom schema") raise SchemaValidationError("Failed to extract custom schema") from e @staticmethod @@ -41,8 +41,8 @@ def parse(self, event: Dict[str, Any], schema: BaseModel): def parse_envelope(event: Dict[str, Any], envelope: BaseEnvelope, schema: BaseModel): - if not callable(envelope) and not isinstance(BaseEnvelope): + try: + logger.debug(f"Parsing and validating event schema, envelope={envelope}") + return envelope().parse(event=event, schema=schema) + except (TypeError, AttributeError): raise InvalidEnvelopeError(f"envelope must be a callable and instance of BaseEnvelope, envelope={envelope}") - - logger.debug(f"Parsing and validating event schema, envelope={envelope}") - return envelope().parse(event=event, schema=schema) diff --git a/tests/functional/parser/conftest.py b/tests/functional/parser/conftest.py new file mode 100644 index 00000000000..8cbed4e857d --- /dev/null +++ b/tests/functional/parser/conftest.py @@ -0,0 +1,47 @@ +from typing import Any, Dict + +import pytest +from pydantic import BaseModel, ValidationError + +from aws_lambda_powertools.utilities.parser.envelopes.base import BaseEnvelope +from aws_lambda_powertools.utilities.parser.exceptions import SchemaValidationError + + +@pytest.fixture +def dummy_event(): + return {"payload": {"message": "hello world"}} + + +@pytest.fixture +def dummy_schema(): + """Wanted payload structure""" + + class MyDummyModel(BaseModel): + message: str + + return MyDummyModel + + +@pytest.fixture +def dummy_envelope_schema(): + """Event wrapper structure""" + + class MyDummyEnvelopeSchema(BaseModel): + payload: Dict + + return MyDummyEnvelopeSchema + + +@pytest.fixture +def dummy_envelope(dummy_envelope_schema): + class MyDummyEnvelope(BaseEnvelope): + """Unwrap dummy event within payload key""" + + def parse(self, event: Dict[str, Any], schema: BaseModel): + try: + parsed_enveloped = dummy_envelope_schema(**event) + except (ValidationError, TypeError) as e: + raise SchemaValidationError("Dummy input doesn't conform with schema") from e + return self._parse_user_dict_schema(user_event=parsed_enveloped.payload, schema=schema) + + return MyDummyEnvelope diff --git a/tests/functional/parser/test_eventbridge.py b/tests/functional/parser/test_eventbridge.py index 7e7e66e2de2..0bd0e68161d 100644 --- a/tests/functional/parser/test_eventbridge.py +++ b/tests/functional/parser/test_eventbridge.py @@ -54,3 +54,8 @@ def test_validate_event_does_not_conform_with_user_dict_schema(): def test_handle_eventbridge_trigger_event_no_envelope(): event_dict = load_event("eventBridgeEvent.json") handle_eventbridge_no_envelope(event_dict, LambdaContext()) + + +def test_handle_invalid_event_with_eventbridge_envelope(): + with pytest.raises(exceptions.SchemaValidationError): + handle_eventbridge(event={}, context=LambdaContext()) diff --git a/tests/functional/parser/test_parser.py b/tests/functional/parser/test_parser.py new file mode 100644 index 00000000000..c79270bf6a5 --- /dev/null +++ b/tests/functional/parser/test_parser.py @@ -0,0 +1,42 @@ +from typing import Dict + +import pytest + +from aws_lambda_powertools.utilities.parser import exceptions, parser +from aws_lambda_powertools.utilities.typing import LambdaContext + + +@pytest.mark.parametrize("invalid_value", [None, bool(), [], (), object]) +def test_parser_unsupported_event(dummy_schema, invalid_value): + @parser(schema=dummy_schema) + def handle_no_envelope(event: Dict, _: LambdaContext): + return event + + with pytest.raises(TypeError): + handle_no_envelope(event=invalid_value, context=LambdaContext()) + + +@pytest.mark.parametrize("invalid_envelope", [bool(), [], (), object]) +def test_parser_invalid_envelope_type(dummy_schema, invalid_envelope): + @parser(schema=dummy_schema, envelope=invalid_envelope) + def handle_no_envelope(event: Dict, _: LambdaContext): + return event + + with pytest.raises(exceptions.InvalidEnvelopeError): + handle_no_envelope(event={}, context=LambdaContext()) + + +def test_parser_schema_with_envelope(dummy_event, dummy_schema, dummy_envelope): + @parser(schema=dummy_schema, envelope=dummy_envelope) + def handle_no_envelope(event: Dict, _: LambdaContext): + return event + + handle_no_envelope(dummy_event, LambdaContext()) + + +def test_parser_schema_no_envelope(dummy_event, dummy_schema): + @parser(schema=dummy_schema) + def handle_no_envelope(event: Dict, _: LambdaContext): + return event + + handle_no_envelope(dummy_event["payload"], LambdaContext()) From f0ff92642dbf63ad645d6e54a0a4603136ed81d2 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sat, 10 Oct 2020 22:19:39 +0200 Subject: [PATCH 09/22] improv: simplify base envelope; increase test cov --- .../utilities/parser/envelopes/base.py | 29 +++++++------------ .../utilities/parser/envelopes/dynamodb.py | 4 +-- .../parser/envelopes/event_bridge.py | 2 +- .../utilities/parser/envelopes/sqs.py | 4 +-- .../utilities/parser/exceptions.py | 4 +++ .../utilities/parser/parser.py | 14 +++++---- tests/functional/parser/conftest.py | 2 +- tests/functional/parser/test_parser.py | 12 +++++++- tests/functional/parser/test_sqs.py | 12 -------- 9 files changed, 38 insertions(+), 45 deletions(-) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/base.py b/aws_lambda_powertools/utilities/parser/envelopes/base.py index 6135c0f183f..8b14245a88a 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/base.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/base.py @@ -11,30 +11,21 @@ class BaseEnvelope(ABC): @staticmethod - def _parse_user_dict_schema(user_event: Dict[str, Any], schema: BaseModel) -> Any: - if user_event is None: - return None + def _parse(event: Dict[str, Any], schema: BaseModel) -> Any: + if event is None: + logger.debug("Skipping parsing as event is None") + return event try: - logger.debug("parsing user dictionary schema") - return schema(**user_event) + logger.debug("parsing event against schema") + if isinstance(event, str): + logger.debug("parsing event as string") + return schema.parse_raw(event) + else: + return schema.parse_obj(event) except (ValidationError, TypeError) as e: raise SchemaValidationError("Failed to extract custom schema") from e - @staticmethod - def _parse_user_json_string_schema(user_event: str, schema: BaseModel) -> Any: - # this is used in cases where the underlying schema is not a Dict that can be parsed as baseModel - # but a plain string as payload i.e. SQS: "body": "Test message." - if schema is str: - logger.debug("input is string, returning") - return user_event - - try: - logger.debug("trying to parse as json encoded string") - return schema.parse_raw(user_event) - except (ValidationError, TypeError) as e: - raise SchemaValidationError("Failed to extract custom schema from JSON string") from e - @abstractmethod def parse(self, event: Dict[str, Any], schema: BaseModel): return NotImplemented # pragma: no cover diff --git a/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py b/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py index 00af540831f..c1c6477e565 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py @@ -47,8 +47,8 @@ def parse(self, event: Dict[str, Any], schema: BaseModel) -> List[Dict[Literal[" for record in parsed_envelope.Records: output.append( { - "NewImage": self._parse_user_dict_schema(record.dynamodb.NewImage, schema), - "OldImage": self._parse_user_dict_schema(record.dynamodb.OldImage, schema), + "NewImage": self._parse(record.dynamodb.NewImage, schema), + "OldImage": self._parse(record.dynamodb.OldImage, schema), } ) return output diff --git a/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py index 8e00b3f2b25..43262976bc0 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py @@ -38,4 +38,4 @@ def parse(self, event: Dict[str, Any], schema: BaseModel) -> BaseModel: parsed_envelope = EventBridgeSchema(**event) except ValidationError as e: raise SchemaValidationError("EventBridge input doesn't conform with schema") from e - return self._parse_user_dict_schema(parsed_envelope.detail, schema) + return self._parse(parsed_envelope.detail, schema) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/sqs.py b/aws_lambda_powertools/utilities/parser/envelopes/sqs.py index 98b94e1580c..c64b7520842 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/sqs.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/sqs.py @@ -44,7 +44,5 @@ def parse(self, event: Dict[str, Any], schema: Union[BaseModel, str]) -> List[Un parsed_envelope = SqsSchema(**event) except (ValidationError, TypeError) as e: raise SchemaValidationError("SQS input doesn't conform with schema") from e - output = [] - for record in parsed_envelope.Records: - output.append(self._parse_user_json_string_schema(record.body, schema)) + output = [self._parse(record.body, schema) for record in parsed_envelope.Records] return output diff --git a/aws_lambda_powertools/utilities/parser/exceptions.py b/aws_lambda_powertools/utilities/parser/exceptions.py index f70b65b1bb0..9f1ac331bef 100644 --- a/aws_lambda_powertools/utilities/parser/exceptions.py +++ b/aws_lambda_powertools/utilities/parser/exceptions.py @@ -4,3 +4,7 @@ class InvalidEnvelopeError(Exception): class SchemaValidationError(Exception): """Input data does not conform with schema""" + + +class InvalidSchemaTypeError(Exception): + """Input schema does not implement BaseModel""" diff --git a/aws_lambda_powertools/utilities/parser/parser.py b/aws_lambda_powertools/utilities/parser/parser.py index 6e663863bd0..c4263638cfa 100644 --- a/aws_lambda_powertools/utilities/parser/parser.py +++ b/aws_lambda_powertools/utilities/parser/parser.py @@ -6,7 +6,7 @@ from aws_lambda_powertools.middleware_factory import lambda_handler_decorator from .envelopes.base import BaseEnvelope, parse_envelope -from .exceptions import SchemaValidationError +from .exceptions import InvalidSchemaTypeError, SchemaValidationError logger = logging.getLogger(__name__) @@ -52,17 +52,19 @@ def handler(event: MyBusiness , context: LambdaContext): ------ SchemaValidationError When input event doesn't conform with schema provided + InvalidSchemaTypeError + When schema given does not implement BaseModel """ if envelope is None: try: logger.debug("Parsing and validating event schema; no envelope used") - parsed_event = schema(**event) - except ValidationError as e: + parsed_event = schema.parse_obj(event) + except (ValidationError, TypeError) as e: raise SchemaValidationError("Input event doesn't conform with schema") from e - + except AttributeError: + raise InvalidSchemaTypeError("Input schema must implement BaseModel") else: parsed_event = parse_envelope(event, envelope, schema) - lambda_handler_name = handler.__name__ - logger.debug(f"Calling handler {lambda_handler_name}") + logger.debug(f"Calling handler {handler.__name__}") return handler(parsed_event, context) diff --git a/tests/functional/parser/conftest.py b/tests/functional/parser/conftest.py index 8cbed4e857d..5af1f757c83 100644 --- a/tests/functional/parser/conftest.py +++ b/tests/functional/parser/conftest.py @@ -42,6 +42,6 @@ def parse(self, event: Dict[str, Any], schema: BaseModel): parsed_enveloped = dummy_envelope_schema(**event) except (ValidationError, TypeError) as e: raise SchemaValidationError("Dummy input doesn't conform with schema") from e - return self._parse_user_dict_schema(user_event=parsed_enveloped.payload, schema=schema) + return self._parse(event=parsed_enveloped.payload, schema=schema) return MyDummyEnvelope diff --git a/tests/functional/parser/test_parser.py b/tests/functional/parser/test_parser.py index c79270bf6a5..969418cfcd3 100644 --- a/tests/functional/parser/test_parser.py +++ b/tests/functional/parser/test_parser.py @@ -12,7 +12,7 @@ def test_parser_unsupported_event(dummy_schema, invalid_value): def handle_no_envelope(event: Dict, _: LambdaContext): return event - with pytest.raises(TypeError): + with pytest.raises(exceptions.SchemaValidationError): handle_no_envelope(event=invalid_value, context=LambdaContext()) @@ -40,3 +40,13 @@ def handle_no_envelope(event: Dict, _: LambdaContext): return event handle_no_envelope(dummy_event["payload"], LambdaContext()) + + +@pytest.mark.parametrize("invalid_schema", [None, str, bool(), [], (), object]) +def test_parser_with_invalid_schema_type(dummy_event, invalid_schema): + @parser(schema=invalid_schema) + def handle_no_envelope(event: Dict, _: LambdaContext): + return event + + with pytest.raises(exceptions.InvalidSchemaTypeError): + handle_no_envelope(event=dummy_event, context=LambdaContext()) diff --git a/tests/functional/parser/test_sqs.py b/tests/functional/parser/test_sqs.py index 6117f63717b..9cb898b6eda 100644 --- a/tests/functional/parser/test_sqs.py +++ b/tests/functional/parser/test_sqs.py @@ -9,18 +9,6 @@ from tests.functional.validator.conftest import sqs_event # noqa: F401 -@parser(schema=str, envelope=envelopes.SQS) -def handle_sqs_str_body(event: List[str], _: LambdaContext): - assert len(event) == 2 - assert event[0] == "Test message." - assert event[1] == "Test message2." - - -def test_handle_sqs_trigger_event_str_body(): - event_dict = load_event("sqsEvent.json") - handle_sqs_str_body(event_dict, LambdaContext()) - - @parser(schema=MySqsBusiness, envelope=envelopes.SQS) def handle_sqs_json_body(event: List[MySqsBusiness], _: LambdaContext): assert len(event) == 1 From b868e1f716add525d3e9b8ed091e6eac0020fb24 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sat, 10 Oct 2020 22:34:03 +0200 Subject: [PATCH 10/22] fix: code inspect issues --- .../utilities/parser/envelopes/base.py | 4 +- .../utilities/parser/envelopes/dynamodb.py | 1 + .../utilities/parser/envelopes/sqs.py | 4 +- .../utilities/parser/parser.py | 61 ++++++++++--------- 4 files changed, 37 insertions(+), 33 deletions(-) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/base.py b/aws_lambda_powertools/utilities/parser/envelopes/base.py index 8b14245a88a..164bf3ca3f7 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/base.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/base.py @@ -1,6 +1,6 @@ import logging from abc import ABC, abstractmethod -from typing import Any, Dict +from typing import Any, Dict, Union from pydantic import BaseModel, ValidationError @@ -11,7 +11,7 @@ class BaseEnvelope(ABC): @staticmethod - def _parse(event: Dict[str, Any], schema: BaseModel) -> Any: + def _parse(event: Union[Dict[str, Any], str], schema: BaseModel) -> Any: if event is None: logger.debug("Skipping parsing as event is None") return event diff --git a/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py b/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py index c1c6477e565..9d6a699e517 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py @@ -51,4 +51,5 @@ def parse(self, event: Dict[str, Any], schema: BaseModel) -> List[Dict[Literal[" "OldImage": self._parse(record.dynamodb.OldImage, schema), } ) + # noinspection PyTypeChecker return output diff --git a/aws_lambda_powertools/utilities/parser/envelopes/sqs.py b/aws_lambda_powertools/utilities/parser/envelopes/sqs.py index c64b7520842..01201e16b02 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/sqs.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/sqs.py @@ -44,5 +44,7 @@ def parse(self, event: Dict[str, Any], schema: Union[BaseModel, str]) -> List[Un parsed_envelope = SqsSchema(**event) except (ValidationError, TypeError) as e: raise SchemaValidationError("SQS input doesn't conform with schema") from e - output = [self._parse(record.body, schema) for record in parsed_envelope.Records] + output = [] + for record in parsed_envelope.Records: + output.append(self._parse(record.body, schema)) return output diff --git a/aws_lambda_powertools/utilities/parser/parser.py b/aws_lambda_powertools/utilities/parser/parser.py index c4263638cfa..9df0f7749e1 100644 --- a/aws_lambda_powertools/utilities/parser/parser.py +++ b/aws_lambda_powertools/utilities/parser/parser.py @@ -19,42 +19,43 @@ def parser( schema: BaseModel, envelope: Optional[BaseEnvelope] = None, ) -> Any: + # noinspection SpellCheckingInspection,SpellCheckingInspection """Decorator to conduct advanced parsing & validation for lambda handlers events - As Lambda follows (event, context) signature we can remove some of the boilerplate - and also capture any exception any Lambda function throws as metadata. - event will be the parsed and passed as a BaseModel pydantic class of the input type "schema" - to the lambda handler. - event will be extracted from the envelope in case envelope is not None. - In case envelope is None, the complete event is parsed to match the schema parameter BaseModel definition. - In case envelope is not None, first the event is parsed as the envelope's schema definition, and the user - message is extracted and parsed again as the schema parameter's definition. + As Lambda follows (event, context) signature we can remove some of the boilerplate + and also capture any exception any Lambda function throws as metadata. + event will be the parsed and passed as a BaseModel pydantic class of the input type "schema" + to the lambda handler. + event will be extracted from the envelope in case envelope is not None. + In case envelope is None, the complete event is parsed to match the schema parameter BaseModel definition. + In case envelope is not None, first the event is parsed as the envelope's schema definition, and the user + message is extracted and parsed again as the schema parameter's definition. - Example - ------- - **Lambda function using validation decorator** + Example + ------- + **Lambda function using validation decorator** - @parser(schema=MyBusiness, envelope=envelopes.EVENTBRIDGE) - def handler(event: MyBusiness , context: LambdaContext): - ... + @parser(schema=MyBusiness, envelope=envelopes.EVENTBRIDGE) + def handler(event: MyBusiness , context: LambdaContext): + ... - Parameters - ---------- - handler: input for lambda_handler_decorator, wraps the handler lambda - event: AWS event dictionary - context: AWS lambda context - schema: pydantic BaseModel class. This is the user data schema that will replace the event. - event parameter will be parsed and a new schema object will be created from it. - envelope: what envelope to extract the schema from, can be any AWS service that is currently - supported in the envelopes module. Can be None. + Parameters + ---------- + handler: input for lambda_handler_decorator, wraps the handler lambda + event: AWS event dictionary + context: AWS lambda context + schema: pydantic BaseModel class. This is the user data schema that will replace the event. + event parameter will be parsed and a new schema object will be created from it. + envelope: what envelope to extract the schema from, can be any AWS service that is currently + supported in the envelopes module. Can be None. - Raises - ------ - SchemaValidationError - When input event doesn't conform with schema provided - InvalidSchemaTypeError - When schema given does not implement BaseModel - """ + Raises + ------ + SchemaValidationError + When input event doesn't conform with schema provided + InvalidSchemaTypeError + When schema given does not implement BaseModel + """ if envelope is None: try: logger.debug("Parsing and validating event schema; no envelope used") From 7840d6b5bd9ea18ad97c7402f3bb9ce6d7c2edf9 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sat, 10 Oct 2020 22:40:29 +0200 Subject: [PATCH 11/22] chore: kwarg over arg to ease refactoring --- aws_lambda_powertools/utilities/parser/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/parser/parser.py b/aws_lambda_powertools/utilities/parser/parser.py index 9df0f7749e1..8bbc8071a7e 100644 --- a/aws_lambda_powertools/utilities/parser/parser.py +++ b/aws_lambda_powertools/utilities/parser/parser.py @@ -65,7 +65,7 @@ def handler(event: MyBusiness , context: LambdaContext): except AttributeError: raise InvalidSchemaTypeError("Input schema must implement BaseModel") else: - parsed_event = parse_envelope(event, envelope, schema) + parsed_event = parse_envelope(event=event, envelope=envelope, schema=schema) logger.debug(f"Calling handler {handler.__name__}") return handler(parsed_event, context) From 89929b3735e068b47d0d7b4ae24851cb3739a016 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sat, 10 Oct 2020 23:19:13 +0200 Subject: [PATCH 12/22] feat: add standalone parse function --- .../utilities/parser/envelopes/base.py | 10 +- .../utilities/parser/parser.py | 137 ++++++++++++------ tests/functional/parser/test_parser.py | 6 +- 3 files changed, 94 insertions(+), 59 deletions(-) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/base.py b/aws_lambda_powertools/utilities/parser/envelopes/base.py index 164bf3ca3f7..d6b12c7b5cd 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/base.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/base.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, ValidationError -from ..exceptions import InvalidEnvelopeError, SchemaValidationError +from ..exceptions import SchemaValidationError logger = logging.getLogger(__name__) @@ -29,11 +29,3 @@ def _parse(event: Union[Dict[str, Any], str], schema: BaseModel) -> Any: @abstractmethod def parse(self, event: Dict[str, Any], schema: BaseModel): return NotImplemented # pragma: no cover - - -def parse_envelope(event: Dict[str, Any], envelope: BaseEnvelope, schema: BaseModel): - try: - logger.debug(f"Parsing and validating event schema, envelope={envelope}") - return envelope().parse(event=event, schema=schema) - except (TypeError, AttributeError): - raise InvalidEnvelopeError(f"envelope must be a callable and instance of BaseEnvelope, envelope={envelope}") diff --git a/aws_lambda_powertools/utilities/parser/parser.py b/aws_lambda_powertools/utilities/parser/parser.py index 8bbc8071a7e..a16c3420234 100644 --- a/aws_lambda_powertools/utilities/parser/parser.py +++ b/aws_lambda_powertools/utilities/parser/parser.py @@ -5,8 +5,8 @@ from aws_lambda_powertools.middleware_factory import lambda_handler_decorator -from .envelopes.base import BaseEnvelope, parse_envelope -from .exceptions import InvalidSchemaTypeError, SchemaValidationError +from .envelopes.base import BaseEnvelope +from .exceptions import InvalidEnvelopeError, InvalidSchemaTypeError, SchemaValidationError logger = logging.getLogger(__name__) @@ -19,53 +19,96 @@ def parser( schema: BaseModel, envelope: Optional[BaseEnvelope] = None, ) -> Any: - # noinspection SpellCheckingInspection,SpellCheckingInspection """Decorator to conduct advanced parsing & validation for lambda handlers events - As Lambda follows (event, context) signature we can remove some of the boilerplate - and also capture any exception any Lambda function throws as metadata. - event will be the parsed and passed as a BaseModel pydantic class of the input type "schema" - to the lambda handler. - event will be extracted from the envelope in case envelope is not None. - In case envelope is None, the complete event is parsed to match the schema parameter BaseModel definition. - In case envelope is not None, first the event is parsed as the envelope's schema definition, and the user - message is extracted and parsed again as the schema parameter's definition. - - Example - ------- - **Lambda function using validation decorator** - - @parser(schema=MyBusiness, envelope=envelopes.EVENTBRIDGE) - def handler(event: MyBusiness , context: LambdaContext): - ... - - Parameters - ---------- - handler: input for lambda_handler_decorator, wraps the handler lambda - event: AWS event dictionary - context: AWS lambda context - schema: pydantic BaseModel class. This is the user data schema that will replace the event. - event parameter will be parsed and a new schema object will be created from it. - envelope: what envelope to extract the schema from, can be any AWS service that is currently - supported in the envelopes module. Can be None. - - Raises - ------ - SchemaValidationError - When input event doesn't conform with schema provided - InvalidSchemaTypeError - When schema given does not implement BaseModel - """ - if envelope is None: - try: - logger.debug("Parsing and validating event schema; no envelope used") - parsed_event = schema.parse_obj(event) - except (ValidationError, TypeError) as e: - raise SchemaValidationError("Input event doesn't conform with schema") from e - except AttributeError: - raise InvalidSchemaTypeError("Input schema must implement BaseModel") - else: - parsed_event = parse_envelope(event=event, envelope=envelope, schema=schema) + As Lambda follows (event, context) signature we can remove some of the boilerplate + and also capture any exception any Lambda function throws as metadata. + event will be the parsed and passed as a BaseModel Pydantic class of the input type "schema" + to the lambda handler. + event will be extracted from the envelope in case envelope is not None. + In case envelope is None, the complete event is parsed to match the schema parameter BaseModel definition. + In case envelope is not None, first the event is parsed as the envelope's schema definition, and the user + message is extracted and parsed again as the schema parameter's definition. + + Example + ------- + **Lambda function using validation decorator** + + @parser(schema=MyBusiness, envelope=envelopes.EVENTBRIDGE) + def handler(event: MyBusiness , context: LambdaContext): + ... + + Parameters + ---------- + handler: input for lambda_handler_decorator, wraps the handler lambda + event: AWS event dictionary + context: AWS lambda context + schema: pydantic BaseModel class. This is the user data schema that will replace the event. + event parameter will be parsed and a new schema object will be created from it. + envelope: what envelope to extract the schema from, can be any AWS service that is currently + supported in the envelopes module. Can be None. + Raises + ------ + SchemaValidationError + When input event doesn't conform with schema provided + InvalidSchemaTypeError + When schema given does not implement BaseModel + """ + parsed_event = parse(event=event, schema=schema, envelope=envelope) logger.debug(f"Calling handler {handler.__name__}") return handler(parsed_event, context) + + +def parse(event: Dict[str, Any], schema: BaseModel, envelope: Optional[BaseEnvelope] = None) -> Any: + """ + Standalone parse function to conduct advanced parsing & validation for lambda handlers events + + As Lambda follows (event, context) signature we can remove some of the boilerplate + and also capture any exception any Lambda function throws as metadata. + event will be the parsed and passed as a BaseModel Pydantic class of the input type "schema" + to the lambda handler. + event will be extracted from the envelope in case envelope is not None. + In case envelope is None, the complete event is parsed to match the schema parameter BaseModel definition. + In case envelope is not None, first the event is parsed as the envelope's schema definition, and the user + message is extracted and parsed again as the schema parameter's definition. + + Example + ------- + **Lambda function using standalone parse** + + def handler(event: MyBusiness , context: LambdaContext): + parse(event=event, schema=MyBusiness, envelope=envelopes.EVENTBRIDGE) + ... + + Parameters + ---------- + event: AWS event dictionary + schema: pydantic BaseModel class. This is the user data schema that will replace the event. + event parameter will be parsed and a new schema object will be created from it. + envelope: what envelope to extract the schema from, can be any AWS service that is currently + supported in the envelopes module. Can be None. + + Raises + ------ + SchemaValidationError + When input event doesn't conform with schema provided + InvalidSchemaTypeError + When schema given does not implement BaseModel + + """ + if envelope: + try: + logger.debug(f"Parsing and validating event schema, envelope={envelope}") + # noinspection PyCallingNonCallable + return envelope().parse(event=event, schema=schema) + except (TypeError, AttributeError): + raise InvalidEnvelopeError(f"envelope must be a callable and instance of BaseEnvelope, envelope={envelope}") + + try: + logger.debug("Parsing and validating event schema; no envelope used") + return schema.parse_obj(event) + except (ValidationError, TypeError) as e: + raise SchemaValidationError("Input event doesn't conform with schema") from e + except AttributeError: + raise InvalidSchemaTypeError("Input schema must implement BaseModel") diff --git a/tests/functional/parser/test_parser.py b/tests/functional/parser/test_parser.py index 969418cfcd3..41f92e50405 100644 --- a/tests/functional/parser/test_parser.py +++ b/tests/functional/parser/test_parser.py @@ -16,14 +16,14 @@ def handle_no_envelope(event: Dict, _: LambdaContext): handle_no_envelope(event=invalid_value, context=LambdaContext()) -@pytest.mark.parametrize("invalid_envelope", [bool(), [], (), object]) -def test_parser_invalid_envelope_type(dummy_schema, invalid_envelope): +@pytest.mark.parametrize("invalid_envelope", [True, ["dummy"], ("dummy"), object]) +def test_parser_invalid_envelope_type(dummy_event, dummy_schema, invalid_envelope): @parser(schema=dummy_schema, envelope=invalid_envelope) def handle_no_envelope(event: Dict, _: LambdaContext): return event with pytest.raises(exceptions.InvalidEnvelopeError): - handle_no_envelope(event={}, context=LambdaContext()) + handle_no_envelope(event=dummy_event["payload"], context=LambdaContext()) def test_parser_schema_with_envelope(dummy_event, dummy_schema, dummy_envelope): From 002cc5422edba75a4c81dfadc4a839c361e995a0 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sat, 10 Oct 2020 23:19:28 +0200 Subject: [PATCH 13/22] chore: lint --- .../utilities/parser/envelopes/event_bridge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py index 43262976bc0..b90ea45a52b 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py @@ -36,6 +36,6 @@ def parse(self, event: Dict[str, Any], schema: BaseModel) -> BaseModel: """ try: parsed_envelope = EventBridgeSchema(**event) + return self._parse(parsed_envelope.detail, schema) except ValidationError as e: raise SchemaValidationError("EventBridge input doesn't conform with schema") from e - return self._parse(parsed_envelope.detail, schema) From 89122ac5ee5def52c7329f5af7b3bb4457d1aede Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sun, 11 Oct 2020 20:53:07 +0200 Subject: [PATCH 14/22] improv: propagate exception to parser --- .../utilities/parser/envelopes/base.py | 21 ++++++++----------- .../utilities/parser/envelopes/dynamodb.py | 14 ++----------- .../parser/envelopes/event_bridge.py | 16 +++----------- .../utilities/parser/envelopes/sqs.py | 13 ++---------- .../utilities/parser/parser.py | 20 ++++++++++-------- tests/functional/parser/conftest.py | 2 +- tests/functional/parser/test_parser.py | 11 +++++++--- 7 files changed, 36 insertions(+), 61 deletions(-) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/base.py b/aws_lambda_powertools/utilities/parser/envelopes/base.py index d6b12c7b5cd..cb8166600b4 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/base.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/base.py @@ -2,29 +2,26 @@ from abc import ABC, abstractmethod from typing import Any, Dict, Union -from pydantic import BaseModel, ValidationError - -from ..exceptions import SchemaValidationError +from pydantic import BaseModel logger = logging.getLogger(__name__) class BaseEnvelope(ABC): + """ABC implementation for creating a supported Envelope""" + @staticmethod def _parse(event: Union[Dict[str, Any], str], schema: BaseModel) -> Any: if event is None: logger.debug("Skipping parsing as event is None") return event - try: - logger.debug("parsing event against schema") - if isinstance(event, str): - logger.debug("parsing event as string") - return schema.parse_raw(event) - else: - return schema.parse_obj(event) - except (ValidationError, TypeError) as e: - raise SchemaValidationError("Failed to extract custom schema") from e + logger.debug("parsing event against schema") + if isinstance(event, str): + logger.debug("parsing event as string") + return schema.parse_raw(event) + + return schema.parse_obj(event) @abstractmethod def parse(self, event: Dict[str, Any], schema: BaseModel): diff --git a/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py b/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py index 9d6a699e517..23b74f1b4d5 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py @@ -1,10 +1,9 @@ import logging from typing import Any, Dict, List -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel from typing_extensions import Literal -from ..exceptions import SchemaValidationError from ..schemas import DynamoDBSchema from .base import BaseEnvelope @@ -32,17 +31,8 @@ def parse(self, event: Dict[str, Any], schema: BaseModel) -> List[Dict[Literal[" ------- List List of records parsed with schema provided - - Raises - ------ - SchemaValidationError - When input event doesn't conform with schema provided """ - try: - parsed_envelope = DynamoDBSchema(**event) - except (ValidationError, TypeError) as e: - raise SchemaValidationError("DynamoDB input doesn't conform with schema") from e - + parsed_envelope = DynamoDBSchema(**event) output = [] for record in parsed_envelope.Records: output.append( diff --git a/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py index b90ea45a52b..f8075384a69 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py @@ -1,13 +1,11 @@ import logging from typing import Any, Dict -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel from aws_lambda_powertools.utilities.parser.envelopes.base import BaseEnvelope from aws_lambda_powertools.utilities.parser.schemas import EventBridgeSchema -from ..exceptions import SchemaValidationError - logger = logging.getLogger(__name__) @@ -28,14 +26,6 @@ def parse(self, event: Dict[str, Any], schema: BaseModel) -> BaseModel: ------- Any Parsed detail payload with schema provided - - Raises - ------ - SchemaValidationError - When input event doesn't conform with schema provided """ - try: - parsed_envelope = EventBridgeSchema(**event) - return self._parse(parsed_envelope.detail, schema) - except ValidationError as e: - raise SchemaValidationError("EventBridge input doesn't conform with schema") from e + parsed_envelope = EventBridgeSchema(**event) + return self._parse(parsed_envelope.detail, schema) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/sqs.py b/aws_lambda_powertools/utilities/parser/envelopes/sqs.py index 01201e16b02..64ec2d50ca5 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/sqs.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/sqs.py @@ -1,9 +1,8 @@ import logging from typing import Any, Dict, List, Union -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel -from ..exceptions import SchemaValidationError from ..schemas import SqsSchema from .base import BaseEnvelope @@ -34,16 +33,8 @@ def parse(self, event: Dict[str, Any], schema: Union[BaseModel, str]) -> List[Un ------- List List of records parsed with schema provided - - Raises - ------ - SchemaValidationError - When input event doesn't conform with schema provided """ - try: - parsed_envelope = SqsSchema(**event) - except (ValidationError, TypeError) as e: - raise SchemaValidationError("SQS input doesn't conform with schema") from e + parsed_envelope = SqsSchema(**event) output = [] for record in parsed_envelope.Records: output.append(self._parse(record.body, schema)) diff --git a/aws_lambda_powertools/utilities/parser/parser.py b/aws_lambda_powertools/utilities/parser/parser.py index a16c3420234..c408830733b 100644 --- a/aws_lambda_powertools/utilities/parser/parser.py +++ b/aws_lambda_powertools/utilities/parser/parser.py @@ -51,7 +51,7 @@ def handler(event: MyBusiness , context: LambdaContext): Raises ------ SchemaValidationError - When input event doesn't conform with schema provided + When input event does not conform with schema provided InvalidSchemaTypeError When schema given does not implement BaseModel """ @@ -92,23 +92,25 @@ def handler(event: MyBusiness , context: LambdaContext): Raises ------ SchemaValidationError - When input event doesn't conform with schema provided + When input event does not conform with schema provided InvalidSchemaTypeError When schema given does not implement BaseModel - + InvalidEnvelopeError + When envelope given does not implement BaseEnvelope """ - if envelope: + if envelope and callable(envelope): try: - logger.debug(f"Parsing and validating event schema, envelope={envelope}") - # noinspection PyCallingNonCallable + logger.debug(f"Parsing and validating event schema with envelope={envelope}") return envelope().parse(event=event, schema=schema) - except (TypeError, AttributeError): - raise InvalidEnvelopeError(f"envelope must be a callable and instance of BaseEnvelope, envelope={envelope}") + except AttributeError: + raise InvalidEnvelopeError(f"Envelope must implement BaseEnvelope, envelope={envelope}") + except (ValidationError, TypeError) as e: + raise SchemaValidationError(f"Input event does not conform with schema, envelope={envelope}") from e try: logger.debug("Parsing and validating event schema; no envelope used") return schema.parse_obj(event) except (ValidationError, TypeError) as e: - raise SchemaValidationError("Input event doesn't conform with schema") from e + raise SchemaValidationError("Input event does not conform with schema") from e except AttributeError: raise InvalidSchemaTypeError("Input schema must implement BaseModel") diff --git a/tests/functional/parser/conftest.py b/tests/functional/parser/conftest.py index 5af1f757c83..9273d2fdca1 100644 --- a/tests/functional/parser/conftest.py +++ b/tests/functional/parser/conftest.py @@ -41,7 +41,7 @@ def parse(self, event: Dict[str, Any], schema: BaseModel): try: parsed_enveloped = dummy_envelope_schema(**event) except (ValidationError, TypeError) as e: - raise SchemaValidationError("Dummy input doesn't conform with schema") from e + raise SchemaValidationError("Dummy input does not conform with schema") from e return self._parse(event=parsed_enveloped.payload, schema=schema) return MyDummyEnvelope diff --git a/tests/functional/parser/test_parser.py b/tests/functional/parser/test_parser.py index 41f92e50405..65cc7a28f81 100644 --- a/tests/functional/parser/test_parser.py +++ b/tests/functional/parser/test_parser.py @@ -16,13 +16,18 @@ def handle_no_envelope(event: Dict, _: LambdaContext): handle_no_envelope(event=invalid_value, context=LambdaContext()) -@pytest.mark.parametrize("invalid_envelope", [True, ["dummy"], ("dummy"), object]) -def test_parser_invalid_envelope_type(dummy_event, dummy_schema, invalid_envelope): +@pytest.mark.parametrize( + "invalid_envelope,expected", [(True, ""), (["dummy"], ""), (object, exceptions.InvalidEnvelopeError)] +) +def test_parser_invalid_envelope_type(dummy_event, dummy_schema, invalid_envelope, expected): @parser(schema=dummy_schema, envelope=invalid_envelope) def handle_no_envelope(event: Dict, _: LambdaContext): return event - with pytest.raises(exceptions.InvalidEnvelopeError): + if hasattr(expected, "__cause__"): + with pytest.raises(expected): + handle_no_envelope(event=dummy_event["payload"], context=LambdaContext()) + else: handle_no_envelope(event=dummy_event["payload"], context=LambdaContext()) From f21b3cac62e6ee66775a0c59bc6b5df90670dcc2 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Sun, 11 Oct 2020 22:12:48 +0200 Subject: [PATCH 15/22] improv: add docstrings; event internal param renamed --- .../utilities/parser/envelopes/base.py | 44 +++++-- .../utilities/parser/envelopes/dynamodb.py | 6 +- .../parser/envelopes/event_bridge.py | 10 +- .../utilities/parser/envelopes/sqs.py | 6 +- .../utilities/parser/parser.py | 119 ++++++++++++------ tests/functional/parser/conftest.py | 6 +- 6 files changed, 131 insertions(+), 60 deletions(-) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/base.py b/aws_lambda_powertools/utilities/parser/envelopes/base.py index cb8166600b4..95a9b24f2b3 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/base.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/base.py @@ -11,18 +11,48 @@ class BaseEnvelope(ABC): """ABC implementation for creating a supported Envelope""" @staticmethod - def _parse(event: Union[Dict[str, Any], str], schema: BaseModel) -> Any: - if event is None: + def _parse(data: Union[Dict[str, Any], str], schema: BaseModel) -> Any: + """Parses envelope data against schema provided + + Parameters + ---------- + data : Dict + Data to be parsed and validated + schema + Schema to parse and validate data against + + Returns + ------- + Any + Parsed data + """ + if data is None: logger.debug("Skipping parsing as event is None") - return event + return data logger.debug("parsing event against schema") - if isinstance(event, str): + if isinstance(data, str): logger.debug("parsing event as string") - return schema.parse_raw(event) + return schema.parse_raw(data) - return schema.parse_obj(event) + return schema.parse_obj(data) @abstractmethod - def parse(self, event: Dict[str, Any], schema: BaseModel): + def parse(self, data: Dict[str, Any], schema: BaseModel): + """Implementation to parse data against envelope schema, then against the schema + + NOTE: Call `_parse` method to fully parse data with schema provided. + + Example + ------- + + **EventBridge envelope implementation example** + + def parse(...): + # 1. parses data against envelope schema + parsed_envelope = EventBridgeSchema(**data) + + # 2. parses portion of data within the envelope against schema + return self._parse(data=parsed_envelope.detail, schema=schema) + """ return NotImplemented # pragma: no cover diff --git a/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py b/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py index 23b74f1b4d5..51e1d4f3381 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py @@ -17,12 +17,12 @@ class DynamoDBEnvelope(BaseEnvelope): length of the list is the record's amount in the original event. """ - def parse(self, event: Dict[str, Any], schema: BaseModel) -> List[Dict[Literal["NewImage", "OldImage"], BaseModel]]: + def parse(self, data: Dict[str, Any], schema: BaseModel) -> List[Dict[Literal["NewImage", "OldImage"], BaseModel]]: """Parses DynamoDB Stream records found in either NewImage and OldImage with schema provided Parameters ---------- - event : Dict + data : Dict Lambda event to be parsed schema : BaseModel User schema provided to parse after extracting data using envelope @@ -32,7 +32,7 @@ def parse(self, event: Dict[str, Any], schema: BaseModel) -> List[Dict[Literal[" List List of records parsed with schema provided """ - parsed_envelope = DynamoDBSchema(**event) + parsed_envelope = DynamoDBSchema(**data) output = [] for record in parsed_envelope.Records: output.append( diff --git a/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py index f8075384a69..9d71dd11882 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py @@ -10,14 +10,14 @@ class EventBridgeEnvelope(BaseEnvelope): - """EventBridge envelope to extract data in detail key""" + """EventBridge envelope to extract data within detail key""" - def parse(self, event: Dict[str, Any], schema: BaseModel) -> BaseModel: + def parse(self, data: Dict[str, Any], schema: BaseModel) -> BaseModel: """Parses data found with schema provided Parameters ---------- - event : Dict + data : Dict Lambda event to be parsed schema : BaseModel User schema provided to parse after extracting data using envelope @@ -27,5 +27,5 @@ def parse(self, event: Dict[str, Any], schema: BaseModel) -> BaseModel: Any Parsed detail payload with schema provided """ - parsed_envelope = EventBridgeSchema(**event) - return self._parse(parsed_envelope.detail, schema) + parsed_envelope = EventBridgeSchema(**data) + return self._parse(data=parsed_envelope.detail, schema=schema) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/sqs.py b/aws_lambda_powertools/utilities/parser/envelopes/sqs.py index 64ec2d50ca5..2e55fe2a82b 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/sqs.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/sqs.py @@ -19,12 +19,12 @@ class SqsEnvelope(BaseEnvelope): all items in the list will be parsed as str and npt as JSON (and vice versa) """ - def parse(self, event: Dict[str, Any], schema: Union[BaseModel, str]) -> List[Union[BaseModel, str]]: + def parse(self, data: Dict[str, Any], schema: Union[BaseModel, str]) -> List[Union[BaseModel, str]]: """Parses records found with schema provided Parameters ---------- - event : Dict + data : Dict Lambda event to be parsed schema : BaseModel User schema provided to parse after extracting data using envelope @@ -34,7 +34,7 @@ def parse(self, event: Dict[str, Any], schema: Union[BaseModel, str]) -> List[Un List List of records parsed with schema provided """ - parsed_envelope = SqsSchema(**event) + parsed_envelope = SqsSchema(**data) output = [] for record in parsed_envelope.Records: output.append(self._parse(record.body, schema)) diff --git a/aws_lambda_powertools/utilities/parser/parser.py b/aws_lambda_powertools/utilities/parser/parser.py index c408830733b..de3ad10f29f 100644 --- a/aws_lambda_powertools/utilities/parser/parser.py +++ b/aws_lambda_powertools/utilities/parser/parser.py @@ -5,6 +5,7 @@ from aws_lambda_powertools.middleware_factory import lambda_handler_decorator +from ..typing import LambdaContext from .envelopes.base import BaseEnvelope from .exceptions import InvalidEnvelopeError, InvalidSchemaTypeError, SchemaValidationError @@ -15,38 +16,60 @@ def parser( handler: Callable[[Dict, Any], Any], event: Dict[str, Any], - context: Dict[str, Any], + context: LambdaContext, schema: BaseModel, envelope: Optional[BaseEnvelope] = None, ) -> Any: - """Decorator to conduct advanced parsing & validation for lambda handlers events + """Lambda handler decorator to parse & validate events using Pydantic models - As Lambda follows (event, context) signature we can remove some of the boilerplate - and also capture any exception any Lambda function throws as metadata. - event will be the parsed and passed as a BaseModel Pydantic class of the input type "schema" - to the lambda handler. - event will be extracted from the envelope in case envelope is not None. - In case envelope is None, the complete event is parsed to match the schema parameter BaseModel definition. - In case envelope is not None, first the event is parsed as the envelope's schema definition, and the user - message is extracted and parsed again as the schema parameter's definition. + It requires a schema that implements Pydantic BaseModel to parse & validate the event. + + When an envelope is given, it'll use the following logic: + + 1. Parse the event against envelope schema first e.g. EnvelopeSchema(**event) + 2. Envelope will extract a given key to be parsed against the schema e.g. event.detail + + This is useful when you need to confirm event wrapper structure, and + b) selectively extract a portion of your payload for parsing & validation. + + NOTE: If envelope is omitted, the complete event is parsed to match the schema parameter BaseModel definition. Example ------- - **Lambda function using validation decorator** + **Lambda handler decorator to parse & validate event** + + class Order(BaseModel): + id: int + description: str + ... + + @parser(schema=Order) + def handler(event: Order, context: LambdaContext): + ... + + **Lambda handler decorator to parse & validate event - using built-in envelope** + + class Order(BaseModel): + id: int + description: str + ... - @parser(schema=MyBusiness, envelope=envelopes.EVENTBRIDGE) - def handler(event: MyBusiness , context: LambdaContext): + @parser(schema=Order, envelope=envelopes.EVENTBRIDGE) + def handler(event: Order, context: LambdaContext): ... Parameters ---------- - handler: input for lambda_handler_decorator, wraps the handler lambda - event: AWS event dictionary - context: AWS lambda context - schema: pydantic BaseModel class. This is the user data schema that will replace the event. - event parameter will be parsed and a new schema object will be created from it. - envelope: what envelope to extract the schema from, can be any AWS service that is currently - supported in the envelopes module. Can be None. + handler: Callable + Method to annotate on + event: Dict + Lambda event to be parsed & validated + context: LambdaContext + Lambda context object + schema: BaseModel + Your data schema that will replace the event. + envelope: BaseEnvelope + Optional envelope to extract the schema from Raises ------ @@ -54,6 +77,8 @@ def handler(event: MyBusiness , context: LambdaContext): When input event does not conform with schema provided InvalidSchemaTypeError When schema given does not implement BaseModel + InvalidEnvelopeError + When envelope given does not implement BaseEnvelope """ parsed_event = parse(event=event, schema=schema, envelope=envelope) logger.debug(f"Calling handler {handler.__name__}") @@ -61,33 +86,49 @@ def handler(event: MyBusiness , context: LambdaContext): def parse(event: Dict[str, Any], schema: BaseModel, envelope: Optional[BaseEnvelope] = None) -> Any: - """ - Standalone parse function to conduct advanced parsing & validation for lambda handlers events + """Standalone function to parse & validate events using Pydantic models - As Lambda follows (event, context) signature we can remove some of the boilerplate - and also capture any exception any Lambda function throws as metadata. - event will be the parsed and passed as a BaseModel Pydantic class of the input type "schema" - to the lambda handler. - event will be extracted from the envelope in case envelope is not None. - In case envelope is None, the complete event is parsed to match the schema parameter BaseModel definition. - In case envelope is not None, first the event is parsed as the envelope's schema definition, and the user - message is extracted and parsed again as the schema parameter's definition. + Typically used when you need fine-grained control over error handling compared to parser decorator. Example ------- - **Lambda function using standalone parse** - def handler(event: MyBusiness , context: LambdaContext): - parse(event=event, schema=MyBusiness, envelope=envelopes.EVENTBRIDGE) + **Lambda handler decorator to parse & validate event** + + from aws_lambda_powertools.utilities.parser.exceptions import SchemaValidationError + + class Order(BaseModel): + id: int + description: str ... + def handler(event: Order, context: LambdaContext): + try: + parse(schema=Order) + except SchemaValidationError: + ... + + **Lambda handler decorator to parse & validate event - using built-in envelope** + + class Order(BaseModel): + id: int + description: str + ... + + def handler(event: Order, context: LambdaContext): + try: + parse(schema=Order, envelope=envelopes.EVENTBRIDGE) + except SchemaValidationError: + ... + Parameters ---------- - event: AWS event dictionary - schema: pydantic BaseModel class. This is the user data schema that will replace the event. - event parameter will be parsed and a new schema object will be created from it. - envelope: what envelope to extract the schema from, can be any AWS service that is currently - supported in the envelopes module. Can be None. + event: Dict + Lambda event to be parsed & validated + schema: BaseModel + Your data schema that will replace the event. + envelope: BaseEnvelope + Optional envelope to extract the schema from Raises ------ @@ -101,7 +142,7 @@ def handler(event: MyBusiness , context: LambdaContext): if envelope and callable(envelope): try: logger.debug(f"Parsing and validating event schema with envelope={envelope}") - return envelope().parse(event=event, schema=schema) + return envelope().parse(data=event, schema=schema) except AttributeError: raise InvalidEnvelopeError(f"Envelope must implement BaseEnvelope, envelope={envelope}") except (ValidationError, TypeError) as e: diff --git a/tests/functional/parser/conftest.py b/tests/functional/parser/conftest.py index 9273d2fdca1..a623f7d6a0d 100644 --- a/tests/functional/parser/conftest.py +++ b/tests/functional/parser/conftest.py @@ -37,11 +37,11 @@ def dummy_envelope(dummy_envelope_schema): class MyDummyEnvelope(BaseEnvelope): """Unwrap dummy event within payload key""" - def parse(self, event: Dict[str, Any], schema: BaseModel): + def parse(self, data: Dict[str, Any], schema: BaseModel): try: - parsed_enveloped = dummy_envelope_schema(**event) + parsed_enveloped = dummy_envelope_schema(**data) except (ValidationError, TypeError) as e: raise SchemaValidationError("Dummy input does not conform with schema") from e - return self._parse(event=parsed_enveloped.payload, schema=schema) + return self._parse(data=parsed_enveloped.payload, schema=schema) return MyDummyEnvelope From 974a98c150c485181c47ccf79ac041943913ed67 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 12 Oct 2020 13:23:38 +0200 Subject: [PATCH 16/22] improv: adjust high level imports --- aws_lambda_powertools/utilities/parser/__init__.py | 5 ++++- aws_lambda_powertools/utilities/parser/envelopes/__init__.py | 5 ++--- .../utilities/parser/envelopes/event_bridge.py | 4 ++-- aws_lambda_powertools/utilities/parser/parser.py | 3 +-- tests/functional/parser/conftest.py | 3 +-- tests/functional/parser/test_dynamodb.py | 2 +- tests/functional/parser/test_eventbridge.py | 2 +- tests/functional/parser/test_sqs.py | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/aws_lambda_powertools/utilities/parser/__init__.py b/aws_lambda_powertools/utilities/parser/__init__.py index be959623ae9..4fbb961fc2f 100644 --- a/aws_lambda_powertools/utilities/parser/__init__.py +++ b/aws_lambda_powertools/utilities/parser/__init__.py @@ -1,6 +1,9 @@ """Advanced parser utility """ from . import envelopes +from .envelopes import BaseEnvelope +from .exceptions import SchemaValidationError from .parser import parser +from .pydantic import BaseModel, root_validator, validator -__all__ = ["envelopes", "parser"] +__all__ = ["parser", "envelopes", "BaseEnvelope", "BaseModel", "validator", "root_validator", "SchemaValidationError"] diff --git a/aws_lambda_powertools/utilities/parser/envelopes/__init__.py b/aws_lambda_powertools/utilities/parser/envelopes/__init__.py index bb7e4761aa9..766021a3f92 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/__init__.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/__init__.py @@ -1,7 +1,6 @@ +from .base import BaseEnvelope from .dynamodb import DynamoDBEnvelope from .event_bridge import EventBridgeEnvelope from .sqs import SqsEnvelope -SQS = SqsEnvelope -DYNAMODB_STREAM = DynamoDBEnvelope -EVENTBRIDGE = EventBridgeEnvelope +__all__ = ["DynamoDBEnvelope", "EventBridgeEnvelope", "SqsEnvelope", "BaseEnvelope"] diff --git a/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py index 9d71dd11882..18996cfda01 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py @@ -3,8 +3,8 @@ from pydantic import BaseModel -from aws_lambda_powertools.utilities.parser.envelopes.base import BaseEnvelope -from aws_lambda_powertools.utilities.parser.schemas import EventBridgeSchema +from ..schemas import EventBridgeSchema +from .base import BaseEnvelope logger = logging.getLogger(__name__) diff --git a/aws_lambda_powertools/utilities/parser/parser.py b/aws_lambda_powertools/utilities/parser/parser.py index de3ad10f29f..f0019df09e5 100644 --- a/aws_lambda_powertools/utilities/parser/parser.py +++ b/aws_lambda_powertools/utilities/parser/parser.py @@ -3,8 +3,7 @@ from pydantic import BaseModel, ValidationError -from aws_lambda_powertools.middleware_factory import lambda_handler_decorator - +from ...middleware_factory import lambda_handler_decorator from ..typing import LambdaContext from .envelopes.base import BaseEnvelope from .exceptions import InvalidEnvelopeError, InvalidSchemaTypeError, SchemaValidationError diff --git a/tests/functional/parser/conftest.py b/tests/functional/parser/conftest.py index a623f7d6a0d..1fb97f61d89 100644 --- a/tests/functional/parser/conftest.py +++ b/tests/functional/parser/conftest.py @@ -3,8 +3,7 @@ import pytest from pydantic import BaseModel, ValidationError -from aws_lambda_powertools.utilities.parser.envelopes.base import BaseEnvelope -from aws_lambda_powertools.utilities.parser.exceptions import SchemaValidationError +from aws_lambda_powertools.utilities.parser import BaseEnvelope, SchemaValidationError @pytest.fixture diff --git a/tests/functional/parser/test_dynamodb.py b/tests/functional/parser/test_dynamodb.py index 3710ff211f9..9f98b49226c 100644 --- a/tests/functional/parser/test_dynamodb.py +++ b/tests/functional/parser/test_dynamodb.py @@ -8,7 +8,7 @@ from tests.functional.parser.utils import load_event -@parser(schema=MyDynamoBusiness, envelope=envelopes.DYNAMODB_STREAM) +@parser(schema=MyDynamoBusiness, envelope=envelopes.DynamoDBEnvelope) def handle_dynamodb(event: List[Dict[str, MyDynamoBusiness]], _: LambdaContext): assert len(event) == 2 assert event[0]["OldImage"] is None diff --git a/tests/functional/parser/test_eventbridge.py b/tests/functional/parser/test_eventbridge.py index 0bd0e68161d..f64fc7deea0 100644 --- a/tests/functional/parser/test_eventbridge.py +++ b/tests/functional/parser/test_eventbridge.py @@ -8,7 +8,7 @@ from tests.functional.parser.utils import load_event -@parser(schema=MyEventbridgeBusiness, envelope=envelopes.EVENTBRIDGE) +@parser(schema=MyEventbridgeBusiness, envelope=envelopes.EventBridgeEnvelope) def handle_eventbridge(event: MyEventbridgeBusiness, _: LambdaContext): assert event.instance_id == "i-1234567890abcdef0" assert event.state == "terminated" diff --git a/tests/functional/parser/test_sqs.py b/tests/functional/parser/test_sqs.py index 9cb898b6eda..2d1ab1457bf 100644 --- a/tests/functional/parser/test_sqs.py +++ b/tests/functional/parser/test_sqs.py @@ -9,7 +9,7 @@ from tests.functional.validator.conftest import sqs_event # noqa: F401 -@parser(schema=MySqsBusiness, envelope=envelopes.SQS) +@parser(schema=MySqsBusiness, envelope=envelopes.SqsEnvelope) def handle_sqs_json_body(event: List[MySqsBusiness], _: LambdaContext): assert len(event) == 1 assert event[0].message == "hello world" From b1a4821075d74debb5bdddccfb76769133256411 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 12 Oct 2020 13:29:41 +0200 Subject: [PATCH 17/22] improv: expose all pydantic imports --- aws_lambda_powertools/utilities/parser/pydantic.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 aws_lambda_powertools/utilities/parser/pydantic.py diff --git a/aws_lambda_powertools/utilities/parser/pydantic.py b/aws_lambda_powertools/utilities/parser/pydantic.py new file mode 100644 index 00000000000..9060eb57749 --- /dev/null +++ b/aws_lambda_powertools/utilities/parser/pydantic.py @@ -0,0 +1,8 @@ +# Pydantic has many utilities that some advanced customers typically use. +# Importing what's currently in the docs would likely miss something. +# As Pydantic export new types, new utilities, we will have to keep up +# with a project that's not used in our core functionalities. +# For this reason, we're relying on Pydantic's __all__ attr to allow customers +# to use `from from aws_lambda_powertools.utilities.parser.pydantic import ` + +from pydantic import * # noqa: F403,F401 From 0189db08590a0439997fa7babfa9f1bbee247055 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 12 Oct 2020 13:40:40 +0200 Subject: [PATCH 18/22] chore: explicit DynamoDB Stream schema naming --- .../utilities/parser/envelopes/dynamodb.py | 4 ++-- .../utilities/parser/schemas/__init__.py | 8 ++++---- .../utilities/parser/schemas/dynamodb.py | 10 +++++----- tests/functional/parser/schemas.py | 14 +++++++------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py b/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py index 51e1d4f3381..a35dd689a57 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py @@ -4,7 +4,7 @@ from pydantic import BaseModel from typing_extensions import Literal -from ..schemas import DynamoDBSchema +from ..schemas import DynamoDBStreamSchema from .base import BaseEnvelope logger = logging.getLogger(__name__) @@ -32,7 +32,7 @@ def parse(self, data: Dict[str, Any], schema: BaseModel) -> List[Dict[Literal["N List List of records parsed with schema provided """ - parsed_envelope = DynamoDBSchema(**data) + parsed_envelope = DynamoDBStreamSchema(**data) output = [] for record in parsed_envelope.Records: output.append( diff --git a/aws_lambda_powertools/utilities/parser/schemas/__init__.py b/aws_lambda_powertools/utilities/parser/schemas/__init__.py index ac470a16c94..914eb9d50d8 100644 --- a/aws_lambda_powertools/utilities/parser/schemas/__init__.py +++ b/aws_lambda_powertools/utilities/parser/schemas/__init__.py @@ -1,12 +1,12 @@ -from .dynamodb import DynamoDBSchema, DynamoRecordSchema, DynamoScheme +from .dynamodb import DynamoDBStreamChangedRecordSchema, DynamoDBStreamRecordSchema, DynamoDBStreamSchema from .event_bridge import EventBridgeSchema from .sqs import SqsRecordSchema, SqsSchema __all__ = [ - "DynamoDBSchema", + "DynamoDBStreamSchema", "EventBridgeSchema", - "DynamoScheme", - "DynamoRecordSchema", + "DynamoDBStreamChangedRecordSchema", + "DynamoDBStreamRecordSchema", "SqsSchema", "SqsRecordSchema", ] diff --git a/aws_lambda_powertools/utilities/parser/schemas/dynamodb.py b/aws_lambda_powertools/utilities/parser/schemas/dynamodb.py index 3dc8b021fa9..d50ad0a891d 100644 --- a/aws_lambda_powertools/utilities/parser/schemas/dynamodb.py +++ b/aws_lambda_powertools/utilities/parser/schemas/dynamodb.py @@ -5,7 +5,7 @@ from typing_extensions import Literal -class DynamoScheme(BaseModel): +class DynamoDBStreamChangedRecordSchema(BaseModel): ApproximateCreationDateTime: Optional[date] Keys: Dict[str, Dict[str, Any]] NewImage: Optional[Dict[str, Any]] @@ -31,16 +31,16 @@ class UserIdentity(BaseModel): principalId: Literal["dynamodb.amazonaws.com"] -class DynamoRecordSchema(BaseModel): +class DynamoDBStreamRecordSchema(BaseModel): eventID: str eventName: Literal["INSERT", "MODIFY", "REMOVE"] eventVersion: float eventSource: Literal["aws:dynamodb"] awsRegion: str eventSourceARN: str - dynamodb: DynamoScheme + dynamodb: DynamoDBStreamChangedRecordSchema userIdentity: Optional[UserIdentity] -class DynamoDBSchema(BaseModel): - Records: List[DynamoRecordSchema] +class DynamoDBStreamSchema(BaseModel): + Records: List[DynamoDBStreamRecordSchema] diff --git a/tests/functional/parser/schemas.py b/tests/functional/parser/schemas.py index b0b61fe61d4..5ed38f0a6d1 100644 --- a/tests/functional/parser/schemas.py +++ b/tests/functional/parser/schemas.py @@ -4,9 +4,9 @@ from typing_extensions import Literal from aws_lambda_powertools.utilities.parser.schemas import ( - DynamoDBSchema, - DynamoRecordSchema, - DynamoScheme, + DynamoDBStreamChangedRecordSchema, + DynamoDBStreamRecordSchema, + DynamoDBStreamSchema, EventBridgeSchema, SqsRecordSchema, SqsSchema, @@ -18,17 +18,17 @@ class MyDynamoBusiness(BaseModel): Id: Dict[Literal["N"], int] -class MyDynamoScheme(DynamoScheme): +class MyDynamoScheme(DynamoDBStreamChangedRecordSchema): NewImage: Optional[MyDynamoBusiness] OldImage: Optional[MyDynamoBusiness] -class MyDynamoRecordSchema(DynamoRecordSchema): +class MyDynamoDBStreamRecordSchema(DynamoDBStreamRecordSchema): dynamodb: MyDynamoScheme -class MyAdvancedDynamoBusiness(DynamoDBSchema): - Records: List[MyDynamoRecordSchema] +class MyAdvancedDynamoBusiness(DynamoDBStreamSchema): + Records: List[MyDynamoDBStreamRecordSchema] class MyEventbridgeBusiness(BaseModel): From 22a9a8428db48a02c8e763cbd8ca3cf9825a3783 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 12 Oct 2020 13:47:09 +0200 Subject: [PATCH 19/22] chore: remove flake8 polyfill as explicit dep --- poetry.lock | 607 +++++++++++++++++++++++-------------------------- pyproject.toml | 1 - 2 files changed, 289 insertions(+), 319 deletions(-) diff --git a/poetry.lock b/poetry.lock index 94bf8b20b7c..b255194ddc5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,41 +1,40 @@ [[package]] -category = "dev" -description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." name = "appdirs" +version = "1.4.4" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" optional = false python-versions = "*" -version = "1.4.4" [[package]] -category = "dev" -description = "Atomic file writes." -marker = "sys_platform == \"win32\"" name = "atomicwrites" +version = "1.4.0" +description = "Atomic file writes." +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "1.4.0" [[package]] -category = "dev" -description = "Classes Without Boilerplate" name = "attrs" +version = "20.2.0" +description = "Classes Without Boilerplate" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "20.2.0" [package.extras] -dev = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "sphinx-rtd-theme", "pre-commit"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "sphinx-rtd-theme", "pre-commit"] docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] -tests = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] -tests_no_zope = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] [[package]] -category = "main" -description = "The AWS X-Ray SDK for Python (the SDK) enables Python developers to record and emit information from within their applications to the AWS X-Ray service." name = "aws-xray-sdk" +version = "2.6.0" +description = "The AWS X-Ray SDK for Python (the SDK) enables Python developers to record and emit information from within their applications to the AWS X-Ray service." +category = "main" optional = false python-versions = "*" -version = "2.6.0" [package.dependencies] botocore = ">=1.11.3" @@ -44,27 +43,27 @@ jsonpickle = "*" wrapt = "*" [[package]] -category = "dev" -description = "Security oriented static analyser for python code." name = "bandit" +version = "1.6.2" +description = "Security oriented static analyser for python code." +category = "dev" optional = false python-versions = "*" -version = "1.6.2" [package.dependencies] +colorama = {version = ">=0.3.9", markers = "platform_system == \"Windows\""} GitPython = ">=1.0.1" PyYAML = ">=3.13" -colorama = ">=0.3.9" six = ">=1.10.0" stevedore = ">=1.20.0" [[package]] -category = "dev" -description = "The uncompromising code formatter." name = "black" +version = "19.10b0" +description = "The uncompromising code formatter." +category = "dev" optional = false python-versions = ">=3.6" -version = "19.10b0" [package.dependencies] appdirs = "*" @@ -79,159 +78,149 @@ typed-ast = ">=1.4.0" d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] [[package]] -category = "main" -description = "The AWS SDK for Python" name = "boto3" +version = "1.15.16" +description = "The AWS SDK for Python" +category = "main" optional = false python-versions = "*" -version = "1.15.5" [package.dependencies] -botocore = ">=1.18.5,<1.19.0" +botocore = ">=1.18.16,<1.19.0" jmespath = ">=0.7.1,<1.0.0" s3transfer = ">=0.3.0,<0.4.0" [[package]] -category = "main" -description = "Low-level, data-driven core of boto 3." name = "botocore" +version = "1.18.16" +description = "Low-level, data-driven core of boto 3." +category = "main" optional = false python-versions = "*" -version = "1.18.5" [package.dependencies] jmespath = ">=0.7.1,<1.0.0" python-dateutil = ">=2.1,<3.0.0" - -[package.dependencies.urllib3] -python = "<3.4.0 || >=3.5.0" -version = ">=1.20,<1.26" +urllib3 = {version = ">=1.20,<1.26", markers = "python_version != \"3.4\""} [[package]] -category = "dev" -description = "Python package for providing Mozilla's CA Bundle." name = "certifi" +version = "2020.6.20" +description = "Python package for providing Mozilla's CA Bundle." +category = "dev" optional = false python-versions = "*" -version = "2020.6.20" [[package]] -category = "dev" -description = "Universal encoding detector for Python 2 and 3" name = "chardet" +version = "3.0.4" +description = "Universal encoding detector for Python 2 and 3" +category = "dev" optional = false python-versions = "*" -version = "3.0.4" [[package]] -category = "dev" -description = "Composable command line interface toolkit" name = "click" +version = "7.1.2" +description = "Composable command line interface toolkit" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "7.1.2" [[package]] -category = "dev" -description = "Cross-platform colored terminal text." -marker = "sys_platform == \"win32\" or platform_system == \"Windows\" or python_version > \"3.4\"" name = "colorama" +version = "0.4.3" +description = "Cross-platform colored terminal text." +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "0.4.3" [[package]] -category = "dev" -description = "Code coverage measurement for Python" name = "coverage" +version = "5.3" +description = "Code coverage measurement for Python" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" -version = "5.3" [package.dependencies] -[package.dependencies.toml] -optional = true -version = "*" +toml = {version = "*", optional = true, markers = "extra == \"toml\""} [package.extras] toml = ["toml"] [[package]] -category = "main" -description = "A backport of the dataclasses module for Python 3.6" -marker = "python_version < \"3.7\"" name = "dataclasses" +version = "0.7" +description = "A backport of the dataclasses module for Python 3.6" +category = "main" optional = true -python-versions = "*" -version = "0.6" +python-versions = ">=3.6, <3.7" [[package]] -category = "dev" -description = "Removes commented-out code." name = "eradicate" +version = "1.0" +description = "Removes commented-out code." +category = "dev" optional = false python-versions = "*" -version = "1.0" [[package]] -category = "main" -description = "Fastest Python implementation of JSON schema" name = "fastjsonschema" +version = "2.14.5" +description = "Fastest Python implementation of JSON schema" +category = "main" optional = false python-versions = "*" -version = "2.14.5" [package.extras] devel = ["colorama", "jsonschema", "json-spec", "pylint", "pytest", "pytest-benchmark", "pytest-cache", "validictory"] [[package]] -category = "dev" -description = "the modular source code checker: pep8 pyflakes and co" name = "flake8" +version = "3.8.4" +description = "the modular source code checker: pep8 pyflakes and co" +category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" -version = "3.8.3" [package.dependencies] +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} mccabe = ">=0.6.0,<0.7.0" pycodestyle = ">=2.6.0a1,<2.7.0" pyflakes = ">=2.2.0,<2.3.0" -[package.dependencies.importlib-metadata] -python = "<3.8" -version = "*" - [[package]] -category = "dev" -description = "flake8 plugin to call black as a code style validator" name = "flake8-black" +version = "0.1.2" +description = "flake8 plugin to call black as a code style validator" +category = "dev" optional = false python-versions = "*" -version = "0.1.2" [package.dependencies] black = ">=19.3b0" flake8 = ">=3.0.0" [[package]] -category = "dev" -description = "A plugin for flake8 finding likely bugs and design problems in your program. Contains warnings that don't belong in pyflakes and pycodestyle." name = "flake8-bugbear" +version = "20.1.4" +description = "A plugin for flake8 finding likely bugs and design problems in your program. Contains warnings that don't belong in pyflakes and pycodestyle." +category = "dev" optional = false python-versions = ">=3.6" -version = "20.1.4" [package.dependencies] attrs = ">=19.2.0" flake8 = ">=3.0.0" [[package]] -category = "dev" -description = "Check for python builtins being used as variables or parameters." name = "flake8-builtins" +version = "1.5.3" +description = "Check for python builtins being used as variables or parameters." +category = "dev" optional = false python-versions = "*" -version = "1.5.3" [package.dependencies] flake8 = "*" @@ -240,39 +229,36 @@ flake8 = "*" test = ["coverage", "coveralls", "mock", "pytest", "pytest-cov"] [[package]] -category = "dev" -description = "A flake8 plugin to help you write better list/set/dict comprehensions." name = "flake8-comprehensions" +version = "3.2.3" +description = "A flake8 plugin to help you write better list/set/dict comprehensions." +category = "dev" optional = false python-versions = ">=3.5" -version = "3.2.3" [package.dependencies] flake8 = ">=3.0,<3.2.0 || >3.2.0,<4" - -[package.dependencies.importlib-metadata] -python = "<3.8" -version = "*" +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [[package]] -category = "dev" -description = "ipdb/pdb statement checker plugin for flake8" name = "flake8-debugger" +version = "3.2.1" +description = "ipdb/pdb statement checker plugin for flake8" +category = "dev" optional = false python-versions = "*" -version = "3.2.1" [package.dependencies] flake8 = ">=1.5" pycodestyle = "*" [[package]] -category = "dev" -description = "Flake8 plugin to find commented out code" name = "flake8-eradicate" +version = "0.3.0" +description = "Flake8 plugin to find commented out code" +category = "dev" optional = false python-versions = ">=3.6,<4.0" -version = "0.3.0" [package.dependencies] attrs = "*" @@ -280,99 +266,93 @@ eradicate = ">=1.0,<2.0" flake8 = ">=3.5,<4.0" [[package]] -category = "dev" -description = "Check for FIXME, TODO and other temporary developer notes. Plugin for flake8." name = "flake8-fixme" +version = "1.1.1" +description = "Check for FIXME, TODO and other temporary developer notes. Plugin for flake8." +category = "dev" optional = false python-versions = "*" -version = "1.1.1" [[package]] -category = "dev" -description = "flake8 plugin that integrates isort ." name = "flake8-isort" +version = "2.9.1" +description = "flake8 plugin that integrates isort ." +category = "dev" optional = false python-versions = "*" -version = "2.9.1" [package.dependencies] flake8 = ">=3.2.1" +isort = {version = ">=4.3.5", extras = ["pyproject"]} testfixtures = "*" -[package.dependencies.isort] -extras = ["pyproject"] -version = ">=4.3.5" - [package.extras] test = ["pytest"] [[package]] -category = "dev" -description = "Polyfill package for Flake8 plugins" name = "flake8-polyfill" +version = "1.0.2" +description = "Polyfill package for Flake8 plugins" +category = "dev" optional = false python-versions = "*" -version = "1.0.2" [package.dependencies] flake8 = "*" [[package]] -category = "dev" -description = "A flake8 extension that helps to make more readable variables names" name = "flake8-variables-names" +version = "0.0.3" +description = "A flake8 extension that helps to make more readable variables names" +category = "dev" optional = false python-versions = "*" -version = "0.0.3" - -[package.dependencies] -setuptools = "*" [[package]] -category = "main" -description = "Clean single-source support for Python 3 and 2" name = "future" +version = "0.18.2" +description = "Clean single-source support for Python 3 and 2" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -version = "0.18.2" [[package]] -category = "dev" -description = "Git Object Database" name = "gitdb" +version = "4.0.5" +description = "Git Object Database" +category = "dev" optional = false python-versions = ">=3.4" -version = "4.0.5" [package.dependencies] smmap = ">=3.0.1,<4" [[package]] -category = "dev" -description = "Python Git Library" name = "gitpython" +version = "3.1.9" +description = "Python Git Library" +category = "dev" optional = false python-versions = ">=3.4" -version = "3.1.8" [package.dependencies] gitdb = ">=4.0.1,<5" [[package]] -category = "dev" -description = "Internationalized Domain Names in Applications (IDNA)" name = "idna" +version = "2.10" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "2.10" [[package]] -category = "main" -description = "Read metadata from Python packages" name = "importlib-metadata" +version = "2.0.0" +description = "Read metadata from Python packages" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -version = "2.0.0" [package.dependencies] zipp = ">=0.5" @@ -382,12 +362,15 @@ docs = ["sphinx", "rst.linker"] testing = ["packaging", "pep517", "importlib-resources (>=1.3)"] [[package]] -category = "dev" -description = "A Python utility / library to sort Python imports." name = "isort" +version = "4.3.21" +description = "A Python utility / library to sort Python imports." +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "4.3.21" + +[package.dependencies] +toml = {version = "*", optional = true, markers = "extra == \"pyproject\""} [package.extras] pipfile = ["pipreqs", "requirementslib"] @@ -396,20 +379,20 @@ requirements = ["pipreqs", "pip-api"] xdg_home = ["appdirs (>=1.4.0)"] [[package]] -category = "main" -description = "JSON Matching Expressions" name = "jmespath" +version = "0.10.0" +description = "JSON Matching Expressions" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -version = "0.10.0" [[package]] -category = "main" -description = "Python library for serializing any arbitrary object graph into JSON" name = "jsonpickle" +version = "1.4.1" +description = "Python library for serializing any arbitrary object graph into JSON" +category = "main" optional = false python-versions = ">=2.7" -version = "1.4.1" [package.dependencies] importlib-metadata = "*" @@ -420,12 +403,12 @@ testing = ["coverage (<5)", "pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs "testing.libs" = ["demjson", "simplejson", "ujson", "yajl"] [[package]] -category = "dev" -description = "A super-fast templating language that borrows the best ideas from the existing templating languages." name = "mako" +version = "1.1.3" +description = "A super-fast templating language that borrows the best ideas from the existing templating languages." +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "1.1.3" [package.dependencies] MarkupSafe = ">=0.9.2" @@ -435,12 +418,12 @@ babel = ["babel"] lingua = ["lingua"] [[package]] -category = "dev" -description = "Create Python CLI apps with little to no effort at all!" name = "mando" +version = "0.6.4" +description = "Create Python CLI apps with little to no effort at all!" +category = "dev" optional = false python-versions = "*" -version = "0.6.4" [package.dependencies] six = "*" @@ -449,129 +432,123 @@ six = "*" restructuredText = ["rst2ansi"] [[package]] -category = "dev" -description = "Python implementation of Markdown." name = "markdown" +version = "3.3" +description = "Python implementation of Markdown." +category = "dev" optional = false -python-versions = ">=3.5" -version = "3.2.2" +python-versions = ">=3.6" [package.dependencies] -[package.dependencies.importlib-metadata] -python = "<3.8" -version = "*" +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [package.extras] testing = ["coverage", "pyyaml"] [[package]] -category = "dev" -description = "Safely add untrusted strings to HTML/XML markup." name = "markupsafe" +version = "1.1.1" +description = "Safely add untrusted strings to HTML/XML markup." +category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" -version = "1.1.1" [[package]] -category = "dev" -description = "McCabe checker, plugin for flake8" name = "mccabe" +version = "0.6.1" +description = "McCabe checker, plugin for flake8" +category = "dev" optional = false python-versions = "*" -version = "0.6.1" [[package]] -category = "dev" -description = "More routines for operating on iterables, beyond itertools" name = "more-itertools" +version = "8.5.0" +description = "More routines for operating on iterables, beyond itertools" +category = "dev" optional = false python-versions = ">=3.5" -version = "8.5.0" [[package]] -category = "dev" -description = "Core utilities for Python packages" name = "packaging" +version = "20.4" +description = "Core utilities for Python packages" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "20.4" [package.dependencies] pyparsing = ">=2.0.2" six = "*" [[package]] -category = "dev" -description = "Utility library for gitignore style pattern matching of file paths." name = "pathspec" +version = "0.8.0" +description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "0.8.0" [[package]] -category = "dev" -description = "Python Build Reasonableness" name = "pbr" +version = "5.5.0" +description = "Python Build Reasonableness" +category = "dev" optional = false python-versions = ">=2.6" -version = "5.5.0" [[package]] -category = "dev" -description = "Auto-generate API documentation for Python projects." name = "pdoc3" +version = "0.7.5" +description = "Auto-generate API documentation for Python projects." +category = "dev" optional = false python-versions = ">= 3.5" -version = "0.7.5" [package.dependencies] mako = "*" markdown = ">=3.0" [[package]] -category = "dev" -description = "plugin and hook calling mechanisms for python" name = "pluggy" +version = "0.13.1" +description = "plugin and hook calling mechanisms for python" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "0.13.1" [package.dependencies] -[package.dependencies.importlib-metadata] -python = "<3.8" -version = ">=0.12" +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} [package.extras] dev = ["pre-commit", "tox"] [[package]] -category = "dev" -description = "library with cross-python path, ini-parsing, io, code, log facilities" name = "py" +version = "1.9.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "1.9.0" [[package]] -category = "dev" -description = "Python style guide checker" name = "pycodestyle" +version = "2.6.0" +description = "Python style guide checker" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "2.6.0" [[package]] -category = "main" -description = "Data validation and settings management using python 3.6 type hinting" name = "pydantic" +version = "1.6.1" +description = "Data validation and settings management using python 3.6 type hinting" +category = "main" optional = true python-versions = ">=3.6" -version = "1.6.1" [package.dependencies] -[package.dependencies.dataclasses] -python = "<3.7" -version = ">=0.6" +dataclasses = {version = ">=0.6", markers = "python_version < \"3.7\""} [package.extras] dotenv = ["python-dotenv (>=0.10.4)"] @@ -579,54 +556,51 @@ email = ["email-validator (>=1.0.3)"] typing_extensions = ["typing-extensions (>=3.7.2)"] [[package]] -category = "dev" -description = "passive checker of Python programs" name = "pyflakes" +version = "2.2.0" +description = "passive checker of Python programs" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "2.2.0" [[package]] -category = "dev" -description = "Python parsing module" name = "pyparsing" +version = "2.4.7" +description = "Python parsing module" +category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -version = "2.4.7" [[package]] -category = "dev" -description = "pytest: simple powerful testing with Python" name = "pytest" +version = "5.4.3" +description = "pytest: simple powerful testing with Python" +category = "dev" optional = false python-versions = ">=3.5" -version = "5.4.3" [package.dependencies] -atomicwrites = ">=1.0" +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} attrs = ">=17.4.0" -colorama = "*" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} more-itertools = ">=4.0.0" packaging = "*" pluggy = ">=0.12,<1.0" py = ">=1.5.0" wcwidth = "*" -[package.dependencies.importlib-metadata] -python = "<3.8" -version = ">=0.12" - [package.extras] checkqa-mypy = ["mypy (v0.761)"] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] [[package]] -category = "dev" -description = "Pytest support for asyncio." name = "pytest-asyncio" +version = "0.12.0" +description = "Pytest support for asyncio." +category = "dev" optional = false python-versions = ">= 3.5" -version = "0.12.0" [package.dependencies] pytest = ">=5.4.0" @@ -635,12 +609,12 @@ pytest = ">=5.4.0" testing = ["async_generator (>=1.3)", "coverage", "hypothesis (>=5.7.1)"] [[package]] -category = "dev" -description = "Pytest plugin for measuring coverage." name = "pytest-cov" +version = "2.10.1" +description = "Pytest plugin for measuring coverage." +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "2.10.1" [package.dependencies] coverage = ">=4.4" @@ -650,12 +624,12 @@ pytest = ">=4.6" testing = ["fields", "hunter", "process-tests (2.0.2)", "six", "pytest-xdist", "virtualenv"] [[package]] -category = "dev" -description = "Thin-wrapper around the mock package for easier use with py.test" name = "pytest-mock" +version = "2.0.0" +description = "Thin-wrapper around the mock package for easier use with py.test" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "2.0.0" [package.dependencies] pytest = ">=2.7" @@ -664,58 +638,56 @@ pytest = ">=2.7" dev = ["pre-commit", "tox"] [[package]] -category = "main" -description = "Extensions to the standard Python datetime module" name = "python-dateutil" +version = "2.8.1" +description = "Extensions to the standard Python datetime module" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -version = "2.8.1" [package.dependencies] six = ">=1.5" [[package]] -category = "dev" -description = "YAML parser and emitter for Python" name = "pyyaml" +version = "5.3.1" +description = "YAML parser and emitter for Python" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "5.3.1" [[package]] -category = "dev" -description = "Code Metrics in Python" name = "radon" +version = "4.3.2" +description = "Code Metrics in Python" +category = "dev" optional = false python-versions = "*" -version = "4.3.2" [package.dependencies] +colorama = {version = ">=0.4.1", markers = "python_version > \"3.4\""} +flake8-polyfill = {version = "*", optional = true, markers = "extra == \"flake8\""} future = "*" mando = ">=0.6,<0.7" -[package.dependencies.colorama] -python = ">=3.5" -version = ">=0.4.1" - [package.extras] flake8 = ["flake8-polyfill"] [[package]] -category = "dev" -description = "Alternative regular expression module, to replace re." name = "regex" +version = "2020.10.11" +description = "Alternative regular expression module, to replace re." +category = "dev" optional = false python-versions = "*" -version = "2020.7.14" [[package]] -category = "dev" -description = "Python HTTP for Humans." name = "requests" +version = "2.24.0" +description = "Python HTTP for Humans." +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -version = "2.24.0" [package.dependencies] certifi = ">=2017.4.17" @@ -728,54 +700,51 @@ security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] [[package]] -category = "main" -description = "An Amazon S3 Transfer Manager" name = "s3transfer" +version = "0.3.3" +description = "An Amazon S3 Transfer Manager" +category = "main" optional = false python-versions = "*" -version = "0.3.3" [package.dependencies] botocore = ">=1.12.36,<2.0a.0" [[package]] -category = "main" -description = "Python 2 and 3 compatibility utilities" name = "six" +version = "1.15.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -version = "1.15.0" [[package]] -category = "dev" -description = "A pure Python implementation of a sliding window memory map manager" name = "smmap" +version = "3.0.4" +description = "A pure Python implementation of a sliding window memory map manager" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "3.0.4" [[package]] -category = "dev" -description = "Manage dynamic plugins for Python applications" name = "stevedore" +version = "3.2.2" +description = "Manage dynamic plugins for Python applications" +category = "dev" optional = false python-versions = ">=3.6" -version = "3.2.2" [package.dependencies] +importlib-metadata = {version = ">=1.7.0", markers = "python_version < \"3.8\""} pbr = ">=2.0.0,<2.1.0 || >2.1.0" -[package.dependencies.importlib-metadata] -python = "<3.8" -version = ">=1.7.0" - [[package]] -category = "dev" -description = "A collection of helpers and mock objects for unit tests and doc tests." name = "testfixtures" +version = "6.15.0" +description = "A collection of helpers and mock objects for unit tests and doc tests." +category = "dev" optional = false python-versions = "*" -version = "6.14.2" [package.extras] build = ["setuptools-git", "wheel", "twine"] @@ -783,36 +752,36 @@ docs = ["sphinx", "zope.component", "sybil", "twisted", "mock", "django (<2)", " test = ["pytest (>=3.6)", "pytest-cov", "pytest-django", "zope.component", "sybil", "twisted", "mock", "django (<2)", "django"] [[package]] -category = "dev" -description = "Python Library for Tom's Obvious, Minimal Language" name = "toml" +version = "0.10.1" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "dev" optional = false python-versions = "*" -version = "0.10.1" [[package]] -category = "dev" -description = "a fork of Python 2 and 3 ast modules with type comment support" name = "typed-ast" +version = "1.4.1" +description = "a fork of Python 2 and 3 ast modules with type comment support" +category = "dev" optional = false python-versions = "*" -version = "1.4.1" [[package]] -category = "main" -description = "Backported and Experimental Type Hints for Python 3.5+" name = "typing-extensions" +version = "3.7.4.3" +description = "Backported and Experimental Type Hints for Python 3.5+" +category = "main" optional = true python-versions = "*" -version = "3.7.4.3" [[package]] -category = "main" -description = "HTTP library with thread-safe connection pooling, file post, and more." name = "urllib3" +version = "1.25.10" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" -version = "1.25.10" [package.extras] brotli = ["brotlipy (>=0.6.0)"] @@ -820,45 +789,41 @@ secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=0 socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] [[package]] -category = "dev" -description = "Measures the displayed width of unicode strings in a terminal" name = "wcwidth" +version = "0.2.5" +description = "Measures the displayed width of unicode strings in a terminal" +category = "dev" optional = false python-versions = "*" -version = "0.2.5" [[package]] -category = "main" -description = "Module for decorators, wrappers and monkey patching." name = "wrapt" +version = "1.12.1" +description = "Module for decorators, wrappers and monkey patching." +category = "main" optional = false python-versions = "*" -version = "1.12.1" [[package]] -category = "dev" -description = "Monitor code metrics for Python on your CI server" name = "xenon" +version = "0.7.1" +description = "Monitor code metrics for Python on your CI server" +category = "dev" optional = false python-versions = "*" -version = "0.7.1" [package.dependencies] PyYAML = ">=4.2b1,<6.0" +radon = {version = ">=4,<5", extras = ["flake8"]} requests = ">=2.0,<3.0" -[package.dependencies.radon] -extras = ["flake8"] -version = ">=4,<5" - [[package]] -category = "main" -description = "Backport of pathlib-compatible object wrapper for zip files" -marker = "python_version < \"3.8\"" name = "zipp" +version = "3.3.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" optional = false python-versions = ">=3.6" -version = "3.2.0" [package.extras] docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] @@ -868,9 +833,9 @@ testing = ["pytest (>=3.5,<3.7.3 || >3.7.3)", "pytest-checkdocs (>=1.2.3)", "pyt pydantic = ["pydantic", "typing_extensions"] [metadata] -content-hash = "f2207b4e243108a8b2b2eee5a56f648519d2ce8cb893f4e3c8fb346a44374eaa" -lock-version = "1.0" +lock-version = "1.1" python-versions = "^3.6" +content-hash = "e18b9f99b7876adb78623fd8b2acb9a6f76a5e427c30d0c9ec7ebb5786bc4a52" [metadata.files] appdirs = [ @@ -898,12 +863,12 @@ black = [ {file = "black-19.10b0.tar.gz", hash = "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"}, ] boto3 = [ - {file = "boto3-1.15.5-py2.py3-none-any.whl", hash = "sha256:0c464a7de522f88b581ca0d41ffa71e9be5e17fbb0456c275421f65b7c5f6a55"}, - {file = "boto3-1.15.5.tar.gz", hash = "sha256:0fce548e19d6db8e11fd0e2ae7809e1e3282080636b4062b2452bfa20e4f0233"}, + {file = "boto3-1.15.16-py2.py3-none-any.whl", hash = "sha256:557320fe8b65cfc85953e6a63d2328e8efec95bf4ec383b92fa2d01119209716"}, + {file = "boto3-1.15.16.tar.gz", hash = "sha256:454a8dfb7b367a058c7967ef6b4e2a192c318f10761769fd1003cf7f2f5a7db9"}, ] botocore = [ - {file = "botocore-1.18.5-py2.py3-none-any.whl", hash = "sha256:e3bf44fba058f6df16006b94a67650418a080a525c82521abb3cb516a4cba362"}, - {file = "botocore-1.18.5.tar.gz", hash = "sha256:7ce7a05b98ffb3170396960273383e8aade9be6026d5a762f5f40969d5d6b761"}, + {file = "botocore-1.18.16-py2.py3-none-any.whl", hash = "sha256:e586e4d6eddbca31e6447a25df9972329ea3de64b1fb0eb17e7ab0c9b91f7720"}, + {file = "botocore-1.18.16.tar.gz", hash = "sha256:f0616d2c719691b94470307cee8adf89ceb1657b7b6f9aa1bf61f9de5543dbbb"}, ] certifi = [ {file = "certifi-2020.6.20-py2.py3-none-any.whl", hash = "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"}, @@ -958,8 +923,8 @@ coverage = [ {file = "coverage-5.3.tar.gz", hash = "sha256:280baa8ec489c4f542f8940f9c4c2181f0306a8ee1a54eceba071a449fb870a0"}, ] dataclasses = [ - {file = "dataclasses-0.6-py3-none-any.whl", hash = "sha256:454a69d788c7fda44efd71e259be79577822f5e3f53f029a22d08004e951dc9f"}, - {file = "dataclasses-0.6.tar.gz", hash = "sha256:6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84"}, + {file = "dataclasses-0.7-py3-none-any.whl", hash = "sha256:3459118f7ede7c8bea0fe795bff7c6c2ce287d01dd226202f7c9ebc0610a7836"}, + {file = "dataclasses-0.7.tar.gz", hash = "sha256:494a6dcae3b8bcf80848eea2ef64c0cc5cd307ffc263e17cdf42f3e5420808e6"}, ] eradicate = [ {file = "eradicate-1.0.tar.gz", hash = "sha256:4ffda82aae6fd49dfffa777a857cb758d77502a1f2e0f54c9ac5155a39d2d01a"}, @@ -969,8 +934,8 @@ fastjsonschema = [ {file = "fastjsonschema-2.14.5.tar.gz", hash = "sha256:afbc235655f06356e46caa80190512e4d9222abfaca856041be5a74c665fa094"}, ] flake8 = [ - {file = "flake8-3.8.3-py2.py3-none-any.whl", hash = "sha256:15e351d19611c887e482fb960eae4d44845013cc142d42896e9862f775d8cf5c"}, - {file = "flake8-3.8.3.tar.gz", hash = "sha256:f04b9fcbac03b0a3e58c0ab3a0ecc462e023a9faf046d57794184028123aa208"}, + {file = "flake8-3.8.4-py2.py3-none-any.whl", hash = "sha256:749dbbd6bfd0cf1318af27bf97a14e28e5ff548ef8e5b1566ccfb25a11e7c839"}, + {file = "flake8-3.8.4.tar.gz", hash = "sha256:aadae8761ec651813c24be05c6f7b4680857ef6afaae4651a4eccaef97ce6c3b"}, ] flake8-black = [ {file = "flake8-black-0.1.2.tar.gz", hash = "sha256:b79d8d868bd42dc2c1f27469b92a984ecab3579ad285a8708ea5f19bf6c1f3a2"}, @@ -1017,8 +982,8 @@ gitdb = [ {file = "gitdb-4.0.5.tar.gz", hash = "sha256:c9e1f2d0db7ddb9a704c2a0217be31214e91a4fe1dea1efad19ae42ba0c285c9"}, ] gitpython = [ - {file = "GitPython-3.1.8-py3-none-any.whl", hash = "sha256:1858f4fd089abe92ae465f01d5aaaf55e937eca565fb2c1fce35a51b5f85c910"}, - {file = "GitPython-3.1.8.tar.gz", hash = "sha256:080bf8e2cf1a2b907634761c2eaefbe83b69930c94c66ad11b65a8252959f912"}, + {file = "GitPython-3.1.9-py3-none-any.whl", hash = "sha256:138016d519bf4dd55b22c682c904ed2fd0235c3612b2f8f65ce218ff358deed8"}, + {file = "GitPython-3.1.9.tar.gz", hash = "sha256:a03f728b49ce9597a6655793207c6ab0da55519368ff5961e4a74ae475b9fa8e"}, ] idna = [ {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, @@ -1049,8 +1014,8 @@ mando = [ {file = "mando-0.6.4.tar.gz", hash = "sha256:79feb19dc0f097daa64a1243db578e7674909b75f88ac2220f1c065c10a0d960"}, ] markdown = [ - {file = "Markdown-3.2.2-py3-none-any.whl", hash = "sha256:c467cd6233885534bf0fe96e62e3cf46cfc1605112356c4f9981512b8174de59"}, - {file = "Markdown-3.2.2.tar.gz", hash = "sha256:1fafe3f1ecabfb514a5285fca634a53c1b32a81cb0feb154264d55bf2ff22c17"}, + {file = "Markdown-3.3-py3-none-any.whl", hash = "sha256:fbb1ba54ca41e8991dc5a561d9c6f752f5e4546f8750e56413ea50f2385761d3"}, + {file = "Markdown-3.3.tar.gz", hash = "sha256:4f4172a4e989b97f96860fa434b89895069c576e2b537c4b4eed265266a7affc"}, ] markupsafe = [ {file = "MarkupSafe-1.1.1-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161"}, @@ -1186,27 +1151,33 @@ radon = [ {file = "radon-4.3.2.tar.gz", hash = "sha256:758b3ab345aa86e95f642713612a57da7c7da6d552c4dbfbe397a67601ace7dd"}, ] regex = [ - {file = "regex-2020.7.14-cp27-cp27m-win32.whl", hash = "sha256:e46d13f38cfcbb79bfdb2964b0fe12561fe633caf964a77a5f8d4e45fe5d2ef7"}, - {file = "regex-2020.7.14-cp27-cp27m-win_amd64.whl", hash = "sha256:6961548bba529cac7c07af2fd4d527c5b91bb8fe18995fed6044ac22b3d14644"}, - {file = "regex-2020.7.14-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c50a724d136ec10d920661f1442e4a8b010a4fe5aebd65e0c2241ea41dbe93dc"}, - {file = "regex-2020.7.14-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:8a51f2c6d1f884e98846a0a9021ff6861bdb98457879f412fdc2b42d14494067"}, - {file = "regex-2020.7.14-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:9c568495e35599625f7b999774e29e8d6b01a6fb684d77dee1f56d41b11b40cd"}, - {file = "regex-2020.7.14-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:51178c738d559a2d1071ce0b0f56e57eb315bcf8f7d4cf127674b533e3101f88"}, - {file = "regex-2020.7.14-cp36-cp36m-win32.whl", hash = "sha256:9eddaafb3c48e0900690c1727fba226c4804b8e6127ea409689c3bb492d06de4"}, - {file = "regex-2020.7.14-cp36-cp36m-win_amd64.whl", hash = "sha256:14a53646369157baa0499513f96091eb70382eb50b2c82393d17d7ec81b7b85f"}, - {file = "regex-2020.7.14-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:1269fef3167bb52631ad4fa7dd27bf635d5a0790b8e6222065d42e91bede4162"}, - {file = "regex-2020.7.14-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:d0a5095d52b90ff38592bbdc2644f17c6d495762edf47d876049cfd2968fbccf"}, - {file = "regex-2020.7.14-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:4c037fd14c5f4e308b8370b447b469ca10e69427966527edcab07f52d88388f7"}, - {file = "regex-2020.7.14-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:bc3d98f621898b4a9bc7fecc00513eec8f40b5b83913d74ccb445f037d58cd89"}, - {file = "regex-2020.7.14-cp37-cp37m-win32.whl", hash = "sha256:46bac5ca10fb748d6c55843a931855e2727a7a22584f302dd9bb1506e69f83f6"}, - {file = "regex-2020.7.14-cp37-cp37m-win_amd64.whl", hash = "sha256:0dc64ee3f33cd7899f79a8d788abfbec168410be356ed9bd30bbd3f0a23a7204"}, - {file = "regex-2020.7.14-cp38-cp38-manylinux1_i686.whl", hash = "sha256:5ea81ea3dbd6767873c611687141ec7b06ed8bab43f68fad5b7be184a920dc99"}, - {file = "regex-2020.7.14-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:bbb332d45b32df41200380fff14712cb6093b61bd142272a10b16778c418e98e"}, - {file = "regex-2020.7.14-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:c11d6033115dc4887c456565303f540c44197f4fc1a2bfb192224a301534888e"}, - {file = "regex-2020.7.14-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:75aaa27aa521a182824d89e5ab0a1d16ca207318a6b65042b046053cfc8ed07a"}, - {file = "regex-2020.7.14-cp38-cp38-win32.whl", hash = "sha256:d6cff2276e502b86a25fd10c2a96973fdb45c7a977dca2138d661417f3728341"}, - {file = "regex-2020.7.14-cp38-cp38-win_amd64.whl", hash = "sha256:7a2dd66d2d4df34fa82c9dc85657c5e019b87932019947faece7983f2089a840"}, - {file = "regex-2020.7.14.tar.gz", hash = "sha256:3a3af27a8d23143c49a3420efe5b3f8cf1a48c6fc8bc6856b03f638abc1833bb"}, + {file = "regex-2020.10.11-cp27-cp27m-win32.whl", hash = "sha256:4f5c0fe46fb79a7adf766b365cae56cafbf352c27358fda811e4a1dc8216d0db"}, + {file = "regex-2020.10.11-cp27-cp27m-win_amd64.whl", hash = "sha256:39a5ef30bca911f5a8a3d4476f5713ed4d66e313d9fb6755b32bec8a2e519635"}, + {file = "regex-2020.10.11-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:7c4fc5a8ec91a2254bb459db27dbd9e16bba1dabff638f425d736888d34aaefa"}, + {file = "regex-2020.10.11-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:d537e270b3e6bfaea4f49eaf267984bfb3628c86670e9ad2a257358d3b8f0955"}, + {file = "regex-2020.10.11-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:a8240df4957a5b0e641998a5d78b3c4ea762c845d8cb8997bf820626826fde9a"}, + {file = "regex-2020.10.11-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:4302153abb96859beb2c778cc4662607a34175065fc2f33a21f49eb3fbd1ccd3"}, + {file = "regex-2020.10.11-cp36-cp36m-win32.whl", hash = "sha256:c077c9d04a040dba001cf62b3aff08fd85be86bccf2c51a770c77377662a2d55"}, + {file = "regex-2020.10.11-cp36-cp36m-win_amd64.whl", hash = "sha256:46ab6070b0d2cb85700b8863b3f5504c7f75d8af44289e9562195fe02a8dd72d"}, + {file = "regex-2020.10.11-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:d629d750ebe75a88184db98f759633b0a7772c2e6f4da529f0027b4a402c0e2f"}, + {file = "regex-2020.10.11-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:8e7ef296b84d44425760fe813cabd7afbb48c8dd62023018b338bbd9d7d6f2f0"}, + {file = "regex-2020.10.11-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:e490f08897cb44e54bddf5c6e27deca9b58c4076849f32aaa7a0b9f1730f2c20"}, + {file = "regex-2020.10.11-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:850339226aa4fec04916386577674bb9d69abe0048f5d1a99f91b0004bfdcc01"}, + {file = "regex-2020.10.11-cp37-cp37m-win32.whl", hash = "sha256:60c4f64d9a326fe48e8738c3dbc068e1edc41ff7895a9e3723840deec4bc1c28"}, + {file = "regex-2020.10.11-cp37-cp37m-win_amd64.whl", hash = "sha256:8ba3efdd60bfee1aa784dbcea175eb442d059b576934c9d099e381e5a9f48930"}, + {file = "regex-2020.10.11-cp38-cp38-manylinux1_i686.whl", hash = "sha256:2308491b3e6c530a3bb38a8a4bb1dc5fd32cbf1e11ca623f2172ba17a81acef1"}, + {file = "regex-2020.10.11-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:b8806649983a1c78874ec7e04393ef076805740f6319e87a56f91f1767960212"}, + {file = "regex-2020.10.11-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:a2a31ee8a354fa3036d12804730e1e20d58bc4e250365ead34b9c30bbe9908c3"}, + {file = "regex-2020.10.11-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:d9d53518eeed12190744d366ec4a3f39b99d7daa705abca95f87dd8b442df4ad"}, + {file = "regex-2020.10.11-cp38-cp38-win32.whl", hash = "sha256:3d5a8d007116021cf65355ada47bf405656c4b3b9a988493d26688275fde1f1c"}, + {file = "regex-2020.10.11-cp38-cp38-win_amd64.whl", hash = "sha256:f579caecbbca291b0fcc7d473664c8c08635da2f9b1567c22ea32311c86ef68c"}, + {file = "regex-2020.10.11-cp39-cp39-manylinux1_i686.whl", hash = "sha256:8c8c42aa5d3ac9a49829c4b28a81bebfa0378996f9e0ca5b5ab8a36870c3e5ee"}, + {file = "regex-2020.10.11-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:c529ba90c1775697a65b46c83d47a2d3de70f24d96da5d41d05a761c73b063af"}, + {file = "regex-2020.10.11-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:6cf527ec2f3565248408b61dd36e380d799c2a1047eab04e13a2b0c15dd9c767"}, + {file = "regex-2020.10.11-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:671c51d352cfb146e48baee82b1ee8d6ffe357c292f5e13300cdc5c00867ebfc"}, + {file = "regex-2020.10.11-cp39-cp39-win32.whl", hash = "sha256:a63907332531a499b8cdfd18953febb5a4c525e9e7ca4ac147423b917244b260"}, + {file = "regex-2020.10.11-cp39-cp39-win_amd64.whl", hash = "sha256:1a16afbfadaadc1397353f9b32e19a65dc1d1804c80ad73a14f435348ca017ad"}, + {file = "regex-2020.10.11.tar.gz", hash = "sha256:463e770c48da76a8da82b8d4a48a541f314e0df91cbb6d873a341dbe578efafd"}, ] requests = [ {file = "requests-2.24.0-py2.py3-none-any.whl", hash = "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"}, @@ -1229,8 +1200,8 @@ stevedore = [ {file = "stevedore-3.2.2.tar.gz", hash = "sha256:f845868b3a3a77a2489d226568abe7328b5c2d4f6a011cc759dfa99144a521f0"}, ] testfixtures = [ - {file = "testfixtures-6.14.2-py2.py3-none-any.whl", hash = "sha256:816557888877f498081c1b5c572049b4a2ddffedb77401308ff4cdc1bb9147b7"}, - {file = "testfixtures-6.14.2.tar.gz", hash = "sha256:14d9907390f5f9c7189b3d511b64f34f1072d07cc13b604a57e1bb79029376e3"}, + {file = "testfixtures-6.15.0-py2.py3-none-any.whl", hash = "sha256:e17f4f526fc90b0ac9bc7f8ca62b7dec17d9faf3d721f56bda4f0fd94d02f85a"}, + {file = "testfixtures-6.15.0.tar.gz", hash = "sha256:409f77cfbdad822d12a8ce5c4aa8fb4d0bb38073f4a5444fede3702716a2cec2"}, ] toml = [ {file = "toml-0.10.1-py2.py3-none-any.whl", hash = "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88"}, @@ -1280,6 +1251,6 @@ xenon = [ {file = "xenon-0.7.1.tar.gz", hash = "sha256:38bf283135f0636355ecf6054b6f37226af12faab152161bda1a4f9e4dc5b701"}, ] zipp = [ - {file = "zipp-3.2.0-py3-none-any.whl", hash = "sha256:43f4fa8d8bb313e65d8323a3952ef8756bf40f9a5c3ea7334be23ee4ec8278b6"}, - {file = "zipp-3.2.0.tar.gz", hash = "sha256:b52f22895f4cfce194bc8172f3819ee8de7540aa6d873535a8668b730b8b411f"}, + {file = "zipp-3.3.0-py3-none-any.whl", hash = "sha256:eed8ec0b8d1416b2ca33516a37a08892442f3954dee131e92cfd92d8fe3e7066"}, + {file = "zipp-3.3.0.tar.gz", hash = "sha256:64ad89efee774d1897a58607895d80789c59778ea02185dd846ac38394a8642b"}, ] diff --git a/pyproject.toml b/pyproject.toml index 61bc92ab26a..e18cfe3e211 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,6 @@ flake8-debugger = "^3.2.1" flake8-fixme = "^1.1.1" flake8-isort = "^2.8.0" flake8-variables-names = "^0.0.3" -flake8_polyfill = "^1.0.2" isort = "^4.3.21" pytest-cov = "^2.8.1" pytest-mock = "^2.0.0" From bcb763b0128145144cae7cbecceab0a603b7cb86 Mon Sep 17 00:00:00 2001 From: Heitor Lessa Date: Tue, 13 Oct 2020 11:01:29 +0200 Subject: [PATCH 20/22] Update aws_lambda_powertools/utilities/parser/pydantic.py Co-authored-by: Joris Conijn --- aws_lambda_powertools/utilities/parser/pydantic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/parser/pydantic.py b/aws_lambda_powertools/utilities/parser/pydantic.py index 9060eb57749..d2551928979 100644 --- a/aws_lambda_powertools/utilities/parser/pydantic.py +++ b/aws_lambda_powertools/utilities/parser/pydantic.py @@ -3,6 +3,6 @@ # As Pydantic export new types, new utilities, we will have to keep up # with a project that's not used in our core functionalities. # For this reason, we're relying on Pydantic's __all__ attr to allow customers -# to use `from from aws_lambda_powertools.utilities.parser.pydantic import ` +# to use `from aws_lambda_powertools.utilities.parser.pydantic import ` from pydantic import * # noqa: F403,F401 From 404530f5f77da03464a6d3fa0fc9da09d8a729dd Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 14 Oct 2020 20:52:28 +0200 Subject: [PATCH 21/22] improv: rename to event_parser as per Tom's review --- aws_lambda_powertools/utilities/parser/__init__.py | 14 +++++++++++--- aws_lambda_powertools/utilities/parser/parser.py | 8 ++++---- tests/functional/parser/test_dynamodb.py | 6 +++--- tests/functional/parser/test_eventbridge.py | 6 +++--- tests/functional/parser/test_parser.py | 12 ++++++------ tests/functional/parser/test_sqs.py | 6 +++--- 6 files changed, 30 insertions(+), 22 deletions(-) diff --git a/aws_lambda_powertools/utilities/parser/__init__.py b/aws_lambda_powertools/utilities/parser/__init__.py index 4fbb961fc2f..fab306c95f0 100644 --- a/aws_lambda_powertools/utilities/parser/__init__.py +++ b/aws_lambda_powertools/utilities/parser/__init__.py @@ -1,9 +1,17 @@ -"""Advanced parser utility +"""Advanced event_parser utility """ from . import envelopes from .envelopes import BaseEnvelope from .exceptions import SchemaValidationError -from .parser import parser +from .parser import event_parser from .pydantic import BaseModel, root_validator, validator -__all__ = ["parser", "envelopes", "BaseEnvelope", "BaseModel", "validator", "root_validator", "SchemaValidationError"] +__all__ = [ + "event_parser", + "envelopes", + "BaseEnvelope", + "BaseModel", + "validator", + "root_validator", + "SchemaValidationError", +] diff --git a/aws_lambda_powertools/utilities/parser/parser.py b/aws_lambda_powertools/utilities/parser/parser.py index f0019df09e5..71733b0264b 100644 --- a/aws_lambda_powertools/utilities/parser/parser.py +++ b/aws_lambda_powertools/utilities/parser/parser.py @@ -12,7 +12,7 @@ @lambda_handler_decorator -def parser( +def event_parser( handler: Callable[[Dict, Any], Any], event: Dict[str, Any], context: LambdaContext, @@ -42,7 +42,7 @@ class Order(BaseModel): description: str ... - @parser(schema=Order) + @event_parser(schema=Order) def handler(event: Order, context: LambdaContext): ... @@ -53,7 +53,7 @@ class Order(BaseModel): description: str ... - @parser(schema=Order, envelope=envelopes.EVENTBRIDGE) + @event_parser(schema=Order, envelope=envelopes.EVENTBRIDGE) def handler(event: Order, context: LambdaContext): ... @@ -87,7 +87,7 @@ def handler(event: Order, context: LambdaContext): def parse(event: Dict[str, Any], schema: BaseModel, envelope: Optional[BaseEnvelope] = None) -> Any: """Standalone function to parse & validate events using Pydantic models - Typically used when you need fine-grained control over error handling compared to parser decorator. + Typically used when you need fine-grained control over error handling compared to event_parser decorator. Example ------- diff --git a/tests/functional/parser/test_dynamodb.py b/tests/functional/parser/test_dynamodb.py index 9f98b49226c..b8b83e21112 100644 --- a/tests/functional/parser/test_dynamodb.py +++ b/tests/functional/parser/test_dynamodb.py @@ -2,13 +2,13 @@ import pytest -from aws_lambda_powertools.utilities.parser import envelopes, exceptions, parser +from aws_lambda_powertools.utilities.parser import envelopes, event_parser, exceptions from aws_lambda_powertools.utilities.typing import LambdaContext from tests.functional.parser.schemas import MyAdvancedDynamoBusiness, MyDynamoBusiness from tests.functional.parser.utils import load_event -@parser(schema=MyDynamoBusiness, envelope=envelopes.DynamoDBEnvelope) +@event_parser(schema=MyDynamoBusiness, envelope=envelopes.DynamoDBEnvelope) def handle_dynamodb(event: List[Dict[str, MyDynamoBusiness]], _: LambdaContext): assert len(event) == 2 assert event[0]["OldImage"] is None @@ -20,7 +20,7 @@ def handle_dynamodb(event: List[Dict[str, MyDynamoBusiness]], _: LambdaContext): assert event[1]["NewImage"].Id["N"] == 101 -@parser(schema=MyAdvancedDynamoBusiness) +@event_parser(schema=MyAdvancedDynamoBusiness) def handle_dynamodb_no_envelope(event: MyAdvancedDynamoBusiness, _: LambdaContext): records = event.Records record = records[0] diff --git a/tests/functional/parser/test_eventbridge.py b/tests/functional/parser/test_eventbridge.py index f64fc7deea0..20ef983fde1 100644 --- a/tests/functional/parser/test_eventbridge.py +++ b/tests/functional/parser/test_eventbridge.py @@ -2,19 +2,19 @@ import pytest -from aws_lambda_powertools.utilities.parser import envelopes, exceptions, parser +from aws_lambda_powertools.utilities.parser import envelopes, event_parser, exceptions from aws_lambda_powertools.utilities.typing import LambdaContext from tests.functional.parser.schemas import MyAdvancedEventbridgeBusiness, MyEventbridgeBusiness from tests.functional.parser.utils import load_event -@parser(schema=MyEventbridgeBusiness, envelope=envelopes.EventBridgeEnvelope) +@event_parser(schema=MyEventbridgeBusiness, envelope=envelopes.EventBridgeEnvelope) def handle_eventbridge(event: MyEventbridgeBusiness, _: LambdaContext): assert event.instance_id == "i-1234567890abcdef0" assert event.state == "terminated" -@parser(schema=MyAdvancedEventbridgeBusiness) +@event_parser(schema=MyAdvancedEventbridgeBusiness) def handle_eventbridge_no_envelope(event: MyAdvancedEventbridgeBusiness, _: LambdaContext): assert event.detail.instance_id == "i-1234567890abcdef0" assert event.detail.state == "terminated" diff --git a/tests/functional/parser/test_parser.py b/tests/functional/parser/test_parser.py index 65cc7a28f81..1b348e8e7cb 100644 --- a/tests/functional/parser/test_parser.py +++ b/tests/functional/parser/test_parser.py @@ -2,13 +2,13 @@ import pytest -from aws_lambda_powertools.utilities.parser import exceptions, parser +from aws_lambda_powertools.utilities.parser import event_parser, exceptions from aws_lambda_powertools.utilities.typing import LambdaContext @pytest.mark.parametrize("invalid_value", [None, bool(), [], (), object]) def test_parser_unsupported_event(dummy_schema, invalid_value): - @parser(schema=dummy_schema) + @event_parser(schema=dummy_schema) def handle_no_envelope(event: Dict, _: LambdaContext): return event @@ -20,7 +20,7 @@ def handle_no_envelope(event: Dict, _: LambdaContext): "invalid_envelope,expected", [(True, ""), (["dummy"], ""), (object, exceptions.InvalidEnvelopeError)] ) def test_parser_invalid_envelope_type(dummy_event, dummy_schema, invalid_envelope, expected): - @parser(schema=dummy_schema, envelope=invalid_envelope) + @event_parser(schema=dummy_schema, envelope=invalid_envelope) def handle_no_envelope(event: Dict, _: LambdaContext): return event @@ -32,7 +32,7 @@ def handle_no_envelope(event: Dict, _: LambdaContext): def test_parser_schema_with_envelope(dummy_event, dummy_schema, dummy_envelope): - @parser(schema=dummy_schema, envelope=dummy_envelope) + @event_parser(schema=dummy_schema, envelope=dummy_envelope) def handle_no_envelope(event: Dict, _: LambdaContext): return event @@ -40,7 +40,7 @@ def handle_no_envelope(event: Dict, _: LambdaContext): def test_parser_schema_no_envelope(dummy_event, dummy_schema): - @parser(schema=dummy_schema) + @event_parser(schema=dummy_schema) def handle_no_envelope(event: Dict, _: LambdaContext): return event @@ -49,7 +49,7 @@ def handle_no_envelope(event: Dict, _: LambdaContext): @pytest.mark.parametrize("invalid_schema", [None, str, bool(), [], (), object]) def test_parser_with_invalid_schema_type(dummy_event, invalid_schema): - @parser(schema=invalid_schema) + @event_parser(schema=invalid_schema) def handle_no_envelope(event: Dict, _: LambdaContext): return event diff --git a/tests/functional/parser/test_sqs.py b/tests/functional/parser/test_sqs.py index 2d1ab1457bf..dec06e1d1e9 100644 --- a/tests/functional/parser/test_sqs.py +++ b/tests/functional/parser/test_sqs.py @@ -2,14 +2,14 @@ import pytest -from aws_lambda_powertools.utilities.parser import envelopes, exceptions, parser +from aws_lambda_powertools.utilities.parser import envelopes, event_parser, exceptions from aws_lambda_powertools.utilities.typing import LambdaContext from tests.functional.parser.schemas import MyAdvancedSqsBusiness, MySqsBusiness from tests.functional.parser.utils import load_event from tests.functional.validator.conftest import sqs_event # noqa: F401 -@parser(schema=MySqsBusiness, envelope=envelopes.SqsEnvelope) +@event_parser(schema=MySqsBusiness, envelope=envelopes.SqsEnvelope) def handle_sqs_json_body(event: List[MySqsBusiness], _: LambdaContext): assert len(event) == 1 assert event[0].message == "hello world" @@ -55,7 +55,7 @@ def test_validate_event_does_not_conform_user_json_string_with_schema(): handle_sqs_json_body(event, LambdaContext()) -@parser(schema=MyAdvancedSqsBusiness) +@event_parser(schema=MyAdvancedSqsBusiness) def handle_sqs_no_envelope(event: MyAdvancedSqsBusiness, _: LambdaContext): records = event.Records record = records[0] From 74809858f72364f8f4e6100f50fd52e824475a5d Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 14 Oct 2020 21:20:14 +0200 Subject: [PATCH 22/22] improv: rename schema to model as per Tom's review --- .../utilities/parser/__init__.py | 4 +- .../utilities/parser/envelopes/base.py | 28 ++++---- .../utilities/parser/envelopes/dynamodb.py | 20 +++--- .../parser/envelopes/event_bridge.py | 16 ++--- .../utilities/parser/envelopes/sqs.py | 20 +++--- .../utilities/parser/exceptions.py | 8 +-- .../utilities/parser/models/__init__.py | 12 ++++ .../parser/{schemas => models}/dynamodb.py | 14 ++-- .../{schemas => models}/event_bridge.py | 2 +- .../parser/{schemas => models}/sqs.py | 14 ++-- .../utilities/parser/parser.py | 72 +++++++++---------- .../utilities/parser/schemas/__init__.py | 12 ---- tests/functional/parser/conftest.py | 8 +-- tests/functional/parser/schemas.py | 30 ++++---- tests/functional/parser/test_dynamodb.py | 12 ++-- tests/functional/parser/test_eventbridge.py | 10 +-- tests/functional/parser/test_parser.py | 14 ++-- tests/functional/parser/test_sqs.py | 12 ++-- 18 files changed, 154 insertions(+), 154 deletions(-) create mode 100644 aws_lambda_powertools/utilities/parser/models/__init__.py rename aws_lambda_powertools/utilities/parser/{schemas => models}/dynamodb.py (76%) rename aws_lambda_powertools/utilities/parser/{schemas => models}/event_bridge.py (90%) rename aws_lambda_powertools/utilities/parser/{schemas => models}/sqs.py (89%) delete mode 100644 aws_lambda_powertools/utilities/parser/schemas/__init__.py diff --git a/aws_lambda_powertools/utilities/parser/__init__.py b/aws_lambda_powertools/utilities/parser/__init__.py index fab306c95f0..62aa4bd73d7 100644 --- a/aws_lambda_powertools/utilities/parser/__init__.py +++ b/aws_lambda_powertools/utilities/parser/__init__.py @@ -2,7 +2,7 @@ """ from . import envelopes from .envelopes import BaseEnvelope -from .exceptions import SchemaValidationError +from .exceptions import ModelValidationError from .parser import event_parser from .pydantic import BaseModel, root_validator, validator @@ -13,5 +13,5 @@ "BaseModel", "validator", "root_validator", - "SchemaValidationError", + "ModelValidationError", ] diff --git a/aws_lambda_powertools/utilities/parser/envelopes/base.py b/aws_lambda_powertools/utilities/parser/envelopes/base.py index 95a9b24f2b3..baf6cd33420 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/base.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/base.py @@ -11,15 +11,15 @@ class BaseEnvelope(ABC): """ABC implementation for creating a supported Envelope""" @staticmethod - def _parse(data: Union[Dict[str, Any], str], schema: BaseModel) -> Any: - """Parses envelope data against schema provided + def _parse(data: Union[Dict[str, Any], str], model: BaseModel) -> Any: + """Parses envelope data against model provided Parameters ---------- data : Dict Data to be parsed and validated - schema - Schema to parse and validate data against + model + Data model to parse and validate data against Returns ------- @@ -30,18 +30,18 @@ def _parse(data: Union[Dict[str, Any], str], schema: BaseModel) -> Any: logger.debug("Skipping parsing as event is None") return data - logger.debug("parsing event against schema") + logger.debug("parsing event against model") if isinstance(data, str): logger.debug("parsing event as string") - return schema.parse_raw(data) + return model.parse_raw(data) - return schema.parse_obj(data) + return model.parse_obj(data) @abstractmethod - def parse(self, data: Dict[str, Any], schema: BaseModel): - """Implementation to parse data against envelope schema, then against the schema + def parse(self, data: Dict[str, Any], model: BaseModel): + """Implementation to parse data against envelope model, then against the data model - NOTE: Call `_parse` method to fully parse data with schema provided. + NOTE: Call `_parse` method to fully parse data with model provided. Example ------- @@ -49,10 +49,10 @@ def parse(self, data: Dict[str, Any], schema: BaseModel): **EventBridge envelope implementation example** def parse(...): - # 1. parses data against envelope schema - parsed_envelope = EventBridgeSchema(**data) + # 1. parses data against envelope model + parsed_envelope = EventBridgeModel(**data) - # 2. parses portion of data within the envelope against schema - return self._parse(data=parsed_envelope.detail, schema=schema) + # 2. parses portion of data within the envelope against model + return self._parse(data=parsed_envelope.detail, model=data_model) """ return NotImplemented # pragma: no cover diff --git a/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py b/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py index a35dd689a57..ef166a5c48f 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/dynamodb.py @@ -4,7 +4,7 @@ from pydantic import BaseModel from typing_extensions import Literal -from ..schemas import DynamoDBStreamSchema +from ..models import DynamoDBStreamModel from .base import BaseEnvelope logger = logging.getLogger(__name__) @@ -13,32 +13,32 @@ class DynamoDBEnvelope(BaseEnvelope): """ DynamoDB Stream Envelope to extract data within NewImage/OldImage - Note: Values are the parsed schema models. Images' values can also be None, and + Note: Values are the parsed models. Images' values can also be None, and length of the list is the record's amount in the original event. """ - def parse(self, data: Dict[str, Any], schema: BaseModel) -> List[Dict[Literal["NewImage", "OldImage"], BaseModel]]: - """Parses DynamoDB Stream records found in either NewImage and OldImage with schema provided + def parse(self, data: Dict[str, Any], model: BaseModel) -> List[Dict[Literal["NewImage", "OldImage"], BaseModel]]: + """Parses DynamoDB Stream records found in either NewImage and OldImage with model provided Parameters ---------- data : Dict Lambda event to be parsed - schema : BaseModel - User schema provided to parse after extracting data using envelope + model : BaseModel + Data model provided to parse after extracting data using envelope Returns ------- List - List of records parsed with schema provided + List of records parsed with model provided """ - parsed_envelope = DynamoDBStreamSchema(**data) + parsed_envelope = DynamoDBStreamModel(**data) output = [] for record in parsed_envelope.Records: output.append( { - "NewImage": self._parse(record.dynamodb.NewImage, schema), - "OldImage": self._parse(record.dynamodb.OldImage, schema), + "NewImage": self._parse(record.dynamodb.NewImage, model), + "OldImage": self._parse(record.dynamodb.OldImage, model), } ) # noinspection PyTypeChecker diff --git a/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py index 18996cfda01..8b91266e848 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/event_bridge.py @@ -3,7 +3,7 @@ from pydantic import BaseModel -from ..schemas import EventBridgeSchema +from ..models import EventBridgeModel from .base import BaseEnvelope logger = logging.getLogger(__name__) @@ -12,20 +12,20 @@ class EventBridgeEnvelope(BaseEnvelope): """EventBridge envelope to extract data within detail key""" - def parse(self, data: Dict[str, Any], schema: BaseModel) -> BaseModel: - """Parses data found with schema provided + def parse(self, data: Dict[str, Any], model: BaseModel) -> BaseModel: + """Parses data found with model provided Parameters ---------- data : Dict Lambda event to be parsed - schema : BaseModel - User schema provided to parse after extracting data using envelope + model : BaseModel + Data model provided to parse after extracting data using envelope Returns ------- Any - Parsed detail payload with schema provided + Parsed detail payload with model provided """ - parsed_envelope = EventBridgeSchema(**data) - return self._parse(data=parsed_envelope.detail, schema=schema) + parsed_envelope = EventBridgeModel(**data) + return self._parse(data=parsed_envelope.detail, model=model) diff --git a/aws_lambda_powertools/utilities/parser/envelopes/sqs.py b/aws_lambda_powertools/utilities/parser/envelopes/sqs.py index 2e55fe2a82b..7bf326206f3 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/sqs.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/sqs.py @@ -3,7 +3,7 @@ from pydantic import BaseModel -from ..schemas import SqsSchema +from ..models import SqsModel from .base import BaseEnvelope logger = logging.getLogger(__name__) @@ -13,29 +13,29 @@ class SqsEnvelope(BaseEnvelope): """SQS Envelope to extract array of Records The record's body parameter is a string, though it can also be a JSON encoded string. - Regardless of it's type it'll be parsed into a BaseModel object. + Regardless of its type it'll be parsed into a BaseModel object. - Note: Records will be parsed the same way so if schema is str, + Note: Records will be parsed the same way so if model is str, all items in the list will be parsed as str and npt as JSON (and vice versa) """ - def parse(self, data: Dict[str, Any], schema: Union[BaseModel, str]) -> List[Union[BaseModel, str]]: - """Parses records found with schema provided + def parse(self, data: Dict[str, Any], model: Union[BaseModel, str]) -> List[Union[BaseModel, str]]: + """Parses records found with model provided Parameters ---------- data : Dict Lambda event to be parsed - schema : BaseModel - User schema provided to parse after extracting data using envelope + model : BaseModel + Data model provided to parse after extracting data using envelope Returns ------- List - List of records parsed with schema provided + List of records parsed with model provided """ - parsed_envelope = SqsSchema(**data) + parsed_envelope = SqsModel(**data) output = [] for record in parsed_envelope.Records: - output.append(self._parse(record.body, schema)) + output.append(self._parse(record.body, model)) return output diff --git a/aws_lambda_powertools/utilities/parser/exceptions.py b/aws_lambda_powertools/utilities/parser/exceptions.py index 9f1ac331bef..93e259df371 100644 --- a/aws_lambda_powertools/utilities/parser/exceptions.py +++ b/aws_lambda_powertools/utilities/parser/exceptions.py @@ -2,9 +2,9 @@ class InvalidEnvelopeError(Exception): """Input envelope is not callable and instance of BaseEnvelope""" -class SchemaValidationError(Exception): - """Input data does not conform with schema""" +class ModelValidationError(Exception): + """Input data does not conform with model""" -class InvalidSchemaTypeError(Exception): - """Input schema does not implement BaseModel""" +class InvalidModelTypeError(Exception): + """Input data model does not implement BaseModel""" diff --git a/aws_lambda_powertools/utilities/parser/models/__init__.py b/aws_lambda_powertools/utilities/parser/models/__init__.py new file mode 100644 index 00000000000..e58a678e959 --- /dev/null +++ b/aws_lambda_powertools/utilities/parser/models/__init__.py @@ -0,0 +1,12 @@ +from .dynamodb import DynamoDBStreamChangedRecordModel, DynamoDBStreamModel, DynamoDBStreamRecordModel +from .event_bridge import EventBridgeModel +from .sqs import SqsModel, SqsRecordModel + +__all__ = [ + "DynamoDBStreamModel", + "EventBridgeModel", + "DynamoDBStreamChangedRecordModel", + "DynamoDBStreamRecordModel", + "SqsModel", + "SqsRecordModel", +] diff --git a/aws_lambda_powertools/utilities/parser/schemas/dynamodb.py b/aws_lambda_powertools/utilities/parser/models/dynamodb.py similarity index 76% rename from aws_lambda_powertools/utilities/parser/schemas/dynamodb.py rename to aws_lambda_powertools/utilities/parser/models/dynamodb.py index d50ad0a891d..7bcf68845cc 100644 --- a/aws_lambda_powertools/utilities/parser/schemas/dynamodb.py +++ b/aws_lambda_powertools/utilities/parser/models/dynamodb.py @@ -5,7 +5,7 @@ from typing_extensions import Literal -class DynamoDBStreamChangedRecordSchema(BaseModel): +class DynamoDBStreamChangedRecordModel(BaseModel): ApproximateCreationDateTime: Optional[date] Keys: Dict[str, Dict[str, Any]] NewImage: Optional[Dict[str, Any]] @@ -16,13 +16,13 @@ class DynamoDBStreamChangedRecordSchema(BaseModel): # context on why it's commented: https://github.com/awslabs/aws-lambda-powertools-python/pull/118 # since both images are optional, they can both be None. However, at least one must - # exist in a legal schema of NEW_AND_OLD_IMAGES type + # exist in a legal model of NEW_AND_OLD_IMAGES type # @root_validator # def check_one_image_exists(cls, values): # noqa: E800 # new_img, old_img = values.get("NewImage"), values.get("OldImage") # noqa: E800 # stream_type = values.get("StreamViewType") # noqa: E800 # if stream_type == "NEW_AND_OLD_IMAGES" and not new_img and not old_img: # noqa: E800 - # raise TypeError("DynamoDB streams schema failed validation, missing both new & old stream images") # noqa: E800,E501 + # raise TypeError("DynamoDB streams model failed validation, missing both new & old stream images") # noqa: E800,E501 # return values # noqa: E800 @@ -31,16 +31,16 @@ class UserIdentity(BaseModel): principalId: Literal["dynamodb.amazonaws.com"] -class DynamoDBStreamRecordSchema(BaseModel): +class DynamoDBStreamRecordModel(BaseModel): eventID: str eventName: Literal["INSERT", "MODIFY", "REMOVE"] eventVersion: float eventSource: Literal["aws:dynamodb"] awsRegion: str eventSourceARN: str - dynamodb: DynamoDBStreamChangedRecordSchema + dynamodb: DynamoDBStreamChangedRecordModel userIdentity: Optional[UserIdentity] -class DynamoDBStreamSchema(BaseModel): - Records: List[DynamoDBStreamRecordSchema] +class DynamoDBStreamModel(BaseModel): + Records: List[DynamoDBStreamRecordModel] diff --git a/aws_lambda_powertools/utilities/parser/schemas/event_bridge.py b/aws_lambda_powertools/utilities/parser/models/event_bridge.py similarity index 90% rename from aws_lambda_powertools/utilities/parser/schemas/event_bridge.py rename to aws_lambda_powertools/utilities/parser/models/event_bridge.py index 40b00848047..ab621d5da9f 100644 --- a/aws_lambda_powertools/utilities/parser/schemas/event_bridge.py +++ b/aws_lambda_powertools/utilities/parser/models/event_bridge.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, Field -class EventBridgeSchema(BaseModel): +class EventBridgeModel(BaseModel): version: str id: str # noqa: A003,VNE003 source: str diff --git a/aws_lambda_powertools/utilities/parser/schemas/sqs.py b/aws_lambda_powertools/utilities/parser/models/sqs.py similarity index 89% rename from aws_lambda_powertools/utilities/parser/schemas/sqs.py rename to aws_lambda_powertools/utilities/parser/models/sqs.py index efb92ad5345..fd708020492 100644 --- a/aws_lambda_powertools/utilities/parser/schemas/sqs.py +++ b/aws_lambda_powertools/utilities/parser/models/sqs.py @@ -5,7 +5,7 @@ from typing_extensions import Literal -class SqsAttributesSchema(BaseModel): +class SqsAttributesModel(BaseModel): ApproximateReceiveCount: str ApproximateFirstReceiveTimestamp: datetime MessageDeduplicationId: Optional[str] @@ -16,7 +16,7 @@ class SqsAttributesSchema(BaseModel): AWSTraceHeader: Optional[str] -class SqsMsgAttributeSchema(BaseModel): +class SqsMsgAttributeModel(BaseModel): stringValue: Optional[str] binaryValue: Optional[str] stringListValues: List[str] = [] @@ -48,12 +48,12 @@ class SqsMsgAttributeSchema(BaseModel): # return values # noqa: E800 -class SqsRecordSchema(BaseModel): +class SqsRecordModel(BaseModel): messageId: str receiptHandle: str body: str - attributes: SqsAttributesSchema - messageAttributes: Dict[str, SqsMsgAttributeSchema] + attributes: SqsAttributesModel + messageAttributes: Dict[str, SqsMsgAttributeModel] md5OfBody: str md5OfMessageAttributes: Optional[str] eventSource: Literal["aws:sqs"] @@ -61,5 +61,5 @@ class SqsRecordSchema(BaseModel): awsRegion: str -class SqsSchema(BaseModel): - Records: List[SqsRecordSchema] +class SqsModel(BaseModel): + Records: List[SqsRecordModel] diff --git a/aws_lambda_powertools/utilities/parser/parser.py b/aws_lambda_powertools/utilities/parser/parser.py index 71733b0264b..a58ee90f4e9 100644 --- a/aws_lambda_powertools/utilities/parser/parser.py +++ b/aws_lambda_powertools/utilities/parser/parser.py @@ -6,7 +6,7 @@ from ...middleware_factory import lambda_handler_decorator from ..typing import LambdaContext from .envelopes.base import BaseEnvelope -from .exceptions import InvalidEnvelopeError, InvalidSchemaTypeError, SchemaValidationError +from .exceptions import InvalidEnvelopeError, InvalidModelTypeError, ModelValidationError logger = logging.getLogger(__name__) @@ -16,22 +16,22 @@ def event_parser( handler: Callable[[Dict, Any], Any], event: Dict[str, Any], context: LambdaContext, - schema: BaseModel, + model: BaseModel, envelope: Optional[BaseEnvelope] = None, ) -> Any: """Lambda handler decorator to parse & validate events using Pydantic models - It requires a schema that implements Pydantic BaseModel to parse & validate the event. + It requires a model that implements Pydantic BaseModel to parse & validate the event. When an envelope is given, it'll use the following logic: - 1. Parse the event against envelope schema first e.g. EnvelopeSchema(**event) - 2. Envelope will extract a given key to be parsed against the schema e.g. event.detail + 1. Parse the event against the envelope model first e.g. EnvelopeModel(**event) + 2. Envelope will extract a given key to be parsed against the model e.g. event.detail This is useful when you need to confirm event wrapper structure, and b) selectively extract a portion of your payload for parsing & validation. - NOTE: If envelope is omitted, the complete event is parsed to match the schema parameter BaseModel definition. + NOTE: If envelope is omitted, the complete event is parsed to match the model parameter BaseModel definition. Example ------- @@ -42,7 +42,7 @@ class Order(BaseModel): description: str ... - @event_parser(schema=Order) + @event_parser(model=Order) def handler(event: Order, context: LambdaContext): ... @@ -53,7 +53,7 @@ class Order(BaseModel): description: str ... - @event_parser(schema=Order, envelope=envelopes.EVENTBRIDGE) + @event_parser(model=Order, envelope=envelopes.EVENTBRIDGE) def handler(event: Order, context: LambdaContext): ... @@ -65,26 +65,26 @@ def handler(event: Order, context: LambdaContext): Lambda event to be parsed & validated context: LambdaContext Lambda context object - schema: BaseModel - Your data schema that will replace the event. + model: BaseModel + Your data model that will replace the event. envelope: BaseEnvelope - Optional envelope to extract the schema from + Optional envelope to extract the model from Raises ------ - SchemaValidationError - When input event does not conform with schema provided - InvalidSchemaTypeError - When schema given does not implement BaseModel + ModelValidationError + When input event does not conform with model provided + InvalidModelTypeError + When model given does not implement BaseModel InvalidEnvelopeError When envelope given does not implement BaseEnvelope """ - parsed_event = parse(event=event, schema=schema, envelope=envelope) + parsed_event = parse(event=event, model=model, envelope=envelope) logger.debug(f"Calling handler {handler.__name__}") return handler(parsed_event, context) -def parse(event: Dict[str, Any], schema: BaseModel, envelope: Optional[BaseEnvelope] = None) -> Any: +def parse(event: Dict[str, Any], model: BaseModel, envelope: Optional[BaseEnvelope] = None) -> Any: """Standalone function to parse & validate events using Pydantic models Typically used when you need fine-grained control over error handling compared to event_parser decorator. @@ -94,7 +94,7 @@ def parse(event: Dict[str, Any], schema: BaseModel, envelope: Optional[BaseEnvel **Lambda handler decorator to parse & validate event** - from aws_lambda_powertools.utilities.parser.exceptions import SchemaValidationError + from aws_lambda_powertools.utilities.parser.exceptions import ModelValidationError class Order(BaseModel): id: int @@ -103,8 +103,8 @@ class Order(BaseModel): def handler(event: Order, context: LambdaContext): try: - parse(schema=Order) - except SchemaValidationError: + parse(model=Order) + except ModelValidationError: ... **Lambda handler decorator to parse & validate event - using built-in envelope** @@ -116,41 +116,41 @@ class Order(BaseModel): def handler(event: Order, context: LambdaContext): try: - parse(schema=Order, envelope=envelopes.EVENTBRIDGE) - except SchemaValidationError: + parse(model=Order, envelope=envelopes.EVENTBRIDGE) + except ModelValidationError: ... Parameters ---------- event: Dict Lambda event to be parsed & validated - schema: BaseModel - Your data schema that will replace the event. + model: BaseModel + Your data model that will replace the event envelope: BaseEnvelope - Optional envelope to extract the schema from + Optional envelope to extract the model from Raises ------ - SchemaValidationError - When input event does not conform with schema provided - InvalidSchemaTypeError - When schema given does not implement BaseModel + ModelValidationError + When input event does not conform with model provided + InvalidModelTypeError + When model given does not implement BaseModel InvalidEnvelopeError When envelope given does not implement BaseEnvelope """ if envelope and callable(envelope): try: - logger.debug(f"Parsing and validating event schema with envelope={envelope}") - return envelope().parse(data=event, schema=schema) + logger.debug(f"Parsing and validating event model with envelope={envelope}") + return envelope().parse(data=event, model=model) except AttributeError: raise InvalidEnvelopeError(f"Envelope must implement BaseEnvelope, envelope={envelope}") except (ValidationError, TypeError) as e: - raise SchemaValidationError(f"Input event does not conform with schema, envelope={envelope}") from e + raise ModelValidationError(f"Input event does not conform with model, envelope={envelope}") from e try: - logger.debug("Parsing and validating event schema; no envelope used") - return schema.parse_obj(event) + logger.debug("Parsing and validating event model; no envelope used") + return model.parse_obj(event) except (ValidationError, TypeError) as e: - raise SchemaValidationError("Input event does not conform with schema") from e + raise ModelValidationError("Input event does not conform with model") from e except AttributeError: - raise InvalidSchemaTypeError("Input schema must implement BaseModel") + raise InvalidModelTypeError("Input model must implement BaseModel") diff --git a/aws_lambda_powertools/utilities/parser/schemas/__init__.py b/aws_lambda_powertools/utilities/parser/schemas/__init__.py deleted file mode 100644 index 914eb9d50d8..00000000000 --- a/aws_lambda_powertools/utilities/parser/schemas/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -from .dynamodb import DynamoDBStreamChangedRecordSchema, DynamoDBStreamRecordSchema, DynamoDBStreamSchema -from .event_bridge import EventBridgeSchema -from .sqs import SqsRecordSchema, SqsSchema - -__all__ = [ - "DynamoDBStreamSchema", - "EventBridgeSchema", - "DynamoDBStreamChangedRecordSchema", - "DynamoDBStreamRecordSchema", - "SqsSchema", - "SqsRecordSchema", -] diff --git a/tests/functional/parser/conftest.py b/tests/functional/parser/conftest.py index 1fb97f61d89..27fd4b2d1f6 100644 --- a/tests/functional/parser/conftest.py +++ b/tests/functional/parser/conftest.py @@ -3,7 +3,7 @@ import pytest from pydantic import BaseModel, ValidationError -from aws_lambda_powertools.utilities.parser import BaseEnvelope, SchemaValidationError +from aws_lambda_powertools.utilities.parser import BaseEnvelope, ModelValidationError @pytest.fixture @@ -36,11 +36,11 @@ def dummy_envelope(dummy_envelope_schema): class MyDummyEnvelope(BaseEnvelope): """Unwrap dummy event within payload key""" - def parse(self, data: Dict[str, Any], schema: BaseModel): + def parse(self, data: Dict[str, Any], model: BaseModel): try: parsed_enveloped = dummy_envelope_schema(**data) except (ValidationError, TypeError) as e: - raise SchemaValidationError("Dummy input does not conform with schema") from e - return self._parse(data=parsed_enveloped.payload, schema=schema) + raise ModelValidationError("Dummy input does not conform with schema") from e + return self._parse(data=parsed_enveloped.payload, model=model) return MyDummyEnvelope diff --git a/tests/functional/parser/schemas.py b/tests/functional/parser/schemas.py index 5ed38f0a6d1..47614cb95d8 100644 --- a/tests/functional/parser/schemas.py +++ b/tests/functional/parser/schemas.py @@ -3,13 +3,13 @@ from pydantic import BaseModel from typing_extensions import Literal -from aws_lambda_powertools.utilities.parser.schemas import ( - DynamoDBStreamChangedRecordSchema, - DynamoDBStreamRecordSchema, - DynamoDBStreamSchema, - EventBridgeSchema, - SqsRecordSchema, - SqsSchema, +from aws_lambda_powertools.utilities.parser.models import ( + DynamoDBStreamChangedRecordModel, + DynamoDBStreamModel, + DynamoDBStreamRecordModel, + EventBridgeModel, + SqsModel, + SqsRecordModel, ) @@ -18,17 +18,17 @@ class MyDynamoBusiness(BaseModel): Id: Dict[Literal["N"], int] -class MyDynamoScheme(DynamoDBStreamChangedRecordSchema): +class MyDynamoScheme(DynamoDBStreamChangedRecordModel): NewImage: Optional[MyDynamoBusiness] OldImage: Optional[MyDynamoBusiness] -class MyDynamoDBStreamRecordSchema(DynamoDBStreamRecordSchema): +class MyDynamoDBStreamRecordModel(DynamoDBStreamRecordModel): dynamodb: MyDynamoScheme -class MyAdvancedDynamoBusiness(DynamoDBStreamSchema): - Records: List[MyDynamoDBStreamRecordSchema] +class MyAdvancedDynamoBusiness(DynamoDBStreamModel): + Records: List[MyDynamoDBStreamRecordModel] class MyEventbridgeBusiness(BaseModel): @@ -36,7 +36,7 @@ class MyEventbridgeBusiness(BaseModel): state: str -class MyAdvancedEventbridgeBusiness(EventBridgeSchema): +class MyAdvancedEventbridgeBusiness(EventBridgeModel): detail: MyEventbridgeBusiness @@ -45,9 +45,9 @@ class MySqsBusiness(BaseModel): username: str -class MyAdvancedSqsRecordSchema(SqsRecordSchema): +class MyAdvancedSqsRecordModel(SqsRecordModel): body: str -class MyAdvancedSqsBusiness(SqsSchema): - Records: List[MyAdvancedSqsRecordSchema] +class MyAdvancedSqsBusiness(SqsModel): + Records: List[MyAdvancedSqsRecordModel] diff --git a/tests/functional/parser/test_dynamodb.py b/tests/functional/parser/test_dynamodb.py index b8b83e21112..ac5ebab40c3 100644 --- a/tests/functional/parser/test_dynamodb.py +++ b/tests/functional/parser/test_dynamodb.py @@ -8,7 +8,7 @@ from tests.functional.parser.utils import load_event -@event_parser(schema=MyDynamoBusiness, envelope=envelopes.DynamoDBEnvelope) +@event_parser(model=MyDynamoBusiness, envelope=envelopes.DynamoDBEnvelope) def handle_dynamodb(event: List[Dict[str, MyDynamoBusiness]], _: LambdaContext): assert len(event) == 2 assert event[0]["OldImage"] is None @@ -20,7 +20,7 @@ def handle_dynamodb(event: List[Dict[str, MyDynamoBusiness]], _: LambdaContext): assert event[1]["NewImage"].Id["N"] == 101 -@event_parser(schema=MyAdvancedDynamoBusiness) +@event_parser(model=MyAdvancedDynamoBusiness) def handle_dynamodb_no_envelope(event: MyAdvancedDynamoBusiness, _: LambdaContext): records = event.Records record = records[0] @@ -57,13 +57,13 @@ def test_dynamo_db_stream_trigger_event_no_envelope(): handle_dynamodb_no_envelope(event_dict, LambdaContext()) -def test_validate_event_does_not_conform_with_schema_no_envelope(): +def test_validate_event_does_not_conform_with_model_no_envelope(): event_dict: Any = {"hello": "s"} - with pytest.raises(exceptions.SchemaValidationError): + with pytest.raises(exceptions.ModelValidationError): handle_dynamodb_no_envelope(event_dict, LambdaContext()) -def test_validate_event_does_not_conform_with_schema(): +def test_validate_event_does_not_conform_with_model(): event_dict: Any = {"hello": "s"} - with pytest.raises(exceptions.SchemaValidationError): + with pytest.raises(exceptions.ModelValidationError): handle_dynamodb(event_dict, LambdaContext()) diff --git a/tests/functional/parser/test_eventbridge.py b/tests/functional/parser/test_eventbridge.py index 20ef983fde1..07387e9ba0a 100644 --- a/tests/functional/parser/test_eventbridge.py +++ b/tests/functional/parser/test_eventbridge.py @@ -8,13 +8,13 @@ from tests.functional.parser.utils import load_event -@event_parser(schema=MyEventbridgeBusiness, envelope=envelopes.EventBridgeEnvelope) +@event_parser(model=MyEventbridgeBusiness, envelope=envelopes.EventBridgeEnvelope) def handle_eventbridge(event: MyEventbridgeBusiness, _: LambdaContext): assert event.instance_id == "i-1234567890abcdef0" assert event.state == "terminated" -@event_parser(schema=MyAdvancedEventbridgeBusiness) +@event_parser(model=MyAdvancedEventbridgeBusiness) def handle_eventbridge_no_envelope(event: MyAdvancedEventbridgeBusiness, _: LambdaContext): assert event.detail.instance_id == "i-1234567890abcdef0" assert event.detail.state == "terminated" @@ -34,7 +34,7 @@ def test_handle_eventbridge_trigger_event(): handle_eventbridge(event_dict, LambdaContext()) -def test_validate_event_does_not_conform_with_user_dict_schema(): +def test_validate_event_does_not_conform_with_user_dict_model(): event_dict: Any = { "version": "0", "id": "6a7e8feb-b491-4cf7-a9f1-bf3703467718", @@ -46,7 +46,7 @@ def test_validate_event_does_not_conform_with_user_dict_schema(): "resources": ["arn:aws:ec2:us-west-1:123456789012:instance/i-1234567890abcdef0"], "detail": {}, } - with pytest.raises(exceptions.SchemaValidationError) as e: + with pytest.raises(exceptions.ModelValidationError) as e: handle_eventbridge(event_dict, LambdaContext()) print(e.exconly()) @@ -57,5 +57,5 @@ def test_handle_eventbridge_trigger_event_no_envelope(): def test_handle_invalid_event_with_eventbridge_envelope(): - with pytest.raises(exceptions.SchemaValidationError): + with pytest.raises(exceptions.ModelValidationError): handle_eventbridge(event={}, context=LambdaContext()) diff --git a/tests/functional/parser/test_parser.py b/tests/functional/parser/test_parser.py index 1b348e8e7cb..162b52ee439 100644 --- a/tests/functional/parser/test_parser.py +++ b/tests/functional/parser/test_parser.py @@ -8,11 +8,11 @@ @pytest.mark.parametrize("invalid_value", [None, bool(), [], (), object]) def test_parser_unsupported_event(dummy_schema, invalid_value): - @event_parser(schema=dummy_schema) + @event_parser(model=dummy_schema) def handle_no_envelope(event: Dict, _: LambdaContext): return event - with pytest.raises(exceptions.SchemaValidationError): + with pytest.raises(exceptions.ModelValidationError): handle_no_envelope(event=invalid_value, context=LambdaContext()) @@ -20,7 +20,7 @@ def handle_no_envelope(event: Dict, _: LambdaContext): "invalid_envelope,expected", [(True, ""), (["dummy"], ""), (object, exceptions.InvalidEnvelopeError)] ) def test_parser_invalid_envelope_type(dummy_event, dummy_schema, invalid_envelope, expected): - @event_parser(schema=dummy_schema, envelope=invalid_envelope) + @event_parser(model=dummy_schema, envelope=invalid_envelope) def handle_no_envelope(event: Dict, _: LambdaContext): return event @@ -32,7 +32,7 @@ def handle_no_envelope(event: Dict, _: LambdaContext): def test_parser_schema_with_envelope(dummy_event, dummy_schema, dummy_envelope): - @event_parser(schema=dummy_schema, envelope=dummy_envelope) + @event_parser(model=dummy_schema, envelope=dummy_envelope) def handle_no_envelope(event: Dict, _: LambdaContext): return event @@ -40,7 +40,7 @@ def handle_no_envelope(event: Dict, _: LambdaContext): def test_parser_schema_no_envelope(dummy_event, dummy_schema): - @event_parser(schema=dummy_schema) + @event_parser(model=dummy_schema) def handle_no_envelope(event: Dict, _: LambdaContext): return event @@ -49,9 +49,9 @@ def handle_no_envelope(event: Dict, _: LambdaContext): @pytest.mark.parametrize("invalid_schema", [None, str, bool(), [], (), object]) def test_parser_with_invalid_schema_type(dummy_event, invalid_schema): - @event_parser(schema=invalid_schema) + @event_parser(model=invalid_schema) def handle_no_envelope(event: Dict, _: LambdaContext): return event - with pytest.raises(exceptions.InvalidSchemaTypeError): + with pytest.raises(exceptions.InvalidModelTypeError): handle_no_envelope(event=dummy_event, context=LambdaContext()) diff --git a/tests/functional/parser/test_sqs.py b/tests/functional/parser/test_sqs.py index dec06e1d1e9..2ee992e2fa1 100644 --- a/tests/functional/parser/test_sqs.py +++ b/tests/functional/parser/test_sqs.py @@ -9,7 +9,7 @@ from tests.functional.validator.conftest import sqs_event # noqa: F401 -@event_parser(schema=MySqsBusiness, envelope=envelopes.SqsEnvelope) +@event_parser(model=MySqsBusiness, envelope=envelopes.SqsEnvelope) def handle_sqs_json_body(event: List[MySqsBusiness], _: LambdaContext): assert len(event) == 1 assert event[0].message == "hello world" @@ -20,14 +20,14 @@ def test_handle_sqs_trigger_event_json_body(sqs_event): # noqa: F811 handle_sqs_json_body(sqs_event, LambdaContext()) -def test_validate_event_does_not_conform_with_schema(): +def test_validate_event_does_not_conform_with_model(): event: Any = {"invalid": "event"} - with pytest.raises(exceptions.SchemaValidationError): + with pytest.raises(exceptions.ModelValidationError): handle_sqs_json_body(event, LambdaContext()) -def test_validate_event_does_not_conform_user_json_string_with_schema(): +def test_validate_event_does_not_conform_user_json_string_with_model(): event: Any = { "Records": [ { @@ -51,11 +51,11 @@ def test_validate_event_does_not_conform_user_json_string_with_schema(): ] } - with pytest.raises(exceptions.SchemaValidationError): + with pytest.raises(exceptions.ModelValidationError): handle_sqs_json_body(event, LambdaContext()) -@event_parser(schema=MyAdvancedSqsBusiness) +@event_parser(model=MyAdvancedSqsBusiness) def handle_sqs_no_envelope(event: MyAdvancedSqsBusiness, _: LambdaContext): records = event.Records record = records[0]