From 9fb7ee05d8cb48fc804c465b5ce1b65b1925a460 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Sat, 20 Mar 2021 08:45:57 -0700 Subject: [PATCH 01/14] feat(data-classes): Add S3 Object Event --- .../logging/correlation_paths.py | 1 + .../utilities/data_classes/s3_event.py | 178 +++++++++++++++++- tests/events/s3ObjectEventIAMUser.json | 29 +++ .../events/s3ObjectEventTempCredentials.json | 42 +++++ .../functional/test_lambda_trigger_events.py | 13 ++ 5 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 tests/events/s3ObjectEventIAMUser.json create mode 100644 tests/events/s3ObjectEventTempCredentials.json diff --git a/aws_lambda_powertools/logging/correlation_paths.py b/aws_lambda_powertools/logging/correlation_paths.py index cbccd85637f..7a4774ea2a3 100644 --- a/aws_lambda_powertools/logging/correlation_paths.py +++ b/aws_lambda_powertools/logging/correlation_paths.py @@ -5,3 +5,4 @@ APPSYNC_RESOLVER = 'request.headers."x-amzn-trace-id"' APPLICATION_LOAD_BALANCER = 'headers."x-amzn-trace-id"' EVENT_BRIDGE = "id" +S3_OBJECT = "xAmzRequestId" diff --git a/aws_lambda_powertools/utilities/data_classes/s3_event.py b/aws_lambda_powertools/utilities/data_classes/s3_event.py index 2670142d575..36100cdc839 100644 --- a/aws_lambda_powertools/utilities/data_classes/s3_event.py +++ b/aws_lambda_powertools/utilities/data_classes/s3_event.py @@ -1,4 +1,4 @@ -from typing import Dict, Iterator, Optional +from typing import Any, Dict, Iterator, Optional from urllib.parse import unquote_plus from aws_lambda_powertools.utilities.data_classes.common import DictWrapper @@ -187,3 +187,179 @@ def bucket_name(self) -> str: def object_key(self) -> str: """Get the object key for the first s3 event record and unquote plus""" return unquote_plus(self["Records"][0]["s3"]["object"]["key"]) + + +class S3ObjectContext(DictWrapper): + """The input and output details for connections to Amazon S3 and S3 Object Lambda.""" + + @property + def input_s3_url(self) -> str: + """A presigned URL that can be used to fetch the original object from Amazon S3. + The URL is signed using the original caller’s identity, and their permissions + will apply when the URL is used. If there are signed headers in the URL, the + Lambda function must include these in the call to Amazon S3, except for the Host.""" + return self["inputS3Url"] + + @property + def output_route(self) -> str: + """A routing token that is added to the S3 Object Lambda URL when the Lambda function + calls `WriteGetObjectResponse`.""" + return self["outputRoute"] + + @property + def output_token(self) -> str: + """An opaque token used by S3 Object Lambda to match the WriteGetObjectResponse call + with the original caller.""" + return self["outputToken"] + + +class S3ObjectConfiguration(DictWrapper): + """Configuration information about the S3 Object Lambda access point.""" + + @property + def access_point_arn(self) -> str: + """The Amazon Resource Name (ARN) of the S3 Object Lambda access point that received + this request.""" + return self["accessPointArn"] + + @property + def supporting_access_point_arn(self) -> str: + """The ARN of the supporting access point that is specified in the S3 Object Lambda + access point configuration.""" + return self["supportingAccessPointArn"] + + @property + def payload(self) -> str: + """Custom data that is applied to the S3 Object Lambda access point configuration. + S3 Object Lambda treats this as an opaque string, so it might need to be decoded + before use.""" + return self["payload"] + + +class S3ObjectUserRequest(DictWrapper): + """ Information about the original call to S3 Object Lambda.""" + + @property + def url(self) -> str: + """The decoded URL of the request as received by S3 Object Lambda, excluding any + authorization-related query parameters.""" + return self["url"] + + @property + def headers(self) -> Dict[str, str]: + """A map of string to strings containing the HTTP headers and their values from the + original call, excluding any authorization-related headers. If the same header appears + multiple times, their values are combined into a comma-delimited list. The case of the + original headers is retained in this map.""" + return self["headers"] + + +class S3ObjectUserIdentity(DictWrapper): + """Details about the identity that made the call to S3 Object Lambda. + + Documentation: + ------------- + - https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-event-reference-user-identity.html + """ + + @property + def get_type(self) -> str: + """The type of identity. + + The following values are possible: + - Root – The request was made with your AWS account credentials. If the userIdentity + type is Root and you set an alias for your account, the userName field contains your account alias. + For more information, see Your AWS Account ID and Its Alias. + - IAMUser – The request was made with the credentials of an IAM user. + - AssumedRole – The request was made with temporary security credentials that were obtained + with a role via a call to the AWS Security Token Service (AWS STS) AssumeRole API. This can include + roles for Amazon EC2 and cross-account API access. + - FederatedUser – The request was made with temporary security credentials that were obtained via a + call to the AWS STS GetFederationToken API. The sessionIssuer element indicates if the API was + called with root or IAM user credentials. + - AWSAccount – The request was made by another AWS account. + - AWSService – The request was made by an AWS account that belongs to an AWS service. + For example, AWS Elastic Beanstalk assumes an IAM role in your account to call other AWS services + on your behalf. + """ + return self["type"] + + @property + def account_id(self) -> str: + """The account that owns the entity that granted permissions for the request. + If the request was made with temporary security credentials, this is the account that owns the IAM + user or role that was used to obtain credentials.""" + return self["accountId"] + + @property + def access_key_id(self) -> str: + """The access key ID that was used to sign the request. + If the request was made with temporary security credentials, this is the access key ID of + the temporary credentials. For security reasons, accessKeyId might not be present, or might + be displayed as an empty string.""" + return self["accessKeyId"] + + @property + def user_name(self) -> str: + """The friendly name of the identity that made the call.""" + return self["userName"] + + @property + def principal_id(self) -> str: + """The unique identifier for the identity that made the call. + For requests made with temporary security credentials, this value includes + the session name that is passed to the AssumeRole, AssumeRoleWithWebIdentity, + or GetFederationToken API call.""" + return self["principalId"] + + @property + def arn(self) -> str: + """The ARN of the principal that made the call. + The last section of the ARN contains the user or role that made the call.""" + return self["arn"] + + @property + def session_context(self) -> Optional[Dict[str, Any]]: + """ If the request was made with temporary security credentials, + this element provides information about the session that was created for those credentials.""" + return self.get("sessionContext") + + +class S3ObjectEvent(DictWrapper): + """S3 object event notification + + Documentation: + ------------- + - https://docs.aws.amazon.com/AmazonS3/latest/userguide/olap-writing-lambda.html + """ + + @property + def request_id(self) -> str: + """The Amazon S3 request ID for this request. We recommend that you log this value to help with debugging.""" + return self["xAmzRequestId"] + + def object_context(self) -> S3ObjectContext: + """The input and output details for connections to Amazon S3 and S3 Object Lambda.""" + return S3ObjectContext(self["getObjectContext"]) + + def configuration(self) -> S3ObjectConfiguration: + """Configuration information about the S3 Object Lambda access point.""" + return S3ObjectConfiguration(self["configuration"]) + + def user_request(self) -> S3ObjectUserRequest: + """Information about the original call to S3 Object Lambda.""" + return S3ObjectUserRequest(self["userRequest"]) + + def user_identity(self) -> S3ObjectUserIdentity: + """Details about the identity that made the call to S3 Object Lambda.""" + return S3ObjectUserIdentity(self["userIdentity"]) + + def protocol_version(self) -> str: + """The version ID of the context provided. + The format of this field is {Major Version}.{Minor Version}. + The minor version numbers are always two-digit numbers. Any removal or change to the semantics of a + field will necessitate a major version bump and will require active opt-in. Amazon S3 can add new + fields at any time, at which point you might experience a minor version bump. Due to the nature of + software rollouts, it is possible that you might see multiple minor versions in use at once. + """ + return self["protocolVersion"] diff --git a/tests/events/s3ObjectEventIAMUser.json b/tests/events/s3ObjectEventIAMUser.json new file mode 100644 index 00000000000..3b5c5fa52bd --- /dev/null +++ b/tests/events/s3ObjectEventIAMUser.json @@ -0,0 +1,29 @@ +{ + "xAmzRequestId": "1a5ed718-5f53-471d-b6fe-5cf62d88d02a", + "getObjectContext": { + "inputS3Url": "https://myap-123412341234.s3-accesspoint.us-east-1.amazonaws.com/s3.txt?X-Amz-Security-Token=...", + "outputRoute": "io-iad-cell001", + "outputToken": "..." + }, + "configuration": { + "accessPointArn": "arn:aws:s3-object-lambda:us-east-1:123412341234:accesspoint/myolap", + "supportingAccessPointArn": "arn:aws:s3:us-east-1:123412341234:accesspoint/myap", + "payload": "test" + }, + "userRequest": { + "url": "/s3.txt", + "headers": { + "Host": "myolap-123412341234.s3-object-lambda.us-east-1.amazonaws.com", + "Accept-Encoding": "identity", + "X-Amz-Content-SHA256": "e3b0c44297fc1c149afbf4c8995fb92427ae41e4649b934ca495991b7852b855" + } + }, + "userIdentity": { + "type": "IAMUser", + "principalId": "...", + "arn": "arn:aws:iam::123412341234:user/myuser", + "accountId": "123412341234", + "accessKeyId": "..." + }, + "protocolVersion": "1.00" +} diff --git a/tests/events/s3ObjectEventTempCredentials.json b/tests/events/s3ObjectEventTempCredentials.json new file mode 100644 index 00000000000..30c70fe6df9 --- /dev/null +++ b/tests/events/s3ObjectEventTempCredentials.json @@ -0,0 +1,42 @@ +{ + "xAmzRequestId": "requestId", + "getObjectContext": { + "inputS3Url": "https://my-s3-ap-111122223333.s3-accesspoint.us-east-1.amazonaws.com/example?X-Amz-Security-Token=", + "outputRoute": "io-use1-001", + "outputToken": "OutputToken" + }, + "configuration": { + "accessPointArn": "arn:aws:s3-object-lambda:us-east-1:111122223333:accesspoint/example-object-lambda-ap", + "supportingAccessPointArn": "arn:aws:s3:us-east-1:111122223333:accesspoint/example-ap", + "payload": "{}" + }, + "userRequest": { + "url": "https://object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com/example", + "headers": { + "Host": "object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com", + "Accept-Encoding": "identity", + "X-Amz-Content-SHA256": "e3b0c44298fc1example" + } + }, + "userIdentity": { + "type": "AssumedRole", + "principalId": "principalId", + "arn": "arn:aws:sts::111122223333:assumed-role/Admin/example", + "accountId": "111122223333", + "accessKeyId": "accessKeyId", + "sessionContext": { + "attributes": { + "mfaAuthenticated": "false", + "creationDate": "Wed Mar 10 23:41:52 UTC 2021" + }, + "sessionIssuer": { + "type": "Role", + "principalId": "principalId", + "arn": "arn:aws:iam::111122223333:role/Admin", + "accountId": "111122223333", + "userName": "Admin" + } + } + }, + "protocolVersion": "1.00" +} diff --git a/tests/functional/test_lambda_trigger_events.py b/tests/functional/test_lambda_trigger_events.py index 7517c1a3c87..33984e63cfd 100644 --- a/tests/functional/test_lambda_trigger_events.py +++ b/tests/functional/test_lambda_trigger_events.py @@ -48,6 +48,7 @@ DynamoDBStreamEvent, StreamViewType, ) +from aws_lambda_powertools.utilities.data_classes.s3_event import S3ObjectEvent def load_event(file_name: str) -> dict: @@ -1005,3 +1006,15 @@ def test_appsync_resolver_event_empty(): assert event.info.field_name is None assert event.info.parent_type_name is None + + +def test_s3_object_event_iam(): + event = S3ObjectEvent(load_event("s3ObjectEventIAMUser.json")) + + assert event.request_id == "1a5ed718-5f53-471d-b6fe-5cf62d88d02a" + + +def test_s3_object_event_temp_creds(): + event = S3ObjectEvent(load_event("s3ObjectEventTempCredentials.json")) + + assert event.request_id == "requestId" From baa536fdc65235f20a2286eb6562ca7fe91c4cb0 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Sat, 20 Mar 2021 10:15:16 -0700 Subject: [PATCH 02/14] feat(data-classes): Complete docs and implementation Changes: - Add support for get_header_value - Add docs with example - Add test coverage - Add example userName example event json --- .flake8 | 2 +- .../utilities/data_classes/s3_event.py | 33 +++++++++++++++- docs/utilities/data_classes.md | 38 +++++++++++++++++++ tests/events/s3ObjectEventIAMUser.json | 3 +- .../functional/test_lambda_trigger_events.py | 35 +++++++++++++++-- 5 files changed, 104 insertions(+), 7 deletions(-) diff --git a/.flake8 b/.flake8 index 6c0c78fa967..09975e87f87 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,5 @@ [flake8] -exclude = docs, .eggs, setup.py, example, .aws-sam, .git, dist, *.md, *.yaml, example/samconfig.toml, *.txt, *.ini +exclude = docs, .eggs, setup.py, example, .aws-sam, .git, dist, *.md, *.yaml, example/samconfig.toml, *.txt, *.ini, tests/THIRD-PARTY-LICENSES ignore = E203, E266, W503, BLK100, W291, I004 max-line-length = 120 max-complexity = 15 diff --git a/aws_lambda_powertools/utilities/data_classes/s3_event.py b/aws_lambda_powertools/utilities/data_classes/s3_event.py index 36100cdc839..5bc8614b5c9 100644 --- a/aws_lambda_powertools/utilities/data_classes/s3_event.py +++ b/aws_lambda_powertools/utilities/data_classes/s3_event.py @@ -1,7 +1,7 @@ from typing import Any, Dict, Iterator, Optional from urllib.parse import unquote_plus -from aws_lambda_powertools.utilities.data_classes.common import DictWrapper +from aws_lambda_powertools.utilities.data_classes.common import DictWrapper, get_header_value class S3Identity(DictWrapper): @@ -253,6 +253,26 @@ def headers(self) -> Dict[str, str]: original headers is retained in this map.""" return self["headers"] + def get_header_value( + self, name: str, default_value: Optional[str] = None, case_sensitive: Optional[bool] = False + ) -> Optional[str]: + """Get header value by name + + Parameters + ---------- + name: str + Header name + default_value: str, optional + Default value if no value was found by name + case_sensitive: bool + Whether to use a case sensitive look up + Returns + ------- + str, optional + Header value + """ + return get_header_value(self.headers, name, default_value, case_sensitive) + class S3ObjectUserIdentity(DictWrapper): """Details about the identity that made the call to S3 Object Lambda. @@ -267,6 +287,7 @@ def get_type(self) -> str: """The type of identity. The following values are possible: + - Root – The request was made with your AWS account credentials. If the userIdentity type is Root and you set an alias for your account, the userName field contains your account alias. For more information, see Your AWS Account ID and Its Alias. @@ -294,6 +315,7 @@ def account_id(self) -> str: @property def access_key_id(self) -> str: """The access key ID that was used to sign the request. + If the request was made with temporary security credentials, this is the access key ID of the temporary credentials. For security reasons, accessKeyId might not be present, or might be displayed as an empty string.""" @@ -307,6 +329,7 @@ def user_name(self) -> str: @property def principal_id(self) -> str: """The unique identifier for the identity that made the call. + For requests made with temporary security credentials, this value includes the session name that is passed to the AssumeRole, AssumeRoleWithWebIdentity, or GetFederationToken API call.""" @@ -338,25 +361,31 @@ def request_id(self) -> str: """The Amazon S3 request ID for this request. We recommend that you log this value to help with debugging.""" return self["xAmzRequestId"] + @property def object_context(self) -> S3ObjectContext: """The input and output details for connections to Amazon S3 and S3 Object Lambda.""" return S3ObjectContext(self["getObjectContext"]) + @property def configuration(self) -> S3ObjectConfiguration: """Configuration information about the S3 Object Lambda access point.""" return S3ObjectConfiguration(self["configuration"]) + @property def user_request(self) -> S3ObjectUserRequest: """Information about the original call to S3 Object Lambda.""" return S3ObjectUserRequest(self["userRequest"]) + @property def user_identity(self) -> S3ObjectUserIdentity: """Details about the identity that made the call to S3 Object Lambda.""" return S3ObjectUserIdentity(self["userIdentity"]) + @property def protocol_version(self) -> str: """The version ID of the context provided. - The format of this field is {Major Version}.{Minor Version}. + + The format of this field is `{Major Version}`.`{Minor Version}`. The minor version numbers are always two-digit numbers. Any removal or change to the semantics of a field will necessitate a major version bump and will require active opt-in. Amazon S3 can add new fields at any time, at which point you might experience a minor version bump. Due to the nature of diff --git a/docs/utilities/data_classes.md b/docs/utilities/data_classes.md index b834390cc15..4394f15c98c 100644 --- a/docs/utilities/data_classes.md +++ b/docs/utilities/data_classes.md @@ -59,6 +59,7 @@ Event Source | Data_class [EventBridge](#eventbridge) | `EventBridgeEvent` [Kinesis Data Stream](#kinesis-streams) | `KinesisStreamEvent` [S3](#s3) | `S3Event` +[S3 Object](#s3-object) | `S3ObjectEvent` [SES](#ses) | `SESEvent` [SNS](#sns) | `SNSEvent` [SQS](#sqs) | `SQSEvent` @@ -547,6 +548,43 @@ or plain text, depending on the original payload. do_something_with(f'{bucket_name}/{object_key}') ``` +### S3 Object + +This example is based on the AWS Blog post [Introducing Amazon S3 Object Lambda – Use Your Code to Process Data as It Is Being Retrieved from S3](https://aws.amazon.com/blogs/aws/introducing-amazon-s3-object-lambda-use-your-code-to-process-data-as-it-is-being-retrieved-from-s3/){target="_blank"}. + +=== "app.py" + + ```python hl_lines="4 8 10" + import requests + from aws_lambda_powertools import Logger + from aws_lambda_powertools.logging.correlation_paths import S3_OBJECT + from aws_lambda_powertools.utilities.data_classes.s3_event import S3ObjectEvent + + logger = Logger() + + @logger.inject_lambda_context(correlation_id_path=S3_OBJECT, log_event=True) + def lambda_handler(event, context): + event = S3ObjectEvent(event) + + object_context = event.object_context + request_route = object_context.output_route + request_token = object_context.output_token + s3_url = object_context.input_s3_url + + # Get object from S3 + response = requests.get(s3_url) + original_object = response.content.decode("utf-8") + + # Transform object + transformed_object = original_object.upper() + + # Write object back to S3 Object Lambda + s3 = boto3.client("s3") + s3.write_get_object_response(Body=transformed_object, RequestRoute=request_route, RequestToken=request_token) + + return {"status_code": 200} + ``` + ### SES === "app.py" diff --git a/tests/events/s3ObjectEventIAMUser.json b/tests/events/s3ObjectEventIAMUser.json index 3b5c5fa52bd..6be41c4352b 100644 --- a/tests/events/s3ObjectEventIAMUser.json +++ b/tests/events/s3ObjectEventIAMUser.json @@ -23,7 +23,8 @@ "principalId": "...", "arn": "arn:aws:iam::123412341234:user/myuser", "accountId": "123412341234", - "accessKeyId": "..." + "accessKeyId": "...", + "userName": "Alice" }, "protocolVersion": "1.00" } diff --git a/tests/functional/test_lambda_trigger_events.py b/tests/functional/test_lambda_trigger_events.py index 33984e63cfd..d2401bcc2f3 100644 --- a/tests/functional/test_lambda_trigger_events.py +++ b/tests/functional/test_lambda_trigger_events.py @@ -1012,9 +1012,38 @@ def test_s3_object_event_iam(): event = S3ObjectEvent(load_event("s3ObjectEventIAMUser.json")) assert event.request_id == "1a5ed718-5f53-471d-b6fe-5cf62d88d02a" - - -def test_s3_object_event_temp_creds(): + assert event.object_context is not None + object_context = event.object_context + assert object_context.input_s3_url == event["getObjectContext"]["inputS3Url"] + assert object_context.output_route == event["getObjectContext"]["outputRoute"] + assert object_context.output_token == event["getObjectContext"]["outputToken"] + assert event.configuration is not None + configuration = event.configuration + assert configuration.access_point_arn == event["configuration"]["accessPointArn"] + assert configuration.supporting_access_point_arn == event["configuration"]["supportingAccessPointArn"] + assert configuration.payload == event["configuration"]["payload"] + assert event.user_request is not None + user_request = event.user_request + assert user_request.url == event["userRequest"]["url"] + assert user_request.headers == event["userRequest"]["headers"] + assert user_request.get_header_value("Accept-Encoding") == "identity" + assert event.user_identity is not None + user_identity = event.user_identity + assert user_identity.get_type == event["userIdentity"]["type"] + assert user_identity.principal_id == event["userIdentity"]["principalId"] + assert user_identity.arn == event["userIdentity"]["arn"] + assert user_identity.account_id == event["userIdentity"]["accountId"] + assert user_identity.access_key_id == event["userIdentity"]["accessKeyId"] + assert user_identity.user_name == event["userIdentity"]["userName"] + assert user_identity.session_context is None + assert event.protocol_version == event["protocolVersion"] + + +def test_s3_object_event_temp_credentials(): event = S3ObjectEvent(load_event("s3ObjectEventTempCredentials.json")) assert event.request_id == "requestId" + session_context = event.user_identity.session_context + assert session_context is not None + # NOTE: This can be a data class to like S3ObjectSessionContext + assert isinstance(session_context, dict) From 107d1d470b970be98863268717526cb77032a79f Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Sat, 20 Mar 2021 11:35:24 -0700 Subject: [PATCH 03/14] chore: revert --- .flake8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index 09975e87f87..6c0c78fa967 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,5 @@ [flake8] -exclude = docs, .eggs, setup.py, example, .aws-sam, .git, dist, *.md, *.yaml, example/samconfig.toml, *.txt, *.ini, tests/THIRD-PARTY-LICENSES +exclude = docs, .eggs, setup.py, example, .aws-sam, .git, dist, *.md, *.yaml, example/samconfig.toml, *.txt, *.ini ignore = E203, E266, W503, BLK100, W291, I004 max-line-length = 120 max-complexity = 15 From f4701439885623eaa577752d6bf9022997928c3c Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Sat, 20 Mar 2021 11:42:02 -0700 Subject: [PATCH 04/14] refactor(data-classes): Rename S3ObjectEvent to S3ObjectLambdaEvent --- aws_lambda_powertools/utilities/data_classes/s3_event.py | 2 +- docs/utilities/data_classes.md | 8 ++++---- tests/functional/test_lambda_trigger_events.py | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_classes/s3_event.py b/aws_lambda_powertools/utilities/data_classes/s3_event.py index 5bc8614b5c9..b8d3a2c123b 100644 --- a/aws_lambda_powertools/utilities/data_classes/s3_event.py +++ b/aws_lambda_powertools/utilities/data_classes/s3_event.py @@ -348,7 +348,7 @@ def session_context(self) -> Optional[Dict[str, Any]]: return self.get("sessionContext") -class S3ObjectEvent(DictWrapper): +class S3ObjectLambdaEvent(DictWrapper): """S3 object event notification Documentation: diff --git a/docs/utilities/data_classes.md b/docs/utilities/data_classes.md index 4394f15c98c..0053f1cb655 100644 --- a/docs/utilities/data_classes.md +++ b/docs/utilities/data_classes.md @@ -59,7 +59,7 @@ Event Source | Data_class [EventBridge](#eventbridge) | `EventBridgeEvent` [Kinesis Data Stream](#kinesis-streams) | `KinesisStreamEvent` [S3](#s3) | `S3Event` -[S3 Object](#s3-object) | `S3ObjectEvent` +[S3 Object Lambda](#s3-object-lambda) | `S3ObjectLambdaEvent` [SES](#ses) | `SESEvent` [SNS](#sns) | `SNSEvent` [SQS](#sqs) | `SQSEvent` @@ -548,7 +548,7 @@ or plain text, depending on the original payload. do_something_with(f'{bucket_name}/{object_key}') ``` -### S3 Object +### S3 Object Lambda This example is based on the AWS Blog post [Introducing Amazon S3 Object Lambda – Use Your Code to Process Data as It Is Being Retrieved from S3](https://aws.amazon.com/blogs/aws/introducing-amazon-s3-object-lambda-use-your-code-to-process-data-as-it-is-being-retrieved-from-s3/){target="_blank"}. @@ -558,13 +558,13 @@ This example is based on the AWS Blog post [Introducing Amazon S3 Object Lambda import requests from aws_lambda_powertools import Logger from aws_lambda_powertools.logging.correlation_paths import S3_OBJECT - from aws_lambda_powertools.utilities.data_classes.s3_event import S3ObjectEvent + from aws_lambda_powertools.utilities.data_classes.s3_event import S3ObjectLambdaEvent logger = Logger() @logger.inject_lambda_context(correlation_id_path=S3_OBJECT, log_event=True) def lambda_handler(event, context): - event = S3ObjectEvent(event) + event = S3ObjectLambdaEvent(event) object_context = event.object_context request_route = object_context.output_route diff --git a/tests/functional/test_lambda_trigger_events.py b/tests/functional/test_lambda_trigger_events.py index d2401bcc2f3..c110272d064 100644 --- a/tests/functional/test_lambda_trigger_events.py +++ b/tests/functional/test_lambda_trigger_events.py @@ -48,7 +48,7 @@ DynamoDBStreamEvent, StreamViewType, ) -from aws_lambda_powertools.utilities.data_classes.s3_event import S3ObjectEvent +from aws_lambda_powertools.utilities.data_classes.s3_event import S3ObjectLambdaEvent def load_event(file_name: str) -> dict: @@ -1009,7 +1009,7 @@ def test_appsync_resolver_event_empty(): def test_s3_object_event_iam(): - event = S3ObjectEvent(load_event("s3ObjectEventIAMUser.json")) + event = S3ObjectLambdaEvent(load_event("s3ObjectEventIAMUser.json")) assert event.request_id == "1a5ed718-5f53-471d-b6fe-5cf62d88d02a" assert event.object_context is not None @@ -1040,7 +1040,7 @@ def test_s3_object_event_iam(): def test_s3_object_event_temp_credentials(): - event = S3ObjectEvent(load_event("s3ObjectEventTempCredentials.json")) + event = S3ObjectLambdaEvent(load_event("s3ObjectEventTempCredentials.json")) assert event.request_id == "requestId" session_context = event.user_identity.session_context From 044c2f80592f2af12cb5af80c4d65f953bf87710 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Sat, 20 Mar 2021 13:45:27 -0700 Subject: [PATCH 05/14] feat(data-classes): Add S3ObjectSessionContext --- .../utilities/data_classes/s3_event.py | 209 +------------- .../utilities/data_classes/s3_object_event.py | 267 ++++++++++++++++++ docs/utilities/data_classes.md | 2 +- .../functional/test_lambda_trigger_events.py | 15 +- 4 files changed, 282 insertions(+), 211 deletions(-) create mode 100644 aws_lambda_powertools/utilities/data_classes/s3_object_event.py diff --git a/aws_lambda_powertools/utilities/data_classes/s3_event.py b/aws_lambda_powertools/utilities/data_classes/s3_event.py index b8d3a2c123b..2670142d575 100644 --- a/aws_lambda_powertools/utilities/data_classes/s3_event.py +++ b/aws_lambda_powertools/utilities/data_classes/s3_event.py @@ -1,7 +1,7 @@ -from typing import Any, Dict, Iterator, Optional +from typing import Dict, Iterator, Optional from urllib.parse import unquote_plus -from aws_lambda_powertools.utilities.data_classes.common import DictWrapper, get_header_value +from aws_lambda_powertools.utilities.data_classes.common import DictWrapper class S3Identity(DictWrapper): @@ -187,208 +187,3 @@ def bucket_name(self) -> str: def object_key(self) -> str: """Get the object key for the first s3 event record and unquote plus""" return unquote_plus(self["Records"][0]["s3"]["object"]["key"]) - - -class S3ObjectContext(DictWrapper): - """The input and output details for connections to Amazon S3 and S3 Object Lambda.""" - - @property - def input_s3_url(self) -> str: - """A presigned URL that can be used to fetch the original object from Amazon S3. - The URL is signed using the original caller’s identity, and their permissions - will apply when the URL is used. If there are signed headers in the URL, the - Lambda function must include these in the call to Amazon S3, except for the Host.""" - return self["inputS3Url"] - - @property - def output_route(self) -> str: - """A routing token that is added to the S3 Object Lambda URL when the Lambda function - calls `WriteGetObjectResponse`.""" - return self["outputRoute"] - - @property - def output_token(self) -> str: - """An opaque token used by S3 Object Lambda to match the WriteGetObjectResponse call - with the original caller.""" - return self["outputToken"] - - -class S3ObjectConfiguration(DictWrapper): - """Configuration information about the S3 Object Lambda access point.""" - - @property - def access_point_arn(self) -> str: - """The Amazon Resource Name (ARN) of the S3 Object Lambda access point that received - this request.""" - return self["accessPointArn"] - - @property - def supporting_access_point_arn(self) -> str: - """The ARN of the supporting access point that is specified in the S3 Object Lambda - access point configuration.""" - return self["supportingAccessPointArn"] - - @property - def payload(self) -> str: - """Custom data that is applied to the S3 Object Lambda access point configuration. - S3 Object Lambda treats this as an opaque string, so it might need to be decoded - before use.""" - return self["payload"] - - -class S3ObjectUserRequest(DictWrapper): - """ Information about the original call to S3 Object Lambda.""" - - @property - def url(self) -> str: - """The decoded URL of the request as received by S3 Object Lambda, excluding any - authorization-related query parameters.""" - return self["url"] - - @property - def headers(self) -> Dict[str, str]: - """A map of string to strings containing the HTTP headers and their values from the - original call, excluding any authorization-related headers. If the same header appears - multiple times, their values are combined into a comma-delimited list. The case of the - original headers is retained in this map.""" - return self["headers"] - - def get_header_value( - self, name: str, default_value: Optional[str] = None, case_sensitive: Optional[bool] = False - ) -> Optional[str]: - """Get header value by name - - Parameters - ---------- - name: str - Header name - default_value: str, optional - Default value if no value was found by name - case_sensitive: bool - Whether to use a case sensitive look up - Returns - ------- - str, optional - Header value - """ - return get_header_value(self.headers, name, default_value, case_sensitive) - - -class S3ObjectUserIdentity(DictWrapper): - """Details about the identity that made the call to S3 Object Lambda. - - Documentation: - ------------- - - https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-event-reference-user-identity.html - """ - - @property - def get_type(self) -> str: - """The type of identity. - - The following values are possible: - - - Root – The request was made with your AWS account credentials. If the userIdentity - type is Root and you set an alias for your account, the userName field contains your account alias. - For more information, see Your AWS Account ID and Its Alias. - - IAMUser – The request was made with the credentials of an IAM user. - - AssumedRole – The request was made with temporary security credentials that were obtained - with a role via a call to the AWS Security Token Service (AWS STS) AssumeRole API. This can include - roles for Amazon EC2 and cross-account API access. - - FederatedUser – The request was made with temporary security credentials that were obtained via a - call to the AWS STS GetFederationToken API. The sessionIssuer element indicates if the API was - called with root or IAM user credentials. - - AWSAccount – The request was made by another AWS account. - - AWSService – The request was made by an AWS account that belongs to an AWS service. - For example, AWS Elastic Beanstalk assumes an IAM role in your account to call other AWS services - on your behalf. - """ - return self["type"] - - @property - def account_id(self) -> str: - """The account that owns the entity that granted permissions for the request. - If the request was made with temporary security credentials, this is the account that owns the IAM - user or role that was used to obtain credentials.""" - return self["accountId"] - - @property - def access_key_id(self) -> str: - """The access key ID that was used to sign the request. - - If the request was made with temporary security credentials, this is the access key ID of - the temporary credentials. For security reasons, accessKeyId might not be present, or might - be displayed as an empty string.""" - return self["accessKeyId"] - - @property - def user_name(self) -> str: - """The friendly name of the identity that made the call.""" - return self["userName"] - - @property - def principal_id(self) -> str: - """The unique identifier for the identity that made the call. - - For requests made with temporary security credentials, this value includes - the session name that is passed to the AssumeRole, AssumeRoleWithWebIdentity, - or GetFederationToken API call.""" - return self["principalId"] - - @property - def arn(self) -> str: - """The ARN of the principal that made the call. - The last section of the ARN contains the user or role that made the call.""" - return self["arn"] - - @property - def session_context(self) -> Optional[Dict[str, Any]]: - """ If the request was made with temporary security credentials, - this element provides information about the session that was created for those credentials.""" - return self.get("sessionContext") - - -class S3ObjectLambdaEvent(DictWrapper): - """S3 object event notification - - Documentation: - ------------- - - https://docs.aws.amazon.com/AmazonS3/latest/userguide/olap-writing-lambda.html - """ - - @property - def request_id(self) -> str: - """The Amazon S3 request ID for this request. We recommend that you log this value to help with debugging.""" - return self["xAmzRequestId"] - - @property - def object_context(self) -> S3ObjectContext: - """The input and output details for connections to Amazon S3 and S3 Object Lambda.""" - return S3ObjectContext(self["getObjectContext"]) - - @property - def configuration(self) -> S3ObjectConfiguration: - """Configuration information about the S3 Object Lambda access point.""" - return S3ObjectConfiguration(self["configuration"]) - - @property - def user_request(self) -> S3ObjectUserRequest: - """Information about the original call to S3 Object Lambda.""" - return S3ObjectUserRequest(self["userRequest"]) - - @property - def user_identity(self) -> S3ObjectUserIdentity: - """Details about the identity that made the call to S3 Object Lambda.""" - return S3ObjectUserIdentity(self["userIdentity"]) - - @property - def protocol_version(self) -> str: - """The version ID of the context provided. - - The format of this field is `{Major Version}`.`{Minor Version}`. - The minor version numbers are always two-digit numbers. Any removal or change to the semantics of a - field will necessitate a major version bump and will require active opt-in. Amazon S3 can add new - fields at any time, at which point you might experience a minor version bump. Due to the nature of - software rollouts, it is possible that you might see multiple minor versions in use at once. - """ - return self["protocolVersion"] diff --git a/aws_lambda_powertools/utilities/data_classes/s3_object_event.py b/aws_lambda_powertools/utilities/data_classes/s3_object_event.py new file mode 100644 index 00000000000..1cd7d9f9320 --- /dev/null +++ b/aws_lambda_powertools/utilities/data_classes/s3_object_event.py @@ -0,0 +1,267 @@ +from typing import Dict, Optional + +from aws_lambda_powertools.utilities.data_classes.common import DictWrapper, get_header_value + + +class S3ObjectContext(DictWrapper): + """The input and output details for connections to Amazon S3 and S3 Object Lambda.""" + + @property + def input_s3_url(self) -> str: + """A pre-signed URL that can be used to fetch the original object from Amazon S3. + The URL is signed using the original caller’s identity, and their permissions + will apply when the URL is used. If there are signed headers in the URL, the + Lambda function must include these in the call to Amazon S3, except for the Host.""" + return self["inputS3Url"] + + @property + def output_route(self) -> str: + """A routing token that is added to the S3 Object Lambda URL when the Lambda function + calls `WriteGetObjectResponse`.""" + return self["outputRoute"] + + @property + def output_token(self) -> str: + """An opaque token used by S3 Object Lambda to match the WriteGetObjectResponse call + with the original caller.""" + return self["outputToken"] + + +class S3ObjectConfiguration(DictWrapper): + """Configuration information about the S3 Object Lambda access point.""" + + @property + def access_point_arn(self) -> str: + """The Amazon Resource Name (ARN) of the S3 Object Lambda access point that received + this request.""" + return self["accessPointArn"] + + @property + def supporting_access_point_arn(self) -> str: + """The ARN of the supporting access point that is specified in the S3 Object Lambda + access point configuration.""" + return self["supportingAccessPointArn"] + + @property + def payload(self) -> str: + """Custom data that is applied to the S3 Object Lambda access point configuration. + S3 Object Lambda treats this as an opaque string, so it might need to be decoded + before use.""" + return self["payload"] + + +class S3ObjectUserRequest(DictWrapper): + """ Information about the original call to S3 Object Lambda.""" + + @property + def url(self) -> str: + """The decoded URL of the request as received by S3 Object Lambda, excluding any + authorization-related query parameters.""" + return self["url"] + + @property + def headers(self) -> Dict[str, str]: + """A map of string to strings containing the HTTP headers and their values from the + original call, excluding any authorization-related headers. If the same header appears + multiple times, their values are combined into a comma-delimited list. The case of the + original headers is retained in this map.""" + return self["headers"] + + def get_header_value( + self, name: str, default_value: Optional[str] = None, case_sensitive: Optional[bool] = False + ) -> Optional[str]: + """Get header value by name + + Parameters + ---------- + name: str + Header name + default_value: str, optional + Default value if no value was found by name + case_sensitive: bool + Whether to use a case sensitive look up + Returns + ------- + str, optional + Header value + """ + return get_header_value(self.headers, name, default_value, case_sensitive) + + +class S3ObjectSessionIssuer(DictWrapper): + @property + def get_type(self) -> str: + """The source of the temporary security credentials, such as Root, IAMUser, or Role.""" + return self["type"] + + @property + def user_name(self) -> str: + """The friendly name of the user or role that issued the session.""" + return self["userName"] + + @property + def principal_id(self) -> str: + """The internal ID of the entity that was used to get credentials.""" + return self["principalId"] + + @property + def arn(self) -> str: + """The ARN of the source (account, IAM user, or role) that was used to get temporary security credentials.""" + return self["arn"] + + @property + def account_id(self) -> str: + """The account that owns the entity that was used to get credentials.""" + return self["accountId"] + + +class S3ObjectSessionAttributes(DictWrapper): + @property + def creation_date(self) -> str: + """The date and time when the temporary security credentials were issued. + Represented in ISO 8601 basic notation.""" + return self["creationDate"] + + @property + def mfa_authenticated(self) -> str: + """The value is true if the root user or IAM user whose credentials were used for the request also was + authenticated with an MFA device; otherwise, false..""" + return self["mfaAuthenticated"] + + +class S3ObjectSessionContext(DictWrapper): + @property + def session_issuer(self) -> S3ObjectSessionIssuer: + """If the request was made with temporary security credentials, an element that provides information + about how the credentials were obtained. """ + return S3ObjectSessionIssuer(self["sessionIssuer"]) + + @property + def attributes(self) -> S3ObjectSessionAttributes: + """User session attributes. """ + return S3ObjectSessionAttributes(self["attributes"]) + + +class S3ObjectUserIdentity(DictWrapper): + """Details about the identity that made the call to S3 Object Lambda. + + Documentation: + ------------- + - https://docs.aws.amazon.com/awscloudtrail/latest/userguide/cloudtrail-event-reference-user-identity.html + """ + + @property + def get_type(self) -> str: + """The type of identity. + + The following values are possible: + + - Root – The request was made with your AWS account credentials. If the userIdentity + type is Root and you set an alias for your account, the userName field contains your account alias. + For more information, see Your AWS Account ID and Its Alias. + - IAMUser – The request was made with the credentials of an IAM user. + - AssumedRole – The request was made with temporary security credentials that were obtained + with a role via a call to the AWS Security Token Service (AWS STS) AssumeRole API. This can include + roles for Amazon EC2 and cross-account API access. + - FederatedUser – The request was made with temporary security credentials that were obtained via a + call to the AWS STS GetFederationToken API. The sessionIssuer element indicates if the API was + called with root or IAM user credentials. + - AWSAccount – The request was made by another AWS account. + - AWSService – The request was made by an AWS account that belongs to an AWS service. + For example, AWS Elastic Beanstalk assumes an IAM role in your account to call other AWS services + on your behalf. + """ + return self["type"] + + @property + def account_id(self) -> str: + """The account that owns the entity that granted permissions for the request. + If the request was made with temporary security credentials, this is the account that owns the IAM + user or role that was used to obtain credentials.""" + return self["accountId"] + + @property + def access_key_id(self) -> str: + """The access key ID that was used to sign the request. + + If the request was made with temporary security credentials, this is the access key ID of + the temporary credentials. For security reasons, accessKeyId might not be present, or might + be displayed as an empty string.""" + return self["accessKeyId"] + + @property + def user_name(self) -> str: + """The friendly name of the identity that made the call.""" + return self["userName"] + + @property + def principal_id(self) -> str: + """The unique identifier for the identity that made the call. + + For requests made with temporary security credentials, this value includes + the session name that is passed to the AssumeRole, AssumeRoleWithWebIdentity, + or GetFederationToken API call.""" + return self["principalId"] + + @property + def arn(self) -> str: + """The ARN of the principal that made the call. + The last section of the ARN contains the user or role that made the call.""" + return self["arn"] + + @property + def session_context(self) -> Optional[S3ObjectSessionContext]: + """ If the request was made with temporary security credentials, + this element provides information about the session that was created for those credentials.""" + session_context = self.get("sessionContext") + + if session_context is None: + return None + + return S3ObjectSessionContext(session_context) + + +class S3ObjectLambdaEvent(DictWrapper): + """S3 object event notification + + Documentation: + ------------- + - https://docs.aws.amazon.com/AmazonS3/latest/userguide/olap-writing-lambda.html + """ + + @property + def request_id(self) -> str: + """The Amazon S3 request ID for this request. We recommend that you log this value to help with debugging.""" + return self["xAmzRequestId"] + + @property + def object_context(self) -> S3ObjectContext: + """The input and output details for connections to Amazon S3 and S3 Object Lambda.""" + return S3ObjectContext(self["getObjectContext"]) + + @property + def configuration(self) -> S3ObjectConfiguration: + """Configuration information about the S3 Object Lambda access point.""" + return S3ObjectConfiguration(self["configuration"]) + + @property + def user_request(self) -> S3ObjectUserRequest: + """Information about the original call to S3 Object Lambda.""" + return S3ObjectUserRequest(self["userRequest"]) + + @property + def user_identity(self) -> S3ObjectUserIdentity: + """Details about the identity that made the call to S3 Object Lambda.""" + return S3ObjectUserIdentity(self["userIdentity"]) + + @property + def protocol_version(self) -> str: + """The version ID of the context provided. + + The format of this field is `{Major Version}`.`{Minor Version}`. + The minor version numbers are always two-digit numbers. Any removal or change to the semantics of a + field will necessitate a major version bump and will require active opt-in. Amazon S3 can add new + fields at any time, at which point you might experience a minor version bump. Due to the nature of + software rollouts, it is possible that you might see multiple minor versions in use at once. + """ + return self["protocolVersion"] diff --git a/docs/utilities/data_classes.md b/docs/utilities/data_classes.md index 0053f1cb655..4b6256f7adb 100644 --- a/docs/utilities/data_classes.md +++ b/docs/utilities/data_classes.md @@ -558,7 +558,7 @@ This example is based on the AWS Blog post [Introducing Amazon S3 Object Lambda import requests from aws_lambda_powertools import Logger from aws_lambda_powertools.logging.correlation_paths import S3_OBJECT - from aws_lambda_powertools.utilities.data_classes.s3_event import S3ObjectLambdaEvent + from aws_lambda_powertools.utilities.data_classes.s3_object_event import S3ObjectLambdaEvent logger = Logger() diff --git a/tests/functional/test_lambda_trigger_events.py b/tests/functional/test_lambda_trigger_events.py index c110272d064..d3d3f4c9fe2 100644 --- a/tests/functional/test_lambda_trigger_events.py +++ b/tests/functional/test_lambda_trigger_events.py @@ -48,7 +48,7 @@ DynamoDBStreamEvent, StreamViewType, ) -from aws_lambda_powertools.utilities.data_classes.s3_event import S3ObjectLambdaEvent +from aws_lambda_powertools.utilities.data_classes.s3_object_event import S3ObjectLambdaEvent def load_event(file_name: str) -> dict: @@ -1045,5 +1045,14 @@ def test_s3_object_event_temp_credentials(): assert event.request_id == "requestId" session_context = event.user_identity.session_context assert session_context is not None - # NOTE: This can be a data class to like S3ObjectSessionContext - assert isinstance(session_context, dict) + session_issuer = session_context.session_issuer + assert session_issuer is not None + assert session_issuer.get_type == session_context["sessionIssuer"]["type"] + assert session_issuer.user_name == session_context["sessionIssuer"]["userName"] + assert session_issuer.principal_id == session_context["sessionIssuer"]["principalId"] + assert session_issuer.arn == session_context["sessionIssuer"]["arn"] + assert session_issuer.account_id == session_context["sessionIssuer"]["accountId"] + session_attributes = session_context.attributes + assert session_attributes is not None + assert session_attributes.mfa_authenticated == session_context["attributes"]["mfaAuthenticated"] + assert session_attributes.creation_date == session_context["attributes"]["creationDate"] From f334c6f6c01d5fa2e6af8dc40fc3e0743deecbf8 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Sat, 20 Mar 2021 13:47:51 -0700 Subject: [PATCH 06/14] refactor(logging): Rename to S3_LAMBDA_OBJECT --- aws_lambda_powertools/logging/correlation_paths.py | 2 +- docs/utilities/data_classes.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aws_lambda_powertools/logging/correlation_paths.py b/aws_lambda_powertools/logging/correlation_paths.py index 7a4774ea2a3..e20a3e690e4 100644 --- a/aws_lambda_powertools/logging/correlation_paths.py +++ b/aws_lambda_powertools/logging/correlation_paths.py @@ -5,4 +5,4 @@ APPSYNC_RESOLVER = 'request.headers."x-amzn-trace-id"' APPLICATION_LOAD_BALANCER = 'headers."x-amzn-trace-id"' EVENT_BRIDGE = "id" -S3_OBJECT = "xAmzRequestId" +S3_LAMBDA_OBJECT = "xAmzRequestId" diff --git a/docs/utilities/data_classes.md b/docs/utilities/data_classes.md index 4b6256f7adb..88e14fd8ecb 100644 --- a/docs/utilities/data_classes.md +++ b/docs/utilities/data_classes.md @@ -557,12 +557,12 @@ This example is based on the AWS Blog post [Introducing Amazon S3 Object Lambda ```python hl_lines="4 8 10" import requests from aws_lambda_powertools import Logger - from aws_lambda_powertools.logging.correlation_paths import S3_OBJECT + from aws_lambda_powertools.logging.correlation_paths import S3_LAMBDA_OBJECT from aws_lambda_powertools.utilities.data_classes.s3_object_event import S3ObjectLambdaEvent logger = Logger() - @logger.inject_lambda_context(correlation_id_path=S3_OBJECT, log_event=True) + @logger.inject_lambda_context(correlation_id_path=S3_LAMBDA_OBJECT, log_event=True) def lambda_handler(event, context): event = S3ObjectLambdaEvent(event) From ecf8ba0585214b8f69f15630577e7ace44f0dcbc Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Sat, 20 Mar 2021 13:49:56 -0700 Subject: [PATCH 07/14] refactor(logging): Rename to S3_OBJECT_LAMBDA --- aws_lambda_powertools/logging/correlation_paths.py | 2 +- docs/utilities/data_classes.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aws_lambda_powertools/logging/correlation_paths.py b/aws_lambda_powertools/logging/correlation_paths.py index e20a3e690e4..004aa2a59a3 100644 --- a/aws_lambda_powertools/logging/correlation_paths.py +++ b/aws_lambda_powertools/logging/correlation_paths.py @@ -5,4 +5,4 @@ APPSYNC_RESOLVER = 'request.headers."x-amzn-trace-id"' APPLICATION_LOAD_BALANCER = 'headers."x-amzn-trace-id"' EVENT_BRIDGE = "id" -S3_LAMBDA_OBJECT = "xAmzRequestId" +S3_OBJECT_LAMBDA = "xAmzRequestId" diff --git a/docs/utilities/data_classes.md b/docs/utilities/data_classes.md index 88e14fd8ecb..51875b585f8 100644 --- a/docs/utilities/data_classes.md +++ b/docs/utilities/data_classes.md @@ -557,12 +557,12 @@ This example is based on the AWS Blog post [Introducing Amazon S3 Object Lambda ```python hl_lines="4 8 10" import requests from aws_lambda_powertools import Logger - from aws_lambda_powertools.logging.correlation_paths import S3_LAMBDA_OBJECT + from aws_lambda_powertools.logging.correlation_paths import S3_OBJECT_LAMBDA from aws_lambda_powertools.utilities.data_classes.s3_object_event import S3ObjectLambdaEvent logger = Logger() - @logger.inject_lambda_context(correlation_id_path=S3_LAMBDA_OBJECT, log_event=True) + @logger.inject_lambda_context(correlation_id_path=S3_OBJECT_LAMBDA, log_event=True) def lambda_handler(event, context): event = S3ObjectLambdaEvent(event) From a5d87c04061a0ed4eb6910a69ef04728d43aa1ec Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Sat, 20 Mar 2021 21:28:54 -0700 Subject: [PATCH 08/14] docs(data-classes): Correct docs --- .../utilities/data_classes/s3_object_event.py | 7 +++++-- docs/utilities/data_classes.md | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_classes/s3_object_event.py b/aws_lambda_powertools/utilities/data_classes/s3_object_event.py index 1cd7d9f9320..79635782936 100644 --- a/aws_lambda_powertools/utilities/data_classes/s3_object_event.py +++ b/aws_lambda_powertools/utilities/data_classes/s3_object_event.py @@ -9,6 +9,7 @@ class S3ObjectContext(DictWrapper): @property def input_s3_url(self) -> str: """A pre-signed URL that can be used to fetch the original object from Amazon S3. + The URL is signed using the original caller’s identity, and their permissions will apply when the URL is used. If there are signed headers in the URL, the Lambda function must include these in the call to Amazon S3, except for the Host.""" @@ -45,6 +46,7 @@ def supporting_access_point_arn(self) -> str: @property def payload(self) -> str: """Custom data that is applied to the S3 Object Lambda access point configuration. + S3 Object Lambda treats this as an opaque string, so it might need to be decoded before use.""" return self["payload"] @@ -138,7 +140,7 @@ def session_issuer(self) -> S3ObjectSessionIssuer: @property def attributes(self) -> S3ObjectSessionAttributes: - """User session attributes. """ + """Session attributes.""" return S3ObjectSessionAttributes(self["attributes"]) @@ -176,6 +178,7 @@ def get_type(self) -> str: @property def account_id(self) -> str: """The account that owns the entity that granted permissions for the request. + If the request was made with temporary security credentials, this is the account that owns the IAM user or role that was used to obtain credentials.""" return self["accountId"] @@ -222,7 +225,7 @@ def session_context(self) -> Optional[S3ObjectSessionContext]: class S3ObjectLambdaEvent(DictWrapper): - """S3 object event notification + """S3 object lambda event Documentation: ------------- diff --git a/docs/utilities/data_classes.md b/docs/utilities/data_classes.md index 51875b585f8..8922318d0b7 100644 --- a/docs/utilities/data_classes.md +++ b/docs/utilities/data_classes.md @@ -554,7 +554,8 @@ This example is based on the AWS Blog post [Introducing Amazon S3 Object Lambda === "app.py" - ```python hl_lines="4 8 10" + ```python hl_lines="5 9 11" + import boto3 import requests from aws_lambda_powertools import Logger from aws_lambda_powertools.logging.correlation_paths import S3_OBJECT_LAMBDA From 98744f86e7581ceb7699691715765dba48f0f769 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Sat, 20 Mar 2021 22:03:52 -0700 Subject: [PATCH 09/14] chore: Bump CI --- .../utilities/data_classes/s3_object_event.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_classes/s3_object_event.py b/aws_lambda_powertools/utilities/data_classes/s3_object_event.py index 79635782936..d7c8bc496c1 100644 --- a/aws_lambda_powertools/utilities/data_classes/s3_object_event.py +++ b/aws_lambda_powertools/utilities/data_classes/s3_object_event.py @@ -63,10 +63,11 @@ def url(self) -> str: @property def headers(self) -> Dict[str, str]: - """A map of string to strings containing the HTTP headers and their values from the - original call, excluding any authorization-related headers. If the same header appears - multiple times, their values are combined into a comma-delimited list. The case of the - original headers is retained in this map.""" + """A map of string to strings containing the HTTP headers and their values from the original call, + excluding any authorization-related headers. + + If the same header appears multiple times, their values are combined into a comma-delimited list. + The case of the original headers is retained in this map.""" return self["headers"] def get_header_value( @@ -135,7 +136,7 @@ class S3ObjectSessionContext(DictWrapper): @property def session_issuer(self) -> S3ObjectSessionIssuer: """If the request was made with temporary security credentials, an element that provides information - about how the credentials were obtained. """ + about how the credentials were obtained.""" return S3ObjectSessionIssuer(self["sessionIssuer"]) @property @@ -214,7 +215,7 @@ def arn(self) -> str: @property def session_context(self) -> Optional[S3ObjectSessionContext]: - """ If the request was made with temporary security credentials, + """If the request was made with temporary security credentials, this element provides information about the session that was created for those credentials.""" session_context = self.get("sessionContext") From 0fad7145555752c01a8b8c872718af5e5f93a2c6 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Mon, 22 Mar 2021 09:51:47 -0700 Subject: [PATCH 10/14] docs(data-classes): Add docstring examples Co-authored-by: Heitor Lessa --- .../utilities/data_classes/s3_object_event.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/aws_lambda_powertools/utilities/data_classes/s3_object_event.py b/aws_lambda_powertools/utilities/data_classes/s3_object_event.py index d7c8bc496c1..02e8ee06cb5 100644 --- a/aws_lambda_powertools/utilities/data_classes/s3_object_event.py +++ b/aws_lambda_powertools/utilities/data_classes/s3_object_event.py @@ -231,6 +231,33 @@ class S3ObjectLambdaEvent(DictWrapper): Documentation: ------------- - https://docs.aws.amazon.com/AmazonS3/latest/userguide/olap-writing-lambda.html + + + Example + ------- + **Fetch and transform original object from Amazon S3** + + import boto3 + import requests + from aws_lambda_powertools.utilities.data_classes.s3_object_event import S3ObjectLambdaEvent + + session = boto3.Session() + s3 = session.client("s3") + + def lambda_handler(event, context): + event = S3ObjectLambdaEvent(event) + + # Get object from S3 + response = requests.get(event.input_s3_url) + original_object = response.content.decode("utf-8") + + # Make changes to the object about to be returned + transformed_object = original_object.upper() + + # Write object back to S3 Object Lambda + s3.write_get_object_response( + Body=transformed_object, RequestRoute=event.request_route, RequestToken=event.request_token + ) """ @property From 6080f70fe3e421a371516d42ce513cacf4471a4f Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Mon, 22 Mar 2021 09:52:55 -0700 Subject: [PATCH 11/14] feat(data-classes): Add helper functions Co-authored-by: Heitor Lessa --- .../utilities/data_classes/s3_object_event.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/aws_lambda_powertools/utilities/data_classes/s3_object_event.py b/aws_lambda_powertools/utilities/data_classes/s3_object_event.py index 02e8ee06cb5..837201e8d01 100644 --- a/aws_lambda_powertools/utilities/data_classes/s3_object_event.py +++ b/aws_lambda_powertools/utilities/data_classes/s3_object_event.py @@ -284,6 +284,41 @@ def user_request(self) -> S3ObjectUserRequest: def user_identity(self) -> S3ObjectUserIdentity: """Details about the identity that made the call to S3 Object Lambda.""" return S3ObjectUserIdentity(self["userIdentity"]) + @property + def request_route(self) -> str: + """A routing token that is added to the S3 Object Lambda URL when the Lambda function + calls `WriteGetObjectResponse`.""" + return S3ObjectContext(self["getObjectContext"]).output_route + + @property + def request_token(self) -> str: + """An opaque token used by S3 Object Lambda to match the WriteGetObjectResponse call + with the original caller.""" + return S3ObjectContext(self["getObjectContext"]).output_token + + @property + def input_s3_url(self) -> str: + """A pre-signed URL that can be used to fetch the original object from Amazon S3. + + The URL is signed using the original caller’s identity, and their permissions + will apply when the URL is used. If there are signed headers in the URL, the + Lambda function must include these in the call to Amazon S3, except for the Host. + + Example + ------- + **Fetch original object from Amazon S3** + + import requests + from aws_lambda_powertools.utilities.data_classes.s3_object_event import S3ObjectLambdaEvent + + def lambda_handler(event, context): + event = S3ObjectLambdaEvent(event) + + response = requests.get(event.input_s3_url) + original_object = response.content.decode("utf-8") + ... + """ + return S3ObjectContext(self["getObjectContext"]).input_s3_url @property def protocol_version(self) -> str: From fbebd319425105152a36812e4e06d4abacd061e2 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Mon, 22 Mar 2021 10:02:26 -0700 Subject: [PATCH 12/14] docs(data-classes): Update docs and cleanup --- .../utilities/data_classes/s3_object_event.py | 10 +++++----- docs/utilities/data_classes.md | 19 +++++++++---------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_classes/s3_object_event.py b/aws_lambda_powertools/utilities/data_classes/s3_object_event.py index 837201e8d01..f653f7aca6e 100644 --- a/aws_lambda_powertools/utilities/data_classes/s3_object_event.py +++ b/aws_lambda_powertools/utilities/data_classes/s3_object_event.py @@ -232,7 +232,6 @@ class S3ObjectLambdaEvent(DictWrapper): ------------- - https://docs.aws.amazon.com/AmazonS3/latest/userguide/olap-writing-lambda.html - Example ------- **Fetch and transform original object from Amazon S3** @@ -243,7 +242,7 @@ class S3ObjectLambdaEvent(DictWrapper): session = boto3.Session() s3 = session.client("s3") - + def lambda_handler(event, context): event = S3ObjectLambdaEvent(event) @@ -284,17 +283,18 @@ def user_request(self) -> S3ObjectUserRequest: def user_identity(self) -> S3ObjectUserIdentity: """Details about the identity that made the call to S3 Object Lambda.""" return S3ObjectUserIdentity(self["userIdentity"]) + @property def request_route(self) -> str: """A routing token that is added to the S3 Object Lambda URL when the Lambda function calls `WriteGetObjectResponse`.""" - return S3ObjectContext(self["getObjectContext"]).output_route + return self.object_context.output_route @property def request_token(self) -> str: """An opaque token used by S3 Object Lambda to match the WriteGetObjectResponse call with the original caller.""" - return S3ObjectContext(self["getObjectContext"]).output_token + return self.object_context.output_token @property def input_s3_url(self) -> str: @@ -318,7 +318,7 @@ def lambda_handler(event, context): original_object = response.content.decode("utf-8") ... """ - return S3ObjectContext(self["getObjectContext"]).input_s3_url + return self.object_context.input_s3_url @property def protocol_version(self) -> str: diff --git a/docs/utilities/data_classes.md b/docs/utilities/data_classes.md index 8922318d0b7..c7c11b6b2f9 100644 --- a/docs/utilities/data_classes.md +++ b/docs/utilities/data_classes.md @@ -554,34 +554,33 @@ This example is based on the AWS Blog post [Introducing Amazon S3 Object Lambda === "app.py" - ```python hl_lines="5 9 11" + ```python hl_lines="4-5 10 12" import boto3 import requests + from aws_lambda_powertools import Logger from aws_lambda_powertools.logging.correlation_paths import S3_OBJECT_LAMBDA from aws_lambda_powertools.utilities.data_classes.s3_object_event import S3ObjectLambdaEvent logger = Logger() + session = boto3.Session() + s3 = session.client("s3") @logger.inject_lambda_context(correlation_id_path=S3_OBJECT_LAMBDA, log_event=True) def lambda_handler(event, context): event = S3ObjectLambdaEvent(event) - object_context = event.object_context - request_route = object_context.output_route - request_token = object_context.output_token - s3_url = object_context.input_s3_url - # Get object from S3 - response = requests.get(s3_url) + response = requests.get(event.input_s3_url) original_object = response.content.decode("utf-8") - # Transform object + # Make changes to the object about to be returned transformed_object = original_object.upper() # Write object back to S3 Object Lambda - s3 = boto3.client("s3") - s3.write_get_object_response(Body=transformed_object, RequestRoute=request_route, RequestToken=request_token) + s3.write_get_object_response( + Body=transformed_object, RequestRoute=event.request_route, RequestToken=event.request_token + ) return {"status_code": 200} ``` From 382ac60ab91b77baa4a80a537b5fa44116d6a871 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Mon, 22 Mar 2021 10:05:55 -0700 Subject: [PATCH 13/14] tests(data-classes): Add missing tests --- tests/functional/test_lambda_trigger_events.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/functional/test_lambda_trigger_events.py b/tests/functional/test_lambda_trigger_events.py index d3d3f4c9fe2..62bcb50762c 100644 --- a/tests/functional/test_lambda_trigger_events.py +++ b/tests/functional/test_lambda_trigger_events.py @@ -1037,6 +1037,9 @@ def test_s3_object_event_iam(): assert user_identity.user_name == event["userIdentity"]["userName"] assert user_identity.session_context is None assert event.protocol_version == event["protocolVersion"] + assert event.request_route == object_context.output_route + assert event.request_token == object_context.output_token + assert event.input_s3_url == object_context.input_s3_url def test_s3_object_event_temp_credentials(): From 47cfe0c50b1582fe11f1c7b7f40a0181577621a0 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Mon, 22 Mar 2021 10:18:07 -0700 Subject: [PATCH 14/14] chore: bump ci