From f68bafe7ec1b952fcd5cca7a285b08343cf4e60b Mon Sep 17 00:00:00 2001 From: Mike Weeks Date: Thu, 24 Oct 2024 13:32:31 -0400 Subject: [PATCH 1/7] feat(event_source): Extend CodePipeline Artifact Capabilities --- .../data_classes/code_pipeline_job_event.py | 78 +++++++++++++++-- docs/utilities/data_classes.md | 7 +- tests/events/codePipelineEventData.json | 4 + .../_boto3/test_code_pipeline_job_event.py | 85 +++++++++++++++++++ 4 files changed, 165 insertions(+), 9 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py b/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py index 8e5fa9ebcb4..684a34956d6 100644 --- a/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py +++ b/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py @@ -236,7 +236,25 @@ def find_input_artifact(self, artifact_name: str) -> CodePipelineArtifact | None return artifact return None - def get_artifact(self, artifact_name: str, filename: str) -> str | None: + def find_output_artifact(self, artifact_name: str) -> CodePipelineArtifact | None: + """Find an output artifact by artifact name + + Parameters + ---------- + artifact_name : str + The name of the output artifact to look for + + Returns + ------- + CodePipelineArtifact, None + Matching CodePipelineArtifact if found + """ + for artifact in self.data.output_artifacts: + if artifact.name == artifact_name: + return artifact + return None + + def get_artifact(self, artifact_name: str, filename: str | None = None) -> str | None: """Get a file within an artifact zip on s3 Parameters @@ -245,6 +263,7 @@ def get_artifact(self, artifact_name: str, filename: str) -> str | None: Name of the S3 artifact to download filename : str The file name within the artifact zip to extract as a string + If None, this will return the raw object body. Returns ------- @@ -255,10 +274,53 @@ def get_artifact(self, artifact_name: str, filename: str) -> str | None: if artifact is None: return None - with tempfile.NamedTemporaryFile() as tmp_file: - s3 = self.setup_s3_client() - bucket = artifact.location.s3_location.bucket_name - key = artifact.location.s3_location.key - s3.download_file(bucket, key, tmp_file.name) - with zipfile.ZipFile(tmp_file.name, "r") as zip_file: - return zip_file.read(filename).decode("UTF-8") + s3 = self.setup_s3_client() + bucket = artifact.location.s3_location.bucket_name + key = artifact.location.s3_location.key + + if filename: + with tempfile.NamedTemporaryFile() as tmp_file: + s3.download_file(bucket, key, tmp_file.name) + with zipfile.ZipFile(tmp_file.name, "r") as zip_file: + return zip_file.read(filename).decode("UTF-8") + + return s3.get_object(Bucket=bucket, Key=key)["Body"].read() + + def put_artifact(self, artifact_name: str, body: Any, content_type: str) -> None: + """Writes an object to an s3 output artifact. + + Parameters + ---------- + artifact_name : str + Name of the S3 artifact to upload + body: Any + The data to be written. Binary files should use io.BytesIO. + content_type: str + The content type of the data. + + Returns + ------- + None + """ + artifact = self.find_output_artifact(artifact_name) + if artifact is None: + raise ValueError(f"Artifact not found: {artifact_name}.") + + s3 = self.setup_s3_client() + bucket = artifact.location.s3_location.bucket_name + key = artifact.location.s3_location.key + encryption_key_id = self.data.encryption_key.get_id + encryption_key_type = self.data.encryption_key.get_type + + if encryption_key_type == "KMS": + encryption_key_type = "aws:kms" + + s3.put_object( + Bucket=bucket, + Key=key, + ContentType=content_type, + Body=body, + ServerSideEncryption=encryption_key_type, + SSEKMSKeyId=encryption_key_id, + BucketKeyEnabled=True, + ) diff --git a/docs/utilities/data_classes.md b/docs/utilities/data_classes.md index 01fe21e20b8..a207a7b787f 100644 --- a/docs/utilities/data_classes.md +++ b/docs/utilities/data_classes.md @@ -679,7 +679,12 @@ Data classes and utility functions to help create continuous delivery pipelines else: template = event.get_artifact(artifact_name, template_file) # Kick off a stack update or create - start_update_or_create(job_id, stack, template) + result = start_update_or_create(job_id, stack, template) + event.put_artifact( + artifact_name="json-artifact", + body=json.dumps(result), + content_type="application/json" + ) except Exception as e: # If any other exceptions which we didn't expect are raised # then fail the job and log the exception message. diff --git a/tests/events/codePipelineEventData.json b/tests/events/codePipelineEventData.json index 7552f19ca93..3635312c38b 100644 --- a/tests/events/codePipelineEventData.json +++ b/tests/events/codePipelineEventData.json @@ -40,6 +40,10 @@ "secretAccessKey": "6CGtmAa3lzWtV7a...", "sessionToken": "IQoJb3JpZ2luX2VjEA...", "expirationTime": 1575493418000 + }, + "encryptionKey": { + "id": "someKey", + "type": "KMS" } } } diff --git a/tests/unit/data_classes/_boto3/test_code_pipeline_job_event.py b/tests/unit/data_classes/_boto3/test_code_pipeline_job_event.py index 75e68b44396..8a5984dc810 100644 --- a/tests/unit/data_classes/_boto3/test_code_pipeline_job_event.py +++ b/tests/unit/data_classes/_boto3/test_code_pipeline_job_event.py @@ -1,7 +1,9 @@ import json import zipfile +from io import StringIO import pytest +from botocore.response import StreamingBody from pytest_mock import MockerFixture from aws_lambda_powertools.utilities.data_classes import CodePipelineJobEvent @@ -184,3 +186,86 @@ def download_file(bucket: str, key: str, tmp_name: str): }, ) assert artifact_str == file_contents + + +def test_raw_code_pipeline_get_artifact(mocker: MockerFixture): + raw_content = json.dumps({"steve": "french"}) + + class MockClient: + @staticmethod + def get_object(Bucket: str, Key: str): + assert Bucket == "us-west-2-123456789012-my-pipeline" + assert Key == "my-pipeline/test-api-2/TdOSFRV" + return {"Body": StreamingBody(StringIO(str(raw_content)), len(str(raw_content)))} + + s3 = mocker.patch("boto3.client") + s3.return_value = MockClient() + + event = CodePipelineJobEvent(load_event("codePipelineEventData.json")) + + artifact_str = event.get_artifact(artifact_name="my-pipeline-SourceArtifact") + + s3.assert_called_once_with( + "s3", + **{ + "aws_access_key_id": event.data.artifact_credentials.access_key_id, + "aws_secret_access_key": event.data.artifact_credentials.secret_access_key, + "aws_session_token": event.data.artifact_credentials.session_token, + }, + ) + assert artifact_str == raw_content + + +def test_code_pipeline_put_artifact(mocker: MockerFixture): + + raw_content = json.dumps({"steve": "french"}) + artifact_content_type = "application/json" + event = CodePipelineJobEvent(load_event("codePipelineEventData.json")) + artifact_name = event.data.output_artifacts[0].name + + class MockClient: + @staticmethod + def put_object( + Bucket: str, + Key: str, + ContentType: str, + Body: str, + ServerSideEncryption: str, + SSEKMSKeyId: str, + BucketKeyEnabled: bool, + ): + output_artifact = event.find_output_artifact(artifact_name) + assert Bucket == output_artifact.location.s3_location.bucket_name + assert Key == output_artifact.location.s3_location.key + assert ContentType == artifact_content_type + assert Body == raw_content + assert ServerSideEncryption == "aws:kms" + assert SSEKMSKeyId == event.data.encryption_key.get_id + assert BucketKeyEnabled is True + + s3 = mocker.patch("boto3.client") + s3.return_value = MockClient() + + event.put_artifact( + artifact_name=artifact_name, + body=raw_content, + content_type=artifact_content_type, + ) + + s3.assert_called_once_with( + "s3", + **{ + "aws_access_key_id": event.data.artifact_credentials.access_key_id, + "aws_secret_access_key": event.data.artifact_credentials.secret_access_key, + "aws_session_token": event.data.artifact_credentials.session_token, + }, + ) + + +def test_code_pipeline_put_output_artifact_not_found(): + raw_event = load_event("codePipelineEventData.json") + parsed_event = CodePipelineJobEvent(raw_event) + + assert parsed_event.find_output_artifact("not-found") is None + with pytest.raises(ValueError): + parsed_event.put_artifact(artifact_name="not-found", body="", content_type="text/plain") From 5a1a038719ab7056ab914d89f61e1a763b57c062 Mon Sep 17 00:00:00 2001 From: Mike Weeks Date: Thu, 14 Nov 2024 09:40:51 -0500 Subject: [PATCH 2/7] Fix mypy warnings --- .../data_classes/code_pipeline_job_event.py | 41 ++++++++++++------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py b/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py index 684a34956d6..841bae8ce5c 100644 --- a/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py +++ b/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py @@ -309,18 +309,29 @@ def put_artifact(self, artifact_name: str, body: Any, content_type: str) -> None s3 = self.setup_s3_client() bucket = artifact.location.s3_location.bucket_name key = artifact.location.s3_location.key - encryption_key_id = self.data.encryption_key.get_id - encryption_key_type = self.data.encryption_key.get_type - - if encryption_key_type == "KMS": - encryption_key_type = "aws:kms" - - s3.put_object( - Bucket=bucket, - Key=key, - ContentType=content_type, - Body=body, - ServerSideEncryption=encryption_key_type, - SSEKMSKeyId=encryption_key_id, - BucketKeyEnabled=True, - ) + + if self.data.encryption_key: + + encryption_key_id = self.data.encryption_key.get_id + encryption_key_type = self.data.encryption_key.get_type + if encryption_key_type == "KMS": + encryption_key_type = "aws:kms" + + s3.put_object( + Bucket=bucket, + Key=key, + ContentType=content_type, + Body=body, + ServerSideEncryption=encryption_key_type, + SSEKMSKeyId=encryption_key_id, + BucketKeyEnabled=True, + ) + + else: + s3.put_object( + Bucket=bucket, + Key=key, + ContentType=content_type, + Body=body, + BucketKeyEnabled=True, + ) From 32611143e62cf9f76857b1b98dda9082a9c4acae Mon Sep 17 00:00:00 2001 From: Mike Weeks Date: Thu, 14 Nov 2024 09:54:11 -0500 Subject: [PATCH 3/7] Update docs --- docs/utilities/data_classes.md | 7 ++++--- poetry.toml | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 poetry.toml diff --git a/docs/utilities/data_classes.md b/docs/utilities/data_classes.md index a207a7b787f..4cd738fad70 100644 --- a/docs/utilities/data_classes.md +++ b/docs/utilities/data_classes.md @@ -680,10 +680,11 @@ Data classes and utility functions to help create continuous delivery pipelines template = event.get_artifact(artifact_name, template_file) # Kick off a stack update or create result = start_update_or_create(job_id, stack, template) + artifact: io.BytesIO = zip_data(result) event.put_artifact( - artifact_name="json-artifact", - body=json.dumps(result), - content_type="application/json" + artifact_name=event.data.output_artifacts[0].name, + body=artifact, + content_type="application/zip" ) except Exception as e: # If any other exceptions which we didn't expect are raised diff --git a/poetry.toml b/poetry.toml new file mode 100644 index 00000000000..ab1033bd372 --- /dev/null +++ b/poetry.toml @@ -0,0 +1,2 @@ +[virtualenvs] +in-project = true From 48d7163ba2d02e9f792c9d6709ce5d5225ccc134 Mon Sep 17 00:00:00 2001 From: Mike Weeks Date: Thu, 14 Nov 2024 13:15:21 -0500 Subject: [PATCH 4/7] Add Unencrypted Artifact Test --- .../_boto3/test_code_pipeline_job_event.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/unit/data_classes/_boto3/test_code_pipeline_job_event.py b/tests/unit/data_classes/_boto3/test_code_pipeline_job_event.py index 8a5984dc810..0306096a8c5 100644 --- a/tests/unit/data_classes/_boto3/test_code_pipeline_job_event.py +++ b/tests/unit/data_classes/_boto3/test_code_pipeline_job_event.py @@ -262,6 +262,51 @@ def put_object( ) +def test_code_pipeline_put_unencrypted_artifact(mocker: MockerFixture): + + raw_content = json.dumps({"steve": "french"}) + artifact_content_type = "application/json" + event_without_artifact_encryption = load_event("codePipelineEventData.json") + event_without_artifact_encryption["CodePipeline.job"]["data"]["encryptionKey"] = None + event = CodePipelineJobEvent(event_without_artifact_encryption) + assert event.data.encryption_key is None + artifact_name = event.data.output_artifacts[0].name + + class MockClient: + @staticmethod + def put_object( + Bucket: str, + Key: str, + ContentType: str, + Body: str, + BucketKeyEnabled: bool, + ): + output_artifact = event.find_output_artifact(artifact_name) + assert Bucket == output_artifact.location.s3_location.bucket_name + assert Key == output_artifact.location.s3_location.key + assert ContentType == artifact_content_type + assert Body == raw_content + assert BucketKeyEnabled is True + + s3 = mocker.patch("boto3.client") + s3.return_value = MockClient() + + event.put_artifact( + artifact_name=artifact_name, + body=raw_content, + content_type=artifact_content_type, + ) + + s3.assert_called_once_with( + "s3", + **{ + "aws_access_key_id": event.data.artifact_credentials.access_key_id, + "aws_secret_access_key": event.data.artifact_credentials.secret_access_key, + "aws_session_token": event.data.artifact_credentials.session_token, + }, + ) + + def test_code_pipeline_put_output_artifact_not_found(): raw_event = load_event("codePipelineEventData.json") parsed_event = CodePipelineJobEvent(raw_event) From 5919ffc2daee8c8ed35cbb8275d87bfd3a532b41 Mon Sep 17 00:00:00 2001 From: Ana Falcao Date: Mon, 18 Nov 2024 09:10:39 -0300 Subject: [PATCH 5/7] deleting poetry.toml file --- poetry.toml | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 poetry.toml diff --git a/poetry.toml b/poetry.toml deleted file mode 100644 index ab1033bd372..00000000000 --- a/poetry.toml +++ /dev/null @@ -1,2 +0,0 @@ -[virtualenvs] -in-project = true From 36f71c0d24c60f0e57e1a0b399b5ef545c733c63 Mon Sep 17 00:00:00 2001 From: Ana Falcao Date: Mon, 18 Nov 2024 11:35:10 -0300 Subject: [PATCH 6/7] add a comment to explain none type for boto3 --- .../utilities/data_classes/code_pipeline_job_event.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py b/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py index 841bae8ce5c..a03e866c513 100644 --- a/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py +++ b/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py @@ -310,6 +310,9 @@ def put_artifact(self, artifact_name: str, body: Any, content_type: str) -> None bucket = artifact.location.s3_location.bucket_name key = artifact.location.s3_location.key + # boto3 doesn't treat None as equivalent to omitting the parameter when using ServerSideEncryption and SSEKMSKeyId + # So we are using if/else instead. + if self.data.encryption_key: encryption_key_id = self.data.encryption_key.get_id From a7831d353d8db6a479ffbb32d048bdb889008c0a Mon Sep 17 00:00:00 2001 From: Ana Falcao Date: Mon, 18 Nov 2024 11:41:20 -0300 Subject: [PATCH 7/7] fix the comment to explain none type for boto3 --- .../utilities/data_classes/code_pipeline_job_event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py b/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py index a03e866c513..a52e5fbc7a2 100644 --- a/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py +++ b/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py @@ -310,7 +310,7 @@ def put_artifact(self, artifact_name: str, body: Any, content_type: str) -> None bucket = artifact.location.s3_location.bucket_name key = artifact.location.s3_location.key - # boto3 doesn't treat None as equivalent to omitting the parameter when using ServerSideEncryption and SSEKMSKeyId + # boto3 doesn't support None to omit the parameter when using ServerSideEncryption and SSEKMSKeyId # So we are using if/else instead. if self.data.encryption_key: