From a60bbb87a91a27a307177ddc582290a374e4b958 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 19 May 2025 15:36:21 -0700 Subject: [PATCH 1/6] sync --- .../aws_dbesdk_dynamodb/encrypted/resource.py | 311 ++++++++++++++++++ .../python/test/integ/encrypted/README.md | 10 + .../test/integ/encrypted/test_resource.py | 154 +++++++++ .../test/unit/encrypted/test_resource.py | 13 + .../batch_read_write_example.py | 183 +++++++++++ ...ypted_tables_collection_manager_example.py | 175 ++++++++++ .../test/encrypted_resource/__init__.py | 3 + .../test_batch_read_write_example.py | 15 + ...ypted_tables_collection_manager_example.py | 17 + 9 files changed, 881 insertions(+) create mode 100644 DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/resource.py create mode 100644 DynamoDbEncryption/runtimes/python/test/integ/encrypted/README.md create mode 100644 DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_resource.py create mode 100644 DynamoDbEncryption/runtimes/python/test/unit/encrypted/test_resource.py create mode 100644 Examples/runtimes/python/DynamoDBEncryption/src/encrypted_resource/batch_read_write_example.py create mode 100644 Examples/runtimes/python/DynamoDBEncryption/src/encrypted_resource/encrypted_tables_collection_manager_example.py create mode 100644 Examples/runtimes/python/DynamoDBEncryption/test/encrypted_resource/__init__.py create mode 100644 Examples/runtimes/python/DynamoDBEncryption/test/encrypted_resource/test_batch_read_write_example.py create mode 100644 Examples/runtimes/python/DynamoDBEncryption/test/encrypted_resource/test_encrypted_tables_collection_manager_example.py diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/resource.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/resource.py new file mode 100644 index 000000000..65cfa53f3 --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/resource.py @@ -0,0 +1,311 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""High-level helper classes to provide encrypting wrappers for boto3 DynamoDB resources.""" +from collections.abc import Callable, Generator +from copy import deepcopy +from typing import Any + +from boto3.resources.base import ServiceResource +from boto3.resources.collection import CollectionManager + +from aws_dbesdk_dynamodb.encrypted.boto3_interface import EncryptedBotoInterface +from aws_dbesdk_dynamodb.encrypted.table import EncryptedTable +from aws_dbesdk_dynamodb.internal.client_to_resource import ClientShapeToResourceShapeConverter +from aws_dbesdk_dynamodb.internal.resource_to_client import ResourceShapeToClientShapeConverter +from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb.models import ( + DynamoDbTablesEncryptionConfig, +) +from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb_transforms.client import ( + DynamoDbEncryptionTransforms, +) +from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb_transforms.models import ( + BatchGetItemInputTransformInput, + BatchGetItemOutputTransformInput, + BatchWriteItemInputTransformInput, + BatchWriteItemOutputTransformInput, +) + + +class EncryptedTablesCollectionManager(EncryptedBotoInterface): + """ + Collection manager that yields EncryptedTable objects. + + The API matches boto3's tables collection manager interface: + + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/service-resource/tables.html + + All operations on this class will yield ``EncryptedTable`` objects. + """ + + def __init__( + self, + *, + collection: CollectionManager, + encryption_config: DynamoDbTablesEncryptionConfig, + ): + """ + Create an ``EncryptedTablesCollectionManager`` object. + + Args: + collection (CollectionManager): Pre-configured boto3 DynamoDB table collection manager + encryption_config (DynamoDbTablesEncryptionConfig): Initialized DynamoDbTablesEncryptionConfig + + """ + self._collection = collection + self._encryption_config = encryption_config + + def all(self) -> Generator[EncryptedTable, None, None]: + """ + Create an iterable of all EncryptedTable resources in the collection. + + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/service-resource/tables.html#DynamoDB.ServiceResource.all + + Returns: + Generator[EncryptedTable, None, None]: An iterable of EncryptedTable objects + + """ + yield from self._transform_table(self._collection.all) + + def filter(self, **kwargs) -> Generator[EncryptedTable, None, None]: + """ + Create an iterable of all EncryptedTable resources in the collection filtered by kwargs passed to method. + + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/service-resource/tables.html#filter + + Returns: + Generator[EncryptedTable, None, None]: An iterable of EncryptedTable objects + + """ + yield from self._transform_table(self._collection.filter, **kwargs) + + def limit(self, **kwargs) -> Generator[EncryptedTable, None, None]: + """ + Create an iterable of all EncryptedTable resources in the collection filtered by kwargs passed to method. + + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/service-resource/tables.html#limit + + Returns: + Generator[EncryptedTable, None, None]: An iterable of EncryptedTable objects + + """ + yield from self._transform_table(self._collection.limit, **kwargs) + + def page_size(self, **kwargs) -> Generator[EncryptedTable, None, None]: + """ + Create an iterable of all EncryptedTable resources in the collection. + + This limits the number of items returned by each service call by the specified amount. + + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/service-resource/tables.html#page_size + + Returns: + Generator[EncryptedTable, None, None]: An iterable of EncryptedTable objects + + """ + yield from self._transform_table(self._collection.page_size, **kwargs) + + def _transform_table( + self, + method: Callable, + **kwargs, + ) -> Generator[EncryptedTable, None, None]: + for table in method(**kwargs): + yield EncryptedTable(table=table, encryption_config=self._encryption_config) + + @property + def _boto_client_attr_name(self) -> str: + """ + Name of the attribute containing the underlying boto3 client. + + Returns: + str: '_collection' + + """ + return "_collection" + + +class EncryptedResource(EncryptedBotoInterface): + """ + Wrapper for a boto3 DynamoDB resource. + + This class implements the complete boto3 DynamoDB resource API, allowing it to serve as a + drop-in replacement that transparently handles encryption and decryption of items. + + The API matches the standard boto3 DynamoDB resource interface: + + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/service-resource/index.html + + This class will encrypt/decrypt items for the following operations: + + * ``batch_get_item`` + * ``batch_write_item`` + + Calling ``Table()`` will return an ``EncryptedTable`` object. + + Any other operations on this class will defer to the underlying boto3 DynamoDB resource's implementation + and will not be encrypted/decrypted. + + """ + + def __init__( + self, + *, + resource: ServiceResource, + encryption_config: DynamoDbTablesEncryptionConfig, + ): + """ + Create an ``EncryptedResource`` object. + + Args: + resource (ServiceResource): Initialized boto3 DynamoDB resource + encryption_config (DynamoDbTablesEncryptionConfig): Initialized DynamoDbTablesEncryptionConfig + + """ + self._resource = resource + self._encryption_config = encryption_config + self._transformer = DynamoDbEncryptionTransforms(config=encryption_config) + self._client_shape_to_resource_shape_converter = ClientShapeToResourceShapeConverter() + self._resource_shape_to_client_shape_converter = ResourceShapeToClientShapeConverter() + self.tables = EncryptedTablesCollectionManager( + collection=self._resource.tables, encryption_config=self._encryption_config + ) + + def Table(self, name): + """ + Create an ``EncryptedTable`` resource. + + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/service-resource/Table.html + + Args: + name (str): The EncryptedTable's name identifier. This must be set. + + Returns: + EncryptedTable: An ``EncryptedTable`` resource + + """ + return EncryptedTable(table=self._resource.Table(name), encryption_config=self._encryption_config) + + def batch_get_item(self, **kwargs): + """ + Get multiple items from one or more tables. Decrypts any returned items. + + The input and output syntaxes match those for the boto3 DynamoDB resource ``batch_get_item`` API: + + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/service-resource/batch_get_item.html + + Args: + **kwargs: Keyword arguments to pass to the operation. These match the boto3 resource ``batch_get_item`` + request syntax. + + Returns: + dict: The response from DynamoDB. This matches the boto3 resource ``batch_get_item`` response syntax. + The ``"Responses"`` field will be decrypted locally after being read from DynamoDB. + + """ + return self._resource_operation_logic( + operation_input=kwargs, + input_resource_to_client_shape_transform_method=self._resource_shape_to_client_shape_converter.batch_get_item_request, + input_client_to_resource_shape_transform_method=self._client_shape_to_resource_shape_converter.batch_get_item_request, + input_encryption_transform_method=self._transformer.batch_get_item_input_transform, + input_encryption_transform_shape=BatchGetItemInputTransformInput, + output_encryption_transform_method=self._transformer.batch_get_item_output_transform, + output_encryption_transform_shape=BatchGetItemOutputTransformInput, + output_resource_to_client_shape_transform_method=self._resource_shape_to_client_shape_converter.batch_get_item_response, + output_client_to_resource_shape_transform_method=self._client_shape_to_resource_shape_converter.batch_get_item_response, + resource_method=self._resource.batch_get_item, + ) + + def batch_write_item(self, **kwargs): + """ + Put or delete multiple items in one or more tables. + + For put operations, encrypts items before writing. + + The input and output syntaxes match those for the boto3 DynamoDB resource ``batch_write_item`` API: + + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/service-resource/batch_write_item.html + + Args: + **kwargs: Keyword arguments to pass to the operation. These match the boto3 resource + ``batch_write_item`` request syntax. Any ``"PutRequest"`` values in the ``"RequestItems"`` + argument will be encrypted locally before being written to DynamoDB. + + Returns: + dict: The response from DynamoDB. This matches the boto3 resource ``batch_write_item`` response syntax. + + """ + return self._resource_operation_logic( + operation_input=kwargs, + input_resource_to_client_shape_transform_method=self._resource_shape_to_client_shape_converter.batch_write_item_request, + input_client_to_resource_shape_transform_method=self._client_shape_to_resource_shape_converter.batch_write_item_request, + input_encryption_transform_method=self._transformer.batch_write_item_input_transform, + input_encryption_transform_shape=BatchWriteItemInputTransformInput, + output_encryption_transform_method=self._transformer.batch_write_item_output_transform, + output_encryption_transform_shape=BatchWriteItemOutputTransformInput, + output_resource_to_client_shape_transform_method=self._resource_shape_to_client_shape_converter.batch_write_item_response, + output_client_to_resource_shape_transform_method=self._client_shape_to_resource_shape_converter.batch_write_item_response, + resource_method=self._resource.batch_write_item, + ) + + def _resource_operation_logic( + self, + *, + operation_input: dict[str, Any], + input_resource_to_client_shape_transform_method: Callable, + input_client_to_resource_shape_transform_method: Callable, + input_encryption_transform_method: Callable, + input_encryption_transform_shape: Any, + output_encryption_transform_method: Callable, + output_encryption_transform_shape: Any, + output_resource_to_client_shape_transform_method: Callable, + output_client_to_resource_shape_transform_method: Callable, + resource_method: Callable, + ): + operation_input = deepcopy(operation_input) + # Table inputs are formatted as Python dictionary JSON, but encryption transformers expect DynamoDB JSON. + # `input_resource_to_client_shape_transform_method` formats the supplied Python dictionary as DynamoDB JSON. + input_transform_input = input_resource_to_client_shape_transform_method(operation_input) + + # Apply encryption transformation to the user-supplied input + input_transform_output = input_encryption_transform_method( + input_encryption_transform_shape(sdk_input=input_transform_input) + ).transformed_input + + # The encryption transformation result is formatted in DynamoDB JSON, + # but the underlying boto3 table expects Python dictionary JSON. + # `input_client_to_resource_shape_transform_method` formats the transformation as Python dictionary JSON. + sdk_input = input_client_to_resource_shape_transform_method(input_transform_output) + + # Call boto3 Table method with Python-dictionary-JSON-formatted, encryption-transformed input, + # and receive Python-dictionary-JSON-formatted boto3 output. + sdk_output = resource_method(**sdk_input) + + # Format Python dictionary JSON-formatted SDK output as DynamoDB JSON for encryption transformer + output_transform_input = output_resource_to_client_shape_transform_method(sdk_output) + + # Apply encryption transformer to boto3 output + output_transform_output = output_encryption_transform_method( + output_encryption_transform_shape( + original_input=input_transform_input, + sdk_output=output_transform_input, + ) + ).transformed_output + + # Format DynamoDB JSON-formatted encryption transformation result as Python dictionary JSON + dbesdk_response = output_client_to_resource_shape_transform_method(output_transform_output) + # Copy any missing fields from the SDK output to the response + # (e.g. `ConsumedCapacity`) + dbesdk_response = self._copy_sdk_response_to_dbesdk_response(sdk_output, dbesdk_response) + + return dbesdk_response + + @property + def _boto_client_attr_name(self) -> str: + """ + Name of the attribute containing the underlying boto3 client. + + Returns: + str: '_resource' + + """ + return "_resource" diff --git a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/README.md b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/README.md new file mode 100644 index 000000000..f6a9abf10 --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/README.md @@ -0,0 +1,10 @@ +Integration tests for encrypted interfaces. + +These integration tests verify that encrypted boto3 interfaces behave as drop-in replacements for plaintext boto3 interfaces. + +Each test runs with both a plaintext client and an encrypted client, using the same request parameters and expecting the same response. + +This validates that encrypted clients expect the same input shapes as plaintext clients +and encrypted clients return the same output shapes as plaintext clients. + +This guarantees that users can substitute encrypted interfaces without modifying their application logic. diff --git a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_resource.py b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_resource.py new file mode 100644 index 000000000..7e48dd2e2 --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_resource.py @@ -0,0 +1,154 @@ +import boto3 +import pytest + +from aws_dbesdk_dynamodb.encrypted.resource import EncryptedResource, EncryptedTablesCollectionManager +from aws_dbesdk_dynamodb.encrypted.table import EncryptedTable + +from ...constants import INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, INTEG_TEST_DEFAULT_TABLE_CONFIGS +from ...items import complex_item_dict, complex_key_dict, simple_item_dict, simple_key_dict +from ...requests import ( + basic_batch_get_item_request_dict, + basic_batch_write_item_delete_request_dict, + basic_batch_write_item_put_request_dict, +) + + +@pytest.fixture(params=[True, False], ids=["encrypted", "plaintext"]) +def encrypted(request): + return request.param + + +def plaintext_resource(): + return boto3.resource("dynamodb") + + +def encrypted_resource(): + return EncryptedResource( + resource=plaintext_resource(), + encryption_config=INTEG_TEST_DEFAULT_TABLE_CONFIGS, + ) + + +@pytest.fixture +def resource(encrypted): + if encrypted: + return encrypted_resource() + else: + return plaintext_resource() + + +@pytest.fixture +def tables(resource): + return resource.tables + + +def test_GIVEN_items_WHEN_batch_write_and_get_THEN_round_trip_passes( + resource, +): + batch_write_item_put_request = basic_batch_write_item_put_request_dict([simple_item_dict, complex_item_dict]) + batch_write_response = resource.batch_write_item(**batch_write_item_put_request) + assert batch_write_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + batch_get_item_request = basic_batch_get_item_request_dict([simple_key_dict, complex_key_dict]) + batch_get_response = resource.batch_get_item(**batch_get_item_request) + assert batch_get_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + responses = batch_get_response["Responses"][INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME] + assert len(responses) == 2 + for response in responses: + assert response in [simple_item_dict, complex_item_dict] + + batch_write_item_delete_request = basic_batch_write_item_delete_request_dict([simple_key_dict, complex_key_dict]) + batch_write_response = resource.batch_write_item(**batch_write_item_delete_request) + assert batch_write_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + batch_get_response = resource.batch_get_item(**batch_get_item_request) + assert batch_get_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert len(batch_get_response["Responses"][INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME]) == 0 + + +def test_GIVEN_encrypted_resource_WHEN_Table_THEN_returns_encrypted_table_with_correct_arguments(): + # Given: Encrypted resource + resource = encrypted_resource() + # When: Table + table = resource.Table(INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME) + # Then: Returns encrypted table + assert isinstance(table, EncryptedTable) + assert table.name == INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME + assert table._encryption_config == resource._encryption_config + + +def test_GIVEN_encrypted_resource_WHEN_tables_THEN_returns_encrypted_tables_collection_manager(): + # Given: Encrypted resource + resource = encrypted_resource() + # When: tables + tables = resource.tables + # Then: Returns encrypted tables collection manager + assert isinstance(tables, EncryptedTablesCollectionManager) + # Given: Encrypted tables collection manager + tables = resource.tables + # When: all + iter = tables.all() + tables_list = [] + for table in iter: + tables_list.append(table) + assert len(tables_list) > 0 + for table in tables_list: + assert isinstance(table, EncryptedTable) + assert table._encryption_config == resource._encryption_config + + +def test_GIVEN_tables_WHEN_all_THEN_returns_tables( + tables, +): + # Given: Tables + # When: all + tables_list = [] + for table in tables.all(): + tables_list.append(table) + # Then: Returns tables + assert len(tables_list) > 0 + table_names = [table.name for table in tables_list] + # "All tables" includes the integ test table + assert INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME in table_names + + +def test_GIVEN_tables_WHEN_filter_THEN_returns_tables( + tables, +): + # Given: Tables + # When: filter + tables_list = [] + for table in tables.filter( + ExclusiveStartTableName=INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, + Limit=1, + ): + tables_list.append(table) + # Then: Returns tables + assert len(tables_list) > 0 + table_names = [table.name for table in tables_list] + # The filter request started from the integ test table, not inclusive; it should not be in the list + assert INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME not in table_names + + +def test_GIVEN_tables_WHEN_limit_THEN_returns_tables( + tables, +): + # Given: Tables + # When: limit + tables_list = [] + for table in tables.limit(count=1): + tables_list.append(table) + # Then: Returns tables + assert len(tables_list) == 1 + + +def test_GIVEN_tables_WHEN_page_size_THEN_returns_tables( + tables, +): + # Given: Tables + # When: page_size + tables_list = [] + for table in tables.page_size(count=1): + tables_list.append(table) + # Then: Returns tables + assert len(tables_list) > 0 diff --git a/DynamoDbEncryption/runtimes/python/test/unit/encrypted/test_resource.py b/DynamoDbEncryption/runtimes/python/test/unit/encrypted/test_resource.py new file mode 100644 index 000000000..aa0e1f2ce --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/test/unit/encrypted/test_resource.py @@ -0,0 +1,13 @@ +from mock import MagicMock + +from aws_dbesdk_dynamodb.encrypted.resource import EncryptedResource, EncryptedTablesCollectionManager + + +def test_WHEN_boto3_client_attr_name_THEN_returns_expected_value(): + # Given: EncryptedResource + assert EncryptedResource(resource=MagicMock(), encryption_config=MagicMock())._boto_client_attr_name == "_resource" + # And: EncryptedTablesCollectionManager + assert ( + EncryptedTablesCollectionManager(collection=MagicMock(), encryption_config=MagicMock())._boto_client_attr_name + == "_collection" + ) diff --git a/Examples/runtimes/python/DynamoDBEncryption/src/encrypted_resource/batch_read_write_example.py b/Examples/runtimes/python/DynamoDBEncryption/src/encrypted_resource/batch_read_write_example.py new file mode 100644 index 000000000..5efb34336 --- /dev/null +++ b/Examples/runtimes/python/DynamoDBEncryption/src/encrypted_resource/batch_read_write_example.py @@ -0,0 +1,183 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +""" +Example for using an EncryptedResource to batch read and write items. + +Running this example requires access to the DDB Table whose name +is provided in the function arguments. +This table must be configured with the following +primary key configuration: +- Partition key is named "partition_key" with type (S) +- Sort key is named "sort_key" with type (N) +""" + +import boto3 +from aws_cryptographic_material_providers.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_material_providers.mpl.config import MaterialProvidersConfig +from aws_cryptographic_material_providers.mpl.models import ( + CreateAwsKmsMrkMultiKeyringInput, + DBEAlgorithmSuiteId, +) +from aws_cryptographic_material_providers.mpl.references import IKeyring +from aws_dbesdk_dynamodb.encrypted.resource import EncryptedResource +from aws_dbesdk_dynamodb.structures.dynamodb import ( + DynamoDbTableEncryptionConfig, + DynamoDbTablesEncryptionConfig, +) +from aws_dbesdk_dynamodb.structures.structured_encryption import ( + CryptoAction, +) + + +def encrypted_resource_batch_read_write_example( + kms_key_id: str, + dynamodb_table_name: str, +): + """Use an EncryptedResource to batch read and write items.""" + # 1. Create a Keyring. This Keyring will be responsible for protecting the data keys that protect your data. + # For this example, we will create a AWS KMS Keyring with the AWS KMS Key we want to use. + # We will use the `CreateMrkMultiKeyring` method to create this keyring, + # as it will correctly handle both single region and Multi-Region KMS Keys. + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders(config=MaterialProvidersConfig()) + kms_mrk_multi_keyring_input: CreateAwsKmsMrkMultiKeyringInput = CreateAwsKmsMrkMultiKeyringInput( + generator=kms_key_id, + ) + kms_mrk_multi_keyring: IKeyring = mat_prov.create_aws_kms_mrk_multi_keyring(input=kms_mrk_multi_keyring_input) + + # 2. Configure which attributes are encrypted and/or signed when writing new items. + # For each attribute that may exist on the items we plan to write to our DynamoDbTable, + # we must explicitly configure how they should be treated during item encryption: + # - ENCRYPT_AND_SIGN: The attribute is encrypted and included in the signature + # - SIGN_ONLY: The attribute not encrypted, but is still included in the signature + # - DO_NOTHING: The attribute is not encrypted and not included in the signature + attribute_actions_on_encrypt = { + "partition_key": CryptoAction.SIGN_ONLY, + "sort_key": CryptoAction.SIGN_ONLY, + "attribute1": CryptoAction.ENCRYPT_AND_SIGN, + "attribute2": CryptoAction.SIGN_ONLY, + ":attribute3": CryptoAction.DO_NOTHING, + } + + # 3. Configure which attributes we expect to be included in the signature + # when reading items. There are two options for configuring this: + # + # - (Recommended) Configure `allowedUnsignedAttributesPrefix`: + # When defining your DynamoDb schema and deciding on attribute names, + # choose a distinguishing prefix (such as ":") for all attributes that + # you do not want to include in the signature. + # This has two main benefits: + # - It is easier to reason about the security and authenticity of data within your item + # when all unauthenticated data is easily distinguishable by their attribute name. + # - If you need to add new unauthenticated attributes in the future, + # you can easily make the corresponding update to your `attributeActionsOnEncrypt` + # and immediately start writing to that new attribute, without + # any other configuration update needed. + # Once you configure this field, it is not safe to update it. + # + # - Configure `allowedUnsignedAttributes`: You may also explicitly list + # a set of attributes that should be considered unauthenticated when encountered + # on read. Be careful if you use this configuration. Do not remove an attribute + # name from this configuration, even if you are no longer writing with that attribute, + # as old items may still include this attribute, and our configuration needs to know + # to continue to exclude this attribute from the signature scope. + # If you add new attribute names to this field, you must first deploy the update to this + # field to all readers in your host fleet before deploying the update to start writing + # with that new attribute. + # + # For this example, we have designed our DynamoDb table such that any attribute name with + # the ":" prefix should be considered unauthenticated. + unsignAttrPrefix: str = ":" + + # 4. Create the DynamoDb Encryption configuration for the table we will be writing to. + table_configs = {} + table_config = DynamoDbTableEncryptionConfig( + logical_table_name=dynamodb_table_name, + partition_key_name="partition_key", + sort_key_name="sort_key", + attribute_actions_on_encrypt=attribute_actions_on_encrypt, + keyring=kms_mrk_multi_keyring, + allowed_unsigned_attribute_prefix=unsignAttrPrefix, + # Specifying an algorithm suite is not required, + # but is done here to demonstrate how to do so. + # We suggest using the + # `ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384_SYMSIG_HMAC_SHA384` suite, + # which includes AES-GCM with key derivation, signing, and key commitment. + # This is also the default algorithm suite if one is not specified in this config. + # For more information on supported algorithm suites, see: + # https://docs.aws.amazon.com/database-encryption-sdk/latest/devguide/supported-algorithms.html + algorithm_suite_id=DBEAlgorithmSuiteId.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384_SYMSIG_HMAC_SHA384, + ) + table_configs[dynamodb_table_name] = table_config + tables_config = DynamoDbTablesEncryptionConfig(table_encryption_configs=table_configs) + + # 5. Create the EncryptedResource + encrypted_resource = EncryptedResource( + resource=boto3.resource("dynamodb"), + encryption_config=tables_config, + ) + + # 6. Write a batch of items to the table. + # Before the items get sent to DynamoDb, they will be encrypted + # client-side, according to our configuration. + items = [ + { + "partition_key": "PythonEncryptedResourceBatchReadWriteExample1", + "sort_key": 0, + "attribute1": "encrypt and sign me!", + "attribute2": "sign me!", + ":attribute3": "ignore me!", + }, + { + "partition_key": "PythonEncryptedResourceBatchReadWriteExample2", + "sort_key": 0, + "attribute1": "encrypt and sign me!", + "attribute2": "sign me!", + ":attribute3": "ignore me!", + }, + ] + + batch_write_items_put_request = { + "RequestItems": { + dynamodb_table_name: [{"PutRequest": {"Item": item}} for item in items], + }, + } + + batch_write_items_put_response = encrypted_resource.batch_write_item(**batch_write_items_put_request) + + # Demonstrate that BatchWriteItem succeeded + assert batch_write_items_put_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # 7. Read the items back from the table. + # After the items are retrieved from DynamoDb, but before the EncryptedResource + # returns them to the caller, they will be decrypted client-side according to our configuration. + batch_get_items_request = { + "RequestItems": { + dynamodb_table_name: { + "Keys": [{"partition_key": item["partition_key"], "sort_key": item["sort_key"]} for item in items], + } + }, + } + + batch_get_items_response = encrypted_resource.batch_get_item(**batch_get_items_request) + + # Demonstrate that BatchGetItem succeeded with the expected result + assert batch_get_items_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + for item in batch_get_items_response["Responses"][dynamodb_table_name]: + assert item["attribute1"] == "encrypt and sign me!" + assert item["attribute2"] == "sign me!" + assert item[":attribute3"] == "ignore me!" + + # 8. Delete the items from the table. + batch_write_items_delete_request = { + "RequestItems": { + dynamodb_table_name: [ + {"DeleteRequest": {"Key": {"partition_key": item["partition_key"], "sort_key": item["sort_key"]}}} + for item in items + ], + }, + } + + batch_write_items_delete_response = encrypted_resource.batch_write_item(**batch_write_items_delete_request) + + # Demonstrate that BatchWriteItem succeeded + assert batch_write_items_delete_response["ResponseMetadata"]["HTTPStatusCode"] == 200 diff --git a/Examples/runtimes/python/DynamoDBEncryption/src/encrypted_resource/encrypted_tables_collection_manager_example.py b/Examples/runtimes/python/DynamoDBEncryption/src/encrypted_resource/encrypted_tables_collection_manager_example.py new file mode 100644 index 000000000..c453ae580 --- /dev/null +++ b/Examples/runtimes/python/DynamoDBEncryption/src/encrypted_resource/encrypted_tables_collection_manager_example.py @@ -0,0 +1,175 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +""" +Example for using the EncryptedTablesCollectionManager provided by EncryptedResource. + +Running this example requires access to the DDB Tables whose names +are provided in the function arguments. +These tables must be configured with the following primary key configuration: +- Partition key is named "partition_key" with type (S) +- Sort key is named "sort_key" with type (N) +""" + +import boto3 +from aws_cryptographic_material_providers.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_material_providers.mpl.config import MaterialProvidersConfig +from aws_cryptographic_material_providers.mpl.models import ( + CreateAwsKmsMrkMultiKeyringInput, + DBEAlgorithmSuiteId, +) +from aws_cryptographic_material_providers.mpl.references import IKeyring +from aws_dbesdk_dynamodb.encrypted.resource import ( + EncryptedResource, +) +from aws_dbesdk_dynamodb.structures.dynamodb import ( + DynamoDbTableEncryptionConfig, + DynamoDbTablesEncryptionConfig, +) +from aws_dbesdk_dynamodb.structures.structured_encryption import ( + CryptoAction, +) + + +def encrypted_tables_collection_manager_example( + kms_key_id: str, + dynamodb_table_names: list[str], +): + """Use an EncryptedTablesCollectionManager to write and read to multiple tables.""" + # 1. Create a Keyring. This Keyring will be responsible for protecting the data keys that protect your data. + # For this example, we will create a AWS KMS Keyring with the AWS KMS Key we want to use. + # We will use the `CreateMrkMultiKeyring` method to create this keyring, + # as it will correctly handle both single region and Multi-Region KMS Keys. + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders(config=MaterialProvidersConfig()) + kms_mrk_multi_keyring_input: CreateAwsKmsMrkMultiKeyringInput = CreateAwsKmsMrkMultiKeyringInput( + generator=kms_key_id, + ) + kms_mrk_multi_keyring: IKeyring = mat_prov.create_aws_kms_mrk_multi_keyring(input=kms_mrk_multi_keyring_input) + + # 2. Configure which attributes are encrypted and/or signed when writing new items. + # For each attribute that may exist on the items we plan to write to our DynamoDbTable, + # we must explicitly configure how they should be treated during item encryption: + # - ENCRYPT_AND_SIGN: The attribute is encrypted and included in the signature + # - SIGN_ONLY: The attribute not encrypted, but is still included in the signature + # - DO_NOTHING: The attribute is not encrypted and not included in the signature + attribute_actions_on_encrypt = { + "partition_key": CryptoAction.SIGN_ONLY, + "sort_key": CryptoAction.SIGN_ONLY, + "attribute1": CryptoAction.ENCRYPT_AND_SIGN, + "attribute2": CryptoAction.SIGN_ONLY, + ":attribute3": CryptoAction.DO_NOTHING, + } + + # 3. Configure which attributes we expect to be included in the signature + # when reading items. There are two options for configuring this: + # + # - (Recommended) Configure `allowedUnsignedAttributesPrefix`: + # When defining your DynamoDb schema and deciding on attribute names, + # choose a distinguishing prefix (such as ":") for all attributes that + # you do not want to include in the signature. + # This has two main benefits: + # - It is easier to reason about the security and authenticity of data within your item + # when all unauthenticated data is easily distinguishable by their attribute name. + # - If you need to add new unauthenticated attributes in the future, + # you can easily make the corresponding update to your `attributeActionsOnEncrypt` + # and immediately start writing to that new attribute, without + # any other configuration update needed. + # Once you configure this field, it is not safe to update it. + # + # - Configure `allowedUnsignedAttributes`: You may also explicitly list + # a set of attributes that should be considered unauthenticated when encountered + # on read. Be careful if you use this configuration. Do not remove an attribute + # name from this configuration, even if you are no longer writing with that attribute, + # as old items may still include this attribute, and our configuration needs to know + # to continue to exclude this attribute from the signature scope. + # If you add new attribute names to this field, you must first deploy the update to this + # field to all readers in your host fleet before deploying the update to start writing + # with that new attribute. + # + # For this example, we have designed our DynamoDb table such that any attribute name with + # the ":" prefix should be considered unauthenticated. + unsignAttrPrefix: str = ":" + + # 4. Create the DynamoDb Encryption configuration for the tables we will be writing to. + # For each table, we create a DynamoDbTableEncryptionConfig and add it to a dictionary. + # This dictionary is then added to a DynamoDbTablesEncryptionConfig, which is used to create the + # EncryptedResource. + table_configs = {} + for dynamodb_table_name in dynamodb_table_names: + table_config = DynamoDbTableEncryptionConfig( + logical_table_name=dynamodb_table_name, + partition_key_name="partition_key", + sort_key_name="sort_key", + attribute_actions_on_encrypt=attribute_actions_on_encrypt, + keyring=kms_mrk_multi_keyring, + allowed_unsigned_attribute_prefix=unsignAttrPrefix, + # Specifying an algorithm suite is not required, + # but is done here to demonstrate how to do so. + # We suggest using the + # `ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384_SYMSIG_HMAC_SHA384` suite, + # which includes AES-GCM with key derivation, signing, and key commitment. + # This is also the default algorithm suite if one is not specified in this config. + # For more information on supported algorithm suites, see: + # https://docs.aws.amazon.com/database-encryption-sdk/latest/devguide/supported-algorithms.html + algorithm_suite_id=DBEAlgorithmSuiteId.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384_SYMSIG_HMAC_SHA384, + ) + table_configs[dynamodb_table_name] = table_config + tables_config = DynamoDbTablesEncryptionConfig(table_encryption_configs=table_configs) + + # 5. Create the EncryptedResource + encrypted_resource = EncryptedResource( + resource=boto3.resource("dynamodb"), + encryption_config=tables_config, + ) + + # 6. Retrieve the EncryptedTablesCollectionManager from the EncryptedResource + encrypted_tables_collection_manager = encrypted_resource.tables + + # 7. Use the EncryptedTablesCollectionManager to get EncryptedTables to write to. + # **IMPORTANT**: This will return all tables in the collection, not just the ones you want to write to. + # This will include all tables that are associated with the current account and endpoint. + # You should consider filtering the tables you write to based on the table name. + encrypted_tables = encrypted_tables_collection_manager.all() + + # 8. Write a batch of items to the table. + # Before the items get sent to DynamoDb, they will be encrypted + # client-side, according to our configuration. + items = [] + for encrypted_table in encrypted_tables: + # Here, you should consider filtering the tables you write. + # If you do not, you will write to all tables in the collection. + # This may include tables with incompatible schemas, or tables that you do not have permission to write to. + if encrypted_table.table_name in dynamodb_table_names: + encrypted_table.put_item( + Item={ + "partition_key": "PythonEncryptedTablesCollectionManagerExample", + "sort_key": 0, + "attribute1": "encrypt and sign me!", + "attribute2": "sign me!", + ":attribute3": "ignore me!", + } + ) + + # 9. Read the items back from the table. + # After the items are retrieved from DynamoDb, but before the EncryptedResource + # returns them to the caller, they will be decrypted client-side according to our configuration. + items = [] + for encrypted_table in encrypted_tables: + # Here, you should consider filtering the tables you read from. + # If you do not, you will read from all tables in the collection. + # This may include tables with incompatible schemas, or tables that you do not have permission to read from. + if encrypted_table.table_name in dynamodb_table_names: + get_item_response = encrypted_table.get_item( + Key={ + "partition_key": "PythonEncryptedTablesCollectionManagerExample", + "sort_key": 0, + } + ) + + item = get_item_response["Item"] + items.append(item) + + # 10. Assert the items are as expected. + for item in items: + assert item["attribute1"] == "encrypt and sign me!" + assert item["attribute2"] == "sign me!" + assert item[":attribute3"] == "ignore me!" diff --git a/Examples/runtimes/python/DynamoDBEncryption/test/encrypted_resource/__init__.py b/Examples/runtimes/python/DynamoDBEncryption/test/encrypted_resource/__init__.py new file mode 100644 index 000000000..fa977e22f --- /dev/null +++ b/Examples/runtimes/python/DynamoDBEncryption/test/encrypted_resource/__init__.py @@ -0,0 +1,3 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Stub to allow relative imports of examples from tests.""" diff --git a/Examples/runtimes/python/DynamoDBEncryption/test/encrypted_resource/test_batch_read_write_example.py b/Examples/runtimes/python/DynamoDBEncryption/test/encrypted_resource/test_batch_read_write_example.py new file mode 100644 index 000000000..259f3925d --- /dev/null +++ b/Examples/runtimes/python/DynamoDBEncryption/test/encrypted_resource/test_batch_read_write_example.py @@ -0,0 +1,15 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Test suite for the EncryptedResource batch read/write example.""" +import pytest + +from ...src.encrypted_resource.batch_read_write_example import encrypted_resource_batch_read_write_example + +pytestmark = [pytest.mark.examples] + + +def test_encrypted_resource_batch_read_write_example(): + """Test function for encrypt and decrypt using the EncryptedClient example.""" + test_kms_key_id = "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f" + test_dynamodb_table_name = "DynamoDbEncryptionInterceptorTestTable" + encrypted_resource_batch_read_write_example(test_kms_key_id, test_dynamodb_table_name) diff --git a/Examples/runtimes/python/DynamoDBEncryption/test/encrypted_resource/test_encrypted_tables_collection_manager_example.py b/Examples/runtimes/python/DynamoDBEncryption/test/encrypted_resource/test_encrypted_tables_collection_manager_example.py new file mode 100644 index 000000000..31e266fbe --- /dev/null +++ b/Examples/runtimes/python/DynamoDBEncryption/test/encrypted_resource/test_encrypted_tables_collection_manager_example.py @@ -0,0 +1,17 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Test suite for the EncryptedResource batch read/write example.""" +import pytest + +from ...src.encrypted_resource.encrypted_tables_collection_manager_example import ( + encrypted_tables_collection_manager_example, +) + +pytestmark = [pytest.mark.examples] + + +def test_encrypted_resource_batch_read_write_example(): + """Test function for encrypt and decrypt using the EncryptedClient example.""" + test_kms_key_id = "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f" + test_dynamodb_table_name = "DynamoDbEncryptionInterceptorTestTable" + encrypted_tables_collection_manager_example(test_kms_key_id, [test_dynamodb_table_name]) From 7577d4b8badb64d47e5142cc0af18169a91d1653 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 May 2025 09:12:11 -0700 Subject: [PATCH 2/6] sync --- .../extern/CreateInterceptedDDBResource.py | 146 ++++++++++++++++++ TestVectors/runtimes/python/test/__init__.py | 2 + .../runtimes/python/test/resource/__init__.py | 0 .../test/resource/test_dafny_wrapper.py | 64 ++++++++ 4 files changed, 212 insertions(+) create mode 100644 TestVectors/runtimes/python/src/aws_dbesdk_dynamodb_test_vectors/internaldafny/extern/CreateInterceptedDDBResource.py create mode 100644 TestVectors/runtimes/python/test/__init__.py create mode 100644 TestVectors/runtimes/python/test/resource/__init__.py create mode 100644 TestVectors/runtimes/python/test/resource/test_dafny_wrapper.py diff --git a/TestVectors/runtimes/python/src/aws_dbesdk_dynamodb_test_vectors/internaldafny/extern/CreateInterceptedDDBResource.py b/TestVectors/runtimes/python/src/aws_dbesdk_dynamodb_test_vectors/internaldafny/extern/CreateInterceptedDDBResource.py new file mode 100644 index 000000000..98af0d1aa --- /dev/null +++ b/TestVectors/runtimes/python/src/aws_dbesdk_dynamodb_test_vectors/internaldafny/extern/CreateInterceptedDDBResource.py @@ -0,0 +1,146 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import boto3 +import types +import aws_dbesdk_dynamodb_test_vectors.internaldafny.generated.CreateInterceptedDDBClient +import aws_cryptography_internal_dynamodb.internaldafny.extern +from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb.dafny_to_smithy import aws_cryptography_dbencryptionsdk_dynamodb_DynamoDbTablesEncryptionConfig +from aws_dbesdk_dynamodb.encrypted.resource import EncryptedResource +from smithy_dafny_standard_library.internaldafny.generated import Wrappers +from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb.errors import _smithy_error_to_dafny_error +from aws_dbesdk_dynamodb_test_vectors.waiting_boto3_ddb_client import WaitingLocalDynamoClient +from aws_dbesdk_dynamodb.internal.resource_to_client import ResourceShapeToClientShapeConverter +from aws_dbesdk_dynamodb.internal.client_to_resource import ClientShapeToResourceShapeConverter + +class DynamoDBClientWrapperForDynamoDBResource: + """ + Internal-only wrapper class for DBESDK TestVectors. + + TestVectors Dafny code only knows how to interact with DynamoDB clients. + However, Python DDBEC and DBESDK have an EncryptedResource class that wraps boto3 DynamoDB Resources. + These classes create EncryptedTables that wrap boto3 DynamoDB Table Resources. + This class interfaces between Dafny TestVectors' DynamoDB client-calling code + and Python DBESDK's EncryptedResource/EncryptedTable classes. + + This class defers to a boto3 client for create_table and delete_table, + which are not supported on boto3 DynamoDB Table resources. + + TODO: Transact not supported on table. What do? + """ + + def __init__(self, resource, client): + self._resource = resource + self._client = client + self._client_shape_to_resource_shape_converter = ClientShapeToResourceShapeConverter() + self._resource_shape_to_client_shape_converter = ResourceShapeToClientShapeConverter() + + def batch_write_item(self, **kwargs): + # The input here is from the DBESDK TestVectors, which is in the shape of a client request. + # Convert the client request to a resource request to be passed to the table. + resource_input = self._client_shape_to_resource_shape_converter.batch_write_item_request(kwargs) + resource_output = self._resource.batch_write_item(**resource_input) + client_output = self._resource_shape_to_client_shape_converter.batch_write_item_response(resource_output) + return client_output + + def batch_get_item(self, **kwargs): + resource_input = self._client_shape_to_resource_shape_converter.batch_get_item_request(kwargs) + resource_output = self._resource.batch_get_item(**resource_input) + client_output = self._resource_shape_to_client_shape_converter.batch_get_item_response(resource_output) + return client_output + + def scan(self, **kwargs): + # Resources don't have scan, but EncryptedResources can provide EncryptedTables that do support scan. + # This path tests that the EncryptedTables provided by EncryptedResources can used for scan. + table_name = kwargs["TableName"] + # Note: Any ConditionExpression strings are not converted to boto3 Condition objects + # and are passed as-is to the resource. + # They absolutely could be converted, but that is tested in the boto3 Table tests. + # Not doing this conversion here expands test coverage to both cases. + table_input = self._client_shape_to_resource_shape_converter.scan_request(kwargs) + encrypted_table = self._resource.Table(table_name) + table_output = encrypted_table.scan(**table_input) + table_shape_converter = ResourceShapeToClientShapeConverter(table_name=table_name) + client_output = table_shape_converter.scan_response(table_output) + return client_output + + def put_item(self, **kwargs): + # Resources don't have put_item, but EncryptedResources can provide EncryptedTables that do support put_item. + # This path tests that the EncryptedTables provided by EncryptedResources can used for put_item. + table_name = kwargs["TableName"] + table_input = self._client_shape_to_resource_shape_converter.put_item_request(kwargs) + encrypted_table = self._resource.Table(table_name) + table_output = encrypted_table.put_item(**table_input) + table_shape_converter = ResourceShapeToClientShapeConverter(table_name=table_name) + client_output = table_shape_converter.put_item_response(table_output) + return client_output + + def get_item(self, **kwargs): + # Resources don't have get_item, but EncryptedResources can provide EncryptedTables that do support get_item. + # This path tests that the EncryptedTables provided by EncryptedResources can used for get_item. + table_name = kwargs["TableName"] + table_input = self._client_shape_to_resource_shape_converter.get_item_request(kwargs) + encrypted_table = self._resource.Table(table_name) + table_output = encrypted_table.get_item(**table_input) + table_shape_converter = ResourceShapeToClientShapeConverter(table_name=table_name) + client_output = table_shape_converter.get_item_response(table_output) + return client_output + + def query(self, **kwargs): + # Resources don't have query, but EncryptedResources can provide EncryptedTables that do support query. + # This path tests that the EncryptedTables provided by EncryptedResources can used for query. + table_name = kwargs["TableName"] + # Note: Any ConditionExpression strings are not converted to boto3 Condition objects + # and are passed as-is to the resource. + # They absolutely could be converted, but that is tested in the boto3 Table tests. + # Not doing this conversion here expands test coverage to both cases. + table_input = self._client_shape_to_resource_shape_converter.query_request(kwargs) + encrypted_table = self._resource.Table(table_name) + table_output = encrypted_table.query(**table_input) + table_shape_converter = ResourceShapeToClientShapeConverter(table_name=table_name) + client_output = table_shape_converter.query_response(table_output) + return client_output + + def transact_get_items(self, **kwargs): + raise NotImplementedError("transact_get_items not supported on resources") + + def transact_write_items(self, **kwargs): + raise NotImplementedError("transact_get_items not supported on resources") + + def delete_table(self, **kwargs): + # Resources don't have delete_table. Plus, DBESDK doesn't intercept DeleteTable calls. + # TestVectors only use this to ensure a new, clean table is created for each test. + # Defer to the underlying boto3 client to delete the table. + return self._client.delete_table(**kwargs) + + def create_table(self, **kwargs): + # Resources don't have create_table. Plus, DBESDK doesn't intercept CreateTable calls. + # TestVectors only use this to ensure a new, clean table is created for each test. + # Defer to the underlying boto3 client to create a table. + return self._client.create_table(**kwargs) + + +class default__: + @staticmethod + def CreateVanillaDDBClient(): + try: + return aws_cryptography_internal_dynamodb.internaldafny.extern.Com_Amazonaws_Dynamodb.default__.DynamoDBClient(WaitingLocalDynamoClient()) + except Exception as e: + return Wrappers.Result_Failure(_smithy_error_to_dafny_error(e)) + + @staticmethod + def CreateInterceptedDDBClient(dafny_encryption_config): + try: + native_encryption_config = aws_cryptography_dbencryptionsdk_dynamodb_DynamoDbTablesEncryptionConfig(dafny_encryption_config) + boto3_client = WaitingLocalDynamoClient() + table_config_names = list(native_encryption_config.table_encryption_configs.keys()) + if len(table_config_names) > 1: + raise ValueError("TODO more than 1 table; need EncryptedTablesManager") + # For TestVectors, use local DynamoDB endpoint + resource = boto3.resource('dynamodb', endpoint_url="http://localhost:8000") + encrypted_resource = EncryptedResource(resource = resource, encryption_config = native_encryption_config) + wrapped_encrypted_resource = DynamoDBClientWrapperForDynamoDBResource(resource = encrypted_resource, client = boto3_client) + return aws_cryptography_internal_dynamodb.internaldafny.extern.Com_Amazonaws_Dynamodb.default__.DynamoDBClient(wrapped_encrypted_resource) + except Exception as e: + return Wrappers.Result_Failure(_smithy_error_to_dafny_error(e)) + +aws_dbesdk_dynamodb_test_vectors.internaldafny.generated.CreateInterceptedDDBClient.default__ = default__ diff --git a/TestVectors/runtimes/python/test/__init__.py b/TestVectors/runtimes/python/test/__init__.py new file mode 100644 index 000000000..f94fd12a2 --- /dev/null +++ b/TestVectors/runtimes/python/test/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/TestVectors/runtimes/python/test/resource/__init__.py b/TestVectors/runtimes/python/test/resource/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/TestVectors/runtimes/python/test/resource/test_dafny_wrapper.py b/TestVectors/runtimes/python/test/resource/test_dafny_wrapper.py new file mode 100644 index 000000000..f19423f2d --- /dev/null +++ b/TestVectors/runtimes/python/test/resource/test_dafny_wrapper.py @@ -0,0 +1,64 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +import sys +from functools import partial + +# Different from standard test_dafny_wrapper due to weird test structure. +test_dir = '/'.join(__file__.split("/")[:-2]) + +sys.path.append(test_dir + "/internaldafny/extern") +sys.path.append(test_dir + "/internaldafny/generated") + +# Import extern to use an EncryptedResource as the wrapped DBESDK client. +import aws_dbesdk_dynamodb_test_vectors.internaldafny.extern.CreateInterceptedDDBResource +# Import extern to use the ItemEncryptor with Python dictionary-formatted items. +# (EncryptedResources use Python dictionary-formatted items.) +import aws_dbesdk_dynamodb_test_vectors.internaldafny.extern.CreateWrappedDictItemEncryptor + +# Remove invalid tests. +# Supported operations on Resources that are also supported by DBESDK are: +# - batch_get_item +# - batch_write_item +# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/service-resource/index.html +# +# However, Resources can provide Tables. +# Unsupported operations on Resources are that are supported by provided Tables are: +# - put_item +# - get_item +# - query +# - scan +# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/index.html#DynamoDB.Table +# These operations will be tested on EncryptedResources via provided EncryptedTables. +# +# Unsupported operations on both Resources and Tables are that are supported by DBESDK are: +# - transact_get_items +# - transact_write_items +# Remove any tests that call unsupported operations by overriding the test method to do nothing.. +# If more tests that call these operations are added, remove them below. +# If the list below becomes unmaintainable, or if other languages add clients with unsupported operations, +# refactor the Dafny code to conditionally call tests based on whether the client supports the operation under test. + +def EmptyTest(*args, **kwargs): + print(f"Skipping test {kwargs['test_name']} because {kwargs['reason']}") + +aws_dbesdk_dynamodb_test_vectors.internaldafny.generated.DdbEncryptionTestVectors.TestVectorConfig.BasicIoTestTransactWriteItems = partial( + EmptyTest, + test_name="BasicIoTestTransactWriteItems", + reason="neither DDB resources nor DDB tables support transact_write_items" +) +aws_dbesdk_dynamodb_test_vectors.internaldafny.generated.DdbEncryptionTestVectors.TestVectorConfig.BasicIoTestTransactGetItems = partial( + EmptyTest, + test_name="BasicIoTestTransactGetItems", + reason="neither DDB resources nor DDB tables support transact_get_items" +) + +def test_dafny(): + from ..internaldafny.generated import __main__ \ No newline at end of file From d68e095281dde5a716f9ac06362888e04a85b266 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 May 2025 09:25:17 -0700 Subject: [PATCH 3/6] sync --- .../python/src/aws_dbesdk_dynamodb/encrypted/resource.py | 3 +++ TestVectors/runtimes/python/test/__init__.py | 1 + TestVectors/runtimes/python/test/resource/__init__.py | 3 +++ 3 files changed, 7 insertions(+) diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/resource.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/resource.py index 65cfa53f3..dc268a615 100644 --- a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/resource.py +++ b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/resource.py @@ -297,6 +297,9 @@ def _resource_operation_logic( # (e.g. `ConsumedCapacity`) dbesdk_response = self._copy_sdk_response_to_dbesdk_response(sdk_output, dbesdk_response) + # Clean up the expression builder for the next operation + self._resource_shape_to_client_shape_converter.expression_builder.reset() + return dbesdk_response @property diff --git a/TestVectors/runtimes/python/test/__init__.py b/TestVectors/runtimes/python/test/__init__.py index f94fd12a2..fa977e22f 100644 --- a/TestVectors/runtimes/python/test/__init__.py +++ b/TestVectors/runtimes/python/test/__init__.py @@ -1,2 +1,3 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +"""Stub to allow relative imports of examples from tests.""" diff --git a/TestVectors/runtimes/python/test/resource/__init__.py b/TestVectors/runtimes/python/test/resource/__init__.py index e69de29bb..fa977e22f 100644 --- a/TestVectors/runtimes/python/test/resource/__init__.py +++ b/TestVectors/runtimes/python/test/resource/__init__.py @@ -0,0 +1,3 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Stub to allow relative imports of examples from tests.""" From 68114f364ae3375bf2e24b9ad49496fd133f8312 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 May 2025 14:06:53 -0700 Subject: [PATCH 4/6] sync --- .../test/integ/encrypted/test_resource.py | 55 +++++++++++++++++-- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_resource.py b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_resource.py index 7e48dd2e2..fe218442e 100644 --- a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_resource.py +++ b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_resource.py @@ -1,3 +1,6 @@ +import uuid +from copy import deepcopy + import boto3 import pytest @@ -5,11 +8,19 @@ from aws_dbesdk_dynamodb.encrypted.table import EncryptedTable from ...constants import INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, INTEG_TEST_DEFAULT_TABLE_CONFIGS -from ...items import complex_item_dict, complex_key_dict, simple_item_dict, simple_key_dict +from ...items import ( + complex_item_ddb, + complex_item_dict, + complex_key_dict, + simple_item_ddb, + simple_item_dict, + simple_key_dict, +) from ...requests import ( basic_batch_get_item_request_dict, basic_batch_write_item_delete_request_dict, basic_batch_write_item_put_request_dict, + basic_delete_item_request_ddb, ) @@ -42,22 +53,45 @@ def tables(resource): return resource.tables +@pytest.fixture(scope="module") +def test_run_suffix(): + return str(uuid.uuid4()) + + +@pytest.fixture +def test_items(test_run_suffix): + items = [deepcopy(complex_item_dict), deepcopy(simple_item_dict)] + for item in items: + item["partition_key"] += test_run_suffix + return items + + +@pytest.fixture +def test_keys(test_run_suffix): + keys = [deepcopy(complex_key_dict), deepcopy(simple_key_dict)] + for key in keys: + key["partition_key"] += test_run_suffix + return keys + + def test_GIVEN_items_WHEN_batch_write_and_get_THEN_round_trip_passes( resource, + test_items, + test_keys, ): - batch_write_item_put_request = basic_batch_write_item_put_request_dict([simple_item_dict, complex_item_dict]) + batch_write_item_put_request = basic_batch_write_item_put_request_dict(test_items) batch_write_response = resource.batch_write_item(**batch_write_item_put_request) assert batch_write_response["ResponseMetadata"]["HTTPStatusCode"] == 200 - batch_get_item_request = basic_batch_get_item_request_dict([simple_key_dict, complex_key_dict]) + batch_get_item_request = basic_batch_get_item_request_dict(test_keys) batch_get_response = resource.batch_get_item(**batch_get_item_request) assert batch_get_response["ResponseMetadata"]["HTTPStatusCode"] == 200 responses = batch_get_response["Responses"][INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME] assert len(responses) == 2 for response in responses: - assert response in [simple_item_dict, complex_item_dict] + assert response in test_items - batch_write_item_delete_request = basic_batch_write_item_delete_request_dict([simple_key_dict, complex_key_dict]) + batch_write_item_delete_request = basic_batch_write_item_delete_request_dict(test_keys) batch_write_response = resource.batch_write_item(**batch_write_item_delete_request) assert batch_write_response["ResponseMetadata"]["HTTPStatusCode"] == 200 @@ -152,3 +186,14 @@ def test_GIVEN_tables_WHEN_page_size_THEN_returns_tables( tables_list.append(table) # Then: Returns tables assert len(tables_list) > 0 + + +# Delete the items in the table after the module runs +@pytest.fixture(scope="module", autouse=True) +def cleanup_after_module(test_run_suffix): + yield + table = boto3.client("dynamodb") + items = [deepcopy(simple_item_ddb), deepcopy(complex_item_ddb)] + for item in items: + item["partition_key"]["S"] += test_run_suffix + table.delete_item(**basic_delete_item_request_ddb(item)) From 2c8877e3413e36f006f3a3b1c7ed6713fd7f7dd1 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 29 May 2025 13:21:50 -0700 Subject: [PATCH 5/6] sync --- .../runtimes/python/test/integ/encrypted/test_resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_resource.py b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_resource.py index fe218442e..9f41550e2 100644 --- a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_resource.py +++ b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_resource.py @@ -55,7 +55,7 @@ def tables(resource): @pytest.fixture(scope="module") def test_run_suffix(): - return str(uuid.uuid4()) + return "-" + str(uuid.uuid4()) @pytest.fixture From 773298c45012c212e4c2953e4bac59327dd80222 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 29 May 2025 14:59:33 -0700 Subject: [PATCH 6/6] sync --- .../runtimes/python/test/integ/encrypted/test_resource.py | 2 ++ .../runtimes/python/test/unit/encrypted/test_resource.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_resource.py b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_resource.py index 9f41550e2..22ddc3d1a 100644 --- a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_resource.py +++ b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_resource.py @@ -1,3 +1,5 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 import uuid from copy import deepcopy diff --git a/DynamoDbEncryption/runtimes/python/test/unit/encrypted/test_resource.py b/DynamoDbEncryption/runtimes/python/test/unit/encrypted/test_resource.py index aa0e1f2ce..12293dd54 100644 --- a/DynamoDbEncryption/runtimes/python/test/unit/encrypted/test_resource.py +++ b/DynamoDbEncryption/runtimes/python/test/unit/encrypted/test_resource.py @@ -1,3 +1,5 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 from mock import MagicMock from aws_dbesdk_dynamodb.encrypted.resource import EncryptedResource, EncryptedTablesCollectionManager