From b6787f95f7227abfc29b86152e797f5762864f10 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Wed, 29 Jul 2020 09:35:46 +0200 Subject: [PATCH 01/30] feat: add get_parameter utility --- aws_lambda_powertools/utilities/__init__.py | 8 ++ aws_lambda_powertools/utilities/parameters.py | 63 +++++++++ poetry.lock | 8 +- pyproject.toml | 1 + tests/functional/test_utilities.py | 128 ++++++++++++++++++ 5 files changed, 204 insertions(+), 4 deletions(-) create mode 100644 aws_lambda_powertools/utilities/__init__.py create mode 100644 aws_lambda_powertools/utilities/parameters.py create mode 100644 tests/functional/test_utilities.py diff --git a/aws_lambda_powertools/utilities/__init__.py b/aws_lambda_powertools/utilities/__init__.py new file mode 100644 index 00000000000..6e8839ad2ee --- /dev/null +++ b/aws_lambda_powertools/utilities/__init__.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- + +"""General utilities for Powertools""" + + +from .parameters import get_parameter + +__all__ = ["get_parameter"] diff --git a/aws_lambda_powertools/utilities/parameters.py b/aws_lambda_powertools/utilities/parameters.py new file mode 100644 index 00000000000..8e17d3e9311 --- /dev/null +++ b/aws_lambda_powertools/utilities/parameters.py @@ -0,0 +1,63 @@ +""" +Parameter retrieval and caching utility +""" + + +from collections import namedtuple +from datetime import datetime, timedelta + +import boto3 + +DEFAULT_MAX_AGE = 5 +ExpirableValue = namedtuple("ExpirableValue", ["value", "ttl"]) +PARAMETER_VALUES = {} +ssm = boto3.client("ssm") + + +def get_parameter(name: str, max_age: int = DEFAULT_MAX_AGE) -> str: + """ + Retrieve a parameter from the AWS Systems Manager (SSM) Parameter Store + + This will keep a local version in cache for `max_age` seconds to prevent + overfetching from SSM Parameter Store. + + See the [AWS Systems Manager Parameter Store documentation] + (https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-parameter-store.html) + for more information. + + Parameters + ---------- + name: str + Name of the SSM Parameter + max_age: int + Duration for which the parameter value can be cached + + Example + ------- + + from aws_lambda_powertools.utilities import get_parameter + + def lambda_handler(event, context): + # This will only make a call to the SSM service every 30 seconds. + value = get_parameter("my-parameter", max_age=30) + + Raises + ------ + ssm.exceptions.InternalServerError + When there is an internal server error from AWS Systems Manager + ssm.exceptions.InvalidKeyId + When the key ID is invalid + ssm.exceptions.ParameterNotFound + When the parameter name is not found in AWS Systems Manager + ssm.exceptions.ParameterVersionNotFound + When a version of the parameter is not found in AWS Systems Manager + """ + + if name not in PARAMETER_VALUES or PARAMETER_VALUES[name].ttl < datetime.now(): + # Retrieve the parameter from AWS Systems Manager + parameter = ssm.get_parameter(Name=name) + PARAMETER_VALUES[name] = ExpirableValue( + parameter["Parameter"]["Value"], datetime.now() + timedelta(seconds=max_age) + ) + + return PARAMETER_VALUES[name].value diff --git a/poetry.lock b/poetry.lock index a5570d1445c..e7b7cdff1db 100644 --- a/poetry.lock +++ b/poetry.lock @@ -162,7 +162,7 @@ typed-ast = ">=1.4.0" d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] [[package]] -category = "dev" +category = "main" description = "The AWS SDK for Python" name = "boto3" optional = false @@ -816,7 +816,7 @@ security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] [[package]] -category = "dev" +category = "main" description = "An Amazon S3 Transfer Manager" name = "s3transfer" optional = false @@ -951,7 +951,6 @@ multidict = ">=4.0" [[package]] category = "main" description = "Backport of pathlib-compatible object wrapper for zip files" -marker = "python_version < \"3.8\"" name = "zipp" optional = false python-versions = ">=3.6" @@ -962,7 +961,8 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["jaraco.itertools", "func-timeout"] [metadata] -content-hash = "a2760fd5f04b7f1841509fcbcb4ccdaf35d92d1395627787e4a11f391a0597d2" +content-hash = "18607a712e4a4a05de7350ecbcf26327a4fb45bb8609dc7f3d19b7610c2faafc" +lock-version = "1.0" python-versions = "^3.6" [metadata.files] diff --git a/pyproject.toml b/pyproject.toml index 0f0d9ac6ebe..3f823717a29 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ license = "MIT-0" python = "^3.6" aws-xray-sdk = "^2.5.0" fastjsonschema = "~=2.14.4" +boto3 = "^1.12" [tool.poetry.dev-dependencies] coverage = {extras = ["toml"], version = "^5.0.3"} diff --git a/tests/functional/test_utilities.py b/tests/functional/test_utilities.py new file mode 100644 index 00000000000..5dff1c73d33 --- /dev/null +++ b/tests/functional/test_utilities.py @@ -0,0 +1,128 @@ +import random +import string +from datetime import datetime, timedelta + +import pytest +from botocore import stub + +from aws_lambda_powertools import utilities +from aws_lambda_powertools.utilities import parameters + + +@pytest.fixture(scope="function") +def mock_name(): + # Parameter name must match [a-zA-Z0-9_.-/]+ + return "".join(random.choices(string.ascii_letters + string.digits + "_.-/", k=random.randrange(3, 200))) + + +@pytest.fixture(scope="function") +def mock_value(): + # Standard parameters can be up to 4 KB + return "".join(random.choices(string.printable, k=random.randrange(100, 4000))) + + +@pytest.fixture(scope="function") +def mock_version(): + return random.randrange(1, 1000) + + +def test_get_parameter_new(monkeypatch, mock_name, mock_value, mock_version): + """ + Test get_parameter() with a new parameter name + """ + + # Patch the parameter value store + monkeypatch.setattr(parameters, "PARAMETER_VALUES", {}) + + # Stub boto3 + stubber = stub.Stubber(parameters.ssm) + response = { + "Parameter": { + "Name": mock_name, + "Type": "String", + "Value": mock_value, + "Version": mock_version, + "Selector": f"{mock_name}:{mock_version}", + "SourceResult": "string", + "LastModifiedDate": datetime(2015, 1, 1), + "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{mock_name}", + } + } + expected_params = {"Name": mock_name} + stubber.add_response("get_parameter", response, expected_params) + stubber.activate() + + # Get the parameter value + try: + value = utilities.get_parameter(mock_name) + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + +def test_get_parameter_cached(monkeypatch, mock_name, mock_value, mock_version): + """ + Test get_parameter() with a cached value for parameter name + """ + + # Patch the parameter value store + monkeypatch.setattr( + parameters, + "PARAMETER_VALUES", + {mock_name: parameters.ExpirableValue(mock_value, datetime.now() + timedelta(seconds=60))}, + ) + + # Stub boto3 + stubber = stub.Stubber(parameters.ssm) + stubber.activate() + + # Get the parameter value + try: + value = utilities.get_parameter(mock_name) + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + +def test_get_parameter_expired(monkeypatch, mock_name, mock_value, mock_version): + """ + Test get_parameter() with a cached, but expired value for parameter name + """ + + # Patch the parameter value store + monkeypatch.setattr( + parameters, + "PARAMETER_VALUES", + {mock_name: parameters.ExpirableValue(mock_value, datetime.now() - timedelta(seconds=60))}, + ) + + # Stub boto3 + stubber = stub.Stubber(parameters.ssm) + response = { + "Parameter": { + "Name": mock_name, + "Type": "String", + "Value": mock_value, + "Version": mock_version, + "Selector": f"{mock_name}:{mock_version}", + "SourceResult": "string", + "LastModifiedDate": datetime(2015, 1, 1), + "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{mock_name}", + } + } + expected_params = {"Name": mock_name} + stubber.add_response("get_parameter", response, expected_params) + stubber.activate() + + # Get the parameter value + try: + value = utilities.get_parameter(mock_name) + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() From 29d27b841bbb8f74127303fe815c8fef73ea24aa Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Wed, 29 Jul 2020 09:47:55 +0200 Subject: [PATCH 02/30] fix: add AWS_DEFAULT_REGION for boto3 tests --- .github/workflows/python_build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python_build.yml b/.github/workflows/python_build.yml index ddef9f6c527..546fef326fe 100644 --- a/.github/workflows/python_build.yml +++ b/.github/workflows/python_build.yml @@ -32,6 +32,8 @@ jobs: run: make lint - name: Test with pytest run: make test + env: + AWS_DEFAULT_REGION: us-east-1 - name: Security baseline run: make security-baseline - name: Complexity baseline From a805e745510215f3f733c890de3726ce2232a85c Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Wed, 29 Jul 2020 11:27:22 +0200 Subject: [PATCH 03/30] revert "fix: add AWS_DEFAULT_REGION for boto3 tests" This reverts commit 29d27b841bbb8f74127303fe815c8fef73ea24aa. --- .github/workflows/python_build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/python_build.yml b/.github/workflows/python_build.yml index 546fef326fe..ddef9f6c527 100644 --- a/.github/workflows/python_build.yml +++ b/.github/workflows/python_build.yml @@ -32,8 +32,6 @@ jobs: run: make lint - name: Test with pytest run: make test - env: - AWS_DEFAULT_REGION: us-east-1 - name: Security baseline run: make security-baseline - name: Complexity baseline From 37964d1aa4e259b1ee5d72922fcd298a08fe4aa4 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Wed, 29 Jul 2020 11:30:12 +0200 Subject: [PATCH 04/30] fix: fix AWS_DEFAULT_REGION for get_parameter tests --- tests/functional/test_utilities.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/functional/test_utilities.py b/tests/functional/test_utilities.py index 5dff1c73d33..9c481530e03 100644 --- a/tests/functional/test_utilities.py +++ b/tests/functional/test_utilities.py @@ -5,9 +5,6 @@ import pytest from botocore import stub -from aws_lambda_powertools import utilities -from aws_lambda_powertools.utilities import parameters - @pytest.fixture(scope="function") def mock_name(): @@ -31,6 +28,12 @@ def test_get_parameter_new(monkeypatch, mock_name, mock_value, mock_version): Test get_parameter() with a new parameter name """ + # Set default region for boto3 + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + + from aws_lambda_powertools import utilities + from aws_lambda_powertools.utilities import parameters + # Patch the parameter value store monkeypatch.setattr(parameters, "PARAMETER_VALUES", {}) @@ -67,6 +70,12 @@ def test_get_parameter_cached(monkeypatch, mock_name, mock_value, mock_version): Test get_parameter() with a cached value for parameter name """ + # Set default region for boto3 + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + + from aws_lambda_powertools import utilities + from aws_lambda_powertools.utilities import parameters + # Patch the parameter value store monkeypatch.setattr( parameters, @@ -93,6 +102,12 @@ def test_get_parameter_expired(monkeypatch, mock_name, mock_value, mock_version) Test get_parameter() with a cached, but expired value for parameter name """ + # Set default region for boto3 + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + + from aws_lambda_powertools import utilities + from aws_lambda_powertools.utilities import parameters + # Patch the parameter value store monkeypatch.setattr( parameters, From dc3a5abc78362d94c32c9437b33cb7163abdedf8 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Wed, 29 Jul 2020 11:30:12 +0200 Subject: [PATCH 05/30] fix: fix AWS_DEFAULT_REGION for get_parameter tests --- aws_lambda_powertools/utilities/__init__.py | 5 - aws_lambda_powertools/utilities/parameters.py | 63 ------ .../utilities/parameters/__init__.py | 131 +++++++++++ .../utilities/parameters/base.py | 92 ++++++++ tests/functional/test_utilities.py | 128 ----------- tests/functional/test_utilities_parameters.py | 210 ++++++++++++++++++ 6 files changed, 433 insertions(+), 196 deletions(-) delete mode 100644 aws_lambda_powertools/utilities/parameters.py create mode 100644 aws_lambda_powertools/utilities/parameters/__init__.py create mode 100644 aws_lambda_powertools/utilities/parameters/base.py delete mode 100644 tests/functional/test_utilities.py create mode 100644 tests/functional/test_utilities_parameters.py diff --git a/aws_lambda_powertools/utilities/__init__.py b/aws_lambda_powertools/utilities/__init__.py index 6e8839ad2ee..67be909187a 100644 --- a/aws_lambda_powertools/utilities/__init__.py +++ b/aws_lambda_powertools/utilities/__init__.py @@ -1,8 +1,3 @@ # -*- coding: utf-8 -*- """General utilities for Powertools""" - - -from .parameters import get_parameter - -__all__ = ["get_parameter"] diff --git a/aws_lambda_powertools/utilities/parameters.py b/aws_lambda_powertools/utilities/parameters.py deleted file mode 100644 index 8e17d3e9311..00000000000 --- a/aws_lambda_powertools/utilities/parameters.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -Parameter retrieval and caching utility -""" - - -from collections import namedtuple -from datetime import datetime, timedelta - -import boto3 - -DEFAULT_MAX_AGE = 5 -ExpirableValue = namedtuple("ExpirableValue", ["value", "ttl"]) -PARAMETER_VALUES = {} -ssm = boto3.client("ssm") - - -def get_parameter(name: str, max_age: int = DEFAULT_MAX_AGE) -> str: - """ - Retrieve a parameter from the AWS Systems Manager (SSM) Parameter Store - - This will keep a local version in cache for `max_age` seconds to prevent - overfetching from SSM Parameter Store. - - See the [AWS Systems Manager Parameter Store documentation] - (https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-parameter-store.html) - for more information. - - Parameters - ---------- - name: str - Name of the SSM Parameter - max_age: int - Duration for which the parameter value can be cached - - Example - ------- - - from aws_lambda_powertools.utilities import get_parameter - - def lambda_handler(event, context): - # This will only make a call to the SSM service every 30 seconds. - value = get_parameter("my-parameter", max_age=30) - - Raises - ------ - ssm.exceptions.InternalServerError - When there is an internal server error from AWS Systems Manager - ssm.exceptions.InvalidKeyId - When the key ID is invalid - ssm.exceptions.ParameterNotFound - When the parameter name is not found in AWS Systems Manager - ssm.exceptions.ParameterVersionNotFound - When a version of the parameter is not found in AWS Systems Manager - """ - - if name not in PARAMETER_VALUES or PARAMETER_VALUES[name].ttl < datetime.now(): - # Retrieve the parameter from AWS Systems Manager - parameter = ssm.get_parameter(Name=name) - PARAMETER_VALUES[name] = ExpirableValue( - parameter["Parameter"]["Value"], datetime.now() + timedelta(seconds=max_age) - ) - - return PARAMETER_VALUES[name].value diff --git a/aws_lambda_powertools/utilities/parameters/__init__.py b/aws_lambda_powertools/utilities/parameters/__init__.py new file mode 100644 index 00000000000..66784cadced --- /dev/null +++ b/aws_lambda_powertools/utilities/parameters/__init__.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- + +""" +Parameter retrieval and caching utility +""" + +from typing import Optional, Union + +import boto3 + +from .base import BaseProvider + +__all__ = ["BaseProvider", "DynamoDBProvider", "SecretsProvider", "SSMProvider", "get_parameter", "get_secret"] + + +class SSMProvider(BaseProvider): + """ + AWS Systems Manager Parameter Store Provider + """ + + client = None + + def __init__( + self, region: Optional[str] = None, + ): + """ + Initialize the SSM Parameter Store client + """ + + if region: + self.client = boto3.client("ssm", region_name=region) + else: + self.client = boto3.client("ssm") + + super().__init__() + + def _get_from_external_store(self, name: str) -> str: + """ + Retrieve a parameter value from AWS Systems Manager Parameter Store + """ + + return self.client.get_parameter(Name=name)["Parameter"]["Value"] + + +class SecretsProvider(BaseProvider): + """ + AWS Secrets Manager Parameter Provider + """ + + client = None + + def __init__(self, region: Optional[str] = None): + """ + Initialize the Secrets Manager client + """ + + if region: + self.client = boto3.client("secretsmanager", region_name=region) + else: + self.client = boto3.client("secretsmanager") + + super().__init__() + + def _get_from_external_store(self, name: str) -> str: + """ + Retrieve a parameter value from AWS Systems Manager Parameter Store + """ + + return self.client.get_secret_value(SecretId=name)["SecretString"] + + +class DynamoDBProvider(BaseProvider): + """ + Amazon DynamoDB Parameter Provider + """ + + table = None + key_attr = None + value_attr = None + + def __init__( + self, table_name: str, key_attr: str = "id", value_attr: str = "value", region: Optional[str] = None, + ): + """ + Initialize the DynamoDB client + """ + + if region: + self.table = boto3.resource("dynamodb", region_name=region).Table(table_name) + else: + self.table = boto3.resource("dynamodb").Table(table_name) + + self.key_attr = key_attr + self.value_attr = value_attr + + super().__init__() + + def _get_from_external_store(self, name: str) -> str: + """ + Retrieve a parameter value from Amazon DynamoDB + """ + + return self.table.get_item(Key={self.key_attr: name})["Item"][self.value_attr] + + +# These providers will be dynamically initialized on first use of the helper functions +_DEFAULT_PROVIDERS = {} + + +def get_parameter(name: str, transform: Optional[str] = None) -> Union[str, dict, bytes]: + """ + Retrieve a parameter value from AWS Systems Manager (SSM) Parameter Store + """ + + # Only create the provider if this function is called at least once + if "ssm" not in _DEFAULT_PROVIDERS: + _DEFAULT_PROVIDERS["ssm"] = SSMProvider() + + return _DEFAULT_PROVIDERS["ssm"].get(name, transform=transform) + + +def get_secret(name: str, transform: Optional[str] = None) -> Union[str, dict, bytes]: + """ + Retrieve a parameter value from AWS Secrets Manager + """ + + # Only create the provider if this function is called at least once + if "secrets" not in _DEFAULT_PROVIDERS: + _DEFAULT_PROVIDERS["secrets"] = SecretsProvider() + + return _DEFAULT_PROVIDERS["secrets"].get(name, transform=transform) diff --git a/aws_lambda_powertools/utilities/parameters/base.py b/aws_lambda_powertools/utilities/parameters/base.py new file mode 100644 index 00000000000..e45a3fd5bbc --- /dev/null +++ b/aws_lambda_powertools/utilities/parameters/base.py @@ -0,0 +1,92 @@ +""" +Base for Parameter providers +""" + +import base64 +import json +from abc import ABC, abstractmethod +from collections import namedtuple +from datetime import datetime, timedelta +from typing import Optional, Union + +DEFAULT_MAX_AGE_SECS = 5 +ExpirableValue = namedtuple("ExpirableValue", ["value", "ttl"]) + + +class GetParameterError(Exception): + """When a provider raises an exception on parameter retrieval""" + + +class BaseProvider(ABC): + """ + Abstract Base Class for Parameter providers + """ + + store = None + + def __init__(self): + """ + Initialize the base provider + """ + + self.store = {} + + def get( + self, name: str, max_age: int = DEFAULT_MAX_AGE_SECS, transform: Optional[str] = None, + ) -> Union[str, dict, bytes]: + """ + Retrieve a parameter value or return the cached value + + Parameters + ---------- + + name: str + Parameter name + max_age: int + Maximum age of the cached value + transform: str + Optional transformation of the parameter value. Supported values + are "json" for JSON strings and "binary" for base 64 encoded + values. + + Raises + ------ + + GetParameterError + When the parameter provider fails to retrieve a parameter value for + a given name. + """ + + # If there are multiple calls to the same parameter but in a different + # transform, they will be stored multiple times. This allows us to + # optimize by transforming the data only once per retrieval, thus there + # is no need to transform cached values multiple times. However, this + # means that we need to make multiple calls to the underlying parameter + # store if we need to return it in different transforms. Since the number + # of supported transform is small and the probability that a given + # parameter will always be used in a specific transform, this should be + # an acceptable tradeoff. + key = (name, transform) + + if key not in self.store or self.store[key].ttl < datetime.now(): + try: + value = self._get_from_external_store(name) + # Encapsulate all errors into a generic GetParameterError + except Exception as exc: + raise GetParameterError(str(exc)) + + if transform == "json": + value = json.loads(value) + elif transform == "binary": + value = base64.b64decode(value) + + self.store[key] = ExpirableValue(value, datetime.now() + timedelta(seconds=max_age),) + + return self.store[key].value + + @abstractmethod + def _get_from_external_store(self, name: str) -> str: + """ + Retrieve paramater value from the underlying parameter store + """ + raise NotImplementedError() diff --git a/tests/functional/test_utilities.py b/tests/functional/test_utilities.py deleted file mode 100644 index 5dff1c73d33..00000000000 --- a/tests/functional/test_utilities.py +++ /dev/null @@ -1,128 +0,0 @@ -import random -import string -from datetime import datetime, timedelta - -import pytest -from botocore import stub - -from aws_lambda_powertools import utilities -from aws_lambda_powertools.utilities import parameters - - -@pytest.fixture(scope="function") -def mock_name(): - # Parameter name must match [a-zA-Z0-9_.-/]+ - return "".join(random.choices(string.ascii_letters + string.digits + "_.-/", k=random.randrange(3, 200))) - - -@pytest.fixture(scope="function") -def mock_value(): - # Standard parameters can be up to 4 KB - return "".join(random.choices(string.printable, k=random.randrange(100, 4000))) - - -@pytest.fixture(scope="function") -def mock_version(): - return random.randrange(1, 1000) - - -def test_get_parameter_new(monkeypatch, mock_name, mock_value, mock_version): - """ - Test get_parameter() with a new parameter name - """ - - # Patch the parameter value store - monkeypatch.setattr(parameters, "PARAMETER_VALUES", {}) - - # Stub boto3 - stubber = stub.Stubber(parameters.ssm) - response = { - "Parameter": { - "Name": mock_name, - "Type": "String", - "Value": mock_value, - "Version": mock_version, - "Selector": f"{mock_name}:{mock_version}", - "SourceResult": "string", - "LastModifiedDate": datetime(2015, 1, 1), - "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{mock_name}", - } - } - expected_params = {"Name": mock_name} - stubber.add_response("get_parameter", response, expected_params) - stubber.activate() - - # Get the parameter value - try: - value = utilities.get_parameter(mock_name) - - assert value == mock_value - stubber.assert_no_pending_responses() - finally: - stubber.deactivate() - - -def test_get_parameter_cached(monkeypatch, mock_name, mock_value, mock_version): - """ - Test get_parameter() with a cached value for parameter name - """ - - # Patch the parameter value store - monkeypatch.setattr( - parameters, - "PARAMETER_VALUES", - {mock_name: parameters.ExpirableValue(mock_value, datetime.now() + timedelta(seconds=60))}, - ) - - # Stub boto3 - stubber = stub.Stubber(parameters.ssm) - stubber.activate() - - # Get the parameter value - try: - value = utilities.get_parameter(mock_name) - - assert value == mock_value - stubber.assert_no_pending_responses() - finally: - stubber.deactivate() - - -def test_get_parameter_expired(monkeypatch, mock_name, mock_value, mock_version): - """ - Test get_parameter() with a cached, but expired value for parameter name - """ - - # Patch the parameter value store - monkeypatch.setattr( - parameters, - "PARAMETER_VALUES", - {mock_name: parameters.ExpirableValue(mock_value, datetime.now() - timedelta(seconds=60))}, - ) - - # Stub boto3 - stubber = stub.Stubber(parameters.ssm) - response = { - "Parameter": { - "Name": mock_name, - "Type": "String", - "Value": mock_value, - "Version": mock_version, - "Selector": f"{mock_name}:{mock_version}", - "SourceResult": "string", - "LastModifiedDate": datetime(2015, 1, 1), - "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{mock_name}", - } - } - expected_params = {"Name": mock_name} - stubber.add_response("get_parameter", response, expected_params) - stubber.activate() - - # Get the parameter value - try: - value = utilities.get_parameter(mock_name) - - assert value == mock_value - stubber.assert_no_pending_responses() - finally: - stubber.deactivate() diff --git a/tests/functional/test_utilities_parameters.py b/tests/functional/test_utilities_parameters.py new file mode 100644 index 00000000000..b301d35648c --- /dev/null +++ b/tests/functional/test_utilities_parameters.py @@ -0,0 +1,210 @@ +import random +import string +from datetime import datetime, timedelta + +import pytest +from botocore import stub + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.parameters.base import ExpirableValue + + +@pytest.fixture(scope="function") +def mock_name(): + # Parameter name must match [a-zA-Z0-9_.-/]+ + return "".join(random.choices(string.ascii_letters + string.digits + "_.-/", k=random.randrange(3, 200))) + + +@pytest.fixture(scope="function") +def mock_value(): + # Standard parameters can be up to 4 KB + return "".join(random.choices(string.printable, k=random.randrange(100, 4000))) + + +@pytest.fixture(scope="function") +def mock_version(): + return random.randrange(1, 1000) + + +def test_ssm_provider_get(monkeypatch, mock_name, mock_value, mock_version): + """ + Test SSMProvider.get() with a non-cached value + """ + + # Create a new provider + provider = parameters.SSMProvider(region="us-east-1") + + # Stub the boto3 client + stubber = stub.Stubber(provider.client) + response = { + "Parameter": { + "Name": mock_name, + "Type": "String", + "Value": mock_value, + "Version": mock_version, + "Selector": f"{mock_name}:{mock_version}", + "SourceResult": "string", + "LastModifiedDate": datetime(2015, 1, 1), + "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{mock_name}", + } + } + expected_params = {"Name": mock_name} + stubber.add_response("get_parameter", response, expected_params) + stubber.activate() + + try: + value = provider.get(mock_name) + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + +def test_ssm_provider_get_cached(monkeypatch, mock_name, mock_value, mock_version): + """ + Test SSMProvider.get() with a cached value + """ + + # Create a new provider + provider = parameters.SSMProvider(region="us-east-1") + + # Inject value in the internal store + provider.store[(mock_name, None)] = ExpirableValue(mock_value, datetime.now() + timedelta(seconds=60)) + + # Stub the boto3 client + stubber = stub.Stubber(provider.client) + stubber.activate() + + try: + value = provider.get(mock_name) + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + +def test_ssm_provider_get_expired(monkeypatch, mock_name, mock_value, mock_version): + """ + Test SSMProvider.get() with a cached but expired value + """ + + # Create a new provider + provider = parameters.SSMProvider(region="us-east-1") + + # Inject value in the internal store + provider.store[(mock_name, None)] = ExpirableValue(mock_value, datetime.now() - timedelta(seconds=60)) + + # Stub the boto3 client + stubber = stub.Stubber(provider.client) + response = { + "Parameter": { + "Name": mock_name, + "Type": "String", + "Value": mock_value, + "Version": mock_version, + "Selector": f"{mock_name}:{mock_version}", + "SourceResult": "string", + "LastModifiedDate": datetime(2015, 1, 1), + "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{mock_name}", + } + } + expected_params = {"Name": mock_name} + stubber.add_response("get_parameter", response, expected_params) + stubber.activate() + + try: + value = provider.get(mock_name) + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + +def test_secrets_provider_get(monkeypatch, mock_name, mock_value): + """ + Test SecretsProvider.get() with a non-cached value + """ + + # Create a new provider + provider = parameters.SecretsProvider(region="us-east-1") + + # Stub the boto3 client + stubber = stub.Stubber(provider.client) + response = { + "ARN": f"arn:aws:secretsmanager:us-east-1:132456789012:secret/{mock_name}", + "Name": mock_name, + "VersionId": "7a9155b8-2dc9-466e-b4f6-5bc46516c84d", + "SecretString": mock_value, + "CreatedDate": datetime(2015, 1, 1), + } + expected_params = {"SecretId": mock_name} + stubber.add_response("get_secret_value", response, expected_params) + stubber.activate() + + try: + value = provider.get(mock_name) + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + +def test_secrets_provider_get_cached(monkeypatch, mock_name, mock_value, mock_version): + """ + Test SecretsProvider.get() with a cached value + """ + + # Create a new provider + provider = parameters.SecretsProvider(region="us-east-1") + + # Inject value in the internal store + provider.store[(mock_name, None)] = ExpirableValue(mock_value, datetime.now() + timedelta(seconds=60)) + + # Stub the boto3 client + stubber = stub.Stubber(provider.client) + stubber.activate() + + try: + value = provider.get(mock_name) + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + +def test_secrets_provider_get_expired(monkeypatch, mock_name, mock_value): + """ + Test SecretsProvider.get() with a cached but expired value + """ + + # Create a new provider + provider = parameters.SecretsProvider(region="us-east-1") + + # Inject value in the internal store + provider.store[(mock_name, None)] = ExpirableValue(mock_value, datetime.now() - timedelta(seconds=60)) + + # Stub the boto3 client + stubber = stub.Stubber(provider.client) + response = { + "ARN": f"arn:aws:secretsmanager:us-east-1:132456789012:secret/{mock_name}", + "Name": mock_name, + "VersionId": "7a9155b8-2dc9-466e-b4f6-5bc46516c84d", + "SecretString": mock_value, + "CreatedDate": datetime(2015, 1, 1), + } + expected_params = {"SecretId": mock_name} + stubber.add_response("get_secret_value", response, expected_params) + stubber.activate() + + try: + value = provider.get(mock_name) + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() From 79bff7e0419d1047fdd3b6a36213784760e70660 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Thu, 30 Jul 2020 14:11:10 +0200 Subject: [PATCH 06/30] chore: rename _get_from_external_store to _get --- aws_lambda_powertools/utilities/parameters/__init__.py | 6 +++--- aws_lambda_powertools/utilities/parameters/base.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/__init__.py b/aws_lambda_powertools/utilities/parameters/__init__.py index 66784cadced..b6e948179e1 100644 --- a/aws_lambda_powertools/utilities/parameters/__init__.py +++ b/aws_lambda_powertools/utilities/parameters/__init__.py @@ -34,7 +34,7 @@ def __init__( super().__init__() - def _get_from_external_store(self, name: str) -> str: + def _get(self, name: str) -> str: """ Retrieve a parameter value from AWS Systems Manager Parameter Store """ @@ -61,7 +61,7 @@ def __init__(self, region: Optional[str] = None): super().__init__() - def _get_from_external_store(self, name: str) -> str: + def _get(self, name: str) -> str: """ Retrieve a parameter value from AWS Systems Manager Parameter Store """ @@ -95,7 +95,7 @@ def __init__( super().__init__() - def _get_from_external_store(self, name: str) -> str: + def _get(self, name: str) -> str: """ Retrieve a parameter value from Amazon DynamoDB """ diff --git a/aws_lambda_powertools/utilities/parameters/base.py b/aws_lambda_powertools/utilities/parameters/base.py index e45a3fd5bbc..a53036be49d 100644 --- a/aws_lambda_powertools/utilities/parameters/base.py +++ b/aws_lambda_powertools/utilities/parameters/base.py @@ -70,7 +70,7 @@ def get( if key not in self.store or self.store[key].ttl < datetime.now(): try: - value = self._get_from_external_store(name) + value = self._get(name) # Encapsulate all errors into a generic GetParameterError except Exception as exc: raise GetParameterError(str(exc)) @@ -85,7 +85,7 @@ def get( return self.store[key].value @abstractmethod - def _get_from_external_store(self, name: str) -> str: + def _get(self, name: str) -> str: """ Retrieve paramater value from the underlying parameter store """ From a1205280fc674bef84dfafab120ec665c11b0e46 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Thu, 30 Jul 2020 15:41:10 +0200 Subject: [PATCH 07/30] feat: add get_multiple for parameter providers --- .../utilities/parameters/__init__.py | 121 ++++++- .../utilities/parameters/base.py | 42 ++- tests/functional/test_utilities_parameters.py | 297 +++++++++++++++++- 3 files changed, 438 insertions(+), 22 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/__init__.py b/aws_lambda_powertools/utilities/parameters/__init__.py index b6e948179e1..c4d0a629f20 100644 --- a/aws_lambda_powertools/utilities/parameters/__init__.py +++ b/aws_lambda_powertools/utilities/parameters/__init__.py @@ -4,9 +4,10 @@ Parameter retrieval and caching utility """ -from typing import Optional, Union +from typing import Dict, Optional, Union import boto3 +from boto3.dynamodb.conditions import Key from .base import BaseProvider @@ -34,12 +35,66 @@ def __init__( super().__init__() - def _get(self, name: str) -> str: + def _get(self, name: str, **kwargs) -> str: """ Retrieve a parameter value from AWS Systems Manager Parameter Store + + Parameters + ---------- + name: str + Parameter name + decrypt: bool + If the parameter value should be decrypted + """ + + # Load kwargs + decrypt = kwargs.get("decrypt", False) + + return self.client.get_parameter(Name=name, WithDecryption=decrypt)["Parameter"]["Value"] + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + """ + Retrieve multiple parameter values from AWS Systems Manager Parameter Store + + Parameters + ---------- + path: str + Path to retrieve the parameters + decrypt: bool + If the parameter values should be decrypted + recursive: bool + If this should retrieve the parameter values recursively or not """ - return self.client.get_parameter(Name=name)["Parameter"]["Value"] + # Load kwargs + decrypt = kwargs.get("decrypt", False) + recursive = kwargs.get("recursive", False) + + response = self.client.get_parameters_by_path(Path=path, WithDecryption=decrypt, Recursive=recursive) + parameters = response.get("Parameters", []) + + # Keep retrieving parameters + while "NextToken" in response: + response = self.client.get_parameters_by_path( + Path=path, WithDecryption=decrypt, Recursive=recursive, NextToken=response["NextToken"] + ) + parameters.extend(response.get("Parameters", [])) + + retval = {} + for parameter in parameters: + + # Standardize the parameter name + # The parameter name returned by SSM will contained the full path. + # However, for readability, we should return only the part after + # the path. + name = parameter["Name"] + if name.startswith(path): + name = name[len(path) :] + name = name.lstrip("/") + + retval[name] = parameter["Value"] + + return retval class SecretsProvider(BaseProvider): @@ -61,13 +116,19 @@ def __init__(self, region: Optional[str] = None): super().__init__() - def _get(self, name: str) -> str: + def _get(self, name: str, **kwargs) -> str: """ Retrieve a parameter value from AWS Systems Manager Parameter Store """ return self.client.get_secret_value(SecretId=name)["SecretString"] + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + """ + Retrieving multiple parameter values is not supported with AWS Secrets Manager + """ + raise NotImplementedError() + class DynamoDBProvider(BaseProvider): """ @@ -95,19 +156,49 @@ def __init__( super().__init__() - def _get(self, name: str) -> str: + def _get(self, name: str, **kwargs) -> str: """ Retrieve a parameter value from Amazon DynamoDB """ return self.table.get_item(Key={self.key_attr: name})["Item"][self.value_attr] + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + """ + Retrieve multiple parameter values from Amazon DynamoDB + + Parameters + ---------- + path: str + Path to retrieve the parameters + sort_attr: str + Name of the DynamoDB table sort key (defaults to 'sk') + """ + + sort_attr = kwargs.get("sort_attr", "sk") + + response = self.table.query(KeyConditionExpression=Key(self.key_attr).eq(path)) + items = response.get("Items", []) + + # Keep querying while there are more items matching the partition key + while "LastEvaluatedKey" in response: + response = self.table.query( + KeyConditionExpression=Key(self.key_attr).eq(path), ExclusiveStartKey=response["LastEvaluatedKey"], + ) + items.extend(response.get("Items", [])) + + retval = {} + for item in items: + retval[item[sort_attr]] = item[self.value_attr] + + return retval + # These providers will be dynamically initialized on first use of the helper functions _DEFAULT_PROVIDERS = {} -def get_parameter(name: str, transform: Optional[str] = None) -> Union[str, dict, bytes]: +def get_parameter(name: str, transform: Optional[str] = None) -> Union[str, list, dict, bytes]: """ Retrieve a parameter value from AWS Systems Manager (SSM) Parameter Store """ @@ -119,7 +210,21 @@ def get_parameter(name: str, transform: Optional[str] = None) -> Union[str, dict return _DEFAULT_PROVIDERS["ssm"].get(name, transform=transform) -def get_secret(name: str, transform: Optional[str] = None) -> Union[str, dict, bytes]: +def get_parameters( + path: str, transform: Optional[str] = None, recursive: bool = False, decrypt: bool = False +) -> Union[Dict[str, str], Dict[str, dict], Dict[str, bytes]]: + """ + Retrieve multiple parameter values from AWS Systems Manager (SSM) Parameter Store + """ + + # Only create the provider if this function is called at least once + if "ssm" not in _DEFAULT_PROVIDERS: + _DEFAULT_PROVIDERS["ssm"] = SSMProvider() + + return _DEFAULT_PROVIDERS["ssm"].get_multiple(path, transform=transform, recursive=recursive, decrypt=decrypt) + + +def get_secret(name: str, transform: Optional[str] = None, decrypt: bool = False) -> Union[str, dict, bytes]: """ Retrieve a parameter value from AWS Secrets Manager """ @@ -128,4 +233,4 @@ def get_secret(name: str, transform: Optional[str] = None) -> Union[str, dict, b if "secrets" not in _DEFAULT_PROVIDERS: _DEFAULT_PROVIDERS["secrets"] = SecretsProvider() - return _DEFAULT_PROVIDERS["secrets"].get(name, transform=transform) + return _DEFAULT_PROVIDERS["secrets"].get(name, transform=transform, decrypt=decrypt) diff --git a/aws_lambda_powertools/utilities/parameters/base.py b/aws_lambda_powertools/utilities/parameters/base.py index a53036be49d..33342eb3f84 100644 --- a/aws_lambda_powertools/utilities/parameters/base.py +++ b/aws_lambda_powertools/utilities/parameters/base.py @@ -7,7 +7,7 @@ from abc import ABC, abstractmethod from collections import namedtuple from datetime import datetime, timedelta -from typing import Optional, Union +from typing import Dict, Optional, Union DEFAULT_MAX_AGE_SECS = 5 ExpirableValue = namedtuple("ExpirableValue", ["value", "ttl"]) @@ -32,8 +32,8 @@ def __init__(self): self.store = {} def get( - self, name: str, max_age: int = DEFAULT_MAX_AGE_SECS, transform: Optional[str] = None, - ) -> Union[str, dict, bytes]: + self, name: str, max_age: int = DEFAULT_MAX_AGE_SECS, transform: Optional[str] = None, **kwargs + ) -> Union[str, list, dict, bytes]: """ Retrieve a parameter value or return the cached value @@ -70,7 +70,7 @@ def get( if key not in self.store or self.store[key].ttl < datetime.now(): try: - value = self._get(name) + value = self._get(name, **kwargs) # Encapsulate all errors into a generic GetParameterError except Exception as exc: raise GetParameterError(str(exc)) @@ -85,8 +85,40 @@ def get( return self.store[key].value @abstractmethod - def _get(self, name: str) -> str: + def _get(self, name: str, **kwargs) -> str: """ Retrieve paramater value from the underlying parameter store """ raise NotImplementedError() + + def get_multiple( + self, path: str, max_age: int = DEFAULT_MAX_AGE_SECS, transform: Optional[str] = None, **kwargs + ) -> Union[Dict[str, str], Dict[str, dict], Dict[str, bytes]]: + """ + Retrieve multiple parameters based on a path prefix + """ + + key = (path, transform) + + if key not in self.store or self.store[key].ttl < datetime.now(): + try: + values = self._get_multiple(path, **kwargs) + # Encapsulate all errors into a generic GetParameterError + except Exception as exc: + raise GetParameterError(str(exc)) + + if transform == "json": + values = {k: json.loads(v) for k, v in values} + elif transform == "binary": + values = {k: base64.b64decode(v) for k, v in values} + + self.store[key] = ExpirableValue(values, datetime.now() + timedelta(seconds=max_age),) + + return self.store[key].value + + @abstractmethod + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + """ + Retrieve multiple parameter values from the underlying parameter store + """ + raise NotImplementedError() diff --git a/tests/functional/test_utilities_parameters.py b/tests/functional/test_utilities_parameters.py index b301d35648c..2e183d82cfe 100644 --- a/tests/functional/test_utilities_parameters.py +++ b/tests/functional/test_utilities_parameters.py @@ -1,12 +1,16 @@ +import base64 +import json import random import string from datetime import datetime, timedelta +from typing import Dict import pytest +from boto3.dynamodb.conditions import Key from botocore import stub from aws_lambda_powertools.utilities import parameters -from aws_lambda_powertools.utilities.parameters.base import ExpirableValue +from aws_lambda_powertools.utilities.parameters.base import BaseProvider, ExpirableValue @pytest.fixture(scope="function") @@ -26,7 +30,124 @@ def mock_version(): return random.randrange(1, 1000) -def test_ssm_provider_get(monkeypatch, mock_name, mock_value, mock_version): +def test_dynamodb_provider_get(mock_name, mock_value): + """ + Test DynamoDBProvider.get() with a non-cached value + """ + + table_name = "TEST_TABLE" + + # Create a new provider + provider = parameters.DynamoDBProvider(table_name, region="us-east-1") + + # Stub the boto3 client + stubber = stub.Stubber(provider.table.meta.client) + response = {"Item": {"id": {"S": mock_name}, "value": {"S": mock_value}}} + expected_params = {"TableName": table_name, "Key": {"id": mock_name}} + stubber.add_response("get_item", response, expected_params) + stubber.activate() + + try: + value = provider.get(mock_name) + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + +def test_dynamodb_provider_get_cached(mock_name, mock_value): + """ + Test DynamoDBProvider.get() with a cached value + """ + + table_name = "TEST_TABLE" + + # Create a new provider + provider = parameters.DynamoDBProvider(table_name, region="us-east-1") + + # Inject value in the internal store + provider.store[(mock_name, None)] = ExpirableValue(mock_value, datetime.now() + timedelta(seconds=60)) + + # Stub the boto3 client + stubber = stub.Stubber(provider.table.meta.client) + stubber.activate() + + try: + value = provider.get(mock_name) + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + +def test_dynamodb_provider_get_expired(mock_name, mock_value): + """ + Test DynamoDBProvider.get() with a cached but expired value + """ + + table_name = "TEST_TABLE" + + # Create a new provider + provider = parameters.DynamoDBProvider(table_name, region="us-east-1") + + # Inject value in the internal store + provider.store[(mock_name, None)] = ExpirableValue(mock_value, datetime.now() - timedelta(seconds=60)) + + # Stub the boto3 client + stubber = stub.Stubber(provider.table.meta.client) + response = {"Item": {"id": {"S": mock_name}, "value": {"S": mock_value}}} + expected_params = {"TableName": table_name, "Key": {"id": mock_name}} + stubber.add_response("get_item", response, expected_params) + stubber.activate() + + try: + value = provider.get(mock_name) + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + +def test_dynamodb_provider_get_multiple(mock_name, mock_value): + """ + Test DynamoDBProvider.get_multiple() with a non-cached path + """ + + mock_param_names = ["A", "B", "C"] + table_name = "TEST_TABLE" + + # Create a new provider + provider = parameters.DynamoDBProvider(table_name, region="us-east-1") + + # Stub the boto3 client + stubber = stub.Stubber(provider.table.meta.client) + response = { + "Items": [ + {"id": {"S": mock_name}, "sk": {"S": name}, "value": {"S": f"{mock_value}/{name}"}} + for name in mock_param_names + ] + } + expected_params = {"TableName": table_name, "KeyConditionExpression": Key("id").eq(mock_name)} + stubber.add_response("query", response, expected_params) + stubber.activate() + + try: + values = provider.get_multiple(mock_name) + + stubber.assert_no_pending_responses() + + assert len(values) == len(mock_param_names) + for name in mock_param_names: + assert name in values + assert values[name] == f"{mock_value}/{name}" + finally: + stubber.deactivate() + + +def test_ssm_provider_get(mock_name, mock_value, mock_version): """ Test SSMProvider.get() with a non-cached value """ @@ -48,7 +169,7 @@ def test_ssm_provider_get(monkeypatch, mock_name, mock_value, mock_version): "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{mock_name}", } } - expected_params = {"Name": mock_name} + expected_params = {"Name": mock_name, "WithDecryption": False} stubber.add_response("get_parameter", response, expected_params) stubber.activate() @@ -61,7 +182,7 @@ def test_ssm_provider_get(monkeypatch, mock_name, mock_value, mock_version): stubber.deactivate() -def test_ssm_provider_get_cached(monkeypatch, mock_name, mock_value, mock_version): +def test_ssm_provider_get_cached(mock_name, mock_value): """ Test SSMProvider.get() with a cached value """ @@ -85,7 +206,7 @@ def test_ssm_provider_get_cached(monkeypatch, mock_name, mock_value, mock_versio stubber.deactivate() -def test_ssm_provider_get_expired(monkeypatch, mock_name, mock_value, mock_version): +def test_ssm_provider_get_expired(mock_name, mock_value, mock_version): """ Test SSMProvider.get() with a cached but expired value """ @@ -110,7 +231,7 @@ def test_ssm_provider_get_expired(monkeypatch, mock_name, mock_value, mock_versi "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{mock_name}", } } - expected_params = {"Name": mock_name} + expected_params = {"Name": mock_name, "WithDecryption": False} stubber.add_response("get_parameter", response, expected_params) stubber.activate() @@ -123,7 +244,117 @@ def test_ssm_provider_get_expired(monkeypatch, mock_name, mock_value, mock_versi stubber.deactivate() -def test_secrets_provider_get(monkeypatch, mock_name, mock_value): +def test_ssm_provider_get_multiple(mock_name, mock_value, mock_version): + """ + Test SSMProvider.get_multiple() with a non-cached path + """ + + mock_param_names = ["A", "B", "C"] + + # Create a new provider + provider = parameters.SSMProvider(region="us-east-1") + + # Stub the boto3 client + stubber = stub.Stubber(provider.client) + response = { + "Parameters": [ + { + "Name": f"{mock_name}/{name}", + "Type": "String", + "Value": f"{mock_value}/{name}", + "Version": mock_version, + "Selector": f"{mock_name}/{name}:{mock_version}", + "SourceResult": "string", + "LastModifiedDate": datetime(2015, 1, 1), + "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{mock_name}/{name}", + } + for name in mock_param_names + ] + } + expected_params = {"Path": mock_name, "Recursive": False, "WithDecryption": False} + stubber.add_response("get_parameters_by_path", response, expected_params) + stubber.activate() + + try: + values = provider.get_multiple(mock_name) + + stubber.assert_no_pending_responses() + + assert len(values) == len(mock_param_names) + for name in mock_param_names: + assert name in values + assert values[name] == f"{mock_value}/{name}" + finally: + stubber.deactivate() + + +def test_ssm_provider_get_multiple_next_token(mock_name, mock_value, mock_version): + """ + Test SSMProvider.get_multiple() with a non-cached path with multiple calls + """ + + mock_param_names = ["A", "B", "C"] + + # Create a new provider + provider = parameters.SSMProvider(region="us-east-1") + + # Stub the boto3 client + stubber = stub.Stubber(provider.client) + + # First call + response = { + "Parameters": [ + { + "Name": f"{mock_name}/{name}", + "Type": "String", + "Value": f"{mock_value}/{name}", + "Version": mock_version, + "Selector": f"{mock_name}/{name}:{mock_version}", + "SourceResult": "string", + "LastModifiedDate": datetime(2015, 1, 1), + "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{mock_name}/{name}", + } + for name in mock_param_names[:1] + ], + "NextToken": "next_token", + } + expected_params = {"Path": mock_name, "Recursive": False, "WithDecryption": False} + stubber.add_response("get_parameters_by_path", response, expected_params) + + # Second call + response = { + "Parameters": [ + { + "Name": f"{mock_name}/{name}", + "Type": "String", + "Value": f"{mock_value}/{name}", + "Version": mock_version, + "Selector": f"{mock_name}/{name}:{mock_version}", + "SourceResult": "string", + "LastModifiedDate": datetime(2015, 1, 1), + "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{mock_name}/{name}", + } + for name in mock_param_names[1:] + ] + } + expected_params = {"Path": mock_name, "Recursive": False, "WithDecryption": False, "NextToken": "next_token"} + stubber.add_response("get_parameters_by_path", response, expected_params) + stubber.activate() + + try: + values = provider.get_multiple(mock_name) + + stubber.assert_no_pending_responses() + + assert len(values) == len(mock_param_names) + for name in mock_param_names: + assert name in values + assert values[name] == f"{mock_value}/{name}" + finally: + stubber.deactivate() + + +def test_secrets_provider_get(mock_name, mock_value): """ Test SecretsProvider.get() with a non-cached value """ @@ -153,7 +384,7 @@ def test_secrets_provider_get(monkeypatch, mock_name, mock_value): stubber.deactivate() -def test_secrets_provider_get_cached(monkeypatch, mock_name, mock_value, mock_version): +def test_secrets_provider_get_cached(mock_name, mock_value): """ Test SecretsProvider.get() with a cached value """ @@ -177,7 +408,7 @@ def test_secrets_provider_get_cached(monkeypatch, mock_name, mock_value, mock_ve stubber.deactivate() -def test_secrets_provider_get_expired(monkeypatch, mock_name, mock_value): +def test_secrets_provider_get_expired(mock_name, mock_value): """ Test SecretsProvider.get() with a cached but expired value """ @@ -208,3 +439,51 @@ def test_secrets_provider_get_expired(monkeypatch, mock_name, mock_value): stubber.assert_no_pending_responses() finally: stubber.deactivate() + + +def test_base_provider_get_transform_json(mock_name, mock_value): + """ + Test BaseProvider.get() with a json transform + """ + + mock_data = json.dumps({mock_name: mock_value}) + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + assert name == mock_name + return mock_data + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + raise NotImplementedError() + + provider = TestProvider() + + value = provider.get(mock_name, transform="json") + + assert isinstance(value, dict) + assert mock_name in value + assert value[mock_name] == mock_value + + +def test_base_provider_get_transform_binary(mock_name, mock_value): + """ + Test BaseProvider.get() with a bianry transform + """ + + mock_binary = mock_value.encode() + mock_data = base64.b64encode(mock_binary).decode() + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + assert name == mock_name + return mock_data + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + raise NotImplementedError() + + provider = TestProvider() + + value = provider.get(mock_name, transform="binary") + + assert isinstance(value, bytes) + assert value == mock_binary From d1c57ef84b1c764dadc78ec712ab71c0720e618f Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Thu, 30 Jul 2020 16:07:48 +0200 Subject: [PATCH 08/30] tests: increase test coverage --- .../utilities/parameters/base.py | 4 +- tests/functional/test_utilities_parameters.py | 108 ++++++++++++++++++ 2 files changed, 110 insertions(+), 2 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/base.py b/aws_lambda_powertools/utilities/parameters/base.py index 33342eb3f84..6bf983ab048 100644 --- a/aws_lambda_powertools/utilities/parameters/base.py +++ b/aws_lambda_powertools/utilities/parameters/base.py @@ -108,9 +108,9 @@ def get_multiple( raise GetParameterError(str(exc)) if transform == "json": - values = {k: json.loads(v) for k, v in values} + values = {k: json.loads(v) for k, v in values.items()} elif transform == "binary": - values = {k: base64.b64decode(v) for k, v in values} + values = {k: base64.b64decode(v) for k, v in values.items()} self.store[key] = ExpirableValue(values, datetime.now() + timedelta(seconds=max_age),) diff --git a/tests/functional/test_utilities_parameters.py b/tests/functional/test_utilities_parameters.py index 2e183d82cfe..3d87656ccd9 100644 --- a/tests/functional/test_utilities_parameters.py +++ b/tests/functional/test_utilities_parameters.py @@ -487,3 +487,111 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: assert isinstance(value, bytes) assert value == mock_binary + + +def test_base_provider_get_multiple_transform_json(mock_name, mock_value): + """ + Test BaseProvider.get_multiple() with a json transform + """ + + mock_data = json.dumps({mock_name: mock_value}) + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + raise NotImplementedError() + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + assert path == mock_name + return {"A": mock_data} + + provider = TestProvider() + + value = provider.get_multiple(mock_name, transform="json") + + assert isinstance(value, dict) + assert value["A"][mock_name] == mock_value + + +def test_base_provider_get_multiple_transform_binary(mock_name, mock_value): + """ + Test BaseProvider.get_multiple() with a binary transform + """ + + mock_binary = mock_value.encode() + mock_data = base64.b64encode(mock_binary).decode() + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + raise NotImplementedError() + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + assert path == mock_name + return {"A": mock_data} + + provider = TestProvider() + + value = provider.get_multiple(mock_name, transform="binary") + + assert isinstance(value, dict) + assert value["A"] == mock_binary + + +def test_get_parameter(monkeypatch, mock_name, mock_value): + """ + Test get_parameter() + """ + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + assert name == mock_name + return mock_value + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + raise NotImplementedError() + + monkeypatch.setitem(parameters._DEFAULT_PROVIDERS, "ssm", TestProvider()) + + value = parameters.get_parameter(mock_name) + + assert value == mock_value + + +def test_get_parameters(monkeypatch, mock_name, mock_value): + """ + Test get_parameters() + """ + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + raise NotImplementedError() + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + assert path == mock_name + return {"A": mock_value} + + monkeypatch.setitem(parameters._DEFAULT_PROVIDERS, "ssm", TestProvider()) + + values = parameters.get_parameters(mock_name) + + assert len(values) == 1 + assert values["A"] == mock_value + + +def test_get_secret(monkeypatch, mock_name, mock_value): + """ + Test get_secret() + """ + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + assert name == mock_name + return mock_value + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + raise NotImplementedError() + + monkeypatch.setitem(parameters._DEFAULT_PROVIDERS, "secrets", TestProvider()) + + value = parameters.get_secret(mock_name) + + assert value == mock_value From 285ac9560eb83f60c862565a2c8e747ff0a5144f Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Thu, 30 Jul 2020 18:15:01 +0200 Subject: [PATCH 09/30] tests: increase test coverage (2) --- .../utilities/parameters/__init__.py | 32 ++-- tests/functional/test_utilities_parameters.py | 160 +++++++++++++++++- 2 files changed, 180 insertions(+), 12 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/__init__.py b/aws_lambda_powertools/utilities/parameters/__init__.py index c4d0a629f20..67e080d11cb 100644 --- a/aws_lambda_powertools/utilities/parameters/__init__.py +++ b/aws_lambda_powertools/utilities/parameters/__init__.py @@ -9,9 +9,17 @@ import boto3 from boto3.dynamodb.conditions import Key -from .base import BaseProvider +from .base import BaseProvider, GetParameterError -__all__ = ["BaseProvider", "DynamoDBProvider", "SecretsProvider", "SSMProvider", "get_parameter", "get_secret"] +__all__ = [ + "BaseProvider", + "GetParameterError", + "DynamoDBProvider", + "SecretsProvider", + "SSMProvider", + "get_parameter", + "get_secret", +] class SSMProvider(BaseProvider): @@ -28,10 +36,11 @@ def __init__( Initialize the SSM Parameter Store client """ + client_kwargs = {} if region: - self.client = boto3.client("ssm", region_name=region) - else: - self.client = boto3.client("ssm") + client_kwargs["region_name"] = region + + self.client = boto3.client("ssm", **client_kwargs) super().__init__() @@ -109,10 +118,11 @@ def __init__(self, region: Optional[str] = None): Initialize the Secrets Manager client """ + client_kwargs = {} if region: - self.client = boto3.client("secretsmanager", region_name=region) - else: - self.client = boto3.client("secretsmanager") + client_kwargs["region_name"] = region + + self.client = boto3.client("secretsmanager", **client_kwargs) super().__init__() @@ -146,10 +156,10 @@ def __init__( Initialize the DynamoDB client """ + client_kwargs = {} if region: - self.table = boto3.resource("dynamodb", region_name=region).Table(table_name) - else: - self.table = boto3.resource("dynamodb").Table(table_name) + client_kwargs["region_name"] = region + self.table = boto3.resource("dynamodb", **client_kwargs).Table(table_name) self.key_attr = key_attr self.value_attr = value_attr diff --git a/tests/functional/test_utilities_parameters.py b/tests/functional/test_utilities_parameters.py index 3d87656ccd9..e715ebd14cb 100644 --- a/tests/functional/test_utilities_parameters.py +++ b/tests/functional/test_utilities_parameters.py @@ -147,6 +147,59 @@ def test_dynamodb_provider_get_multiple(mock_name, mock_value): stubber.deactivate() +def test_dynamodb_provider_get_multiple_next_token(mock_name, mock_value): + """ + Test DynamoDBProvider.get_multiple() with a non-cached path + """ + + mock_param_names = ["A", "B", "C"] + table_name = "TEST_TABLE" + + # Create a new provider + provider = parameters.DynamoDBProvider(table_name, region="us-east-1") + + # Stub the boto3 client + stubber = stub.Stubber(provider.table.meta.client) + + # First call + response = { + "Items": [ + {"id": {"S": mock_name}, "sk": {"S": name}, "value": {"S": f"{mock_value}/{name}"}} + for name in mock_param_names[:1] + ], + "LastEvaluatedKey": {"id": {"S": mock_name}, "sk": {"S": mock_param_names[0]}}, + } + expected_params = {"TableName": table_name, "KeyConditionExpression": Key("id").eq(mock_name)} + stubber.add_response("query", response, expected_params) + + # Second call + response = { + "Items": [ + {"id": {"S": mock_name}, "sk": {"S": name}, "value": {"S": f"{mock_value}/{name}"}} + for name in mock_param_names[1:] + ] + } + expected_params = { + "TableName": table_name, + "KeyConditionExpression": Key("id").eq(mock_name), + "ExclusiveStartKey": {"id": mock_name, "sk": mock_param_names[0]}, + } + stubber.add_response("query", response, expected_params) + stubber.activate() + + try: + values = provider.get_multiple(mock_name) + + stubber.assert_no_pending_responses() + + assert len(values) == len(mock_param_names) + for name in mock_param_names: + assert name in values + assert values[name] == f"{mock_value}/{name}" + finally: + stubber.deactivate() + + def test_ssm_provider_get(mock_name, mock_value, mock_version): """ Test SSMProvider.get() with a non-cached value @@ -465,9 +518,51 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: assert value[mock_name] == mock_value +def test_base_provider_get_exception(mock_name): + """ + Test BaseProvider.get() that raises an exception + """ + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + assert name == mock_name + raise Exception("test exception raised") + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + raise NotImplementedError() + + provider = TestProvider() + + with pytest.raises(parameters.GetParameterError) as excinfo: + provider.get(mock_name) + + assert "test exception raised" in str(excinfo) + + +def test_base_provider_get_multiple_exception(mock_name): + """ + Test BaseProvider.get_multiple() that raises an exception + """ + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + raise NotImplementedError() + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + assert path == mock_name + raise Exception("test exception raised") + + provider = TestProvider() + + with pytest.raises(parameters.GetParameterError) as excinfo: + provider.get_multiple(mock_name) + + assert "test exception raised" in str(excinfo) + + def test_base_provider_get_transform_binary(mock_name, mock_value): """ - Test BaseProvider.get() with a bianry transform + Test BaseProvider.get() with a binary transform """ mock_binary = mock_value.encode() @@ -556,6 +651,27 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: assert value == mock_value +def test_get_parameter_new(monkeypatch, mock_name, mock_value): + """ + Test get_parameter() without a default provider + """ + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + assert name == mock_name + return mock_value + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + raise NotImplementedError() + + monkeypatch.setattr(parameters, "_DEFAULT_PROVIDERS", {}) + monkeypatch.setattr(parameters, "SSMProvider", TestProvider) + + value = parameters.get_parameter(mock_name) + + assert value == mock_value + + def test_get_parameters(monkeypatch, mock_name, mock_value): """ Test get_parameters() @@ -577,6 +693,27 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: assert values["A"] == mock_value +def test_get_parameters_new(monkeypatch, mock_name, mock_value): + """ + Test get_parameters() without a default provider + """ + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + raise NotImplementedError() + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + assert path == mock_name + return mock_value + + monkeypatch.setattr(parameters, "_DEFAULT_PROVIDERS", {}) + monkeypatch.setattr(parameters, "SSMProvider", TestProvider) + + value = parameters.get_parameters(mock_name) + + assert value == mock_value + + def test_get_secret(monkeypatch, mock_name, mock_value): """ Test get_secret() @@ -595,3 +732,24 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: value = parameters.get_secret(mock_name) assert value == mock_value + + +def test_get_secret_new(monkeypatch, mock_name, mock_value): + """ + Test get_secret() without a default provider + """ + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + assert name == mock_name + return mock_value + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + raise NotImplementedError() + + monkeypatch.setattr(parameters, "_DEFAULT_PROVIDERS", {}) + monkeypatch.setattr(parameters, "SecretsProvider", TestProvider) + + value = parameters.get_secret(mock_name) + + assert value == mock_value From 4c762765568ff0695ad7644cb19b73fb0ae029ea Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Thu, 30 Jul 2020 18:40:14 +0200 Subject: [PATCH 10/30] tests: increase coverage to 100% --- tests/functional/test_utilities_parameters.py | 186 ++++++++++++++++++ 1 file changed, 186 insertions(+) diff --git a/tests/functional/test_utilities_parameters.py b/tests/functional/test_utilities_parameters.py index e715ebd14cb..c0ab8232fd5 100644 --- a/tests/functional/test_utilities_parameters.py +++ b/tests/functional/test_utilities_parameters.py @@ -56,6 +56,34 @@ def test_dynamodb_provider_get(mock_name, mock_value): stubber.deactivate() +def test_dynamodb_provider_get_default_region(monkeypatch, mock_name, mock_value): + """ + Test DynamoDBProvider.get() without setting a region + """ + + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + + table_name = "TEST_TABLE" + + # Create a new provider + provider = parameters.DynamoDBProvider(table_name) + + # Stub the boto3 client + stubber = stub.Stubber(provider.table.meta.client) + response = {"Item": {"id": {"S": mock_name}, "value": {"S": mock_value}}} + expected_params = {"TableName": table_name, "Key": {"id": mock_name}} + stubber.add_response("get_item", response, expected_params) + stubber.activate() + + try: + value = provider.get(mock_name) + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + def test_dynamodb_provider_get_cached(mock_name, mock_value): """ Test DynamoDBProvider.get() with a cached value @@ -235,6 +263,43 @@ def test_ssm_provider_get(mock_name, mock_value, mock_version): stubber.deactivate() +def test_ssm_provider_get_default_region(monkeypatch, mock_name, mock_value, mock_version): + """ + Test SSMProvider.get() without specifying the region + """ + + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + + # Create a new provider + provider = parameters.SSMProvider() + + # Stub the boto3 client + stubber = stub.Stubber(provider.client) + response = { + "Parameter": { + "Name": mock_name, + "Type": "String", + "Value": mock_value, + "Version": mock_version, + "Selector": f"{mock_name}:{mock_version}", + "SourceResult": "string", + "LastModifiedDate": datetime(2015, 1, 1), + "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{mock_name}", + } + } + expected_params = {"Name": mock_name, "WithDecryption": False} + stubber.add_response("get_parameter", response, expected_params) + stubber.activate() + + try: + value = provider.get(mock_name) + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + def test_ssm_provider_get_cached(mock_name, mock_value): """ Test SSMProvider.get() with a cached value @@ -341,6 +406,50 @@ def test_ssm_provider_get_multiple(mock_name, mock_value, mock_version): stubber.deactivate() +def test_ssm_provider_get_multiple_different_path(mock_name, mock_value, mock_version): + """ + Test SSMProvider.get_multiple() with a non-cached path and names that don't start with the path + """ + + mock_param_names = ["A", "B", "C"] + + # Create a new provider + provider = parameters.SSMProvider(region="us-east-1") + + # Stub the boto3 client + stubber = stub.Stubber(provider.client) + response = { + "Parameters": [ + { + "Name": f"{name}", + "Type": "String", + "Value": f"{mock_value}/{name}", + "Version": mock_version, + "Selector": f"{mock_name}/{name}:{mock_version}", + "SourceResult": "string", + "LastModifiedDate": datetime(2015, 1, 1), + "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{mock_name}/{name}", + } + for name in mock_param_names + ] + } + expected_params = {"Path": mock_name, "Recursive": False, "WithDecryption": False} + stubber.add_response("get_parameters_by_path", response, expected_params) + stubber.activate() + + try: + values = provider.get_multiple(mock_name) + + stubber.assert_no_pending_responses() + + assert len(values) == len(mock_param_names) + for name in mock_param_names: + assert name in values + assert values[name] == f"{mock_value}/{name}" + finally: + stubber.deactivate() + + def test_ssm_provider_get_multiple_next_token(mock_name, mock_value, mock_version): """ Test SSMProvider.get_multiple() with a non-cached path with multiple calls @@ -437,6 +546,38 @@ def test_secrets_provider_get(mock_name, mock_value): stubber.deactivate() +def test_secrets_provider_get_default_region(monkeypatch, mock_name, mock_value): + """ + Test SecretsProvider.get() without specifying a region + """ + + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + + # Create a new provider + provider = parameters.SecretsProvider() + + # Stub the boto3 client + stubber = stub.Stubber(provider.client) + response = { + "ARN": f"arn:aws:secretsmanager:us-east-1:132456789012:secret/{mock_name}", + "Name": mock_name, + "VersionId": "7a9155b8-2dc9-466e-b4f6-5bc46516c84d", + "SecretString": mock_value, + "CreatedDate": datetime(2015, 1, 1), + } + expected_params = {"SecretId": mock_name} + stubber.add_response("get_secret_value", response, expected_params) + stubber.activate() + + try: + value = provider.get(mock_name) + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + def test_secrets_provider_get_cached(mock_name, mock_value): """ Test SecretsProvider.get() with a cached value @@ -631,6 +772,51 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: assert value["A"] == mock_binary +def test_base_provider_get_multiple_cached(mock_name, mock_value): + """ + Test BaseProvider.get_multiple() with cached values + """ + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + raise NotImplementedError() + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + raise NotImplementedError() + + provider = TestProvider() + + provider.store[(mock_name, None)] = ExpirableValue({"A": mock_value}, datetime.now() + timedelta(seconds=60)) + + value = provider.get_multiple(mock_name) + + assert isinstance(value, dict) + assert value["A"] == mock_value + + +def test_base_provider_get_multiple_expired(mock_name, mock_value): + """ + Test BaseProvider.get_multiple() with expired values + """ + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + raise NotImplementedError() + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + assert path == mock_name + return {"A": mock_value} + + provider = TestProvider() + + provider.store[(mock_name, None)] = ExpirableValue({"B": mock_value}, datetime.now() - timedelta(seconds=60)) + + value = provider.get_multiple(mock_name) + + assert isinstance(value, dict) + assert value["A"] == mock_value + + def test_get_parameter(monkeypatch, mock_name, mock_value): """ Test get_parameter() From 6eb012f1c96251b267cd7a4f6f4f1b51a388b8a6 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Thu, 30 Jul 2020 20:52:38 +0200 Subject: [PATCH 11/30] fix: add get_parameters in __all__ --- aws_lambda_powertools/utilities/parameters/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aws_lambda_powertools/utilities/parameters/__init__.py b/aws_lambda_powertools/utilities/parameters/__init__.py index 67e080d11cb..3488b050903 100644 --- a/aws_lambda_powertools/utilities/parameters/__init__.py +++ b/aws_lambda_powertools/utilities/parameters/__init__.py @@ -18,6 +18,7 @@ "SecretsProvider", "SSMProvider", "get_parameter", + "get_parameters", "get_secret", ] From e6416c0c818e34028f52432459265277425c5bef Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Mon, 17 Aug 2020 19:00:25 +0200 Subject: [PATCH 12/30] chore: split parameter utilities into smaller files --- .../utilities/parameters/__init__.py | 235 +----------------- .../utilities/parameters/base.py | 8 +- .../utilities/parameters/dynamodb.py | 75 ++++++ .../utilities/parameters/exceptions.py | 7 + .../utilities/parameters/secrets.py | 56 +++++ .../utilities/parameters/ssm.py | 120 +++++++++ tests/functional/test_utilities_parameters.py | 18 +- 7 files changed, 276 insertions(+), 243 deletions(-) create mode 100644 aws_lambda_powertools/utilities/parameters/dynamodb.py create mode 100644 aws_lambda_powertools/utilities/parameters/exceptions.py create mode 100644 aws_lambda_powertools/utilities/parameters/secrets.py create mode 100644 aws_lambda_powertools/utilities/parameters/ssm.py diff --git a/aws_lambda_powertools/utilities/parameters/__init__.py b/aws_lambda_powertools/utilities/parameters/__init__.py index 3488b050903..3c18301a1dd 100644 --- a/aws_lambda_powertools/utilities/parameters/__init__.py +++ b/aws_lambda_powertools/utilities/parameters/__init__.py @@ -4,12 +4,11 @@ Parameter retrieval and caching utility """ -from typing import Dict, Optional, Union - -import boto3 -from boto3.dynamodb.conditions import Key - -from .base import BaseProvider, GetParameterError +from .base import BaseProvider +from .dynamodb import DynamoDBProvider +from .exceptions import GetParameterError +from .secrets import SecretsProvider, get_secret +from .ssm import SSMProvider, get_parameter, get_parameters __all__ = [ "BaseProvider", @@ -21,227 +20,3 @@ "get_parameters", "get_secret", ] - - -class SSMProvider(BaseProvider): - """ - AWS Systems Manager Parameter Store Provider - """ - - client = None - - def __init__( - self, region: Optional[str] = None, - ): - """ - Initialize the SSM Parameter Store client - """ - - client_kwargs = {} - if region: - client_kwargs["region_name"] = region - - self.client = boto3.client("ssm", **client_kwargs) - - super().__init__() - - def _get(self, name: str, **kwargs) -> str: - """ - Retrieve a parameter value from AWS Systems Manager Parameter Store - - Parameters - ---------- - name: str - Parameter name - decrypt: bool - If the parameter value should be decrypted - """ - - # Load kwargs - decrypt = kwargs.get("decrypt", False) - - return self.client.get_parameter(Name=name, WithDecryption=decrypt)["Parameter"]["Value"] - - def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: - """ - Retrieve multiple parameter values from AWS Systems Manager Parameter Store - - Parameters - ---------- - path: str - Path to retrieve the parameters - decrypt: bool - If the parameter values should be decrypted - recursive: bool - If this should retrieve the parameter values recursively or not - """ - - # Load kwargs - decrypt = kwargs.get("decrypt", False) - recursive = kwargs.get("recursive", False) - - response = self.client.get_parameters_by_path(Path=path, WithDecryption=decrypt, Recursive=recursive) - parameters = response.get("Parameters", []) - - # Keep retrieving parameters - while "NextToken" in response: - response = self.client.get_parameters_by_path( - Path=path, WithDecryption=decrypt, Recursive=recursive, NextToken=response["NextToken"] - ) - parameters.extend(response.get("Parameters", [])) - - retval = {} - for parameter in parameters: - - # Standardize the parameter name - # The parameter name returned by SSM will contained the full path. - # However, for readability, we should return only the part after - # the path. - name = parameter["Name"] - if name.startswith(path): - name = name[len(path) :] - name = name.lstrip("/") - - retval[name] = parameter["Value"] - - return retval - - -class SecretsProvider(BaseProvider): - """ - AWS Secrets Manager Parameter Provider - """ - - client = None - - def __init__(self, region: Optional[str] = None): - """ - Initialize the Secrets Manager client - """ - - client_kwargs = {} - if region: - client_kwargs["region_name"] = region - - self.client = boto3.client("secretsmanager", **client_kwargs) - - super().__init__() - - def _get(self, name: str, **kwargs) -> str: - """ - Retrieve a parameter value from AWS Systems Manager Parameter Store - """ - - return self.client.get_secret_value(SecretId=name)["SecretString"] - - def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: - """ - Retrieving multiple parameter values is not supported with AWS Secrets Manager - """ - raise NotImplementedError() - - -class DynamoDBProvider(BaseProvider): - """ - Amazon DynamoDB Parameter Provider - """ - - table = None - key_attr = None - value_attr = None - - def __init__( - self, table_name: str, key_attr: str = "id", value_attr: str = "value", region: Optional[str] = None, - ): - """ - Initialize the DynamoDB client - """ - - client_kwargs = {} - if region: - client_kwargs["region_name"] = region - self.table = boto3.resource("dynamodb", **client_kwargs).Table(table_name) - - self.key_attr = key_attr - self.value_attr = value_attr - - super().__init__() - - def _get(self, name: str, **kwargs) -> str: - """ - Retrieve a parameter value from Amazon DynamoDB - """ - - return self.table.get_item(Key={self.key_attr: name})["Item"][self.value_attr] - - def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: - """ - Retrieve multiple parameter values from Amazon DynamoDB - - Parameters - ---------- - path: str - Path to retrieve the parameters - sort_attr: str - Name of the DynamoDB table sort key (defaults to 'sk') - """ - - sort_attr = kwargs.get("sort_attr", "sk") - - response = self.table.query(KeyConditionExpression=Key(self.key_attr).eq(path)) - items = response.get("Items", []) - - # Keep querying while there are more items matching the partition key - while "LastEvaluatedKey" in response: - response = self.table.query( - KeyConditionExpression=Key(self.key_attr).eq(path), ExclusiveStartKey=response["LastEvaluatedKey"], - ) - items.extend(response.get("Items", [])) - - retval = {} - for item in items: - retval[item[sort_attr]] = item[self.value_attr] - - return retval - - -# These providers will be dynamically initialized on first use of the helper functions -_DEFAULT_PROVIDERS = {} - - -def get_parameter(name: str, transform: Optional[str] = None) -> Union[str, list, dict, bytes]: - """ - Retrieve a parameter value from AWS Systems Manager (SSM) Parameter Store - """ - - # Only create the provider if this function is called at least once - if "ssm" not in _DEFAULT_PROVIDERS: - _DEFAULT_PROVIDERS["ssm"] = SSMProvider() - - return _DEFAULT_PROVIDERS["ssm"].get(name, transform=transform) - - -def get_parameters( - path: str, transform: Optional[str] = None, recursive: bool = False, decrypt: bool = False -) -> Union[Dict[str, str], Dict[str, dict], Dict[str, bytes]]: - """ - Retrieve multiple parameter values from AWS Systems Manager (SSM) Parameter Store - """ - - # Only create the provider if this function is called at least once - if "ssm" not in _DEFAULT_PROVIDERS: - _DEFAULT_PROVIDERS["ssm"] = SSMProvider() - - return _DEFAULT_PROVIDERS["ssm"].get_multiple(path, transform=transform, recursive=recursive, decrypt=decrypt) - - -def get_secret(name: str, transform: Optional[str] = None, decrypt: bool = False) -> Union[str, dict, bytes]: - """ - Retrieve a parameter value from AWS Secrets Manager - """ - - # Only create the provider if this function is called at least once - if "secrets" not in _DEFAULT_PROVIDERS: - _DEFAULT_PROVIDERS["secrets"] = SecretsProvider() - - return _DEFAULT_PROVIDERS["secrets"].get(name, transform=transform, decrypt=decrypt) diff --git a/aws_lambda_powertools/utilities/parameters/base.py b/aws_lambda_powertools/utilities/parameters/base.py index 6bf983ab048..0ce24cce857 100644 --- a/aws_lambda_powertools/utilities/parameters/base.py +++ b/aws_lambda_powertools/utilities/parameters/base.py @@ -9,12 +9,12 @@ from datetime import datetime, timedelta from typing import Dict, Optional, Union +from .exceptions import GetParameterError + DEFAULT_MAX_AGE_SECS = 5 ExpirableValue = namedtuple("ExpirableValue", ["value", "ttl"]) - - -class GetParameterError(Exception): - """When a provider raises an exception on parameter retrieval""" +# These providers will be dynamically initialized on first use of the helper functions +DEFAULT_PROVIDERS = {} class BaseProvider(ABC): diff --git a/aws_lambda_powertools/utilities/parameters/dynamodb.py b/aws_lambda_powertools/utilities/parameters/dynamodb.py new file mode 100644 index 00000000000..8ed2d0f535a --- /dev/null +++ b/aws_lambda_powertools/utilities/parameters/dynamodb.py @@ -0,0 +1,75 @@ +""" +Amazon DynamoDB parameter retrieval and caching utility +""" + + +from typing import Dict, Optional + +import boto3 +from boto3.dynamodb.conditions import Key + +from .base import BaseProvider + + +class DynamoDBProvider(BaseProvider): + """ + Amazon DynamoDB Parameter Provider + """ + + table = None + key_attr = None + value_attr = None + + def __init__( + self, table_name: str, key_attr: str = "id", value_attr: str = "value", region: Optional[str] = None, + ): + """ + Initialize the DynamoDB client + """ + + client_kwargs = {} + if region: + client_kwargs["region_name"] = region + self.table = boto3.resource("dynamodb", **client_kwargs).Table(table_name) + + self.key_attr = key_attr + self.value_attr = value_attr + + super().__init__() + + def _get(self, name: str, **kwargs) -> str: + """ + Retrieve a parameter value from Amazon DynamoDB + """ + + return self.table.get_item(Key={self.key_attr: name})["Item"][self.value_attr] + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + """ + Retrieve multiple parameter values from Amazon DynamoDB + + Parameters + ---------- + path: str + Path to retrieve the parameters + sort_attr: str + Name of the DynamoDB table sort key (defaults to 'sk') + """ + + sort_attr = kwargs.get("sort_attr", "sk") + + response = self.table.query(KeyConditionExpression=Key(self.key_attr).eq(path)) + items = response.get("Items", []) + + # Keep querying while there are more items matching the partition key + while "LastEvaluatedKey" in response: + response = self.table.query( + KeyConditionExpression=Key(self.key_attr).eq(path), ExclusiveStartKey=response["LastEvaluatedKey"], + ) + items.extend(response.get("Items", [])) + + retval = {} + for item in items: + retval[item[sort_attr]] = item[self.value_attr] + + return retval diff --git a/aws_lambda_powertools/utilities/parameters/exceptions.py b/aws_lambda_powertools/utilities/parameters/exceptions.py new file mode 100644 index 00000000000..683ad319b09 --- /dev/null +++ b/aws_lambda_powertools/utilities/parameters/exceptions.py @@ -0,0 +1,7 @@ +""" +Parameter retrieval exceptions +""" + + +class GetParameterError(Exception): + """When a provider raises an exception on parameter retrieval""" diff --git a/aws_lambda_powertools/utilities/parameters/secrets.py b/aws_lambda_powertools/utilities/parameters/secrets.py new file mode 100644 index 00000000000..63732b219d0 --- /dev/null +++ b/aws_lambda_powertools/utilities/parameters/secrets.py @@ -0,0 +1,56 @@ +""" +AWS Secrets Manager parameter retrieval and caching utility +""" + + +from typing import Dict, Optional, Union + +import boto3 + +from .base import DEFAULT_PROVIDERS, BaseProvider + + +class SecretsProvider(BaseProvider): + """ + AWS Secrets Manager Parameter Provider + """ + + client = None + + def __init__(self, region: Optional[str] = None): + """ + Initialize the Secrets Manager client + """ + + client_kwargs = {} + if region: + client_kwargs["region_name"] = region + + self.client = boto3.client("secretsmanager", **client_kwargs) + + super().__init__() + + def _get(self, name: str, **kwargs) -> str: + """ + Retrieve a parameter value from AWS Systems Manager Parameter Store + """ + + return self.client.get_secret_value(SecretId=name)["SecretString"] + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + """ + Retrieving multiple parameter values is not supported with AWS Secrets Manager + """ + raise NotImplementedError() + + +def get_secret(name: str, transform: Optional[str] = None, decrypt: bool = False) -> Union[str, dict, bytes]: + """ + Retrieve a parameter value from AWS Secrets Manager + """ + + # Only create the provider if this function is called at least once + if "secrets" not in DEFAULT_PROVIDERS: + DEFAULT_PROVIDERS["secrets"] = SecretsProvider() + + return DEFAULT_PROVIDERS["secrets"].get(name, transform=transform, decrypt=decrypt) diff --git a/aws_lambda_powertools/utilities/parameters/ssm.py b/aws_lambda_powertools/utilities/parameters/ssm.py new file mode 100644 index 00000000000..3811e6bcce4 --- /dev/null +++ b/aws_lambda_powertools/utilities/parameters/ssm.py @@ -0,0 +1,120 @@ +""" +AWS SSM Parameter retrieval and caching utility +""" + + +from typing import Dict, Optional, Union + +import boto3 + +from .base import DEFAULT_PROVIDERS, BaseProvider + + +class SSMProvider(BaseProvider): + """ + AWS Systems Manager Parameter Store Provider + """ + + client = None + + def __init__( + self, region: Optional[str] = None, + ): + """ + Initialize the SSM Parameter Store client + """ + + client_kwargs = {} + if region: + client_kwargs["region_name"] = region + + self.client = boto3.client("ssm", **client_kwargs) + + super().__init__() + + def _get(self, name: str, **kwargs) -> str: + """ + Retrieve a parameter value from AWS Systems Manager Parameter Store + + Parameters + ---------- + name: str + Parameter name + decrypt: bool + If the parameter value should be decrypted + """ + + # Load kwargs + decrypt = kwargs.get("decrypt", False) + + return self.client.get_parameter(Name=name, WithDecryption=decrypt)["Parameter"]["Value"] + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + """ + Retrieve multiple parameter values from AWS Systems Manager Parameter Store + + Parameters + ---------- + path: str + Path to retrieve the parameters + decrypt: bool + If the parameter values should be decrypted + recursive: bool + If this should retrieve the parameter values recursively or not + """ + + # Load kwargs + decrypt = kwargs.get("decrypt", False) + recursive = kwargs.get("recursive", False) + + response = self.client.get_parameters_by_path(Path=path, WithDecryption=decrypt, Recursive=recursive) + parameters = response.get("Parameters", []) + + # Keep retrieving parameters + while "NextToken" in response: + response = self.client.get_parameters_by_path( + Path=path, WithDecryption=decrypt, Recursive=recursive, NextToken=response["NextToken"] + ) + parameters.extend(response.get("Parameters", [])) + + retval = {} + for parameter in parameters: + + # Standardize the parameter name + # The parameter name returned by SSM will contained the full path. + # However, for readability, we should return only the part after + # the path. + name = parameter["Name"] + if name.startswith(path): + name = name[len(path) :] + name = name.lstrip("/") + + retval[name] = parameter["Value"] + + return retval + + +def get_parameter(name: str, transform: Optional[str] = None) -> Union[str, list, dict, bytes]: + """ + Retrieve a parameter value from AWS Systems Manager (SSM) Parameter Store + """ + + # Only create the provider if this function is called at least once + if "ssm" not in DEFAULT_PROVIDERS: + DEFAULT_PROVIDERS["ssm"] = SSMProvider() + + return DEFAULT_PROVIDERS["ssm"].get(name, transform=transform) + + +def get_parameters( + path: str, transform: Optional[str] = None, recursive: bool = False, decrypt: bool = False +) -> Union[Dict[str, str], Dict[str, dict], Dict[str, bytes]]: + """ + Retrieve multiple parameter values from AWS Systems Manager (SSM) Parameter Store + """ + + # Only create the provider if this function is called at least once + if "ssm" not in DEFAULT_PROVIDERS: + DEFAULT_PROVIDERS["ssm"] = SSMProvider() + + return DEFAULT_PROVIDERS["ssm"].get_multiple(path, transform=transform, recursive=recursive, decrypt=decrypt) diff --git a/tests/functional/test_utilities_parameters.py b/tests/functional/test_utilities_parameters.py index c0ab8232fd5..39978df4f64 100644 --- a/tests/functional/test_utilities_parameters.py +++ b/tests/functional/test_utilities_parameters.py @@ -830,7 +830,7 @@ def _get(self, name: str, **kwargs) -> str: def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: raise NotImplementedError() - monkeypatch.setitem(parameters._DEFAULT_PROVIDERS, "ssm", TestProvider()) + monkeypatch.setitem(parameters.base.DEFAULT_PROVIDERS, "ssm", TestProvider()) value = parameters.get_parameter(mock_name) @@ -850,8 +850,8 @@ def _get(self, name: str, **kwargs) -> str: def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: raise NotImplementedError() - monkeypatch.setattr(parameters, "_DEFAULT_PROVIDERS", {}) - monkeypatch.setattr(parameters, "SSMProvider", TestProvider) + monkeypatch.setattr(parameters.ssm, "DEFAULT_PROVIDERS", {}) + monkeypatch.setattr(parameters.ssm, "SSMProvider", TestProvider) value = parameters.get_parameter(mock_name) @@ -871,7 +871,7 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: assert path == mock_name return {"A": mock_value} - monkeypatch.setitem(parameters._DEFAULT_PROVIDERS, "ssm", TestProvider()) + monkeypatch.setitem(parameters.base.DEFAULT_PROVIDERS, "ssm", TestProvider()) values = parameters.get_parameters(mock_name) @@ -892,8 +892,8 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: assert path == mock_name return mock_value - monkeypatch.setattr(parameters, "_DEFAULT_PROVIDERS", {}) - monkeypatch.setattr(parameters, "SSMProvider", TestProvider) + monkeypatch.setattr(parameters.ssm, "DEFAULT_PROVIDERS", {}) + monkeypatch.setattr(parameters.ssm, "SSMProvider", TestProvider) value = parameters.get_parameters(mock_name) @@ -913,7 +913,7 @@ def _get(self, name: str, **kwargs) -> str: def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: raise NotImplementedError() - monkeypatch.setitem(parameters._DEFAULT_PROVIDERS, "secrets", TestProvider()) + monkeypatch.setitem(parameters.base.DEFAULT_PROVIDERS, "secrets", TestProvider()) value = parameters.get_secret(mock_name) @@ -933,8 +933,8 @@ def _get(self, name: str, **kwargs) -> str: def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: raise NotImplementedError() - monkeypatch.setattr(parameters, "_DEFAULT_PROVIDERS", {}) - monkeypatch.setattr(parameters, "SecretsProvider", TestProvider) + monkeypatch.setattr(parameters.secrets, "DEFAULT_PROVIDERS", {}) + monkeypatch.setattr(parameters.secrets, "SecretsProvider", TestProvider) value = parameters.get_secret(mock_name) From de842a86e3493679116fd11cdbbe4ca24f31c977 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Mon, 17 Aug 2020 19:16:43 +0200 Subject: [PATCH 13/30] feat: use botocore.config.Config for parameter providers --- .../utilities/parameters/dynamodb.py | 9 +-- .../utilities/parameters/secrets.py | 9 +-- .../utilities/parameters/ssm.py | 10 +-- tests/functional/test_utilities_parameters.py | 74 ++++++++++--------- 4 files changed, 52 insertions(+), 50 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/dynamodb.py b/aws_lambda_powertools/utilities/parameters/dynamodb.py index 8ed2d0f535a..6c15e3316a1 100644 --- a/aws_lambda_powertools/utilities/parameters/dynamodb.py +++ b/aws_lambda_powertools/utilities/parameters/dynamodb.py @@ -7,6 +7,7 @@ import boto3 from boto3.dynamodb.conditions import Key +from botocore.config import Config from .base import BaseProvider @@ -21,16 +22,14 @@ class DynamoDBProvider(BaseProvider): value_attr = None def __init__( - self, table_name: str, key_attr: str = "id", value_attr: str = "value", region: Optional[str] = None, + self, table_name: str, key_attr: str = "id", value_attr: str = "value", config: Optional[Config] = None, ): """ Initialize the DynamoDB client """ - client_kwargs = {} - if region: - client_kwargs["region_name"] = region - self.table = boto3.resource("dynamodb", **client_kwargs).Table(table_name) + config = config or Config() + self.table = boto3.resource("dynamodb", config=config).Table(table_name) self.key_attr = key_attr self.value_attr = value_attr diff --git a/aws_lambda_powertools/utilities/parameters/secrets.py b/aws_lambda_powertools/utilities/parameters/secrets.py index 63732b219d0..8d55527f72c 100644 --- a/aws_lambda_powertools/utilities/parameters/secrets.py +++ b/aws_lambda_powertools/utilities/parameters/secrets.py @@ -6,6 +6,7 @@ from typing import Dict, Optional, Union import boto3 +from botocore.config import Config from .base import DEFAULT_PROVIDERS, BaseProvider @@ -17,16 +18,14 @@ class SecretsProvider(BaseProvider): client = None - def __init__(self, region: Optional[str] = None): + def __init__(self, config: Optional[Config] = None): """ Initialize the Secrets Manager client """ - client_kwargs = {} - if region: - client_kwargs["region_name"] = region + config = config or Config() - self.client = boto3.client("secretsmanager", **client_kwargs) + self.client = boto3.client("secretsmanager", config=config) super().__init__() diff --git a/aws_lambda_powertools/utilities/parameters/ssm.py b/aws_lambda_powertools/utilities/parameters/ssm.py index 3811e6bcce4..b53808a856b 100644 --- a/aws_lambda_powertools/utilities/parameters/ssm.py +++ b/aws_lambda_powertools/utilities/parameters/ssm.py @@ -6,6 +6,7 @@ from typing import Dict, Optional, Union import boto3 +from botocore.config import Config from .base import DEFAULT_PROVIDERS, BaseProvider @@ -18,17 +19,14 @@ class SSMProvider(BaseProvider): client = None def __init__( - self, region: Optional[str] = None, + self, config: Optional[Config] = None, ): """ Initialize the SSM Parameter Store client """ - client_kwargs = {} - if region: - client_kwargs["region_name"] = region - - self.client = boto3.client("ssm", **client_kwargs) + config = config or Config() + self.client = boto3.client("ssm", config=config) super().__init__() diff --git a/tests/functional/test_utilities_parameters.py b/tests/functional/test_utilities_parameters.py index 39978df4f64..60622ceff96 100644 --- a/tests/functional/test_utilities_parameters.py +++ b/tests/functional/test_utilities_parameters.py @@ -8,6 +8,7 @@ import pytest from boto3.dynamodb.conditions import Key from botocore import stub +from botocore.config import Config from aws_lambda_powertools.utilities import parameters from aws_lambda_powertools.utilities.parameters.base import BaseProvider, ExpirableValue @@ -30,7 +31,12 @@ def mock_version(): return random.randrange(1, 1000) -def test_dynamodb_provider_get(mock_name, mock_value): +@pytest.fixture(scope="module") +def config(): + return Config(region_name="us-east-1") + + +def test_dynamodb_provider_get(mock_name, mock_value, config): """ Test DynamoDBProvider.get() with a non-cached value """ @@ -38,7 +44,7 @@ def test_dynamodb_provider_get(mock_name, mock_value): table_name = "TEST_TABLE" # Create a new provider - provider = parameters.DynamoDBProvider(table_name, region="us-east-1") + provider = parameters.DynamoDBProvider(table_name, config=config) # Stub the boto3 client stubber = stub.Stubber(provider.table.meta.client) @@ -56,9 +62,9 @@ def test_dynamodb_provider_get(mock_name, mock_value): stubber.deactivate() -def test_dynamodb_provider_get_default_region(monkeypatch, mock_name, mock_value): +def test_dynamodb_provider_get_default_config(monkeypatch, mock_name, mock_value): """ - Test DynamoDBProvider.get() without setting a region + Test DynamoDBProvider.get() without setting a config """ monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") @@ -84,7 +90,7 @@ def test_dynamodb_provider_get_default_region(monkeypatch, mock_name, mock_value stubber.deactivate() -def test_dynamodb_provider_get_cached(mock_name, mock_value): +def test_dynamodb_provider_get_cached(mock_name, mock_value, config): """ Test DynamoDBProvider.get() with a cached value """ @@ -92,7 +98,7 @@ def test_dynamodb_provider_get_cached(mock_name, mock_value): table_name = "TEST_TABLE" # Create a new provider - provider = parameters.DynamoDBProvider(table_name, region="us-east-1") + provider = parameters.DynamoDBProvider(table_name, config=config) # Inject value in the internal store provider.store[(mock_name, None)] = ExpirableValue(mock_value, datetime.now() + timedelta(seconds=60)) @@ -110,7 +116,7 @@ def test_dynamodb_provider_get_cached(mock_name, mock_value): stubber.deactivate() -def test_dynamodb_provider_get_expired(mock_name, mock_value): +def test_dynamodb_provider_get_expired(mock_name, mock_value, config): """ Test DynamoDBProvider.get() with a cached but expired value """ @@ -118,7 +124,7 @@ def test_dynamodb_provider_get_expired(mock_name, mock_value): table_name = "TEST_TABLE" # Create a new provider - provider = parameters.DynamoDBProvider(table_name, region="us-east-1") + provider = parameters.DynamoDBProvider(table_name, config=config) # Inject value in the internal store provider.store[(mock_name, None)] = ExpirableValue(mock_value, datetime.now() - timedelta(seconds=60)) @@ -139,7 +145,7 @@ def test_dynamodb_provider_get_expired(mock_name, mock_value): stubber.deactivate() -def test_dynamodb_provider_get_multiple(mock_name, mock_value): +def test_dynamodb_provider_get_multiple(mock_name, mock_value, config): """ Test DynamoDBProvider.get_multiple() with a non-cached path """ @@ -148,7 +154,7 @@ def test_dynamodb_provider_get_multiple(mock_name, mock_value): table_name = "TEST_TABLE" # Create a new provider - provider = parameters.DynamoDBProvider(table_name, region="us-east-1") + provider = parameters.DynamoDBProvider(table_name, config=config) # Stub the boto3 client stubber = stub.Stubber(provider.table.meta.client) @@ -175,7 +181,7 @@ def test_dynamodb_provider_get_multiple(mock_name, mock_value): stubber.deactivate() -def test_dynamodb_provider_get_multiple_next_token(mock_name, mock_value): +def test_dynamodb_provider_get_multiple_next_token(mock_name, mock_value, config): """ Test DynamoDBProvider.get_multiple() with a non-cached path """ @@ -184,7 +190,7 @@ def test_dynamodb_provider_get_multiple_next_token(mock_name, mock_value): table_name = "TEST_TABLE" # Create a new provider - provider = parameters.DynamoDBProvider(table_name, region="us-east-1") + provider = parameters.DynamoDBProvider(table_name, config=config) # Stub the boto3 client stubber = stub.Stubber(provider.table.meta.client) @@ -228,13 +234,13 @@ def test_dynamodb_provider_get_multiple_next_token(mock_name, mock_value): stubber.deactivate() -def test_ssm_provider_get(mock_name, mock_value, mock_version): +def test_ssm_provider_get(mock_name, mock_value, mock_version, config): """ Test SSMProvider.get() with a non-cached value """ # Create a new provider - provider = parameters.SSMProvider(region="us-east-1") + provider = parameters.SSMProvider(config=config) # Stub the boto3 client stubber = stub.Stubber(provider.client) @@ -263,9 +269,9 @@ def test_ssm_provider_get(mock_name, mock_value, mock_version): stubber.deactivate() -def test_ssm_provider_get_default_region(monkeypatch, mock_name, mock_value, mock_version): +def test_ssm_provider_get_default_config(monkeypatch, mock_name, mock_value, mock_version): """ - Test SSMProvider.get() without specifying the region + Test SSMProvider.get() without specifying the config """ monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") @@ -300,13 +306,13 @@ def test_ssm_provider_get_default_region(monkeypatch, mock_name, mock_value, moc stubber.deactivate() -def test_ssm_provider_get_cached(mock_name, mock_value): +def test_ssm_provider_get_cached(mock_name, mock_value, config): """ Test SSMProvider.get() with a cached value """ # Create a new provider - provider = parameters.SSMProvider(region="us-east-1") + provider = parameters.SSMProvider(config=config) # Inject value in the internal store provider.store[(mock_name, None)] = ExpirableValue(mock_value, datetime.now() + timedelta(seconds=60)) @@ -324,13 +330,13 @@ def test_ssm_provider_get_cached(mock_name, mock_value): stubber.deactivate() -def test_ssm_provider_get_expired(mock_name, mock_value, mock_version): +def test_ssm_provider_get_expired(mock_name, mock_value, mock_version, config): """ Test SSMProvider.get() with a cached but expired value """ # Create a new provider - provider = parameters.SSMProvider(region="us-east-1") + provider = parameters.SSMProvider(config=config) # Inject value in the internal store provider.store[(mock_name, None)] = ExpirableValue(mock_value, datetime.now() - timedelta(seconds=60)) @@ -362,7 +368,7 @@ def test_ssm_provider_get_expired(mock_name, mock_value, mock_version): stubber.deactivate() -def test_ssm_provider_get_multiple(mock_name, mock_value, mock_version): +def test_ssm_provider_get_multiple(mock_name, mock_value, mock_version, config): """ Test SSMProvider.get_multiple() with a non-cached path """ @@ -370,7 +376,7 @@ def test_ssm_provider_get_multiple(mock_name, mock_value, mock_version): mock_param_names = ["A", "B", "C"] # Create a new provider - provider = parameters.SSMProvider(region="us-east-1") + provider = parameters.SSMProvider(config=config) # Stub the boto3 client stubber = stub.Stubber(provider.client) @@ -406,7 +412,7 @@ def test_ssm_provider_get_multiple(mock_name, mock_value, mock_version): stubber.deactivate() -def test_ssm_provider_get_multiple_different_path(mock_name, mock_value, mock_version): +def test_ssm_provider_get_multiple_different_path(mock_name, mock_value, mock_version, config): """ Test SSMProvider.get_multiple() with a non-cached path and names that don't start with the path """ @@ -414,7 +420,7 @@ def test_ssm_provider_get_multiple_different_path(mock_name, mock_value, mock_ve mock_param_names = ["A", "B", "C"] # Create a new provider - provider = parameters.SSMProvider(region="us-east-1") + provider = parameters.SSMProvider(config=config) # Stub the boto3 client stubber = stub.Stubber(provider.client) @@ -450,7 +456,7 @@ def test_ssm_provider_get_multiple_different_path(mock_name, mock_value, mock_ve stubber.deactivate() -def test_ssm_provider_get_multiple_next_token(mock_name, mock_value, mock_version): +def test_ssm_provider_get_multiple_next_token(mock_name, mock_value, mock_version, config): """ Test SSMProvider.get_multiple() with a non-cached path with multiple calls """ @@ -458,7 +464,7 @@ def test_ssm_provider_get_multiple_next_token(mock_name, mock_value, mock_versio mock_param_names = ["A", "B", "C"] # Create a new provider - provider = parameters.SSMProvider(region="us-east-1") + provider = parameters.SSMProvider(config=config) # Stub the boto3 client stubber = stub.Stubber(provider.client) @@ -516,13 +522,13 @@ def test_ssm_provider_get_multiple_next_token(mock_name, mock_value, mock_versio stubber.deactivate() -def test_secrets_provider_get(mock_name, mock_value): +def test_secrets_provider_get(mock_name, mock_value, config): """ Test SecretsProvider.get() with a non-cached value """ # Create a new provider - provider = parameters.SecretsProvider(region="us-east-1") + provider = parameters.SecretsProvider(config=config) # Stub the boto3 client stubber = stub.Stubber(provider.client) @@ -546,9 +552,9 @@ def test_secrets_provider_get(mock_name, mock_value): stubber.deactivate() -def test_secrets_provider_get_default_region(monkeypatch, mock_name, mock_value): +def test_secrets_provider_get_default_config(monkeypatch, mock_name, mock_value): """ - Test SecretsProvider.get() without specifying a region + Test SecretsProvider.get() without specifying a config """ monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") @@ -578,13 +584,13 @@ def test_secrets_provider_get_default_region(monkeypatch, mock_name, mock_value) stubber.deactivate() -def test_secrets_provider_get_cached(mock_name, mock_value): +def test_secrets_provider_get_cached(mock_name, mock_value, config): """ Test SecretsProvider.get() with a cached value """ # Create a new provider - provider = parameters.SecretsProvider(region="us-east-1") + provider = parameters.SecretsProvider(config=config) # Inject value in the internal store provider.store[(mock_name, None)] = ExpirableValue(mock_value, datetime.now() + timedelta(seconds=60)) @@ -602,13 +608,13 @@ def test_secrets_provider_get_cached(mock_name, mock_value): stubber.deactivate() -def test_secrets_provider_get_expired(mock_name, mock_value): +def test_secrets_provider_get_expired(mock_name, mock_value, config): """ Test SecretsProvider.get() with a cached but expired value """ # Create a new provider - provider = parameters.SecretsProvider(region="us-east-1") + provider = parameters.SecretsProvider(config=config) # Inject value in the internal store provider.store[(mock_name, None)] = ExpirableValue(mock_value, datetime.now() - timedelta(seconds=60)) From 3569e2112bf18f79c8d0be921246a10b303ac7a4 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Mon, 17 Aug 2020 19:19:46 +0200 Subject: [PATCH 14/30] feat: make arguments explicits in parameter utilities --- .../utilities/parameters/dynamodb.py | 4 +--- aws_lambda_powertools/utilities/parameters/ssm.py | 11 ++--------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/dynamodb.py b/aws_lambda_powertools/utilities/parameters/dynamodb.py index 6c15e3316a1..3dcefb44eed 100644 --- a/aws_lambda_powertools/utilities/parameters/dynamodb.py +++ b/aws_lambda_powertools/utilities/parameters/dynamodb.py @@ -43,7 +43,7 @@ def _get(self, name: str, **kwargs) -> str: return self.table.get_item(Key={self.key_attr: name})["Item"][self.value_attr] - def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + def _get_multiple(self, path: str, sort_attr: str = "sk", **kwargs) -> Dict[str, str]: """ Retrieve multiple parameter values from Amazon DynamoDB @@ -55,8 +55,6 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: Name of the DynamoDB table sort key (defaults to 'sk') """ - sort_attr = kwargs.get("sort_attr", "sk") - response = self.table.query(KeyConditionExpression=Key(self.key_attr).eq(path)) items = response.get("Items", []) diff --git a/aws_lambda_powertools/utilities/parameters/ssm.py b/aws_lambda_powertools/utilities/parameters/ssm.py index b53808a856b..dadd65c7790 100644 --- a/aws_lambda_powertools/utilities/parameters/ssm.py +++ b/aws_lambda_powertools/utilities/parameters/ssm.py @@ -30,7 +30,7 @@ def __init__( super().__init__() - def _get(self, name: str, **kwargs) -> str: + def _get(self, name: str, decrypt: bool = False, **kwargs) -> str: """ Retrieve a parameter value from AWS Systems Manager Parameter Store @@ -42,12 +42,9 @@ def _get(self, name: str, **kwargs) -> str: If the parameter value should be decrypted """ - # Load kwargs - decrypt = kwargs.get("decrypt", False) - return self.client.get_parameter(Name=name, WithDecryption=decrypt)["Parameter"]["Value"] - def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + def _get_multiple(self, path: str, decrypt: bool = False, recursive: bool = False, **kwargs) -> Dict[str, str]: """ Retrieve multiple parameter values from AWS Systems Manager Parameter Store @@ -61,10 +58,6 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: If this should retrieve the parameter values recursively or not """ - # Load kwargs - decrypt = kwargs.get("decrypt", False) - recursive = kwargs.get("recursive", False) - response = self.client.get_parameters_by_path(Path=path, WithDecryption=decrypt, Recursive=recursive) parameters = response.get("Parameters", []) From 1278d2cf4cd8d32f2be09a870ad5928120699601 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Mon, 17 Aug 2020 19:34:36 +0200 Subject: [PATCH 15/30] docs: add examples for parameter utilities --- .../utilities/parameters/dynamodb.py | 54 +++++++++++++++++++ .../utilities/parameters/secrets.py | 19 +++++++ .../utilities/parameters/ssm.py | 26 +++++++++ 3 files changed, 99 insertions(+) diff --git a/aws_lambda_powertools/utilities/parameters/dynamodb.py b/aws_lambda_powertools/utilities/parameters/dynamodb.py index 3dcefb44eed..e43522e0848 100644 --- a/aws_lambda_powertools/utilities/parameters/dynamodb.py +++ b/aws_lambda_powertools/utilities/parameters/dynamodb.py @@ -15,6 +15,60 @@ class DynamoDBProvider(BaseProvider): """ Amazon DynamoDB Parameter Provider + + Example + ------- + **Retrieves a parameter value from a DynamoDB table** + + In this example, the DynamoDB table uses `id` as hash key and stores the value in the `value` + attribute. + + >>> from aws_lambda_powertools.utilities.parameters import DynamoDBProvider + >>> ddb_provider = DynamoDBProvider("ParametersTable") + >>> + >>> ddb_provider.get("my-parameter") + + **Retrieves a parameter value from a DynamoDB table that has custom attribute names** + + >>> from aws_lambda_powertools.utilities.parameters import DynamoDBProvider + >>> ddb_provider = DynamoDBProvider( + ... "ParametersTable", + ... key_attr="my-id", + ... value_attr="my-value" + ... ) + >>> + >>> ddb_provider.get("my-parameter") + + **Retrieves a parameter value from a DynamoDB table in another AWS region** + + >>> from botocore.config import Config + >>> from aws_lambda_powertools.utilities.parameters import DynamoDBProvider + >>> + >>> config = Config(region_name="us-west-1") + >>> ddb_provider = DynamoDBProvider("ParametersTable", config=config) + >>> + >>> ddb_provider.get("my-parameter") + + **Retrieves multiple values from a DynamoDB table** + + In this case, the provider will use a sort key to retrieve multiple values using a query under + the hood. This expects that the sort key is named `sk`. + + >>> from aws_lambda_powertools.utilities.parameters import DynamoDBProvider + >>> ddb_provider = DynamoDBProvider("ParametersTable") + >>> + >>> ddb_provider.get_multiple("my-parameters") + + **Retrieves multiple values from a DynamoDB table with a custom sort key** + + In this case, the provider will use a sort key to retrieve multiple values using a query under + the hood. + + >>> from aws_lambda_powertools.utilities.parameters import DynamoDBProvider + >>> ddb_provider = DynamoDBProvider("ParametersTable") + >>> + >>> ddb_provider.get_multiple("my-parameters", sort_attr="my-sort-attr") + """ table = None diff --git a/aws_lambda_powertools/utilities/parameters/secrets.py b/aws_lambda_powertools/utilities/parameters/secrets.py index 8d55527f72c..36b7209d595 100644 --- a/aws_lambda_powertools/utilities/parameters/secrets.py +++ b/aws_lambda_powertools/utilities/parameters/secrets.py @@ -14,6 +14,25 @@ class SecretsProvider(BaseProvider): """ AWS Secrets Manager Parameter Provider + + Example + ------- + **Retrieves a parameter value from Secrets Manager** + + >>> from aws_lambda_powertools.utilities.parameters import SecretsProvider + >>> secrets_provider = SecretsProvider() + >>> + >>> secrets_provider.get("my-parameter") + + **Retrieves a parameter value from Secrets Manager in another AWS region** + + >>> from botocore.config import Config + >>> from aws_lambda_powertools.utilities.parameters import SecretsProvider + >>> + >>> config = Config(region_name="us-west-1") + >>> secrets_provider = SecretsProvider(config=config) + >>> + >>> secrets_provider.get("my-parameter") """ client = None diff --git a/aws_lambda_powertools/utilities/parameters/ssm.py b/aws_lambda_powertools/utilities/parameters/ssm.py index dadd65c7790..2fddb036a8c 100644 --- a/aws_lambda_powertools/utilities/parameters/ssm.py +++ b/aws_lambda_powertools/utilities/parameters/ssm.py @@ -14,6 +14,32 @@ class SSMProvider(BaseProvider): """ AWS Systems Manager Parameter Store Provider + + Example + ------- + **Retrieves a parameter value from Systems Manager Parameter Store** + + >>> from aws_lambda_powertools.utilities.parameters import SSMProvider + >>> ssm_provider = SSMProvider() + >>> + >>> ssm_provider.get("/my/parameter") + + **Retrieves a parameter value from Systems Manager Parameter Store in another AWS region** + + >>> from botocore.config import Config + >>> from aws_lambda_powertools.utilities.parameters import SSMProvider + >>> + >>> config = Config(region_name="us-west-1") + >>> ssm_provider = SSMProvider(config=config) + >>> + >>> ssm_provider.get("/my/parameter") + + **Retrieves multiple parameter values from Systes Manager Parameter Store using a path prefix** + + >>> from aws_lambda_powertools.utilities.parameters import SSMProvider + >>> ssm_provider = SSMProvider() + >>> + >>> ssm_provider.get_multiple("/my/path/prefix") """ client = None From 5fb929de48421862a3f5bdd3b26ba65eccbaceba Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Mon, 17 Aug 2020 20:05:05 +0200 Subject: [PATCH 16/30] feat: add override SDK options for parameter utilities --- .../utilities/parameters/dynamodb.py | 41 ++- .../utilities/parameters/secrets.py | 23 +- .../utilities/parameters/ssm.py | 30 +- tests/functional/test_utilities_parameters.py | 316 ++++++++++++++++++ 4 files changed, 395 insertions(+), 15 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/dynamodb.py b/aws_lambda_powertools/utilities/parameters/dynamodb.py index e43522e0848..7e54308c8eb 100644 --- a/aws_lambda_powertools/utilities/parameters/dynamodb.py +++ b/aws_lambda_powertools/utilities/parameters/dynamodb.py @@ -49,6 +49,13 @@ class DynamoDBProvider(BaseProvider): >>> >>> ddb_provider.get("my-parameter") + **Retrieves a parameter value from a DynamoDB table passing options to the SDK call** + + >>> from aws_lambda_powertools.utilities.parameters import DynamoDBProvider + >>> ddb_provider = DynamoDBProvider("ParametersTable") + >>> + >>> ddb_provider.get("my-parameter", ConsistentRead=True) + **Retrieves multiple values from a DynamoDB table** In this case, the provider will use a sort key to retrieve multiple values using a query under @@ -69,6 +76,12 @@ class DynamoDBProvider(BaseProvider): >>> >>> ddb_provider.get_multiple("my-parameters", sort_attr="my-sort-attr") + **Retrieves multiple values from a DynamoDB table passing options to the SDK calls** + + >>> from aws_lambda_powertools.utilities.parameters import DynamoDBProvider + >>> ddb_provider = DynamoDBProvider("ParametersTable") + >>> + >>> ddb_provider.get("my-parameter", ConsistentRead=True) """ table = None @@ -90,14 +103,24 @@ def __init__( super().__init__() - def _get(self, name: str, **kwargs) -> str: + def _get(self, name: str, **sdk_options) -> str: """ Retrieve a parameter value from Amazon DynamoDB + + Parameters + ---------- + name: str + Name of the parameter + sdk_options: dict + Dictionary of options that will be passed to the get_item call """ - return self.table.get_item(Key={self.key_attr: name})["Item"][self.value_attr] + # Explicit arguments will take precedence over keyword arguments + sdk_options["Key"] = {self.key_attr: name} + + return self.table.get_item(**sdk_options)["Item"][self.value_attr] - def _get_multiple(self, path: str, sort_attr: str = "sk", **kwargs) -> Dict[str, str]: + def _get_multiple(self, path: str, sort_attr: str = "sk", **sdk_options) -> Dict[str, str]: """ Retrieve multiple parameter values from Amazon DynamoDB @@ -107,16 +130,20 @@ def _get_multiple(self, path: str, sort_attr: str = "sk", **kwargs) -> Dict[str, Path to retrieve the parameters sort_attr: str Name of the DynamoDB table sort key (defaults to 'sk') + sdk_options: dict + Dictionary of options that will be passed to the query call """ - response = self.table.query(KeyConditionExpression=Key(self.key_attr).eq(path)) + # Explicit arguments will take precedence over keyword arguments + sdk_options["KeyConditionExpression"] = Key(self.key_attr).eq(path) + + response = self.table.query(**sdk_options) items = response.get("Items", []) # Keep querying while there are more items matching the partition key while "LastEvaluatedKey" in response: - response = self.table.query( - KeyConditionExpression=Key(self.key_attr).eq(path), ExclusiveStartKey=response["LastEvaluatedKey"], - ) + sdk_options["ExclusiveStartKey"] = response["LastEvaluatedKey"] + response = self.table.query(**sdk_options) items.extend(response.get("Items", [])) retval = {} diff --git a/aws_lambda_powertools/utilities/parameters/secrets.py b/aws_lambda_powertools/utilities/parameters/secrets.py index 36b7209d595..4290faec146 100644 --- a/aws_lambda_powertools/utilities/parameters/secrets.py +++ b/aws_lambda_powertools/utilities/parameters/secrets.py @@ -33,6 +33,13 @@ class SecretsProvider(BaseProvider): >>> secrets_provider = SecretsProvider(config=config) >>> >>> secrets_provider.get("my-parameter") + + **Retrieves a parameter value from Secrets Manager passing options to the SDK call** + + >>> from aws_lambda_powertools.utilities.parameters import SecretsProvider + >>> secrets_provider = SecretsProvider() + >>> + >>> secrets_provider.get("my-parameter", VersionId="f658cac0-98a5-41d9-b993-8a76a7799194") """ client = None @@ -48,14 +55,24 @@ def __init__(self, config: Optional[Config] = None): super().__init__() - def _get(self, name: str, **kwargs) -> str: + def _get(self, name: str, **sdk_options) -> str: """ Retrieve a parameter value from AWS Systems Manager Parameter Store + + Parameters + ---------- + name: str + Name of the parameter + sdk_options: dict + Dictionary of options that will be passed to the get_secret_value call """ - return self.client.get_secret_value(SecretId=name)["SecretString"] + # Explicit arguments will take precedence over keyword arguments + sdk_options["SecretId"] = name + + return self.client.get_secret_value(**sdk_options)["SecretString"] - def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + def _get_multiple(self, path: str, **sdk_options) -> Dict[str, str]: """ Retrieving multiple parameter values is not supported with AWS Secrets Manager """ diff --git a/aws_lambda_powertools/utilities/parameters/ssm.py b/aws_lambda_powertools/utilities/parameters/ssm.py index 2fddb036a8c..36bf1f9902f 100644 --- a/aws_lambda_powertools/utilities/parameters/ssm.py +++ b/aws_lambda_powertools/utilities/parameters/ssm.py @@ -34,12 +34,19 @@ class SSMProvider(BaseProvider): >>> >>> ssm_provider.get("/my/parameter") - **Retrieves multiple parameter values from Systes Manager Parameter Store using a path prefix** + **Retrieves multiple parameter values from Systems Manager Parameter Store using a path prefix** >>> from aws_lambda_powertools.utilities.parameters import SSMProvider >>> ssm_provider = SSMProvider() >>> >>> ssm_provider.get_multiple("/my/path/prefix") + + **Retrieves multiple parameter values from Systems Manager Parameter Store passing options to the SDK call** + + >>> from aws_lambda_powertools.utilities.parameters import SSMProvider + >>> ssm_provider = SSMProvider() + >>> + >>> ssm_provider.get_multiple("/my/path/prefix", MaxResults=10) """ client = None @@ -56,7 +63,7 @@ def __init__( super().__init__() - def _get(self, name: str, decrypt: bool = False, **kwargs) -> str: + def _get(self, name: str, decrypt: bool = False, **sdk_options) -> str: """ Retrieve a parameter value from AWS Systems Manager Parameter Store @@ -66,11 +73,17 @@ def _get(self, name: str, decrypt: bool = False, **kwargs) -> str: Parameter name decrypt: bool If the parameter value should be decrypted + sdk_options: dict + Dictionary of options that will be passed to the get_parameter call """ - return self.client.get_parameter(Name=name, WithDecryption=decrypt)["Parameter"]["Value"] + # Explicit arguments will take precedence over keyword arguments + sdk_options["Name"] = name + sdk_options["WithDecryption"] = decrypt + + return self.client.get_parameter(**sdk_options)["Parameter"]["Value"] - def _get_multiple(self, path: str, decrypt: bool = False, recursive: bool = False, **kwargs) -> Dict[str, str]: + def _get_multiple(self, path: str, decrypt: bool = False, recursive: bool = False, **sdk_options) -> Dict[str, str]: """ Retrieve multiple parameter values from AWS Systems Manager Parameter Store @@ -82,9 +95,16 @@ def _get_multiple(self, path: str, decrypt: bool = False, recursive: bool = Fals If the parameter values should be decrypted recursive: bool If this should retrieve the parameter values recursively or not + sdk_options: dict + Dictionary of options that will be passed to the get_parameters_by_path call """ - response = self.client.get_parameters_by_path(Path=path, WithDecryption=decrypt, Recursive=recursive) + # Explicit arguments will take precedence over keyword arguments + sdk_options["Path"] = path + sdk_options["WithDecryption"] = decrypt + sdk_options["Recursive"] = recursive + + response = self.client.get_parameters_by_path(**sdk_options) parameters = response.get("Parameters", []) # Keep retrieving parameters diff --git a/tests/functional/test_utilities_parameters.py b/tests/functional/test_utilities_parameters.py index 60622ceff96..d0997393d73 100644 --- a/tests/functional/test_utilities_parameters.py +++ b/tests/functional/test_utilities_parameters.py @@ -145,6 +145,58 @@ def test_dynamodb_provider_get_expired(mock_name, mock_value, config): stubber.deactivate() +def test_dynamodb_provider_get_sdk_options(mock_name, mock_value, config): + """ + Test DynamoDBProvider.get() with SDK options + """ + + table_name = "TEST_TABLE" + + # Create a new provider + provider = parameters.DynamoDBProvider(table_name, config=config) + + # Stub the boto3 client + stubber = stub.Stubber(provider.table.meta.client) + response = {"Item": {"id": {"S": mock_name}, "value": {"S": mock_value}}} + expected_params = {"TableName": table_name, "Key": {"id": mock_name}, "ConsistentRead": True} + stubber.add_response("get_item", response, expected_params) + stubber.activate() + + try: + value = provider.get(mock_name, ConsistentRead=True) + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + +def test_dynamodb_provider_get_sdk_options_overwrite(mock_name, mock_value, config): + """ + Test DynamoDBProvider.get() with SDK options that should be overwritten + """ + + table_name = "TEST_TABLE" + + # Create a new provider + provider = parameters.DynamoDBProvider(table_name, config=config) + + # Stub the boto3 client + stubber = stub.Stubber(provider.table.meta.client) + response = {"Item": {"id": {"S": mock_name}, "value": {"S": mock_value}}} + expected_params = {"TableName": table_name, "Key": {"id": mock_name}} + stubber.add_response("get_item", response, expected_params) + stubber.activate() + + try: + value = provider.get(mock_name, Key="THIS_SHOULD_BE_OVERWRITTEN") + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + def test_dynamodb_provider_get_multiple(mock_name, mock_value, config): """ Test DynamoDBProvider.get_multiple() with a non-cached path @@ -234,6 +286,85 @@ def test_dynamodb_provider_get_multiple_next_token(mock_name, mock_value, config stubber.deactivate() +def test_dynamodb_provider_get_multiple_sdk_options(mock_name, mock_value, config): + """ + Test DynamoDBProvider.get_multiple() with custom SDK options + """ + + mock_param_names = ["A", "B", "C"] + table_name = "TEST_TABLE" + + # Create a new provider + provider = parameters.DynamoDBProvider(table_name, config=config) + + # Stub the boto3 client + stubber = stub.Stubber(provider.table.meta.client) + response = { + "Items": [ + {"id": {"S": mock_name}, "sk": {"S": name}, "value": {"S": f"{mock_value}/{name}"}} + for name in mock_param_names + ] + } + expected_params = { + "TableName": table_name, + "KeyConditionExpression": Key("id").eq(mock_name), + "ConsistentRead": True, + } + stubber.add_response("query", response, expected_params) + stubber.activate() + + try: + values = provider.get_multiple(mock_name, ConsistentRead=True) + + stubber.assert_no_pending_responses() + + assert len(values) == len(mock_param_names) + for name in mock_param_names: + assert name in values + assert values[name] == f"{mock_value}/{name}" + finally: + stubber.deactivate() + + +def test_dynamodb_provider_get_multiple_sdk_options_overwrite(mock_name, mock_value, config): + """ + Test DynamoDBProvider.get_multiple() with custom SDK options that should be overwritten + """ + + mock_param_names = ["A", "B", "C"] + table_name = "TEST_TABLE" + + # Create a new provider + provider = parameters.DynamoDBProvider(table_name, config=config) + + # Stub the boto3 client + stubber = stub.Stubber(provider.table.meta.client) + response = { + "Items": [ + {"id": {"S": mock_name}, "sk": {"S": name}, "value": {"S": f"{mock_value}/{name}"}} + for name in mock_param_names + ] + } + expected_params = { + "TableName": table_name, + "KeyConditionExpression": Key("id").eq(mock_name), + } + stubber.add_response("query", response, expected_params) + stubber.activate() + + try: + values = provider.get_multiple(mock_name, KeyConditionExpression="THIS_SHOULD_BE_OVERWRITTEN") + + stubber.assert_no_pending_responses() + + assert len(values) == len(mock_param_names) + for name in mock_param_names: + assert name in values + assert values[name] == f"{mock_value}/{name}" + finally: + stubber.deactivate() + + def test_ssm_provider_get(mock_name, mock_value, mock_version, config): """ Test SSMProvider.get() with a non-cached value @@ -368,6 +499,41 @@ def test_ssm_provider_get_expired(mock_name, mock_value, mock_version, config): stubber.deactivate() +def test_ssm_provider_get_sdk_options_overwrite(mock_name, mock_value, mock_version, config): + """ + Test SSMProvider.get() with custom SDK options overwritten + """ + + # Create a new provider + provider = parameters.SSMProvider(config=config) + + # Stub the boto3 client + stubber = stub.Stubber(provider.client) + response = { + "Parameter": { + "Name": mock_name, + "Type": "String", + "Value": mock_value, + "Version": mock_version, + "Selector": f"{mock_name}:{mock_version}", + "SourceResult": "string", + "LastModifiedDate": datetime(2015, 1, 1), + "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{mock_name}", + } + } + expected_params = {"Name": mock_name, "WithDecryption": False} + stubber.add_response("get_parameter", response, expected_params) + stubber.activate() + + try: + value = provider.get(mock_name, Name="THIS_SHOULD_BE_OVERWRITTEN", WithDecryption=True) + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + def test_ssm_provider_get_multiple(mock_name, mock_value, mock_version, config): """ Test SSMProvider.get_multiple() with a non-cached path @@ -522,6 +688,96 @@ def test_ssm_provider_get_multiple_next_token(mock_name, mock_value, mock_versio stubber.deactivate() +def test_ssm_provider_get_multiple_sdk_options(mock_name, mock_value, mock_version, config): + """ + Test SSMProvider.get_multiple() with SDK options + """ + + mock_param_names = ["A", "B", "C"] + + # Create a new provider + provider = parameters.SSMProvider(config=config) + + # Stub the boto3 client + stubber = stub.Stubber(provider.client) + response = { + "Parameters": [ + { + "Name": f"{mock_name}/{name}", + "Type": "String", + "Value": f"{mock_value}/{name}", + "Version": mock_version, + "Selector": f"{mock_name}/{name}:{mock_version}", + "SourceResult": "string", + "LastModifiedDate": datetime(2015, 1, 1), + "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{mock_name}/{name}", + } + for name in mock_param_names + ] + } + expected_params = {"Path": mock_name, "Recursive": False, "WithDecryption": False, "MaxResults": 10} + stubber.add_response("get_parameters_by_path", response, expected_params) + stubber.activate() + + try: + values = provider.get_multiple(mock_name, MaxResults=10) + + stubber.assert_no_pending_responses() + + assert len(values) == len(mock_param_names) + for name in mock_param_names: + assert name in values + assert values[name] == f"{mock_value}/{name}" + finally: + stubber.deactivate() + + +def test_ssm_provider_get_multiple_sdk_options_overwrite(mock_name, mock_value, mock_version, config): + """ + Test SSMProvider.get_multiple() with SDK options overwritten + """ + + mock_param_names = ["A", "B", "C"] + + # Create a new provider + provider = parameters.SSMProvider(config=config) + + # Stub the boto3 client + stubber = stub.Stubber(provider.client) + response = { + "Parameters": [ + { + "Name": f"{mock_name}/{name}", + "Type": "String", + "Value": f"{mock_value}/{name}", + "Version": mock_version, + "Selector": f"{mock_name}/{name}:{mock_version}", + "SourceResult": "string", + "LastModifiedDate": datetime(2015, 1, 1), + "ARN": f"arn:aws:ssm:us-east-2:111122223333:parameter/{mock_name}/{name}", + } + for name in mock_param_names + ] + } + expected_params = {"Path": mock_name, "Recursive": False, "WithDecryption": False} + stubber.add_response("get_parameters_by_path", response, expected_params) + stubber.activate() + + try: + values = provider.get_multiple( + mock_name, Path="THIS_SHOULD_BE_OVERWRITTEN", Recursive=True, WithDecryption=True, + ) + + stubber.assert_no_pending_responses() + + assert len(values) == len(mock_param_names) + for name in mock_param_names: + assert name in values + assert values[name] == f"{mock_value}/{name}" + finally: + stubber.deactivate() + + def test_secrets_provider_get(mock_name, mock_value, config): """ Test SecretsProvider.get() with a non-cached value @@ -641,6 +897,66 @@ def test_secrets_provider_get_expired(mock_name, mock_value, config): stubber.deactivate() +def test_secrets_provider_get_sdk_options(mock_name, mock_value, config): + """ + Test SecretsProvider.get() with custom SDK options + """ + + # Create a new provider + provider = parameters.SecretsProvider(config=config) + + # Stub the boto3 client + stubber = stub.Stubber(provider.client) + response = { + "ARN": f"arn:aws:secretsmanager:us-east-1:132456789012:secret/{mock_name}", + "Name": mock_name, + "VersionId": "7a9155b8-2dc9-466e-b4f6-5bc46516c84d", + "SecretString": mock_value, + "CreatedDate": datetime(2015, 1, 1), + } + expected_params = {"SecretId": mock_name, "VersionId": "7a9155b8-2dc9-466e-b4f6-5bc46516c84d"} + stubber.add_response("get_secret_value", response, expected_params) + stubber.activate() + + try: + value = provider.get(mock_name, VersionId="7a9155b8-2dc9-466e-b4f6-5bc46516c84d") + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + +def test_secrets_provider_get_sdk_options_overwrite(mock_name, mock_value, config): + """ + Test SecretsProvider.get() with custom SDK options overwritten + """ + + # Create a new provider + provider = parameters.SecretsProvider(config=config) + + # Stub the boto3 client + stubber = stub.Stubber(provider.client) + response = { + "ARN": f"arn:aws:secretsmanager:us-east-1:132456789012:secret/{mock_name}", + "Name": mock_name, + "VersionId": "7a9155b8-2dc9-466e-b4f6-5bc46516c84d", + "SecretString": mock_value, + "CreatedDate": datetime(2015, 1, 1), + } + expected_params = {"SecretId": mock_name} + stubber.add_response("get_secret_value", response, expected_params) + stubber.activate() + + try: + value = provider.get(mock_name, SecretId="THIS_SHOULD_BE_OVERWRITTEN") + + assert value == mock_value + stubber.assert_no_pending_responses() + finally: + stubber.deactivate() + + def test_base_provider_get_transform_json(mock_name, mock_value): """ Test BaseProvider.get() with a json transform From 7438766ef75d00951029ea3f93c94afacfa48c5e Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Mon, 17 Aug 2020 20:16:56 +0200 Subject: [PATCH 17/30] docs: add examples for shorthands in the parameter utility --- .../utilities/parameters/dynamodb.py | 17 ++++- .../utilities/parameters/secrets.py | 38 +++++++++- .../utilities/parameters/ssm.py | 69 +++++++++++++++++-- 3 files changed, 112 insertions(+), 12 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/dynamodb.py b/aws_lambda_powertools/utilities/parameters/dynamodb.py index 7e54308c8eb..9d1db3cbcd4 100644 --- a/aws_lambda_powertools/utilities/parameters/dynamodb.py +++ b/aws_lambda_powertools/utilities/parameters/dynamodb.py @@ -16,6 +16,17 @@ class DynamoDBProvider(BaseProvider): """ Amazon DynamoDB Parameter Provider + Parameters + ---------- + table_name: str + Name of the DynamoDB table that stores parameters + key_attr: str, optional + Hash key for the DynamoDB table + value_attr: str, optional + Attribute that contains the values in the DynamoDB table + config: botocore.config.Config, optional + Botocore configuration to pass during client initialization + Example ------- **Retrieves a parameter value from a DynamoDB table** @@ -111,7 +122,7 @@ def _get(self, name: str, **sdk_options) -> str: ---------- name: str Name of the parameter - sdk_options: dict + sdk_options: dict, optional Dictionary of options that will be passed to the get_item call """ @@ -128,9 +139,9 @@ def _get_multiple(self, path: str, sort_attr: str = "sk", **sdk_options) -> Dict ---------- path: str Path to retrieve the parameters - sort_attr: str + sort_attr: str, optional Name of the DynamoDB table sort key (defaults to 'sk') - sdk_options: dict + sdk_options: dict, optional Dictionary of options that will be passed to the query call """ diff --git a/aws_lambda_powertools/utilities/parameters/secrets.py b/aws_lambda_powertools/utilities/parameters/secrets.py index 4290faec146..417bbbb1f8e 100644 --- a/aws_lambda_powertools/utilities/parameters/secrets.py +++ b/aws_lambda_powertools/utilities/parameters/secrets.py @@ -15,6 +15,11 @@ class SecretsProvider(BaseProvider): """ AWS Secrets Manager Parameter Provider + Parameters + ---------- + config: botocore.config.Config, optional + Botocore configuration to pass during client initialization + Example ------- **Retrieves a parameter value from Secrets Manager** @@ -79,13 +84,42 @@ def _get_multiple(self, path: str, **sdk_options) -> Dict[str, str]: raise NotImplementedError() -def get_secret(name: str, transform: Optional[str] = None, decrypt: bool = False) -> Union[str, dict, bytes]: +def get_secret(name: str, transform: Optional[str] = None, **sdk_options) -> Union[str, dict, bytes]: """ Retrieve a parameter value from AWS Secrets Manager + + Parameters + ---------- + name: str + Name of the parameter + transform: str, optional + Transforms the content from a JSON object ('json') or base64 binary string ('binary') + sdk_options: dict, optional + Dictionary of options that will be passed to the get_secret_value call + + Example + ------- + **Retrieves a secret*** + + >>> from aws_lambda_powertools.utilities.parameters import get_secret + >>> + >>> get_secret("my-secret") + + **Retrieves a secret and transforms using a JSON deserializer*** + + >>> from aws_lambda_powertools.utilities.parameters import get_secret + >>> + >>> get_secret("my-secret", transform="json") + + **Retrieves a secret and passes custom arguments to the SDK** + + >>> from aws_lambda_powertools.utilities.parameters import get_secret + >>> + >>> get_secret("my-secret", VersionId="f658cac0-98a5-41d9-b993-8a76a7799194") """ # Only create the provider if this function is called at least once if "secrets" not in DEFAULT_PROVIDERS: DEFAULT_PROVIDERS["secrets"] = SecretsProvider() - return DEFAULT_PROVIDERS["secrets"].get(name, transform=transform, decrypt=decrypt) + return DEFAULT_PROVIDERS["secrets"].get(name, transform=transform, **sdk_options) diff --git a/aws_lambda_powertools/utilities/parameters/ssm.py b/aws_lambda_powertools/utilities/parameters/ssm.py index 36bf1f9902f..5e86431cc74 100644 --- a/aws_lambda_powertools/utilities/parameters/ssm.py +++ b/aws_lambda_powertools/utilities/parameters/ssm.py @@ -15,6 +15,11 @@ class SSMProvider(BaseProvider): """ AWS Systems Manager Parameter Store Provider + Parameters + ---------- + config: botocore.config.Config, optional + Botocore configuration to pass during client initialization + Example ------- **Retrieves a parameter value from Systems Manager Parameter Store** @@ -71,9 +76,9 @@ def _get(self, name: str, decrypt: bool = False, **sdk_options) -> str: ---------- name: str Parameter name - decrypt: bool + decrypt: bool, optional If the parameter value should be decrypted - sdk_options: dict + sdk_options: dict, optional Dictionary of options that will be passed to the get_parameter call """ @@ -91,11 +96,11 @@ def _get_multiple(self, path: str, decrypt: bool = False, recursive: bool = Fals ---------- path: str Path to retrieve the parameters - decrypt: bool + decrypt: bool, optional If the parameter values should be decrypted - recursive: bool + recursive: bool, optional If this should retrieve the parameter values recursively or not - sdk_options: dict + sdk_options: dict, optional Dictionary of options that will be passed to the get_parameters_by_path call """ @@ -131,9 +136,32 @@ def _get_multiple(self, path: str, decrypt: bool = False, recursive: bool = Fals return retval -def get_parameter(name: str, transform: Optional[str] = None) -> Union[str, list, dict, bytes]: +def get_parameter(name: str, transform: Optional[str] = None, **sdk_options) -> Union[str, list, dict, bytes]: """ Retrieve a parameter value from AWS Systems Manager (SSM) Parameter Store + + Parameters + ---------- + name: str + Name of the parameter + transform: str, optional + Transforms the content from a JSON object ('json') or base64 binary string ('binary') + sdk_options: dict, optional + Dictionary of options that will be passed to the get_secret_value call + + Example + ------- + **Retrieves a parameter value from Systems Manager Parameter Store** + + >>> from aws_lambda_powertools.utilities.parameters import get_parameter + >>> + >>> get_parameter("/my/parameter") + + **Retrieves a parameter value and decodes it using a Base64 decoder** + + >>> from aws_lambda_powertools.utilities.parameters import get_parameter + >>> + >>> get_parameter("/my/parameter", transform='binary') """ # Only create the provider if this function is called at least once @@ -144,10 +172,37 @@ def get_parameter(name: str, transform: Optional[str] = None) -> Union[str, list def get_parameters( - path: str, transform: Optional[str] = None, recursive: bool = False, decrypt: bool = False + path: str, transform: Optional[str] = None, recursive: bool = False, decrypt: bool = False, **sdk_options ) -> Union[Dict[str, str], Dict[str, dict], Dict[str, bytes]]: """ Retrieve multiple parameter values from AWS Systems Manager (SSM) Parameter Store + + Parameters + ---------- + path: str + Path to retrieve the parameters + transform: str, optional + Transforms the content from a JSON object ('json') or base64 binary string ('binary') + decrypt: bool, optional + If the parameter values should be decrypted + recursive: bool, optional + If this should retrieve the parameter values recursively or not + sdk_options: dict, optional + Dictionary of options that will be passed to the get_parameters_by_path call + + Example + ------- + **Retrieves parameter values from Systems Manager Parameter Store** + + >>> from aws_lambda_powertools.utilities.parameters import get_parameter + >>> + >>> get_parameters("/my/path/parameter") + + **Retrieves parameter values and decodes them using a Base64 decoder** + + >>> from aws_lambda_powertools.utilities.parameters import get_parameter + >>> + >>> get_parameters("/my/path/parameter", transform='binary') """ # Only create the provider if this function is called at least once From d53c373b594b593983dbbc1010a42caceba04458 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Tue, 18 Aug 2020 17:40:03 +0200 Subject: [PATCH 18/30] fix: fix typo in DynamoDB parameter example --- aws_lambda_powertools/utilities/parameters/dynamodb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/parameters/dynamodb.py b/aws_lambda_powertools/utilities/parameters/dynamodb.py index 9d1db3cbcd4..d0b3cac37eb 100644 --- a/aws_lambda_powertools/utilities/parameters/dynamodb.py +++ b/aws_lambda_powertools/utilities/parameters/dynamodb.py @@ -92,7 +92,7 @@ class DynamoDBProvider(BaseProvider): >>> from aws_lambda_powertools.utilities.parameters import DynamoDBProvider >>> ddb_provider = DynamoDBProvider("ParametersTable") >>> - >>> ddb_provider.get("my-parameter", ConsistentRead=True) + >>> ddb_provider.get_multiple("my-parameters", ConsistentRead=True) """ table = None From fce326885bc6204999d15292b6c0456416b38e57 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Tue, 18 Aug 2020 18:17:49 +0200 Subject: [PATCH 19/30] feat: throw exception on failed transform for parameter utility --- .../utilities/parameters/__init__.py | 3 +- .../utilities/parameters/base.py | 61 +++++--- .../utilities/parameters/exceptions.py | 4 + tests/functional/test_utilities_parameters.py | 135 +++++++++++++++--- 4 files changed, 164 insertions(+), 39 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/__init__.py b/aws_lambda_powertools/utilities/parameters/__init__.py index 3c18301a1dd..07f9cb2ca76 100644 --- a/aws_lambda_powertools/utilities/parameters/__init__.py +++ b/aws_lambda_powertools/utilities/parameters/__init__.py @@ -6,7 +6,7 @@ from .base import BaseProvider from .dynamodb import DynamoDBProvider -from .exceptions import GetParameterError +from .exceptions import GetParameterError, TransformParameterError from .secrets import SecretsProvider, get_secret from .ssm import SSMProvider, get_parameter, get_parameters @@ -16,6 +16,7 @@ "DynamoDBProvider", "SecretsProvider", "SSMProvider", + "TransformParameterError", "get_parameter", "get_parameters", "get_secret", diff --git a/aws_lambda_powertools/utilities/parameters/base.py b/aws_lambda_powertools/utilities/parameters/base.py index 0ce24cce857..560a21ea657 100644 --- a/aws_lambda_powertools/utilities/parameters/base.py +++ b/aws_lambda_powertools/utilities/parameters/base.py @@ -9,7 +9,7 @@ from datetime import datetime, timedelta from typing import Dict, Optional, Union -from .exceptions import GetParameterError +from .exceptions import GetParameterError, TransformParameterError DEFAULT_MAX_AGE_SECS = 5 ExpirableValue = namedtuple("ExpirableValue", ["value", "ttl"]) @@ -32,14 +32,13 @@ def __init__(self): self.store = {} def get( - self, name: str, max_age: int = DEFAULT_MAX_AGE_SECS, transform: Optional[str] = None, **kwargs + self, name: str, max_age: int = DEFAULT_MAX_AGE_SECS, transform: Optional[str] = None, **sdk_options ) -> Union[str, list, dict, bytes]: """ Retrieve a parameter value or return the cached value Parameters ---------- - name: str Parameter name max_age: int @@ -51,10 +50,11 @@ def get( Raises ------ - GetParameterError When the parameter provider fails to retrieve a parameter value for a given name. + TransformParameterError + When the parameter provider fails to transform a parameter value. """ # If there are multiple calls to the same parameter but in a different @@ -70,54 +70,81 @@ def get( if key not in self.store or self.store[key].ttl < datetime.now(): try: - value = self._get(name, **kwargs) + value = self._get(name, **sdk_options) # Encapsulate all errors into a generic GetParameterError except Exception as exc: raise GetParameterError(str(exc)) - if transform == "json": - value = json.loads(value) - elif transform == "binary": - value = base64.b64decode(value) + try: + if transform == "json": + value = json.loads(value) + elif transform == "binary": + value = base64.b64decode(value) + # Encapsulate transform exceptions into TransformParameterError + except Exception as exc: + raise TransformParameterError(str(exc)) self.store[key] = ExpirableValue(value, datetime.now() + timedelta(seconds=max_age),) return self.store[key].value @abstractmethod - def _get(self, name: str, **kwargs) -> str: + def _get(self, name: str, **sdk_options) -> str: """ Retrieve paramater value from the underlying parameter store """ raise NotImplementedError() def get_multiple( - self, path: str, max_age: int = DEFAULT_MAX_AGE_SECS, transform: Optional[str] = None, **kwargs + self, path: str, max_age: int = DEFAULT_MAX_AGE_SECS, transform: Optional[str] = None, **sdk_options ) -> Union[Dict[str, str], Dict[str, dict], Dict[str, bytes]]: """ Retrieve multiple parameters based on a path prefix + + Parameters + ---------- + path: str + Parameter path used to retrieve multiple parameters + max_age: int + Maximum age of the cached value + transform: str + Optional transformation of the parameter value. Supported values + are "json" for JSON strings and "binary" for base 64 encoded + values. + + Raises + ------ + GetParameterError + When the parameter provider fails to retrieve parameter values for + a given path. + TransformParameterError + When the parameter provider fails to transform a parameter value. """ key = (path, transform) if key not in self.store or self.store[key].ttl < datetime.now(): try: - values = self._get_multiple(path, **kwargs) + values = self._get_multiple(path, **sdk_options) # Encapsulate all errors into a generic GetParameterError except Exception as exc: raise GetParameterError(str(exc)) - if transform == "json": - values = {k: json.loads(v) for k, v in values.items()} - elif transform == "binary": - values = {k: base64.b64decode(v) for k, v in values.items()} + try: + if transform == "json": + values = {k: json.loads(v) for k, v in values.items()} + elif transform == "binary": + values = {k: base64.b64decode(v) for k, v in values.items()} + # Encapsulate transform exceptions into TransformParameterError + except Exception as exc: + raise TransformParameterError(str(exc)) self.store[key] = ExpirableValue(values, datetime.now() + timedelta(seconds=max_age),) return self.store[key].value @abstractmethod - def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + def _get_multiple(self, path: str, **sdk_options) -> Dict[str, str]: """ Retrieve multiple parameter values from the underlying parameter store """ diff --git a/aws_lambda_powertools/utilities/parameters/exceptions.py b/aws_lambda_powertools/utilities/parameters/exceptions.py index 683ad319b09..1287568b463 100644 --- a/aws_lambda_powertools/utilities/parameters/exceptions.py +++ b/aws_lambda_powertools/utilities/parameters/exceptions.py @@ -5,3 +5,7 @@ class GetParameterError(Exception): """When a provider raises an exception on parameter retrieval""" + + +class TransformParameterError(Exception): + """When a provider fails to transform a parameter value""" diff --git a/tests/functional/test_utilities_parameters.py b/tests/functional/test_utilities_parameters.py index d0997393d73..a09ad756633 100644 --- a/tests/functional/test_utilities_parameters.py +++ b/tests/functional/test_utilities_parameters.py @@ -957,6 +957,48 @@ def test_secrets_provider_get_sdk_options_overwrite(mock_name, mock_value, confi stubber.deactivate() +def test_base_provider_get_exception(mock_name): + """ + Test BaseProvider.get() that raises an exception + """ + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + assert name == mock_name + raise Exception("test exception raised") + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + raise NotImplementedError() + + provider = TestProvider() + + with pytest.raises(parameters.GetParameterError) as excinfo: + provider.get(mock_name) + + assert "test exception raised" in str(excinfo) + + +def test_base_provider_get_multiple_exception(mock_name): + """ + Test BaseProvider.get_multiple() that raises an exception + """ + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + raise NotImplementedError() + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + assert path == mock_name + raise Exception("test exception raised") + + provider = TestProvider() + + with pytest.raises(parameters.GetParameterError) as excinfo: + provider.get_multiple(mock_name) + + assert "test exception raised" in str(excinfo) + + def test_base_provider_get_transform_json(mock_name, mock_value): """ Test BaseProvider.get() with a json transform @@ -981,55 +1023,60 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: assert value[mock_name] == mock_value -def test_base_provider_get_exception(mock_name): +def test_base_provider_get_transform_json_exception(mock_name, mock_value): """ - Test BaseProvider.get() that raises an exception + Test BaseProvider.get() with a json transform that raises an exception """ + mock_data = json.dumps({mock_name: mock_value}) + "{" + class TestProvider(BaseProvider): def _get(self, name: str, **kwargs) -> str: assert name == mock_name - raise Exception("test exception raised") + return mock_data def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: raise NotImplementedError() provider = TestProvider() - with pytest.raises(parameters.GetParameterError) as excinfo: - provider.get(mock_name) + with pytest.raises(parameters.TransformParameterError) as excinfo: + provider.get(mock_name, transform="json") - assert "test exception raised" in str(excinfo) + assert "Extra data" in str(excinfo) -def test_base_provider_get_multiple_exception(mock_name): +def test_base_provider_get_transform_binary(mock_name, mock_value): """ - Test BaseProvider.get_multiple() that raises an exception + Test BaseProvider.get() with a binary transform """ + mock_binary = mock_value.encode() + mock_data = base64.b64encode(mock_binary).decode() + class TestProvider(BaseProvider): def _get(self, name: str, **kwargs) -> str: - raise NotImplementedError() + assert name == mock_name + return mock_data def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: - assert path == mock_name - raise Exception("test exception raised") + raise NotImplementedError() provider = TestProvider() - with pytest.raises(parameters.GetParameterError) as excinfo: - provider.get_multiple(mock_name) + value = provider.get(mock_name, transform="binary") - assert "test exception raised" in str(excinfo) + assert isinstance(value, bytes) + assert value == mock_binary -def test_base_provider_get_transform_binary(mock_name, mock_value): +def test_base_provider_get_transform_binary_exception(mock_name): """ - Test BaseProvider.get() with a binary transform + Test BaseProvider.get() with a binary transform that raises an exception """ - mock_binary = mock_value.encode() - mock_data = base64.b64encode(mock_binary).decode() + mock_data = "qw" + print(mock_data) class TestProvider(BaseProvider): def _get(self, name: str, **kwargs) -> str: @@ -1041,10 +1088,10 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: provider = TestProvider() - value = provider.get(mock_name, transform="binary") + with pytest.raises(parameters.TransformParameterError) as excinfo: + provider.get(mock_name, transform="binary") - assert isinstance(value, bytes) - assert value == mock_binary + assert "Incorrect padding" in str(excinfo) def test_base_provider_get_multiple_transform_json(mock_name, mock_value): @@ -1070,6 +1117,29 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: assert value["A"][mock_name] == mock_value +def test_base_provider_get_multiple_transform_json_exception(mock_name, mock_value): + """ + Test BaseProvider.get_multiple() with a json transform that raises an exception + """ + + mock_data = json.dumps({mock_name: mock_value}) + "{" + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + raise NotImplementedError() + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + assert path == mock_name + return {"A": mock_data} + + provider = TestProvider() + + with pytest.raises(parameters.TransformParameterError) as excinfo: + provider.get_multiple(mock_name, transform="json") + + assert "Extra data" in str(excinfo) + + def test_base_provider_get_multiple_transform_binary(mock_name, mock_value): """ Test BaseProvider.get_multiple() with a binary transform @@ -1094,6 +1164,29 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: assert value["A"] == mock_binary +def test_base_provider_get_multiple_transform_binary_exception(mock_name): + """ + Test BaseProvider.get_multiple() with a binary transform that raises an exception + """ + + mock_data = "qw" + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + raise NotImplementedError() + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + assert path == mock_name + return {"A": mock_data} + + provider = TestProvider() + + with pytest.raises(parameters.TransformParameterError) as excinfo: + provider.get_multiple(mock_name, transform="binary") + + assert "Incorrect padding" in str(excinfo) + + def test_base_provider_get_multiple_cached(mock_name, mock_value): """ Test BaseProvider.get_multiple() with cached values From c765c9091f5b068d8af03dc3c0ce17a7efe09a0f Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Tue, 18 Aug 2020 18:38:12 +0200 Subject: [PATCH 20/30] docs: add examples on how to retrieve parameters in the parameter utility --- .../utilities/parameters/dynamodb.py | 59 +++++++++++++++---- .../utilities/parameters/secrets.py | 25 ++++++-- .../utilities/parameters/ssm.py | 50 +++++++++++++--- 3 files changed, 111 insertions(+), 23 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/dynamodb.py b/aws_lambda_powertools/utilities/parameters/dynamodb.py index d0b3cac37eb..cf098171580 100644 --- a/aws_lambda_powertools/utilities/parameters/dynamodb.py +++ b/aws_lambda_powertools/utilities/parameters/dynamodb.py @@ -32,12 +32,17 @@ class DynamoDBProvider(BaseProvider): **Retrieves a parameter value from a DynamoDB table** In this example, the DynamoDB table uses `id` as hash key and stores the value in the `value` - attribute. + attribute. The parameter item looks like this: + + { "id": "my-parameters", "value": "Parameter value a" } >>> from aws_lambda_powertools.utilities.parameters import DynamoDBProvider >>> ddb_provider = DynamoDBProvider("ParametersTable") >>> - >>> ddb_provider.get("my-parameter") + >>> value = ddb_provider.get("my-parameter") + >>> + >>> print(value) + My parameter value **Retrieves a parameter value from a DynamoDB table that has custom attribute names** @@ -48,7 +53,10 @@ class DynamoDBProvider(BaseProvider): ... value_attr="my-value" ... ) >>> - >>> ddb_provider.get("my-parameter") + >>> value = ddb_provider.get("my-parameter") + >>> + >>> print(value) + My parameter value **Retrieves a parameter value from a DynamoDB table in another AWS region** @@ -58,24 +66,41 @@ class DynamoDBProvider(BaseProvider): >>> config = Config(region_name="us-west-1") >>> ddb_provider = DynamoDBProvider("ParametersTable", config=config) >>> - >>> ddb_provider.get("my-parameter") + >>> value = ddb_provider.get("my-parameter") + >>> + >>> print(value) + My parameter value **Retrieves a parameter value from a DynamoDB table passing options to the SDK call** >>> from aws_lambda_powertools.utilities.parameters import DynamoDBProvider >>> ddb_provider = DynamoDBProvider("ParametersTable") >>> - >>> ddb_provider.get("my-parameter", ConsistentRead=True) + >>> value = ddb_provider.get("my-parameter", ConsistentRead=True) + >>> + >>> print(value) + My parameter value **Retrieves multiple values from a DynamoDB table** In this case, the provider will use a sort key to retrieve multiple values using a query under - the hood. This expects that the sort key is named `sk`. + the hood. This expects that the sort key is named `sk`. The DynamoDB table contains three items + looking like this: + + { "id": "my-parameters", "sk": "a", "value": "Parameter value a" } + { "id": "my-parameters", "sk": "b", "value": "Parameter value b" } + { "id": "my-parameters", "sk": "c", "value": "Parameter value c" } >>> from aws_lambda_powertools.utilities.parameters import DynamoDBProvider >>> ddb_provider = DynamoDBProvider("ParametersTable") >>> - >>> ddb_provider.get_multiple("my-parameters") + >>> values = ddb_provider.get_multiple("my-parameters") + >>> + >>> for key, value in values.items(): + ... print(key, value) + a Parameter value a + b Parameter value b + c Parameter value c **Retrieves multiple values from a DynamoDB table with a custom sort key** @@ -85,14 +110,26 @@ class DynamoDBProvider(BaseProvider): >>> from aws_lambda_powertools.utilities.parameters import DynamoDBProvider >>> ddb_provider = DynamoDBProvider("ParametersTable") >>> - >>> ddb_provider.get_multiple("my-parameters", sort_attr="my-sort-attr") + >>> values = ddb_provider.get_multiple("my-parameters", sort_attr="my-sort-attr") + >>> + >>> for key, value in values.items(): + ... print(key, value) + a Parameter value a + b Parameter value b + c Parameter value c **Retrieves multiple values from a DynamoDB table passing options to the SDK calls** >>> from aws_lambda_powertools.utilities.parameters import DynamoDBProvider >>> ddb_provider = DynamoDBProvider("ParametersTable") >>> - >>> ddb_provider.get_multiple("my-parameters", ConsistentRead=True) + >>> values = ddb_provider.get_multiple("my-parameters", ConsistentRead=True) + >>> + >>> for key, value in values.items(): + ... print(key, value) + a Parameter value a + b Parameter value b + c Parameter value c """ table = None @@ -123,7 +160,7 @@ def _get(self, name: str, **sdk_options) -> str: name: str Name of the parameter sdk_options: dict, optional - Dictionary of options that will be passed to the get_item call + Dictionary of options that will be passed to the DynamoDB get_item API call """ # Explicit arguments will take precedence over keyword arguments @@ -142,7 +179,7 @@ def _get_multiple(self, path: str, sort_attr: str = "sk", **sdk_options) -> Dict sort_attr: str, optional Name of the DynamoDB table sort key (defaults to 'sk') sdk_options: dict, optional - Dictionary of options that will be passed to the query call + Dictionary of options that will be passed to the DynamoDB query API call """ # Explicit arguments will take precedence over keyword arguments diff --git a/aws_lambda_powertools/utilities/parameters/secrets.py b/aws_lambda_powertools/utilities/parameters/secrets.py index 417bbbb1f8e..ee4585309fe 100644 --- a/aws_lambda_powertools/utilities/parameters/secrets.py +++ b/aws_lambda_powertools/utilities/parameters/secrets.py @@ -27,7 +27,10 @@ class SecretsProvider(BaseProvider): >>> from aws_lambda_powertools.utilities.parameters import SecretsProvider >>> secrets_provider = SecretsProvider() >>> - >>> secrets_provider.get("my-parameter") + >>> value secrets_provider.get("my-parameter") + >>> + >>> print(value) + My parameter value **Retrieves a parameter value from Secrets Manager in another AWS region** @@ -37,14 +40,20 @@ class SecretsProvider(BaseProvider): >>> config = Config(region_name="us-west-1") >>> secrets_provider = SecretsProvider(config=config) >>> - >>> secrets_provider.get("my-parameter") + >>> value = secrets_provider.get("my-parameter") + >>> + >>> print(value) + My parameter value **Retrieves a parameter value from Secrets Manager passing options to the SDK call** >>> from aws_lambda_powertools.utilities.parameters import SecretsProvider >>> secrets_provider = SecretsProvider() >>> - >>> secrets_provider.get("my-parameter", VersionId="f658cac0-98a5-41d9-b993-8a76a7799194") + >>> value = secrets_provider.get("my-parameter", VersionId="f658cac0-98a5-41d9-b993-8a76a7799194") + >>> + >>> print(value) + My parameter value """ client = None @@ -69,7 +78,7 @@ def _get(self, name: str, **sdk_options) -> str: name: str Name of the parameter sdk_options: dict - Dictionary of options that will be passed to the get_secret_value call + Dictionary of options that will be passed to the Secrets Manager get_secret_value API call """ # Explicit arguments will take precedence over keyword arguments @@ -97,6 +106,14 @@ def get_secret(name: str, transform: Optional[str] = None, **sdk_options) -> Uni sdk_options: dict, optional Dictionary of options that will be passed to the get_secret_value call + Raises + ------ + GetParameterError + When the parameter provider fails to retrieve a parameter value for + a given name. + TransformParameterError + When the parameter provider fails to transform a parameter value. + Example ------- **Retrieves a secret*** diff --git a/aws_lambda_powertools/utilities/parameters/ssm.py b/aws_lambda_powertools/utilities/parameters/ssm.py index 5e86431cc74..ff9780a7462 100644 --- a/aws_lambda_powertools/utilities/parameters/ssm.py +++ b/aws_lambda_powertools/utilities/parameters/ssm.py @@ -27,7 +27,10 @@ class SSMProvider(BaseProvider): >>> from aws_lambda_powertools.utilities.parameters import SSMProvider >>> ssm_provider = SSMProvider() >>> - >>> ssm_provider.get("/my/parameter") + >>> value = ssm_provider.get("/my/parameter") + >>> + >>> print(value) + My parameter value **Retrieves a parameter value from Systems Manager Parameter Store in another AWS region** @@ -37,21 +40,36 @@ class SSMProvider(BaseProvider): >>> config = Config(region_name="us-west-1") >>> ssm_provider = SSMProvider(config=config) >>> - >>> ssm_provider.get("/my/parameter") + >>> value = ssm_provider.get("/my/parameter") + >>> + >>> print(value) + My parameter value **Retrieves multiple parameter values from Systems Manager Parameter Store using a path prefix** >>> from aws_lambda_powertools.utilities.parameters import SSMProvider >>> ssm_provider = SSMProvider() >>> - >>> ssm_provider.get_multiple("/my/path/prefix") + >>> values = ssm_provider.get_multiple("/my/path/prefix") + >>> + >>> for key, value in values.items(): + ... print(key, value) + /my/path/prefix/a Parameter value a + /my/path/prefix/b Parameter value b + /my/path/prefix/c Parameter value c **Retrieves multiple parameter values from Systems Manager Parameter Store passing options to the SDK call** >>> from aws_lambda_powertools.utilities.parameters import SSMProvider >>> ssm_provider = SSMProvider() >>> - >>> ssm_provider.get_multiple("/my/path/prefix", MaxResults=10) + >>> values = ssm_provider.get_multiple("/my/path/prefix", MaxResults=10) + >>> + >>> for key, value in values.items(): + ... print(key, value) + /my/path/prefix/a Parameter value a + /my/path/prefix/b Parameter value b + /my/path/prefix/c Parameter value c """ client = None @@ -79,7 +97,7 @@ def _get(self, name: str, decrypt: bool = False, **sdk_options) -> str: decrypt: bool, optional If the parameter value should be decrypted sdk_options: dict, optional - Dictionary of options that will be passed to the get_parameter call + Dictionary of options that will be passed to the Parameter Store get_parameter API call """ # Explicit arguments will take precedence over keyword arguments @@ -101,7 +119,7 @@ def _get_multiple(self, path: str, decrypt: bool = False, recursive: bool = Fals recursive: bool, optional If this should retrieve the parameter values recursively or not sdk_options: dict, optional - Dictionary of options that will be passed to the get_parameters_by_path call + Dictionary of options that will be passed to the Parameter Store get_parameters_by_path API call """ # Explicit arguments will take precedence over keyword arguments @@ -147,7 +165,15 @@ def get_parameter(name: str, transform: Optional[str] = None, **sdk_options) -> transform: str, optional Transforms the content from a JSON object ('json') or base64 binary string ('binary') sdk_options: dict, optional - Dictionary of options that will be passed to the get_secret_value call + Dictionary of options that will be passed to the Parameter Store get_parameter API call + + Raises + ------ + GetParameterError + When the parameter provider fails to retrieve a parameter value for + a given name. + TransformParameterError + When the parameter provider fails to transform a parameter value. Example ------- @@ -188,7 +214,15 @@ def get_parameters( recursive: bool, optional If this should retrieve the parameter values recursively or not sdk_options: dict, optional - Dictionary of options that will be passed to the get_parameters_by_path call + Dictionary of options that will be passed to the Parameter Store get_parameters_by_path API call + + Raises + ------ + GetParameterError + When the parameter provider fails to retrieve parameter values for + a given path. + TransformParameterError + When the parameter provider fails to transform a parameter value. Example ------- From bec8de34c0b934faf5bf62ad4bd116c824be69fa Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Tue, 18 Aug 2020 19:04:58 +0200 Subject: [PATCH 21/30] feat: use paginator for SSM parameter utility --- .../utilities/parameters/ssm.py | 40 +++++++------------ 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/ssm.py b/aws_lambda_powertools/utilities/parameters/ssm.py index ff9780a7462..041793478a4 100644 --- a/aws_lambda_powertools/utilities/parameters/ssm.py +++ b/aws_lambda_powertools/utilities/parameters/ssm.py @@ -127,31 +127,21 @@ def _get_multiple(self, path: str, decrypt: bool = False, recursive: bool = Fals sdk_options["WithDecryption"] = decrypt sdk_options["Recursive"] = recursive - response = self.client.get_parameters_by_path(**sdk_options) - parameters = response.get("Parameters", []) - - # Keep retrieving parameters - while "NextToken" in response: - response = self.client.get_parameters_by_path( - Path=path, WithDecryption=decrypt, Recursive=recursive, NextToken=response["NextToken"] - ) - parameters.extend(response.get("Parameters", [])) - - retval = {} - for parameter in parameters: - - # Standardize the parameter name - # The parameter name returned by SSM will contained the full path. - # However, for readability, we should return only the part after - # the path. - name = parameter["Name"] - if name.startswith(path): - name = name[len(path) :] - name = name.lstrip("/") - - retval[name] = parameter["Value"] - - return retval + parameters = {} + for page in self.client.get_paginator("get_parameters_by_path").paginate(**sdk_options): + for parameter in page.get("Parameters", []): + # Standardize the parameter name + # The parameter name returned by SSM will contained the full path. + # However, for readability, we should return only the part after + # the path. + name = parameter["Name"] + if name.startswith(path): + name = name[len(path) :] + name = name.lstrip("/") + + parameters[name] = parameter["Value"] + + return parameters def get_parameter(name: str, transform: Optional[str] = None, **sdk_options) -> Union[str, list, dict, bytes]: From 3ddc3bdbbe3698d8819b0bd0fad7fbddfb6a4ad1 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Tue, 18 Aug 2020 19:08:20 +0200 Subject: [PATCH 22/30] feat: make SSM parameter provider recursive by default --- .../utilities/parameters/ssm.py | 24 ++++++++++++++----- tests/functional/test_utilities_parameters.py | 2 +- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/ssm.py b/aws_lambda_powertools/utilities/parameters/ssm.py index 041793478a4..b458f8690d0 100644 --- a/aws_lambda_powertools/utilities/parameters/ssm.py +++ b/aws_lambda_powertools/utilities/parameters/ssm.py @@ -171,13 +171,19 @@ def get_parameter(name: str, transform: Optional[str] = None, **sdk_options) -> >>> from aws_lambda_powertools.utilities.parameters import get_parameter >>> - >>> get_parameter("/my/parameter") + >>> value = get_parameter("/my/parameter") + >>> + >>> print(value) + My parameter value **Retrieves a parameter value and decodes it using a Base64 decoder** >>> from aws_lambda_powertools.utilities.parameters import get_parameter >>> - >>> get_parameter("/my/parameter", transform='binary') + >>> value = get_parameter("/my/parameter", transform='binary') + >>> + >>> print(value) + My parameter value """ # Only create the provider if this function is called at least once @@ -188,7 +194,7 @@ def get_parameter(name: str, transform: Optional[str] = None, **sdk_options) -> def get_parameters( - path: str, transform: Optional[str] = None, recursive: bool = False, decrypt: bool = False, **sdk_options + path: str, transform: Optional[str] = None, recursive: bool = True, decrypt: bool = False, **sdk_options ) -> Union[Dict[str, str], Dict[str, dict], Dict[str, bytes]]: """ Retrieve multiple parameter values from AWS Systems Manager (SSM) Parameter Store @@ -202,7 +208,7 @@ def get_parameters( decrypt: bool, optional If the parameter values should be decrypted recursive: bool, optional - If this should retrieve the parameter values recursively or not + If this should retrieve the parameter values recursively or not, defaults to True sdk_options: dict, optional Dictionary of options that will be passed to the Parameter Store get_parameters_by_path API call @@ -220,13 +226,19 @@ def get_parameters( >>> from aws_lambda_powertools.utilities.parameters import get_parameter >>> - >>> get_parameters("/my/path/parameter") + >>> values = get_parameters("/my/path/prefix") + >>> + >>> for key, value in values.items(): + ... print(key, value) + /my/path/prefix/a Parameter value a + /my/path/prefix/b Parameter value b + /my/path/prefix/c Parameter value c **Retrieves parameter values and decodes them using a Base64 decoder** >>> from aws_lambda_powertools.utilities.parameters import get_parameter >>> - >>> get_parameters("/my/path/parameter", transform='binary') + >>> values = get_parameters("/my/path/prefix", transform='binary') """ # Only create the provider if this function is called at least once diff --git a/tests/functional/test_utilities_parameters.py b/tests/functional/test_utilities_parameters.py index a09ad756633..d648c8357a1 100644 --- a/tests/functional/test_utilities_parameters.py +++ b/tests/functional/test_utilities_parameters.py @@ -765,7 +765,7 @@ def test_ssm_provider_get_multiple_sdk_options_overwrite(mock_name, mock_value, try: values = provider.get_multiple( - mock_name, Path="THIS_SHOULD_BE_OVERWRITTEN", Recursive=True, WithDecryption=True, + mock_name, Path="THIS_SHOULD_BE_OVERWRITTEN", Recursive=False, WithDecryption=True, ) stubber.assert_no_pending_responses() From 55022157de843602cc53a7d1770674fc30bebbf6 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Tue, 18 Aug 2020 19:14:13 +0200 Subject: [PATCH 23/30] feat: move sort_attr to init for DynamoDB parameter provider --- .../utilities/parameters/dynamodb.py | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/dynamodb.py b/aws_lambda_powertools/utilities/parameters/dynamodb.py index cf098171580..4132697f0b9 100644 --- a/aws_lambda_powertools/utilities/parameters/dynamodb.py +++ b/aws_lambda_powertools/utilities/parameters/dynamodb.py @@ -21,9 +21,11 @@ class DynamoDBProvider(BaseProvider): table_name: str Name of the DynamoDB table that stores parameters key_attr: str, optional - Hash key for the DynamoDB table + Hash key for the DynamoDB table (default to 'id') + sort_attr: str, optional + Name of the DynamoDB table sort key (defaults to 'sk'), used only for get_multiple value_attr: str, optional - Attribute that contains the values in the DynamoDB table + Attribute that contains the values in the DynamoDB table (defaults to 'value') config: botocore.config.Config, optional Botocore configuration to pass during client initialization @@ -102,15 +104,20 @@ class DynamoDBProvider(BaseProvider): b Parameter value b c Parameter value c - **Retrieves multiple values from a DynamoDB table with a custom sort key** + **Retrieves multiple values from a DynamoDB table that has custom attribute names** In this case, the provider will use a sort key to retrieve multiple values using a query under the hood. >>> from aws_lambda_powertools.utilities.parameters import DynamoDBProvider - >>> ddb_provider = DynamoDBProvider("ParametersTable") + >>> ddb_provider = DynamoDBProvider( + ... "ParametersTable", + ... key_attr="my-id", + ... sort_attr="my-sort-key", + ... value_attr="my-value" + ... ) >>> - >>> values = ddb_provider.get_multiple("my-parameters", sort_attr="my-sort-attr") + >>> values = ddb_provider.get_multiple("my-parameters") >>> >>> for key, value in values.items(): ... print(key, value) @@ -134,10 +141,16 @@ class DynamoDBProvider(BaseProvider): table = None key_attr = None + sort_attr = None value_attr = None def __init__( - self, table_name: str, key_attr: str = "id", value_attr: str = "value", config: Optional[Config] = None, + self, + table_name: str, + key_attr: str = "id", + sort_attr: str = "sk", + value_attr: str = "value", + config: Optional[Config] = None, ): """ Initialize the DynamoDB client @@ -147,6 +160,7 @@ def __init__( self.table = boto3.resource("dynamodb", config=config).Table(table_name) self.key_attr = key_attr + self.sort_attr = sort_attr self.value_attr = value_attr super().__init__() @@ -168,7 +182,7 @@ def _get(self, name: str, **sdk_options) -> str: return self.table.get_item(**sdk_options)["Item"][self.value_attr] - def _get_multiple(self, path: str, sort_attr: str = "sk", **sdk_options) -> Dict[str, str]: + def _get_multiple(self, path: str, **sdk_options) -> Dict[str, str]: """ Retrieve multiple parameter values from Amazon DynamoDB @@ -176,8 +190,6 @@ def _get_multiple(self, path: str, sort_attr: str = "sk", **sdk_options) -> Dict ---------- path: str Path to retrieve the parameters - sort_attr: str, optional - Name of the DynamoDB table sort key (defaults to 'sk') sdk_options: dict, optional Dictionary of options that will be passed to the DynamoDB query API call """ @@ -196,6 +208,6 @@ def _get_multiple(self, path: str, sort_attr: str = "sk", **sdk_options) -> Dict retval = {} for item in items: - retval[item[sort_attr]] = item[self.value_attr] + retval[item[self.sort_attr]] = item[self.value_attr] return retval From c8c970fc2d1ab9cca10215dc7db384e061f09b55 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Wed, 19 Aug 2020 09:59:14 +0200 Subject: [PATCH 24/30] feat: add 'raise_on_transform_error' for get_multiple parameter utility --- .../utilities/parameters/base.py | 73 ++++++++--- tests/functional/test_utilities_parameters.py | 118 +++++++++++++++++- 2 files changed, 170 insertions(+), 21 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/base.py b/aws_lambda_powertools/utilities/parameters/base.py index 560a21ea657..6602928c949 100644 --- a/aws_lambda_powertools/utilities/parameters/base.py +++ b/aws_lambda_powertools/utilities/parameters/base.py @@ -75,14 +75,8 @@ def get( except Exception as exc: raise GetParameterError(str(exc)) - try: - if transform == "json": - value = json.loads(value) - elif transform == "binary": - value = base64.b64decode(value) - # Encapsulate transform exceptions into TransformParameterError - except Exception as exc: - raise TransformParameterError(str(exc)) + if transform is not None: + value = transform_value(value, transform) self.store[key] = ExpirableValue(value, datetime.now() + timedelta(seconds=max_age),) @@ -96,7 +90,12 @@ def _get(self, name: str, **sdk_options) -> str: raise NotImplementedError() def get_multiple( - self, path: str, max_age: int = DEFAULT_MAX_AGE_SECS, transform: Optional[str] = None, **sdk_options + self, + path: str, + max_age: int = DEFAULT_MAX_AGE_SECS, + transform: Optional[str] = None, + raise_on_transform_error: bool = False, + **sdk_options, ) -> Union[Dict[str, str], Dict[str, dict], Dict[str, bytes]]: """ Retrieve multiple parameters based on a path prefix @@ -105,12 +104,15 @@ def get_multiple( ---------- path: str Parameter path used to retrieve multiple parameters - max_age: int + max_age: int, optional Maximum age of the cached value - transform: str + transform: str, optional Optional transformation of the parameter value. Supported values are "json" for JSON strings and "binary" for base 64 encoded values. + raise_on_transform_error: bool, optional + Raises an exception if any transform fails, otherwise this will + return a None value for each transform that failed Raises ------ @@ -130,14 +132,18 @@ def get_multiple( except Exception as exc: raise GetParameterError(str(exc)) - try: - if transform == "json": - values = {k: json.loads(v) for k, v in values.items()} - elif transform == "binary": - values = {k: base64.b64decode(v) for k, v in values.items()} - # Encapsulate transform exceptions into TransformParameterError - except Exception as exc: - raise TransformParameterError(str(exc)) + if transform is not None: + new_values = {} + for key, value in values.items(): + try: + new_values[key] = transform_value(value, transform) + except Exception as exc: + if raise_on_transform_error: + raise exc + else: + new_values[key] = None + + values = new_values self.store[key] = ExpirableValue(values, datetime.now() + timedelta(seconds=max_age),) @@ -149,3 +155,32 @@ def _get_multiple(self, path: str, **sdk_options) -> Dict[str, str]: Retrieve multiple parameter values from the underlying parameter store """ raise NotImplementedError() + + +def transform_value(value: str, transform: str) -> Union[dict, bytes]: + """ + Apply a transform to a value + + Parameters + --------- + value: str + Parameter alue to transform + transform: str + Type of transform, supported values are "json" and "binary" + + Raises + ------ + TransformParameterError: + When the parameter value could not be transformed + """ + + try: + if transform == "json": + return json.loads(value) + elif transform == "binary": + return base64.b64decode(value) + else: + raise ValueError(f"Invalid transform type '{transform}'") + + except Exception as exc: + raise TransformParameterError(str(exc)) diff --git a/tests/functional/test_utilities_parameters.py b/tests/functional/test_utilities_parameters.py index d648c8357a1..7a0677b2197 100644 --- a/tests/functional/test_utilities_parameters.py +++ b/tests/functional/test_utilities_parameters.py @@ -1117,6 +1117,30 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: assert value["A"][mock_name] == mock_value +def test_base_provider_get_multiple_transform_json_partial_failure(mock_name, mock_value): + """ + Test BaseProvider.get_multiple() with a json transform that contains a partial failure + """ + + mock_data = json.dumps({mock_name: mock_value}) + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + raise NotImplementedError() + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + assert path == mock_name + return {"A": mock_data, "B": mock_data + "{"} + + provider = TestProvider() + + value = provider.get_multiple(mock_name, transform="json") + + assert isinstance(value, dict) + assert value["A"][mock_name] == mock_value + assert value["B"] is None + + def test_base_provider_get_multiple_transform_json_exception(mock_name, mock_value): """ Test BaseProvider.get_multiple() with a json transform that raises an exception @@ -1135,7 +1159,7 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: provider = TestProvider() with pytest.raises(parameters.TransformParameterError) as excinfo: - provider.get_multiple(mock_name, transform="json") + provider.get_multiple(mock_name, transform="json", raise_on_transform_error=True) assert "Extra data" in str(excinfo) @@ -1164,6 +1188,32 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: assert value["A"] == mock_binary +def test_base_provider_get_multiple_transform_binary_partial_failure(mock_name, mock_value): + """ + Test BaseProvider.get_multiple() with a binary transform that contains a partial failure + """ + + mock_binary = mock_value.encode() + mock_data_a = base64.b64encode(mock_binary).decode() + mock_data_b = "qw" + + class TestProvider(BaseProvider): + def _get(self, name: str, **kwargs) -> str: + raise NotImplementedError() + + def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: + assert path == mock_name + return {"A": mock_data_a, "B": mock_data_b} + + provider = TestProvider() + + value = provider.get_multiple(mock_name, transform="binary") + + assert isinstance(value, dict) + assert value["A"] == mock_binary + assert value["B"] is None + + def test_base_provider_get_multiple_transform_binary_exception(mock_name): """ Test BaseProvider.get_multiple() with a binary transform that raises an exception @@ -1182,7 +1232,7 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: provider = TestProvider() with pytest.raises(parameters.TransformParameterError) as excinfo: - provider.get_multiple(mock_name, transform="binary") + provider.get_multiple(mock_name, transform="binary", raise_on_transform_error=True) assert "Incorrect padding" in str(excinfo) @@ -1354,3 +1404,67 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]: value = parameters.get_secret(mock_name) assert value == mock_value + + +def test_transform_value_json(mock_value): + """ + Test transform_value() with a json transform + """ + + mock_data = json.dumps({"A": mock_value}) + + value = parameters.base.transform_value(mock_data, "json") + + assert isinstance(value, dict) + assert value["A"] == mock_value + + +def test_transform_value_json_exception(mock_value): + """ + Test transform_value() with a json transform that fails + """ + + mock_data = json.dumps({"A": mock_value}) + "{" + + with pytest.raises(parameters.TransformParameterError) as excinfo: + parameters.base.transform_value(mock_data, "json") + + assert "Extra data" in str(excinfo) + + +def test_transform_value_binary(mock_value): + """ + Test transform_value() with a binary transform + """ + + mock_binary = mock_value.encode() + mock_data = base64.b64encode(mock_binary).decode() + + value = parameters.base.transform_value(mock_data, "binary") + + assert isinstance(value, bytes) + assert value == mock_binary + + +def test_transform_value_binary_exception(): + """ + Test transform_value() with a binary transform that fails + """ + + mock_data = "qw" + + with pytest.raises(parameters.TransformParameterError) as excinfo: + parameters.base.transform_value(mock_data, "binary") + + assert "Incorrect padding" in str(excinfo) + + +def test_transform_value_wrong(mock_value): + """ + Test transform_value() with an incorrect transform + """ + + with pytest.raises(parameters.TransformParameterError) as excinfo: + parameters.base.transform_value(mock_value, "INCORRECT") + + assert "Invalid transform type" in str(excinfo) From ed45c4bd715b9425bc6f500562880dd6064b643c Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Wed, 19 Aug 2020 10:01:35 +0200 Subject: [PATCH 25/30] docs: add sdk_options to parameters for get and get_multiple --- aws_lambda_powertools/utilities/parameters/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/aws_lambda_powertools/utilities/parameters/base.py b/aws_lambda_powertools/utilities/parameters/base.py index 6602928c949..8a552b53bcb 100644 --- a/aws_lambda_powertools/utilities/parameters/base.py +++ b/aws_lambda_powertools/utilities/parameters/base.py @@ -47,6 +47,8 @@ def get( Optional transformation of the parameter value. Supported values are "json" for JSON strings and "binary" for base 64 encoded values. + sdk_options: dict, optional + Arguments that will be passed directly to the underlying API call Raises ------ @@ -113,6 +115,8 @@ def get_multiple( raise_on_transform_error: bool, optional Raises an exception if any transform fails, otherwise this will return a None value for each transform that failed + sdk_options: dict, optional + Arguments that will be passed directly to the underlying API call Raises ------ From 4ecb17b52a517cbd5af8922b3230669d453877c5 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Wed, 19 Aug 2020 10:49:36 +0200 Subject: [PATCH 26/30] docs: add documentation for parameters utility --- docs/content/index.mdx | 1 + docs/content/utilities/parameters.mdx | 172 ++++++++++++++++++++++++++ docs/gatsby-config.js | 1 + 3 files changed, 174 insertions(+) create mode 100644 docs/content/utilities/parameters.mdx diff --git a/docs/content/index.mdx b/docs/content/index.mdx index 9966a08deb3..a5e27f469d3 100644 --- a/docs/content/index.mdx +++ b/docs/content/index.mdx @@ -18,6 +18,7 @@ Powertools is available in PyPi. You can use your favourite dependency managemen * [Logging](./core/logger) - Structured logging made easier, and decorator to enrich structured logging with key Lambda context details * [Metrics](./core/metrics) - Custom Metrics created asynchronously via CloudWatch Embedded Metric Format (EMF) * [Bring your own middleware](./utilities/middleware_factory) - Decorator factory to create your own middleware to run logic before, and after each Lambda invocation +* [Parameters utility](./utilities/parameters) - Retrieve parameter values and cache them for a specific amount of time ## Tenets diff --git a/docs/content/utilities/parameters.mdx b/docs/content/utilities/parameters.mdx new file mode 100644 index 00000000000..8f34713cd2a --- /dev/null +++ b/docs/content/utilities/parameters.mdx @@ -0,0 +1,172 @@ +--- +title: Parameters utility +description: Utility +--- + +import Note from "../../src/components/Note" + +The parameters utility provides a way to retrieve parameter values from [AWS Systems Manager Parameter Store](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-parameter-store.html), [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/) or [Amazon DynamoDB](https://aws.amazon.com/dynamodb/). It also provides a base class to create your parameter provider implementation. + +**Key features** + +* Retrieve one or multiple parameters from the underlying provider +* Cache parameter values for a given amount of time + * By default, parameters are cached for 5 seconds +* Transform parameter values from JSON or base 64 encoded strings + +## High-level functions + +This utility provides high-level functions for quick use. These support both the AWS Systems Manager Parameter Store and AWS Secrets Manager. + +```python:title=ssm.py +from aws_lambda_powertools.utilities import parameters + +def handler(event, context): + # Retrieve a single parameter + value = parameters.get_parameter("/my/parameter") + + # Retrieve multiple parameters from a path prefix + # This returns a dict with the parameter name as key + values = parameters.get_parameters("/my/path/prefix") + for k, v in values.items(): + print(f"{k}: {v}") +``` + +```python:title=secrets.py +from aws_lambda_powertools.utilities import parameters + +def handler(event, context): + # Retrieve a single secret + value = parameters.get_secret("my-secret") +``` + +## Provider classes + +Alternatively, you can use the provider classes, which give more flexibility, such as the ability to configure the underlying SDK client. + +```python:title=dynamodb.py +from aws_lambda_powertools.utilities import parameters + +dynamodb_provider = parameters.DynamoDBProvider(table_name="my-table") + +def handler(event, context): + # Retrieve a value from DynamoDB + value = dynamodb_provider.get("my-parameter") + + # Retrieve multiple values by performing a Query on the DynamoDB table + # This returns a dict with the sort key attribute as dict key. + values = dynamodb_provider.get_multiple("my-hash-key") + for k, v in values.items(): + print(f"{k}: {v}") +``` + +```python:title=ssm.py +from aws_lambda_powertools.utilities import parameters +from botocore.config import Config + +config = Config(region_name="us-west-1") +ssm_provider = parameters.SSMProvider(config=config) + +def handler(event, context): + # Retrieve a single parameter + value = ssm_provider.get("/my/parameter") + + # Retrieve multiple parameters from a path prefix + values = ssm_provider.get_multiple("/my/path/prefix") + for k, v in values.items(): + print(f"{k}: {v}") +``` + +```python:title=secrets.py +from aws_lambda_powertools.utilities import parameters +from botocore.config import Config + +config = Config(region_name="us-west-1") +secrets_provider = parameters.SecretsProvider(config=config) + +def handler(event, context): + # Retrieve a single secret + value = secrets_provider.get("my-secret") +``` + +## Transform values + +You can transform the incoming values if they are storing in JSON or Base64 format using the `transform` argument for the `get()` and `get_multiple()` methods. This is also supported by the helper functions. + +```python:title=transform.py +from aws_lambda_powertools.utilities import parameters + +ssm_provider = parameters.SSMProvider() + +def handler(event, context): + # Transform a JSON string + value_from_json = ssm_provider.get("/my/json/parameter", transform="json") + + # Transform a Base64 encoded string + value_from_binary = ssm_provider.get("/my/binary/parameter", transform="binary") +``` + +## DynamoDB provider options + +The Amazon DynamoDB provider supports four additional arguments at initialization: + +* __table_name__ (mandatory): Name of the DynamoDB table containing the parameter values. +* __key_attr__: Hash key for the DynamoDB table, defaults to `id`. +* __sort_attr__: Range key for the DynamoDB table , defaults to `sk`. You don't need to set this if you don't use the `get_multiple()` method. +* __value_attr__: Name of the attribute containing the parameter value, defaults to `value`. + +```python:title=dynamodb.py +from aws_lambda_powertools.utilities import parameters + +dynamodb_provider = parameters.DynamoDBProvider( + table_name="my-table", + key_attr="MyKeyAttr", + sort_attr="MySortAttr", + value_attr="MyvalueAttr" +) + +def handler(event, context): + value = dynamodb_provider.get("my-parameter") +``` + +## Parameter Store provider options + +The AWS Systems Manager Parameter Store provider supports two additional arguments for the `get()` and `get_multiple()` methods: + +* **decrypt**: Will automatically decrypt the parameter. +* **recursive**: For `get_multiple()` only, will fetch all parameter values recursively based on a path prefix. Defaults to `True`. + +```python:title=ssm.py +from aws_lambda_powertools.utilities import parameters + +ssm_provider = parameters.SSMProvider() + +def handler(event, context): + decrypted_value = ssm_provider.get("/my/encrypted/parameter", decrypt=True) + + no_recursive_values = ssm_provider.get_multiple("/my/path/prefix", recursive=False) +``` + +## Partial transform failures with `get_multiple()` + +If you use `transform` with `get_multiple()`, you can have a single malformed parameter value. To prevent failing the entire request, the method will return a `None` value for the parameters that failed to transform. You can override this by setting the `raise_on_transform_error` argument to `True`. + +For example, if you have three parameters (*/param/a*, */param/b* and */param/c*) but */param/c* is malformed: + +```python:title=partial_failures.py +from aws_lambda_powertools.utilities import parameters + +ssm_provider = parameters.SSMProvider() + +def handler(event, context): + # This will display: + # /param/a: [some value] + # /param/b: [some value] + # /param/c: None + values = ssm_provider.get_multiple("/param", transform="json") + for k, v in values.items(): + print(f"{k}: {v}") + + # This will raise a TransformParameterError exception + values = ssm_provider.get_multiple("/param", transform="json", raise_on_transform_error=True) +``` diff --git a/docs/gatsby-config.js b/docs/gatsby-config.js index f62c2d4e30e..d518ee8e715 100644 --- a/docs/gatsby-config.js +++ b/docs/gatsby-config.js @@ -31,6 +31,7 @@ module.exports = { ], 'Utilities': [ 'utilities/middleware_factory', + 'utilities/parameters', ], }, navConfig: { From 616a98d36cf97303e4c3003e4b861c1cc32db354 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Wed, 19 Aug 2020 14:01:27 +0200 Subject: [PATCH 27/30] docs: add passing arguments to SDK --- README.md | 1 + docs/content/utilities/parameters.mdx | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/README.md b/README.md index 234ba219a32..f608489a756 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ A suite of utilities for AWS Lambda Functions that makes tracing with AWS X-Ray, * **[Logging](https://awslabs.github.io/aws-lambda-powertools-python/core/logger/)** - Structured logging made easier, and decorator to enrich structured logging with key Lambda context details * **[Metrics](https://awslabs.github.io/aws-lambda-powertools-python/core/metrics/)** - Custom Metrics created asynchronously via CloudWatch Embedded Metric Format (EMF) * **[Bring your own middleware](https://awslabs.github.io/aws-lambda-powertools-python/utilities/middleware_factory/)** - Decorator factory to create your own middleware to run logic before, and after each Lambda invocation +* **[Parameters utility](https://awslabs.github.io/aws-lambda-powertools-python/utilities/parameters/)** - Retrieve and cache parameter values ### Installation diff --git a/docs/content/utilities/parameters.mdx b/docs/content/utilities/parameters.mdx index 8f34713cd2a..57aee701f7f 100644 --- a/docs/content/utilities/parameters.mdx +++ b/docs/content/utilities/parameters.mdx @@ -170,3 +170,17 @@ def handler(event, context): # This will raise a TransformParameterError exception values = ssm_provider.get_multiple("/param", transform="json", raise_on_transform_error=True) ``` + +## Passing arguments to the underlying SDK + +You can use arbitrary keyword arguments to pass it directly to the underlying SDK method. + +```python:title=ssm.py +from aws_lambda_powertools.utilities import parameters + +secrets_provider = parameters.SecretsProvider() + +def handler(event, context): + # The 'VersionId' argument will be passed to the underlying get_secret_value() call. + value = secrets_provider.get("my-secret", VersionId="e62ec170-6b01-48c7-94f3-d7497851a8d2") +``` From 68f6beb05f1a819f88c523be374e3d4a03d753c7 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Fri, 21 Aug 2020 14:12:19 +0200 Subject: [PATCH 28/30] docs: restructure based on feedback --- README.md | 2 +- docs/content/index.mdx | 2 +- docs/content/utilities/parameters.mdx | 242 ++++++++++++++++++++------ 3 files changed, 186 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index f608489a756..68090913dbe 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ A suite of utilities for AWS Lambda Functions that makes tracing with AWS X-Ray, * **[Logging](https://awslabs.github.io/aws-lambda-powertools-python/core/logger/)** - Structured logging made easier, and decorator to enrich structured logging with key Lambda context details * **[Metrics](https://awslabs.github.io/aws-lambda-powertools-python/core/metrics/)** - Custom Metrics created asynchronously via CloudWatch Embedded Metric Format (EMF) * **[Bring your own middleware](https://awslabs.github.io/aws-lambda-powertools-python/utilities/middleware_factory/)** - Decorator factory to create your own middleware to run logic before, and after each Lambda invocation -* **[Parameters utility](https://awslabs.github.io/aws-lambda-powertools-python/utilities/parameters/)** - Retrieve and cache parameter values +* **[Parameters utility](https://awslabs.github.io/aws-lambda-powertools-python/utilities/parameters/)** - Retrieve and cache parameter values from Parameter Store, Secrets Manager, or DynamoDB ### Installation diff --git a/docs/content/index.mdx b/docs/content/index.mdx index a5e27f469d3..e5c2688ecc7 100644 --- a/docs/content/index.mdx +++ b/docs/content/index.mdx @@ -18,7 +18,7 @@ Powertools is available in PyPi. You can use your favourite dependency managemen * [Logging](./core/logger) - Structured logging made easier, and decorator to enrich structured logging with key Lambda context details * [Metrics](./core/metrics) - Custom Metrics created asynchronously via CloudWatch Embedded Metric Format (EMF) * [Bring your own middleware](./utilities/middleware_factory) - Decorator factory to create your own middleware to run logic before, and after each Lambda invocation -* [Parameters utility](./utilities/parameters) - Retrieve parameter values and cache them for a specific amount of time +* [Parameters utility](./utilities/parameters) - Retrieve parameter values from AWS Systems Manager Parameter Store, AWS Secrets Manager, or Amazon DynamoDB, and cache them for a specific amount of time ## Tenets diff --git a/docs/content/utilities/parameters.mdx b/docs/content/utilities/parameters.mdx index 57aee701f7f..07b2cfdb4eb 100644 --- a/docs/content/utilities/parameters.mdx +++ b/docs/content/utilities/parameters.mdx @@ -1,5 +1,5 @@ --- -title: Parameters utility +title: Parameters description: Utility --- @@ -10,57 +10,34 @@ The parameters utility provides a way to retrieve parameter values from [AWS Sys **Key features** * Retrieve one or multiple parameters from the underlying provider -* Cache parameter values for a given amount of time - * By default, parameters are cached for 5 seconds +* Cache parameter values for a given amount of time (defaults to 5 seconds) * Transform parameter values from JSON or base 64 encoded strings -## High-level functions +## SSM Parameter Store -This utility provides high-level functions for quick use. These support both the AWS Systems Manager Parameter Store and AWS Secrets Manager. +You can retrieve a single parameter using `get_parameter` high-level function. For multiple parameters, you can use `get_parameters` and pass a path to retrieve them recursively. -```python:title=ssm.py +```python:title=ssm_parameter_store.py from aws_lambda_powertools.utilities import parameters def handler(event, context): # Retrieve a single parameter value = parameters.get_parameter("/my/parameter") - # Retrieve multiple parameters from a path prefix + # Retrieve multiple parameters from a path prefix recursively # This returns a dict with the parameter name as key values = parameters.get_parameters("/my/path/prefix") for k, v in values.items(): print(f"{k}: {v}") ``` -```python:title=secrets.py -from aws_lambda_powertools.utilities import parameters - -def handler(event, context): - # Retrieve a single secret - value = parameters.get_secret("my-secret") -``` - -## Provider classes +### SSMProvider class -Alternatively, you can use the provider classes, which give more flexibility, such as the ability to configure the underlying SDK client. - -```python:title=dynamodb.py -from aws_lambda_powertools.utilities import parameters - -dynamodb_provider = parameters.DynamoDBProvider(table_name="my-table") - -def handler(event, context): - # Retrieve a value from DynamoDB - value = dynamodb_provider.get("my-parameter") +Alternatively, you can use the `SSMProvider` class, which give more flexibility, such as the ability to configure the underlying SDK client. - # Retrieve multiple values by performing a Query on the DynamoDB table - # This returns a dict with the sort key attribute as dict key. - values = dynamodb_provider.get_multiple("my-hash-key") - for k, v in values.items(): - print(f"{k}: {v}") -``` +This can be used to retrieve values from other regions, change the retry behavior, etc. -```python:title=ssm.py +```python:title=ssm_parameter_store.py from aws_lambda_powertools.utilities import parameters from botocore.config import Config @@ -77,7 +54,47 @@ def handler(event, context): print(f"{k}: {v}") ``` -```python:title=secrets.py +### SSMProvider arguments + +The AWS Systems Manager Parameter Store provider supports two additional arguments for the `get()` and `get_multiple()` methods: + +| Parameter | Default | Description | +|---------------|---------|-------------| +| **decrypt** | `False` | Will automatically decrypt the parameter. | +| **recursive** | `True` | For `get_multiple()` only, will fetch all parameter values recursively based on a path prefix. | + +**Example:** + +```python:title=ssm_parameter_store.py +from aws_lambda_powertools.utilities import parameters + +ssm_provider = parameters.SSMProvider() + +def handler(event, context): + decrypted_value = ssm_provider.get("/my/encrypted/parameter", decrypt=True) + + no_recursive_values = ssm_provider.get_multiple("/my/path/prefix", recursive=False) +``` + +## Secrets Manager + +For secrets stored in Secrets Manager, use `get_secret`. + +```python:title=secrets_manager.py +from aws_lambda_powertools.utilities import parameters + +def handler(event, context): + # Retrieve a single secret + value = parameters.get_secret("my-secret") +``` + +### SecretsProvider class + +Alternatively, you can use the `SecretsProvider` class, which give more flexibility, such as the ability to configure the underlying SDK client. + +This can be used to retrieve values from other regions, change the retry behavior, etc. + +```python:title=secrets_manager.py from aws_lambda_powertools.utilities import parameters from botocore.config import Config @@ -89,31 +106,40 @@ def handler(event, context): value = secrets_provider.get("my-secret") ``` -## Transform values +## DynamoDB -You can transform the incoming values if they are storing in JSON or Base64 format using the `transform` argument for the `get()` and `get_multiple()` methods. This is also supported by the helper functions. +To use the DynamoDB provider, you need to import and instantiate the `DynamoDBProvider` class. -```python:title=transform.py +The DynamoDB Provider does not have any high-level functions, as it needs to know the name of the DynamoDB table containing the parameters. + +**Example:** + +```python:title=dynamodb.py from aws_lambda_powertools.utilities import parameters -ssm_provider = parameters.SSMProvider() +dynamodb_provider = parameters.DynamoDBProvider(table_name="my-table") def handler(event, context): - # Transform a JSON string - value_from_json = ssm_provider.get("/my/json/parameter", transform="json") + # Retrieve a value from DynamoDB + value = dynamodb_provider.get("my-parameter") - # Transform a Base64 encoded string - value_from_binary = ssm_provider.get("/my/binary/parameter", transform="binary") + # Retrieve multiple values by performing a Query on the DynamoDB table + # This returns a dict with the sort key attribute as dict key. + values = dynamodb_provider.get_multiple("my-hash-key") + for k, v in values.items(): + print(f"{k}: {v}") ``` -## DynamoDB provider options +### DynamoDBProvider options The Amazon DynamoDB provider supports four additional arguments at initialization: -* __table_name__ (mandatory): Name of the DynamoDB table containing the parameter values. -* __key_attr__: Hash key for the DynamoDB table, defaults to `id`. -* __sort_attr__: Range key for the DynamoDB table , defaults to `sk`. You don't need to set this if you don't use the `get_multiple()` method. -* __value_attr__: Name of the attribute containing the parameter value, defaults to `value`. +| Parameter | Mandatory | Default | Description | +|----------------|-----------|---------|-------------| +| **table_name** | **Yes** | *(N/A)* | Name of the DynamoDB table containing the parameter values. +| **key_attr** | No | `id` | Hash key for the DynamoDB table. +| **sort_attr** | No | `sk` | Range key for the DynamoDB table. You don't need to set this if you don't use the `get_multiple()` method. +| **value_attr** | No | `value` | Name of the attribute containing the parameter value. ```python:title=dynamodb.py from aws_lambda_powertools.utilities import parameters @@ -129,27 +155,127 @@ def handler(event, context): value = dynamodb_provider.get("my-parameter") ``` -## Parameter Store provider options +### DynamoDB table structure -The AWS Systems Manager Parameter Store provider supports two additional arguments for the `get()` and `get_multiple()` methods: +When using the default options, if you want to retrieve only single parameters, your table should be structured as such, assuming a parameter named **my-param** with a value of **my-value**. The `id` attribute should be the [partition key](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.CoreComponents.html#HowItWorks.CoreComponents.PrimaryKey) for that table. + +| `id` | `value` | +|----------|----------| +| my-param | my-value | + +With this table, when you do a `dynamodb_provider.get("my-param")` call, this will return `my-value`. + +**Multiple values:** + +If you want to be able to retrieve multiple parameters at once sharing the same `id`, your table needs to contain a sort key name `sk`. For example, if you want to retrieve multiple parameters having `my-params` as ID: + +| `id` | `sk` | `value` | +|-----------|---------|------------| +| my-params | param-a | my-value-a | +| my-params | param-b | my-value-b | +| my-params | param-c | my-value-c | + +With this table, when you do a `dynamodb_provider.get_multiple("my-params")` call, you will receive the following dict as a response: + +``` +{ + "param-a": "my-value-a", + "param-b": "my-value-b", + "param-c": "my-value-c" +} +``` + +## Create your own provider + +If you want to use a parameter store other than DynamoDB, Parameter store or Secrets Manager, you can create a custom provider that inherits from the base class. You then need to create your own `_get()` and `_get_multiple()` methods that will handle retrieving data from your parameter provider. + +All transformation and caching logic is handled by the `get()` and `get_multiple()` methods from the base provider class. + +```python:title=custom_provider.py +import copy + +from aws_lambda_powertools.utilities import BaseProvider +import boto3 + +class S3Provider(BaseProvider): + bucket_name = None + client = None + + def __init__(self, bucket_name: str): + # Initialize the client to your custom parameter store + # E.g.: + + self.bucket_name = bucket_name + self.client = boto3.client("s3") + + def _get(self, name: str, **sdk_options) -> str: + # Retrieve a single value + # E.g.: + + sdk_options["Bucket"] = self.bucket_name + sdk_options["Key"] = name + + response = self.client.get_object(**sdk_options) + return -* **decrypt**: Will automatically decrypt the parameter. -* **recursive**: For `get_multiple()` only, will fetch all parameter values recursively based on a path prefix. Defaults to `True`. + def _get_multiple(self, path: str, **sdk_options) -> Dict[str, str]: + # Retrieve multiple values + # E.g.: -```python:title=ssm.py + list_sdk_options = copy.deepcopy(sdk_options) + + list_sdk_options["Bucket"] = self.bucket_name + list_sdk_options["Prefix"] = path + + list_response = self.client.list_objects_v2(**list_sdk_options) + + parameters = {} + + for obj in list_response.get("Contents", []): + get_sdk_options = copy.deepcopy(sdk_options) + + get_sdk_options["Bucket"] = self.bucket_name + get_sdk_options["Key"] = obj["Key"] + + get_response = self.client.get_object(**get_sdk_options) + + parameters[obj["Key"]] = get_response["Body"].read().decode() + + return parameters + +``` + +## Transform values + +You can transform the incoming values if they are storing in JSON or Base64 format using the `transform` argument for the `get()` and `get_multiple()` methods. This is also supported by the helper functions. + +```python:title=transform.py from aws_lambda_powertools.utilities import parameters ssm_provider = parameters.SSMProvider() def handler(event, context): - decrypted_value = ssm_provider.get("/my/encrypted/parameter", decrypt=True) + # Transform a JSON string + value_from_json = ssm_provider.get("/my/json/parameter", transform="json") - no_recursive_values = ssm_provider.get_multiple("/my/path/prefix", recursive=False) + # Transform a Base64 encoded string + value_from_binary = ssm_provider.get("/my/binary/parameter", transform="binary") ``` -## Partial transform failures with `get_multiple()` +You can also use the `transform` argument with high-level functions: + +```python:title=transform.py +from aws_lambda_powertools.utilities import parameters + +def handler(event, context): + value_from_json = parameters.get_parameter("/my/json/parameter", transform="json") +``` + +### Partial transform failures with `get_multiple()` + +If you use `transform` with `get_multiple()`, you can have a single malformed parameter value. To prevent failing the entire request, the method will return a `None` value for the parameters that failed to transform. -If you use `transform` with `get_multiple()`, you can have a single malformed parameter value. To prevent failing the entire request, the method will return a `None` value for the parameters that failed to transform. You can override this by setting the `raise_on_transform_error` argument to `True`. +You can override this by setting the `raise_on_transform_error` argument to `True`. If you do so, a single transform error will raise a `TransformParameterError` exception. For example, if you have three parameters (*/param/a*, */param/b* and */param/c*) but */param/c* is malformed: @@ -171,11 +297,11 @@ def handler(event, context): values = ssm_provider.get_multiple("/param", transform="json", raise_on_transform_error=True) ``` -## Passing arguments to the underlying SDK +## Additional SDK arguments You can use arbitrary keyword arguments to pass it directly to the underlying SDK method. -```python:title=ssm.py +```python:title=ssm_parameter_store.py from aws_lambda_powertools.utilities import parameters secrets_provider = parameters.SecretsProvider() From dd3053dc92a0403b031e05b774a26361c4a51836 Mon Sep 17 00:00:00 2001 From: Nicolas Moutschen Date: Fri, 21 Aug 2020 15:23:10 +0200 Subject: [PATCH 29/30] docs: tweaks based on feedback --- docs/content/utilities/parameters.mdx | 107 +++++++++++++++++--------- 1 file changed, 72 insertions(+), 35 deletions(-) diff --git a/docs/content/utilities/parameters.mdx b/docs/content/utilities/parameters.mdx index 07b2cfdb4eb..9cd19c1e148 100644 --- a/docs/content/utilities/parameters.mdx +++ b/docs/content/utilities/parameters.mdx @@ -13,6 +13,21 @@ The parameters utility provides a way to retrieve parameter values from [AWS Sys * Cache parameter values for a given amount of time (defaults to 5 seconds) * Transform parameter values from JSON or base 64 encoded strings +**IAM Permissions** + +This utility requires additional permissions to work as expected. See the table below: + +| Provider | Function/Method | IAM Permission | +|---------------------|---------------------------------|---------------------------------| +| SSM Parameter Store | `get_parameter` | `ssm:GetParameter` | +| SSM Parameter Store | `get_parameters` | `ssm:GetParametersByPath` | +| SSM Parameter Store | `SSMProvider.get` | `ssm:GetParameter` | +| SSM Parameter Store | `SSMProvider.get_multiple` | `ssm:GetParametersByPath` | +| Secrets Manager | `get_secret` | `secretsmanager:GetSecretValue` | +| Secrets Manager | `SecretsManager.get` | `secretsmanager:GetSecretValue` | +| DynamoDB | `DynamoDBProvider.get` | `dynamodb:GetItem` | +| DynamoDB | `DynamoDBProvider.get_multiple` | `dynamodb:Query` | + ## SSM Parameter Store You can retrieve a single parameter using `get_parameter` high-level function. For multiple parameters, you can use `get_parameters` and pass a path to retrieve them recursively. @@ -54,7 +69,7 @@ def handler(event, context): print(f"{k}: {v}") ``` -### SSMProvider arguments +**Additional arguments** The AWS Systems Manager Parameter Store provider supports two additional arguments for the `get()` and `get_multiple()` methods: @@ -112,7 +127,15 @@ To use the DynamoDB provider, you need to import and instantiate the `DynamoDBPr The DynamoDB Provider does not have any high-level functions, as it needs to know the name of the DynamoDB table containing the parameters. -**Example:** +**DynamoDB table structure** + +When using the default options, if you want to retrieve only single parameters, your table should be structured as such, assuming a parameter named **my-parameter** with a value of **my-value**. The `id` attribute should be the [partition key](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.CoreComponents.html#HowItWorks.CoreComponents.PrimaryKey) for that table. + +| `id` | `value` | +|--------------|----------| +| my-parameter | my-value | + +With this table, when you do a `dynamodb_provider.get("my-param")` call, this will return `my-value`. ```python:title=dynamodb.py from aws_lambda_powertools.utilities import parameters @@ -122,7 +145,36 @@ dynamodb_provider = parameters.DynamoDBProvider(table_name="my-table") def handler(event, context): # Retrieve a value from DynamoDB value = dynamodb_provider.get("my-parameter") +``` + +**Retrieve multiple values** + +If you want to be able to retrieve multiple parameters at once sharing the same `id`, your table needs to contain a sort key name `sk`. For example, if you want to retrieve multiple parameters having `my-hash-key` as ID: + +| `id` | `sk` | `value` | +|-------------|---------|------------| +| my-hash-key | param-a | my-value-a | +| my-hash-key | param-b | my-value-b | +| my-hash-key | param-c | my-value-c | +With this table, when you do a `dynamodb_provider.get_multiple("my-hash-key")` call, you will receive the following dict as a response: + +``` +{ + "param-a": "my-value-a", + "param-b": "my-value-b", + "param-c": "my-value-c" +} +``` + +**Example:** + +```python:title="dynamodb_multiple.py +from aws_lambda_powertools.utilities import parameters + +dynamodb_provider = parameters.DynamoDBProvider(table_name="my-table") + +def handler(event, context): # Retrieve multiple values by performing a Query on the DynamoDB table # This returns a dict with the sort key attribute as dict key. values = dynamodb_provider.get_multiple("my-hash-key") @@ -130,7 +182,7 @@ def handler(event, context): print(f"{k}: {v}") ``` -### DynamoDBProvider options +**Additional arguments** The Amazon DynamoDB provider supports four additional arguments at initialization: @@ -155,42 +207,14 @@ def handler(event, context): value = dynamodb_provider.get("my-parameter") ``` -### DynamoDB table structure - -When using the default options, if you want to retrieve only single parameters, your table should be structured as such, assuming a parameter named **my-param** with a value of **my-value**. The `id` attribute should be the [partition key](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.CoreComponents.html#HowItWorks.CoreComponents.PrimaryKey) for that table. - -| `id` | `value` | -|----------|----------| -| my-param | my-value | - -With this table, when you do a `dynamodb_provider.get("my-param")` call, this will return `my-value`. - -**Multiple values:** - -If you want to be able to retrieve multiple parameters at once sharing the same `id`, your table needs to contain a sort key name `sk`. For example, if you want to retrieve multiple parameters having `my-params` as ID: - -| `id` | `sk` | `value` | -|-----------|---------|------------| -| my-params | param-a | my-value-a | -| my-params | param-b | my-value-b | -| my-params | param-c | my-value-c | - -With this table, when you do a `dynamodb_provider.get_multiple("my-params")` call, you will receive the following dict as a response: - -``` -{ - "param-a": "my-value-a", - "param-b": "my-value-b", - "param-c": "my-value-c" -} -``` - ## Create your own provider -If you want to use a parameter store other than DynamoDB, Parameter store or Secrets Manager, you can create a custom provider that inherits from the base class. You then need to create your own `_get()` and `_get_multiple()` methods that will handle retrieving data from your parameter provider. +You can create your own custom parameter store provider by inheriting the `BaseProvider` class, and implementing both `_get()` and `_get_multiple()` methods to retrieve a single, or multiple parameters from your custom store. All transformation and caching logic is handled by the `get()` and `get_multiple()` methods from the base provider class. +Here is an example implementation using S3 as a custom parameter store: + ```python:title=custom_provider.py import copy @@ -247,7 +271,7 @@ class S3Provider(BaseProvider): ## Transform values -You can transform the incoming values if they are storing in JSON or Base64 format using the `transform` argument for the `get()` and `get_multiple()` methods. This is also supported by the helper functions. +For parameters stored in JSON or Base64 format, you can use the `transform` argument for deserialization - The `transform` argument is available across all providers, including the high level functions. ```python:title=transform.py from aws_lambda_powertools.utilities import parameters @@ -310,3 +334,16 @@ def handler(event, context): # The 'VersionId' argument will be passed to the underlying get_secret_value() call. value = secrets_provider.get("my-secret", VersionId="e62ec170-6b01-48c7-94f3-d7497851a8d2") ``` + +Here is the mapping between this utility's functions and methods and the underlying SDK: + +| Provider | Function/Method | Client name | Function name | +|---------------------|---------------------------------|-------------|---------------| +| SSM Parameter Store | `get_parameter` | `ssm` | [get_parameter](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ssm.html#SSM.Client.get_parameter) | +| SSM Parameter Store | `get_parameters` | `ssm` | [get_parameters_by_path](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ssm.html#SSM.Client.get_parameters_by_path) | +| SSM Parameter Store | `SSMProvider.get` | `ssm` | [get_parameter](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ssm.html#SSM.Client.get_parameter) | +| SSM Parameter Store | `SSMProvider.get_multiple` | `ssm` | [get_parameters_by_path](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ssm.html#SSM.Client.get_parameters_by_path) | +| Secrets Manager | `get_secret` | `secretsmanager` | [get_secret_value](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/secretsmanager.html#SecretsManager.Client.get_secret_value) | +| Secrets Manager | `SecretsManager.get` | `secretsmanager` | [get_secret_value](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/secretsmanager.html#SecretsManager.Client.get_secret_value) | +| DynamoDB | `DynamoDBProvider.get` | `dynamodb` ([Table resource](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#table)) | [get_item](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table.get_item) +| DynamoDB | `DynamoDBProvider.get_multiple` | `dynamodb` ([Table resource](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#table)) | [query](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table.query) From 7b87dfa35d050c0622f756e0037cb617de0abfa2 Mon Sep 17 00:00:00 2001 From: Heitor Lessa Date: Fri, 21 Aug 2020 15:45:15 +0200 Subject: [PATCH 30/30] improv: iam permissions table Merge both high level and class provider functions and methods, since they require the same IAM permission. --- docs/content/utilities/parameters.mdx | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/docs/content/utilities/parameters.mdx b/docs/content/utilities/parameters.mdx index 9cd19c1e148..b40bfe2c885 100644 --- a/docs/content/utilities/parameters.mdx +++ b/docs/content/utilities/parameters.mdx @@ -17,16 +17,13 @@ The parameters utility provides a way to retrieve parameter values from [AWS Sys This utility requires additional permissions to work as expected. See the table below: -| Provider | Function/Method | IAM Permission | -|---------------------|---------------------------------|---------------------------------| -| SSM Parameter Store | `get_parameter` | `ssm:GetParameter` | -| SSM Parameter Store | `get_parameters` | `ssm:GetParametersByPath` | -| SSM Parameter Store | `SSMProvider.get` | `ssm:GetParameter` | -| SSM Parameter Store | `SSMProvider.get_multiple` | `ssm:GetParametersByPath` | -| Secrets Manager | `get_secret` | `secretsmanager:GetSecretValue` | -| Secrets Manager | `SecretsManager.get` | `secretsmanager:GetSecretValue` | -| DynamoDB | `DynamoDBProvider.get` | `dynamodb:GetItem` | -| DynamoDB | `DynamoDBProvider.get_multiple` | `dynamodb:Query` | +Provider | Function/Method | IAM Permission +------------------------------------------------- | ------------------------------------------------- | --------------------------------------------------------------------------------- +SSM Parameter Store | `get_parameter`, `SSMProvider.get` | `ssm:GetParameter` +SSM Parameter Store | `get_parameters`, `SSMProvider.get_multiple` | `ssm:GetParametersByPath` +Secrets Manager | `get_secret`, `SecretsManager.get` | `secretsmanager:GetSecretValue` +DynamoDB | `DynamoDBProvider.get` | `dynamodb:GetItem` +DynamoDB | `DynamoDBProvider.get_multiple` | `dynamodb:Query` ## SSM Parameter Store