Skip to content

chore(ci): add the Idempotency feature to nox tests #4585

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 20, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions noxfile.py
Original file line number Diff line number Diff line change
@@ -110,6 +110,7 @@ def test_with_boto3_sdk_as_required_package(session: nox.Session):
f"{PREFIX_TESTS_FUNCTIONAL}/feature_flags/_boto3/",
f"{PREFIX_TESTS_UNIT}/data_classes/_boto3/",
f"{PREFIX_TESTS_FUNCTIONAL}/streaming/_boto3/",
f"{PREFIX_TESTS_FUNCTIONAL}/idempotency/_boto3/",
],
extras="aws-sdk",
)
@@ -158,3 +159,37 @@ def test_with_pydantic_required_package(session: nox.Session, pydantic: str):
f"{PREFIX_TESTS_UNIT}/parser/_pydantic/",
],
)


@nox.session()
@nox.parametrize("pydantic", ["1.10", "2.0"])
def test_with_boto3_and_pydantic_required_package(session: nox.Session, pydantic: str):
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I need to install pydantic + boto3 because we export the DynamoDBPersistenceLayer at top level and it requires boto3.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you meant you only needed boto3 when using DynamoDBPersistenceLayer, whereas for this test you also need pydantic because it'll use serializers and models.

"""Tests that only depends for Boto3 + Pydantic library v1 and v2"""
# Idempotency with custom serializer

session.install(f"pydantic>={pydantic}")

build_and_run_test(
session,
folders=[
f"{PREFIX_TESTS_FUNCTIONAL}/idempotency/_pydantic/",
],
extras="aws-sdk",
)


@nox.session()
def test_with_redis_and_boto3_sdk_as_required_package(session: nox.Session):
"""Tests that depends on Redis library"""
# Idempotency - Redis backend

# Our Redis tests requires multiprocess library to simulate Race Condition
session.run("pip", "install", "multiprocess")

build_and_run_test(
session,
folders=[
f"{PREFIX_TESTS_FUNCTIONAL}/idempotency/_redis/",
],
extras="redis,aws-sdk",
)
Empty file.
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -8,7 +8,6 @@
import pytest
from botocore import stub
from botocore.config import Config
from pydantic import BaseModel
from pytest import FixtureRequest
from pytest_mock import MockerFixture

@@ -32,7 +31,6 @@
IdempotencyInconsistentStateError,
IdempotencyInvalidStatusError,
IdempotencyKeyError,
IdempotencyModelTypeError,
IdempotencyNoSerializationModelError,
IdempotencyPersistenceLayerError,
IdempotencyValidationError,
@@ -47,9 +45,6 @@
from aws_lambda_powertools.utilities.idempotency.serialization.dataclass import (
DataclassSerializer,
)
from aws_lambda_powertools.utilities.idempotency.serialization.pydantic import (
PydanticSerializer,
)
from aws_lambda_powertools.utilities.validation import envelopes, validator
from aws_lambda_powertools.warnings import PowertoolsUserWarning
from tests.functional.idempotency.utils import (
@@ -61,7 +56,7 @@
from tests.functional.utils import json_serialize, load_event

TABLE_NAME = "TEST_TABLE"
TESTS_MODULE_PREFIX = "test-func.tests.functional.idempotency.test_idempotency"
TESTS_MODULE_PREFIX = "test-func.tests.functional.idempotency._boto3.test_idempotency"


def get_dataclasses_lib():
@@ -1315,106 +1310,6 @@ def record_handler(record):
assert from_dict_called is False, "in case response is None, from_dict should not be called"


@pytest.mark.parametrize("output_serializer_type", ["explicit", "deduced"])
def test_idempotent_function_serialization_pydantic(output_serializer_type: str):
# GIVEN
config = IdempotencyConfig(use_local_cache=True)
mock_event = {"customer_id": "fake", "transaction_id": "fake-id"}
idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic.<locals>.collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501
persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key)

class PaymentInput(BaseModel):
customer_id: str
transaction_id: str

class PaymentOutput(BaseModel):
customer_id: str
transaction_id: str

if output_serializer_type == "explicit":
output_serializer = PydanticSerializer(
model=PaymentOutput,
)
else:
output_serializer = PydanticSerializer

@idempotent_function(
data_keyword_argument="payment",
persistence_store=persistence_layer,
config=config,
output_serializer=output_serializer,
)
def collect_payment(payment: PaymentInput) -> PaymentOutput:
return PaymentOutput(**payment.dict())

# WHEN
payment = PaymentInput(**mock_event)
first_call: PaymentOutput = collect_payment(payment=payment)
assert first_call.customer_id == payment.customer_id
assert first_call.transaction_id == payment.transaction_id
assert isinstance(first_call, PaymentOutput)
second_call: PaymentOutput = collect_payment(payment=payment)
assert isinstance(second_call, PaymentOutput)
assert second_call.customer_id == payment.customer_id
assert second_call.transaction_id == payment.transaction_id


def test_idempotent_function_serialization_pydantic_failure_no_return_type():
# GIVEN
config = IdempotencyConfig(use_local_cache=True)
mock_event = {"customer_id": "fake", "transaction_id": "fake-id"}
idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic_failure_no_return_type.<locals>.collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501
persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key)

class PaymentInput(BaseModel):
customer_id: str
transaction_id: str

class PaymentOutput(BaseModel):
customer_id: str
transaction_id: str

idempotent_function_decorator = idempotent_function(
data_keyword_argument="payment",
persistence_store=persistence_layer,
config=config,
output_serializer=PydanticSerializer,
)
with pytest.raises(IdempotencyNoSerializationModelError, match="No serialization model was supplied"):

@idempotent_function_decorator
def collect_payment(payment: PaymentInput):
return PaymentOutput(**payment.dict())


def test_idempotent_function_serialization_pydantic_failure_bad_type():
# GIVEN
config = IdempotencyConfig(use_local_cache=True)
mock_event = {"customer_id": "fake", "transaction_id": "fake-id"}
idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic_failure_no_return_type.<locals>.collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501
persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key)

class PaymentInput(BaseModel):
customer_id: str
transaction_id: str

class PaymentOutput(BaseModel):
customer_id: str
transaction_id: str

idempotent_function_decorator = idempotent_function(
data_keyword_argument="payment",
persistence_store=persistence_layer,
config=config,
output_serializer=PydanticSerializer,
)
with pytest.raises(IdempotencyModelTypeError, match="Model type is not inherited from pydantic BaseModel"):

@idempotent_function_decorator
def collect_payment(payment: PaymentInput) -> dict:
return PaymentOutput(**payment.dict())


@pytest.mark.parametrize("output_serializer_type", ["explicit", "deduced"])
def test_idempotent_function_serialization_dataclass(output_serializer_type: str):
# GIVEN
@@ -1493,37 +1388,6 @@ def collect_payment(payment: PaymentInput):
return PaymentOutput(**payment.dict())


def test_idempotent_function_serialization_dataclass_failure_bad_type():
# GIVEN
dataclasses = get_dataclasses_lib()
config = IdempotencyConfig(use_local_cache=True)
mock_event = {"customer_id": "fake", "transaction_id": "fake-id"}
idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic_failure_no_return_type.<locals>.collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501
persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key)

@dataclasses.dataclass
class PaymentInput:
customer_id: str
transaction_id: str

@dataclasses.dataclass
class PaymentOutput:
customer_id: str
transaction_id: str

idempotent_function_decorator = idempotent_function(
data_keyword_argument="payment",
persistence_store=persistence_layer,
config=config,
output_serializer=PydanticSerializer,
)
with pytest.raises(IdempotencyModelTypeError, match="Model type is not inherited from pydantic BaseModel"):

@idempotent_function_decorator
def collect_payment(payment: PaymentInput) -> dict:
return PaymentOutput(**payment.dict())


def test_idempotent_function_arbitrary_args_kwargs():
# Scenario to validate we can use idempotent_function with a function
# with an arbitrary number of args and kwargs
@@ -1804,24 +1668,6 @@ class Foo:
assert as_dict == expected_result


def test_idempotent_function_pydantic():
# Scenario _prepare_data should convert a pydantic to a dict
class Foo(BaseModel):
name: str

expected_result = {"name": "Bar"}
data = Foo(name="Bar")
as_dict = _prepare_data(data)
assert as_dict == data.dict()
assert as_dict == expected_result


@pytest.mark.parametrize("data", [None, "foo", ["foo"], 1, True, {}])
def test_idempotent_function_other(data):
# All other data types should be left as is
assert _prepare_data(data) == data


def test_idempotent_function_dataclass_with_jmespath():
# GIVEN
dataclasses = get_dataclasses_lib()
@@ -1847,29 +1693,6 @@ def collect_payment(payment: Payment):
assert result == payment.transaction_id


def test_idempotent_function_pydantic_with_jmespath():
# GIVEN
config = IdempotencyConfig(event_key_jmespath="transaction_id", use_local_cache=True)
mock_event = {"customer_id": "fake", "transaction_id": "fake-id"}
idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_pydantic_with_jmespath.<locals>.collect_payment#{hash_idempotency_key(mock_event['transaction_id'])}" # noqa E501
persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key)

class Payment(BaseModel):
customer_id: str
transaction_id: str

@idempotent_function(data_keyword_argument="payment", persistence_store=persistence_layer, config=config)
def collect_payment(payment: Payment):
return payment.transaction_id

# WHEN
payment = Payment(**mock_event)
result = collect_payment(payment=payment)

# THEN idempotency key assertion happens at MockPersistenceLayer
assert result == payment.transaction_id


@pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}], indirect=True)
def test_idempotent_lambda_compound_already_completed(
idempotency_config: IdempotencyConfig,
Empty file.
Loading