Fix botocore tests

newrelic/hooks/external_botocore.py

@@ -12,15 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from newrelic.api.message_trace import message_trace
 from newrelic.api.datastore_trace import datastore_trace
 from newrelic.api.external_trace import ExternalTrace
+from newrelic.api.message_trace import message_trace
 from newrelic.common.object_wrapper import wrap_function_wrapper
 
 
 def extract_sqs(*args, **kwargs):
-    queue_value = kwargs.get('QueueUrl', 'Unknown')
-    return queue_value.rsplit('/', 1)[-1]
+    queue_value = kwargs.get("QueueUrl", "Unknown")
+    return queue_value.rsplit("/", 1)[-1]
 
 
 def extract(argument_names, default=None):
@@ -41,42 +41,27 @@ def extractor_string(*args, **kwargs):
 
 
 CUSTOM_TRACE_POINTS = {
-    ('sns', 'publish'): message_trace(
-            'SNS', 'Produce', 'Topic',
-            extract(('TopicArn', 'TargetArn'), 'PhoneNumber')),
-    ('dynamodb', 'put_item'): datastore_trace(
-            'DynamoDB', extract('TableName'), 'put_item'),
-    ('dynamodb', 'get_item'): datastore_trace(
-            'DynamoDB', extract('TableName'), 'get_item'),
-    ('dynamodb', 'update_item'): datastore_trace(
-            'DynamoDB', extract('TableName'), 'update_item'),
-    ('dynamodb', 'delete_item'): datastore_trace(
-            'DynamoDB', extract('TableName'), 'delete_item'),
-    ('dynamodb', 'create_table'): datastore_trace(
-            'DynamoDB', extract('TableName'), 'create_table'),
-    ('dynamodb', 'delete_table'): datastore_trace(
-            'DynamoDB', extract('TableName'), 'delete_table'),
-    ('dynamodb', 'query'): datastore_trace(
-            'DynamoDB', extract('TableName'), 'query'),
-    ('dynamodb', 'scan'): datastore_trace(
-            'DynamoDB', extract('TableName'), 'scan'),
-    ('sqs', 'send_message'): message_trace(
-            'SQS', 'Produce', 'Queue', extract_sqs),
-    ('sqs', 'send_message_batch'): message_trace(
-            'SQS', 'Produce', 'Queue', extract_sqs),
-    ('sqs', 'receive_message'): message_trace(
-            'SQS', 'Consume', 'Queue', extract_sqs),
+    ("sns", "publish"): message_trace("SNS", "Produce", "Topic", extract(("TopicArn", "TargetArn"), "PhoneNumber")),
+    ("dynamodb", "put_item"): datastore_trace("DynamoDB", extract("TableName"), "put_item"),
+    ("dynamodb", "get_item"): datastore_trace("DynamoDB", extract("TableName"), "get_item"),
+    ("dynamodb", "update_item"): datastore_trace("DynamoDB", extract("TableName"), "update_item"),
+    ("dynamodb", "delete_item"): datastore_trace("DynamoDB", extract("TableName"), "delete_item"),
+    ("dynamodb", "create_table"): datastore_trace("DynamoDB", extract("TableName"), "create_table"),
+    ("dynamodb", "delete_table"): datastore_trace("DynamoDB", extract("TableName"), "delete_table"),
+    ("dynamodb", "query"): datastore_trace("DynamoDB", extract("TableName"), "query"),
+    ("dynamodb", "scan"): datastore_trace("DynamoDB", extract("TableName"), "scan"),
+    ("sqs", "send_message"): message_trace("SQS", "Produce", "Queue", extract_sqs),
+    ("sqs", "send_message_batch"): message_trace("SQS", "Produce", "Queue", extract_sqs),
+    ("sqs", "receive_message"): message_trace("SQS", "Consume", "Queue", extract_sqs),
 }
 
 
-def bind__create_api_method(py_operation_name, operation_name, service_model,
-        *args, **kwargs):
+def bind__create_api_method(py_operation_name, operation_name, service_model, *args, **kwargs):
     return (py_operation_name, service_model)
 
 
 def _nr_clientcreator__create_api_method_(wrapped, instance, args, kwargs):
-    (py_operation_name, service_model) = \
-            bind__create_api_method(*args, **kwargs)
+    (py_operation_name, service_model) = bind__create_api_method(*args, **kwargs)
 
     service_name = service_model.service_name.lower()
     tracer = CUSTOM_TRACE_POINTS.get((service_name, py_operation_name))
@@ -95,30 +80,27 @@ def _bind_make_request_params(operation_model, request_dict, *args, **kwargs):
 
 def _nr_endpoint_make_request_(wrapped, instance, args, kwargs):
     operation_model, request_dict = _bind_make_request_params(*args, **kwargs)
-    url = request_dict.get('url', '')
-    method = request_dict.get('method', None)
-
-    with ExternalTrace(library='botocore', url=url, method=method, source=wrapped) as trace:
+    url = request_dict.get("url", "")
+    method = request_dict.get("method", None)
 
+    with ExternalTrace(library="botocore", url=url, method=method, source=wrapped) as trace:
         try:
-            trace._add_agent_attribute('aws.operation', operation_model.name)
+            trace._add_agent_attribute("aws.operation", operation_model.name)
         except:
             pass
 
         result = wrapped(*args, **kwargs)
         try:
-            request_id = result[1]['ResponseMetadata']['RequestId']
-            trace._add_agent_attribute('aws.requestId', request_id)
+            request_id = result[1]["ResponseMetadata"]["RequestId"]
+            trace._add_agent_attribute("aws.requestId", request_id)
         except:
             pass
         return result
 
 
 def instrument_botocore_endpoint(module):
-    wrap_function_wrapper(module, 'Endpoint.make_request',
-            _nr_endpoint_make_request_)
+    wrap_function_wrapper(module, "Endpoint.make_request", _nr_endpoint_make_request_)
 
 
 def instrument_botocore_client(module):
-    wrap_function_wrapper(module, 'ClientCreator._create_api_method',
-            _nr_clientcreator__create_api_method_)
+    wrap_function_wrapper(module, "ClientCreator._create_api_method", _nr_clientcreator__create_api_method_)

tests/external_botocore/conftest.py

@@ -12,19 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import pytest
-
-from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture  # noqa: F401; pylint: disable=W0611
-
+from testing_support.fixtures import (  # noqa: F401; pylint: disable=W0611
+    collector_agent_registration_fixture,
+    collector_available_fixture,
+)
 
 _default_settings = {
-    'transaction_tracer.explain_threshold': 0.0,
-    'transaction_tracer.transaction_threshold': 0.0,
-    'transaction_tracer.stack_trace_threshold': 0.0,
-    'debug.log_data_collector_payloads': True,
-    'debug.record_transaction_failure': True,
+    "transaction_tracer.explain_threshold": 0.0,
+    "transaction_tracer.transaction_threshold": 0.0,
+    "transaction_tracer.stack_trace_threshold": 0.0,
+    "debug.log_data_collector_payloads": True,
+    "debug.record_transaction_failure": True,
 }
 
 collector_agent_registration = collector_agent_registration_fixture(
-        app_name='Python Agent Test (external_botocore)',
-        default_settings=_default_settings)
+    app_name="Python Agent Test (external_botocore)", default_settings=_default_settings
+)

tests/external_boto3/test_boto3_iam.py → tests/external_botocore/test_boto3_iam.py

@@ -17,7 +17,7 @@
 
 import boto3
 import moto
-from testing_support.fixtures import override_application_settings
+from testing_support.fixtures import dt_enabled
 from testing_support.validators.validate_span_events import validate_span_events
 from testing_support.validators.validate_transaction_metrics import (
     validate_transaction_metrics,
@@ -53,7 +53,7 @@
 ]
 
 
-@override_application_settings({"distributed_tracing.enabled": True})
+@dt_enabled
 @validate_span_events(exact_agents={"http.url": "https://iam.amazonaws.com/"}, count=3)
 @validate_span_events(expected_agents=("aws.requestId",), count=3)
 @validate_span_events(exact_agents={"aws.operation": "CreateUser"}, count=1)

tests/external_boto3/test_boto3_s3.py → tests/external_botocore/test_boto3_s3.py

@@ -18,7 +18,7 @@
 import boto3
 import botocore
 import moto
-from testing_support.fixtures import override_application_settings
+from testing_support.fixtures import dt_enabled
 from testing_support.validators.validate_span_events import validate_span_events
 from testing_support.validators.validate_transaction_metrics import (
     validate_transaction_metrics,
@@ -73,7 +73,7 @@
 ]
 
 
-@override_application_settings({"distributed_tracing.enabled": True})
+@dt_enabled
 @validate_span_events(exact_agents={"aws.operation": "CreateBucket"}, count=1)
 @validate_span_events(exact_agents={"aws.operation": "PutObject"}, count=1)
 @validate_span_events(exact_agents={"aws.operation": "ListObjects"}, count=1)

tests/external_boto3/test_boto3_sns.py → tests/external_botocore/test_boto3_sns.py

@@ -17,7 +17,7 @@
 import boto3
 import moto
 import pytest
-from testing_support.fixtures import override_application_settings
+from testing_support.fixtures import dt_enabled
 from testing_support.validators.validate_span_events import validate_span_events
 from testing_support.validators.validate_transaction_metrics import (
     validate_transaction_metrics,
@@ -45,7 +45,7 @@
 sns_metrics_phone = [("MessageBroker/SNS/Topic" "/Produce/Named/PhoneNumber", 1)]
 
 
-@override_application_settings({"distributed_tracing.enabled": True})
+@dt_enabled
 @validate_span_events(expected_agents=("aws.requestId",), count=2)
 @validate_span_events(exact_agents={"aws.operation": "CreateTopic"}, count=1)
 @validate_span_events(exact_agents={"aws.operation": "Publish"}, count=1)
@@ -74,7 +74,7 @@ def test_publish_to_sns_topic(topic_argument):
     assert "MessageId" in published_message
 
 
-@override_application_settings({"distributed_tracing.enabled": True})
+@dt_enabled
 @validate_span_events(expected_agents=("aws.requestId",), count=3)
 @validate_span_events(exact_agents={"aws.operation": "CreateTopic"}, count=1)
 @validate_span_events(exact_agents={"aws.operation": "Subscribe"}, count=1)

tests/external_botocore/test_botocore_dynamodb.py

@@ -17,7 +17,7 @@
 
 import botocore.session
 import moto
-from testing_support.fixtures import override_application_settings
+from testing_support.fixtures import dt_enabled
 from testing_support.validators.validate_span_events import validate_span_events
 from testing_support.validators.validate_transaction_metrics import (
     validate_transaction_metrics,
@@ -63,7 +63,7 @@
 ]
 
 
-@override_application_settings({"distributed_tracing.enabled": True})
+@dt_enabled
 @validate_span_events(expected_agents=("aws.requestId",), count=8)
 @validate_span_events(exact_agents={"aws.operation": "PutItem"}, count=1)
 @validate_span_events(exact_agents={"aws.operation": "GetItem"}, count=1)
@@ -80,7 +80,7 @@
     background_task=True,
 )
 @background_task()
-@moto.mock_dynamodb2
+@moto.mock_dynamodb
 def test_dynamodb():
     session = botocore.session.get_session()
     client = session.create_client(

tests/external_botocore/test_botocore_ec2.py

@@ -17,7 +17,7 @@
 
 import botocore.session
 import moto
-from testing_support.fixtures import override_application_settings
+from testing_support.fixtures import dt_enabled
 from testing_support.validators.validate_span_events import validate_span_events
 from testing_support.validators.validate_transaction_metrics import (
     validate_transaction_metrics,
@@ -55,7 +55,7 @@
 ]
 
 
-@override_application_settings({"distributed_tracing.enabled": True})
+@dt_enabled
 @validate_span_events(expected_agents=("aws.requestId",), count=3)
 @validate_span_events(exact_agents={"aws.operation": "RunInstances"}, count=1)
 @validate_span_events(exact_agents={"aws.operation": "DescribeInstances"}, count=1)

tests/external_botocore/test_botocore_s3.py

@@ -18,7 +18,7 @@
 import botocore
 import botocore.session
 import moto
-from testing_support.fixtures import override_application_settings
+from testing_support.fixtures import dt_enabled
 from testing_support.validators.validate_span_events import validate_span_events
 from testing_support.validators.validate_transaction_metrics import (
     validate_transaction_metrics,
@@ -67,7 +67,7 @@
 ]
 
 
-@override_application_settings({"distributed_tracing.enabled": True})
+@dt_enabled
 @validate_span_events(exact_agents={"aws.operation": "CreateBucket"}, count=1)
 @validate_span_events(exact_agents={"aws.operation": "PutObject"}, count=1)
 @validate_span_events(exact_agents={"aws.operation": "ListObjects"}, count=1)

tests/external_botocore/test_botocore_sqs.py

@@ -18,7 +18,7 @@
 import botocore.session
 import moto
 import pytest
-from testing_support.fixtures import override_application_settings
+from testing_support.fixtures import dt_enabled
 from testing_support.validators.validate_span_events import validate_span_events
 from testing_support.validators.validate_transaction_metrics import (
     validate_transaction_metrics,
@@ -70,7 +70,7 @@
 ]
 
 
-@override_application_settings({"distributed_tracing.enabled": True})
+@dt_enabled
 @validate_span_events(exact_agents={"aws.operation": "CreateQueue"}, count=1)
 @validate_span_events(exact_agents={"aws.operation": "SendMessage"}, count=1)
 @validate_span_events(exact_agents={"aws.operation": "ReceiveMessage"}, count=1)
@@ -124,7 +124,7 @@ def test_sqs():
     assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200
 
 
-@override_application_settings({"distributed_tracing.enabled": True})
+@dt_enabled
 @validate_transaction_metrics(
     "test_botocore_sqs:test_sqs_malformed",
     scoped_metrics=_sqs_scoped_metrics_malformed,

tests/testing_support/validators/validate_ml_events.py

@@ -24,7 +24,6 @@
 def validate_ml_events(events):
     @function_wrapper
     def _validate_wrapper(wrapped, instance, args, kwargs):
-
         record_called = []
         recorded_events = []
 
@@ -55,7 +54,7 @@ def _validate_ml_events(wrapped, instance, args, kwargs):
             for captured in found_events:
                 if _check_event_attributes(expected, captured, mismatches):
                     matching_ml_events += 1
-            assert matching_ml_events == 1, _event_details(matching_ml_events, events, mismatches)
+            assert matching_ml_events == 1, _event_details(matching_ml_events, found_events, mismatches)
 
         return val
 

tox.ini

@@ -97,7 +97,6 @@ envlist =
     redis-datastore_redis-{py37,py38,py39,py310,py311,pypy38}-redis{0400,latest},
     rediscluster-datastore_rediscluster-{py37,py311,pypy38}-redis{latest},
     python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,pypy27,pypy38},
-    python-external_boto3-{py27,py37,py38,py39,py310,py311}-boto01,
     python-external_botocore-{py37,py38,py39,py310,py311}-botocorelatest,
     python-external_botocore-{py311}-botocore128,
     python-external_botocore-py310-botocore0125,
@@ -251,15 +250,13 @@ deps =
     datastore_redis-redislatest: redis
     datastore_rediscluster-redislatest: redis
     datastore_redis-redis0400: redis<4.1
-    external_boto3-boto01: boto3<2.0
-    external_boto3-boto01: moto<2.0
-    external_boto3-py27: rsa<4.7.1
     external_botocore-botocorelatest: botocore
+    external_botocore-botocorelatest: boto3
     external_botocore-botocore128: botocore<1.29
     external_botocore-botocore0125: botocore<1.26
-    external_botocore-{py37,py38,py39,py310,py311}: moto[awslambda,ec2,iam]<3.0
+    external_botocore-{py37,py38,py39,py310,py311}: moto[awslambda,ec2,iam,sqs]
     external_botocore-py27: rsa<4.7.1
-    external_botocore-py27: moto[awslambda,ec2,iam]<2.0
+    external_botocore-py27: moto[awslambda,ec2,iam,sqs]<2.0
     external_feedparser-feedparser05: feedparser<6
     external_feedparser-feedparser06: feedparser<7
     external_httplib2: httplib2<1.0
@@ -437,7 +434,6 @@ changedir =
     datastore_redis: tests/datastore_redis
     datastore_rediscluster: tests/datastore_rediscluster
     datastore_sqlite: tests/datastore_sqlite
-    external_boto3: tests/external_boto3
     external_botocore: tests/external_botocore
     external_feedparser: tests/external_feedparser
     external_http: tests/external_http