diff --git a/newrelic/hooks/external_botocore.py b/newrelic/hooks/external_botocore.py index 7d49fbd031..2f2b8a1134 100644 --- a/newrelic/hooks/external_botocore.py +++ b/newrelic/hooks/external_botocore.py @@ -12,15 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from newrelic.api.message_trace import message_trace from newrelic.api.datastore_trace import datastore_trace from newrelic.api.external_trace import ExternalTrace +from newrelic.api.message_trace import message_trace from newrelic.common.object_wrapper import wrap_function_wrapper def extract_sqs(*args, **kwargs): - queue_value = kwargs.get('QueueUrl', 'Unknown') - return queue_value.rsplit('/', 1)[-1] + queue_value = kwargs.get("QueueUrl", "Unknown") + return queue_value.rsplit("/", 1)[-1] def extract(argument_names, default=None): @@ -41,42 +41,27 @@ def extractor_string(*args, **kwargs): CUSTOM_TRACE_POINTS = { - ('sns', 'publish'): message_trace( - 'SNS', 'Produce', 'Topic', - extract(('TopicArn', 'TargetArn'), 'PhoneNumber')), - ('dynamodb', 'put_item'): datastore_trace( - 'DynamoDB', extract('TableName'), 'put_item'), - ('dynamodb', 'get_item'): datastore_trace( - 'DynamoDB', extract('TableName'), 'get_item'), - ('dynamodb', 'update_item'): datastore_trace( - 'DynamoDB', extract('TableName'), 'update_item'), - ('dynamodb', 'delete_item'): datastore_trace( - 'DynamoDB', extract('TableName'), 'delete_item'), - ('dynamodb', 'create_table'): datastore_trace( - 'DynamoDB', extract('TableName'), 'create_table'), - ('dynamodb', 'delete_table'): datastore_trace( - 'DynamoDB', extract('TableName'), 'delete_table'), - ('dynamodb', 'query'): datastore_trace( - 'DynamoDB', extract('TableName'), 'query'), - ('dynamodb', 'scan'): datastore_trace( - 'DynamoDB', extract('TableName'), 'scan'), - ('sqs', 'send_message'): message_trace( - 'SQS', 'Produce', 'Queue', extract_sqs), - ('sqs', 'send_message_batch'): message_trace( - 'SQS', 'Produce', 'Queue', extract_sqs), - ('sqs', 'receive_message'): message_trace( - 'SQS', 'Consume', 'Queue', extract_sqs), + ("sns", "publish"): message_trace("SNS", "Produce", "Topic", extract(("TopicArn", "TargetArn"), "PhoneNumber")), + ("dynamodb", "put_item"): datastore_trace("DynamoDB", extract("TableName"), "put_item"), + ("dynamodb", "get_item"): datastore_trace("DynamoDB", extract("TableName"), "get_item"), + ("dynamodb", "update_item"): datastore_trace("DynamoDB", extract("TableName"), "update_item"), + ("dynamodb", "delete_item"): datastore_trace("DynamoDB", extract("TableName"), "delete_item"), + ("dynamodb", "create_table"): datastore_trace("DynamoDB", extract("TableName"), "create_table"), + ("dynamodb", "delete_table"): datastore_trace("DynamoDB", extract("TableName"), "delete_table"), + ("dynamodb", "query"): datastore_trace("DynamoDB", extract("TableName"), "query"), + ("dynamodb", "scan"): datastore_trace("DynamoDB", extract("TableName"), "scan"), + ("sqs", "send_message"): message_trace("SQS", "Produce", "Queue", extract_sqs), + ("sqs", "send_message_batch"): message_trace("SQS", "Produce", "Queue", extract_sqs), + ("sqs", "receive_message"): message_trace("SQS", "Consume", "Queue", extract_sqs), } -def bind__create_api_method(py_operation_name, operation_name, service_model, - *args, **kwargs): +def bind__create_api_method(py_operation_name, operation_name, service_model, *args, **kwargs): return (py_operation_name, service_model) def _nr_clientcreator__create_api_method_(wrapped, instance, args, kwargs): - (py_operation_name, service_model) = \ - bind__create_api_method(*args, **kwargs) + (py_operation_name, service_model) = bind__create_api_method(*args, **kwargs) service_name = service_model.service_name.lower() tracer = CUSTOM_TRACE_POINTS.get((service_name, py_operation_name)) @@ -95,30 +80,27 @@ def _bind_make_request_params(operation_model, request_dict, *args, **kwargs): def _nr_endpoint_make_request_(wrapped, instance, args, kwargs): operation_model, request_dict = _bind_make_request_params(*args, **kwargs) - url = request_dict.get('url', '') - method = request_dict.get('method', None) - - with ExternalTrace(library='botocore', url=url, method=method, source=wrapped) as trace: + url = request_dict.get("url", "") + method = request_dict.get("method", None) + with ExternalTrace(library="botocore", url=url, method=method, source=wrapped) as trace: try: - trace._add_agent_attribute('aws.operation', operation_model.name) + trace._add_agent_attribute("aws.operation", operation_model.name) except: pass result = wrapped(*args, **kwargs) try: - request_id = result[1]['ResponseMetadata']['RequestId'] - trace._add_agent_attribute('aws.requestId', request_id) + request_id = result[1]["ResponseMetadata"]["RequestId"] + trace._add_agent_attribute("aws.requestId", request_id) except: pass return result def instrument_botocore_endpoint(module): - wrap_function_wrapper(module, 'Endpoint.make_request', - _nr_endpoint_make_request_) + wrap_function_wrapper(module, "Endpoint.make_request", _nr_endpoint_make_request_) def instrument_botocore_client(module): - wrap_function_wrapper(module, 'ClientCreator._create_api_method', - _nr_clientcreator__create_api_method_) + wrap_function_wrapper(module, "ClientCreator._create_api_method", _nr_clientcreator__create_api_method_) diff --git a/tests/external_boto3/conftest.py b/tests/external_boto3/conftest.py deleted file mode 100644 index 90d82f0072..0000000000 --- a/tests/external_boto3/conftest.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - - -_default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, -} - -collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (external_boto3)', - default_settings=_default_settings) diff --git a/tests/external_botocore/conftest.py b/tests/external_botocore/conftest.py index e5cf155336..fb703c85e1 100644 --- a/tests/external_botocore/conftest.py +++ b/tests/external_botocore/conftest.py @@ -12,19 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (external_botocore)', - default_settings=_default_settings) + app_name="Python Agent Test (external_botocore)", default_settings=_default_settings +) diff --git a/tests/external_boto3/test_boto3_iam.py b/tests/external_botocore/test_boto3_iam.py similarity index 95% rename from tests/external_boto3/test_boto3_iam.py rename to tests/external_botocore/test_boto3_iam.py index a2237dc936..3d672f3751 100644 --- a/tests/external_boto3/test_boto3_iam.py +++ b/tests/external_botocore/test_boto3_iam.py @@ -17,7 +17,7 @@ import boto3 import moto -from testing_support.fixtures import override_application_settings +from testing_support.fixtures import dt_enabled from testing_support.validators.validate_span_events import validate_span_events from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, @@ -53,7 +53,7 @@ ] -@override_application_settings({"distributed_tracing.enabled": True}) +@dt_enabled @validate_span_events(exact_agents={"http.url": "https://iam.amazonaws.com/"}, count=3) @validate_span_events(expected_agents=("aws.requestId",), count=3) @validate_span_events(exact_agents={"aws.operation": "CreateUser"}, count=1) diff --git a/tests/external_boto3/test_boto3_s3.py b/tests/external_botocore/test_boto3_s3.py similarity index 97% rename from tests/external_boto3/test_boto3_s3.py rename to tests/external_botocore/test_boto3_s3.py index a7ecf034ab..b6299d9f6e 100644 --- a/tests/external_boto3/test_boto3_s3.py +++ b/tests/external_botocore/test_boto3_s3.py @@ -18,7 +18,7 @@ import boto3 import botocore import moto -from testing_support.fixtures import override_application_settings +from testing_support.fixtures import dt_enabled from testing_support.validators.validate_span_events import validate_span_events from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, @@ -73,7 +73,7 @@ ] -@override_application_settings({"distributed_tracing.enabled": True}) +@dt_enabled @validate_span_events(exact_agents={"aws.operation": "CreateBucket"}, count=1) @validate_span_events(exact_agents={"aws.operation": "PutObject"}, count=1) @validate_span_events(exact_agents={"aws.operation": "ListObjects"}, count=1) diff --git a/tests/external_boto3/test_boto3_sns.py b/tests/external_botocore/test_boto3_sns.py similarity index 94% rename from tests/external_boto3/test_boto3_sns.py rename to tests/external_botocore/test_boto3_sns.py index bafe68611d..5e6c7c4b4e 100644 --- a/tests/external_boto3/test_boto3_sns.py +++ b/tests/external_botocore/test_boto3_sns.py @@ -17,7 +17,7 @@ import boto3 import moto import pytest -from testing_support.fixtures import override_application_settings +from testing_support.fixtures import dt_enabled from testing_support.validators.validate_span_events import validate_span_events from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, @@ -45,7 +45,7 @@ sns_metrics_phone = [("MessageBroker/SNS/Topic" "/Produce/Named/PhoneNumber", 1)] -@override_application_settings({"distributed_tracing.enabled": True}) +@dt_enabled @validate_span_events(expected_agents=("aws.requestId",), count=2) @validate_span_events(exact_agents={"aws.operation": "CreateTopic"}, count=1) @validate_span_events(exact_agents={"aws.operation": "Publish"}, count=1) @@ -74,7 +74,7 @@ def test_publish_to_sns_topic(topic_argument): assert "MessageId" in published_message -@override_application_settings({"distributed_tracing.enabled": True}) +@dt_enabled @validate_span_events(expected_agents=("aws.requestId",), count=3) @validate_span_events(exact_agents={"aws.operation": "CreateTopic"}, count=1) @validate_span_events(exact_agents={"aws.operation": "Subscribe"}, count=1) diff --git a/tests/external_botocore/test_botocore_dynamodb.py b/tests/external_botocore/test_botocore_dynamodb.py index 30114d53b1..932fb1743a 100644 --- a/tests/external_botocore/test_botocore_dynamodb.py +++ b/tests/external_botocore/test_botocore_dynamodb.py @@ -17,7 +17,7 @@ import botocore.session import moto -from testing_support.fixtures import override_application_settings +from testing_support.fixtures import dt_enabled from testing_support.validators.validate_span_events import validate_span_events from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, @@ -63,7 +63,7 @@ ] -@override_application_settings({"distributed_tracing.enabled": True}) +@dt_enabled @validate_span_events(expected_agents=("aws.requestId",), count=8) @validate_span_events(exact_agents={"aws.operation": "PutItem"}, count=1) @validate_span_events(exact_agents={"aws.operation": "GetItem"}, count=1) @@ -80,7 +80,7 @@ background_task=True, ) @background_task() -@moto.mock_dynamodb2 +@moto.mock_dynamodb def test_dynamodb(): session = botocore.session.get_session() client = session.create_client( diff --git a/tests/external_botocore/test_botocore_ec2.py b/tests/external_botocore/test_botocore_ec2.py index 28a8ff63ae..3cb83e3185 100644 --- a/tests/external_botocore/test_botocore_ec2.py +++ b/tests/external_botocore/test_botocore_ec2.py @@ -17,7 +17,7 @@ import botocore.session import moto -from testing_support.fixtures import override_application_settings +from testing_support.fixtures import dt_enabled from testing_support.validators.validate_span_events import validate_span_events from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, @@ -55,7 +55,7 @@ ] -@override_application_settings({"distributed_tracing.enabled": True}) +@dt_enabled @validate_span_events(expected_agents=("aws.requestId",), count=3) @validate_span_events(exact_agents={"aws.operation": "RunInstances"}, count=1) @validate_span_events(exact_agents={"aws.operation": "DescribeInstances"}, count=1) diff --git a/tests/external_botocore/test_botocore_s3.py b/tests/external_botocore/test_botocore_s3.py index 1984d8103e..ea0c225390 100644 --- a/tests/external_botocore/test_botocore_s3.py +++ b/tests/external_botocore/test_botocore_s3.py @@ -18,7 +18,7 @@ import botocore import botocore.session import moto -from testing_support.fixtures import override_application_settings +from testing_support.fixtures import dt_enabled from testing_support.validators.validate_span_events import validate_span_events from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, @@ -67,7 +67,7 @@ ] -@override_application_settings({"distributed_tracing.enabled": True}) +@dt_enabled @validate_span_events(exact_agents={"aws.operation": "CreateBucket"}, count=1) @validate_span_events(exact_agents={"aws.operation": "PutObject"}, count=1) @validate_span_events(exact_agents={"aws.operation": "ListObjects"}, count=1) diff --git a/tests/external_botocore/test_botocore_sqs.py b/tests/external_botocore/test_botocore_sqs.py index 3f7d8c0220..63f15801b5 100644 --- a/tests/external_botocore/test_botocore_sqs.py +++ b/tests/external_botocore/test_botocore_sqs.py @@ -18,7 +18,7 @@ import botocore.session import moto import pytest -from testing_support.fixtures import override_application_settings +from testing_support.fixtures import dt_enabled from testing_support.validators.validate_span_events import validate_span_events from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, @@ -70,7 +70,7 @@ ] -@override_application_settings({"distributed_tracing.enabled": True}) +@dt_enabled @validate_span_events(exact_agents={"aws.operation": "CreateQueue"}, count=1) @validate_span_events(exact_agents={"aws.operation": "SendMessage"}, count=1) @validate_span_events(exact_agents={"aws.operation": "ReceiveMessage"}, count=1) @@ -124,7 +124,7 @@ def test_sqs(): assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 -@override_application_settings({"distributed_tracing.enabled": True}) +@dt_enabled @validate_transaction_metrics( "test_botocore_sqs:test_sqs_malformed", scoped_metrics=_sqs_scoped_metrics_malformed, diff --git a/tests/testing_support/validators/validate_ml_events.py b/tests/testing_support/validators/validate_ml_events.py index 251e8dbe79..275a9b2e1b 100644 --- a/tests/testing_support/validators/validate_ml_events.py +++ b/tests/testing_support/validators/validate_ml_events.py @@ -24,7 +24,6 @@ def validate_ml_events(events): @function_wrapper def _validate_wrapper(wrapped, instance, args, kwargs): - record_called = [] recorded_events = [] @@ -55,7 +54,7 @@ def _validate_ml_events(wrapped, instance, args, kwargs): for captured in found_events: if _check_event_attributes(expected, captured, mismatches): matching_ml_events += 1 - assert matching_ml_events == 1, _event_details(matching_ml_events, events, mismatches) + assert matching_ml_events == 1, _event_details(matching_ml_events, found_events, mismatches) return val diff --git a/tox.ini b/tox.ini index bdcaa745ab..bb6102d895 100644 --- a/tox.ini +++ b/tox.ini @@ -97,7 +97,6 @@ envlist = redis-datastore_redis-{py37,py38,py39,py310,py311,pypy38}-redis{0400,latest}, rediscluster-datastore_rediscluster-{py37,py311,pypy38}-redis{latest}, python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, - python-external_boto3-{py27,py37,py38,py39,py310,py311}-boto01, python-external_botocore-{py37,py38,py39,py310,py311}-botocorelatest, python-external_botocore-{py311}-botocore128, python-external_botocore-py310-botocore0125, @@ -251,15 +250,13 @@ deps = datastore_redis-redislatest: redis datastore_rediscluster-redislatest: redis datastore_redis-redis0400: redis<4.1 - external_boto3-boto01: boto3<2.0 - external_boto3-boto01: moto<2.0 - external_boto3-py27: rsa<4.7.1 external_botocore-botocorelatest: botocore + external_botocore-botocorelatest: boto3 external_botocore-botocore128: botocore<1.29 external_botocore-botocore0125: botocore<1.26 - external_botocore-{py37,py38,py39,py310,py311}: moto[awslambda,ec2,iam]<3.0 + external_botocore-{py37,py38,py39,py310,py311}: moto[awslambda,ec2,iam,sqs] external_botocore-py27: rsa<4.7.1 - external_botocore-py27: moto[awslambda,ec2,iam]<2.0 + external_botocore-py27: moto[awslambda,ec2,iam,sqs]<2.0 external_feedparser-feedparser05: feedparser<6 external_feedparser-feedparser06: feedparser<7 external_httplib2: httplib2<1.0 @@ -437,7 +434,6 @@ changedir = datastore_redis: tests/datastore_redis datastore_rediscluster: tests/datastore_rediscluster datastore_sqlite: tests/datastore_sqlite - external_boto3: tests/external_boto3 external_botocore: tests/external_botocore external_feedparser: tests/external_feedparser external_http: tests/external_http