Benchmarking Infrastructure

.github/workflows/benchmarks.yml

@@ -0,0 +1,70 @@
+# Copyright 2010 New Relic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+name: Benchmarks
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.ref || github.run_id }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  # Benchmarks
+  benchmark:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        python: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
+
+    env:
+      ASV_FACTOR: "1.1"
+      BASE_SHA: ${{ github.event.pull_request.base.sha }}
+
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # 6.0.0
+        with:
+          python-version: "${{ matrix.python }}"
+
+      - name: Fetch git tags
+        run: |
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+          git fetch --tags origin
+
+      - name: Install Dependencies
+        run: |
+          pip install --upgrade pip
+          pip install asv virtualenv
+
+      - name: Configure Machine Information
+        run: |
+          asv machine --yes
+
+      - name: Run Benchmark
+        run: |
+          asv continuous \
+            --show-stderr \
+            --split \
+            --factor "${ASV_FACTOR}" \
+            --python=${{ matrix.python }} \
+            "${BASE_SHA}" "${GITHUB_SHA}"

.gitignore

@@ -4,6 +4,9 @@
 # Linter
 megalinter-reports/
 
+# Benchmarks
+.asv/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

asv.conf.json

@@ -0,0 +1,100 @@
+{
+    "version": 1,  // The version of the config file format.
+    "project": "newrelic",
+    "project_url": "https://github.com/newrelic/newrelic-python-agent",
+    "show_commit_url": "https://github.com/newrelic/newrelic-python-agent/commit/",
+    "repo": ".",
+    "environment_type": "virtualenv",
+    "install_timeout": 120,
+    "pythons": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"],
+    "benchmark_dir": "tests/agent_benchmarks",
+    "env_dir": ".asv/env",
+    "results_dir": ".asv/results",
+    "html_dir": ".asv/html",
+    "regressions_thresholds": {
+       ".*": 0.2,     // Threshold of 20%
+    },
+
+    // The matrix of dependencies to test.  Each key of the "req"
+    // requirements dictionary is the name of a package (in PyPI) and
+    // the values are version numbers.  An empty list or empty string
+    // indicates to just test against the default (latest)
+    // version. null indicates that the package is to not be
+    // installed. If the package to be tested is only available from
+    // PyPi, and the 'environment_type' is conda, then you can preface
+    // the package name by 'pip+', and the package will be installed
+    // via pip (with all the conda available packages installed first,
+    // followed by the pip installed packages).
+    //
+    // The ``@env`` and ``@env_nobuild`` keys contain the matrix of
+    // environment variables to pass to build and benchmark commands.
+    // An environment will be created for every combination of the
+    // cartesian product of the "@env" variables in this matrix.
+    // Variables in "@env_nobuild" will be passed to every environment
+    // during the benchmark phase, but will not trigger creation of
+    // new environments.  A value of ``null`` means that the variable
+    // will not be set for the current combination.
+    //
+    // "matrix": {
+    //     "req": {
+    //         "numpy": ["1.6", "1.7"],
+    //         "six": ["", null],  // test with and without six installed
+    //         "pip+emcee": [""]   // emcee is only available for install with pip.
+    //     },
+    //     "env": {"ENV_VAR_1": ["val1", "val2"]},
+    //     "env_nobuild": {"ENV_VAR_2": ["val3", null]},
+    // },
+
+    // Combinations of libraries/python versions can be excluded/included
+    // from the set to test. Each entry is a dictionary containing additional
+    // key-value pairs to include/exclude.
+    //
+    // An exclude entry excludes entries where all values match. The
+    // values are regexps that should match the whole string.
+    //
+    // An include entry adds an environment. Only the packages listed
+    // are installed. The 'python' key is required. The exclude rules
+    // do not apply to includes.
+    //
+    // In addition to package names, the following keys are available:
+    //
+    // - python
+    //     Python version, as in the *pythons* variable above.
+    // - environment_type
+    //     Environment type, as above.
+    // - sys_platform
+    //     Platform, as in sys.platform. Possible values for the common
+    //     cases: 'linux2', 'win32', 'cygwin', 'darwin'.
+    // - req
+    //     Required packages
+    // - env
+    //     Environment variables
+    // - env_nobuild
+    //     Non-build environment variables
+    //
+    // "exclude": [
+    //     {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows
+    //     {"environment_type": "conda", "req": {"six": null}}, // don't run without six on conda
+    //     {"env": {"ENV_VAR_1": "val2"}}, // skip val2 for ENV_VAR_1
+    // ],
+    //
+    // "include": [
+    //     // additional env for python3.12
+    //     {"python": "3.12", "req": {"numpy": "1.26"}, "env_nobuild": {"FOO": "123"}},
+    //     // additional env if run on windows+conda
+    //     {"platform": "win32", "environment_type": "conda", "python": "3.12", "req": {"libpython": ""}},
+    // ],
+
+    // The commits after which the regression search in `asv publish`
+    // should start looking for regressions. Dictionary whose keys are
+    // regexps matching to benchmark names, and values corresponding to
+    // the commit (exclusive) after which to start looking for
+    // regressions.  The default is to start from the first commit
+    // with results. If the commit is `null`, regression detection is
+    // skipped for the matching benchmark.
+    //
+    // "regressions_first_commits": {
+    //    "some_benchmark": "352cdf",  // Consider regressions only after this commit
+    //    "another_benchmark": null,   // Skip regression detection altogether
+    // },
+}

tests/agent_benchmarks/__init__.py

@@ -0,0 +1,47 @@
+# Copyright 2010 New Relic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._agent_initialization import collector_agent_registration
+
+BENCHMARK_PREFIXES = ("time", "mem")
+REPLACE_PREFIX = "bench_"
+
+
+def benchmark(cls):
+    # Find all methods not prefixed with underscores and treat them as benchmark methods
+    benchmark_methods = {
+        name: method for name, method in vars(cls).items() if callable(method) and name.startswith(REPLACE_PREFIX)
+    }
+
+    # Remove setup function from benchmark methods and save it
+    cls._setup = benchmark_methods.pop("setup", None)
+
+    # Patch in benchmark methods for each prefix
+    for name, method in benchmark_methods.items():
+        name = name[len(REPLACE_PREFIX) :]  # Remove "bench_" prefix
+        for prefix in BENCHMARK_PREFIXES:
+            setattr(cls, f"{prefix}_{name}", method)
+
+    # Define agent activation as setup function
+    def setup(self):
+        collector_agent_registration(self)
+
+        # Call the original setup if it exists
+        if getattr(self, "_setup", None) is not None:
+            self._setup()
+
+    # Patch in new setup method
+    cls.setup = setup
+
+    return cls

tests/agent_benchmarks/_agent_initialization.py

@@ -0,0 +1,70 @@
+# Copyright 2010 New Relic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+from pathlib import Path
+from threading import Lock
+
+# Amend sys.path to allow importing fixtures from testing_support
+tests_path = Path(__file__).parent.parent
+sys.path.append(str(tests_path))
+
+from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture  # noqa: E402
+
+_default_settings = {
+    "package_reporting.enabled": False,  # Turn off package reporting for testing as it causes slow downs.
+    "transaction_tracer.explain_threshold": 0.0,
+    "transaction_tracer.transaction_threshold": 0.0,
+    "transaction_tracer.stack_trace_threshold": 0.0,
+    "debug.log_data_collector_payloads": True,
+    "debug.record_transaction_failure": True,
+}
+
+_collector_agent_registration_fixture = collector_agent_registration_fixture(
+    app_name="Python Agent Test (benchmarks)", default_settings=_default_settings
+)
+
+INITIALIZATION_LOCK = Lock()
+APPLICATIONS = []
+
+DISALLOWED_ENV_VARS = ("NEW_RELIC_CONFIG_FILE", "NEW_RELIC_LICENSE_KEY")
+
+
+def collector_agent_registration(instance):
+    # If the application is already registered, exit early
+    if APPLICATIONS:
+        instance.application = APPLICATIONS[0]  # Make application accessible to benchmarks
+        return
+
+    # Register the agent with the collector using the pytest fixture manually
+    with INITIALIZATION_LOCK:
+        if APPLICATIONS:  # Must re-check this condition just in case
+            instance.application = APPLICATIONS[0]  # Make application accessible to benchmarks
+            return
+
+        # Force benchmarking to always use developer mode
+        os.environ["NEW_RELIC_DEVELOPER_MODE"] = "true"  # Force developer mode
+        for env_var in DISALLOWED_ENV_VARS:  # Drop disallowed env vars
+            os.environ.pop(env_var, None)
+
+        # Use pytest fixture by hand to start the agent
+        fixture = _collector_agent_registration_fixture()
+        APPLICATIONS.append(next(fixture))
+
+        # Wait for the application to become active
+        collector_available_fixture(APPLICATIONS[0])
+
+        # Make application accessible to benchmarks
+        instance.application = APPLICATIONS[0]

tests/agent_benchmarks/bench_agent_active.py

@@ -0,0 +1,34 @@
+# Copyright 2010 New Relic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from newrelic.agent import background_task, current_transaction
+
+from . import benchmark
+
+# This benchmark suite is a placeholder until actual benchmark suites can be added.
+# For now, this ensures the infrastructure works as intended.
+
+
+@benchmark
+class Suite:
+    def bench_application_active(self):
+        from newrelic.agent import application
+
+        assert application().active
+
+    @background_task()
+    def bench_transaction_active(self):
+        from newrelic.agent import application
+
+        assert current_transaction()

tests/testing_support/fixtures.py

@@ -23,8 +23,6 @@
 from pathlib import Path
 from queue import Queue
 
-import pytest
-
 from newrelic.admin.record_deploy import record_deploy
 from newrelic.api.application import application_instance, application_settings, register_application
 from newrelic.api.ml_model import set_llm_token_count_callback
@@ -47,6 +45,22 @@
 
 _logger = logging.getLogger("newrelic.tests")
 
+try:
+    import pytest
+except ImportError:
+    # When running benchmarks, we don't use pytest.
+    # Instead, make these fixtures into functions and generators we can use manually.
+    pytest = type("pytest", (), {})
+
+    def fixture(func=None, **kwargs):
+        # Passthrough to make this transparent for benchmarks
+        if func:
+            return func
+        else:
+            return fixture
+
+    pytest.fixture = fixture
+
 
 def _environ_as_bool(name, default=False):
     flag = os.environ.get(name, default)
@@ -188,7 +202,7 @@ def collector_agent_registration_fixture(
     linked_applications = linked_applications or []
 
     @pytest.fixture(scope=scope)
-    def _collector_agent_registration_fixture(request):
+    def _collector_agent_registration_fixture():
         if should_initialize_agent:
             initialize_agent(app_name=app_name, default_settings=default_settings)
 
@@ -332,7 +346,7 @@ def _collector_agent_registration_fixture(request, settings_fixture):
 
 
 @pytest.fixture
-def collector_available_fixture(request, collector_agent_registration):
+def collector_available_fixture(collector_agent_registration):
     application = collector_agent_registration
     settings = global_settings()