diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 000000000..b2718cb9d --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,70 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +name: Benchmarks + +on: + pull_request: + +permissions: + contents: read + +concurrency: + group: ${{ github.ref || github.run_id }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + # Benchmarks + benchmark: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + strategy: + matrix: + python: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + + env: + ASV_FACTOR: "1.1" + BASE_SHA: ${{ github.event.pull_request.base.sha }} + + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + with: + fetch-depth: 0 + + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # 6.0.0 + with: + python-version: "${{ matrix.python }}" + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin + + - name: Install Dependencies + run: | + pip install --upgrade pip + pip install asv virtualenv + + - name: Configure Machine Information + run: | + asv machine --yes + + - name: Run Benchmark + run: | + asv continuous \ + --show-stderr \ + --split \ + --factor "${ASV_FACTOR}" \ + --python=${{ matrix.python }} \ + "${BASE_SHA}" "${GITHUB_SHA}" diff --git a/.gitignore b/.gitignore index eba56bbc9..4acad5ce1 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ # Linter megalinter-reports/ +# Benchmarks +.asv/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/asv.conf.json b/asv.conf.json new file mode 100644 index 000000000..5826289a5 --- /dev/null +++ b/asv.conf.json @@ -0,0 +1,100 @@ +{ + "version": 1, // The version of the config file format. + "project": "newrelic", + "project_url": "https://github.com/newrelic/newrelic-python-agent", + "show_commit_url": "https://github.com/newrelic/newrelic-python-agent/commit/", + "repo": ".", + "environment_type": "virtualenv", + "install_timeout": 120, + "pythons": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"], + "benchmark_dir": "tests/agent_benchmarks", + "env_dir": ".asv/env", + "results_dir": ".asv/results", + "html_dir": ".asv/html", + "regressions_thresholds": { + ".*": 0.2, // Threshold of 20% + }, + + // The matrix of dependencies to test. Each key of the "req" + // requirements dictionary is the name of a package (in PyPI) and + // the values are version numbers. An empty list or empty string + // indicates to just test against the default (latest) + // version. null indicates that the package is to not be + // installed. If the package to be tested is only available from + // PyPi, and the 'environment_type' is conda, then you can preface + // the package name by 'pip+', and the package will be installed + // via pip (with all the conda available packages installed first, + // followed by the pip installed packages). + // + // The ``@env`` and ``@env_nobuild`` keys contain the matrix of + // environment variables to pass to build and benchmark commands. + // An environment will be created for every combination of the + // cartesian product of the "@env" variables in this matrix. + // Variables in "@env_nobuild" will be passed to every environment + // during the benchmark phase, but will not trigger creation of + // new environments. A value of ``null`` means that the variable + // will not be set for the current combination. + // + // "matrix": { + // "req": { + // "numpy": ["1.6", "1.7"], + // "six": ["", null], // test with and without six installed + // "pip+emcee": [""] // emcee is only available for install with pip. + // }, + // "env": {"ENV_VAR_1": ["val1", "val2"]}, + // "env_nobuild": {"ENV_VAR_2": ["val3", null]}, + // }, + + // Combinations of libraries/python versions can be excluded/included + // from the set to test. Each entry is a dictionary containing additional + // key-value pairs to include/exclude. + // + // An exclude entry excludes entries where all values match. The + // values are regexps that should match the whole string. + // + // An include entry adds an environment. Only the packages listed + // are installed. The 'python' key is required. The exclude rules + // do not apply to includes. + // + // In addition to package names, the following keys are available: + // + // - python + // Python version, as in the *pythons* variable above. + // - environment_type + // Environment type, as above. + // - sys_platform + // Platform, as in sys.platform. Possible values for the common + // cases: 'linux2', 'win32', 'cygwin', 'darwin'. + // - req + // Required packages + // - env + // Environment variables + // - env_nobuild + // Non-build environment variables + // + // "exclude": [ + // {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows + // {"environment_type": "conda", "req": {"six": null}}, // don't run without six on conda + // {"env": {"ENV_VAR_1": "val2"}}, // skip val2 for ENV_VAR_1 + // ], + // + // "include": [ + // // additional env for python3.12 + // {"python": "3.12", "req": {"numpy": "1.26"}, "env_nobuild": {"FOO": "123"}}, + // // additional env if run on windows+conda + // {"platform": "win32", "environment_type": "conda", "python": "3.12", "req": {"libpython": ""}}, + // ], + + // The commits after which the regression search in `asv publish` + // should start looking for regressions. Dictionary whose keys are + // regexps matching to benchmark names, and values corresponding to + // the commit (exclusive) after which to start looking for + // regressions. The default is to start from the first commit + // with results. If the commit is `null`, regression detection is + // skipped for the matching benchmark. + // + // "regressions_first_commits": { + // "some_benchmark": "352cdf", // Consider regressions only after this commit + // "another_benchmark": null, // Skip regression detection altogether + // }, +} diff --git a/tests/agent_benchmarks/__init__.py b/tests/agent_benchmarks/__init__.py new file mode 100644 index 000000000..97e9818c1 --- /dev/null +++ b/tests/agent_benchmarks/__init__.py @@ -0,0 +1,47 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ._agent_initialization import collector_agent_registration + +BENCHMARK_PREFIXES = ("time", "mem") +REPLACE_PREFIX = "bench_" + + +def benchmark(cls): + # Find all methods not prefixed with underscores and treat them as benchmark methods + benchmark_methods = { + name: method for name, method in vars(cls).items() if callable(method) and name.startswith(REPLACE_PREFIX) + } + + # Remove setup function from benchmark methods and save it + cls._setup = benchmark_methods.pop("setup", None) + + # Patch in benchmark methods for each prefix + for name, method in benchmark_methods.items(): + name = name[len(REPLACE_PREFIX) :] # Remove "bench_" prefix + for prefix in BENCHMARK_PREFIXES: + setattr(cls, f"{prefix}_{name}", method) + + # Define agent activation as setup function + def setup(self): + collector_agent_registration(self) + + # Call the original setup if it exists + if getattr(self, "_setup", None) is not None: + self._setup() + + # Patch in new setup method + cls.setup = setup + + return cls diff --git a/tests/agent_benchmarks/_agent_initialization.py b/tests/agent_benchmarks/_agent_initialization.py new file mode 100644 index 000000000..65a0ccb9a --- /dev/null +++ b/tests/agent_benchmarks/_agent_initialization.py @@ -0,0 +1,70 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +from pathlib import Path +from threading import Lock + +# Amend sys.path to allow importing fixtures from testing_support +tests_path = Path(__file__).parent.parent +sys.path.append(str(tests_path)) + +from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: E402 + +_default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, +} + +_collector_agent_registration_fixture = collector_agent_registration_fixture( + app_name="Python Agent Test (benchmarks)", default_settings=_default_settings +) + +INITIALIZATION_LOCK = Lock() +APPLICATIONS = [] + +DISALLOWED_ENV_VARS = ("NEW_RELIC_CONFIG_FILE", "NEW_RELIC_LICENSE_KEY") + + +def collector_agent_registration(instance): + # If the application is already registered, exit early + if APPLICATIONS: + instance.application = APPLICATIONS[0] # Make application accessible to benchmarks + return + + # Register the agent with the collector using the pytest fixture manually + with INITIALIZATION_LOCK: + if APPLICATIONS: # Must re-check this condition just in case + instance.application = APPLICATIONS[0] # Make application accessible to benchmarks + return + + # Force benchmarking to always use developer mode + os.environ["NEW_RELIC_DEVELOPER_MODE"] = "true" # Force developer mode + for env_var in DISALLOWED_ENV_VARS: # Drop disallowed env vars + os.environ.pop(env_var, None) + + # Use pytest fixture by hand to start the agent + fixture = _collector_agent_registration_fixture() + APPLICATIONS.append(next(fixture)) + + # Wait for the application to become active + collector_available_fixture(APPLICATIONS[0]) + + # Make application accessible to benchmarks + instance.application = APPLICATIONS[0] diff --git a/tests/agent_benchmarks/bench_agent_active.py b/tests/agent_benchmarks/bench_agent_active.py new file mode 100644 index 000000000..de7a695e4 --- /dev/null +++ b/tests/agent_benchmarks/bench_agent_active.py @@ -0,0 +1,34 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from newrelic.agent import background_task, current_transaction + +from . import benchmark + +# This benchmark suite is a placeholder until actual benchmark suites can be added. +# For now, this ensures the infrastructure works as intended. + + +@benchmark +class Suite: + def bench_application_active(self): + from newrelic.agent import application + + assert application().active + + @background_task() + def bench_transaction_active(self): + from newrelic.agent import application + + assert current_transaction() diff --git a/tests/testing_support/fixtures.py b/tests/testing_support/fixtures.py index d12311f9c..00cfd7368 100644 --- a/tests/testing_support/fixtures.py +++ b/tests/testing_support/fixtures.py @@ -23,8 +23,6 @@ from pathlib import Path from queue import Queue -import pytest - from newrelic.admin.record_deploy import record_deploy from newrelic.api.application import application_instance, application_settings, register_application from newrelic.api.ml_model import set_llm_token_count_callback @@ -47,6 +45,22 @@ _logger = logging.getLogger("newrelic.tests") +try: + import pytest +except ImportError: + # When running benchmarks, we don't use pytest. + # Instead, make these fixtures into functions and generators we can use manually. + pytest = type("pytest", (), {}) + + def fixture(func=None, **kwargs): + # Passthrough to make this transparent for benchmarks + if func: + return func + else: + return fixture + + pytest.fixture = fixture + def _environ_as_bool(name, default=False): flag = os.environ.get(name, default) @@ -188,7 +202,7 @@ def collector_agent_registration_fixture( linked_applications = linked_applications or [] @pytest.fixture(scope=scope) - def _collector_agent_registration_fixture(request): + def _collector_agent_registration_fixture(): if should_initialize_agent: initialize_agent(app_name=app_name, default_settings=default_settings) @@ -332,7 +346,7 @@ def _collector_agent_registration_fixture(request, settings_fixture): @pytest.fixture -def collector_available_fixture(request, collector_agent_registration): +def collector_available_fixture(collector_agent_registration): application = collector_agent_registration settings = global_settings()