Skip to content

Commit 5706b4f

Browse files
TimPansinomergify[bot]
TimPansino
and
mergify[bot]
authored
Benchmarking Infrastructure (#1506)
* Use asv for benchmarking * Linting --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
1 parent e2d411c commit 5706b4f

File tree

7 files changed

+342
-4
lines changed

7 files changed

+342
-4
lines changed

‎.github/workflows/benchmarks.yml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# Copyright 2010 New Relic, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
---
15+
name: Benchmarks
16+
17+
on:
18+
pull_request:
19+
20+
permissions:
21+
contents: read
22+
23+
concurrency:
24+
group: ${{ github.ref || github.run_id }}-${{ github.workflow }}
25+
cancel-in-progress: true
26+
27+
jobs:
28+
# Benchmarks
29+
benchmark:
30+
runs-on: ubuntu-24.04
31+
timeout-minutes: 30
32+
strategy:
33+
matrix:
34+
python: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
35+
36+
env:
37+
ASV_FACTOR: "1.1"
38+
BASE_SHA: ${{ github.event.pull_request.base.sha }}
39+
40+
steps:
41+
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0
42+
with:
43+
fetch-depth: 0
44+
45+
- uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # 6.0.0
46+
with:
47+
python-version: "${{ matrix.python }}"
48+
49+
- name: Fetch git tags
50+
run: |
51+
git config --global --add safe.directory "$GITHUB_WORKSPACE"
52+
git fetch --tags origin
53+
54+
- name: Install Dependencies
55+
run: |
56+
pip install --upgrade pip
57+
pip install asv virtualenv
58+
59+
- name: Configure Machine Information
60+
run: |
61+
asv machine --yes
62+
63+
- name: Run Benchmark
64+
run: |
65+
asv continuous \
66+
--show-stderr \
67+
--split \
68+
--factor "${ASV_FACTOR}" \
69+
--python=${{ matrix.python }} \
70+
"${BASE_SHA}" "${GITHUB_SHA}"

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44
# Linter
55
megalinter-reports/
66

7+
# Benchmarks
8+
.asv/
9+
710
# Byte-compiled / optimized / DLL files
811
__pycache__/
912
*.py[cod]

‎asv.conf.json‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
{
2+
"version": 1, // The version of the config file format.
3+
"project": "newrelic",
4+
"project_url": "https://github.com/newrelic/newrelic-python-agent",
5+
"show_commit_url": "https://github.com/newrelic/newrelic-python-agent/commit/",
6+
"repo": ".",
7+
"environment_type": "virtualenv",
8+
"install_timeout": 120,
9+
"pythons": ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"],
10+
"benchmark_dir": "tests/agent_benchmarks",
11+
"env_dir": ".asv/env",
12+
"results_dir": ".asv/results",
13+
"html_dir": ".asv/html",
14+
"regressions_thresholds": {
15+
".*": 0.2, // Threshold of 20%
16+
},
17+
18+
// The matrix of dependencies to test. Each key of the "req"
19+
// requirements dictionary is the name of a package (in PyPI) and
20+
// the values are version numbers. An empty list or empty string
21+
// indicates to just test against the default (latest)
22+
// version. null indicates that the package is to not be
23+
// installed. If the package to be tested is only available from
24+
// PyPi, and the 'environment_type' is conda, then you can preface
25+
// the package name by 'pip+', and the package will be installed
26+
// via pip (with all the conda available packages installed first,
27+
// followed by the pip installed packages).
28+
//
29+
// The ``@env`` and ``@env_nobuild`` keys contain the matrix of
30+
// environment variables to pass to build and benchmark commands.
31+
// An environment will be created for every combination of the
32+
// cartesian product of the "@env" variables in this matrix.
33+
// Variables in "@env_nobuild" will be passed to every environment
34+
// during the benchmark phase, but will not trigger creation of
35+
// new environments. A value of ``null`` means that the variable
36+
// will not be set for the current combination.
37+
//
38+
// "matrix": {
39+
// "req": {
40+
// "numpy": ["1.6", "1.7"],
41+
// "six": ["", null], // test with and without six installed
42+
// "pip+emcee": [""] // emcee is only available for install with pip.
43+
// },
44+
// "env": {"ENV_VAR_1": ["val1", "val2"]},
45+
// "env_nobuild": {"ENV_VAR_2": ["val3", null]},
46+
// },
47+
48+
// Combinations of libraries/python versions can be excluded/included
49+
// from the set to test. Each entry is a dictionary containing additional
50+
// key-value pairs to include/exclude.
51+
//
52+
// An exclude entry excludes entries where all values match. The
53+
// values are regexps that should match the whole string.
54+
//
55+
// An include entry adds an environment. Only the packages listed
56+
// are installed. The 'python' key is required. The exclude rules
57+
// do not apply to includes.
58+
//
59+
// In addition to package names, the following keys are available:
60+
//
61+
// - python
62+
// Python version, as in the *pythons* variable above.
63+
// - environment_type
64+
// Environment type, as above.
65+
// - sys_platform
66+
// Platform, as in sys.platform. Possible values for the common
67+
// cases: 'linux2', 'win32', 'cygwin', 'darwin'.
68+
// - req
69+
// Required packages
70+
// - env
71+
// Environment variables
72+
// - env_nobuild
73+
// Non-build environment variables
74+
//
75+
// "exclude": [
76+
// {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows
77+
// {"environment_type": "conda", "req": {"six": null}}, // don't run without six on conda
78+
// {"env": {"ENV_VAR_1": "val2"}}, // skip val2 for ENV_VAR_1
79+
// ],
80+
//
81+
// "include": [
82+
// // additional env for python3.12
83+
// {"python": "3.12", "req": {"numpy": "1.26"}, "env_nobuild": {"FOO": "123"}},
84+
// // additional env if run on windows+conda
85+
// {"platform": "win32", "environment_type": "conda", "python": "3.12", "req": {"libpython": ""}},
86+
// ],
87+
88+
// The commits after which the regression search in `asv publish`
89+
// should start looking for regressions. Dictionary whose keys are
90+
// regexps matching to benchmark names, and values corresponding to
91+
// the commit (exclusive) after which to start looking for
92+
// regressions. The default is to start from the first commit
93+
// with results. If the commit is `null`, regression detection is
94+
// skipped for the matching benchmark.
95+
//
96+
// "regressions_first_commits": {
97+
// "some_benchmark": "352cdf", // Consider regressions only after this commit
98+
// "another_benchmark": null, // Skip regression detection altogether
99+
// },
100+
}

‎tests/agent_benchmarks/__init__.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# Copyright 2010 New Relic, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from ._agent_initialization import collector_agent_registration
16+
17+
BENCHMARK_PREFIXES = ("time", "mem")
18+
REPLACE_PREFIX = "bench_"
19+
20+
21+
def benchmark(cls):
22+
# Find all methods not prefixed with underscores and treat them as benchmark methods
23+
benchmark_methods = {
24+
name: method for name, method in vars(cls).items() if callable(method) and name.startswith(REPLACE_PREFIX)
25+
}
26+
27+
# Remove setup function from benchmark methods and save it
28+
cls._setup = benchmark_methods.pop("setup", None)
29+
30+
# Patch in benchmark methods for each prefix
31+
for name, method in benchmark_methods.items():
32+
name = name[len(REPLACE_PREFIX) :] # Remove "bench_" prefix
33+
for prefix in BENCHMARK_PREFIXES:
34+
setattr(cls, f"{prefix}_{name}", method)
35+
36+
# Define agent activation as setup function
37+
def setup(self):
38+
collector_agent_registration(self)
39+
40+
# Call the original setup if it exists
41+
if getattr(self, "_setup", None) is not None:
42+
self._setup()
43+
44+
# Patch in new setup method
45+
cls.setup = setup
46+
47+
return cls

‎tests/agent_benchmarks/_agent_initialization.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# Copyright 2010 New Relic, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import os
16+
import sys
17+
from pathlib import Path
18+
from threading import Lock
19+
20+
# Amend sys.path to allow importing fixtures from testing_support
21+
tests_path = Path(__file__).parent.parent
22+
sys.path.append(str(tests_path))
23+
24+
from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: E402
25+
26+
_default_settings = {
27+
"package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs.
28+
"transaction_tracer.explain_threshold": 0.0,
29+
"transaction_tracer.transaction_threshold": 0.0,
30+
"transaction_tracer.stack_trace_threshold": 0.0,
31+
"debug.log_data_collector_payloads": True,
32+
"debug.record_transaction_failure": True,
33+
}
34+
35+
_collector_agent_registration_fixture = collector_agent_registration_fixture(
36+
app_name="Python Agent Test (benchmarks)", default_settings=_default_settings
37+
)
38+
39+
INITIALIZATION_LOCK = Lock()
40+
APPLICATIONS = []
41+
42+
DISALLOWED_ENV_VARS = ("NEW_RELIC_CONFIG_FILE", "NEW_RELIC_LICENSE_KEY")
43+
44+
45+
def collector_agent_registration(instance):
46+
# If the application is already registered, exit early
47+
if APPLICATIONS:
48+
instance.application = APPLICATIONS[0] # Make application accessible to benchmarks
49+
return
50+
51+
# Register the agent with the collector using the pytest fixture manually
52+
with INITIALIZATION_LOCK:
53+
if APPLICATIONS: # Must re-check this condition just in case
54+
instance.application = APPLICATIONS[0] # Make application accessible to benchmarks
55+
return
56+
57+
# Force benchmarking to always use developer mode
58+
os.environ["NEW_RELIC_DEVELOPER_MODE"] = "true" # Force developer mode
59+
for env_var in DISALLOWED_ENV_VARS: # Drop disallowed env vars
60+
os.environ.pop(env_var, None)
61+
62+
# Use pytest fixture by hand to start the agent
63+
fixture = _collector_agent_registration_fixture()
64+
APPLICATIONS.append(next(fixture))
65+
66+
# Wait for the application to become active
67+
collector_available_fixture(APPLICATIONS[0])
68+
69+
# Make application accessible to benchmarks
70+
instance.application = APPLICATIONS[0]

‎tests/agent_benchmarks/bench_agent_active.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Copyright 2010 New Relic, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from newrelic.agent import background_task, current_transaction
16+
17+
from . import benchmark
18+
19+
# This benchmark suite is a placeholder until actual benchmark suites can be added.
20+
# For now, this ensures the infrastructure works as intended.
21+
22+
23+
@benchmark
24+
class Suite:
25+
def bench_application_active(self):
26+
from newrelic.agent import application
27+
28+
assert application().active
29+
30+
@background_task()
31+
def bench_transaction_active(self):
32+
from newrelic.agent import application
33+
34+
assert current_transaction()

‎tests/testing_support/fixtures.py‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,6 @@
2323
from pathlib import Path
2424
from queue import Queue
2525

26-
import pytest
27-
2826
from newrelic.admin.record_deploy import record_deploy
2927
from newrelic.api.application import application_instance, application_settings, register_application
3028
from newrelic.api.ml_model import set_llm_token_count_callback
@@ -47,6 +45,22 @@
4745

4846
_logger = logging.getLogger("newrelic.tests")
4947

48+
try:
49+
import pytest
50+
except ImportError:
51+
# When running benchmarks, we don't use pytest.
52+
# Instead, make these fixtures into functions and generators we can use manually.
53+
pytest = type("pytest", (), {})
54+
55+
def fixture(func=None, **kwargs):
56+
# Passthrough to make this transparent for benchmarks
57+
if func:
58+
return func
59+
else:
60+
return fixture
61+
62+
pytest.fixture = fixture
63+
5064

5165
def _environ_as_bool(name, default=False):
5266
flag = os.environ.get(name, default)
@@ -188,7 +202,7 @@ def collector_agent_registration_fixture(
188202
linked_applications = linked_applications or []
189203

190204
@pytest.fixture(scope=scope)
191-
def _collector_agent_registration_fixture(request):
205+
def _collector_agent_registration_fixture():
192206
if should_initialize_agent:
193207
initialize_agent(app_name=app_name, default_settings=default_settings)
194208

@@ -332,7 +346,7 @@ def _collector_agent_registration_fixture(request, settings_fixture):
332346

333347

334348
@pytest.fixture
335-
def collector_available_fixture(request, collector_agent_registration):
349+
def collector_available_fixture(collector_agent_registration):
336350
application = collector_agent_registration
337351
settings = global_settings()
338352

0 commit comments

Comments
 (0)