From 18ef346dbb19471001fdc9a178e640912e60440b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 30 Jun 2021 19:25:09 -0600 Subject: [PATCH 01/32] Move the benchmarks into individual projects. --- benchmarks/MANIFEST | 28 +++++++++++++++++++ benchmarks/base.toml | 21 ++++++++++++++ benchmarks/bm_aiohttp/pyproject.toml | 18 ++++++++++++ .../requirements.txt} | 0 .../run_benchmark.py} | 0 benchmarks/bm_djangocms/pyproject.toml | 24 ++++++++++++++++ .../requirements.txt} | 0 .../run_benchmark.py} | 0 benchmarks/bm_flaskblogging/pyproject.toml | 17 +++++++++++ .../requirements.txt} | 0 .../run_benchmark.py} | 0 benchmarks/bm_gevent_hub/pyproject.toml | 15 ++++++++++ .../requirements.txt} | 0 .../run_benchmark.py} | 0 benchmarks/bm_gunicorn/pyproject.toml | 17 +++++++++++ .../requirements.txt} | 0 .../run_benchmark.py} | 0 benchmarks/bm_json/pyproject.toml | 13 +++++++++ .../requirements.txt} | 0 .../run_benchmark.py} | 0 benchmarks/bm_mypy/pyproject.toml | 15 ++++++++++ .../requirements.txt} | 0 .../run_benchmark.py} | 0 benchmarks/bm_pycparser/pyproject.toml | 15 ++++++++++ .../requirements.txt} | 0 .../run_benchmark.py} | 0 benchmarks/bm_pylint/pyproject.toml | 15 ++++++++++ .../requirements.txt} | 0 .../run_benchmark.py} | 0 .../pyproject.toml | 16 +++++++++++ .../requirements.txt} | 0 .../run_benchmark.py} | 0 benchmarks/bm_thrift/pyproject.toml | 15 ++++++++++ .../requirements.txt} | 0 .../run_benchmark.py} | 0 35 files changed, 229 insertions(+) create mode 100644 benchmarks/MANIFEST create mode 100644 benchmarks/base.toml create mode 100644 benchmarks/bm_aiohttp/pyproject.toml rename benchmarks/{aiohttp_requirements.txt => bm_aiohttp/requirements.txt} (100%) rename benchmarks/{aiohttp.py => bm_aiohttp/run_benchmark.py} (100%) create mode 100644 benchmarks/bm_djangocms/pyproject.toml rename benchmarks/{djangocms_requirements.txt => bm_djangocms/requirements.txt} (100%) rename benchmarks/{djangocms.py => bm_djangocms/run_benchmark.py} (100%) create mode 100644 benchmarks/bm_flaskblogging/pyproject.toml rename benchmarks/{flaskblogging_requirements.txt => bm_flaskblogging/requirements.txt} (100%) rename benchmarks/{flaskblogging.py => bm_flaskblogging/run_benchmark.py} (100%) create mode 100644 benchmarks/bm_gevent_hub/pyproject.toml rename benchmarks/{gevent_bench_hub_requirements.txt => bm_gevent_hub/requirements.txt} (100%) rename benchmarks/{gevent_bench_hub.py => bm_gevent_hub/run_benchmark.py} (100%) create mode 100644 benchmarks/bm_gunicorn/pyproject.toml rename benchmarks/{gunicorn_requirements.txt => bm_gunicorn/requirements.txt} (100%) rename benchmarks/{gunicorn.py => bm_gunicorn/run_benchmark.py} (100%) create mode 100644 benchmarks/bm_json/pyproject.toml rename benchmarks/{json_bench_requirements.txt => bm_json/requirements.txt} (100%) rename benchmarks/{json_bench.py => bm_json/run_benchmark.py} (100%) create mode 100644 benchmarks/bm_mypy/pyproject.toml rename benchmarks/{mypy_bench_requirements.txt => bm_mypy/requirements.txt} (100%) rename benchmarks/{mypy_bench.py => bm_mypy/run_benchmark.py} (100%) create mode 100644 benchmarks/bm_pycparser/pyproject.toml rename benchmarks/{pycparser_bench_requirements.txt => bm_pycparser/requirements.txt} (100%) rename benchmarks/{pycparser_bench.py => bm_pycparser/run_benchmark.py} (100%) create mode 100644 benchmarks/bm_pylint/pyproject.toml rename benchmarks/{pylint_bench_requirements.txt => bm_pylint/requirements.txt} (100%) rename benchmarks/{pylint_bench.py => bm_pylint/run_benchmark.py} (100%) create mode 100644 benchmarks/bm_pytorch_alexnet_inference/pyproject.toml rename benchmarks/{pytorch_alexnet_inference_requirements.txt => bm_pytorch_alexnet_inference/requirements.txt} (100%) rename benchmarks/{pytorch_alexnet_inference.py => bm_pytorch_alexnet_inference/run_benchmark.py} (100%) create mode 100644 benchmarks/bm_thrift/pyproject.toml rename benchmarks/{thrift_bench_requirements.txt => bm_thrift/requirements.txt} (100%) rename benchmarks/{thrift_bench.py => bm_thrift/run_benchmark.py} (100%) diff --git a/benchmarks/MANIFEST b/benchmarks/MANIFEST new file mode 100644 index 0000000..a429270 --- /dev/null +++ b/benchmarks/MANIFEST @@ -0,0 +1,28 @@ +[benchmarks] + +name version origin metafile +aiohttp - - +djangocms - - +flaskblogging - - +gevent_hub - - +gunicorn - - +json - - +mypy - - +pycparser - - +pylint - - +pytorch_alexnet_inference - - +thrift - - + + +[group default] +aiohttp +djangocms +flaskblogging +gevent_hub +gunicorn +json +mypy +pycparser +pylint +pytorch_alexnet_inference +thrift diff --git a/benchmarks/base.toml b/benchmarks/base.toml new file mode 100644 index 0000000..fb3c610 --- /dev/null +++ b/benchmarks/base.toml @@ -0,0 +1,21 @@ +[project] +#description = "a pyperformance benchmark" +#readme = "README.rst" +#requires-python = ">=3.8" +#license = {file = "LICENSE.txt"} + +dependencies = [ + "pyperf", +] + +urls = {repository = "https://github.com/pyston/python-macrobenchmarks"} + +dynamic = [ + "name", + "version", +] + +[tool.pyperformance] +metabase = "" +#tags = [] +#extra_opts = "" diff --git a/benchmarks/bm_aiohttp/pyproject.toml b/benchmarks/bm_aiohttp/pyproject.toml new file mode 100644 index 0000000..cb31e8e --- /dev/null +++ b/benchmarks/bm_aiohttp/pyproject.toml @@ -0,0 +1,18 @@ +[project] +name = "bm_aiohttp" +dependencies = [ + "aiohttp", + "gunicorn", + "requests", + "uvloop", +] + +# XXX This should be inherited from metabase. +dynamic = [ + "name", + "version", +] + +[tool.pyperformance] +# "name" is set automatically. +metabase = ".." diff --git a/benchmarks/aiohttp_requirements.txt b/benchmarks/bm_aiohttp/requirements.txt similarity index 100% rename from benchmarks/aiohttp_requirements.txt rename to benchmarks/bm_aiohttp/requirements.txt diff --git a/benchmarks/aiohttp.py b/benchmarks/bm_aiohttp/run_benchmark.py similarity index 100% rename from benchmarks/aiohttp.py rename to benchmarks/bm_aiohttp/run_benchmark.py diff --git a/benchmarks/bm_djangocms/pyproject.toml b/benchmarks/bm_djangocms/pyproject.toml new file mode 100644 index 0000000..0297f4f --- /dev/null +++ b/benchmarks/bm_djangocms/pyproject.toml @@ -0,0 +1,24 @@ +[project] +name = "bm_djangocms" +dependencies = [ + "Django", + "django-cms", + "djangocms-bootstrap4", + "djangocms-file", + "djangocms-googlemap", + "djangocms-installer", + "djangocms-snippet", + "djangocms-style", + "djangocms-video", + "requests", +] + +# XXX This should be inherited from metabase. +dynamic = [ + "name", + "version", +] + +[tool.pyperformance] +# "name" is set automatically. +metabase = ".." diff --git a/benchmarks/djangocms_requirements.txt b/benchmarks/bm_djangocms/requirements.txt similarity index 100% rename from benchmarks/djangocms_requirements.txt rename to benchmarks/bm_djangocms/requirements.txt diff --git a/benchmarks/djangocms.py b/benchmarks/bm_djangocms/run_benchmark.py similarity index 100% rename from benchmarks/djangocms.py rename to benchmarks/bm_djangocms/run_benchmark.py diff --git a/benchmarks/bm_flaskblogging/pyproject.toml b/benchmarks/bm_flaskblogging/pyproject.toml new file mode 100644 index 0000000..8208132 --- /dev/null +++ b/benchmarks/bm_flaskblogging/pyproject.toml @@ -0,0 +1,17 @@ +[project] +name = "bm_flaskblogging" +dependencies = [ + "Flask", + "Flask-Blogging", + "requests", +] + +# XXX This should be inherited from metabase. +dynamic = [ + "name", + "version", +] + +[tool.pyperformance] +# "name" is set automatically. +metabase = ".." diff --git a/benchmarks/flaskblogging_requirements.txt b/benchmarks/bm_flaskblogging/requirements.txt similarity index 100% rename from benchmarks/flaskblogging_requirements.txt rename to benchmarks/bm_flaskblogging/requirements.txt diff --git a/benchmarks/flaskblogging.py b/benchmarks/bm_flaskblogging/run_benchmark.py similarity index 100% rename from benchmarks/flaskblogging.py rename to benchmarks/bm_flaskblogging/run_benchmark.py diff --git a/benchmarks/bm_gevent_hub/pyproject.toml b/benchmarks/bm_gevent_hub/pyproject.toml new file mode 100644 index 0000000..09f610e --- /dev/null +++ b/benchmarks/bm_gevent_hub/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "bm_gevent_hub" +dependencies = [ + "gevent", +] + +# XXX This should be inherited from metabase. +dynamic = [ + "name", + "version", +] + +[tool.pyperformance] +# "name" is set automatically. +metabase = ".." diff --git a/benchmarks/gevent_bench_hub_requirements.txt b/benchmarks/bm_gevent_hub/requirements.txt similarity index 100% rename from benchmarks/gevent_bench_hub_requirements.txt rename to benchmarks/bm_gevent_hub/requirements.txt diff --git a/benchmarks/gevent_bench_hub.py b/benchmarks/bm_gevent_hub/run_benchmark.py similarity index 100% rename from benchmarks/gevent_bench_hub.py rename to benchmarks/bm_gevent_hub/run_benchmark.py diff --git a/benchmarks/bm_gunicorn/pyproject.toml b/benchmarks/bm_gunicorn/pyproject.toml new file mode 100644 index 0000000..e40ca2c --- /dev/null +++ b/benchmarks/bm_gunicorn/pyproject.toml @@ -0,0 +1,17 @@ +[project] +name = "bm_gunicorn" +dependencies = [ + "gunicorn", + "requests", + "uvloop", +] + +# XXX This should be inherited from metabase. +dynamic = [ + "name", + "version", +] + +[tool.pyperformance] +# "name" is set automatically. +metabase = ".." diff --git a/benchmarks/gunicorn_requirements.txt b/benchmarks/bm_gunicorn/requirements.txt similarity index 100% rename from benchmarks/gunicorn_requirements.txt rename to benchmarks/bm_gunicorn/requirements.txt diff --git a/benchmarks/gunicorn.py b/benchmarks/bm_gunicorn/run_benchmark.py similarity index 100% rename from benchmarks/gunicorn.py rename to benchmarks/bm_gunicorn/run_benchmark.py diff --git a/benchmarks/bm_json/pyproject.toml b/benchmarks/bm_json/pyproject.toml new file mode 100644 index 0000000..f2433db --- /dev/null +++ b/benchmarks/bm_json/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "bm_json" +#dependencies = [] + +# XXX This should be inherited from metabase. +dynamic = [ + "name", + "version", +] + +[tool.pyperformance] +# "name" is set automatically. +metabase = ".." diff --git a/benchmarks/json_bench_requirements.txt b/benchmarks/bm_json/requirements.txt similarity index 100% rename from benchmarks/json_bench_requirements.txt rename to benchmarks/bm_json/requirements.txt diff --git a/benchmarks/json_bench.py b/benchmarks/bm_json/run_benchmark.py similarity index 100% rename from benchmarks/json_bench.py rename to benchmarks/bm_json/run_benchmark.py diff --git a/benchmarks/bm_mypy/pyproject.toml b/benchmarks/bm_mypy/pyproject.toml new file mode 100644 index 0000000..8dd39e2 --- /dev/null +++ b/benchmarks/bm_mypy/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "bm_mypy" +dependencies = [ + "mypy", +] + +# XXX This should be inherited from metabase. +dynamic = [ + "name", + "version", +] + +[tool.pyperformance] +# "name" is set automatically. +metabase = ".." diff --git a/benchmarks/mypy_bench_requirements.txt b/benchmarks/bm_mypy/requirements.txt similarity index 100% rename from benchmarks/mypy_bench_requirements.txt rename to benchmarks/bm_mypy/requirements.txt diff --git a/benchmarks/mypy_bench.py b/benchmarks/bm_mypy/run_benchmark.py similarity index 100% rename from benchmarks/mypy_bench.py rename to benchmarks/bm_mypy/run_benchmark.py diff --git a/benchmarks/bm_pycparser/pyproject.toml b/benchmarks/bm_pycparser/pyproject.toml new file mode 100644 index 0000000..f7a352c --- /dev/null +++ b/benchmarks/bm_pycparser/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "bm_pycparser" +dependencies = [ + "pycparser", +] + +# XXX This should be inherited from metabase. +dynamic = [ + "name", + "version", +] + +[tool.pyperformance] +# "name" is set automatically. +metabase = ".." diff --git a/benchmarks/pycparser_bench_requirements.txt b/benchmarks/bm_pycparser/requirements.txt similarity index 100% rename from benchmarks/pycparser_bench_requirements.txt rename to benchmarks/bm_pycparser/requirements.txt diff --git a/benchmarks/pycparser_bench.py b/benchmarks/bm_pycparser/run_benchmark.py similarity index 100% rename from benchmarks/pycparser_bench.py rename to benchmarks/bm_pycparser/run_benchmark.py diff --git a/benchmarks/bm_pylint/pyproject.toml b/benchmarks/bm_pylint/pyproject.toml new file mode 100644 index 0000000..441bcb1 --- /dev/null +++ b/benchmarks/bm_pylint/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "bm_pylint" +dependencies = [ + "pylint", +] + +# XXX This should be inherited from metabase. +dynamic = [ + "name", + "version", +] + +[tool.pyperformance] +# "name" is set automatically. +metabase = ".." diff --git a/benchmarks/pylint_bench_requirements.txt b/benchmarks/bm_pylint/requirements.txt similarity index 100% rename from benchmarks/pylint_bench_requirements.txt rename to benchmarks/bm_pylint/requirements.txt diff --git a/benchmarks/pylint_bench.py b/benchmarks/bm_pylint/run_benchmark.py similarity index 100% rename from benchmarks/pylint_bench.py rename to benchmarks/bm_pylint/run_benchmark.py diff --git a/benchmarks/bm_pytorch_alexnet_inference/pyproject.toml b/benchmarks/bm_pytorch_alexnet_inference/pyproject.toml new file mode 100644 index 0000000..59496d6 --- /dev/null +++ b/benchmarks/bm_pytorch_alexnet_inference/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "bm_pytorch_alexnet_inference" +dependencies = [ + "torch", + "Pillow", +] + +# XXX This should be inherited from metabase. +dynamic = [ + "name", + "version", +] + +[tool.pyperformance] +# "name" is set automatically. +metabase = ".." diff --git a/benchmarks/pytorch_alexnet_inference_requirements.txt b/benchmarks/bm_pytorch_alexnet_inference/requirements.txt similarity index 100% rename from benchmarks/pytorch_alexnet_inference_requirements.txt rename to benchmarks/bm_pytorch_alexnet_inference/requirements.txt diff --git a/benchmarks/pytorch_alexnet_inference.py b/benchmarks/bm_pytorch_alexnet_inference/run_benchmark.py similarity index 100% rename from benchmarks/pytorch_alexnet_inference.py rename to benchmarks/bm_pytorch_alexnet_inference/run_benchmark.py diff --git a/benchmarks/bm_thrift/pyproject.toml b/benchmarks/bm_thrift/pyproject.toml new file mode 100644 index 0000000..97bf416 --- /dev/null +++ b/benchmarks/bm_thrift/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "bm_thrift" +dependencies = [ + "thrift", +] + +# XXX This should be inherited from metabase. +dynamic = [ + "name", + "version", +] + +[tool.pyperformance] +# "name" is set automatically. +metabase = ".." diff --git a/benchmarks/thrift_bench_requirements.txt b/benchmarks/bm_thrift/requirements.txt similarity index 100% rename from benchmarks/thrift_bench_requirements.txt rename to benchmarks/bm_thrift/requirements.txt diff --git a/benchmarks/thrift_bench.py b/benchmarks/bm_thrift/run_benchmark.py similarity index 100% rename from benchmarks/thrift_bench.py rename to benchmarks/bm_thrift/run_benchmark.py From 3b1fbc9cd062aa63fcb43e807fcc8ab96e3105ce Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 20 Jul 2021 10:22:14 -0600 Subject: [PATCH 02/32] Fix the manifest. --- benchmarks/MANIFEST | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/MANIFEST b/benchmarks/MANIFEST index a429270..a257be0 100644 --- a/benchmarks/MANIFEST +++ b/benchmarks/MANIFEST @@ -1,6 +1,6 @@ [benchmarks] -name version origin metafile +name version origin metafile aiohttp - - djangocms - - flaskblogging - - From 431d5a35cad26af296fa4f8726a8f0b825be5752 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 20 Jul 2021 11:00:26 -0600 Subject: [PATCH 03/32] Add a version to the base metadata. --- benchmarks/base.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/base.toml b/benchmarks/base.toml index fb3c610..e575389 100644 --- a/benchmarks/base.toml +++ b/benchmarks/base.toml @@ -3,6 +3,7 @@ #readme = "README.rst" #requires-python = ">=3.8" #license = {file = "LICENSE.txt"} +version = "0.9.0" # XXX an arbitrary value; the repo doesn't have one dependencies = [ "pyperf", From 440762de42657cc762e11cdd847d824cd286c6a2 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 20 Jul 2021 11:03:03 -0600 Subject: [PATCH 04/32] Add basic networking-related utils to share. --- benchmarks/.libs/netutils.py | 157 +++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 benchmarks/.libs/netutils.py diff --git a/benchmarks/.libs/netutils.py b/benchmarks/.libs/netutils.py new file mode 100644 index 0000000..cb44e3b --- /dev/null +++ b/benchmarks/.libs/netutils.py @@ -0,0 +1,157 @@ +import contextlib +import ipaddress +import re +import socket +import subprocess +import time +import urllib.parse + + +# See: https://validators.readthedocs.io/en/latest/_modules/validators/domain.html +DOMAIN_RE = re.compile( + r'^(([a-zA-Z]{1})|([a-zA-Z]{1}[a-zA-Z]{1})|' + r'([a-zA-Z]{1}[0-9]{1})|([0-9]{1}[a-zA-Z]{1})|' + r'([a-zA-Z0-9][-_.a-zA-Z0-9]{0,61}[a-zA-Z0-9]))\.' + r'([a-zA-Z]{2,13}|[a-zA-Z0-9-]{2,30}.[a-zA-Z]{2,3})$' +) + +LOCALHOST = '127.0.0.1' +HTTP_PORT = 8080 + + +@contextlib.contextmanager +def serving(argv, sitedir, addr='127.0.0.1'): + p = subprocess.Popen( + argv, + cwd=sitedir, + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + ) + try: + waitUntilUp(addr) + yield + assert p.poll() is None, p.poll() + finally: + p.terminate() + p.wait() + + +def waitUntilUp(addr, timeout=10.0): + end = time.time() + timeout + addr = parse_socket_addr(addr) + while True: + try: + with socket.create_connection(addr) as sock: + return + except ConnectionRefusedError: + if time.time() > end: + raise Exception("Timeout reached when trying to connect") + time.sleep(0.001) + + +def parse_socket_addr(addr, *, resolve=True): + orig = addr + if not isinstance(addr, str): + addr = _render_attr(addr) + scheme, domain, ipaddr, port = _parse_addr(addr) + + domain, ipaddr = _resolve_hostname(domain, ipaddr, dns=resolve) + hostname = ipaddr or domain or LOCALHOST + port = _resolve_port(port, scheme, names=resolve) + + return hostname, port + + +def _resolve_port(port, scheme, *, names=False): + if port is None: + if not scheme or scheme in ('http', 'https'): + return HTTP_PORT + else: + raise ValueError(f'missing port in {addr!r}') + + try: + port = int(port) + except (TypeError, ValueError): + if not isinstance(port, str): + raise # re-raise + if names: + raise NotImplementedError + # else check bounds? + return port + + +def _parse_addr(addr): + parsed = urllib.parse.urlparse(addr) + scheme = parsed.scheme + if not scheme: + parsed = urllib.parse.urlparse(f'spam://{addr}') + + hostname = parsed.hostname + try: + port = parsed.port + except ValueError: + if hostname: + raise NotImplementedError(addr) + port = None + + if not hostname and not port: + if not parsed.netloc: + raise NotImplementedError(addr) + hostname, _, port = parsed.netloc.partition(':') + if port: + if port.isdigit(): + port = int(port) + elif port.isalpha(): + port = port.upper() + else: + raise NotImplementedError(addr) + else: + port = None + + domain = addr = None + if hostname: + try: + ipaddress.ip_address(hostname) + except ValueError: + domain, err = _normalize_domainname(hostname) + if err: + raise ValueError(f'bad hostname in addr {addr!r}') + else: + addr = hostname + + return scheme, domain, addr, port + + +def _resolve_hostname(domain, addr=None, *, dns=False): + if addr: + # XXX Resolve other addresses? + domain = 'localhost' if addr == LOCALHOST else None + elif not domain: + domain = 'localhost' + addr = LOCALHOST + elif domain.lower() == 'localhost': + domain = 'localhost' + addr = LOCALHOST + elif dns: + raise NotImplementedError + else: + raise ValueError(f'could not resolve hostname {domain!r}') + return domain, addr + + +def _normalize_domainname(name): + # See: + # * https://datatracker.ietf.org/doc/html/rfc3696#section-2 + if not name: + raise ValueError('missing hostname') + + if name[-1] == '.': + name = name[:-1] + + if len(name) > 253: + err = f'domain name too long {name!r}' + elif not DOMAIN_RE.match(name): + err = f'invalid domain name {name!r}' + else: + err = None + return name, err From 5a683f71aa93e0c664af9c7d0818dd591b5be718 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 20 Jul 2021 16:08:40 -0600 Subject: [PATCH 05/32] Drop a buch of unnecessary code in netutils. --- benchmarks/.libs/netutils.py | 130 ++++------------------------------- 1 file changed, 15 insertions(+), 115 deletions(-) diff --git a/benchmarks/.libs/netutils.py b/benchmarks/.libs/netutils.py index cb44e3b..af213b3 100644 --- a/benchmarks/.libs/netutils.py +++ b/benchmarks/.libs/netutils.py @@ -1,26 +1,12 @@ import contextlib import ipaddress -import re import socket import subprocess import time -import urllib.parse - - -# See: https://validators.readthedocs.io/en/latest/_modules/validators/domain.html -DOMAIN_RE = re.compile( - r'^(([a-zA-Z]{1})|([a-zA-Z]{1}[a-zA-Z]{1})|' - r'([a-zA-Z]{1}[0-9]{1})|([0-9]{1}[a-zA-Z]{1})|' - r'([a-zA-Z0-9][-_.a-zA-Z0-9]{0,61}[a-zA-Z0-9]))\.' - r'([a-zA-Z]{2,13}|[a-zA-Z0-9-]{2,30}.[a-zA-Z]{2,3})$' -) - -LOCALHOST = '127.0.0.1' -HTTP_PORT = 8080 @contextlib.contextmanager -def serving(argv, sitedir, addr='127.0.0.1'): +def serving(argv, sitedir, addr): p = subprocess.Popen( argv, cwd=sitedir, @@ -45,113 +31,27 @@ def waitUntilUp(addr, timeout=10.0): return except ConnectionRefusedError: if time.time() > end: - raise Exception("Timeout reached when trying to connect") + raise Exception('Timeout reached when trying to connect') time.sleep(0.001) def parse_socket_addr(addr, *, resolve=True): - orig = addr if not isinstance(addr, str): - addr = _render_attr(addr) - scheme, domain, ipaddr, port = _parse_addr(addr) - - domain, ipaddr = _resolve_hostname(domain, ipaddr, dns=resolve) - hostname = ipaddr or domain or LOCALHOST - port = _resolve_port(port, scheme, names=resolve) - - return hostname, port - - -def _resolve_port(port, scheme, *, names=False): - if port is None: - if not scheme or scheme in ('http', 'https'): - return HTTP_PORT - else: - raise ValueError(f'missing port in {addr!r}') + raise NotImplementedError(addr) + host, _, port = addr.partition(':') + if not host: + raise NotImplementedError(addr) try: - port = int(port) - except (TypeError, ValueError): - if not isinstance(port, str): - raise # re-raise - if names: - raise NotImplementedError - # else check bounds? - return port - - -def _parse_addr(addr): - parsed = urllib.parse.urlparse(addr) - scheme = parsed.scheme - if not scheme: - parsed = urllib.parse.urlparse(f'spam://{addr}') - - hostname = parsed.hostname - try: - port = parsed.port + host = ipaddress.ip_address(host) except ValueError: - if hostname: - raise NotImplementedError(addr) - port = None - - if not hostname and not port: - if not parsed.netloc: - raise NotImplementedError(addr) - hostname, _, port = parsed.netloc.partition(':') - if port: - if port.isdigit(): - port = int(port) - elif port.isalpha(): - port = port.upper() - else: - raise NotImplementedError(addr) - else: - port = None - - domain = addr = None - if hostname: - try: - ipaddress.ip_address(hostname) - except ValueError: - domain, err = _normalize_domainname(hostname) - if err: - raise ValueError(f'bad hostname in addr {addr!r}') - else: - addr = hostname - - return scheme, domain, addr, port - - -def _resolve_hostname(domain, addr=None, *, dns=False): - if addr: - # XXX Resolve other addresses? - domain = 'localhost' if addr == LOCALHOST else None - elif not domain: - domain = 'localhost' - addr = LOCALHOST - elif domain.lower() == 'localhost': - domain = 'localhost' - addr = LOCALHOST - elif dns: - raise NotImplementedError - else: - raise ValueError(f'could not resolve hostname {domain!r}') - return domain, addr - - -def _normalize_domainname(name): - # See: - # * https://datatracker.ietf.org/doc/html/rfc3696#section-2 - if not name: - raise ValueError('missing hostname') + raise NotImplementedError(addr) + host = str(host) - if name[-1] == '.': - name = name[:-1] + if not port: + raise NotImplementedError(addr) + if not port.isdigit(): + raise NotImplementedError(addr) + port = int(port) - if len(name) > 253: - err = f'domain name too long {name!r}' - elif not DOMAIN_RE.match(name): - err = f'invalid domain name {name!r}' - else: - err = None - return name, err + return (host, port) From 19592052f3dc57430874fbecb1e14178e1e89860 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 21 Jul 2021 18:24:45 -0600 Subject: [PATCH 06/32] Clean up waitUntilUp(). --- benchmarks/.libs/netutils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/benchmarks/.libs/netutils.py b/benchmarks/.libs/netutils.py index af213b3..64e2919 100644 --- a/benchmarks/.libs/netutils.py +++ b/benchmarks/.libs/netutils.py @@ -25,14 +25,17 @@ def serving(argv, sitedir, addr): def waitUntilUp(addr, timeout=10.0): end = time.time() + timeout addr = parse_socket_addr(addr) - while True: + started = False + current = time.time() + while not started or current <= end: try: with socket.create_connection(addr) as sock: return except ConnectionRefusedError: - if time.time() > end: - raise Exception('Timeout reached when trying to connect') time.sleep(0.001) + started = True + current = time.time() + raise Exception('Timeout reached when trying to connect') def parse_socket_addr(addr, *, resolve=True): From e5f88611563aee6ea5d9d0f46b70f370d6ab2b41 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 20 Jul 2021 11:04:26 -0600 Subject: [PATCH 07/32] Finish updating the benchmarks to the pyperformance format. --- benchmarks/base.toml | 1 + .../bm_aiohttp/data/serve.py | 0 benchmarks/bm_aiohttp/run_benchmark.py | 50 ++-- benchmarks/bm_djangocms/run_benchmark.py | 237 ++++++++++++------ .../bm_flaskblogging/data/serve.py | 0 benchmarks/bm_flaskblogging/run_benchmark.py | 50 ++-- .../bm_gevent_hub/bm_gevent_cancel_wait.toml | 17 ++ .../bm_gevent_hub/bm_gevent_switch.toml | 17 ++ .../bm_gevent_wait_func_ready.toml | 17 ++ .../bm_gevent_hub/bm_gevent_wait_ready.toml | 17 ++ benchmarks/bm_gevent_hub/run_benchmark.py | 157 ++++++------ benchmarks/bm_gunicorn/data/serve_aiohttp.py | 12 + benchmarks/bm_gunicorn/run_benchmark.py | 68 +++-- .../bm_json/data}/reddit_comments.json | 0 benchmarks/bm_json/run_benchmark.py | 42 ++-- .../bm_mypy/data}/mypy_target.py | 0 benchmarks/bm_mypy/run_benchmark.py | 61 +++-- .../data}/pycparser_target/README | 0 .../data}/pycparser_target/redis.c.ppout | 0 .../pycparser_target/sqlite-btree.c.ppout | 0 .../data}/pycparser_target/tccgen.c.ppout | 0 benchmarks/bm_pycparser/run_benchmark.py | 41 +-- .../bm_pylint/data}/pylint_target/__init__.py | 0 .../bm_pylint/data}/pylint_target/dist.py | 0 benchmarks/bm_pylint/run_benchmark.py | 53 ++-- .../requirements.txt | 1 + .../run_benchmark.py | 57 +++-- {data => benchmarks/bm_thrift/data}/Makefile | 0 .../bm_thrift/data}/addressbook.thrift | 0 .../bm_thrift/data}/thrift/__init__.py | 0 .../data}/thrift/addressbook/__init__.py | 0 .../data}/thrift/addressbook/constants.py | 0 .../data}/thrift/addressbook/ttypes.py | 0 benchmarks/bm_thrift/run_benchmark.py | 51 ++-- run_all.sh | 14 +- run_mypy.sh | 34 ++- 36 files changed, 592 insertions(+), 405 deletions(-) rename data/gunicorn_serve.py => benchmarks/bm_aiohttp/data/serve.py (100%) rename data/flaskblogging_serve.py => benchmarks/bm_flaskblogging/data/serve.py (100%) create mode 100644 benchmarks/bm_gevent_hub/bm_gevent_cancel_wait.toml create mode 100644 benchmarks/bm_gevent_hub/bm_gevent_switch.toml create mode 100644 benchmarks/bm_gevent_hub/bm_gevent_wait_func_ready.toml create mode 100644 benchmarks/bm_gevent_hub/bm_gevent_wait_ready.toml create mode 100644 benchmarks/bm_gunicorn/data/serve_aiohttp.py rename {data => benchmarks/bm_json/data}/reddit_comments.json (100%) rename {data => benchmarks/bm_mypy/data}/mypy_target.py (100%) rename {data => benchmarks/bm_pycparser/data}/pycparser_target/README (100%) rename {data => benchmarks/bm_pycparser/data}/pycparser_target/redis.c.ppout (100%) rename {data => benchmarks/bm_pycparser/data}/pycparser_target/sqlite-btree.c.ppout (100%) rename {data => benchmarks/bm_pycparser/data}/pycparser_target/tccgen.c.ppout (100%) rename {data => benchmarks/bm_pylint/data}/pylint_target/__init__.py (100%) rename {data => benchmarks/bm_pylint/data}/pylint_target/dist.py (100%) rename {data => benchmarks/bm_thrift/data}/Makefile (100%) rename {data => benchmarks/bm_thrift/data}/addressbook.thrift (100%) rename {data => benchmarks/bm_thrift/data}/thrift/__init__.py (100%) rename {data => benchmarks/bm_thrift/data}/thrift/addressbook/__init__.py (100%) rename {data => benchmarks/bm_thrift/data}/thrift/addressbook/constants.py (100%) rename {data => benchmarks/bm_thrift/data}/thrift/addressbook/ttypes.py (100%) diff --git a/benchmarks/base.toml b/benchmarks/base.toml index e575389..61b4a89 100644 --- a/benchmarks/base.toml +++ b/benchmarks/base.toml @@ -20,3 +20,4 @@ dynamic = [ metabase = "" #tags = [] #extra_opts = "" +libsdir = ".libs" diff --git a/data/gunicorn_serve.py b/benchmarks/bm_aiohttp/data/serve.py similarity index 100% rename from data/gunicorn_serve.py rename to benchmarks/bm_aiohttp/data/serve.py diff --git a/benchmarks/bm_aiohttp/run_benchmark.py b/benchmarks/bm_aiohttp/run_benchmark.py index 45f26ca..5021425 100644 --- a/benchmarks/bm_aiohttp/run_benchmark.py +++ b/benchmarks/bm_aiohttp/run_benchmark.py @@ -1,41 +1,29 @@ -import json -import os +import os.path import requests -import subprocess import sys -import threading -import time -from djangocms import waitUntilUp +import pyperf +import netutils -if __name__ == "__main__": - exe = sys.executable - - times = [] - p = subprocess.Popen([exe, "gunicorn_serve.py"], stdout=open("/dev/null", "w"), stderr=subprocess.STDOUT, cwd=os.path.join(os.path.dirname(__file__), "../data")) - try: - waitUntilUp(("127.0.0.1", 8080)) +DATADIR = os.path.join( + os.path.dirname(__file__), + "data", +) +ARGV = [sys.executable, "serve.py"] - n = 3000 - if len(sys.argv) > 1: - n = int(sys.argv[1]) - start = time.time() - for i in range(n): - times.append(time.time()) - if i % 100 == 0: - print(i, time.time() - start) +def bench_aiohttp(loops=3000): + loops = iter(range(loops)) + with netutils.serving(ARGV, DATADIR, "127.0.0.1:8080"): + t0 = pyperf.perf_counter() + for _ in loops: requests.get("http://localhost:8080/blog/").text - times.append(time.time()) - elapsed = time.time() - start - print("%.2fs (%.3freq/s)" % (elapsed, n / elapsed)) + return pyperf.perf_counter() - t0 - assert p.poll() is None, p.poll() - finally: - p.terminate() - p.wait() - - if len(sys.argv) > 2: - json.dump(times, open(sys.argv[2], 'w')) +if __name__ == "__main__": + runner = pyperf.Runner() + runner.metadata['description'] = "Test the performance of aiohttp" + #runner.bench_func("aiohttp", bench_aiohttp) + runner.bench_time_func("aiohttp", bench_aiohttp) diff --git a/benchmarks/bm_djangocms/run_benchmark.py b/benchmarks/bm_djangocms/run_benchmark.py index af0d3ca..71633c1 100644 --- a/benchmarks/bm_djangocms/run_benchmark.py +++ b/benchmarks/bm_djangocms/run_benchmark.py @@ -9,28 +9,34 @@ and browsing around. """ +import contextlib import os +import os.path import requests -import socket +import shutil import subprocess import sys import tempfile -import time -import json -def setup(): - """ - Set up a djangocms installation. - Runs the initial bootstrapping without the db migration, - so that we can turn off sqlite synchronous and avoid fs time. - Rough testing shows that setting synchronous=OFF is basically - the same performance as running on /dev/shm - """ +import pyperf +import netutils + - subprocess.check_call([exe.replace("python3", "djangocms"), "testsite", "--verbose", "--no-sync"]) +DATADIR = os.path.join( + os.path.dirname(__file__), + "data", +) +PID_FILE = os.path.join(DATADIR, "setup.pid") +# It might be interesting to put the temporary directory in /dev/shm, +# which makes the initial db migration about 20% faster. +TEMP_DIR = None +TEMP_PREFIX = "djangocms_bench_" - with open("testsite/testsite/settings.py", "a") as f: - f.write(""" +INNER_LOOPS = 800 + +# site +SITE_NAME = "testsite" +SETTINGS = """ from django.db.backends.signals import connection_created def set_no_sychronous(sender, connection, **kwargs): if connection.vendor == 'sqlite': @@ -38,62 +44,92 @@ def set_no_sychronous(sender, connection, **kwargs): cursor.execute('PRAGMA synchronous = OFF;') connection_created.connect(set_no_sychronous) -""") - start = time.time() - subprocess.check_call([exe, "manage.py", "migrate"], cwd="testsite") - elapsed = time.time() - start - print("%.2fs to initialize db" % (elapsed,)) - -def waitUntilUp(addr, timeout=10.0): - start = time.time() - while True: - try: - with socket.create_connection(addr) as sock: - return - except ConnectionRefusedError: - if time.time() > start + timeout: - raise Exception("Timeout reached when trying to connect") - time.sleep(0.001) - -def runbenchmark(n=800, out_file=None): - p = subprocess.Popen([exe, "manage.py", "runserver", "--noreload"], cwd="testsite", stdout=open("/dev/null", "w"), stderr=subprocess.STDOUT) - try: - waitUntilUp(("127.0.0.1", 8000)) +""" - start = time.time() - times = [] - for i in range(n): - times.append(time.time()) - if i % 100 == 0: - print(i, time.time() - start) - requests.get("http://localhost:8000/").text - times.append(time.time()) - elapsed = time.time() - start - print("%.2fs (%.3freq/s)" % (elapsed, n / elapsed)) +# django commands +DJANGOCMS = os.path.join( + os.path.dirname(sys.executable), + "djangocms", +) +ARGV_CREATE = [DJANGOCMS, SITE_NAME, "--verbose", "--no-sync"] +ARGV_MIGRATE = [sys.executable, "manage.py", "migrate"] +ARGV_SERVE = [sys.executable, "manage.py", "runserver", "--noreload"] - exitcode = p.poll() - assert exitcode is None, exitcode - if out_file: - json.dump(times, open(out_file, 'w')) +def setup(rootdir): + """ + Set up a djangocms installation. + Runs the initial bootstrapping without the db migration, + so that we can turn off sqlite synchronous and avoid fs time. + Rough testing shows that setting synchronous=OFF is basically + the same performance as running on /dev/shm. + """ + sitedir = os.path.join(rootdir, SITE_NAME) # This is where Django puts it. + # Delete the site dir if it already exists. + if os.path.exists(sitedir): + shutil.rmtree(datadir, ignore_errors=False) + + # First, create the site. + subprocess.check_call(ARGV_CREATE, cwd=rootdir) + + # Add customizations. + settingsfile = os.path.join(sitedir, SITE_NAME, "settings.py") + with open(settingsfile, "a") as f: + f.write(SETTINGS) + + # Finalize the site. + t0 = pyperf.perf_counter() + subprocess.check_call(ARGV_MIGRATE, cwd=sitedir) + elapsed = pyperf.perf_counter() - t0 + + return sitedir, elapsed + + +# This is a generic util that might make sense to put in a separate lib. +def _ensure_python_on_PATH(python=sys.executable): + PATH = os.environ["PATH"].split(os.pathsep) + PATH.insert(0, os.path.dirname(python)) + os.environ["PATH"] = os.pathsep.join(PATH) + + +@contextlib.contextmanager +def _ensure_datadir(datadir, preserve=True): + if datadir: + try: + os.makedirs(datadir) + except FileExistsError: + if preserve is None: + preserve = True + elif not preserve: + raise NotImplementedError(datadir) + else: + datadir = tempfile.mkdtemp(prefix=TEMP_PREFIX, dir=TEMP_DIR) + + try: + yield datadir finally: - p.terminate() - p.wait() + if not preserve: + shutil.rmtree(datadir, ignore_errors=True) -if __name__ == "__main__": - exe = sys.executable - # Hack: make sure this file gets run as "python3" so that perf will collate across different processes - if not exe.endswith('3'): - os.execv(exe + '3', [exe + '3'] + sys.argv) - os.environ["PATH"] = os.path.dirname(exe) + ":" + os.environ["PATH"] +def bench_djangocms(sitedir, loops=INNER_LOOPS): + requests_get = requests.get + loops = iter(range(loops)) + + with netutils.serving(ARGV_SERVE, sitedir, "127.0.0.1:8000"): + t0 = pyperf.perf_counter() + for _ in loops: + requests_get("http://localhost:8000/").text + return pyperf.perf_counter() - t0 + +if __name__ == "__main__": """ Usage: python djangocms.py - python djangocms.py --setup DIR - python djangocms.py --serve DIR + python djangocms.py --setup DATADIR + python djangocms.py --serve DATADIR The first form creates a temporary directory, sets up djangocms in it, serves out of it, and removes the directory. @@ -102,28 +138,63 @@ def runbenchmark(n=800, out_file=None): The second and third forms are useful if you want to benchmark the initial migration phase separately from the second serving phase. """ - if "--setup" in sys.argv: - assert len(sys.argv) > 2 - dir = sys.argv[-1] - os.makedirs(dir, exist_ok=True) - os.chdir(dir) - setup() - elif "--serve" in sys.argv: - assert len(sys.argv) > 2 - os.chdir(sys.argv[-1]) - runbenchmark() + runner = pyperf.Runner() + runner.metadata['description'] = "Test the performance of a Django data migration" + + # Parse the CLI args. + runner.argparser.add_argument("--setup", action="store_const", const=True) + group = runner.argparser.add_mutually_exclusive_group() + group.add_argument("--serve") + group.add_argument("datadir", nargs="?") + args = runner.argparser.parse_args() + + if args.serve is not None: + args.datadir = args.serve + args.serve = True + if not args.setup: + args.setup = False + if not args.datadir: + runner.argparser.error("missing datadir") + elif not os.path.exists(args.datadir): + cmd = f"{sys.executable} {sys.argv[0]} --setup {args.datadir}?" + sys.exit(f"ERROR: Did you forget to run {cmd}?") + default = False + elif args.setup is not None: + args.serve = False + default = False else: - n = 800 - if len(sys.argv) > 1: - n = int(sys.argv[1]) - out_file = None - if len(sys.argv) > 2: - out_file = os.path.abspath(sys.argv[2]) - - # It might be interesting to put the temporary directory in /dev/shm, - # which makes the initial db migration about 20% faster. - with tempfile.TemporaryDirectory(prefix="djangocms_test_") as d: - os.chdir(d) - - setup() - runbenchmark(n, out_file) + args.setup = True + args.serve = True + default = True + + # DjangoCMS looks for Python on $PATH? + _ensure_python_on_PATH() + + # Get everything ready and then perform the requested operations. + preserve = True if args.setup and not args.serve else None + with _ensure_datadir(args.datadir, preserve) as datadir: + # First, set up the site. + if args.setup: + sitedir, elapsed = setup(datadir) + print("%.2fs to initialize db" % (elapsed,)) + print(f"site created in {sitedir}") + if not args.serve: + print(f"now run {sys.executable} {sys.argv[0]} --serve {datadir}") + else: + # This is what a previous call to setup() would have returned. + sitedir = os.path.join(datadir, SITE_NAME) + + # Then run the benchmark. + if args.serve: + def add_worker_args(cmd, _, _datadir=datadir): + cmd.extend([ + '--serve', _datadir, + ]) + # XXX This is an internal attr but we don't have any other good option. + runner._add_cmdline_args = add_worker_args + + def time_func(loops, *args): + return bench_djangocms(*args, loops=loops) + + runner.bench_time_func("djangocms", time_func, sitedir, + inner_loops=INNER_LOOPS) diff --git a/data/flaskblogging_serve.py b/benchmarks/bm_flaskblogging/data/serve.py similarity index 100% rename from data/flaskblogging_serve.py rename to benchmarks/bm_flaskblogging/data/serve.py diff --git a/benchmarks/bm_flaskblogging/run_benchmark.py b/benchmarks/bm_flaskblogging/run_benchmark.py index 40fa891..8a977f1 100644 --- a/benchmarks/bm_flaskblogging/run_benchmark.py +++ b/benchmarks/bm_flaskblogging/run_benchmark.py @@ -1,41 +1,29 @@ -import json -import os +import os.path import requests -import subprocess import sys -import threading -import time -from djangocms import waitUntilUp +import pyperf +import netutils -if __name__ == "__main__": - exe = sys.executable - - times = [] - p = subprocess.Popen([exe, "../data/flaskblogging_serve.py"], stdout=open("/dev/null", "w"), stderr=subprocess.STDOUT, cwd=os.path.dirname(__file__)) - try: - waitUntilUp(("127.0.0.1", 8000)) +DATADIR = os.path.join( + os.path.dirname(__file__), + "data", +) +ARGV = [sys.executable, "serve.py"] - n = 1800 - if len(sys.argv) > 1: - n = int(sys.argv[1]) - start = time.time() - for i in range(n): - times.append(time.time()) - if i % 100 == 0: - print(i, time.time() - start) +def bench_flask_requests(loops=1800): + loops = iter(range(loops)) + with netutils.serving(ARGV, DATADIR, "127.0.0.1:8000"): + t0 = pyperf.perf_counter() + for _ in loops: requests.get("http://localhost:8000/blog/").text - times.append(time.time()) - elapsed = time.time() - start - print("%.2fs (%.3freq/s)" % (elapsed, n / elapsed)) + return pyperf.perf_counter() - t0 - assert p.poll() is None, p.poll() - finally: - p.terminate() - p.wait() - - if len(sys.argv) > 2: - json.dump(times, open(sys.argv[2], 'w')) +if __name__ == "__main__": + runner = pyperf.Runner() + runner.metadata['description'] = "Test the performance of flask" + #runner.bench_func("flaskblogging", bench_flask) + runner.bench_time_func("flaskblogging", bench_flask_requests) diff --git a/benchmarks/bm_gevent_hub/bm_gevent_cancel_wait.toml b/benchmarks/bm_gevent_hub/bm_gevent_cancel_wait.toml new file mode 100644 index 0000000..a44a58d --- /dev/null +++ b/benchmarks/bm_gevent_hub/bm_gevent_cancel_wait.toml @@ -0,0 +1,17 @@ +[project] +name = "bm_gevent_cancel_wait" +# XXX Can these be inherited? +dependencies = [ + "gevent", +] + +# XXX This should be inherited from metabase. +dynamic = [ + "name", + "version", +] + +[tool.pyperformance] +# "name" is set automatically. +# "metabase" is set automatically +extra_opts = ["gevent_cancel_wait"] diff --git a/benchmarks/bm_gevent_hub/bm_gevent_switch.toml b/benchmarks/bm_gevent_hub/bm_gevent_switch.toml new file mode 100644 index 0000000..218b14a --- /dev/null +++ b/benchmarks/bm_gevent_hub/bm_gevent_switch.toml @@ -0,0 +1,17 @@ +[project] +name = "bm_gevent_switch" +# XXX Can these be inherited? +dependencies = [ + "gevent", +] + +# XXX This should be inherited from metabase. +dynamic = [ + "name", + "version", +] + +[tool.pyperformance] +# "name" is set automatically. +# "metabase" is set automatically +extra_opts = ["gevent_switch"] diff --git a/benchmarks/bm_gevent_hub/bm_gevent_wait_func_ready.toml b/benchmarks/bm_gevent_hub/bm_gevent_wait_func_ready.toml new file mode 100644 index 0000000..72d8ef2 --- /dev/null +++ b/benchmarks/bm_gevent_hub/bm_gevent_wait_func_ready.toml @@ -0,0 +1,17 @@ +[project] +name = "bm_gevent_wait_func_ready" +# XXX Can these be inherited? +dependencies = [ + "gevent", +] + +# XXX This should be inherited from metabase. +dynamic = [ + "name", + "version", +] + +[tool.pyperformance] +# "name" is set automatically. +# "metabase" is set automatically +extra_opts = ["gevent_wait_func_ready"] diff --git a/benchmarks/bm_gevent_hub/bm_gevent_wait_ready.toml b/benchmarks/bm_gevent_hub/bm_gevent_wait_ready.toml new file mode 100644 index 0000000..c2722d7 --- /dev/null +++ b/benchmarks/bm_gevent_hub/bm_gevent_wait_ready.toml @@ -0,0 +1,17 @@ +[project] +name = "bm_gevent_wait_ready" +# XXX Can these be inherited? +dependencies = [ + "gevent", +] + +# XXX This should be inherited from metabase. +dynamic = [ + "name", + "version", +] + +[tool.pyperformance] +# "name" is set automatically. +# "metabase" is set automatically +extra_opts = ["gevent_wait_ready"] diff --git a/benchmarks/bm_gevent_hub/run_benchmark.py b/benchmarks/bm_gevent_hub/run_benchmark.py index 7e27737..d74796d 100644 --- a/benchmarks/bm_gevent_hub/run_benchmark.py +++ b/benchmarks/bm_gevent_hub/run_benchmark.py @@ -9,114 +9,131 @@ from __future__ import division from __future__ import print_function -# import perf -# from perf import perf_counter - +import pyperf import gevent +import gevent.hub from greenlet import greenlet from greenlet import getcurrent -N = 1000 +class NoopWatcher: + def start(self, cb, obj): + # Immediately switch back to the waiter, mark as ready + cb(obj) + + def stop(self): + pass + + +class ActiveWatcher: + active = True + callback = object() + + def close(self): + pass + + +class NoopWatchTarget(object): + def rawlink(self, cb): + cb(self) -def bench_switch(): +def get_switching_greenlets(nswitches=1000): class Parent(type(gevent.get_hub())): - def run(self): - parent = self.parent - for _ in range(N): - parent.switch() + def run(self, *, _loops=nswitches): + switch = self.parent.switch + for _ in range(_loops): + switch() - def child(): - parent = getcurrent().parent + def child(*, _loops=nswitches): + switch = getcurrent().parent.switch # Back to the hub, which in turn goes # back to the main greenlet - for _ in range(N): - parent.switch() + for _ in range(_loops): + switch() hub = Parent(None, None) child_greenlet = greenlet(child, hub) - for _ in range(N): - child_greenlet.switch() + return hub, child_greenlet -def bench_wait_ready(): - class Watcher(object): - def start(self, cb, obj): - # Immediately switch back to the waiter, mark as ready - cb(obj) +def bench_switch(loops=1000): + _, child = get_switching_greenlets(loops) + child_switch = child.switch + loops = iter(range(loops)) - def stop(self): - pass + t0 = pyperf.perf_counter() + for _ in loops: + child_switch() + return pyperf.perf_counter() - t0 - watcher = Watcher() - hub = gevent.get_hub() - - for _ in range(1000): - hub.wait(watcher) -def bench_cancel_wait(): +def bench_wait_ready(loops=1000): + watcher = NoopWatcher() + hub = gevent.get_hub() + hub_wait = hub.wait + loops = iter(range(loops)) - class Watcher(object): - active = True - callback = object() + t0 = pyperf.perf_counter() + for _ in loops: + hub_wait(watcher) + return pyperf.perf_counter() - t0 - def close(self): - pass - watcher = Watcher() +def bench_cancel_wait(loops=1000): + watcher = ActiveWatcher() hub = gevent.get_hub() - loop = hub.loop + hub_cancel_wait = hub.cancel_wait + hub_loop = hub.loop + loops = iter(range(loops)) - for _ in range(1000): + t0 = pyperf.perf_counter() + for _ in loops: # Schedule all the callbacks. - hub.cancel_wait(watcher, None, True) - + hub_cancel_wait(watcher, None, True) # Run them! - for cb in loop._callbacks: + # XXX Start timing here? + for cb in hub_loop._callbacks: if cb.callback: cb.callback(*cb.args) - cb.stop() # so the real loop won't do it + cb.stop() # so the real loop won't do it + elapsed = pyperf.perf_counter() - t0 - # destroy the loop so we don't keep building these functions - # up + # Destroy the loop so we don't keep building these functions up. hub.destroy(True) -def bench_wait_func_ready(): - from gevent.hub import wait - class ToWatch(object): - def rawlink(self, cb): - cb(self) + return elapsed - watched_objects = [ToWatch() for _ in range(N)] - t0 = perf_counter() +def bench_wait_func_ready(loops=1000): + watched_objects = [NoopWatchTarget() for _ in range(loops)] + wait = gevent.hub.wait + t0 = pyperf.perf_counter() wait(watched_objects) + return pyperf.perf_counter() - t0 - return perf_counter() - t0 - -def main(): - - runner = perf.Runner() - runner.bench_func('multiple wait ready', - bench_wait_func_ready, - inner_loops=N) +BENCHMARKS = { + "gevent_hub": bench_switch, # XXX Release 10,000 times? + "gevent_wait_func_ready": bench_wait_func_ready, + "gevent_wait_ready": bench_wait_ready, + "gevent_cancel_wait": bench_cancel_wait, + "gevent_switch": bench_switch, +} - runner.bench_func('wait ready', - bench_wait_ready, - inner_loops=N) - runner.bench_func('cancel wait', - bench_cancel_wait, - inner_loops=N) +if __name__ == "__main__": + runner = pyperf.Runner() + runner.metadata['description'] = "Test the performance of gevent" + runner.argparser.add_argument("benchmark", nargs="?", + choices=sorted(BENCHMARKS), + default="gevent_hub") - runner.bench_func('switch', - bench_switch, - inner_loops=N) + args = runner.parse_args() + name = args.benchmark + bench = BENCHMARKS[name] + assert(bench.__code__.co_varnames[0] == 'loops') + inner_loops = bench.__defaults__[0] -if __name__ == '__main__': - # main() - for i in range(10000): - bench_switch() + runner.bench_time_func(name, bench, inner_loops=inner_loops) diff --git a/benchmarks/bm_gunicorn/data/serve_aiohttp.py b/benchmarks/bm_gunicorn/data/serve_aiohttp.py new file mode 100644 index 0000000..f87888a --- /dev/null +++ b/benchmarks/bm_gunicorn/data/serve_aiohttp.py @@ -0,0 +1,12 @@ +from aiohttp import web + +async def hello(request): + return web.Response(text="Hello, world") + +async def main(): + app = web.Application() + app.add_routes([web.get('/', hello)]) + return app + +if __name__ == "__main__": + web.run_app(main()) diff --git a/benchmarks/bm_gunicorn/run_benchmark.py b/benchmarks/bm_gunicorn/run_benchmark.py index aa39d5f..9e09f8c 100644 --- a/benchmarks/bm_gunicorn/run_benchmark.py +++ b/benchmarks/bm_gunicorn/run_benchmark.py @@ -1,41 +1,39 @@ -import json -import os +import os.path import requests -import subprocess import sys -import threading -import time -from djangocms import waitUntilUp - -if __name__ == "__main__": - exe = sys.executable - - times = [] - - p = subprocess.Popen([os.path.join(os.path.dirname(exe), "gunicorn"), "gunicorn_serve:main", "--bind", "127.0.0.1:8000", "-w", "1", "--worker-class", "aiohttp.GunicornWebWorker"], stdout=open("/dev/null", "w"), stderr=subprocess.STDOUT, cwd=os.path.join(os.path.dirname(__file__), "../data")) - try: - waitUntilUp(("127.0.0.1", 8000)) - - n = 3000 - if len(sys.argv) > 1: - n = int(sys.argv[1]) - - start = time.time() - for i in range(n): - times.append(time.time()) - if i % 100 == 0: - print(i, time.time() - start) +import pyperf +import netutils + + +DATADIR = os.path.join( + os.path.dirname(__file__), + "data", +) +GUNICORN = os.path.join( + os.path.dirname(sys.executable), + "gunicorn", +) +ADDR = "127.0.0.1:8000" +ARGV = [ + GUNICORN, "serve_aiohttp:main", + "--bind", ADDR, + "-w", "1", + "--worker-class", "aiohttp.GunicornWebWorker", +] + + +def bench_gunicorn(loops=3000): + loops = iter(range(loops)) + with netutils.serving(ARGV, DATADIR, ADDR): + t0 = pyperf.perf_counter() + for _ in loops: requests.get("http://localhost:8000/blog/").text - times.append(time.time()) - elapsed = time.time() - start - print("%.2fs (%.3freq/s)" % (elapsed, n / elapsed)) + return pyperf.perf_counter() - t0 - assert p.poll() is None, p.poll() - finally: - p.terminate() - p.wait() - - if len(sys.argv) > 2: - json.dump(times, open(sys.argv[2], 'w')) +if __name__ == "__main__": + runner = pyperf.Runner() + runner.metadata['description'] = "Test the performance of gunicorn" + #runner.bench_func("gunicorn", bench_gunicorn) + runner.bench_time_func("gunicorn", bench_gunicorn) diff --git a/data/reddit_comments.json b/benchmarks/bm_json/data/reddit_comments.json similarity index 100% rename from data/reddit_comments.json rename to benchmarks/bm_json/data/reddit_comments.json diff --git a/benchmarks/bm_json/run_benchmark.py b/benchmarks/bm_json/run_benchmark.py index c762cd9..583348d 100644 --- a/benchmarks/bm_json/run_benchmark.py +++ b/benchmarks/bm_json/run_benchmark.py @@ -1,31 +1,33 @@ import json -import os -import sys -import time +import os.path -if __name__ == "__main__": - exe = sys.executable - - times = [] +import pyperf - with open(os.path.join(os.path.dirname(__file__), "../data/reddit_comments.json")) as f: - s = f.read() - data = s.split('\n') +DATADIR = os.path.join( + os.path.dirname(__file__), + "data", +) +TARGET = os.path.join(DATADIR, "reddit_comments.json") - n = 400 - if len(sys.argv) > 1: - n = int(sys.argv[1]) - times = [] +def bench_json(loops=400): + with open(TARGET) as f: + s = f.read() + data = s.split('\n') - for i in range(n): - times.append(time.time()) + json_loads = json.loads + loops = iter(range(loops)) + t0 = pyperf.perf_counter() + for _ in loops: for s in data: if not s: continue - json.loads(s) - times.append(time.time()) + json_loads(s) + return pyperf.perf_counter() - t0 - if len(sys.argv) > 2: - json.dump(times, open(sys.argv[2], 'w')) + +if __name__ == "__main__": + runner = pyperf.Runner() + runner.metadata['description'] = "Test the performance of json" + runner.bench_time_func("json", bench_json) diff --git a/data/mypy_target.py b/benchmarks/bm_mypy/data/mypy_target.py similarity index 100% rename from data/mypy_target.py rename to benchmarks/bm_mypy/data/mypy_target.py diff --git a/benchmarks/bm_mypy/run_benchmark.py b/benchmarks/bm_mypy/run_benchmark.py index b704e8c..710f810 100644 --- a/benchmarks/bm_mypy/run_benchmark.py +++ b/benchmarks/bm_mypy/run_benchmark.py @@ -1,7 +1,33 @@ -import json -import os -import sys -import time +import os.path + +import pyperf +from mypy.main import main + + +DATADIR = os.path.join( + os.path.dirname(__file__), + "data", +) +TARGET = os.path.join(DATADIR, "mypy_target.py") + + +def bench_mypy(devnull): + try: + main(None, devnull, devnull, [TARGET]) + except SystemExit: + pass + + +#def bench_mypy(loops, devnull): +# range_it = range(loops) +# t0 = pyperf.perf_counter() +# for _ in range_it: +# try: +# main(None, devnull, devnull, [TARGET]) +# except SystemExit: +# pass +# return pyperf.perf_counter() - t0 + """ I tested it, and it looks like we get the same performance conclusions @@ -10,24 +36,11 @@ So for convenience run on a single file multiple times. """ + if __name__ == "__main__": - from mypy.main import main - - n = 20 - if len(sys.argv) > 1: - n = int(sys.argv[1]) - target = os.path.join(os.path.dirname(__file__), "../data/mypy_target.py") - - times = [] - devnull = open("/dev/null", "w") - for i in range(n): - times.append(time.time()) - print(i) - try: - main(None, devnull, devnull, [target]) - except SystemExit: - pass - times.append(time.time()) - - if len(sys.argv) > 2: - json.dump(times, open(sys.argv[2], 'w')) + runner = pyperf.Runner() + runner.metadata['description'] = "Test the performance of mypy types" + with open(os.devnull, "w") as devnull: + # XXX 20 loops (by default)? + runner.bench_func("mypy", bench_mypy, devnull) +# runner.bench_time_func("mypy", bench_mypy, devnull) diff --git a/data/pycparser_target/README b/benchmarks/bm_pycparser/data/pycparser_target/README similarity index 100% rename from data/pycparser_target/README rename to benchmarks/bm_pycparser/data/pycparser_target/README diff --git a/data/pycparser_target/redis.c.ppout b/benchmarks/bm_pycparser/data/pycparser_target/redis.c.ppout similarity index 100% rename from data/pycparser_target/redis.c.ppout rename to benchmarks/bm_pycparser/data/pycparser_target/redis.c.ppout diff --git a/data/pycparser_target/sqlite-btree.c.ppout b/benchmarks/bm_pycparser/data/pycparser_target/sqlite-btree.c.ppout similarity index 100% rename from data/pycparser_target/sqlite-btree.c.ppout rename to benchmarks/bm_pycparser/data/pycparser_target/sqlite-btree.c.ppout diff --git a/data/pycparser_target/tccgen.c.ppout b/benchmarks/bm_pycparser/data/pycparser_target/tccgen.c.ppout similarity index 100% rename from data/pycparser_target/tccgen.c.ppout rename to benchmarks/bm_pycparser/data/pycparser_target/tccgen.c.ppout diff --git a/benchmarks/bm_pycparser/run_benchmark.py b/benchmarks/bm_pycparser/run_benchmark.py index 9a01cf4..3559440 100644 --- a/benchmarks/bm_pycparser/run_benchmark.py +++ b/benchmarks/bm_pycparser/run_benchmark.py @@ -1,37 +1,42 @@ -import json import os -import sys -import time +import os.path +import pyperf from pycparser import c_parser, c_ast + +DATADIR = os.path.join( + os.path.dirname(__file__), + "data", +) +TARGET = os.path.join(DATADIR, "pycparser_target") + + def parse_files(files): for code in files: parser = c_parser.CParser() ast = parser.parse(code, '') assert isinstance(ast, c_ast.FileAST) -if __name__ == "__main__": - n = 20 - if len(sys.argv) > 1: - n = int(sys.argv[1]) +def bench_pycparser(loops=20): files = [] - directory = os.path.abspath(__file__ + "/../../data/pycparser_target") - for filename in os.listdir(directory): - filename = os.path.join(directory, filename) + for filename in os.listdir(TARGET): + filename = os.path.join(TARGET, filename) if not filename.endswith(".ppout"): continue with open(filename) as f: files.append(f.read()) - times = [] - for i in range(n): - times.append(time.time()) - - parse_files(files) + _parse = parse_files + loops = iter(range(loops)) + t0 = pyperf.perf_counter() + for _ in loops: + _parse(files) + return pyperf.perf_counter() - t0 - times.append(time.time()) - if len(sys.argv) > 2: - json.dump(times, open(sys.argv[2], 'w')) +if __name__ == "__main__": + runner = pyperf.Runner() + runner.metadata['description'] = "Test the performance of pycparser" + runner.bench_time_func("pycparser", bench_pycparser) diff --git a/data/pylint_target/__init__.py b/benchmarks/bm_pylint/data/pylint_target/__init__.py similarity index 100% rename from data/pylint_target/__init__.py rename to benchmarks/bm_pylint/data/pylint_target/__init__.py diff --git a/data/pylint_target/dist.py b/benchmarks/bm_pylint/data/pylint_target/dist.py similarity index 100% rename from data/pylint_target/dist.py rename to benchmarks/bm_pylint/data/pylint_target/dist.py diff --git a/benchmarks/bm_pylint/run_benchmark.py b/benchmarks/bm_pylint/run_benchmark.py index 6736b88..655591f 100644 --- a/benchmarks/bm_pylint/run_benchmark.py +++ b/benchmarks/bm_pylint/run_benchmark.py @@ -1,12 +1,17 @@ -import json -import os -import subprocess -import sys -import time +import os.path -from pylint import epylint as lint +import pyperf +#from pylint import epylint as lint from pylint.lint import Run + +DATADIR = os.path.join( + os.path.dirname(__file__), + "data", +) +TARGET = os.path.join(DATADIR, "pylint_target", "dist.py") + + """ pylint benchmark @@ -14,24 +19,22 @@ benchmark includes that """ -if __name__ == "__main__": - def noop(*args, **kw): - pass +def bench_pylint(loops=10): class NullReporter: path_strip_prefix = "/" - def __getattr__(self, attr): - return noop - - n = 10 - if len(sys.argv) > 1: - n = int(sys.argv[1]) - - times = [] - for i in range(n): - times.append(time.time()) - print(i) - Run([os.path.join(os.path.dirname(__file__), "../data/pylint_target/dist.py")], exit=False, reporter=NullReporter()) - times.append(time.time()) - - if len(sys.argv) > 2: - json.dump(times, open(sys.argv[2], 'w')) + def __getattr__(self, attr, _noop=(lambda *a, **k: None)): + return _noop + reporter = NullReporter() + _run = Run + loops = iter(range(loops)) + + t0 = pyperf.perf_counter() + for _ in loops: + _run([TARGET], exit=False, reporter=reporter) + return pyperf.perf_counter() - t0 + + +if __name__ == "__main__": + runner = pyperf.Runner() + runner.metadata['description'] = "Test the performance of pylint" + runner.bench_time_func("pylint", bench_pylint) diff --git a/benchmarks/bm_pytorch_alexnet_inference/requirements.txt b/benchmarks/bm_pytorch_alexnet_inference/requirements.txt index d3cba68..49338ed 100644 --- a/benchmarks/bm_pytorch_alexnet_inference/requirements.txt +++ b/benchmarks/bm_pytorch_alexnet_inference/requirements.txt @@ -2,3 +2,4 @@ future==0.18.2 numpy==1.19.0 Pillow==8.0.0 torch==1.5.1 +torchvision==0.6.1 diff --git a/benchmarks/bm_pytorch_alexnet_inference/run_benchmark.py b/benchmarks/bm_pytorch_alexnet_inference/run_benchmark.py index 0ccf648..3f047d3 100644 --- a/benchmarks/bm_pytorch_alexnet_inference/run_benchmark.py +++ b/benchmarks/bm_pytorch_alexnet_inference/run_benchmark.py @@ -1,21 +1,33 @@ -import json -import time -import torch -import urllib +import os +import os.path import sys +import urllib.request -if __name__ == "__main__": - start = time.time() +import pyperf +from PIL import Image +import torch +from torchvision import transforms + + +DATADIR = os.path.join( + os.path.dirname(__file__), + "data", +) +if not os.path.exists(DATADIR): + os.mkdir(DATADIR) + +URL = "https://github.com/pytorch/hub/raw/master/images/dog.jpg" +FILENAME = os.path.join(DATADIR, "dog.jpg") + + +def bench_pytorch(loops=1000): + #start = time.time() model = torch.hub.load('pytorch/vision:v0.6.0', 'alexnet', pretrained=True) # assert time.time() - start < 3, "looks like we just did the first-time download, run this benchmark again to get a clean run" model.eval() - url, filename = ("https://github.com/pytorch/hub/raw/master/images/dog.jpg", "dog.jpg") - urllib.request.urlretrieve(url, filename) - - from PIL import Image - from torchvision import transforms - input_image = Image.open(filename) + urllib.request.urlretrieve(URL, FILENAME) + input_image = Image.open(FILENAME) preprocess = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), @@ -25,19 +37,16 @@ input_tensor = preprocess(input_image) input_batch = input_tensor.unsqueeze(0) # create a mini-batch as expected by the model - n = 1000 - if len(sys.argv) > 1: - n = int(sys.argv[1]) + loops = iter(range(loops)) with torch.no_grad(): - times = [] - for i in range(n): - times.append(time.time()) - if i % 10 == 0: - print(i) + t0 = pyperf.perf_counter() + for _ in loops: output = model(input_batch) - times.append(time.time()) - print((len(times) - 1) / (times[-1] - times[0]) , "/s") + return pyperf.perf_counter() - t0 - if len(sys.argv) > 2: - json.dump(times, open(sys.argv[2], 'w')) + +if __name__ == "__main__": + runner = pyperf.Runner() + runner.metadata['description'] = "Test the performance of pytorch" + runner.bench_time_func("pytorch", bench_pytorch) diff --git a/data/Makefile b/benchmarks/bm_thrift/data/Makefile similarity index 100% rename from data/Makefile rename to benchmarks/bm_thrift/data/Makefile diff --git a/data/addressbook.thrift b/benchmarks/bm_thrift/data/addressbook.thrift similarity index 100% rename from data/addressbook.thrift rename to benchmarks/bm_thrift/data/addressbook.thrift diff --git a/data/thrift/__init__.py b/benchmarks/bm_thrift/data/thrift/__init__.py similarity index 100% rename from data/thrift/__init__.py rename to benchmarks/bm_thrift/data/thrift/__init__.py diff --git a/data/thrift/addressbook/__init__.py b/benchmarks/bm_thrift/data/thrift/addressbook/__init__.py similarity index 100% rename from data/thrift/addressbook/__init__.py rename to benchmarks/bm_thrift/data/thrift/addressbook/__init__.py diff --git a/data/thrift/addressbook/constants.py b/benchmarks/bm_thrift/data/thrift/addressbook/constants.py similarity index 100% rename from data/thrift/addressbook/constants.py rename to benchmarks/bm_thrift/data/thrift/addressbook/constants.py diff --git a/data/thrift/addressbook/ttypes.py b/benchmarks/bm_thrift/data/thrift/addressbook/ttypes.py similarity index 100% rename from data/thrift/addressbook/ttypes.py rename to benchmarks/bm_thrift/data/thrift/addressbook/ttypes.py diff --git a/benchmarks/bm_thrift/run_benchmark.py b/benchmarks/bm_thrift/run_benchmark.py index 68d5a69..64ee538 100644 --- a/benchmarks/bm_thrift/run_benchmark.py +++ b/benchmarks/bm_thrift/run_benchmark.py @@ -1,17 +1,23 @@ # Adapted from https://raw.githubusercontent.com/Thriftpy/thriftpy2/master/benchmark/benchmark_apache_thrift_struct.py -import json -import time +import os.path +import sys +import pyperf from thrift.TSerialization import serialize, deserialize from thrift.protocol.TBinaryProtocol import ( TBinaryProtocolFactory, TBinaryProtocolAcceleratedFactory ) -import os -import sys -sys.path.append(os.path.join(os.path.dirname(__file__), "../data/thrift")) + +DATADIR = os.path.join( + os.path.dirname(__file__), + "data", +) +TARGET = os.path.join(DATADIR, "thrift") + +sys.path.insert(0, TARGET) from addressbook import ttypes @@ -32,27 +38,26 @@ def make_addressbook(): return ab -def main(): +def bench_thrift(loops=1000): # proto_factory = TBinaryProtocolFactory() proto_factory = TBinaryProtocolAcceleratedFactory() - - n = 1000 - if len(sys.argv) > 1: - n = int(sys.argv[1]) - - times = [] - - for i in range(n): - times.append(time.time()) + _serialize = serialize + _deserialize = deserialize + _AddressBook = ttypes.AddressBook + _mkaddr = make_addressbook + loops = iter(range(loops)) + + t0 = pyperf.perf_counter() + for _ in loops: for j in range(100): - ab = make_addressbook() - encoded = serialize(ab, proto_factory) - ab2 = ttypes.AddressBook() - deserialize(ab2, encoded, proto_factory) - times.append(time.time()) + ab = _mkaddr() + encoded = _serialize(ab, proto_factory) + ab2 = _AddressBook() + _deserialize(ab2, encoded, proto_factory) + return pyperf.perf_counter() - t0 - if len(sys.argv) > 2: - json.dump(times, open(sys.argv[2], 'w')) if __name__ == "__main__": - main() + runner = pyperf.Runner() + runner.metadata['description'] = "Test the performance of thrift" + runner.bench_time_func("thrift", bench_thrift) diff --git a/run_all.sh b/run_all.sh index 68bd24e..ebd8e68 100755 --- a/run_all.sh +++ b/run_all.sh @@ -10,12 +10,8 @@ BINARY=$1 set -u set -x -mkdir -p results - -ENV=/tmp/macrobenchmark_env -for bench in flaskblogging djangocms mypy_bench pylint_bench pycparser_bench pytorch_alexnet_inference gunicorn aiohttp thrift_bench gevent_bench_hub; do - rm -rf $ENV - $BINARY -m venv $ENV - $ENV/bin/pip install -r $(dirname $0)/benchmarks/${bench}_requirements.txt - /usr/bin/time --verbose --output=results/${bench}.out $ENV/bin/python $(dirname $0)/benchmarks/${bench}.py -done +#python3 -m pip install pyperformance +python3 -m pyperformance run \ + --manifest $(dirname $0)/benchmarks/MANIFEST \ + --python $BINARY \ + --output results.json diff --git a/run_mypy.sh b/run_mypy.sh index 363fc7c..e8b2dbf 100644 --- a/run_mypy.sh +++ b/run_mypy.sh @@ -18,19 +18,29 @@ set -x ENV=/tmp/macrobenchmark_env rm -rf $ENV -virtualenv -p $BINARY $ENV +python3 -m venv -p $BINARY $ENV +PYTHON=$ENV/bin/python + +#$PYTHON -m pip install pyperformance rm -rf /tmp/mypy git clone --depth 1 --branch v0.790 https://github.com/python/mypy/ /tmp/mypy -cd /tmp/mypy - -$ENV/bin/pip install -r mypy-requirements.txt -$ENV/bin/pip install --upgrade setuptools +pushd /tmp/mypy +$PYTHON -m pip install -r mypy-requirements.txt +$PYTHON -m pip install --upgrade setuptools git submodule update --init mypy/typeshed -$ENV/bin/python setup.py --use-mypyc install - -cd - -time $ENV/bin/python benchmarks/mypy_bench.py 50 -time $ENV/bin/python benchmarks/mypy_bench.py 50 -time $ENV/bin/python benchmarks/mypy_bench.py 50 - +$PYTHON setup.py --use-mypyc install +popd + +OPTS=" \ + --manifest $(dirname $0)/benchmarks/MANIFEST \ + --venv $ENV \ + --benchmarks mypy \ +" +# XXX Run 50 loops each instead of the default 20. +$PYTHON -m pyperformance run $OPTS \ + #--output results.json +$PYTHON -m pyperformance run $OPTS \ + #--append results.json +$PYTHON -m pyperformance run $OPTS \ + #--append results.json From 7e73fe05ed014215f4ba2415918c40ce9799d724 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 22 Jul 2021 09:30:41 -0600 Subject: [PATCH 08/32] Introduce metadata for "pre" and "post" scripts. --- benchmarks/bm_djangocms/pyproject.toml | 5 +++++ benchmarks/bm_djangocms/run_benchmark.py | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/benchmarks/bm_djangocms/pyproject.toml b/benchmarks/bm_djangocms/pyproject.toml index 0297f4f..1ac07d9 100644 --- a/benchmarks/bm_djangocms/pyproject.toml +++ b/benchmarks/bm_djangocms/pyproject.toml @@ -22,3 +22,8 @@ dynamic = [ [tool.pyperformance] # "name" is set automatically. metabase = ".." + +prescript = "run_benchmark.py" +pre_extra_opts = ["--pre"] +postscript = "run_benchmark.py" +post_extra_opts = ["--post"] diff --git a/benchmarks/bm_djangocms/run_benchmark.py b/benchmarks/bm_djangocms/run_benchmark.py index 71633c1..ae713bd 100644 --- a/benchmarks/bm_djangocms/run_benchmark.py +++ b/benchmarks/bm_djangocms/run_benchmark.py @@ -146,6 +146,16 @@ def bench_djangocms(sitedir, loops=INNER_LOOPS): group = runner.argparser.add_mutually_exclusive_group() group.add_argument("--serve") group.add_argument("datadir", nargs="?") + if True: + runner.argparser.add_argument("--pre", action="store_true") + runner.argparser.add_argument("--post", action="store_true") + args = runner.argparser.parse_args() + if args.pre: + print('### running as prescript ###') + sys.exit() + if args.post: + print('### running as postscript ###') + sys.exit() args = runner.argparser.parse_args() if args.serve is not None: From 50bae0906bfeac6aded1a1b8185b8602ce698576 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 22 Jul 2021 11:11:38 -0600 Subject: [PATCH 09/32] Drop "pre" and "post" script support. --- benchmarks/bm_djangocms/pyproject.toml | 5 ----- benchmarks/bm_djangocms/run_benchmark.py | 10 ---------- 2 files changed, 15 deletions(-) diff --git a/benchmarks/bm_djangocms/pyproject.toml b/benchmarks/bm_djangocms/pyproject.toml index 1ac07d9..0297f4f 100644 --- a/benchmarks/bm_djangocms/pyproject.toml +++ b/benchmarks/bm_djangocms/pyproject.toml @@ -22,8 +22,3 @@ dynamic = [ [tool.pyperformance] # "name" is set automatically. metabase = ".." - -prescript = "run_benchmark.py" -pre_extra_opts = ["--pre"] -postscript = "run_benchmark.py" -post_extra_opts = ["--post"] diff --git a/benchmarks/bm_djangocms/run_benchmark.py b/benchmarks/bm_djangocms/run_benchmark.py index ae713bd..71633c1 100644 --- a/benchmarks/bm_djangocms/run_benchmark.py +++ b/benchmarks/bm_djangocms/run_benchmark.py @@ -146,16 +146,6 @@ def bench_djangocms(sitedir, loops=INNER_LOOPS): group = runner.argparser.add_mutually_exclusive_group() group.add_argument("--serve") group.add_argument("datadir", nargs="?") - if True: - runner.argparser.add_argument("--pre", action="store_true") - runner.argparser.add_argument("--post", action="store_true") - args = runner.argparser.parse_args() - if args.pre: - print('### running as prescript ###') - sys.exit() - if args.post: - print('### running as postscript ###') - sys.exit() args = runner.argparser.parse_args() if args.serve is not None: From 4c85a42a3c4dfda5e757d60fbad40f94c77e4447 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 22 Jul 2021 11:28:41 -0600 Subject: [PATCH 10/32] Use symlinks instead of the "libsdir" metadata. --- benchmarks/base.toml | 1 - benchmarks/bm_aiohttp/netutils.py | 1 + benchmarks/bm_djangocms/netutils.py | 1 + benchmarks/bm_flaskblogging/netutils.py | 1 + benchmarks/bm_gunicorn/netutils.py | 1 + 5 files changed, 4 insertions(+), 1 deletion(-) create mode 120000 benchmarks/bm_aiohttp/netutils.py create mode 120000 benchmarks/bm_djangocms/netutils.py create mode 120000 benchmarks/bm_flaskblogging/netutils.py create mode 120000 benchmarks/bm_gunicorn/netutils.py diff --git a/benchmarks/base.toml b/benchmarks/base.toml index 61b4a89..e575389 100644 --- a/benchmarks/base.toml +++ b/benchmarks/base.toml @@ -20,4 +20,3 @@ dynamic = [ metabase = "" #tags = [] #extra_opts = "" -libsdir = ".libs" diff --git a/benchmarks/bm_aiohttp/netutils.py b/benchmarks/bm_aiohttp/netutils.py new file mode 120000 index 0000000..3afa43f --- /dev/null +++ b/benchmarks/bm_aiohttp/netutils.py @@ -0,0 +1 @@ +../.libs/netutils.py \ No newline at end of file diff --git a/benchmarks/bm_djangocms/netutils.py b/benchmarks/bm_djangocms/netutils.py new file mode 120000 index 0000000..3afa43f --- /dev/null +++ b/benchmarks/bm_djangocms/netutils.py @@ -0,0 +1 @@ +../.libs/netutils.py \ No newline at end of file diff --git a/benchmarks/bm_flaskblogging/netutils.py b/benchmarks/bm_flaskblogging/netutils.py new file mode 120000 index 0000000..3afa43f --- /dev/null +++ b/benchmarks/bm_flaskblogging/netutils.py @@ -0,0 +1 @@ +../.libs/netutils.py \ No newline at end of file diff --git a/benchmarks/bm_gunicorn/netutils.py b/benchmarks/bm_gunicorn/netutils.py new file mode 120000 index 0000000..3afa43f --- /dev/null +++ b/benchmarks/bm_gunicorn/netutils.py @@ -0,0 +1 @@ +../.libs/netutils.py \ No newline at end of file From b131693dd8efa5a0ad45a99782f69480e4058aa9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 22 Jul 2021 13:01:10 -0600 Subject: [PATCH 11/32] Don't rely on an internal Runner attr. --- benchmarks/bm_djangocms/run_benchmark.py | 27 +++++++++++++++++------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/benchmarks/bm_djangocms/run_benchmark.py b/benchmarks/bm_djangocms/run_benchmark.py index 71633c1..bbb66dd 100644 --- a/benchmarks/bm_djangocms/run_benchmark.py +++ b/benchmarks/bm_djangocms/run_benchmark.py @@ -124,6 +124,23 @@ def bench_djangocms(sitedir, loops=INNER_LOOPS): return pyperf.perf_counter() - t0 +# We can't set "add_cmdline_args" on pyperf.Runner +# once we've created one. We work around this with a subclass. + +class _Runner(pyperf.Runner): + datadir = None + + def __init__(self): + def add_worker_args(cmd, _): + assert self.datadir + cmd.extend([ + '--serve', self.datadir, + ]) + super().__init__( + add_cmdline_args=add_worker_args, + ) + + if __name__ == "__main__": """ Usage: @@ -138,7 +155,7 @@ def bench_djangocms(sitedir, loops=INNER_LOOPS): The second and third forms are useful if you want to benchmark the initial migration phase separately from the second serving phase. """ - runner = pyperf.Runner() + runner = _Runner() runner.metadata['description'] = "Test the performance of a Django data migration" # Parse the CLI args. @@ -186,15 +203,9 @@ def bench_djangocms(sitedir, loops=INNER_LOOPS): # Then run the benchmark. if args.serve: - def add_worker_args(cmd, _, _datadir=datadir): - cmd.extend([ - '--serve', _datadir, - ]) - # XXX This is an internal attr but we don't have any other good option. - runner._add_cmdline_args = add_worker_args + runner.datadir = datadir def time_func(loops, *args): return bench_djangocms(*args, loops=loops) - runner.bench_time_func("djangocms", time_func, sitedir, inner_loops=INNER_LOOPS) From e3a397058ff8cc60ae6d423b3145520388a81ec9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 23 Jul 2021 14:04:14 -0600 Subject: [PATCH 12/32] Be clear about what is getting benchmarked. --- benchmarks/bm_aiohttp/run_benchmark.py | 30 ++-- benchmarks/bm_djangocms/run_benchmark.py | 30 +++- benchmarks/bm_flaskblogging/run_benchmark.py | 29 +++- benchmarks/bm_gevent_hub/run_benchmark.py | 134 ++++++++++-------- benchmarks/bm_gunicorn/run_benchmark.py | 26 +++- benchmarks/bm_json/run_benchmark.py | 30 +++- benchmarks/bm_mypy/run_benchmark.py | 48 +++---- benchmarks/bm_pycparser/run_benchmark.py | 55 ++++--- benchmarks/bm_pylint/run_benchmark.py | 30 ++-- .../run_benchmark.py | 30 +++- benchmarks/bm_thrift/run_benchmark.py | 60 ++++---- 11 files changed, 324 insertions(+), 178 deletions(-) diff --git a/benchmarks/bm_aiohttp/run_benchmark.py b/benchmarks/bm_aiohttp/run_benchmark.py index 5021425..80e1eec 100644 --- a/benchmarks/bm_aiohttp/run_benchmark.py +++ b/benchmarks/bm_aiohttp/run_benchmark.py @@ -13,17 +13,31 @@ ARGV = [sys.executable, "serve.py"] -def bench_aiohttp(loops=3000): - loops = iter(range(loops)) +def bench_aiohttp_requests(loops=3000): + """Measure N HTTP requests to a local server. + + Note that the server is freshly started here. + + Only the time for requests is measured here. The following are not: + + * preparing the site the server will serve + * starting the server + * stopping the server + + Hence this should be used with bench_time_func() + insted of bench_func(). + """ + elapsed = 0 with netutils.serving(ARGV, DATADIR, "127.0.0.1:8080"): - t0 = pyperf.perf_counter() - for _ in loops: - requests.get("http://localhost:8080/blog/").text - return pyperf.perf_counter() - t0 + requests_get = requests.get + for _ in range(loops): + t0 = pyperf.perf_counter() + requests_get("http://localhost:8080/blog/").text + elapsed += pyperf.perf_counter() - t0 + return elapsed if __name__ == "__main__": runner = pyperf.Runner() runner.metadata['description'] = "Test the performance of aiohttp" - #runner.bench_func("aiohttp", bench_aiohttp) - runner.bench_time_func("aiohttp", bench_aiohttp) + runner.bench_time_func("aiohttp", bench_aiohttp_requests) diff --git a/benchmarks/bm_djangocms/run_benchmark.py b/benchmarks/bm_djangocms/run_benchmark.py index bbb66dd..9a93fbd 100644 --- a/benchmarks/bm_djangocms/run_benchmark.py +++ b/benchmarks/bm_djangocms/run_benchmark.py @@ -113,15 +113,31 @@ def _ensure_datadir(datadir, preserve=True): shutil.rmtree(datadir, ignore_errors=True) -def bench_djangocms(sitedir, loops=INNER_LOOPS): - requests_get = requests.get - loops = iter(range(loops)) +############################# +# benchmarks +def bench_djangocms_requests(sitedir, loops=INNER_LOOPS): + """Measure N HTTP requests to a local server. + + Note that the server is freshly started here. + + Only the time for requests is measured here. The following are not: + + * preparing the site the server will serve + * starting the server + * stopping the server + + Hence this should be used with bench_time_func() + insted of bench_func(). + """ + elapsed = 0 with netutils.serving(ARGV_SERVE, sitedir, "127.0.0.1:8000"): - t0 = pyperf.perf_counter() - for _ in loops: + requests_get = requests.get + for _ in range(loops): + t0 = pyperf.perf_counter() requests_get("http://localhost:8000/").text - return pyperf.perf_counter() - t0 + elapsed += pyperf.perf_counter() - t0 + return elapsed # We can't set "add_cmdline_args" on pyperf.Runner @@ -206,6 +222,6 @@ def add_worker_args(cmd, _): runner.datadir = datadir def time_func(loops, *args): - return bench_djangocms(*args, loops=loops) + return bench_djangocms_requests(*args, loops=loops) runner.bench_time_func("djangocms", time_func, sitedir, inner_loops=INNER_LOOPS) diff --git a/benchmarks/bm_flaskblogging/run_benchmark.py b/benchmarks/bm_flaskblogging/run_benchmark.py index 8a977f1..589180d 100644 --- a/benchmarks/bm_flaskblogging/run_benchmark.py +++ b/benchmarks/bm_flaskblogging/run_benchmark.py @@ -13,17 +13,34 @@ ARGV = [sys.executable, "serve.py"] +############################# +# benchmarks + def bench_flask_requests(loops=1800): - loops = iter(range(loops)) + """Measure N HTTP requests to a local server. + + Note that the server is freshly started here. + + Only the time for requests is measured here. The following are not: + + * preparing the site the server will serve + * starting the server + * stopping the server + + Hence this should be used with bench_time_func() + insted of bench_func(). + """ + elapsed = 0 with netutils.serving(ARGV, DATADIR, "127.0.0.1:8000"): - t0 = pyperf.perf_counter() - for _ in loops: - requests.get("http://localhost:8000/blog/").text - return pyperf.perf_counter() - t0 + requests_get = requests.get + for _ in range(loops): + t0 = pyperf.perf_counter() + requests_get("http://localhost:8000/blog/").text + elapsed += pyperf.perf_counter() - t0 + return elapsed if __name__ == "__main__": runner = pyperf.Runner() runner.metadata['description'] = "Test the performance of flask" - #runner.bench_func("flaskblogging", bench_flask) runner.bench_time_func("flaskblogging", bench_flask_requests) diff --git a/benchmarks/bm_gevent_hub/run_benchmark.py b/benchmarks/bm_gevent_hub/run_benchmark.py index d74796d..f928014 100644 --- a/benchmarks/bm_gevent_hub/run_benchmark.py +++ b/benchmarks/bm_gevent_hub/run_benchmark.py @@ -5,9 +5,7 @@ Taken from https://github.com/gevent/gevent/blob/master/benchmarks/bench_hub.py Modified to remove perf and not need any command line arguments """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +import contextlib import pyperf import gevent @@ -16,6 +14,39 @@ from greenlet import getcurrent +@contextlib.contextmanager +def active_hub(hub=None): + if hub is None: + hub = gevent.get_hub() + try: + yield hub + finally: + # Destroy the loop so we don't keep building up state (e.g. callbacks). + hub.destroy(True) + + +class SwitchingParent(gevent.hub.Hub): + """A gevent hub greenlet that switches back and forth with its child.""" + + def __init__(self, nswitches): + super().__init__(None, None) + self.nswitches = nswitches + self.child = greenlet(self._run_child, self) + + def _run_child(self): + # Back to the hub, which in turn goes + # back to the main greenlet + switch = getcurrent().parent.switch + for _ in range(self.nswitches): + switch() + + def run(self): + # Return to the main greenlet. + switch = self.parent.switch + for _ in range(self.nswitches): + switch() + + class NoopWatcher: def start(self, cb, obj): # Immediately switch back to the waiter, mark as ready @@ -38,84 +69,75 @@ def rawlink(self, cb): cb(self) -def get_switching_greenlets(nswitches=1000): - class Parent(type(gevent.get_hub())): - def run(self, *, _loops=nswitches): - switch = self.parent.switch - for _ in range(_loops): - switch() - - def child(*, _loops=nswitches): - switch = getcurrent().parent.switch - # Back to the hub, which in turn goes - # back to the main greenlet - for _ in range(_loops): - switch() - - hub = Parent(None, None) - child_greenlet = greenlet(child, hub) - return hub, child_greenlet - +############################# +# benchmarks def bench_switch(loops=1000): - _, child = get_switching_greenlets(loops) - child_switch = child.switch - loops = iter(range(loops)) + """Measure switching between a greenlet and the gevent hub N^2 times.""" + hub = SwitchingParent(loops) + child = hub.child - t0 = pyperf.perf_counter() - for _ in loops: - child_switch() - return pyperf.perf_counter() - t0 + with active_hub(hub): + elapsed = 0 + child_switch = child.switch + for _ in range(loops): + t0 = pyperf.perf_counter() + child_switch() + elapsed += pyperf.perf_counter() - t0 + return elapsed def bench_wait_ready(loops=1000): + """Measure waiting for a "noop" watcher to become ready N times.""" watcher = NoopWatcher() - hub = gevent.get_hub() - hub_wait = hub.wait - loops = iter(range(loops)) - t0 = pyperf.perf_counter() - for _ in loops: - hub_wait(watcher) - return pyperf.perf_counter() - t0 + with active_hub() as hub: + elapsed = 0 + hub_wait = hub.wait + for _ in range(loops): + t0 = pyperf.perf_counter() + hub_wait(watcher) + elapsed += pyperf.perf_counter() - t0 + return elapsed def bench_cancel_wait(loops=1000): + """Measure canceling N watchers. + + Note that it is the same watcher N times and that it is a fake + that pretends to already be started. + """ watcher = ActiveWatcher() - hub = gevent.get_hub() - hub_cancel_wait = hub.cancel_wait - hub_loop = hub.loop - loops = iter(range(loops)) - t0 = pyperf.perf_counter() - for _ in loops: - # Schedule all the callbacks. - hub_cancel_wait(watcher, None, True) - # Run them! - # XXX Start timing here? - for cb in hub_loop._callbacks: - if cb.callback: - cb.callback(*cb.args) - cb.stop() # so the real loop won't do it - elapsed = pyperf.perf_counter() - t0 + with active_hub() as hub: + t0 = pyperf.perf_counter() + + # Cancel the fake wait requests. + for _ in range(loops): + # Schedule all the callbacks. + hub.cancel_wait(watcher, None, True) - # Destroy the loop so we don't keep building these functions up. - hub.destroy(True) + # Wait for all the watchers to be closed. + # TODO Start timing here? + for cb in hub.loop._callbacks: + if cb.callback: + cb.callback(*cb.args) + cb.stop() # so the real loop won't do it - return elapsed + return pyperf.perf_counter() - t0 def bench_wait_func_ready(loops=1000): + """Measure waiting for N noop watch targets to become ready.""" watched_objects = [NoopWatchTarget() for _ in range(loops)] - wait = gevent.hub.wait t0 = pyperf.perf_counter() - wait(watched_objects) + gevent.hub.wait(watched_objects) return pyperf.perf_counter() - t0 BENCHMARKS = { - "gevent_hub": bench_switch, # XXX Release 10,000 times? + "gevent_hub": bench_switch, "gevent_wait_func_ready": bench_wait_func_ready, "gevent_wait_ready": bench_wait_ready, "gevent_cancel_wait": bench_cancel_wait, diff --git a/benchmarks/bm_gunicorn/run_benchmark.py b/benchmarks/bm_gunicorn/run_benchmark.py index 9e09f8c..f9fae51 100644 --- a/benchmarks/bm_gunicorn/run_benchmark.py +++ b/benchmarks/bm_gunicorn/run_benchmark.py @@ -24,16 +24,30 @@ def bench_gunicorn(loops=3000): - loops = iter(range(loops)) + """Measure N HTTP requests to a local server. + + Note that the server is freshly started here. + + Only the time for requests is measured here. The following are not: + + * preparing the site the server will serve + * starting the server + * stopping the server + + Hence this should be used with bench_time_func() + insted of bench_func(). + """ + elapsed = 0 with netutils.serving(ARGV, DATADIR, ADDR): - t0 = pyperf.perf_counter() - for _ in loops: - requests.get("http://localhost:8000/blog/").text - return pyperf.perf_counter() - t0 + requests_get = requests.get + for _ in range(loops): + t0 = pyperf.perf_counter() + requests_get("http://localhost:8000/blog/").text + elapsed += pyperf.perf_counter() - t0 + return elapsed if __name__ == "__main__": runner = pyperf.Runner() runner.metadata['description'] = "Test the performance of gunicorn" - #runner.bench_func("gunicorn", bench_gunicorn) runner.bench_time_func("gunicorn", bench_gunicorn) diff --git a/benchmarks/bm_json/run_benchmark.py b/benchmarks/bm_json/run_benchmark.py index 583348d..6793f3a 100644 --- a/benchmarks/bm_json/run_benchmark.py +++ b/benchmarks/bm_json/run_benchmark.py @@ -11,23 +11,39 @@ TARGET = os.path.join(DATADIR, "reddit_comments.json") -def bench_json(loops=400): +def bench_json_loads(loops=400): + """Measure running json.loads() N times. + + The target data is nearly 1100 JSON objects, each on a single line, + from a file. The objects: + + * are all flat (no compound values) + * vary a little in number of properties, though none are big + * have a mix of values, both of type and size + + Only the json.loads() calls are measured. The following are not: + + * reading the text from the file + * looping through the lines + """ with open(TARGET) as f: s = f.read() - data = s.split('\n') +# data = s.split('\n') + data = s.splitlines() + elapsed = 0 json_loads = json.loads - loops = iter(range(loops)) - t0 = pyperf.perf_counter() - for _ in loops: + for _ in range(loops): for s in data: if not s: continue + t0 = pyperf.perf_counter() json_loads(s) - return pyperf.perf_counter() - t0 + elapsed += pyperf.perf_counter() - t0 + return elapsed if __name__ == "__main__": runner = pyperf.Runner() runner.metadata['description'] = "Test the performance of json" - runner.bench_time_func("json", bench_json) + runner.bench_time_func("json", bench_json_loads) diff --git a/benchmarks/bm_mypy/run_benchmark.py b/benchmarks/bm_mypy/run_benchmark.py index 710f810..2bd8526 100644 --- a/benchmarks/bm_mypy/run_benchmark.py +++ b/benchmarks/bm_mypy/run_benchmark.py @@ -8,39 +8,39 @@ os.path.dirname(__file__), "data", ) -TARGET = os.path.join(DATADIR, "mypy_target.py") - - -def bench_mypy(devnull): - try: - main(None, devnull, devnull, [TARGET]) - except SystemExit: - pass - - -#def bench_mypy(loops, devnull): -# range_it = range(loops) -# t0 = pyperf.perf_counter() -# for _ in range_it: -# try: -# main(None, devnull, devnull, [TARGET]) -# except SystemExit: -# pass -# return pyperf.perf_counter() - t0 - - """ I tested it, and it looks like we get the same performance conclusions when we run on the same file multiple times as if we run on a set of files once. So for convenience run on a single file multiple times. """ +TARGETS = [ + os.path.join(DATADIR, "mypy_target.py"), +] + + +def bench_mypy(devnull): + """Meansure running mypy on a file N times. + + The target file is large (over 2300 lines) with extensive use + of type hints. + + Note that mypy's main() is called directly, which means + the measurement includes the time it takes to read the file + from disk. Also, all output is discarded (sent to /dev/null). + """ + try: + main(None, devnull, devnull, TARGETS) + except SystemExit: + pass if __name__ == "__main__": runner = pyperf.Runner() runner.metadata['description'] = "Test the performance of mypy types" + runner.argparser.add_argument("loops", nargs="?", type=int, default=1) + args = runner.argparser.parse_args() + with open(os.devnull, "w") as devnull: - # XXX 20 loops (by default)? - runner.bench_func("mypy", bench_mypy, devnull) -# runner.bench_time_func("mypy", bench_mypy, devnull) + runner.bench_func("mypy", bench_mypy, devnull, + inner_loops=args.loops) diff --git a/benchmarks/bm_pycparser/run_benchmark.py b/benchmarks/bm_pycparser/run_benchmark.py index 3559440..6698625 100644 --- a/benchmarks/bm_pycparser/run_benchmark.py +++ b/benchmarks/bm_pycparser/run_benchmark.py @@ -12,28 +12,45 @@ TARGET = os.path.join(DATADIR, "pycparser_target") -def parse_files(files): - for code in files: - parser = c_parser.CParser() - ast = parser.parse(code, '') - assert isinstance(ast, c_ast.FileAST) +def _iter_files(rootdir=TARGET, *, _cache={}): + if not _cache: + files = _cache['files'] = [] + for name in os.listdir(rootdir): + if not name.endswith(".ppout"): + continue + filename = os.path.join(TARGET, name) + with open(filename) as f: + data = (filename, f.read()) + files.append(data) + yield data + else: + yield from _cache['files'] def bench_pycparser(loops=20): - files = [] - for filename in os.listdir(TARGET): - filename = os.path.join(TARGET, filename) - if not filename.endswith(".ppout"): - continue - with open(filename) as f: - files.append(f.read()) - - _parse = parse_files - loops = iter(range(loops)) - t0 = pyperf.perf_counter() - for _ in loops: - _parse(files) - return pyperf.perf_counter() - t0 + """Measure running pycparser on several large C files N times. + + The files are all relatively large, from well-known projects. + Each is already preprocessed. + + Only the CParser.parse() calls are measured. The following are not: + + * finding the target files + * reading them from disk + * creating the CParser object + """ + elapsed = 0 + parse = c_parser.CParser.parse + for _ in range(loops): + for filename, text in _iter_files(): + # We use a new parser each time because CParser objects + # aren't designed for re-use. + parser = c_parser.CParser() + t0 = pyperf.perf_counter() + ast = parse(parser, text, filename) + elapsed += pyperf.perf_counter() - t0 + assert isinstance(ast, c_ast.FileAST) + return elapsed if __name__ == "__main__": diff --git a/benchmarks/bm_pylint/run_benchmark.py b/benchmarks/bm_pylint/run_benchmark.py index 655591f..5e95041 100644 --- a/benchmarks/bm_pylint/run_benchmark.py +++ b/benchmarks/bm_pylint/run_benchmark.py @@ -9,29 +9,33 @@ os.path.dirname(__file__), "data", ) -TARGET = os.path.join(DATADIR, "pylint_target", "dist.py") +TARGETS = [ + os.path.join(DATADIR, "pylint_target", "dist.py"), +] -""" -pylint benchmark +def bench_pylint(loops=10): + """Measure running pylint on a file N times. -pylint seems to speed up considerably as it progresses, and this -benchmark includes that -""" + The target file is a relatively large, complex one copied + from distutils in the stdlib. -def bench_pylint(loops=10): + pylint seems to speed up considerably as it progresses, and this + benchmark includes that. + """ class NullReporter: path_strip_prefix = "/" def __getattr__(self, attr, _noop=(lambda *a, **k: None)): return _noop reporter = NullReporter() - _run = Run - loops = iter(range(loops)) - t0 = pyperf.perf_counter() - for _ in loops: - _run([TARGET], exit=False, reporter=reporter) - return pyperf.perf_counter() - t0 + elapsed = 0 + _run = Run + for _ in range(loops): + t0 = pyperf.perf_counter() + _run(TARGETS, exit=False, reporter=reporter) + elapsed += pyperf.perf_counter() - t0 + return elapsed if __name__ == "__main__": diff --git a/benchmarks/bm_pytorch_alexnet_inference/run_benchmark.py b/benchmarks/bm_pytorch_alexnet_inference/run_benchmark.py index 3f047d3..45e3184 100644 --- a/benchmarks/bm_pytorch_alexnet_inference/run_benchmark.py +++ b/benchmarks/bm_pytorch_alexnet_inference/run_benchmark.py @@ -16,14 +16,30 @@ if not os.path.exists(DATADIR): os.mkdir(DATADIR) +# TODO: Vendor this file (and the pytorch hub model) into the data dir, +# to avoid network access and to pin the data for consistent results. URL = "https://github.com/pytorch/hub/raw/master/images/dog.jpg" FILENAME = os.path.join(DATADIR, "dog.jpg") +############################# +# benchmarks + def bench_pytorch(loops=1000): - #start = time.time() + """Measure using pytorch to transform an image N times. + + This involves the following steps: + + * load a pre-trained model (alexnet) + * mark it for evaluation + * download an image + * prepare it to be run through the model + * turn off gradients computation + * run the image through the model + + Only that last step is measured (and repeated N times). + """ model = torch.hub.load('pytorch/vision:v0.6.0', 'alexnet', pretrained=True) - # assert time.time() - start < 3, "looks like we just did the first-time download, run this benchmark again to get a clean run" model.eval() urllib.request.urlretrieve(URL, FILENAME) @@ -37,13 +53,13 @@ def bench_pytorch(loops=1000): input_tensor = preprocess(input_image) input_batch = input_tensor.unsqueeze(0) # create a mini-batch as expected by the model - loops = iter(range(loops)) - with torch.no_grad(): - t0 = pyperf.perf_counter() - for _ in loops: + elapsed = 0 + for _ in range(loops): + t0 = pyperf.perf_counter() output = model(input_batch) - return pyperf.perf_counter() - t0 + elapsed += pyperf.perf_counter() - t0 + return elapsed if __name__ == "__main__": diff --git a/benchmarks/bm_thrift/run_benchmark.py b/benchmarks/bm_thrift/run_benchmark.py index 64ee538..ade0d89 100644 --- a/benchmarks/bm_thrift/run_benchmark.py +++ b/benchmarks/bm_thrift/run_benchmark.py @@ -15,46 +15,56 @@ os.path.dirname(__file__), "data", ) +# The target files were generated using the make file in the data dir. TARGET = os.path.join(DATADIR, "thrift") -sys.path.insert(0, TARGET) -from addressbook import ttypes +def bench_thrift(loops=1000): + """Measure using a thrift-generated library N times. -def make_addressbook(): - phone1 = ttypes.PhoneNumber() - phone1.type = ttypes.PhoneType.MOBILE - phone1.number = '555-1212' - phone2 = ttypes.PhoneNumber() - phone2.type = ttypes.PhoneType.HOME - phone2.number = '555-1234' - person = ttypes.Person() - person.name = "Alice" - person.phones = [phone1, phone2] - person.created_at = 1400000000 + The target is a simple addressbook. We measure the following: - ab = ttypes.AddressBook() - ab.people = {person.name: person} - return ab + * create an addressbook with 1 person in it + * serialize it + * deserialize it into a new addressbook + For each iteration we repeat this 100 times. + """ + sys.path.insert(0, TARGET) + from addressbook import ttypes -def bench_thrift(loops=1000): + elapsed = 0 # proto_factory = TBinaryProtocolFactory() proto_factory = TBinaryProtocolAcceleratedFactory() _serialize = serialize _deserialize = deserialize _AddressBook = ttypes.AddressBook - _mkaddr = make_addressbook - loops = iter(range(loops)) - - t0 = pyperf.perf_counter() - for _ in loops: - for j in range(100): - ab = _mkaddr() + _Person = ttypes.Person + _PhoneNumber = ttypes.PhoneNumber + MOBILE = ttypes.PhoneType.MOBILE + HOME = ttypes.PhoneType.HOME + for _ in range(loops): + t0 = pyperf.perf_counter() + for _ in range(100): + # First, create the addressbook. + ab = _AddressBook() + phone1 = _PhoneNumber() + phone1.type = MOBILE + phone1.number = '555-1212' + phone2 = _PhoneNumber() + phone2.type = HOME + phone2.number = '555-1234' + person = _Person() + person.name = "Alice" + person.phones = [phone1, phone2] + person.created_at = 1400000000 + ab.people = {person.name: person} + # Then, round-trip through serialization. encoded = _serialize(ab, proto_factory) ab2 = _AddressBook() _deserialize(ab2, encoded, proto_factory) - return pyperf.perf_counter() - t0 + elapsed += pyperf.perf_counter() - t0 + return elapsed if __name__ == "__main__": From 09197481d18bf5e5848a222febd68ff6b24c8b26 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 3 Nov 2021 17:08:13 -0600 Subject: [PATCH 13/32] Update for changes to pyperformance. --- benchmarks/MANIFEST | 38 ++++++------------- benchmarks/base.toml | 23 +---------- benchmarks/bm_aiohttp/pyproject.toml | 8 +--- benchmarks/bm_djangocms/pyproject.toml | 8 +--- benchmarks/bm_flaskblogging/pyproject.toml | 8 +--- .../bm_gevent_hub/bm_gevent_cancel_wait.toml | 14 +------ .../bm_gevent_hub/bm_gevent_switch.toml | 14 +------ .../bm_gevent_wait_func_ready.toml | 14 +------ .../bm_gevent_hub/bm_gevent_wait_ready.toml | 14 +------ benchmarks/bm_gevent_hub/pyproject.toml | 12 +----- benchmarks/bm_gunicorn/pyproject.toml | 8 +--- benchmarks/bm_json/pyproject.toml | 9 +---- benchmarks/bm_mypy/pyproject.toml | 8 +--- benchmarks/bm_pycparser/pyproject.toml | 8 +--- benchmarks/bm_pylint/pyproject.toml | 8 +--- .../pyproject.toml | 8 +--- benchmarks/bm_thrift/pyproject.toml | 8 +--- pyproject.toml | 10 +++++ 18 files changed, 43 insertions(+), 177 deletions(-) mode change 100644 => 120000 benchmarks/base.toml create mode 100644 pyproject.toml diff --git a/benchmarks/MANIFEST b/benchmarks/MANIFEST index a257be0..24f6685 100644 --- a/benchmarks/MANIFEST +++ b/benchmarks/MANIFEST @@ -1,28 +1,14 @@ [benchmarks] -name version origin metafile -aiohttp - - -djangocms - - -flaskblogging - - -gevent_hub - - -gunicorn - - -json - - -mypy - - -pycparser - - -pylint - - -pytorch_alexnet_inference - - -thrift - - - - -[group default] -aiohttp -djangocms -flaskblogging -gevent_hub -gunicorn -json -mypy -pycparser -pylint -pytorch_alexnet_inference -thrift +name metafile +aiohttp +djangocms +flaskblogging +gevent_hub +gunicorn +json +mypy +pycparser +pylint +pytorch_alexnet_inference +thrift diff --git a/benchmarks/base.toml b/benchmarks/base.toml deleted file mode 100644 index e575389..0000000 --- a/benchmarks/base.toml +++ /dev/null @@ -1,22 +0,0 @@ -[project] -#description = "a pyperformance benchmark" -#readme = "README.rst" -#requires-python = ">=3.8" -#license = {file = "LICENSE.txt"} -version = "0.9.0" # XXX an arbitrary value; the repo doesn't have one - -dependencies = [ - "pyperf", -] - -urls = {repository = "https://github.com/pyston/python-macrobenchmarks"} - -dynamic = [ - "name", - "version", -] - -[tool.pyperformance] -metabase = "" -#tags = [] -#extra_opts = "" diff --git a/benchmarks/base.toml b/benchmarks/base.toml new file mode 120000 index 0000000..1e11d78 --- /dev/null +++ b/benchmarks/base.toml @@ -0,0 +1 @@ +../pyproject.toml \ No newline at end of file diff --git a/benchmarks/bm_aiohttp/pyproject.toml b/benchmarks/bm_aiohttp/pyproject.toml index cb31e8e..b390285 100644 --- a/benchmarks/bm_aiohttp/pyproject.toml +++ b/benchmarks/bm_aiohttp/pyproject.toml @@ -6,13 +6,7 @@ dependencies = [ "requests", "uvloop", ] - -# XXX This should be inherited from metabase. -dynamic = [ - "name", - "version", -] +dynamic = ["version"] [tool.pyperformance] -# "name" is set automatically. metabase = ".." diff --git a/benchmarks/bm_djangocms/pyproject.toml b/benchmarks/bm_djangocms/pyproject.toml index 0297f4f..41fe30f 100644 --- a/benchmarks/bm_djangocms/pyproject.toml +++ b/benchmarks/bm_djangocms/pyproject.toml @@ -12,13 +12,7 @@ dependencies = [ "djangocms-video", "requests", ] - -# XXX This should be inherited from metabase. -dynamic = [ - "name", - "version", -] +dynamic = ["version"] [tool.pyperformance] -# "name" is set automatically. metabase = ".." diff --git a/benchmarks/bm_flaskblogging/pyproject.toml b/benchmarks/bm_flaskblogging/pyproject.toml index 8208132..8648436 100644 --- a/benchmarks/bm_flaskblogging/pyproject.toml +++ b/benchmarks/bm_flaskblogging/pyproject.toml @@ -5,13 +5,7 @@ dependencies = [ "Flask-Blogging", "requests", ] - -# XXX This should be inherited from metabase. -dynamic = [ - "name", - "version", -] +dynamic = ["version"] [tool.pyperformance] -# "name" is set automatically. metabase = ".." diff --git a/benchmarks/bm_gevent_hub/bm_gevent_cancel_wait.toml b/benchmarks/bm_gevent_hub/bm_gevent_cancel_wait.toml index a44a58d..40bbf1e 100644 --- a/benchmarks/bm_gevent_hub/bm_gevent_cancel_wait.toml +++ b/benchmarks/bm_gevent_hub/bm_gevent_cancel_wait.toml @@ -1,17 +1,7 @@ [project] name = "bm_gevent_cancel_wait" -# XXX Can these be inherited? -dependencies = [ - "gevent", -] - -# XXX This should be inherited from metabase. -dynamic = [ - "name", - "version", -] +dependencies = ["gevent"] +dynamic = ["version"] [tool.pyperformance] -# "name" is set automatically. -# "metabase" is set automatically extra_opts = ["gevent_cancel_wait"] diff --git a/benchmarks/bm_gevent_hub/bm_gevent_switch.toml b/benchmarks/bm_gevent_hub/bm_gevent_switch.toml index 218b14a..3f96f66 100644 --- a/benchmarks/bm_gevent_hub/bm_gevent_switch.toml +++ b/benchmarks/bm_gevent_hub/bm_gevent_switch.toml @@ -1,17 +1,7 @@ [project] name = "bm_gevent_switch" -# XXX Can these be inherited? -dependencies = [ - "gevent", -] - -# XXX This should be inherited from metabase. -dynamic = [ - "name", - "version", -] +dependencies = ["gevent"] +dynamic = ["version"] [tool.pyperformance] -# "name" is set automatically. -# "metabase" is set automatically extra_opts = ["gevent_switch"] diff --git a/benchmarks/bm_gevent_hub/bm_gevent_wait_func_ready.toml b/benchmarks/bm_gevent_hub/bm_gevent_wait_func_ready.toml index 72d8ef2..11bc9c6 100644 --- a/benchmarks/bm_gevent_hub/bm_gevent_wait_func_ready.toml +++ b/benchmarks/bm_gevent_hub/bm_gevent_wait_func_ready.toml @@ -1,17 +1,7 @@ [project] name = "bm_gevent_wait_func_ready" -# XXX Can these be inherited? -dependencies = [ - "gevent", -] - -# XXX This should be inherited from metabase. -dynamic = [ - "name", - "version", -] +dependencies = ["gevent"] +dynamic = ["version"] [tool.pyperformance] -# "name" is set automatically. -# "metabase" is set automatically extra_opts = ["gevent_wait_func_ready"] diff --git a/benchmarks/bm_gevent_hub/bm_gevent_wait_ready.toml b/benchmarks/bm_gevent_hub/bm_gevent_wait_ready.toml index c2722d7..6e673c7 100644 --- a/benchmarks/bm_gevent_hub/bm_gevent_wait_ready.toml +++ b/benchmarks/bm_gevent_hub/bm_gevent_wait_ready.toml @@ -1,17 +1,7 @@ [project] name = "bm_gevent_wait_ready" -# XXX Can these be inherited? -dependencies = [ - "gevent", -] - -# XXX This should be inherited from metabase. -dynamic = [ - "name", - "version", -] +dependencies = ["gevent"] +dynamic = ["version"] [tool.pyperformance] -# "name" is set automatically. -# "metabase" is set automatically extra_opts = ["gevent_wait_ready"] diff --git a/benchmarks/bm_gevent_hub/pyproject.toml b/benchmarks/bm_gevent_hub/pyproject.toml index 09f610e..ab85b0e 100644 --- a/benchmarks/bm_gevent_hub/pyproject.toml +++ b/benchmarks/bm_gevent_hub/pyproject.toml @@ -1,15 +1,7 @@ [project] name = "bm_gevent_hub" -dependencies = [ - "gevent", -] - -# XXX This should be inherited from metabase. -dynamic = [ - "name", - "version", -] +dependencies = ["gevent"] +dynamic = ["version"] [tool.pyperformance] -# "name" is set automatically. metabase = ".." diff --git a/benchmarks/bm_gunicorn/pyproject.toml b/benchmarks/bm_gunicorn/pyproject.toml index e40ca2c..9bc64e5 100644 --- a/benchmarks/bm_gunicorn/pyproject.toml +++ b/benchmarks/bm_gunicorn/pyproject.toml @@ -5,13 +5,7 @@ dependencies = [ "requests", "uvloop", ] - -# XXX This should be inherited from metabase. -dynamic = [ - "name", - "version", -] +dynamic = ["version"] [tool.pyperformance] -# "name" is set automatically. metabase = ".." diff --git a/benchmarks/bm_json/pyproject.toml b/benchmarks/bm_json/pyproject.toml index f2433db..38ddd5b 100644 --- a/benchmarks/bm_json/pyproject.toml +++ b/benchmarks/bm_json/pyproject.toml @@ -1,13 +1,6 @@ [project] name = "bm_json" -#dependencies = [] - -# XXX This should be inherited from metabase. -dynamic = [ - "name", - "version", -] +dynamic = ["version"] [tool.pyperformance] -# "name" is set automatically. metabase = ".." diff --git a/benchmarks/bm_mypy/pyproject.toml b/benchmarks/bm_mypy/pyproject.toml index 8dd39e2..7e371f9 100644 --- a/benchmarks/bm_mypy/pyproject.toml +++ b/benchmarks/bm_mypy/pyproject.toml @@ -3,13 +3,7 @@ name = "bm_mypy" dependencies = [ "mypy", ] - -# XXX This should be inherited from metabase. -dynamic = [ - "name", - "version", -] +dynamic = ["version"] [tool.pyperformance] -# "name" is set automatically. metabase = ".." diff --git a/benchmarks/bm_pycparser/pyproject.toml b/benchmarks/bm_pycparser/pyproject.toml index f7a352c..6e36e4f 100644 --- a/benchmarks/bm_pycparser/pyproject.toml +++ b/benchmarks/bm_pycparser/pyproject.toml @@ -3,13 +3,7 @@ name = "bm_pycparser" dependencies = [ "pycparser", ] - -# XXX This should be inherited from metabase. -dynamic = [ - "name", - "version", -] +dynamic = ["version"] [tool.pyperformance] -# "name" is set automatically. metabase = ".." diff --git a/benchmarks/bm_pylint/pyproject.toml b/benchmarks/bm_pylint/pyproject.toml index 441bcb1..e8120e7 100644 --- a/benchmarks/bm_pylint/pyproject.toml +++ b/benchmarks/bm_pylint/pyproject.toml @@ -3,13 +3,7 @@ name = "bm_pylint" dependencies = [ "pylint", ] - -# XXX This should be inherited from metabase. -dynamic = [ - "name", - "version", -] +dynamic = ["version"] [tool.pyperformance] -# "name" is set automatically. metabase = ".." diff --git a/benchmarks/bm_pytorch_alexnet_inference/pyproject.toml b/benchmarks/bm_pytorch_alexnet_inference/pyproject.toml index 59496d6..47090c2 100644 --- a/benchmarks/bm_pytorch_alexnet_inference/pyproject.toml +++ b/benchmarks/bm_pytorch_alexnet_inference/pyproject.toml @@ -4,13 +4,7 @@ dependencies = [ "torch", "Pillow", ] - -# XXX This should be inherited from metabase. -dynamic = [ - "name", - "version", -] +dynamic = ["version"] [tool.pyperformance] -# "name" is set automatically. metabase = ".." diff --git a/benchmarks/bm_thrift/pyproject.toml b/benchmarks/bm_thrift/pyproject.toml index 97bf416..aa0ea61 100644 --- a/benchmarks/bm_thrift/pyproject.toml +++ b/benchmarks/bm_thrift/pyproject.toml @@ -3,13 +3,7 @@ name = "bm_thrift" dependencies = [ "thrift", ] - -# XXX This should be inherited from metabase. -dynamic = [ - "name", - "version", -] +dynamic = ["version"] [tool.pyperformance] -# "name" is set automatically. metabase = ".." diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3901717 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,10 @@ +[project] +name = "python-macrobenchmarks" +version = "0.9.0" # XXX an arbitrary value; the repo doesn't have one +description = "Pyston benchmarks" +#requires-python = ">=3.8" +dependencies = ["pyperf"] +urls = {repository = "https://github.com/pyston/python-macrobenchmarks"} + +[tool.pyperformance] +manifest = "benchmarks/MANIFEST" From ef8353760f38fca6f344dabc42f52f2cb3ff78f4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 Nov 2021 10:51:32 -0600 Subject: [PATCH 14/32] metabase -> inherits --- benchmarks/bm_aiohttp/pyproject.toml | 2 +- benchmarks/bm_djangocms/pyproject.toml | 2 +- benchmarks/bm_flaskblogging/pyproject.toml | 2 +- benchmarks/bm_gevent_hub/pyproject.toml | 2 +- benchmarks/bm_gunicorn/pyproject.toml | 2 +- benchmarks/bm_json/pyproject.toml | 2 +- benchmarks/bm_mypy/pyproject.toml | 2 +- benchmarks/bm_pycparser/pyproject.toml | 2 +- benchmarks/bm_pylint/pyproject.toml | 2 +- benchmarks/bm_pytorch_alexnet_inference/pyproject.toml | 2 +- benchmarks/bm_thrift/pyproject.toml | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/benchmarks/bm_aiohttp/pyproject.toml b/benchmarks/bm_aiohttp/pyproject.toml index b390285..dbe5021 100644 --- a/benchmarks/bm_aiohttp/pyproject.toml +++ b/benchmarks/bm_aiohttp/pyproject.toml @@ -9,4 +9,4 @@ dependencies = [ dynamic = ["version"] [tool.pyperformance] -metabase = ".." +inherits = ".." diff --git a/benchmarks/bm_djangocms/pyproject.toml b/benchmarks/bm_djangocms/pyproject.toml index 41fe30f..84a16e4 100644 --- a/benchmarks/bm_djangocms/pyproject.toml +++ b/benchmarks/bm_djangocms/pyproject.toml @@ -15,4 +15,4 @@ dependencies = [ dynamic = ["version"] [tool.pyperformance] -metabase = ".." +inherits = ".." diff --git a/benchmarks/bm_flaskblogging/pyproject.toml b/benchmarks/bm_flaskblogging/pyproject.toml index 8648436..f07e1e4 100644 --- a/benchmarks/bm_flaskblogging/pyproject.toml +++ b/benchmarks/bm_flaskblogging/pyproject.toml @@ -8,4 +8,4 @@ dependencies = [ dynamic = ["version"] [tool.pyperformance] -metabase = ".." +inherits = ".." diff --git a/benchmarks/bm_gevent_hub/pyproject.toml b/benchmarks/bm_gevent_hub/pyproject.toml index ab85b0e..b2eb678 100644 --- a/benchmarks/bm_gevent_hub/pyproject.toml +++ b/benchmarks/bm_gevent_hub/pyproject.toml @@ -4,4 +4,4 @@ dependencies = ["gevent"] dynamic = ["version"] [tool.pyperformance] -metabase = ".." +inherits = ".." diff --git a/benchmarks/bm_gunicorn/pyproject.toml b/benchmarks/bm_gunicorn/pyproject.toml index 9bc64e5..0f3d550 100644 --- a/benchmarks/bm_gunicorn/pyproject.toml +++ b/benchmarks/bm_gunicorn/pyproject.toml @@ -8,4 +8,4 @@ dependencies = [ dynamic = ["version"] [tool.pyperformance] -metabase = ".." +inherits = ".." diff --git a/benchmarks/bm_json/pyproject.toml b/benchmarks/bm_json/pyproject.toml index 38ddd5b..f8bf3fc 100644 --- a/benchmarks/bm_json/pyproject.toml +++ b/benchmarks/bm_json/pyproject.toml @@ -3,4 +3,4 @@ name = "bm_json" dynamic = ["version"] [tool.pyperformance] -metabase = ".." +inherits = ".." diff --git a/benchmarks/bm_mypy/pyproject.toml b/benchmarks/bm_mypy/pyproject.toml index 7e371f9..5da0cd8 100644 --- a/benchmarks/bm_mypy/pyproject.toml +++ b/benchmarks/bm_mypy/pyproject.toml @@ -6,4 +6,4 @@ dependencies = [ dynamic = ["version"] [tool.pyperformance] -metabase = ".." +inherits = ".." diff --git a/benchmarks/bm_pycparser/pyproject.toml b/benchmarks/bm_pycparser/pyproject.toml index 6e36e4f..38020ba 100644 --- a/benchmarks/bm_pycparser/pyproject.toml +++ b/benchmarks/bm_pycparser/pyproject.toml @@ -6,4 +6,4 @@ dependencies = [ dynamic = ["version"] [tool.pyperformance] -metabase = ".." +inherits = ".." diff --git a/benchmarks/bm_pylint/pyproject.toml b/benchmarks/bm_pylint/pyproject.toml index e8120e7..207eec9 100644 --- a/benchmarks/bm_pylint/pyproject.toml +++ b/benchmarks/bm_pylint/pyproject.toml @@ -6,4 +6,4 @@ dependencies = [ dynamic = ["version"] [tool.pyperformance] -metabase = ".." +inherits = ".." diff --git a/benchmarks/bm_pytorch_alexnet_inference/pyproject.toml b/benchmarks/bm_pytorch_alexnet_inference/pyproject.toml index 47090c2..1225fd2 100644 --- a/benchmarks/bm_pytorch_alexnet_inference/pyproject.toml +++ b/benchmarks/bm_pytorch_alexnet_inference/pyproject.toml @@ -7,4 +7,4 @@ dependencies = [ dynamic = ["version"] [tool.pyperformance] -metabase = ".." +inherits = ".." diff --git a/benchmarks/bm_thrift/pyproject.toml b/benchmarks/bm_thrift/pyproject.toml index aa0ea61..8d12b9e 100644 --- a/benchmarks/bm_thrift/pyproject.toml +++ b/benchmarks/bm_thrift/pyproject.toml @@ -6,4 +6,4 @@ dependencies = [ dynamic = ["version"] [tool.pyperformance] -metabase = ".." +inherits = ".." From 8db13db58177eab516dc1f5d60426b3b324c2ba7 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 Nov 2021 13:59:18 -0600 Subject: [PATCH 15/32] Ignore data generated during benchmark runs. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6cb24dd..8a120cd 100644 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,4 @@ dmypy.json .pyre/ results +benchmarks/bm_pytorch_alexnet_inference/data/dog.jpg From 8a6a487dfd20bd147e0d5fe3b5e110ad14544c7c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 4 Nov 2021 17:23:59 -0600 Subject: [PATCH 16/32] Run pyperformance more uniformly. --- run_all.sh | 13 ++++--- run_benchmarks.sh | 98 +++++++++++++++++++++++++++++++++++++++++++++++ run_mypy.sh | 34 ++++------------ 3 files changed, 113 insertions(+), 32 deletions(-) create mode 100755 run_benchmarks.sh diff --git a/run_all.sh b/run_all.sh index ebd8e68..678a853 100755 --- a/run_all.sh +++ b/run_all.sh @@ -8,10 +8,11 @@ fi BINARY=$1 set -u -set -x -#python3 -m pip install pyperformance -python3 -m pyperformance run \ - --manifest $(dirname $0)/benchmarks/MANIFEST \ - --python $BINARY \ - --output results.json +ENV=/tmp/macrobenchmark_env +rm -rf $ENV +export PYPERFORMANCE= +for bench in flaskblogging djangocms mypy_bench pylint_bench pycparser_bench pytorch_alexnet_inference gunicorn aiohttp thrift_bench gevent_bench_hub; do + ./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks $bench + # XXX Convert results to verbose "time" output. +done diff --git a/run_benchmarks.sh b/run_benchmarks.sh new file mode 100755 index 0000000..455eeb6 --- /dev/null +++ b/run_benchmarks.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +set -e + +function verbose() { + ( + set -x + "$@" + ) +} + +now=$(date --utc +'%Y%m%d-%H%M%S') +outdir=results + + +# Extract values from env vars. +pyperformance=pyperformance +clone_pp='no' +if [ -n "$PYPERFORMANCE" ]; then + pyperformance=$PYPERFORMANCE + if [ ! -e $pyperformance ]; then + clone_pp='yes' + fi +fi +if [ -z "$WITH_MYPYC" ] || [ "$WITH_MYPYC" = 'no' ] || [ "$WITH_MYPYC" -eq 0 ]; then + mypy= + reset_mypy='no' +elif [ "$WITH_MYPYC" = 'yes' ] || [ "$WITH_MYPYC" -eq 1 ]; then + mypy=/tmp/mypy + reset_mypy='yes' +elif [ -e "$WITH_MYPYC" ]; then + mypy=$WITH_MYPYC + reset_mypy='no' +else + mypy=$WITH_MYPYC + reset_mypy='yes' +fi + +set -u + + +# Parse the command-line. +target_python= +reset_venv='no' +venv= +prev= +for arg in "$@"; do + if [ "$prev" = "-p" -o "$prev" = "--python" ]; then + target_python=$arg + elif [ "$prev" = "--venv" ]; then + venv=$arg + fi + prev=$arg +done +if [ -z "$target_python" ]; then + if [ -z "$venv" ]; then + >&2 echo "ERROR: missing -p/--python arg" + exit 1 + fi + target_python=$venv/bin/python3 +fi +if [ -z "$venv" ]; then + reset_venv='yes' + venv=venv/pyston-python-macrobenchmarks +elif [ ! -e "$venv" ]; then + reset_venv='yes' +fi + + +# Set up the execution environment. +if [ $reset_venv = 'yes' ]; then + verbose rm -rf $venv + verbose $target_python -m venv $venv +fi +if [ $clone_pp = 'yes' ]; then + verbose git clone https://github.com/python/pyperformance "$pyperformance" +fi +verbose $venv/bin/pip install --upgrade "$pyperformance" +if [ $reset_mypy = 'yes' ]; then + verbose rm -rf $mypy + verbose git clone --depth 1 --branch v0.790 https://github.com/python/mypy/ $mypy + + pushd $mypy + verbose $venv/bin/pip install -r mypy-requirements.txt + verbose $venv/bin/pip install --upgrade setuptools + verbose git submodule update --init mypy/typeshed + verbose $venv/bin/python setup.py --use-mypyc install + popd +fi +verbose mkdir -p $outdir + + +# Run the benchmarks. +verbose $venv/bin/python3 -m pyperformance run \ + --venv $venv \ + --manifest benchmarks/MANIFEST \ + --output $outdir/results-$now.json \ + "$@" diff --git a/run_mypy.sh b/run_mypy.sh index e8b2dbf..31c9940 100644 --- a/run_mypy.sh +++ b/run_mypy.sh @@ -14,33 +14,15 @@ fi BINARY=$1 set -u -set -x ENV=/tmp/macrobenchmark_env rm -rf $ENV -python3 -m venv -p $BINARY $ENV -PYTHON=$ENV/bin/python - -#$PYTHON -m pip install pyperformance - rm -rf /tmp/mypy -git clone --depth 1 --branch v0.790 https://github.com/python/mypy/ /tmp/mypy -pushd /tmp/mypy -$PYTHON -m pip install -r mypy-requirements.txt -$PYTHON -m pip install --upgrade setuptools -git submodule update --init mypy/typeshed -$PYTHON setup.py --use-mypyc install -popd - -OPTS=" \ - --manifest $(dirname $0)/benchmarks/MANIFEST \ - --venv $ENV \ - --benchmarks mypy \ -" -# XXX Run 50 loops each instead of the default 20. -$PYTHON -m pyperformance run $OPTS \ - #--output results.json -$PYTHON -m pyperformance run $OPTS \ - #--append results.json -$PYTHON -m pyperformance run $OPTS \ - #--append results.json +export PYPERFORMANCE= +export WITH_MYPYC=/tmp/mypy +./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks mypy #50 +# XXX Convert results to verbose "time" output. +./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks mypy #50 +# XXX Convert results to verbose "time" output. +./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks mypy #50 +# XXX Convert results to verbose "time" output. From c1bbc795105ee20fc2ef8229313e2b146b95e191 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 Nov 2021 14:01:39 -0600 Subject: [PATCH 17/32] Make the script better. --- run_benchmarks.sh | 75 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 19 deletions(-) diff --git a/run_benchmarks.sh b/run_benchmarks.sh index 455eeb6..9b67a7d 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -10,7 +10,6 @@ function verbose() { } now=$(date --utc +'%Y%m%d-%H%M%S') -outdir=results # Extract values from env vars. @@ -41,29 +40,67 @@ set -u # Parse the command-line. target_python= -reset_venv='no' venv= -prev= -for arg in "$@"; do - if [ "$prev" = "-p" -o "$prev" = "--python" ]; then - target_python=$arg - elif [ "$prev" = "--venv" ]; then - venv=$arg - fi - prev=$arg +reset_venv='no' +manifest= +outfile= +argv=() +while [ $# -gt 0 ]; do + arg=$1 + shift + case $arg in + -p|--python) + target_python=$1 + shift + ;; + --venv) + venv=$1 + shift + ;; + --manifest) + manifest=$1 + shift + ;; + -o|--output) + outfile=$1 + shift + ;; + *) + argv+=("$arg") + ;; + esac done if [ -z "$target_python" ]; then - if [ -z "$venv" ]; then + target_python=$venv/bin/python3 + if [ -z "$venv" -o ! -e $target_python ]; then >&2 echo "ERROR: missing -p/--python arg" exit 1 fi - target_python=$venv/bin/python3 -fi -if [ -z "$venv" ]; then - reset_venv='yes' +elif [ -z "$venv" ]; then venv=venv/pyston-python-macrobenchmarks -elif [ ! -e "$venv" ]; then reset_venv='yes' +elif [ ! -e $venv ]; then + reset_venv='yes' + >&2 echo "WARNING: requested venv does not exist but will be created" +elif [ "$(realpath $venv/bin/python3)" != $target_python ]; then + reset_venv='yes' + >&2 echo "WARNING: requested venv is outdated and will be reset" +fi +if [ -z "$manifest" ]; then + manifest=benchmarks/MANIFEST +fi +if [ -z "$outfile" ]; then + outdir=results + outfile=$outdir/results-$now.json +elif [ -d $outfile ]; then + outdir=$outfile + outfile=$outdir/results-$now.json +else + outdir=$(dirname $outfile) + if [ -z "$outdir" ]; then + outdir = '.' + outfile=./$outfile + fi fi @@ -93,6 +130,6 @@ verbose mkdir -p $outdir # Run the benchmarks. verbose $venv/bin/python3 -m pyperformance run \ --venv $venv \ - --manifest benchmarks/MANIFEST \ - --output $outdir/results-$now.json \ - "$@" + --manifest $manifest \ + --output $outfile \ + "${argv[@]}" From 3fd8188a69dbe2c9b7d8cd1f9253e511f364e1ce Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 Nov 2021 14:13:05 -0600 Subject: [PATCH 18/32] Fix a numeric test in the script. --- run_benchmarks.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_benchmarks.sh b/run_benchmarks.sh index 9b67a7d..a7d43ce 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -21,10 +21,10 @@ if [ -n "$PYPERFORMANCE" ]; then clone_pp='yes' fi fi -if [ -z "$WITH_MYPYC" ] || [ "$WITH_MYPYC" = 'no' ] || [ "$WITH_MYPYC" -eq 0 ]; then +if [ -z "$WITH_MYPYC" ] || [ "$WITH_MYPYC" = 'no' ] || [ "$WITH_MYPYC" = '0' ]; then mypy= reset_mypy='no' -elif [ "$WITH_MYPYC" = 'yes' ] || [ "$WITH_MYPYC" -eq 1 ]; then +elif [ "$WITH_MYPYC" = 'yes' ] || [ "$WITH_MYPYC" = '1' ]; then mypy=/tmp/mypy reset_mypy='yes' elif [ -e "$WITH_MYPYC" ]; then From 18988a97a1c7b7e0d264a9c139c449b35348f282 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 Nov 2021 14:57:44 -0600 Subject: [PATCH 19/32] Print dividers. --- run_benchmarks.sh | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/run_benchmarks.sh b/run_benchmarks.sh index a7d43ce..79a5178 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -2,6 +2,15 @@ set -e +function divider() { + local title=$1 + echo + echo '##################################################' + echo "# $title" + echo '##################################################' + echo +} + function verbose() { ( set -x @@ -105,29 +114,48 @@ fi # Set up the execution environment. + if [ $reset_venv = 'yes' ]; then + divider "creating the top-level venv at $venv" verbose rm -rf $venv verbose $target_python -m venv $venv fi +divider "ensuring setuptools is up-to-date in $venv" +verbose $venv/bin/pip install --upgrade setuptools + if [ $clone_pp = 'yes' ]; then + divider "preparing pyperformance at $pyperformance" verbose git clone https://github.com/python/pyperformance "$pyperformance" fi +if [ $pyperformance = 'pyperformance' ]; then + divider "installing pyperformance from PyPI" +else + divider "installing pyperformance into $venv from $pyperformance" +fi verbose $venv/bin/pip install --upgrade "$pyperformance" + if [ $reset_mypy = 'yes' ]; then + divider "getting a fresh copy of the mypy repo" verbose rm -rf $mypy verbose git clone --depth 1 --branch v0.790 https://github.com/python/mypy/ $mypy + pushd $mypy + verbose git submodule update --init mypy/typeshed + popd pushd $mypy + divider "installing the mypy requirements into $venv" verbose $venv/bin/pip install -r mypy-requirements.txt - verbose $venv/bin/pip install --upgrade setuptools - verbose git submodule update --init mypy/typeshed + divider "building mypyc and installing it in $venv" verbose $venv/bin/python setup.py --use-mypyc install popd fi + +divider "other setup" verbose mkdir -p $outdir # Run the benchmarks. +divider "running the benchmarks" verbose $venv/bin/python3 -m pyperformance run \ --venv $venv \ --manifest $manifest \ From 0061184a5bd910037500f705e1b71c1cc8a046aa Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 Nov 2021 14:59:02 -0600 Subject: [PATCH 20/32] Prepare mypy even if not resetting it. --- run_benchmarks.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/run_benchmarks.sh b/run_benchmarks.sh index 79a5178..2a621ff 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -134,14 +134,15 @@ else fi verbose $venv/bin/pip install --upgrade "$pyperformance" -if [ $reset_mypy = 'yes' ]; then - divider "getting a fresh copy of the mypy repo" - verbose rm -rf $mypy - verbose git clone --depth 1 --branch v0.790 https://github.com/python/mypy/ $mypy - pushd $mypy - verbose git submodule update --init mypy/typeshed - popd - +if [ -n "$mypy" ]; then + if [ $reset_mypy = 'yes' ]; then + divider "getting a fresh copy of the mypy repo" + verbose rm -rf $mypy + verbose git clone --depth 1 --branch v0.790 https://github.com/python/mypy/ $mypy + pushd $mypy + verbose git submodule update --init mypy/typeshed + popd + fi pushd $mypy divider "installing the mypy requirements into $venv" verbose $venv/bin/pip install -r mypy-requirements.txt From 094e17011d4bd188ed49cc2dc0c1d26d673fa687 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 Nov 2021 15:00:54 -0600 Subject: [PATCH 21/32] Add --skip-setup CLI to the runner script. --- run_benchmarks.sh | 73 +++++++++++++++++++++++++---------------------- run_mypy.sh | 7 +++-- 2 files changed, 43 insertions(+), 37 deletions(-) diff --git a/run_benchmarks.sh b/run_benchmarks.sh index 2a621ff..4c931d1 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -53,6 +53,7 @@ venv= reset_venv='no' manifest= outfile= +skip_setup='no' argv=() while [ $# -gt 0 ]; do arg=$1 @@ -74,6 +75,9 @@ while [ $# -gt 0 ]; do outfile=$1 shift ;; + --skip-setup) + skip_setup='yes' + ;; *) argv+=("$arg") ;; @@ -114,45 +118,46 @@ fi # Set up the execution environment. +if [ $skip_setup = 'no' ]; then + if [ $reset_venv = 'yes' ]; then + divider "creating the top-level venv at $venv" + verbose rm -rf $venv + verbose $target_python -m venv $venv + fi + divider "ensuring setuptools is up-to-date in $venv" + verbose $venv/bin/pip install --upgrade setuptools -if [ $reset_venv = 'yes' ]; then - divider "creating the top-level venv at $venv" - verbose rm -rf $venv - verbose $target_python -m venv $venv -fi -divider "ensuring setuptools is up-to-date in $venv" -verbose $venv/bin/pip install --upgrade setuptools - -if [ $clone_pp = 'yes' ]; then - divider "preparing pyperformance at $pyperformance" - verbose git clone https://github.com/python/pyperformance "$pyperformance" -fi -if [ $pyperformance = 'pyperformance' ]; then - divider "installing pyperformance from PyPI" -else - divider "installing pyperformance into $venv from $pyperformance" -fi -verbose $venv/bin/pip install --upgrade "$pyperformance" - -if [ -n "$mypy" ]; then - if [ $reset_mypy = 'yes' ]; then - divider "getting a fresh copy of the mypy repo" - verbose rm -rf $mypy - verbose git clone --depth 1 --branch v0.790 https://github.com/python/mypy/ $mypy + if [ $clone_pp = 'yes' ]; then + divider "preparing pyperformance at $pyperformance" + verbose git clone https://github.com/python/pyperformance "$pyperformance" + fi + if [ $pyperformance = 'pyperformance' ]; then + divider "installing pyperformance from PyPI" + else + divider "installing pyperformance into $venv from $pyperformance" + fi + verbose $venv/bin/pip install --upgrade "$pyperformance" + + if [ -n "$mypy" ]; then + if [ $reset_mypy = 'yes' ]; then + divider "getting a fresh copy of the mypy repo" + verbose rm -rf $mypy + verbose git clone --depth 1 --branch v0.790 https://github.com/python/mypy/ $mypy + pushd $mypy + verbose git submodule update --init mypy/typeshed + popd + fi pushd $mypy - verbose git submodule update --init mypy/typeshed + divider "installing the mypy requirements into $venv" + verbose $venv/bin/pip install -r mypy-requirements.txt + divider "building mypyc and installing it in $venv" + verbose $venv/bin/python setup.py --use-mypyc install popd fi - pushd $mypy - divider "installing the mypy requirements into $venv" - verbose $venv/bin/pip install -r mypy-requirements.txt - divider "building mypyc and installing it in $venv" - verbose $venv/bin/python setup.py --use-mypyc install - popd -fi -divider "other setup" -verbose mkdir -p $outdir + divider "other setup" + verbose mkdir -p $outdir +fi # Run the benchmarks. diff --git a/run_mypy.sh b/run_mypy.sh index 31c9940..22ea31d 100644 --- a/run_mypy.sh +++ b/run_mypy.sh @@ -20,9 +20,10 @@ rm -rf $ENV rm -rf /tmp/mypy export PYPERFORMANCE= export WITH_MYPYC=/tmp/mypy -./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks mypy #50 +# XXX Do it 50 times. +./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks mypy # XXX Convert results to verbose "time" output. -./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks mypy #50 +./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks mypy --skip-setup # XXX Convert results to verbose "time" output. -./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks mypy #50 +./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks mypy --skip-setup # XXX Convert results to verbose "time" output. From 52a08582b8624b48113e07dd71655bd37ec13908 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 Nov 2021 15:01:15 -0600 Subject: [PATCH 22/32] Make the script executable. --- run_mypy.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 run_mypy.sh diff --git a/run_mypy.sh b/run_mypy.sh old mode 100644 new mode 100755 From f810156d8d328c75d2058d007804d3f926026514 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 Nov 2021 15:43:34 -0600 Subject: [PATCH 23/32] Run with mypyc 50 times. --- benchmarks/MANIFEST | 4 ++++ benchmarks/bm_mypy/bm_mypyc.toml | 9 +++++++++ run_mypy.sh | 7 +++---- 3 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 benchmarks/bm_mypy/bm_mypyc.toml diff --git a/benchmarks/MANIFEST b/benchmarks/MANIFEST index 24f6685..d0b677d 100644 --- a/benchmarks/MANIFEST +++ b/benchmarks/MANIFEST @@ -8,7 +8,11 @@ gevent_hub gunicorn json mypy +mypyc pycparser pylint pytorch_alexnet_inference thrift + +[group default] +-mypyc diff --git a/benchmarks/bm_mypy/bm_mypyc.toml b/benchmarks/bm_mypy/bm_mypyc.toml new file mode 100644 index 0000000..ca894a6 --- /dev/null +++ b/benchmarks/bm_mypy/bm_mypyc.toml @@ -0,0 +1,9 @@ +[project] +name = "bm_mypyc" +dependencies = [ + "mypy", +] +dynamic = ["version"] + +[tool.pyperformance] +extra_opts = ["--loops", "50"] diff --git a/run_mypy.sh b/run_mypy.sh index 22ea31d..0eb79ca 100755 --- a/run_mypy.sh +++ b/run_mypy.sh @@ -20,10 +20,9 @@ rm -rf $ENV rm -rf /tmp/mypy export PYPERFORMANCE= export WITH_MYPYC=/tmp/mypy -# XXX Do it 50 times. -./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks mypy +./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks mypyc # XXX Convert results to verbose "time" output. -./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks mypy --skip-setup +./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks mypyc --skip-setup # XXX Convert results to verbose "time" output. -./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks mypy --skip-setup +./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks mypyc --skip-setup # XXX Convert results to verbose "time" output. From c71124fa49e9bf577ef66b12928d4afe2ba0a488 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 Nov 2021 16:23:35 -0600 Subject: [PATCH 24/32] Add --with-mypyc CLI to the runner script. --- run_benchmarks.sh | 66 +++++++++++++++++++++++++++++++++++++---------- run_mypy.sh | 7 +++-- 2 files changed, 55 insertions(+), 18 deletions(-) diff --git a/run_benchmarks.sh b/run_benchmarks.sh index 4c931d1..13bba87 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -30,19 +30,7 @@ if [ -n "$PYPERFORMANCE" ]; then clone_pp='yes' fi fi -if [ -z "$WITH_MYPYC" ] || [ "$WITH_MYPYC" = 'no' ] || [ "$WITH_MYPYC" = '0' ]; then - mypy= - reset_mypy='no' -elif [ "$WITH_MYPYC" = 'yes' ] || [ "$WITH_MYPYC" = '1' ]; then - mypy=/tmp/mypy - reset_mypy='yes' -elif [ -e "$WITH_MYPYC" ]; then - mypy=$WITH_MYPYC - reset_mypy='no' -else - mypy=$WITH_MYPYC - reset_mypy='yes' -fi + set -u @@ -53,12 +41,14 @@ venv= reset_venv='no' manifest= outfile= +benchmarks= +mypy='no' skip_setup='no' argv=() while [ $# -gt 0 ]; do arg=$1 shift - case $arg in + case "$arg" in -p|--python) target_python=$1 shift @@ -75,6 +65,23 @@ while [ $# -gt 0 ]; do outfile=$1 shift ;; + -b|--benchmarks) + benchmarks=$1 + shift + ;; + --with-mypyc) + mypy='yes' + if [ $# -gt 0 ]; then + case $1 in + -*) + ;; + *) + mypy=$1 + shift + ;; + esac + fi + ;; --skip-setup) skip_setup='yes' ;; @@ -83,6 +90,7 @@ while [ $# -gt 0 ]; do ;; esac done + if [ -z "$target_python" ]; then target_python=$venv/bin/python3 if [ -z "$venv" -o ! -e $target_python ]; then @@ -115,6 +123,35 @@ else outfile=./$outfile fi fi +if [ -z "$benchmarks" ]; then + if [ "$mypy" != "no" ]; then + benchmarks='mypyc' + else + benchmarks='default' + fi +else + case $benchmarks in + *mypyc*) + if [ "$mypy" = 'no' ]; then + mypy='yes' + fi + ;; + *mypy*) + benchmarks=${benchmarks/mypy/mypyc} + ;; + esac +fi +if [ -z "$mypy" -o "$mypy" = 'yes' ]; then + mypy=/tmp/mypy + reset_mypy='yes' +elif [ $mypy == 'no' ]; then + mypy= + reset_mypy='no' +elif [ ! -e $mypy ]; then + reset_mypy='yes' +else + reset_mypy='no' +fi # Set up the execution environment. @@ -165,5 +202,6 @@ divider "running the benchmarks" verbose $venv/bin/python3 -m pyperformance run \ --venv $venv \ --manifest $manifest \ + --benchmarks $benchmarks \ --output $outfile \ "${argv[@]}" diff --git a/run_mypy.sh b/run_mypy.sh index 0eb79ca..a4271cd 100755 --- a/run_mypy.sh +++ b/run_mypy.sh @@ -19,10 +19,9 @@ ENV=/tmp/macrobenchmark_env rm -rf $ENV rm -rf /tmp/mypy export PYPERFORMANCE= -export WITH_MYPYC=/tmp/mypy -./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks mypyc +./run_benchmarks.sh --python $BINARY --venv $ENV --with-mypyc # XXX Convert results to verbose "time" output. -./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks mypyc --skip-setup +./run_benchmarks.sh --python $BINARY --venv $ENV --with-mypyc --skip-setup # XXX Convert results to verbose "time" output. -./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks mypyc --skip-setup +./run_benchmarks.sh --python $BINARY --venv $ENV --with-mypyc --skip-setup # XXX Convert results to verbose "time" output. From c6072d63484f8a329a6be7eb9c940f9e140427a6 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 5 Nov 2021 18:21:35 -0600 Subject: [PATCH 25/32] Add --clean CLI to the runner script. --- run_all.sh | 11 +++-- run_benchmarks.sh | 103 +++++++++++++++++++++++++++++++++++----------- run_mypy.sh | 10 ++--- 3 files changed, 87 insertions(+), 37 deletions(-) diff --git a/run_all.sh b/run_all.sh index 678a853..ea211af 100755 --- a/run_all.sh +++ b/run_all.sh @@ -7,12 +7,11 @@ fi BINARY=$1 -set -u - ENV=/tmp/macrobenchmark_env -rm -rf $ENV -export PYPERFORMANCE= -for bench in flaskblogging djangocms mypy_bench pylint_bench pycparser_bench pytorch_alexnet_inference gunicorn aiohttp thrift_bench gevent_bench_hub; do - ./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks $bench +./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks flaskblogging --clean +# XXX Convert results to verbose "time" output. +for bench in djangocms mypy_bench pylint_bench pycparser_bench pytorch_alexnet_inference gunicorn aiohttp thrift_bench gevent_bench_hub; do + #/usr/bin/time --verbose --output=results/${bench}.out $ENV/bin/python $(dirname $0)/benchmarks/${bench}.py + ./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks $bench --no-clean # XXX Convert results to verbose "time" output. done diff --git a/run_benchmarks.sh b/run_benchmarks.sh index 13bba87..5af3759 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -18,6 +18,24 @@ function verbose() { ) } +function rotate() { + local force='no' + if [ "$1" = '--force' ]; then + shift + force='yes' + fi + local oldfile=$1 + local newfile=$oldfile.bak + if [ -e $newfile ]; then + if [ $force = 'no' ]; then + >&2 echo "ERROR: $newfile already exists" + exit 1 + fi + verbose rm $newfile + fi + verbose mv $oldfile $newfile +} + now=$(date --utc +'%Y%m%d-%H%M%S') @@ -43,6 +61,7 @@ manifest= outfile= benchmarks= mypy='no' +clean= skip_setup='no' argv=() while [ $# -gt 0 ]; do @@ -82,8 +101,23 @@ while [ $# -gt 0 ]; do esac fi ;; + --clean) + if [ "$skip_setup" = 'yes' ]; then + >&2 echo "ERROR: got --clean and --skip-setup" + exit 1 + fi + clean='yes' + ;; + --no-clean) + clean='no' + ;; --skip-setup) + if [ "$clean" = 'yes' ]; then + >&2 echo "ERROR: got --clean and --skip-setup" + exit 1 + fi skip_setup='yes' + clean='no' ;; *) argv+=("$arg") @@ -93,17 +127,17 @@ done if [ -z "$target_python" ]; then target_python=$venv/bin/python3 - if [ -z "$venv" -o ! -e $target_python ]; then + if [ -z "$venv" -o ! -e "$target_python" ]; then >&2 echo "ERROR: missing -p/--python arg" exit 1 fi elif [ -z "$venv" ]; then venv=venv/pyston-python-macrobenchmarks reset_venv='yes' -elif [ ! -e $venv ]; then +elif [ ! -e "$venv" ]; then reset_venv='yes' >&2 echo "WARNING: requested venv does not exist but will be created" -elif [ "$(realpath $venv/bin/python3)" != $target_python ]; then +elif [ "$(realpath "$venv/bin/python3")" != "$target_python" ]; then reset_venv='yes' >&2 echo "WARNING: requested venv is outdated and will be reset" fi @@ -117,9 +151,9 @@ elif [ -d $outfile ]; then outdir=$outfile outfile=$outdir/results-$now.json else - outdir=$(dirname $outfile) + outdir=$(dirname "$outfile") if [ -z "$outdir" ]; then - outdir = '.' + outdir='.' outfile=./$outfile fi fi @@ -141,67 +175,86 @@ else ;; esac fi +MYPY_REPO_ROOT=/tmp/mypy if [ -z "$mypy" -o "$mypy" = 'yes' ]; then - mypy=/tmp/mypy + mypy=MYPY_REPO_ROOT reset_mypy='yes' -elif [ $mypy == 'no' ]; then +elif [ "$mypy" == 'no' ]; then mypy= reset_mypy='no' -elif [ ! -e $mypy ]; then +elif [ ! -e "$mypy" ]; then reset_mypy='yes' else reset_mypy='no' fi +if [ "$clean" = 'yes' ]; then + if [ "$skip_setup" != 'no' ]; then + >&2 echo "ERROR: got --clean and --skip-setup" + exit 1 + fi + reset_mypy='yes' + reset_venv='yes' +fi # Set up the execution environment. if [ $skip_setup = 'no' ]; then if [ $reset_venv = 'yes' ]; then divider "creating the top-level venv at $venv" - verbose rm -rf $venv - verbose $target_python -m venv $venv + if [ -e "$venv" ]; then + #verbose rm -rf $venv + rotate --force "$venv" + fi + verbose "$target_python" -m venv "$venv" fi divider "ensuring setuptools is up-to-date in $venv" - verbose $venv/bin/pip install --upgrade setuptools + verbose "$venv/bin/pip" install --upgrade setuptools if [ $clone_pp = 'yes' ]; then divider "preparing pyperformance at $pyperformance" + if [ -e "$pyperformance" ]; then + rotate "$pyperformance" + fi verbose git clone https://github.com/python/pyperformance "$pyperformance" fi - if [ $pyperformance = 'pyperformance' ]; then + if [ "$pyperformance" = 'pyperformance' ]; then divider "installing pyperformance from PyPI" + verbose "$venv/bin/pip" install --upgrade "$pyperformance" else divider "installing pyperformance into $venv from $pyperformance" + verbose "$venv/bin/pip" install --force-reinstall "$pyperformance" fi - verbose $venv/bin/pip install --upgrade "$pyperformance" if [ -n "$mypy" ]; then if [ $reset_mypy = 'yes' ]; then divider "getting a fresh copy of the mypy repo" - verbose rm -rf $mypy - verbose git clone --depth 1 --branch v0.790 https://github.com/python/mypy/ $mypy - pushd $mypy + if [ -e "$mypy" ]; then + #verbose rm -rf $mypy + rotate "$mypy" + fi + verbose git clone --depth 1 --branch v0.790 https://github.com/python/mypy/ "$mypy" + pushd "$mypy" verbose git submodule update --init mypy/typeshed popd fi - pushd $mypy + pushd "$mypy" divider "installing the mypy requirements into $venv" - verbose $venv/bin/pip install -r mypy-requirements.txt + verbose "$venv/bin/pip" install -r mypy-requirements.txt divider "building mypyc and installing it in $venv" - verbose $venv/bin/python setup.py --use-mypyc install + verbose "$venv/bin/python3" setup.py --use-mypyc install popd fi divider "other setup" - verbose mkdir -p $outdir + verbose mkdir -p "$outdir" fi # Run the benchmarks. divider "running the benchmarks" -verbose $venv/bin/python3 -m pyperformance run \ - --venv $venv \ - --manifest $manifest \ - --benchmarks $benchmarks \ - --output $outfile \ +verbose "$venv/bin/python3" -m pyperformance run \ + --venv "$venv" \ + --manifest "$manifest" \ + --benchmarks "$benchmarks" \ + --output "$outfile" \ "${argv[@]}" diff --git a/run_mypy.sh b/run_mypy.sh index a4271cd..f4820b3 100755 --- a/run_mypy.sh +++ b/run_mypy.sh @@ -13,13 +13,11 @@ fi BINARY=$1 -set -u - ENV=/tmp/macrobenchmark_env -rm -rf $ENV -rm -rf /tmp/mypy -export PYPERFORMANCE= -./run_benchmarks.sh --python $BINARY --venv $ENV --with-mypyc +#time $ENV/bin/python benchmarks/mypy_bench.py 50 +#time $ENV/bin/python benchmarks/mypy_bench.py 50 +#time $ENV/bin/python benchmarks/mypy_bench.py 50 +./run_benchmarks.sh --python $BINARY --venv $ENV --with-mypyc --clean # XXX Convert results to verbose "time" output. ./run_benchmarks.sh --python $BINARY --venv $ENV --with-mypyc --skip-setup # XXX Convert results to verbose "time" output. From 08c918367abce4a1795adbf0e4b0ecafe45113e0 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 9 Nov 2021 16:58:16 -0700 Subject: [PATCH 26/32] Make kinto_bench a pyperformance benchmark. --- benchmarks/.libs/netutils.py | 36 ++++++++- benchmarks/MANIFEST | 1 + .../bm_kinto/data}/.coveragerc | 0 .../bm_kinto/data}/CHANGES.txt | 0 .../bm_kinto/data}/MANIFEST.in | 0 .../bm_kinto/data}/README.txt | 0 .../bm_kinto/data}/app.wsgi | 0 .../bm_kinto/data}/config/kinto.ini | 0 .../bm_kinto/data}/development.ini | 0 .../bm_kinto/data}/kinto_project/__init__.py | 0 .../kinto_project/static/pyramid-16x16.png | Bin .../data}/kinto_project/static/pyramid.png | Bin .../data}/kinto_project/static/theme.css | 0 .../kinto_project/templates/layout.jinja2 | 0 .../kinto_project/templates/mytemplate.jinja2 | 0 .../bm_kinto/data}/kinto_project/tests.py | 0 .../bm_kinto/data}/kinto_project/views.py | 0 .../bm_kinto/data}/nginx.conf | 0 .../bm_kinto/data}/production.ini | 0 .../bm_kinto/data}/pytest.ini | 0 .../bm_kinto/data}/setup.py | 0 .../bm_kinto/data}/uwsgi_params | 0 benchmarks/bm_kinto/netutils.py | 1 + benchmarks/bm_kinto/pyproject.toml | 15 ++++ .../requirements.txt} | 0 benchmarks/bm_kinto/run_benchmark.py | 60 ++++++++++++++ benchmarks/kinto_bench.py | 75 ------------------ 27 files changed, 109 insertions(+), 79 deletions(-) rename {data/kinto_project => benchmarks/bm_kinto/data}/.coveragerc (100%) rename {data/kinto_project => benchmarks/bm_kinto/data}/CHANGES.txt (100%) rename {data/kinto_project => benchmarks/bm_kinto/data}/MANIFEST.in (100%) rename {data/kinto_project => benchmarks/bm_kinto/data}/README.txt (100%) rename {data/kinto_project => benchmarks/bm_kinto/data}/app.wsgi (100%) rename {data/kinto_project => benchmarks/bm_kinto/data}/config/kinto.ini (100%) rename {data/kinto_project => benchmarks/bm_kinto/data}/development.ini (100%) rename {data/kinto_project => benchmarks/bm_kinto/data}/kinto_project/__init__.py (100%) rename {data/kinto_project => benchmarks/bm_kinto/data}/kinto_project/static/pyramid-16x16.png (100%) rename {data/kinto_project => benchmarks/bm_kinto/data}/kinto_project/static/pyramid.png (100%) rename {data/kinto_project => benchmarks/bm_kinto/data}/kinto_project/static/theme.css (100%) rename {data/kinto_project => benchmarks/bm_kinto/data}/kinto_project/templates/layout.jinja2 (100%) rename {data/kinto_project => benchmarks/bm_kinto/data}/kinto_project/templates/mytemplate.jinja2 (100%) rename {data/kinto_project => benchmarks/bm_kinto/data}/kinto_project/tests.py (100%) rename {data/kinto_project => benchmarks/bm_kinto/data}/kinto_project/views.py (100%) rename {data/kinto_project => benchmarks/bm_kinto/data}/nginx.conf (100%) rename {data/kinto_project => benchmarks/bm_kinto/data}/production.ini (100%) rename {data/kinto_project => benchmarks/bm_kinto/data}/pytest.ini (100%) rename {data/kinto_project => benchmarks/bm_kinto/data}/setup.py (100%) rename {data/kinto_project => benchmarks/bm_kinto/data}/uwsgi_params (100%) create mode 120000 benchmarks/bm_kinto/netutils.py create mode 100644 benchmarks/bm_kinto/pyproject.toml rename benchmarks/{kinto_bench_requirements.txt => bm_kinto/requirements.txt} (100%) create mode 100644 benchmarks/bm_kinto/run_benchmark.py delete mode 100644 benchmarks/kinto_bench.py diff --git a/benchmarks/.libs/netutils.py b/benchmarks/.libs/netutils.py index 64e2919..9bd9833 100644 --- a/benchmarks/.libs/netutils.py +++ b/benchmarks/.libs/netutils.py @@ -1,24 +1,52 @@ import contextlib import ipaddress +import os.path import socket import subprocess import time @contextlib.contextmanager -def serving(argv, sitedir, addr): +def serving(argv, sitedir, addr, *, + pause=None, + kill=False, + quiet=True, + ): + if os.path.exists(addr): + sock = addr + addr = None + try: + os.remove(sock) + except FileNotFoundError: + pass + else: + sock = None + p = subprocess.Popen( argv, cwd=sitedir, - stdout=subprocess.DEVNULL, - stderr=subprocess.STDOUT, + stdout=subprocess.DEVNULL if quiet else None, + stderr=subprocess.STDOUT if quiet else None, ) try: - waitUntilUp(addr) + if pause: + time.sleep(pause) + if not sock: + try: + waitUntilUp(addr) + except NotImplementedError: + sock = addr + addr = None + if sock: + while not os.path.exists(sock): + time.sleep(0.001) + assert p.poll() is None, p.poll() yield assert p.poll() is None, p.poll() finally: p.terminate() + if kill: + p.kill() p.wait() diff --git a/benchmarks/MANIFEST b/benchmarks/MANIFEST index d0b677d..7af33eb 100644 --- a/benchmarks/MANIFEST +++ b/benchmarks/MANIFEST @@ -7,6 +7,7 @@ flaskblogging gevent_hub gunicorn json +kinto mypy mypyc pycparser diff --git a/data/kinto_project/.coveragerc b/benchmarks/bm_kinto/data/.coveragerc similarity index 100% rename from data/kinto_project/.coveragerc rename to benchmarks/bm_kinto/data/.coveragerc diff --git a/data/kinto_project/CHANGES.txt b/benchmarks/bm_kinto/data/CHANGES.txt similarity index 100% rename from data/kinto_project/CHANGES.txt rename to benchmarks/bm_kinto/data/CHANGES.txt diff --git a/data/kinto_project/MANIFEST.in b/benchmarks/bm_kinto/data/MANIFEST.in similarity index 100% rename from data/kinto_project/MANIFEST.in rename to benchmarks/bm_kinto/data/MANIFEST.in diff --git a/data/kinto_project/README.txt b/benchmarks/bm_kinto/data/README.txt similarity index 100% rename from data/kinto_project/README.txt rename to benchmarks/bm_kinto/data/README.txt diff --git a/data/kinto_project/app.wsgi b/benchmarks/bm_kinto/data/app.wsgi similarity index 100% rename from data/kinto_project/app.wsgi rename to benchmarks/bm_kinto/data/app.wsgi diff --git a/data/kinto_project/config/kinto.ini b/benchmarks/bm_kinto/data/config/kinto.ini similarity index 100% rename from data/kinto_project/config/kinto.ini rename to benchmarks/bm_kinto/data/config/kinto.ini diff --git a/data/kinto_project/development.ini b/benchmarks/bm_kinto/data/development.ini similarity index 100% rename from data/kinto_project/development.ini rename to benchmarks/bm_kinto/data/development.ini diff --git a/data/kinto_project/kinto_project/__init__.py b/benchmarks/bm_kinto/data/kinto_project/__init__.py similarity index 100% rename from data/kinto_project/kinto_project/__init__.py rename to benchmarks/bm_kinto/data/kinto_project/__init__.py diff --git a/data/kinto_project/kinto_project/static/pyramid-16x16.png b/benchmarks/bm_kinto/data/kinto_project/static/pyramid-16x16.png similarity index 100% rename from data/kinto_project/kinto_project/static/pyramid-16x16.png rename to benchmarks/bm_kinto/data/kinto_project/static/pyramid-16x16.png diff --git a/data/kinto_project/kinto_project/static/pyramid.png b/benchmarks/bm_kinto/data/kinto_project/static/pyramid.png similarity index 100% rename from data/kinto_project/kinto_project/static/pyramid.png rename to benchmarks/bm_kinto/data/kinto_project/static/pyramid.png diff --git a/data/kinto_project/kinto_project/static/theme.css b/benchmarks/bm_kinto/data/kinto_project/static/theme.css similarity index 100% rename from data/kinto_project/kinto_project/static/theme.css rename to benchmarks/bm_kinto/data/kinto_project/static/theme.css diff --git a/data/kinto_project/kinto_project/templates/layout.jinja2 b/benchmarks/bm_kinto/data/kinto_project/templates/layout.jinja2 similarity index 100% rename from data/kinto_project/kinto_project/templates/layout.jinja2 rename to benchmarks/bm_kinto/data/kinto_project/templates/layout.jinja2 diff --git a/data/kinto_project/kinto_project/templates/mytemplate.jinja2 b/benchmarks/bm_kinto/data/kinto_project/templates/mytemplate.jinja2 similarity index 100% rename from data/kinto_project/kinto_project/templates/mytemplate.jinja2 rename to benchmarks/bm_kinto/data/kinto_project/templates/mytemplate.jinja2 diff --git a/data/kinto_project/kinto_project/tests.py b/benchmarks/bm_kinto/data/kinto_project/tests.py similarity index 100% rename from data/kinto_project/kinto_project/tests.py rename to benchmarks/bm_kinto/data/kinto_project/tests.py diff --git a/data/kinto_project/kinto_project/views.py b/benchmarks/bm_kinto/data/kinto_project/views.py similarity index 100% rename from data/kinto_project/kinto_project/views.py rename to benchmarks/bm_kinto/data/kinto_project/views.py diff --git a/data/kinto_project/nginx.conf b/benchmarks/bm_kinto/data/nginx.conf similarity index 100% rename from data/kinto_project/nginx.conf rename to benchmarks/bm_kinto/data/nginx.conf diff --git a/data/kinto_project/production.ini b/benchmarks/bm_kinto/data/production.ini similarity index 100% rename from data/kinto_project/production.ini rename to benchmarks/bm_kinto/data/production.ini diff --git a/data/kinto_project/pytest.ini b/benchmarks/bm_kinto/data/pytest.ini similarity index 100% rename from data/kinto_project/pytest.ini rename to benchmarks/bm_kinto/data/pytest.ini diff --git a/data/kinto_project/setup.py b/benchmarks/bm_kinto/data/setup.py similarity index 100% rename from data/kinto_project/setup.py rename to benchmarks/bm_kinto/data/setup.py diff --git a/data/kinto_project/uwsgi_params b/benchmarks/bm_kinto/data/uwsgi_params similarity index 100% rename from data/kinto_project/uwsgi_params rename to benchmarks/bm_kinto/data/uwsgi_params diff --git a/benchmarks/bm_kinto/netutils.py b/benchmarks/bm_kinto/netutils.py new file mode 120000 index 0000000..3afa43f --- /dev/null +++ b/benchmarks/bm_kinto/netutils.py @@ -0,0 +1 @@ +../.libs/netutils.py \ No newline at end of file diff --git a/benchmarks/bm_kinto/pyproject.toml b/benchmarks/bm_kinto/pyproject.toml new file mode 100644 index 0000000..72cdb08 --- /dev/null +++ b/benchmarks/bm_kinto/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "bm_kinto" +dependencies = [ + "kinto", + "uWSGI", + "pyramind", + #"pyramid_jinja2", + #"pyramid_debugtoolbar", + "waitress", + "requests", +] +dynamic = ["version"] + +[tool.pyperformance] +inherits = ".." diff --git a/benchmarks/kinto_bench_requirements.txt b/benchmarks/bm_kinto/requirements.txt similarity index 100% rename from benchmarks/kinto_bench_requirements.txt rename to benchmarks/bm_kinto/requirements.txt diff --git a/benchmarks/bm_kinto/run_benchmark.py b/benchmarks/bm_kinto/run_benchmark.py new file mode 100644 index 0000000..357c384 --- /dev/null +++ b/benchmarks/bm_kinto/run_benchmark.py @@ -0,0 +1,60 @@ +import os +import os.path +import requests +import shutil +import subprocess +import sys +import urllib + +import pyperf +import netutils + + +PYTHON = os.path.abspath(sys.executable) +UWSGI = os.path.join(os.path.dirname(PYTHON), "uwsgi") +NGINX = shutil.which("nginx") + +SOCK = "/tmp/kinto.sock" +ADDR = "127.0.0.1:8000" + +DATADIR = os.path.join( + os.path.dirname(__file__), + "data", +) +SETUP_PY = os.path.join(DATADIR, "setup.py") +PRODUCTION_INI = os.path.join(DATADIR, "production.ini") +NGINX_CONF = os.path.join(DATADIR, "nginx.conf") + + +def bench_kinto(loops=5000): + elapsed = 0 + + subprocess.check_call( + [PYTHON, SETUP_PY, "develop"], + cwd=DATADIR, + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + ) + + cmd_app = [UWSGI, PRODUCTION_INI] + cmd_web = [NGINX, "-c", NGINX_CONF, "-p", DATADIR] + with netutils.serving(cmd_app, DATADIR, SOCK, kill=True): + with netutils.serving(cmd_web, DATADIR, ADDR, pause=0.010, quiet=False): + print(requests.get("http://localhost:8000/v1").text) + # print(requests.put("http://localhost:8000/v1/accounts/testuser", json={"data": {"password": "password1"}}).text) + + for _ in range(loops): + t0 = pyperf.perf_counter() + # requests.get("http://localhost:8000/v1/").text + urllib.request.urlopen("http://localhost:8000/v1/").read() + elapsed += pyperf.perf_counter() - t0 + + return elapsed + + +if __name__ == "__main__": + if NGINX is None: + raise Exception("nginx is not installed") + runner = pyperf.Runner() + runner.metadata['description'] = "Test the performance of kinto" + runner.bench_time_func("kinto", bench_kinto) diff --git a/benchmarks/kinto_bench.py b/benchmarks/kinto_bench.py deleted file mode 100644 index ced23d7..0000000 --- a/benchmarks/kinto_bench.py +++ /dev/null @@ -1,75 +0,0 @@ -import json -import os -import requests -import subprocess -import sys -import threading -import time -import urllib - -from djangocms import waitUntilUp - -from os.path import join, abspath, dirname - -if __name__ == "__main__": - exe = sys.executable - def bin(name): - return join(dirname(exe), name) - def rel(path): - return abspath(join(dirname(__file__), path)) - - times = [] - - subprocess.check_call([abspath(exe), rel("../data/kinto_project/setup.py"), "develop"], cwd=rel("../data/kinto_project"), stdout=open("/dev/null", "w"), stderr=subprocess.STDOUT) - - try: - os.remove("/tmp/kinto.sock") - except FileNotFoundError: - pass - p1 = subprocess.Popen([bin("uwsgi"), rel("../data/kinto_project/production.ini")], cwd=rel("../data/kinto_project"), stdout=open("/dev/null", "w"), stderr=subprocess.STDOUT) - # p1 = subprocess.Popen([bin("uwsgi"), rel("../data/kinto_project/production.ini")], cwd=rel("../data/kinto_project")) - while not os.path.exists("/tmp/kinto.sock"): - time.sleep(0.001) - - # p2 = subprocess.Popen(["nginx", "-c", abspath("../data/kinto_project/nginx.conf"), "-p", abspath("../data/kinto_project")], cwd="../data/kinto_project", stdout=open("/dev/null", "w"), stderr=subprocess.STDOUT) - p2 = subprocess.Popen(["nginx", "-c", rel("../data/kinto_project/nginx.conf"), "-p", rel("../data/kinto_project")], cwd=rel("../data/kinto_project")) - - time.sleep(0.010) - - try: - waitUntilUp(("127.0.0.1", 8000)) - - assert p1.poll() is None, p1.poll() - assert p2.poll() is None, p2.poll() - - print(requests.get("http://localhost:8000/v1").text) - # print(requests.put("http://localhost:8000/v1/accounts/testuser", json={"data": {"password": "password1"}}).text) - - n = 5000 - if len(sys.argv) > 1: - n = int(sys.argv[1]) - - start = time.time() - for i in range(n): - times.append(time.time()) - if i % 100 == 0: - print(i, time.time() - start) - # requests.get("http://localhost:8000/v1/").text - urllib.request.urlopen("http://localhost:8000/v1/").read() - times.append(time.time()) - elapsed = time.time() - start - print("%.2fs (%.3freq/s)" % (elapsed, n / elapsed)) - - assert p1.poll() is None, p1.poll() - assert p2.poll() is None, p2.poll() - - finally: - p1.terminate() - p1.kill() - p1.wait() - # p2.kill() - p2.terminate() - p2.wait() - - if len(sys.argv) > 2: - json.dump(times, open(sys.argv[2], 'w')) From 1d82993717d60308475c2a36cafbd5fc7cffb7ed Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 19 Nov 2021 14:40:28 -0700 Subject: [PATCH 27/32] Show how to run the benchmarks. --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index b0b1524..f42abf7 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,21 @@ # python-macrobenchmarks A collection of macro benchmarks for the Python programming language + + +## usage + +```shell +# Run the default benchmarks: +python3 -m pyperformance run --manifest ./benchmarks/MANIFEST +``` + +The benchmarks can still be run without pyperformance. This will produce + the old results format. + +```shell +# Run the benchmarks: +sh ./run_all.sh + +# Run the mypy benchmark using mypyc: +sh ./run_mypy.sh +``` From e6164a8fa9a4f07687e174f98ed36b8047ca8a89 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 6 Dec 2021 16:24:45 -0700 Subject: [PATCH 28/32] Allow running the benchmarks the old way. --- benchmarks/.libs/legacyutils.py | 21 ++ benchmarks/bm_aiohttp/legacyutils.py | 1 + benchmarks/bm_aiohttp/run_benchmark.py | 33 ++- benchmarks/bm_djangocms/legacyutils.py | 1 + benchmarks/bm_djangocms/run_benchmark.py | 75 +++-- benchmarks/bm_flaskblogging/legacyutils.py | 1 + benchmarks/bm_flaskblogging/run_benchmark.py | 30 +- benchmarks/bm_gevent_hub/run_benchmark.py | 18 +- benchmarks/bm_gunicorn/legacyutils.py | 1 + benchmarks/bm_gunicorn/run_benchmark.py | 33 ++- benchmarks/bm_json/legacyutils.py | 1 + benchmarks/bm_json/run_benchmark.py | 37 ++- benchmarks/bm_kinto/legacyutils.py | 1 + benchmarks/bm_kinto/run_benchmark.py | 40 ++- benchmarks/bm_mypy/legacyutils.py | 1 + benchmarks/bm_mypy/run_benchmark.py | 46 +++- benchmarks/bm_pycparser/legacyutils.py | 1 + benchmarks/bm_pycparser/run_benchmark.py | 66 +++-- benchmarks/bm_pylint/legacyutils.py | 1 + benchmarks/bm_pylint/run_benchmark.py | 48 +++- .../legacyutils.py | 1 + .../run_benchmark.py | 30 +- benchmarks/bm_thrift/legacyutils.py | 1 + benchmarks/bm_thrift/run_benchmark.py | 79 ++++-- run_all.sh | 30 +- run_benchmarks.sh | 260 ------------------ run_mypy.sh | 30 +- 27 files changed, 482 insertions(+), 405 deletions(-) create mode 100644 benchmarks/.libs/legacyutils.py create mode 120000 benchmarks/bm_aiohttp/legacyutils.py create mode 120000 benchmarks/bm_djangocms/legacyutils.py create mode 120000 benchmarks/bm_flaskblogging/legacyutils.py create mode 120000 benchmarks/bm_gunicorn/legacyutils.py create mode 120000 benchmarks/bm_json/legacyutils.py create mode 120000 benchmarks/bm_kinto/legacyutils.py create mode 120000 benchmarks/bm_mypy/legacyutils.py create mode 120000 benchmarks/bm_pycparser/legacyutils.py create mode 120000 benchmarks/bm_pylint/legacyutils.py create mode 120000 benchmarks/bm_pytorch_alexnet_inference/legacyutils.py create mode 120000 benchmarks/bm_thrift/legacyutils.py delete mode 100755 run_benchmarks.sh mode change 100755 => 100644 run_mypy.sh diff --git a/benchmarks/.libs/legacyutils.py b/benchmarks/.libs/legacyutils.py new file mode 100644 index 0000000..d232c46 --- /dev/null +++ b/benchmarks/.libs/legacyutils.py @@ -0,0 +1,21 @@ +import sys + + +def maybe_handle_legacy(bench_func, *args, loopsarg='loops', legacyarg=None): + if '--legacy' not in sys.argv: + return + argv = list(sys.argv[1:]) + argv.remove('--legacy') + + kwargs = {} + if legacyarg: + kwargs[legacyarg] = True + if argv: + assert loopsarg + kwargs[loopsarg] = int(argv[0]) + + _, times = bench_func(*args, **kwargs) + if len(argv) > 1: + json.dump(times, open(argv[1], 'w')) + + sys.exit(0) diff --git a/benchmarks/bm_aiohttp/legacyutils.py b/benchmarks/bm_aiohttp/legacyutils.py new file mode 120000 index 0000000..644cca6 --- /dev/null +++ b/benchmarks/bm_aiohttp/legacyutils.py @@ -0,0 +1 @@ +../.libs/legacyutils.py \ No newline at end of file diff --git a/benchmarks/bm_aiohttp/run_benchmark.py b/benchmarks/bm_aiohttp/run_benchmark.py index 80e1eec..bf3ecf5 100644 --- a/benchmarks/bm_aiohttp/run_benchmark.py +++ b/benchmarks/bm_aiohttp/run_benchmark.py @@ -13,7 +13,15 @@ ARGV = [sys.executable, "serve.py"] +############################# +# benchmarks + def bench_aiohttp_requests(loops=3000): + elapsed, _ = _bench_aiohttp_requests(loops) + return elapsed + + +def _bench_aiohttp_requests(loops=3000, legacy=False): """Measure N HTTP requests to a local server. Note that the server is freshly started here. @@ -27,17 +35,36 @@ def bench_aiohttp_requests(loops=3000): Hence this should be used with bench_time_func() insted of bench_func(). """ + start = pyperf.perf_counter() elapsed = 0 + times = [] with netutils.serving(ARGV, DATADIR, "127.0.0.1:8080"): requests_get = requests.get - for _ in range(loops): + for i in range(loops): + # This is a macro benchmark for a Python implementation + # so "elapsed" covers more than just how long a request takes. t0 = pyperf.perf_counter() requests_get("http://localhost:8080/blog/").text - elapsed += pyperf.perf_counter() - t0 - return elapsed + t1 = pyperf.perf_counter() + + elapsed += t1 - t0 + times.append(t0) + if legacy and (i % 100 == 0): + print(i, t0 - start) + times.append(pyperf.perf_counter()) + if legacy: + total = times[-1] - start + print("%.2fs (%.3freq/s)" % (total, loops / total)) + return elapsed, times +############################# +# the script + if __name__ == "__main__": + from legacyutils import maybe_handle_legacy + maybe_handle_legacy(_bench_aiohttp_requests, legacyarg='legacy') + runner = pyperf.Runner() runner.metadata['description'] = "Test the performance of aiohttp" runner.bench_time_func("aiohttp", bench_aiohttp_requests) diff --git a/benchmarks/bm_djangocms/legacyutils.py b/benchmarks/bm_djangocms/legacyutils.py new file mode 120000 index 0000000..644cca6 --- /dev/null +++ b/benchmarks/bm_djangocms/legacyutils.py @@ -0,0 +1 @@ +../.libs/legacyutils.py \ No newline at end of file diff --git a/benchmarks/bm_djangocms/run_benchmark.py b/benchmarks/bm_djangocms/run_benchmark.py index 9a93fbd..e02fa60 100644 --- a/benchmarks/bm_djangocms/run_benchmark.py +++ b/benchmarks/bm_djangocms/run_benchmark.py @@ -117,6 +117,11 @@ def _ensure_datadir(datadir, preserve=True): # benchmarks def bench_djangocms_requests(sitedir, loops=INNER_LOOPS): + elapsed, _ = _bench_djangocms_requests(loops) + return elapsed + + +def _bench_djangocms_requests(sitedir, loops=INNER_LOOPS, legacy=False): """Measure N HTTP requests to a local server. Note that the server is freshly started here. @@ -130,14 +135,26 @@ def bench_djangocms_requests(sitedir, loops=INNER_LOOPS): Hence this should be used with bench_time_func() insted of bench_func(). """ + start = pyperf.perf_counter() elapsed = 0 + times = [] with netutils.serving(ARGV_SERVE, sitedir, "127.0.0.1:8000"): - requests_get = requests.get - for _ in range(loops): + for i in range(loops): + # This is a macro benchmark for a Python implementation + # so "elapsed" covers more than just how long a request takes. t0 = pyperf.perf_counter() - requests_get("http://localhost:8000/").text - elapsed += pyperf.perf_counter() - t0 - return elapsed + requests.get("http://localhost:8000/").text + t1 = pyperf.perf_counter() + + elapsed += t1 - t0 + times.append(t0) + if legacy and (i % 100 == 0): + print(i, t0 - start) + times.append(pyperf.perf_counter()) + if legacy: + total = times[-1] - start + print("%.2fs (%.3freq/s)" % (total, loops / total)) + return elapsed, times # We can't set "add_cmdline_args" on pyperf.Runner @@ -157,17 +174,20 @@ def add_worker_args(cmd, _): ) +############################# +# the script + if __name__ == "__main__": """ Usage: - python djangocms.py - python djangocms.py --setup DATADIR - python djangocms.py --serve DATADIR + python benchmarks/bm_djangocms/run_benchmark.py + python benchmarks/bm_djangocms/run_benchmark.py --setup DIR + python benchmarks/bm_djangocms/run_benchmark.py --serve DIR The first form creates a temporary directory, sets up djangocms in it, serves out of it, and removes the directory. The second form sets up a djangocms installation in the given directory. - The third form runs a benchmark out of an already-set-up directory + The third form runs the benchmark out of an already-set-up directory The second and third forms are useful if you want to benchmark the initial migration phase separately from the second serving phase. """ @@ -175,30 +195,27 @@ def add_worker_args(cmd, _): runner.metadata['description'] = "Test the performance of a Django data migration" # Parse the CLI args. - runner.argparser.add_argument("--setup", action="store_const", const=True) + runner.argparser.add_argument("--legacy", action='store_true') group = runner.argparser.add_mutually_exclusive_group() + group.add_argument("--setup") group.add_argument("--serve") - group.add_argument("datadir", nargs="?") args = runner.argparser.parse_args() - if args.serve is not None: + if args.setup is not None: + args.datadir = args.setup + args.setup = True + args.serve = False + elif args.serve is not None: args.datadir = args.serve + args.setup = False args.serve = True - if not args.setup: - args.setup = False - if not args.datadir: - runner.argparser.error("missing datadir") - elif not os.path.exists(args.datadir): - cmd = f"{sys.executable} {sys.argv[0]} --setup {args.datadir}?" - sys.exit(f"ERROR: Did you forget to run {cmd}?") - default = False - elif args.setup is not None: - args.serve = False - default = False + if not os.path.exists(args.datadir): + cmd = f"{sys.executable} {sys.argv[0]} --setup {args.datadir}?" + sys.exit(f"ERROR: Did you forget to run {cmd}?") else: + args.datadir = None args.setup = True args.serve = True - default = True # DjangoCMS looks for Python on $PATH? _ensure_python_on_PATH() @@ -209,8 +226,9 @@ def add_worker_args(cmd, _): # First, set up the site. if args.setup: sitedir, elapsed = setup(datadir) - print("%.2fs to initialize db" % (elapsed,)) - print(f"site created in {sitedir}") + if args.legacy: + print("%.2fs to initialize db" % (elapsed,)) + print(f"site created in {sitedir}") if not args.serve: print(f"now run {sys.executable} {sys.argv[0]} --serve {datadir}") else: @@ -219,6 +237,11 @@ def add_worker_args(cmd, _): # Then run the benchmark. if args.serve: + if args.legacy: + from legacyutils import maybe_handle_legacy + maybe_handle_legacy(_bench_djangocms_requests, sitedir, legacyarg='legacy') + sys.exit(0) + runner.datadir = datadir def time_func(loops, *args): diff --git a/benchmarks/bm_flaskblogging/legacyutils.py b/benchmarks/bm_flaskblogging/legacyutils.py new file mode 120000 index 0000000..644cca6 --- /dev/null +++ b/benchmarks/bm_flaskblogging/legacyutils.py @@ -0,0 +1 @@ +../.libs/legacyutils.py \ No newline at end of file diff --git a/benchmarks/bm_flaskblogging/run_benchmark.py b/benchmarks/bm_flaskblogging/run_benchmark.py index 589180d..45089e0 100644 --- a/benchmarks/bm_flaskblogging/run_benchmark.py +++ b/benchmarks/bm_flaskblogging/run_benchmark.py @@ -17,6 +17,11 @@ # benchmarks def bench_flask_requests(loops=1800): + elapsed, _ = _bench_flask_requests(loops) + return elapsed + + +def _bench_flask_requests(loops=1800, legacy=False): """Measure N HTTP requests to a local server. Note that the server is freshly started here. @@ -30,17 +35,36 @@ def bench_flask_requests(loops=1800): Hence this should be used with bench_time_func() insted of bench_func(). """ + start = pyperf.perf_counter() elapsed = 0 + times = [] with netutils.serving(ARGV, DATADIR, "127.0.0.1:8000"): requests_get = requests.get - for _ in range(loops): + for i in range(loops): + # This is a macro benchmark for a Python implementation + # so "elapsed" covers more than just how long a request takes. t0 = pyperf.perf_counter() requests_get("http://localhost:8000/blog/").text - elapsed += pyperf.perf_counter() - t0 - return elapsed + t1 = pyperf.perf_counter() + elapsed += t1 - t0 + times.append(t0) + if legacy and (i % 100 == 0): + print(i, t0 - start) + times.append(pyperf.perf_counter()) + if legacy: + total = times[-1] - start + print("%.2fs (%.3freq/s)" % (total, loops / total)) + return elapsed, times + + +############################# +# the script if __name__ == "__main__": + from legacyutils import maybe_handle_legacy + maybe_handle_legacy(_bench_flask_requests, legacyarg='legacy') + runner = pyperf.Runner() runner.metadata['description'] = "Test the performance of flask" runner.bench_time_func("flaskblogging", bench_flask_requests) diff --git a/benchmarks/bm_gevent_hub/run_benchmark.py b/benchmarks/bm_gevent_hub/run_benchmark.py index f928014..4479492 100644 --- a/benchmarks/bm_gevent_hub/run_benchmark.py +++ b/benchmarks/bm_gevent_hub/run_benchmark.py @@ -83,7 +83,9 @@ def bench_switch(loops=1000): for _ in range(loops): t0 = pyperf.perf_counter() child_switch() - elapsed += pyperf.perf_counter() - t0 + t1 = pyperf.perf_counter() + + elapsed += t1 - t0 return elapsed @@ -97,7 +99,9 @@ def bench_wait_ready(loops=1000): for _ in range(loops): t0 = pyperf.perf_counter() hub_wait(watcher) - elapsed += pyperf.perf_counter() - t0 + t1 = pyperf.perf_counter() + + elapsed += t1 - t0 return elapsed @@ -145,9 +149,19 @@ def bench_wait_func_ready(loops=1000): } +############################# +# the script + if __name__ == "__main__": + import sys + if '--legacy' in sys.argv: + for i in range(10000): + bench_switch() + sys.exit(0) + runner = pyperf.Runner() runner.metadata['description'] = "Test the performance of gevent" + runner.argparser.add_argument("--legacy", action='store_true') runner.argparser.add_argument("benchmark", nargs="?", choices=sorted(BENCHMARKS), default="gevent_hub") diff --git a/benchmarks/bm_gunicorn/legacyutils.py b/benchmarks/bm_gunicorn/legacyutils.py new file mode 120000 index 0000000..644cca6 --- /dev/null +++ b/benchmarks/bm_gunicorn/legacyutils.py @@ -0,0 +1 @@ +../.libs/legacyutils.py \ No newline at end of file diff --git a/benchmarks/bm_gunicorn/run_benchmark.py b/benchmarks/bm_gunicorn/run_benchmark.py index f9fae51..41934f7 100644 --- a/benchmarks/bm_gunicorn/run_benchmark.py +++ b/benchmarks/bm_gunicorn/run_benchmark.py @@ -23,7 +23,15 @@ ] +############################# +# benchmarks + def bench_gunicorn(loops=3000): + elapsed, _ = _bench_gunicorn(loops) + return elapsed + + +def _bench_gunicorn(loops=3000, legacy=False): """Measure N HTTP requests to a local server. Note that the server is freshly started here. @@ -37,17 +45,36 @@ def bench_gunicorn(loops=3000): Hence this should be used with bench_time_func() insted of bench_func(). """ + start = pyperf.perf_counter() elapsed = 0 + times = [] with netutils.serving(ARGV, DATADIR, ADDR): requests_get = requests.get - for _ in range(loops): + for i in range(loops): + # This is a macro benchmark for a Python implementation + # so "elapsed" covers more than just how long a request takes. t0 = pyperf.perf_counter() requests_get("http://localhost:8000/blog/").text - elapsed += pyperf.perf_counter() - t0 - return elapsed + t1 = pyperf.perf_counter() + + elapsed += t1 - t0 + times.append(t0) + if legacy and (i % 100 == 0): + print(i, t0 - start) + times.append(pyperf.perf_counter()) + if legacy: + total = times[-1] - start + print("%.2fs (%.3freq/s)" % (total, loops / total)) + return elapsed, times +############################# +# the script + if __name__ == "__main__": + from legacyutils import maybe_handle_legacy + maybe_handle_legacy(_bench_gunicorn, legacyarg='legacy') + runner = pyperf.Runner() runner.metadata['description'] = "Test the performance of gunicorn" runner.bench_time_func("gunicorn", bench_gunicorn) diff --git a/benchmarks/bm_json/legacyutils.py b/benchmarks/bm_json/legacyutils.py new file mode 120000 index 0000000..644cca6 --- /dev/null +++ b/benchmarks/bm_json/legacyutils.py @@ -0,0 +1 @@ +../.libs/legacyutils.py \ No newline at end of file diff --git a/benchmarks/bm_json/run_benchmark.py b/benchmarks/bm_json/run_benchmark.py index 6793f3a..201a0a1 100644 --- a/benchmarks/bm_json/run_benchmark.py +++ b/benchmarks/bm_json/run_benchmark.py @@ -11,7 +11,15 @@ TARGET = os.path.join(DATADIR, "reddit_comments.json") +############################# +# benchmarks + def bench_json_loads(loops=400): + elapsed, _ = _bench_json_loads(loops) + return elapsed + + +def _bench_json_loads(loops=400): """Measure running json.loads() N times. The target data is nearly 1100 JSON objects, each on a single line, @@ -28,22 +36,33 @@ def bench_json_loads(loops=400): """ with open(TARGET) as f: s = f.read() -# data = s.split('\n') - data = s.splitlines() + lines = s.splitlines() elapsed = 0 - json_loads = json.loads + times = [] for _ in range(loops): - for s in data: - if not s: + # This is a macro benchmark for a Python implementation + # so "elapsed" covers more than just how long json.loads() takes. + t0 = pyperf.perf_counter() + for text in lines: + if not text: continue - t0 = pyperf.perf_counter() - json_loads(s) - elapsed += pyperf.perf_counter() - t0 - return elapsed + json.loads(text) + t1 = pyperf.perf_counter() + + elapsed += t1 - t0 + times.append(t0) + times.append(pyperf.perf_counter()) + return elapsed, times +############################# +# the script + if __name__ == "__main__": + from legacyutils import maybe_handle_legacy + maybe_handle_legacy(_bench_json_loads) + runner = pyperf.Runner() runner.metadata['description'] = "Test the performance of json" runner.bench_time_func("json", bench_json_loads) diff --git a/benchmarks/bm_kinto/legacyutils.py b/benchmarks/bm_kinto/legacyutils.py new file mode 120000 index 0000000..644cca6 --- /dev/null +++ b/benchmarks/bm_kinto/legacyutils.py @@ -0,0 +1 @@ +../.libs/legacyutils.py \ No newline at end of file diff --git a/benchmarks/bm_kinto/run_benchmark.py b/benchmarks/bm_kinto/run_benchmark.py index 357c384..33ea3e4 100644 --- a/benchmarks/bm_kinto/run_benchmark.py +++ b/benchmarks/bm_kinto/run_benchmark.py @@ -26,9 +26,15 @@ NGINX_CONF = os.path.join(DATADIR, "nginx.conf") +############################# +# benchmarks + def bench_kinto(loops=5000): - elapsed = 0 + elapsed, _ = _bench_kinto(loops) + return elapsed + +def _bench_kinto(loops=5000, legacy=False): subprocess.check_call( [PYTHON, SETUP_PY, "develop"], cwd=DATADIR, @@ -37,22 +43,42 @@ def bench_kinto(loops=5000): ) cmd_app = [UWSGI, PRODUCTION_INI] - cmd_web = [NGINX, "-c", NGINX_CONF, "-p", DATADIR] with netutils.serving(cmd_app, DATADIR, SOCK, kill=True): + cmd_web = [NGINX, "-c", NGINX_CONF, "-p", DATADIR] with netutils.serving(cmd_web, DATADIR, ADDR, pause=0.010, quiet=False): - print(requests.get("http://localhost:8000/v1").text) - # print(requests.put("http://localhost:8000/v1/accounts/testuser", json={"data": {"password": "password1"}}).text) + if legacy: + print(requests.get("http://localhost:8000/v1").text) + # print(requests.put("http://localhost:8000/v1/accounts/testuser", json={"data": {"password": "password1"}}).text) - for _ in range(loops): + start = pyperf.perf_counter() + elapsed = 0 + times = [] + for i in range(loops): + # This is a macro benchmark for a Python implementation + # so "elapsed" covers more than just how long a request takes. t0 = pyperf.perf_counter() # requests.get("http://localhost:8000/v1/").text urllib.request.urlopen("http://localhost:8000/v1/").read() - elapsed += pyperf.perf_counter() - t0 + t1 = pyperf.perf_counter() + + elapsed += t1 - t0 + times.append(t0) + if legacy and (i % 100 == 0): + print(i, t0 - start) + times.append(pyperf.perf_counter()) + if legacy: + total = times[-1] - start + print("%.2fs (%.3freq/s)" % (total, loops / total)) + return elapsed, times - return elapsed +############################# +# the script if __name__ == "__main__": + from legacyutils import maybe_handle_legacy + maybe_handle_legacy(_bench_kinto, legacyarg='legacy') + if NGINX is None: raise Exception("nginx is not installed") runner = pyperf.Runner() diff --git a/benchmarks/bm_mypy/legacyutils.py b/benchmarks/bm_mypy/legacyutils.py new file mode 120000 index 0000000..644cca6 --- /dev/null +++ b/benchmarks/bm_mypy/legacyutils.py @@ -0,0 +1 @@ +../.libs/legacyutils.py \ No newline at end of file diff --git a/benchmarks/bm_mypy/run_benchmark.py b/benchmarks/bm_mypy/run_benchmark.py index 2bd8526..67e8833 100644 --- a/benchmarks/bm_mypy/run_benchmark.py +++ b/benchmarks/bm_mypy/run_benchmark.py @@ -19,7 +19,15 @@ ] -def bench_mypy(devnull): +############################# +# benchmarks + +def bench_mypy(loops=20): + elapsed, _ = _bench_mypy(loops) + return elapsed + + +def _bench_mypy(loops=20, *, legacy=False): """Meansure running mypy on a file N times. The target file is large (over 2300 lines) with extensive use @@ -29,18 +37,34 @@ def bench_mypy(devnull): the measurement includes the time it takes to read the file from disk. Also, all output is discarded (sent to /dev/null). """ - try: - main(None, devnull, devnull, TARGETS) - except SystemExit: - pass + elapsed = 0 + times = [] + with open(os.devnull, "w") as devnull: + for i in range(loops): + if legacy: + print(i) + # This is a macro benchmark for a Python implementation + # so "elapsed" covers more than just how long main() takes. + t0 = pyperf.perf_counter() + try: + main(None, devnull, devnull, TARGETS) + except SystemExit: + pass + t1 = pyperf.perf_counter() + + elapsed += t1 - t0 + times.append(t0) + times.append(pyperf.perf_counter()) + return elapsed, times +############################# +# the script + if __name__ == "__main__": + from legacyutils import maybe_handle_legacy + maybe_handle_legacy(_bench_mypy, legacyarg='legacy') + runner = pyperf.Runner() runner.metadata['description'] = "Test the performance of mypy types" - runner.argparser.add_argument("loops", nargs="?", type=int, default=1) - args = runner.argparser.parse_args() - - with open(os.devnull, "w") as devnull: - runner.bench_func("mypy", bench_mypy, devnull, - inner_loops=args.loops) + runner.bench_time_func("mypy", bench_mypy) diff --git a/benchmarks/bm_pycparser/legacyutils.py b/benchmarks/bm_pycparser/legacyutils.py new file mode 120000 index 0000000..644cca6 --- /dev/null +++ b/benchmarks/bm_pycparser/legacyutils.py @@ -0,0 +1 @@ +../.libs/legacyutils.py \ No newline at end of file diff --git a/benchmarks/bm_pycparser/run_benchmark.py b/benchmarks/bm_pycparser/run_benchmark.py index 6698625..eca4236 100644 --- a/benchmarks/bm_pycparser/run_benchmark.py +++ b/benchmarks/bm_pycparser/run_benchmark.py @@ -12,22 +12,33 @@ TARGET = os.path.join(DATADIR, "pycparser_target") -def _iter_files(rootdir=TARGET, *, _cache={}): - if not _cache: - files = _cache['files'] = [] - for name in os.listdir(rootdir): - if not name.endswith(".ppout"): - continue - filename = os.path.join(TARGET, name) - with open(filename) as f: - data = (filename, f.read()) - files.append(data) - yield data - else: - yield from _cache['files'] +def _iter_files(rootdir=TARGET): + for name in os.listdir(rootdir): + if not name.endswith(".ppout"): + continue + filename = os.path.join(TARGET, name) + with open(filename) as f: + yield (filename, f.read()) +def parse_files(files): + for _, text in files: + # We use a new parser each time because CParser objects + # aren't designed for re-use. + parser = c_parser.CParser() + ast = parser.parse(text, '') + assert isinstance(ast, c_ast.FileAST) + + +############################# +# benchmarks + def bench_pycparser(loops=20): + elapsed, _ = _bench_pycparser(loops) + return elapsed + + +def _bench_pycparser(loops=20): """Measure running pycparser on several large C files N times. The files are all relatively large, from well-known projects. @@ -39,21 +50,30 @@ def bench_pycparser(loops=20): * reading them from disk * creating the CParser object """ + files = list(_iter_files()) + elapsed = 0 - parse = c_parser.CParser.parse + times = [] for _ in range(loops): - for filename, text in _iter_files(): - # We use a new parser each time because CParser objects - # aren't designed for re-use. - parser = c_parser.CParser() - t0 = pyperf.perf_counter() - ast = parse(parser, text, filename) - elapsed += pyperf.perf_counter() - t0 - assert isinstance(ast, c_ast.FileAST) - return elapsed + times.append(pyperf.perf_counter()) + # This is a macro benchmark for a Python implementation + # so "elapsed" covers more than just how long parser.parse() takes. + t0 = pyperf.perf_counter() + parse_files(files) + t1 = pyperf.perf_counter() + + elapsed += t1 - t0 + times.append(pyperf.perf_counter()) + return elapsed, times +############################# +# the script + if __name__ == "__main__": + from legacyutils import maybe_handle_legacy + maybe_handle_legacy(_bench_pycparser) + runner = pyperf.Runner() runner.metadata['description'] = "Test the performance of pycparser" runner.bench_time_func("pycparser", bench_pycparser) diff --git a/benchmarks/bm_pylint/legacyutils.py b/benchmarks/bm_pylint/legacyutils.py new file mode 120000 index 0000000..644cca6 --- /dev/null +++ b/benchmarks/bm_pylint/legacyutils.py @@ -0,0 +1 @@ +../.libs/legacyutils.py \ No newline at end of file diff --git a/benchmarks/bm_pylint/run_benchmark.py b/benchmarks/bm_pylint/run_benchmark.py index 5e95041..40ad2c8 100644 --- a/benchmarks/bm_pylint/run_benchmark.py +++ b/benchmarks/bm_pylint/run_benchmark.py @@ -14,7 +14,25 @@ ] +def noop(*args, **kw): + pass + + +class NullReporter: + path_strip_prefix = "/" + def __getattr__(self, attr): + return noop + + +############################# +# benchmarks + def bench_pylint(loops=10): + elapsed, _ = _bench_pylint(loops) + return elapsed + + +def _bench_pylint(loops=10): """Measure running pylint on a file N times. The target file is a relatively large, complex one copied @@ -23,22 +41,30 @@ def bench_pylint(loops=10): pylint seems to speed up considerably as it progresses, and this benchmark includes that. """ - class NullReporter: - path_strip_prefix = "/" - def __getattr__(self, attr, _noop=(lambda *a, **k: None)): - return _noop - reporter = NullReporter() - elapsed = 0 - _run = Run - for _ in range(loops): + times = [] + for i in range(loops): + print(i) + # This is a macro benchmark for a Python implementation + # so "elapsed" covers more than just how long Run() takes. t0 = pyperf.perf_counter() - _run(TARGETS, exit=False, reporter=reporter) - elapsed += pyperf.perf_counter() - t0 - return elapsed + reporter = NullReporter() + Run(TARGETS, exit=False, reporter=reporter) + t1 = pyperf.perf_counter() + elapsed += t1 - t0 + times.append(t0) + times.append(pyperf.perf_counter()) + return elapsed, times + + +############################# +# the script if __name__ == "__main__": + from legacyutils import maybe_handle_legacy + maybe_handle_legacy(_bench_pylint) + runner = pyperf.Runner() runner.metadata['description'] = "Test the performance of pylint" runner.bench_time_func("pylint", bench_pylint) diff --git a/benchmarks/bm_pytorch_alexnet_inference/legacyutils.py b/benchmarks/bm_pytorch_alexnet_inference/legacyutils.py new file mode 120000 index 0000000..644cca6 --- /dev/null +++ b/benchmarks/bm_pytorch_alexnet_inference/legacyutils.py @@ -0,0 +1 @@ +../.libs/legacyutils.py \ No newline at end of file diff --git a/benchmarks/bm_pytorch_alexnet_inference/run_benchmark.py b/benchmarks/bm_pytorch_alexnet_inference/run_benchmark.py index 45e3184..5c46955 100644 --- a/benchmarks/bm_pytorch_alexnet_inference/run_benchmark.py +++ b/benchmarks/bm_pytorch_alexnet_inference/run_benchmark.py @@ -26,6 +26,11 @@ # benchmarks def bench_pytorch(loops=1000): + elapsed, _ = _bench_pytorch(loops) + return elapsed + + +def _bench_pytorch(loops=1000, *, legacy=False): """Measure using pytorch to transform an image N times. This involves the following steps: @@ -39,7 +44,9 @@ def bench_pytorch(loops=1000): Only that last step is measured (and repeated N times). """ + start = pyperf.perf_counter() model = torch.hub.load('pytorch/vision:v0.6.0', 'alexnet', pretrained=True) + # assert pyperf.perf_counter() - start < 3, "looks like we just did the first-time download, run this benchmark again to get a clean run" model.eval() urllib.request.urlretrieve(URL, FILENAME) @@ -51,18 +58,33 @@ def bench_pytorch(loops=1000): transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) input_tensor = preprocess(input_image) - input_batch = input_tensor.unsqueeze(0) # create a mini-batch as expected by the model + input_batch = input_tensor.unsqueeze(0) # create a mini-batch as expected by the model with torch.no_grad(): elapsed = 0 - for _ in range(loops): + times = [] + for i in range(loops): + if legacy and (i % 10 == 0): + print(i) + # This is a macro benchmark for a Python implementation + # so "elapsed" covers more than just how long model() takes. t0 = pyperf.perf_counter() output = model(input_batch) - elapsed += pyperf.perf_counter() - t0 - return elapsed + t1 = pyperf.perf_counter() + + elapsed += t1 - t0 + times.append(t0) + times.append(pyperf.perf_counter()) + return elapsed, times +############################# +# the script + if __name__ == "__main__": + from legacyutils import maybe_handle_legacy + maybe_handle_legacy(_bench_pytorch, legacyarg='legacy') + runner = pyperf.Runner() runner.metadata['description'] = "Test the performance of pytorch" runner.bench_time_func("pytorch", bench_pytorch) diff --git a/benchmarks/bm_thrift/legacyutils.py b/benchmarks/bm_thrift/legacyutils.py new file mode 120000 index 0000000..644cca6 --- /dev/null +++ b/benchmarks/bm_thrift/legacyutils.py @@ -0,0 +1 @@ +../.libs/legacyutils.py \ No newline at end of file diff --git a/benchmarks/bm_thrift/run_benchmark.py b/benchmarks/bm_thrift/run_benchmark.py index ade0d89..9b45efd 100644 --- a/benchmarks/bm_thrift/run_benchmark.py +++ b/benchmarks/bm_thrift/run_benchmark.py @@ -19,7 +19,37 @@ TARGET = os.path.join(DATADIR, "thrift") +if TARGET not in sys.path: + sys.path.insert(0, TARGET) +from addressbook import ttypes + + +def make_addressbook(): + phone1 = ttypes.PhoneNumber() + phone1.type = ttypes.PhoneType.MOBILE + phone1.number = '555-1212' + phone2 = ttypes.PhoneNumber() + phone2.type = ttypes.PhoneType.HOME + phone2.number = '555-1234' + person = ttypes.Person() + person.name = "Alice" + person.phones = [phone1, phone2] + person.created_at = 1400000000 + + ab = ttypes.AddressBook() + ab.people = {person.name: person} + return ab + + +############################# +# benchmarks + def bench_thrift(loops=1000): + elapsed, _ = _bench_thrift(loops) + return elapsed + + +def _bench_thrift(loops=1000): """Measure using a thrift-generated library N times. The target is a simple addressbook. We measure the following: @@ -30,44 +60,37 @@ def bench_thrift(loops=1000): For each iteration we repeat this 100 times. """ - sys.path.insert(0, TARGET) - from addressbook import ttypes - - elapsed = 0 # proto_factory = TBinaryProtocolFactory() proto_factory = TBinaryProtocolAcceleratedFactory() - _serialize = serialize - _deserialize = deserialize - _AddressBook = ttypes.AddressBook - _Person = ttypes.Person - _PhoneNumber = ttypes.PhoneNumber - MOBILE = ttypes.PhoneType.MOBILE - HOME = ttypes.PhoneType.HOME + + elapsed = 0 + times = [] for _ in range(loops): + # This is a macro benchmark for a Python implementation + # so "elapsed" covers more than just how long the Addressbook ops take. t0 = pyperf.perf_counter() for _ in range(100): # First, create the addressbook. - ab = _AddressBook() - phone1 = _PhoneNumber() - phone1.type = MOBILE - phone1.number = '555-1212' - phone2 = _PhoneNumber() - phone2.type = HOME - phone2.number = '555-1234' - person = _Person() - person.name = "Alice" - person.phones = [phone1, phone2] - person.created_at = 1400000000 - ab.people = {person.name: person} + ab = make_addressbook() # Then, round-trip through serialization. - encoded = _serialize(ab, proto_factory) - ab2 = _AddressBook() - _deserialize(ab2, encoded, proto_factory) - elapsed += pyperf.perf_counter() - t0 - return elapsed + encoded = serialize(ab, proto_factory) + ab2 = ttypes.AddressBook() + deserialize(ab2, encoded, proto_factory) + t1 = pyperf.perf_counter() + + elapsed += t1 - t0 + times.append(t0) + times.append(pyperf.perf_counter()) + return elapsed, times +############################# +# the script + if __name__ == "__main__": + from legacyutils import maybe_handle_legacy + maybe_handle_legacy(_bench_thrift) + runner = pyperf.Runner() runner.metadata['description'] = "Test the performance of thrift" runner.bench_time_func("thrift", bench_thrift) diff --git a/run_all.sh b/run_all.sh index daa323e..d4924ab 100755 --- a/run_all.sh +++ b/run_all.sh @@ -7,11 +7,29 @@ fi BINARY=$1 +set -u +set -x + +mkdir -p results + ENV=/tmp/macrobenchmark_env -./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks flaskblogging --clean -# XXX Convert results to verbose "time" output. -for bench in djangocms mypy_bench pylint_bench pycparser_bench pytorch_alexnet_inference gunicorn aiohttp thrift_bench gevent_bench_hub kinto_bench; do - #/usr/bin/time --verbose --output=results/${bench}.out $ENV/bin/python $(dirname $0)/benchmarks/${bench}.py - ./run_benchmarks.sh --python $BINARY --venv $ENV --benchmarks $bench --no-clean - # XXX Convert results to verbose "time" output. +for bench in kinto; do +#for bench in flaskblogging djangocms mypy pylint pycparser pytorch_alexnet_inference gunicorn aiohttp thrift gevent_hub kinto; do + case $bench in + gevent_hub) + outname=gevent_bench_hub + ;; + mypy|pylint|pycparser|thrift|kinto) + outname=${bench}_bench + ;; + *) + outname=$bench + ;; + esac + + rm -rf $ENV + $BINARY -m venv $ENV + $ENV/bin/pip install pyperf==2.2.0 + $ENV/bin/pip install -r $(dirname $0)/benchmarks/bm_${bench}/requirements.txt + /usr/bin/time --verbose --output=results/${outname}.out $ENV/bin/python $(dirname $0)/benchmarks/bm_${bench}/run_benchmark.py --legacy done diff --git a/run_benchmarks.sh b/run_benchmarks.sh deleted file mode 100755 index 5af3759..0000000 --- a/run_benchmarks.sh +++ /dev/null @@ -1,260 +0,0 @@ -#!/usr/bin/env bash - -set -e - -function divider() { - local title=$1 - echo - echo '##################################################' - echo "# $title" - echo '##################################################' - echo -} - -function verbose() { - ( - set -x - "$@" - ) -} - -function rotate() { - local force='no' - if [ "$1" = '--force' ]; then - shift - force='yes' - fi - local oldfile=$1 - local newfile=$oldfile.bak - if [ -e $newfile ]; then - if [ $force = 'no' ]; then - >&2 echo "ERROR: $newfile already exists" - exit 1 - fi - verbose rm $newfile - fi - verbose mv $oldfile $newfile -} - -now=$(date --utc +'%Y%m%d-%H%M%S') - - -# Extract values from env vars. -pyperformance=pyperformance -clone_pp='no' -if [ -n "$PYPERFORMANCE" ]; then - pyperformance=$PYPERFORMANCE - if [ ! -e $pyperformance ]; then - clone_pp='yes' - fi -fi - - -set -u - - -# Parse the command-line. -target_python= -venv= -reset_venv='no' -manifest= -outfile= -benchmarks= -mypy='no' -clean= -skip_setup='no' -argv=() -while [ $# -gt 0 ]; do - arg=$1 - shift - case "$arg" in - -p|--python) - target_python=$1 - shift - ;; - --venv) - venv=$1 - shift - ;; - --manifest) - manifest=$1 - shift - ;; - -o|--output) - outfile=$1 - shift - ;; - -b|--benchmarks) - benchmarks=$1 - shift - ;; - --with-mypyc) - mypy='yes' - if [ $# -gt 0 ]; then - case $1 in - -*) - ;; - *) - mypy=$1 - shift - ;; - esac - fi - ;; - --clean) - if [ "$skip_setup" = 'yes' ]; then - >&2 echo "ERROR: got --clean and --skip-setup" - exit 1 - fi - clean='yes' - ;; - --no-clean) - clean='no' - ;; - --skip-setup) - if [ "$clean" = 'yes' ]; then - >&2 echo "ERROR: got --clean and --skip-setup" - exit 1 - fi - skip_setup='yes' - clean='no' - ;; - *) - argv+=("$arg") - ;; - esac -done - -if [ -z "$target_python" ]; then - target_python=$venv/bin/python3 - if [ -z "$venv" -o ! -e "$target_python" ]; then - >&2 echo "ERROR: missing -p/--python arg" - exit 1 - fi -elif [ -z "$venv" ]; then - venv=venv/pyston-python-macrobenchmarks - reset_venv='yes' -elif [ ! -e "$venv" ]; then - reset_venv='yes' - >&2 echo "WARNING: requested venv does not exist but will be created" -elif [ "$(realpath "$venv/bin/python3")" != "$target_python" ]; then - reset_venv='yes' - >&2 echo "WARNING: requested venv is outdated and will be reset" -fi -if [ -z "$manifest" ]; then - manifest=benchmarks/MANIFEST -fi -if [ -z "$outfile" ]; then - outdir=results - outfile=$outdir/results-$now.json -elif [ -d $outfile ]; then - outdir=$outfile - outfile=$outdir/results-$now.json -else - outdir=$(dirname "$outfile") - if [ -z "$outdir" ]; then - outdir='.' - outfile=./$outfile - fi -fi -if [ -z "$benchmarks" ]; then - if [ "$mypy" != "no" ]; then - benchmarks='mypyc' - else - benchmarks='default' - fi -else - case $benchmarks in - *mypyc*) - if [ "$mypy" = 'no' ]; then - mypy='yes' - fi - ;; - *mypy*) - benchmarks=${benchmarks/mypy/mypyc} - ;; - esac -fi -MYPY_REPO_ROOT=/tmp/mypy -if [ -z "$mypy" -o "$mypy" = 'yes' ]; then - mypy=MYPY_REPO_ROOT - reset_mypy='yes' -elif [ "$mypy" == 'no' ]; then - mypy= - reset_mypy='no' -elif [ ! -e "$mypy" ]; then - reset_mypy='yes' -else - reset_mypy='no' -fi -if [ "$clean" = 'yes' ]; then - if [ "$skip_setup" != 'no' ]; then - >&2 echo "ERROR: got --clean and --skip-setup" - exit 1 - fi - reset_mypy='yes' - reset_venv='yes' -fi - - -# Set up the execution environment. -if [ $skip_setup = 'no' ]; then - if [ $reset_venv = 'yes' ]; then - divider "creating the top-level venv at $venv" - if [ -e "$venv" ]; then - #verbose rm -rf $venv - rotate --force "$venv" - fi - verbose "$target_python" -m venv "$venv" - fi - divider "ensuring setuptools is up-to-date in $venv" - verbose "$venv/bin/pip" install --upgrade setuptools - - if [ $clone_pp = 'yes' ]; then - divider "preparing pyperformance at $pyperformance" - if [ -e "$pyperformance" ]; then - rotate "$pyperformance" - fi - verbose git clone https://github.com/python/pyperformance "$pyperformance" - fi - if [ "$pyperformance" = 'pyperformance' ]; then - divider "installing pyperformance from PyPI" - verbose "$venv/bin/pip" install --upgrade "$pyperformance" - else - divider "installing pyperformance into $venv from $pyperformance" - verbose "$venv/bin/pip" install --force-reinstall "$pyperformance" - fi - - if [ -n "$mypy" ]; then - if [ $reset_mypy = 'yes' ]; then - divider "getting a fresh copy of the mypy repo" - if [ -e "$mypy" ]; then - #verbose rm -rf $mypy - rotate "$mypy" - fi - verbose git clone --depth 1 --branch v0.790 https://github.com/python/mypy/ "$mypy" - pushd "$mypy" - verbose git submodule update --init mypy/typeshed - popd - fi - pushd "$mypy" - divider "installing the mypy requirements into $venv" - verbose "$venv/bin/pip" install -r mypy-requirements.txt - divider "building mypyc and installing it in $venv" - verbose "$venv/bin/python3" setup.py --use-mypyc install - popd - fi - - divider "other setup" - verbose mkdir -p "$outdir" -fi - - -# Run the benchmarks. -divider "running the benchmarks" -verbose "$venv/bin/python3" -m pyperformance run \ - --venv "$venv" \ - --manifest "$manifest" \ - --benchmarks "$benchmarks" \ - --output "$outfile" \ - "${argv[@]}" diff --git a/run_mypy.sh b/run_mypy.sh old mode 100755 new mode 100644 index f4820b3..d030b5f --- a/run_mypy.sh +++ b/run_mypy.sh @@ -13,13 +13,25 @@ fi BINARY=$1 +set -u +set -x + ENV=/tmp/macrobenchmark_env -#time $ENV/bin/python benchmarks/mypy_bench.py 50 -#time $ENV/bin/python benchmarks/mypy_bench.py 50 -#time $ENV/bin/python benchmarks/mypy_bench.py 50 -./run_benchmarks.sh --python $BINARY --venv $ENV --with-mypyc --clean -# XXX Convert results to verbose "time" output. -./run_benchmarks.sh --python $BINARY --venv $ENV --with-mypyc --skip-setup -# XXX Convert results to verbose "time" output. -./run_benchmarks.sh --python $BINARY --venv $ENV --with-mypyc --skip-setup -# XXX Convert results to verbose "time" output. +rm -rf $ENV +virtualenv -p $BINARY $ENV + +rm -rf /tmp/mypy +git clone --depth 1 --branch v0.790 https://github.com/python/mypy/ /tmp/mypy +cd /tmp/mypy + +$ENV/bin/pip install pyperf==2.2.0 +$ENV/bin/pip install -r mypy-requirements.txt +$ENV/bin/pip install --upgrade setuptools +git submodule update --init mypy/typeshed +$ENV/bin/python setup.py --use-mypyc install + +cd - +time $ENV/bin/python benchmarks/bm_mypy/run_benchmark.py --legacy 50 +time $ENV/bin/python benchmarks/bm_mypy/run_benchmark.py --legacy 50 +time $ENV/bin/python benchmarks/bm_mypy/run_benchmark.py --legacy 50 + From c533adb99d9684e8335bc00ec5b01a7abb2fdc57 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 6 Dec 2021 17:35:54 -0700 Subject: [PATCH 29/32] Fix bm_kinto. --- benchmarks/bm_kinto/run_benchmark.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/benchmarks/bm_kinto/run_benchmark.py b/benchmarks/bm_kinto/run_benchmark.py index 33ea3e4..97122d3 100644 --- a/benchmarks/bm_kinto/run_benchmark.py +++ b/benchmarks/bm_kinto/run_benchmark.py @@ -18,7 +18,7 @@ ADDR = "127.0.0.1:8000" DATADIR = os.path.join( - os.path.dirname(__file__), + os.path.abspath(os.path.dirname(__file__)), "data", ) SETUP_PY = os.path.join(DATADIR, "setup.py") @@ -35,12 +35,16 @@ def bench_kinto(loops=5000): def _bench_kinto(loops=5000, legacy=False): - subprocess.check_call( - [PYTHON, SETUP_PY, "develop"], + cmd = [PYTHON, SETUP_PY, "develop"] + proc = subprocess.run( + cmd, cwd=DATADIR, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, ) + if proc.returncode != 0: + print(f'# running: {" ".join(cmd)} (in {DATADIR})') + subprocess.run(cmd, cwd=DATADIR, check=True) cmd_app = [UWSGI, PRODUCTION_INI] with netutils.serving(cmd_app, DATADIR, SOCK, kill=True): From 7523c3f05db507a790ee6c725ca41b7ee6b6d6ba Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 6 Dec 2021 17:36:24 -0700 Subject: [PATCH 30/32] Run all the benchmarks. --- run_all.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/run_all.sh b/run_all.sh index d4924ab..c1eeafb 100755 --- a/run_all.sh +++ b/run_all.sh @@ -13,8 +13,7 @@ set -x mkdir -p results ENV=/tmp/macrobenchmark_env -for bench in kinto; do -#for bench in flaskblogging djangocms mypy pylint pycparser pytorch_alexnet_inference gunicorn aiohttp thrift gevent_hub kinto; do +for bench in flaskblogging djangocms mypy pylint pycparser pytorch_alexnet_inference gunicorn aiohttp thrift gevent_hub kinto; do case $bench in gevent_hub) outname=gevent_bench_hub From 9828f98a5b7e3f4c9dc23f975b858f606fd68570 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 20 Jan 2022 13:51:40 -0700 Subject: [PATCH 31/32] Use an absolute path in the README. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f42abf7..a7b9ef0 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A collection of macro benchmarks for the Python programming language ```shell # Run the default benchmarks: -python3 -m pyperformance run --manifest ./benchmarks/MANIFEST +python3 -m pyperformance run --manifest $PWD/benchmarks/MANIFEST ``` The benchmarks can still be run without pyperformance. This will produce From 766642a5b2e6add317da94829cf55e7c5d1b3d58 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 20 Jan 2022 13:52:50 -0700 Subject: [PATCH 32/32] Add a missing import. --- benchmarks/.libs/legacyutils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/.libs/legacyutils.py b/benchmarks/.libs/legacyutils.py index d232c46..c91e136 100644 --- a/benchmarks/.libs/legacyutils.py +++ b/benchmarks/.libs/legacyutils.py @@ -1,3 +1,4 @@ +import json import sys