From bffa11af17e065133f8e5e165520f06955912fee Mon Sep 17 00:00:00 2001
From: brettlangdon <brett.langdon@datadoghq.com>
Date: Tue, 23 Jun 2020 19:17:27 -0400
Subject: [PATCH 1/3] Precompile common regular expressions
For very large tox.ini configurations there is a lot of time spent calling re._compile
by precompiling these common/static regular expressions I was able to get our
`tox -l` run from 7.79s to 6.63s
---
src/tox/config/__init__.py | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py
index 2ac4cf3aa..f963733e9 100644
--- a/src/tox/config/__init__.py
+++ b/src/tox/config/__init__.py
@@ -57,6 +57,11 @@
INTERRUPT_TIMEOUT = 0.3
TERMINATE_TIMEOUT = 0.2
+_FACTOR_LINE_PATTERN = re.compile(r"^([\w{}\.!,-]+)\:\s+(.+)")
+_ENVSTR_SPLIT_PATTERN = re.compile(r"((?:\{[^}]+\})+)|,")
+_ENVSTR_EXPAND_PATTERN = re.compile(r"\{([^}]+)\}")
+_WHITESPACE_PATTERN = re.compile(r"\s+")
+
def get_plugin_manager(plugins=()):
# initialize plugin manager
@@ -1438,12 +1443,12 @@ def _split_factor_expr_all(expr):
def _expand_envstr(envstr):
# split by commas not in groups
- tokens = re.split(r"((?:\{[^}]+\})+)|,", envstr)
+ tokens = _ENVSTR_SPLIT_PATTERN.split(envstr)
envlist = ["".join(g).strip() for k, g in itertools.groupby(tokens, key=bool) if k]
def expand(env):
- tokens = re.split(r"\{([^}]+)\}", env)
- parts = [re.sub(r"\s+", "", token).split(",") for token in tokens]
+ tokens = _ENVSTR_EXPAND_PATTERN.split(env)
+ parts = [_WHITESPACE_PATTERN.sub("", token).split(",") for token in tokens]
return ["".join(variant) for variant in itertools.product(*parts)]
return mapcat(expand, envlist)
@@ -1607,7 +1612,7 @@ def _replace_if_needed(self, x, name, replace, crossonly):
def _apply_factors(self, s):
def factor_line(line):
- m = re.search(r"^([\w{}\.!,-]+)\:\s+(.+)", line)
+ m = _FACTOR_LINE_PATTERN.search(line)
if not m:
return line
From 35ed510d80e0066a27b8ff5a0073792eecc2e3a4 Mon Sep 17 00:00:00 2001
From: brettlangdon <brett.langdon@datadoghq.com>
Date: Tue, 23 Jun 2020 19:18:46 -0400
Subject: [PATCH 2/3] Add Brett Langdon to CONTRIBUTORS
---
CONTRIBUTORS | 1 +
1 file changed, 1 insertion(+)
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 80dda65ca..76b744e67 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -16,6 +16,7 @@ Barry Warsaw
Bartolome Sanchez Salado
Benoit Pierre
Bernat Gabor
+Brett Langdon
Bruno Oliveira
Carl Meyer
Charles Brunet
From 542d76ac90b4f15f3986b1bcfb1a1c3d79ab5cd0 Mon Sep 17 00:00:00 2001
From: brettlangdon <brett.langdon@datadoghq.com>
Date: Tue, 23 Jun 2020 19:31:15 -0400
Subject: [PATCH 3/3] Add 1603 changelog
---
docs/changelog/1603.misc.rst | 1 +
1 file changed, 1 insertion(+)
create mode 100644 docs/changelog/1603.misc.rst
diff --git a/docs/changelog/1603.misc.rst b/docs/changelog/1603.misc.rst
new file mode 100644
index 000000000..56ab17c97
--- /dev/null
+++ b/docs/changelog/1603.misc.rst
@@ -0,0 +1 @@
+Improve config parsing performance by precompiling commonly used regular expressions - by :user:`brettlangdon`