Skip to content

Split Session._collection_node_cache to 3 mutually exclusive parts #6555

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jan 28, 2020
Merged

Split Session._collection_node_cache to 3 mutually exclusive parts #6555

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 28 additions & 14 deletions src/_pytest/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from typing import Dict
from typing import FrozenSet
from typing import List
from typing import Sequence
from typing import Tuple

import attr
Expand All @@ -21,13 +22,15 @@
from _pytest.config import hookimpl
from _pytest.config import UsageError
from _pytest.fixtures import FixtureManager
from _pytest.nodes import Node
from _pytest.outcomes import exit
from _pytest.reports import CollectReport
from _pytest.runner import collect_one_node
from _pytest.runner import SetupState


if TYPE_CHECKING:
from typing import Type

from _pytest.python import Package


Expand Down Expand Up @@ -407,7 +410,16 @@ def __init__(self, config: Config) -> None:
self._initialpaths = frozenset() # type: FrozenSet[py.path.local]

# Keep track of any collected nodes in here, so we don't duplicate fixtures
self._collection_node_cache = {} # type: Dict[str, List[Node]]
self._collection_node_cache1 = (
{}
) # type: Dict[py.path.local, Sequence[nodes.Collector]]
self._collection_node_cache2 = (
{}
) # type: Dict[Tuple[Type[nodes.Collector], py.path.local], nodes.Collector]
self._collection_node_cache3 = (
{}
) # type: Dict[Tuple[Type[nodes.Collector], str], CollectReport]

# Dirnames of pkgs with dunder-init files.
self._collection_pkg_roots = {} # type: Dict[py.path.local, Package]

Expand Down Expand Up @@ -525,7 +537,9 @@ def collect(self):
self._notfound.append((report_arg, sys.exc_info()[1]))

self.trace.root.indent -= 1
self._collection_node_cache.clear()
self._collection_node_cache1.clear()
self._collection_node_cache2.clear()
self._collection_node_cache3.clear()
self._collection_pkg_roots.clear()

def _collect(self, argpath, names):
Expand All @@ -543,13 +557,13 @@ def _collect(self, argpath, names):
if parent.isdir():
pkginit = parent.join("__init__.py")
if pkginit.isfile():
if pkginit not in self._collection_node_cache:
if pkginit not in self._collection_node_cache1:
col = self._collectfile(pkginit, handle_dupes=False)
if col:
if isinstance(col[0], Package):
self._collection_pkg_roots[parent] = col[0]
# always store a list in the cache, matchnodes expects it
self._collection_node_cache[col[0].fspath] = [col[0]]
self._collection_node_cache1[col[0].fspath] = [col[0]]

# If it's a directory argument, recurse and look for any Subpackages.
# Let the Package collector deal with subnodes, don't collect here.
Expand All @@ -576,21 +590,21 @@ def _collect(self, argpath, names):

for x in self._collectfile(path):
key = (type(x), x.fspath)
if key in self._collection_node_cache:
yield self._collection_node_cache[key]
if key in self._collection_node_cache2:
yield self._collection_node_cache2[key]
else:
self._collection_node_cache[key] = x
self._collection_node_cache2[key] = x
yield x
else:
assert argpath.check(file=1)

if argpath in self._collection_node_cache:
col = self._collection_node_cache[argpath]
if argpath in self._collection_node_cache1:
col = self._collection_node_cache1[argpath]
else:
collect_root = self._collection_pkg_roots.get(argpath.dirname, self)
col = collect_root._collectfile(argpath, handle_dupes=False)
if col:
self._collection_node_cache[argpath] = col
self._collection_node_cache1[argpath] = col
m = self.matchnodes(col, names)
# If __init__.py was the only file requested, then the matched node will be
# the corresponding Package, and the first yielded item will be the __init__
Expand Down Expand Up @@ -703,11 +717,11 @@ def _matchnodes(self, matching, names):
continue
assert isinstance(node, nodes.Collector)
key = (type(node), node.nodeid)
if key in self._collection_node_cache:
rep = self._collection_node_cache[key]
if key in self._collection_node_cache3:
rep = self._collection_node_cache3[key]
else:
rep = collect_one_node(node)
self._collection_node_cache[key] = rep
self._collection_node_cache3[key] = rep
if rep.passed:
has_matched = False
for x in rep.result:
Expand Down