Fix nondeterminism in fixture collection order

fixtures.reorder_items is non-deterministic because it reorders based
on iteration over an (unordered) set.  Change the code to use an
OrderedDict instead, so that we get deterministic behaviour.

Author: wence-

AUTHORS

@@ -91,6 +91,7 @@ Kale Kundert
 Katarzyna Jachim
 Kevin Cox
 Kodi B. Arfer
+Lawrence Mitchell
 Lee Kamentsky
 Lev Maximov
 Llandy Riveron Del Risco

_pytest/fixtures.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import, division, print_function
 import sys
 
+from collections import OrderedDict
 from py._code.code import FormattedExcinfo
 
 import py
@@ -136,10 +137,10 @@ def get_parametrized_fixture_keys(item, scopenum):
     except AttributeError:
         pass
     else:
-        # cs.indictes.items() is random order of argnames but
-        # then again different functions (items) can change order of
-        # arguments so it doesn't matter much probably
-        for argname, param_index in cs.indices.items():
+        # cs.indices.items() is random order of argnames.  Need to
+        # sort this so that different calls to
+        # get_parametrized_fixture_keys will be deterministic.
+        for argname, param_index in sorted(cs.indices.items()):
             if cs._arg2scopenum[argname] != scopenum:
                 continue
             if scopenum == 0:    # session
@@ -161,7 +162,7 @@ def reorder_items(items):
     for scopenum in range(0, scopenum_function):
         argkeys_cache[scopenum] = d = {}
         for item in items:
-            keys = set(get_parametrized_fixture_keys(item, scopenum))
+            keys = OrderedDict.fromkeys(get_parametrized_fixture_keys(item, scopenum))
             if keys:
                 d[item] = keys
     return reorder_items_atscope(items, set(), argkeys_cache, 0)
@@ -196,9 +197,9 @@ def slice_items(items, ignore, scoped_argkeys_cache):
         for i, item in enumerate(it):
             argkeys = scoped_argkeys_cache.get(item)
             if argkeys is not None:
-                argkeys = argkeys.difference(ignore)
-                if argkeys:  # found a slicing key
-                    slicing_argkey = argkeys.pop()
+                newargkeys = OrderedDict.fromkeys(k for k in argkeys if k not in ignore)
+                if newargkeys:  # found a slicing key
+                    slicing_argkey, _ = newargkeys.popitem()
                     items_before = items[:i]
                     items_same = [item]
                     items_other = []

changelog/920.bugfix

@@ -0,0 +1 @@
+Fix non-determinism in order of fixture collection.