Merge pull request #11 from jbasko/dev

Seed

Author: jbasko

README.rst

@@ -5,8 +5,9 @@ pytest-random-order
     :target: https://travis-ci.org/jbasko/pytest-random-order
 
 pytest-random-order is a plugin for `pytest <http://pytest.org>`_ that randomises the order in which
-tests are run to reveal unwanted coupling between tests. The plugin allows user to control the level
+tests are run to reveal unwanted coupling between tests. It allows user to control the level
 of randomness they want to introduce and to disable reordering on subsets of tests.
+Tests can be rerun in a specific order by passing a seed value reported in a previous test run.
 
 
 Quick Start
@@ -36,6 +37,13 @@ To disable reordering of tests in a module or class, use pytest marker notation:
 
     pytestmark = pytest.mark.random_order(disabled=True)
 
+To rerun tests in a particular order:
+
+::
+
+    $ pytest -v --random-order-seed=<value-reported-in-previous-run>
+
+
 Design
 ------
 
@@ -117,6 +125,28 @@ No matter what will be the bucket type for the test run, ``test_number_one`` wil
 before ``test_number_two``.
 
 
+Rerun Tests in the Same Order (Same Seed)
++++++++++++++++++++++++++++++++++++++++++
+
+If you discover a failing test because you reordered tests, you will probably want to be able to rerun the tests
+in the same failing order. To allow reproducing test order, the plugin reports the seed value it used with pseudo random number
+generator:
+
+::
+
+    ============================= test session starts ==============================
+    ..
+    Using --random-order-bucket=module
+    Using --random-order-seed=24775
+    ...
+
+You can now the ``--random-order-seed=...`` bit as an argument to the next run to produce the same order:
+
+::
+
+    $ pytest -v --random-order-seed=24775
+
+
 Disable the Plugin
 ++++++++++++++++++
 

pytest_random_order/plugin.py

@@ -1,3 +1,4 @@
+import random
 import sys
 import traceback
 
@@ -14,18 +15,34 @@ def pytest_addoption(parser):
         choices=('global', 'package', 'module', 'class'),
         help='Limit reordering of test items across units of code',
     )
+    group.addoption(
+        '--random-order-seed',
+        action='store',
+        dest='random_order_seed',
+        default=None,
+        help='Seed for the test order randomiser to produce a random order that can be reproduced using this seed',
+    )
 
 
 def pytest_configure(config):
     config.addinivalue_line("markers", "random_order(disabled=True): disable reordering of tests within a module or class")
 
+    if config.getoption('random_order_seed'):
+        seed = str(config.getoption('random_order_seed'))
+    else:
+        seed = str(random.randint(1, 1000000))
+    config.random_order_seed = seed
+
 
 def pytest_report_header(config):
-    out = None
+    out = ''
 
     if config.getoption('random_order_bucket'):
         bucket = config.getoption('random_order_bucket')
-        out = "Using --random-order-bucket={0}".format(bucket)
+        out += "Using --random-order-bucket={}\n".format(bucket)
+
+    if hasattr(config, 'random_order_seed'):
+        out += 'Using --random-order-seed={}\n'.format(getattr(config, 'random_order_seed'))
 
     return out
 
@@ -36,8 +53,9 @@ def pytest_collection_modifyitems(session, config, items):
     item_ids = _get_set_of_item_ids(items)
 
     try:
+        seed = getattr(config, 'random_order_seed', None)
         bucket_type = config.getoption('random_order_bucket')
-        _shuffle_items(items, bucket_key=_random_order_item_keys[bucket_type], disable=_disable)
+        _shuffle_items(items, bucket_key=_random_order_item_keys[bucket_type], disable=_disable, seed=seed)
 
     except Exception as e:
         # See the finally block -- we only fail if we have lost user's tests.

pytest_random_order/shuffler.py

@@ -19,7 +19,7 @@
 ItemKey.__new__.__defaults__ = (None, None)
 
 
-def _shuffle_items(items, bucket_key=None, disable=None, _shuffle_buckets=True):
+def _shuffle_items(items, bucket_key=None, disable=None, seed=None):
     """
     Shuffles a list of `items` in place.
 
@@ -31,26 +31,15 @@ def _shuffle_items(items, bucket_key=None, disable=None, _shuffle_buckets=True):
     Bucket defines the boundaries across which items will not
     be shuffled.
 
-    If `disable` is function and returns True for ALL items
-    in a bucket, items in this bucket will remain in their original order.
-
-    `_shuffle_buckets` is for testing only. Setting it to False may not produce
-    the outcome you'd expect in all scenarios because if two non-contiguous sections of items belong
-    to the same bucket, the items in these sections will be reshuffled as if they all belonged
-    to the first section.
-        Example:
-            [A1, A2, B1, B2, A3, A4]
-
-            where letter denotes bucket key,
-            with _shuffle_buckets=False may be reshuffled to:
-                [B2, B1, A3, A1, A4, A2]
-
-            or as well to:
-                [A3, A2, A4, A1, B1, B2]
-
-            because all A's belong to the same bucket and will be grouped together.
+    `disable` is a function that takes an item and returns a falsey value
+    if this item is ok to be shuffled. It returns a truthy value otherwise and
+    the truthy value is used as part of the item's key when determining the bucket
+    it belongs to.
     """
 
+    if seed is not None:
+        random.seed(seed)
+
     # If `bucket_key` is falsey, shuffle is global.
     if not bucket_key and not disable:
         random.shuffle(items)

setup.py

@@ -13,7 +13,7 @@ def read(fname):
 
 setup(
     name='pytest-random-order',
-    version='0.5.1',
+    version='0.5.2',
     author='Jazeps Basko',
     author_email='[email protected]',
     maintainer='Jazeps Basko',

tests/conftest.py

@@ -31,3 +31,19 @@ def get_test_calls():
     Returns a function to get runtest calls out from testdir.pytestrun result object.
     """
     return _get_test_calls
+
+
+@pytest.fixture
+def twenty_tests():
+    code = []
+    for i in range(20):
+        code.append('def test_a{}(): assert True\n'.format(str(i).zfill(2)))
+    return ''.join(code)
+
+
+@pytest.fixture
+def twenty_cls_tests():
+    code = []
+    for i in range(20):
+        code.append('\tdef test_b{}(self): self.assertTrue\n'.format(str(i).zfill(2)))
+    return ''.join(code)

tests/test_actual_test_runs.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 import collections
+import re
 
 import py
 import pytest
@@ -148,3 +149,52 @@ def test_it_works_with_actual_tests(tmp_tree_of_tests, get_test_calls, bucket):
         sequences.add(seq)
 
     assert 1 < len(sequences) <= 5
+
+
+def test_random_order_seed_is_respected(testdir, twenty_tests, get_test_calls):
+    testdir.makepyfile(twenty_tests)
+    call_sequences = {
+        '1': None,
+        '2': None,
+        '3': None,
+    }
+    for seed in call_sequences.keys():
+        result = testdir.runpytest('--random-order-seed={}'.format(seed))
+
+        result.stdout.fnmatch_lines([
+            '*Using --random-order-seed={}*'.format(seed),
+        ])
+
+        result.assert_outcomes(passed=20)
+        call_sequences[seed] = get_test_calls(result)
+
+    for seed in call_sequences.keys():
+        result = testdir.runpytest('--random-order-seed={}'.format(seed))
+        result.assert_outcomes(passed=20)
+        assert call_sequences[seed] == get_test_calls(result)
+
+    assert call_sequences['1'] != call_sequences['2'] != call_sequences['3']
+
+
+def test_generated_seed_is_reported_and_run_can_be_reproduced(testdir, twenty_tests, get_test_calls):
+    testdir.makepyfile(twenty_tests)
+    result = testdir.runpytest('-v')
+    result.assert_outcomes(passed=20)
+    result.stdout.fnmatch_lines([
+        '*Using --random-order-seed=*'
+    ])
+    calls = get_test_calls(result)
+
+    # find the seed in output
+    seed = None
+    for line in result.outlines:
+        g = re.match('^Using --random-order-seed=(.+)$', line)
+        if g:
+            seed = g.group(1)
+            break
+    assert seed
+
+    result2 = testdir.runpytest('-v', '--random-order-seed={}'.format(seed))
+    result2.assert_outcomes(passed=20)
+    calls2 = get_test_calls(result2)
+    assert calls == calls2

tests/test_cli.py

@@ -5,6 +5,7 @@ def test_help_message(testdir):
     result.stdout.fnmatch_lines([
         'random-order:',
         '*--random-order-bucket={global,package,module,class}*',
+        '*--random-order-seed=*',
     ])
 
 

tests/test_markers.py

@@ -1,22 +1,6 @@
 import pytest
 
 
-@pytest.fixture
-def twenty_tests():
-    code = []
-    for i in range(20):
-        code.append('def test_a{}(): assert True\n'.format(str(i).zfill(2)))
-    return ''.join(code)
-
-
-@pytest.fixture
-def twenty_cls_tests():
-    code = []
-    for i in range(20):
-        code.append('\tdef test_b{}(self): self.assertTrue\n'.format(str(i).zfill(2)))
-    return ''.join(code)
-
-
 @pytest.mark.parametrize('disabled', [True, False])
 def test_marker_disables_random_order_in_module(testdir, twenty_tests, get_test_calls, disabled):
     testdir.makepyfile(

tests/test_shuffle.py

@@ -130,3 +130,19 @@ def test_shuffle_respects_two_distinct_disabled_groups_in_one_bucket():
             return
 
     assert False
+
+
+def test_shuffle_respects_seed():
+    sorted_items = list(range(30))
+
+    for seed in range(20):
+        # Reset
+        items1 = list(range(30))
+        _shuffle_items(items1, seed=seed)
+
+        assert items1 != sorted_items
+
+        items2 = list(range(30))
+        _shuffle_items(items2, seed=seed)
+
+        assert items2 == items1