From 01aa08fb9c3115ebd309d80ac8fd6687f96d1ce6 Mon Sep 17 00:00:00 2001 From: Romel Tolos Date: Tue, 12 Sep 2023 18:09:17 +0300 Subject: [PATCH 1/3] added marker --- README.rst | 9 ++ pyproject.toml | 3 + src/pytest_randomly/__init__.py | 23 ++- tests/test_pytest_randomly.py | 246 ++++++++++++++++++++++++++++++++ 4 files changed, 274 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 098b0fa..87e5390 100644 --- a/README.rst +++ b/README.rst @@ -160,6 +160,15 @@ You can disable behaviours you don't like with the following flags: the start of every test * ``--randomly-dont-reorganize`` - turn off the shuffling of the order of tests +Or you can selectively disable the shuffling of the test order with the +``randomly_dont_reorganize`` marker: + +* ``pytestmark = pytest.mark.randomly_dont_reorganize`` - disable shuffling for + all tests in a module +* ``@pytest.mark.randomly_dont_reorganize`` - disable shuffling for a class or function +Note: if using the marker on individual tests, all functions inside a module must be marked, +due to how the plugin shuffles test execution order + The plugin appears to Pytest with the name 'randomly'. To disable it altogether, you can use the ``-p`` argument, for example: diff --git a/pyproject.toml b/pyproject.toml index 74a500d..fdce64f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,9 @@ addopts = """\ --strict-config --strict-markers """ +markers = [ + "randomly_dont_reorganize: Do not shuffle marked class or module", +] [tool.mypy] mypy_path = "src/" diff --git a/src/pytest_randomly/__init__.py b/src/pytest_randomly/__init__.py index 984469f..3a7a2d5 100644 --- a/src/pytest_randomly/__init__.py +++ b/src/pytest_randomly/__init__.py @@ -229,12 +229,15 @@ def pytest_collection_modifyitems(config: Config, items: list[Item]) -> None: modules_items: list[tuple[ModuleType | None, list[Item]]] = [] for module, group in groupby(items, _get_module): - modules_items.append( - ( - module, - _shuffle_by_class(list(group), seed), - ) - ) + module_items = list(group) + + for item in module_items: + marker_finder = getattr(item, "get_closest_marker", getattr(item, "get_marker", None)) + if not marker_finder("randomly_dont_reorganize"): + module_items = _shuffle_by_class(module_items, seed) + break + + modules_items.append((module, module_items)) def _module_key(module_item: tuple[ModuleType | None, list[Item]]) -> bytes: module, _items = module_item @@ -262,7 +265,13 @@ def _item_key(item: Item) -> bytes: for klass, group in groupby(items, _get_cls): klass_items = list(group) - klass_items.sort(key=_item_key) + + for item in klass_items: + marker_finder = getattr(item, "get_closest_marker", getattr(item, "get_marker", None)) + if not marker_finder("randomly_dont_reorganize"): + klass_items.sort(key=_item_key) + break + klasses_items.append((klass, klass_items)) def _cls_key(klass_items: tuple[type[Any] | None, list[Item]]) -> bytes: diff --git a/tests/test_pytest_randomly.py b/tests/test_pytest_randomly.py index 9449aca..d4742f5 100644 --- a/tests/test_pytest_randomly.py +++ b/tests/test_pytest_randomly.py @@ -350,6 +350,107 @@ def test_d(self): ] +def test_class_test_methods_not_reordered_if_marked(ourtester): + ourtester.makepyfile( + test_one=""" + import pytest + + from unittest import TestCase + + @pytest.mark.randomly_dont_reorganize + class T(TestCase): + def test_a(self): + pass + + def test_b(self): + pass + + def test_c(self): + pass + + def test_d(self): + pass + """ + ) + args = ["-v", "--randomly-seed=15"] + + out = ourtester.runpytest(*args) + + out.assert_outcomes(passed=4, failed=0) + assert out.outlines[9:13] == [ + "test_one.py::T::test_a PASSED", + "test_one.py::T::test_b PASSED", + "test_one.py::T::test_c PASSED", + "test_one.py::T::test_d PASSED", + ] + + +def test_classes_reordered_unless_marked(ourtester): + ourtester.makepyfile( + test_one=""" + import pytest + + from unittest import TestCase + + + class A(TestCase): + def test_aa(self): + pass + def test_ab(self): + pass + def test_ac(self): + pass + + + class B(TestCase): + def test_ba(self): + pass + def test_bb(self): + pass + def test_bc(self): + pass + + + @pytest.mark.randomly_dont_reorganize + class C(TestCase): + def test_ca(self): + pass + def test_cb(self): + pass + def test_cc(self): + pass + + + class D(TestCase): + def test_da(self): + pass + def test_db(self): + pass + def test_dc(self): + pass + """ + ) + args = ["-v", "--randomly-seed=1"] + + out = ourtester.runpytest(*args) + + out.assert_outcomes(passed=12, failed=0) + assert out.outlines[9:21] == [ + "test_one.py::A::test_ab PASSED", + "test_one.py::A::test_aa PASSED", + "test_one.py::A::test_ac PASSED", + "test_one.py::C::test_ca PASSED", + "test_one.py::C::test_cb PASSED", + "test_one.py::C::test_cc PASSED", + "test_one.py::B::test_bb PASSED", + "test_one.py::B::test_ba PASSED", + "test_one.py::B::test_bc PASSED", + "test_one.py::D::test_da PASSED", + "test_one.py::D::test_dc PASSED", + "test_one.py::D::test_db PASSED", + ] + + def test_test_functions_reordered(ourtester): ourtester.makepyfile( test_one=""" @@ -379,6 +480,151 @@ def test_d(): ] +def test_test_functions_not_reordered_if_marked(ourtester): + ourtester.makepyfile( + test_one=""" + import pytest + + pytestmark = pytest.mark.randomly_dont_reorganize + + + def test_a(): + pass + + def test_b(): + pass + + def test_c(): + pass + + def test_d(): + pass + """ + ) + args = ["-v", "--randomly-seed=15"] + + out = ourtester.runpytest(*args) + + out.assert_outcomes(passed=4, failed=0) + assert out.outlines[9:13] == [ + "test_one.py::test_a PASSED", + "test_one.py::test_b PASSED", + "test_one.py::test_c PASSED", + "test_one.py::test_d PASSED", + ] + + +def test_test_functions_reordered_if_individual_function_is_marked(ourtester): + ourtester.makepyfile( + test_one=""" + import pytest + + + def test_a(): + pass + + @pytest.mark.randomly_dont_reorganize + def test_b(): + pass + + def test_c(): + pass + + def test_d(): + pass + """ + ) + args = ["-v", "--randomly-seed=15"] + + out = ourtester.runpytest(*args) + + out.assert_outcomes(passed=4, failed=0) + assert out.outlines[9:13] == [ + "test_one.py::test_c PASSED", + "test_one.py::test_a PASSED", + "test_one.py::test_b PASSED", + "test_one.py::test_d PASSED", + ] + + +def test_test_functions_not_reordered_if_all_functions_are_marked(ourtester): + ourtester.makepyfile( + test_one=""" + import pytest + + + @pytest.mark.randomly_dont_reorganize + def test_a(): + pass + + @pytest.mark.randomly_dont_reorganize + def test_b(): + pass + + @pytest.mark.randomly_dont_reorganize + def test_c(): + pass + + @pytest.mark.randomly_dont_reorganize + def test_d(): + pass + """ + ) + args = ["-v", "--randomly-seed=15"] + + out = ourtester.runpytest(*args) + + out.assert_outcomes(passed=4, failed=0) + assert out.outlines[9:13] == [ + "test_one.py::test_a PASSED", + "test_one.py::test_b PASSED", + "test_one.py::test_c PASSED", + "test_one.py::test_d PASSED", + ] + + +def test_test_files_reordered_unless_marked(ourtester): + code = """ + def test_1(): + pass + + def test_2(): + pass + + def test_3(): + pass + """ + ourtester.makepyfile(test_a=code, + test_b=code, + test_c=f""" + import pytest + + pytestmark = pytest.mark.randomly_dont_reorganize + + {code} + """, + test_d=code) + args = ["-v", "--randomly-seed=15"] + + out = ourtester.runpytest(*args) + + out.assert_outcomes(passed=12, failed=0) + assert out.outlines[9:21] == [ + "test_b.py::test_3 PASSED", + "test_b.py::test_1 PASSED", + "test_b.py::test_2 PASSED", + "test_a.py::test_3 PASSED", + "test_a.py::test_2 PASSED", + "test_a.py::test_1 PASSED", + "test_d.py::test_3 PASSED", + "test_d.py::test_1 PASSED", + "test_d.py::test_2 PASSED", + "test_c.py::test_1 PASSED", + "test_c.py::test_2 PASSED", + "test_c.py::test_3 PASSED", + ] + + def test_test_functions_reordered_when_randomness_in_module(ourtester): ourtester.makepyfile( test_one=""" From e28bfeb9fc188d8a95f6792efef9f6e6063486b1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 15:16:45 +0000 Subject: [PATCH 2/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/pytest_randomly/__init__.py | 8 ++++++-- tests/test_pytest_randomly.py | 10 ++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/pytest_randomly/__init__.py b/src/pytest_randomly/__init__.py index 3a7a2d5..6e37bff 100644 --- a/src/pytest_randomly/__init__.py +++ b/src/pytest_randomly/__init__.py @@ -232,7 +232,9 @@ def pytest_collection_modifyitems(config: Config, items: list[Item]) -> None: module_items = list(group) for item in module_items: - marker_finder = getattr(item, "get_closest_marker", getattr(item, "get_marker", None)) + marker_finder = getattr( + item, "get_closest_marker", getattr(item, "get_marker", None) + ) if not marker_finder("randomly_dont_reorganize"): module_items = _shuffle_by_class(module_items, seed) break @@ -267,7 +269,9 @@ def _item_key(item: Item) -> bytes: klass_items = list(group) for item in klass_items: - marker_finder = getattr(item, "get_closest_marker", getattr(item, "get_marker", None)) + marker_finder = getattr( + item, "get_closest_marker", getattr(item, "get_marker", None) + ) if not marker_finder("randomly_dont_reorganize"): klass_items.sort(key=_item_key) break diff --git a/tests/test_pytest_randomly.py b/tests/test_pytest_randomly.py index d4742f5..85f4318 100644 --- a/tests/test_pytest_randomly.py +++ b/tests/test_pytest_randomly.py @@ -594,16 +594,18 @@ def test_2(): def test_3(): pass """ - ourtester.makepyfile(test_a=code, - test_b=code, - test_c=f""" + ourtester.makepyfile( + test_a=code, + test_b=code, + test_c=f""" import pytest pytestmark = pytest.mark.randomly_dont_reorganize {code} """, - test_d=code) + test_d=code, + ) args = ["-v", "--randomly-seed=15"] out = ourtester.runpytest(*args) From 98379d2223761993a1f3af8493277103ca4f475b Mon Sep 17 00:00:00 2001 From: Romel Tolos Date: Tue, 12 Sep 2023 18:35:16 +0300 Subject: [PATCH 3/3] remove support for "get_marker" method, as it is pre pytest 4.0 --- src/pytest_randomly/__init__.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/pytest_randomly/__init__.py b/src/pytest_randomly/__init__.py index 6e37bff..6f79e60 100644 --- a/src/pytest_randomly/__init__.py +++ b/src/pytest_randomly/__init__.py @@ -232,10 +232,7 @@ def pytest_collection_modifyitems(config: Config, items: list[Item]) -> None: module_items = list(group) for item in module_items: - marker_finder = getattr( - item, "get_closest_marker", getattr(item, "get_marker", None) - ) - if not marker_finder("randomly_dont_reorganize"): + if not item.get_closest_marker("randomly_dont_reorganize"): module_items = _shuffle_by_class(module_items, seed) break @@ -269,10 +266,7 @@ def _item_key(item: Item) -> bytes: klass_items = list(group) for item in klass_items: - marker_finder = getattr( - item, "get_closest_marker", getattr(item, "get_marker", None) - ) - if not marker_finder("randomly_dont_reorganize"): + if not item.get_closest_marker("randomly_dont_reorganize"): klass_items.sort(key=_item_key) break