diff --git a/CHANGELOG.md b/CHANGELOG.md index c80bafc..620a3fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +### New features +* added option `--fail-all-on-failed-ordering` to abort the whole test run + without executing any tests if some tests could not be ordered + ### Fixes - transitive relative chains are resolved as a single globally consistent order diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index e8bb36e..a1ac89c 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -653,6 +653,36 @@ If you use the option `--error-on-failed-ordering`, "test_two" will now error: ERROR test_failed_ordering.py::test_two - Failed: The test could not be ordered ========================= 1 passed, 1 error in 0.75s ========================== +``--fail-all-on-failed-ordering`` +--------------------------------- +This option is analogous to ``--error-on-failed-ordering``, but instead of failing only +the tests that could not be ordered, it aborts the whole test run immediately, without +executing any tests. This is useful if running the tests in a wrong order has +unwanted effects. + +Using the example shown above:: + + $ pytest tests -vv --fail-all-on-failed-ordering + ============================= test session starts ============================== + ... + collected 2 items + + ============================ no tests ran in 0.02s ============================= + ERROR: pytest-order: cannot execute 'test_two' relative to others: 'test_three' + +The test run exits with the usage error exit code (4). + +The option also works with ``--collect-only``, as the ordering happens during test +collection. This allows validating the ordering, e.g. in CI, without executing any +tests:: + + $ pytest tests --collect-only --fail-all-on-failed-ordering + ERROR: pytest-order: cannot execute 'test_two' relative to others: 'test_three' + ============================= test session starts ============================== + ... + ========================== 2 tests collected in 0.01s ========================== + +The exit code is 4 in this case as well, and 0 if all tests could be ordered. .. _`pytest-dependency`: https://pypi.org/project/pytest-dependency/ diff --git a/src/pytest_order/item.py b/src/pytest_order/item.py index 36205a3..7c6be64 100644 --- a/src/pytest_order/item.py +++ b/src/pytest_order/item.py @@ -2,7 +2,7 @@ from typing import Optional, Generic, TypeVar from collections import defaultdict -from _pytest.python import Function +from pytest import Function, UsageError from .settings import Scope, Settings @@ -176,6 +176,10 @@ def print_unhandled_items(self) -> None: msg = " ".join([item.node_id for item in failed_items]) sys.stdout.write("\nWARNING: cannot execute test relative to others: ") sys.stdout.write(msg) + if self.settings.fail_all_on_failed_ordering: + raise UsageError( + f"pytest-order: cannot execute test relative to others: {msg}" + ) if self.settings.error_on_failed_ordering: sys.stdout.write(" - ignoring the marker.\n") else: diff --git a/src/pytest_order/plugin.py b/src/pytest_order/plugin.py index 0fb9361..ce0a618 100644 --- a/src/pytest_order/plugin.py +++ b/src/pytest_order/plugin.py @@ -1,11 +1,11 @@ from collections.abc import Generator, Callable import pytest +from pytest import Function from _pytest.config import Config from _pytest.config.argparsing import Parser from _pytest.main import Session from _pytest.mark import Mark -from _pytest.python import Function from .sorter import Sorter @@ -126,6 +126,15 @@ def pytest_addoption(parser: Parser) -> None: "will error instead of generating only a warning." ), ) + group.addoption( + "--fail-all-on-failed-ordering", + action="store_true", + dest="fail_all_on_failed_ordering", + help=( + "If set, the whole test run fails immediately without running " + "any tests if some tests with relative markers could not be ordered." + ), + ) group.addoption( "--order-after-ff", action="store_true", diff --git a/src/pytest_order/settings.py b/src/pytest_order/settings.py index e020adf..fe4c5ea 100644 --- a/src/pytest_order/settings.py +++ b/src/pytest_order/settings.py @@ -26,6 +26,9 @@ def __init__(self, config: Config) -> None: self.error_on_failed_ordering: str = config.getoption( "error_on_failed_ordering" ) + self.fail_all_on_failed_ordering: bool = config.getoption( + "fail_all_on_failed_ordering" + ) scope: str = config.getoption("order_scope") if scope in self.valid_scopes: self.scope: Scope = self.valid_scopes[scope] diff --git a/src/pytest_order/sorter.py b/src/pytest_order/sorter.py index c6eb367..f4aac11 100644 --- a/src/pytest_order/sorter.py +++ b/src/pytest_order/sorter.py @@ -7,7 +7,7 @@ from _pytest.config import Config from _pytest.mark import Mark -from _pytest.python import Function +from pytest import Function, UsageError from .item import Item, ItemList, ItemGroup, filter_marks, move_item, RelativeMark from .settings import Settings, Scope @@ -270,15 +270,15 @@ def handle_relative_marks(self, item: Item, mark: Mark) -> bool: return has_relative_marks def warn_about_unknown_test(self, item: Item, rel_mark: str) -> None: + msg = f"cannot execute '{item.item.name}' relative to others: '{rel_mark}'" + if self.settings.fail_all_on_failed_ordering: + raise UsageError(f"pytest-order: {msg}") if self.settings.error_on_failed_ordering: item.item.fixturenames.insert(0, "fail_after_cannot_order") ignore_msg = "" else: ignore_msg = " - ignoring the marker" - sys.stdout.write( - f"\nWARNING: cannot execute '{item.item.name}' relative to others: " - f"'{rel_mark}'{ignore_msg}." - ) + sys.stdout.write(f"\nWARNING: {msg}{ignore_msg}.") def collect_markers(self) -> None: aliases: dict[str, list[Item]] = {} diff --git a/tests/test_relative_ordering.py b/tests/test_relative_ordering.py index 3f04f38..3758717 100644 --- a/tests/test_relative_ordering.py +++ b/tests/test_relative_ordering.py @@ -474,6 +474,30 @@ def test_3(): ) +def test_failing_dependency_fails_run(test_path): + test_path.makepyfile( + test_failed_ordering=""" + import pytest + + def test_1(): + pass + + @pytest.mark.order(before="test_4") + def test_2(): + pass + + def test_3(): + pass + """ + ) + result = test_path.runpytest("-v", "--fail-all-on-failed-ordering") + assert result.ret == pytest.ExitCode.USAGE_ERROR + result.assert_outcomes(passed=0, failed=0) + result.stderr.fnmatch_lines( + ["ERROR: pytest-order: cannot execute 'test_2' relative to others: 'test_4'"] + ) + + def test_dependency_in_class_before_unknown_test(item_names_for, capsys): test_content = """ import pytest @@ -565,6 +589,35 @@ def test_4(): ) +def test_failed_run_after_dependency_loop(test_path): + test_path.makepyfile( + test_failed_ordering=""" + import pytest + + @pytest.mark.order(after="test_3") + def test_1(): + pass + + @pytest.mark.order(1) + def test_2(): + pass + + @pytest.mark.order(after="test_1") + def test_3(): + pass + """ + ) + result = test_path.runpytest("-v", "--fail-all-on-failed-ordering") + assert result.ret == pytest.ExitCode.USAGE_ERROR + result.assert_outcomes(passed=0, failed=0) + result.stderr.fnmatch_lines( + [ + "ERROR: pytest-order: cannot execute test relative to others: " + "test_failed_ordering.py::test_* test_failed_ordering.py::test_*" + ] + ) + + def test_dependency_on_parametrized_test(item_names_for): test_content = """ import pytest