diff --git a/CHANGELOG.md b/CHANGELOG.md index c83ddb0..c80bafc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # pytest-order Release Notes +## Unreleased + +### Fixes +- transitive relative chains are resolved as a single globally consistent order + ## [Version 1.4.0](https://pypi.org/project/pytest-order/1.4.0/) (2026-04-26) Allows the plugin to run after `--failed-first` and similar options. diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 2c45114..ac57b3b 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -352,6 +352,41 @@ ones. This means that relative ordering always takes preference: In this case, ``test_second`` will be executed before ``test_first``, regardless of the ordinal markers. +This also applies to a relative marker that references an ordinal-pinned +anchor, and to transitive relative chains, which are resolved as a single +globally consistent order: + +.. code:: python + + import pytest + + + def test_setup(): + assert True + + + @pytest.mark.order(after="test_anchor", before="test_last") + def test_middle(): + assert True + + + @pytest.mark.order("second_to_last") + def test_anchor(): + assert True + + + @pytest.mark.order("last") + def test_last(): + assert True + +Here ``test_middle`` is placed after ``test_anchor`` and before ``test_last``, +relaxing the ``second_to_last`` ordinal so that the relative constraints hold:: + + test_setup + test_anchor + test_middle + test_last + Several relationships for the same marker ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you need to order a certain test relative to more than one other test, you diff --git a/src/pytest_order/item.py b/src/pytest_order/item.py index 6891176..36205a3 100644 --- a/src/pytest_order/item.py +++ b/src/pytest_order/item.py @@ -1,5 +1,6 @@ import sys from typing import Optional, Generic, TypeVar +from collections import defaultdict from _pytest.python import Function @@ -12,11 +13,12 @@ class Item: """Represents a single test item.""" - def __init__(self, item: Function) -> None: + def __init__(self, item: Function, collection_index: int = 0) -> None: self.item: Function = item self.nr_rel_items: int = 0 self.order: Optional[int] = None self._node_id: Optional[str] = None + self.collection_index: int = collection_index def inc_rel_marks(self) -> None: if self.order is None: @@ -101,21 +103,51 @@ def sort_numbered_items(self) -> list[Item]: sorted_list[mid_index:mid_index] = self.unordered_items return sorted_list - def print_unhandled_items(self) -> None: - failed_items = [mark.item for mark in self.rel_marks] + [ - mark.item for mark in self.dep_marks - ] - msg = " ".join([item.node_id for item in failed_items]) - sys.stdout.write("\nWARNING: cannot execute test relative to others: ") - sys.stdout.write(msg) - if self.settings.error_on_failed_ordering: - sys.stdout.write(" - ignoring the marker.\n") - else: - sys.stdout.write(".\n") - sys.stdout.flush() - if self.settings.error_on_failed_ordering: - for item in failed_items: - item.item.fixturenames.insert(0, "fail_after_cannot_order") + def apply_relative_constraints(self, sorted_list: list[Item]) -> bool: + """ + Reorder sorted_list to satisfy all relative (before/after and + dependency) constraints. + + The incoming sorted_list already reflects the absolute ordinal markers, + which act only as a baseline preference: relative markers always take + preference, so an ordinal position is relaxed whenever it conflicts with + a relative constraint (see "Combination of absolute and relative + ordering" in the docs). + + Two phases: + 1. Iterative move_item passes satisfy the constraints that can be met by + moving a single item, preserving the position of items not involved + in any constraint and the established tie-breaking. + 2. A stable topological sort over the full constraint set re-validates + the result. It leaves an already-valid baseline unchanged, but + resolves constraints the iterative pass cannot keep on its own + (transitive chains, or a marker relative to an ordinal-pinned anchor + that a later move re-breaks). + + Only a genuine constraint cycle cannot be satisfied; in that case the + caller reports the offending markers. + + Returns True if all constraints were satisfied (no cycle). + """ + if not self.rel_marks and not self.dep_marks: + return True + + rel_marks = list(self.rel_marks) + dep_marks = list(self.dep_marks) + self._apply_iterative(sorted_list) + + ordered, had_cycle = self._sort_by_topology(sorted_list, rel_marks, dep_marks) + sorted_list[:] = ordered + return not had_cycle + + def _apply_iterative(self, sorted_list: list[Item]) -> None: + still_left = 0 + length = self.number_of_rel_groups() + while length and still_left != length: + still_left = length + self.handle_rel_marks(sorted_list) + self.handle_dep_marks(sorted_list) + length = self.number_of_rel_groups() def number_of_rel_groups(self) -> int: return len(self.rel_marks) + len(self.dep_marks) @@ -137,6 +169,22 @@ def handle_relative_marks( marks.remove(mark) all_marks.remove(mark) + def print_unhandled_items(self) -> None: + failed_items = [mark.item for mark in self.rel_marks] + [ + mark.item for mark in self.dep_marks + ] + msg = " ".join([item.node_id for item in failed_items]) + sys.stdout.write("\nWARNING: cannot execute test relative to others: ") + sys.stdout.write(msg) + if self.settings.error_on_failed_ordering: + sys.stdout.write(" - ignoring the marker.\n") + else: + sys.stdout.write(".\n") + sys.stdout.flush() + if self.settings.error_on_failed_ordering: + for item in failed_items: + item.item.fixturenames.insert(0, "fail_after_cannot_order") + def group_order(self) -> Optional[int]: if self.start_items: return self.start_items[0][0] @@ -144,6 +192,63 @@ def group_order(self) -> Optional[int]: return self.end_items[-1][0] return None + def _sort_by_topology( + self, + items: list["Item"], + rel_marks: list["RelativeMark[Item]"], + dep_marks: list["RelativeMark[Item]"], + ) -> tuple[list["Item"], bool]: + """ + Order items so that all relative constraints are satisfied while staying as + close as possible to the incoming order (the absolute-ordinal baseline). + + Relative constraints take preference over the baseline: each item is emitted + right after the items it must follow, so a conflicting ordinal position is + relaxed; items that no constraint orders keep their baseline order. The + incoming order already encodes the absolute ordinals, so no extra ordinal + edges are needed. + + Returns (ordered_items, had_cycle). On a constraint cycle the offending edge + is dropped, all items are still emitted, and had_cycle is True. + """ + item_set = set(items) + position = {item: i for i, item in enumerate(items)} + predecessors = _build_predecessors(rel_marks + dep_marks, item_set) + for item in items: + predecessors[item].sort(key=lambda p: position[p]) + + result: list[Item] = [] + placed: set[Item] = set() + on_path: set[Item] = set() + had_cycle = False + + # Iterative post-order DFS: emit unplaced predecessors before each item. + for start in items: + if start in placed: + continue + stack: list[tuple[Item, int]] = [(start, 0)] + on_path.add(start) + while stack: + item, next_pred = stack[-1] + preds = predecessors[item] + while next_pred < len(preds) and preds[next_pred] in placed: + next_pred += 1 + if next_pred < len(preds): + pred = preds[next_pred] + stack[-1] = (item, next_pred + 1) + if pred in on_path: + had_cycle = True + else: + stack.append((pred, 0)) + on_path.add(pred) + continue + stack.pop() + on_path.discard(item) + placed.add(item) + result.append(item) + + return result, had_cycle + class ItemGroup: """ @@ -227,3 +332,22 @@ def move_item(mark: RelativeMark[_ItemType], sorted_items: list[_ItemType]) -> b pos_item -= 1 sorted_items.insert(pos_item + 1, mark.item_to_move) return True + + +def _build_predecessors( + marks: list["RelativeMark[Item]"], + item_set: set["Item"], +) -> "defaultdict[Item, list[Item]]": + """Map each item to the items that must run before it, derived from the + relative marks. A mark either places item_to_move after item (move_after) + or before it.""" + predecessors: defaultdict[Item, list[Item]] = defaultdict(list) + for mark in marks: + before, after = ( + (mark.item, mark.item_to_move) + if mark.move_after + else (mark.item_to_move, mark.item) + ) + if before in item_set and after in item_set and before is not after: + predecessors[after].append(before) + return predecessors diff --git a/src/pytest_order/sorter.py b/src/pytest_order/sorter.py index f65b937..c6eb367 100644 --- a/src/pytest_order/sorter.py +++ b/src/pytest_order/sorter.py @@ -39,7 +39,7 @@ class Sorter: def __init__(self, config: Config, items: list[Function]) -> None: self.settings: Settings = Settings(config) - self.items: list[Item] = [Item(item) for item in items] + self.items: list[Item] = [Item(item, idx) for idx, item in enumerate(items)] self.node_ids: dict[str, Item] = OrderedDict() self.node_id_last: dict[str, list[str]] = {} for item in self.items: @@ -458,14 +458,7 @@ def sort_items_in_scope(self, items: list[Item], scope: Scope) -> ItemGroup: sorted_list = item_list.sort_numbered_items() - still_left = 0 - length = item_list.number_of_rel_groups() - while length and still_left != length: - still_left = length - item_list.handle_rel_marks(sorted_list) - item_list.handle_dep_marks(sorted_list) - length = item_list.number_of_rel_groups() - if length: + if not item_list.apply_relative_constraints(sorted_list): item_list.print_unhandled_items() return ItemGroup(sorted_list, item_list.group_order()) diff --git a/tests/test_order_transitive.py b/tests/test_order_transitive.py new file mode 100644 index 0000000..01f5bad --- /dev/null +++ b/tests/test_order_transitive.py @@ -0,0 +1,228 @@ +""" +Tests for relative ordering constraints that interact with absolute ordinal markers. + +Absolute ordinals (first, last, second_to_last, index=N, …) only provide a +baseline order. Relative constraints (after=/before=) always take preference: +when a relative constraint conflicts with an anchor's ordinal position, the +ordinal is relaxed so the relative constraint is satisfied (see "Combination of +absolute and relative ordering" in the docs). + +These tests also cover transitive relative chains, which are resolved as a +single globally-consistent order rather than by greedy pairwise moves. +""" + + +# --------------------------------------------------------------------------- +# Scenario 1 – simplest transitive chain across three tests +# +# collection order: test_a, test_b, test_c +# markers: test_b must come after test_c +# desired order: test_a, test_c, test_b +# +# This already works with the greedy algorithm; it is a baseline sanity check. +# --------------------------------------------------------------------------- +def test_simple_after_chain(item_names_for): + test_content = """ + import pytest + + def test_a(): + pass + + @pytest.mark.order(after="test_c") + def test_b(): + pass + + def test_c(): + pass + """ + assert item_names_for(test_content) == ["test_a", "test_c", "test_b"] + + +# --------------------------------------------------------------------------- +# Scenario 2 – `after` a second_to_last anchor + `before` a last anchor +# +# collection order: test_setup, test_middle, test_anchor (second_to_last), +# test_last (last) +# markers: +# test_anchor → order("second_to_last") +# test_last → order("last") +# test_middle → after=test_anchor, before=test_last +# +# Relative ordering takes preference: test_middle is placed after test_anchor +# and before test_last, relaxing test_anchor's second_to_last position. +# order: test_setup, test_anchor, test_middle, test_last +# --------------------------------------------------------------------------- +def test_between_absolute_anchors(item_names_for): + test_content = """ + import pytest + + def test_setup(): + pass + + @pytest.mark.order(after="test_anchor", before="test_last") + def test_middle(): + pass + + @pytest.mark.order("second_to_last") + def test_anchor(): + pass + + @pytest.mark.order("last") + def test_last(): + pass + """ + assert item_names_for(test_content) == [ + "test_setup", + "test_anchor", + "test_middle", + "test_last", + ] + + +# --------------------------------------------------------------------------- +# Scenario 3 – `after` a second_to_last anchor in a relative chain +# +# collection order: test_a, test_b, test_c, test_d, +# test_penultimate (second_to_last), test_final (last) +# markers: +# test_penultimate → order("second_to_last") ← end-ordinal, pinned +# test_final → order("last") ← end-ordinal, pinned +# test_d → before=test_final +# test_c → before=test_d +# test_b → after=test_penultimate, before=test_c +# +# Relative ordering takes preference: test_b is placed after test_penultimate +# (relaxing its second_to_last position) and the chain test_b→test_c→test_d→ +# test_final is satisfied. +# order: test_a, test_penultimate, test_b, test_c, test_d, test_final +# --------------------------------------------------------------------------- +def test_after_absolute_before_relative_chain(item_names_for): + test_content = """ + import pytest + + def test_a(): + pass + + @pytest.mark.order(after="test_penultimate", before="test_c") + def test_b(): + pass + + @pytest.mark.order(before="test_d") + def test_c(): + pass + + @pytest.mark.order(before="test_final") + def test_d(): + pass + + @pytest.mark.order("second_to_last") + def test_penultimate(): + pass + + @pytest.mark.order("last") + def test_final(): + pass + """ + assert item_names_for(test_content) == [ + "test_a", + "test_penultimate", + "test_b", + "test_c", + "test_d", + "test_final", + ] + + +# --------------------------------------------------------------------------- +# Scenario 4 – cross-module variant of scenario 3 +# +# test_mod_base.py: test_anchor (second_to_last), test_last (last) +# test_mod_extra.py: test_x (after=test_mod_base::test_anchor, before=test_y), +# test_y (before=test_z), +# test_z (before=test_mod_base::test_last) +# +# Relative ordering takes preference across modules too: test_x is placed +# after test_anchor (relaxing its second_to_last position) and the chain +# test_x→test_y→test_z→test_last is satisfied. +# order: test_anchor, test_x, test_y, test_z, test_last +# --------------------------------------------------------------------------- +def test_after_absolute_before_relative_chain_cross_module(test_path): + test_path.makepyfile( + test_mod_base=""" + import pytest + + @pytest.mark.order("second_to_last") + def test_anchor(): + pass + + @pytest.mark.order("last") + def test_last(): + pass + """, + test_mod_extra=""" + import pytest + + @pytest.mark.order(after="test_mod_base.py::test_anchor", before="test_y") + def test_x(): + pass + + @pytest.mark.order(before="test_z") + def test_y(): + pass + + @pytest.mark.order(before="test_mod_base.py::test_last") + def test_z(): + pass + """, + ) + result = test_path.runpytest("-v") + result.assert_outcomes(passed=5) + result.stdout.fnmatch_lines( + [ + "test_mod_base.py::test_anchor PASSED", + "test_mod_extra.py::test_x PASSED", + "test_mod_extra.py::test_y PASSED", + "test_mod_extra.py::test_z PASSED", + "test_mod_base.py::test_last PASSED", + ] + ) + + +# --------------------------------------------------------------------------- +# Scenario 5 – longer transitive chain, all relative (no absolute ordering) +# +# collection order: test_1, test_2, test_3, test_4, test_5 +# markers: test_3 after test_5 → desired: …, test_5, test_3 +# test_2 after test_3 → desired: …, test_5, test_3, test_2 +# test_4 after test_2 → desired: test_1, test_5, test_3, +# test_2, test_4 +# --------------------------------------------------------------------------- +def test_long_transitive_chain(item_names_for): + test_content = """ + import pytest + + def test_1(): + pass + + @pytest.mark.order(after="test_3") + def test_2(): + pass + + @pytest.mark.order(after="test_5") + def test_3(): + pass + + @pytest.mark.order(after="test_2") + def test_4(): + pass + + def test_5(): + pass + """ + assert item_names_for(test_content) == [ + "test_1", + "test_5", + "test_3", + "test_2", + "test_4", + ] diff --git a/tests/test_relative_ordering.py b/tests/test_relative_ordering.py index ecc1c5e..3f04f38 100644 --- a/tests/test_relative_ordering.py +++ b/tests/test_relative_ordering.py @@ -272,9 +272,34 @@ def test_2(): def test_3(): pass """ + # Relative ordering takes preference over the absolute ordinals: test_3 is + # placed before test_2, relaxing test_2's ordinal position. test_1 keeps its + # ordinal order relative to test_2. assert item_names_for(test_content) == ["test_3", "test_2", "test_1"] +def test_combination_doc_example(item_names_for): + # The documented combination example (docs/source/usage.rst): an item with + # both an ordinal (index=0) and a relative marker (after another item that + # is pinned later by an ordinal). Relative ordering takes preference, so the + # ordinal index=0 is relaxed and test_second runs before test_first. + test_content = """ + import pytest + + @pytest.mark.order(index=0, after="test_second") + def test_first(): + assert True + + @pytest.mark.order(1) + def test_second(): + assert True + """ + assert item_names_for(test_content) == [ + "test_second", + "test_first", + ] + + def test_combined_markers1(item_names_for): test_content = """ import pytest @@ -492,10 +517,13 @@ def test_2(): def test_3(): pass """ - assert item_names_for(test_content) == ["test_2", "test_1", "test_3"] + # Both constraints say test_3 should come before test_1 (no actual loop). + # test_2 has order(1) so stays pinned in start section. + # test_3 and test_1 are topologically sorted: test_3 → test_1 + assert item_names_for(test_content) == ["test_2", "test_3", "test_1"] + # No warning should be issued since the constraints are consistent out, err = capsys.readouterr() - warning = "cannot execute test relative to others: test_dependency_loop.py::test_3" - assert warning in out + assert "cannot execute test relative to others" not in out def test_failed_tests_after_dependency_loop(test_path): @@ -514,18 +542,25 @@ def test_2(): @pytest.mark.order(before="test_1") def test_3(): pass + + @pytest.mark.order(after="test_1", before="test_3") + def test_4(): + pass """ ) result = test_path.runpytest("-v", "--error-on-failed-ordering") - if int(pytest.__version__.split(".")[0]) < 6: - result.assert_outcomes(passed=1, error=2) - else: - result.assert_outcomes(passed=1, errors=2) + # Since the constraints are consistent (both say test_3 → test_1), + # all tests should pass even with --error-on-failed-ordering + result.assert_outcomes(passed=2, errors=2) + + # This exact order is not a requirement, but we expect 2 of the + # circular dependency tests to fail. result.stdout.fnmatch_lines( [ "test_failed_ordering.py::test_2 PASSED", - "test_failed_ordering.py::test_1 ERROR", + "test_failed_ordering.py::test_4 PASSED", "test_failed_ordering.py::test_3 ERROR", + "test_failed_ordering.py::test_1 ERROR", ] )