From cc80b730946ac0232443d15fc9a1d52d188f7577 Mon Sep 17 00:00:00 2001 From: AenEnlil Date: Fri, 26 Sep 2025 17:11:40 +0300 Subject: [PATCH 1/2] implemented steps reporting configuration. Now steps of skipped or failed tests reported by default. For reporting passed test steps added separate option. Also added option to disable steps reporting for all tests. --- README.md | 49 +++++++++++++++++---- pytestomatio/main.py | 9 ++-- pytestomatio/testomatio/testRunConfig.py | 4 ++ tests/test_testomatio/test_testRunConfig.py | 40 ++++++++++++++++- 4 files changed, 90 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index d2e0a8b..b987530 100644 --- a/README.md +++ b/README.md @@ -170,13 +170,15 @@ You can use environment variable to control certain features of testomat.io #### Test Run configuration -| Env variable | What it does | Examples | -|--------------------------|----------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------| -| TESTOMATIO_TITLE | Name of a test run to create on testomat.io | TESTOMATIO_TITLE="Nightly Smoke Tests" pytest --testomatio report | -| TESTOMATIO_RUN_ID | Id of existing test run to use for sending test results to | TESTOMATIO_RUN_ID=98dfas0 pytest --testomatio report | -| TESTOMATIO_RUNGROUP_TITLE | Create a group (folder) for a test run. If group already exists, attach test run to it | TESTOMATIO_RUNGROUP_TITLE="Release 2.0" pytest --testomatio report | -| TESTOMATIO_ENV | Assign environment to a test run, env variant of **testRunEnv** option. Has a lower precedence than **testRunEnv** option. | TESTOMATIO_ENV="linux,chrome,1920x1080" pytest --testomatio report | -| TESTOMATIO_LABEL | Assign labels to a test run. Labels must exist in project and their scope must be enabled for runs | TESTOMATIO_LABEL="smoke,regression" pytest --testomatio report | +| Env variable | What it does | Examples | +|---------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------| +| TESTOMATIO_TITLE | Name of a test run to create on testomat.io | TESTOMATIO_TITLE="Nightly Smoke Tests" pytest --testomatio report | +| TESTOMATIO_RUN_ID | Id of existing test run to use for sending test results to | TESTOMATIO_RUN_ID=98dfas0 pytest --testomatio report | +| TESTOMATIO_RUNGROUP_TITLE | Create a group (folder) for a test run. If group already exists, attach test run to it | TESTOMATIO_RUNGROUP_TITLE="Release 2.0" pytest --testomatio report | +| TESTOMATIO_ENV | Assign environment to a test run, env variant of **testRunEnv** option. Has a lower precedence than **testRunEnv** option. | TESTOMATIO_ENV="linux,chrome,1920x1080" pytest --testomatio report | +| TESTOMATIO_LABEL | Assign labels to a test run. Labels must exist in project and their scope must be enabled for runs | TESTOMATIO_LABEL="smoke,regression" pytest --testomatio report | +| TESTOMATIO_NO_STEPS | Disable reporting of all steps completely. When enabled, no steps will be included in the test report regardless of test status | TESTOMATIO_NO_STEPS=True pytest --testomatio report | +| TESTOMATIO_STEPS_PASSED | Enable steps reporting for passed tests(disabled by default). When disabled, only failed and skipped tests will include step details to reduce noise | TESTOMATIO_STEPS_PASSED=True pytest --testomatio report | #### S3 Bucket configuration | Env variable | Description | @@ -214,7 +216,7 @@ Environment values are comma separated, please use double quotation. The plugin supports dividing tests into separate, trackable steps. When reporting to testomat.io, you can view detailed information for each step including execution status, duration, and any errors that occurred. -**Important**: This plugin only supports **reporting** test steps to testomat.io during test execution. Test steps cannot be imported to testomat.io using **sync** option. +**Important**: This plugin only supports **reporting** test steps to testomat.io during test execution. Test steps cannot be imported to testomat.io using **sync** option. Steps reported for skipped and failed test by default. To enable steps reporting for passed tests use **TESTOMATIO_STEPS_PASSED** env variable. Test steps can be implemented using either decorators or context managers, giving you flexibility in how you structure your tests. @@ -272,6 +274,37 @@ def test_book_read(): assert book.read() == text ``` +**Note:** Step is registered when the step code is executed. Therefore, if test mark as skipped(not executed at all) or test code execution stops before step code is executed, step will not be attached to test: +```python +import pytest +from pytestomatio.utils.steps import step + +# Step will not be added in report +@pytest.mark.skip +def test_skipped(): + with step('Step1', 'user'): + assert True + +# Step will not be added in report +def test_exception_raised(): + raise ValueError() + with step('Step1', 'user'): + assert True + +# Step will not be added in report +def test_early_skip(): + pytest.skip() + with step('Step1', 'user'): + assert True + +# Step1 will be added in report, Step2 will not be +def test_nested_step_skip_or_exception(): + with step('Step1', 'user'): + with step('Step2', 'user'): + pytest.skip() # or AttributeError() +``` + + ### Submitting Test Artifacts diff --git a/pytestomatio/main.py b/pytestomatio/main.py index 5382ee3..8aa241b 100644 --- a/pytestomatio/main.py +++ b/pytestomatio/main.py @@ -193,9 +193,12 @@ def pytest_runtest_makereport(item: Item, call: CallInfo): if hasattr(item, 'callspec'): request['example'] = test_item.safe_params(item.callspec.params) - step_manager = _step_managers.get(item.nodeid) - if step_manager: - request['steps'] = step_manager.get_steps() + if not pytest.testomatio.test_run_config.disable_steps: + step_manager = _step_managers.get(item.nodeid) + if step_manager: + if call.excinfo is not None or \ + (call.excinfo is None and pytest.testomatio.test_run_config.enable_steps_for_passed_test): + request['steps'] = step_manager.get_steps() if item.nodeid not in pytest.testomatio.test_run_config.status_request: pytest.testomatio.test_run_config.status_request[item.nodeid] = request diff --git a/pytestomatio/testomatio/testRunConfig.py b/pytestomatio/testomatio/testRunConfig.py index b58421f..ac569c4 100644 --- a/pytestomatio/testomatio/testRunConfig.py +++ b/pytestomatio/testomatio/testRunConfig.py @@ -11,8 +11,12 @@ def __init__(self): run_id = os.environ.get('TESTOMATIO_RUN_ID') or os.environ.get('TESTOMATIO_RUN') title = os.environ.get('TESTOMATIO_TITLE') if os.environ.get('TESTOMATIO_TITLE') else 'test run at ' + dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S") shared_run = os.environ.get('TESTOMATIO_SHARED_RUN') in ['True', 'true', '1'] + disable_steps = os.environ.get('TESTOMATIO_NO_STEPS') in ['True', 'true', '1'] + enable_steps_for_passed_test = os.environ.get('TESTOMATIO_STEPS_PASSED') in ['True', 'true', '1'] self.test_run_id = run_id self.title = title + self.enable_steps_for_passed_test = enable_steps_for_passed_test + self.disable_steps = disable_steps self.environment = safe_string_list(os.environ.get('TESTOMATIO_ENV')) self.label = safe_string_list(os.environ.get('TESTOMATIO_LABEL')) self.group_title = os.environ.get('TESTOMATIO_RUNGROUP_TITLE') diff --git a/tests/test_testomatio/test_testRunConfig.py b/tests/test_testomatio/test_testRunConfig.py index 3acad4b..8c608fe 100644 --- a/tests/test_testomatio/test_testRunConfig.py +++ b/tests/test_testomatio/test_testRunConfig.py @@ -19,6 +19,8 @@ def test_init_default_values(self): assert config.test_run_id is None assert config.title == "test run at 2024-01-15 10:30:45" assert config.environment is None + assert config.enable_steps_for_passed_test is False + assert config.disable_steps is False assert config.label is None assert config.group_title is None assert config.parallel is True @@ -32,7 +34,9 @@ def test_init_with_env_variables(self): 'TESTOMATIO_TITLE': 'Custom Test Run', 'TESTOMATIO_ENV': 'linux,chrome,1920x1080', 'TESTOMATIO_LABEL': 'smoke,regression', - 'TESTOMATIO_RUNGROUP_TITLE': 'Release 2.0' + 'TESTOMATIO_RUNGROUP_TITLE': 'Release 2.0', + 'TESTOMATIO_NO_STEPS': '1', + 'TESTOMATIO_STEPS_PASSED': '1' } with patch.dict(os.environ, env_vars, clear=True): @@ -41,6 +45,8 @@ def test_init_with_env_variables(self): assert config.test_run_id == 'run_12345' assert config.title == 'Custom Test Run' assert config.environment == 'linux,chrome,1920x1080' + assert config.enable_steps_for_passed_test is True + assert config.disable_steps is True assert config.label == 'smoke,regression' assert config.group_title == 'Release 2.0' assert config.parallel is True @@ -64,6 +70,38 @@ def test_init_shared_run_false_variations(self, value): assert config.shared_run is False assert config.parallel is True + @pytest.mark.parametrize('value', ['True', 'true', '1']) + def test_init_disable_steps_true_variations(self, value): + """Test different true values for TESTOMATIO_NO_STEPS""" + with patch.dict(os.environ, {'TESTOMATIO_NO_STEPS': value}, clear=True): + config = TestRunConfig() + + assert config.disable_steps is True + + @pytest.mark.parametrize('value', ['False', 'false', '0', 'anything']) + def test_init_disable_steps_false_variations(self, value): + """Test different false values TESTOMATIO_NO_STEPS""" + with patch.dict(os.environ, {'TESTOMATIO_NO_STEPS': value}, clear=True): + config = TestRunConfig() + + assert config.disable_steps is False + + @pytest.mark.parametrize('value', ['True', 'true', '1']) + def test_enable_passed_steps_run_true_variations(self, value): + """Test different true values for TESTOMATIO_STEPS_PASSED""" + with patch.dict(os.environ, {'TESTOMATIO_STEPS_PASSED': value}, clear=True): + config = TestRunConfig() + + assert config.enable_steps_for_passed_test is True + + @pytest.mark.parametrize('value', ['False', 'false', '0', 'anything']) + def test_enable_passed_steps_false_variations(self, value): + """Test different false values TESTOMATIO_STEPS_PASSED""" + with patch.dict(os.environ, {'TESTOMATIO_STEPS_PASSED': value}, clear=True): + config = TestRunConfig() + + assert config.enable_steps_for_passed_test is False + def test_to_dict_full_data(self): """Test to_dict with full data""" env_vars = { From 15d5778043b6ca8604d5f4875e4073aebf68c179 Mon Sep 17 00:00:00 2001 From: AenEnlil Date: Fri, 26 Sep 2025 18:28:53 +0300 Subject: [PATCH 2/2] bump: version 2.10.2b0 -> 2.10.2b9 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e7ba53b..346faf3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ version_provider = "pep621" update_changelog_on_bump = false [project] name = "pytestomatio" -version = "2.10.2b0" +version = "2.10.2b9" dependencies = [ "requests>=2.32.4",