diff --git a/qase-behave/changelog.md b/qase-behave/changelog.md index 888a3188..b72e5676 100644 --- a/qase-behave/changelog.md +++ b/qase-behave/changelog.md @@ -1,3 +1,21 @@ +# qase-behave 1.1.6 + +## What's new + +- Improved test failure status handling +- Enhanced error classification to distinguish assertion errors from other failures +- Assertion errors (containing keywords like 'assert', 'AssertionError', 'expect', 'should', 'must') now map to `failed` status +- Non-assertion errors (setup failures, exceptions, etc.) now map to `invalid` status +- Updated dependency on qase-python-commons to version 3.5.5 + +## Migration Guide + +The formatter now provides more accurate test result reporting by distinguishing between: +- `failed`: Test failed due to assertion error (test logic issue) +- `invalid`: Test failed due to non-assertion error (infrastructure/setup issue) + +This change provides better insights into test failures and helps identify whether issues are related to test logic or infrastructure problems. + # qase-behave 1.1.5 ## What's new diff --git a/qase-behave/pyproject.toml b/qase-behave/pyproject.toml index 12e96533..88397a54 100644 --- a/qase-behave/pyproject.toml +++ b/qase-behave/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "qase-behave" -version = "1.1.5" +version = "1.1.6" description = "Qase Behave Plugin for Qase TestOps and Qase Report" readme = "README.md" keywords = ["qase", "behave", "plugin", "testops", "report", "qase reporting", "test observability"] @@ -29,7 +29,7 @@ classifiers = [ ] requires-python = ">=3.7" dependencies = [ - "qase-python-commons~=3.5.4", + "qase-python-commons~=3.5.5", "behave>=1.2.6", "more_itertools", ] diff --git a/qase-behave/src/qase/behave/formatter.py b/qase-behave/src/qase/behave/formatter.py index 418e992c..f0cbed92 100644 --- a/qase-behave/src/qase/behave/formatter.py +++ b/qase-behave/src/qase/behave/formatter.py @@ -46,7 +46,21 @@ def scenario(self, scenario: Scenario): def result(self, result: Step): step = parse_step(result) if step.execution.status != 'passed': - self.__current_scenario.execution.set_status(step.execution.status) + # Check if it's an assertion error or other error + is_assertion_error = False + if result.error_message: + # Check if the error message contains assertion-related keywords + assertion_keywords = ['assert', 'AssertionError', 'expect', 'should', 'must'] + is_assertion_error = any(keyword in result.error_message for keyword in assertion_keywords) + + # Set appropriate status + if step.execution.status == 'failed': + status = 'failed' if is_assertion_error else 'invalid' + step.execution.set_status(status) + self.__current_scenario.execution.set_status(status) + else: + self.__current_scenario.execution.set_status(step.execution.status) + if result.error_message: self.__current_scenario.execution.stacktrace = result.error_message self.__current_scenario.steps.append(step) diff --git a/qase-behave/src/qase/behave/utils.py b/qase-behave/src/qase/behave/utils.py index 0f6c5673..519b2325 100644 --- a/qase-behave/src/qase/behave/utils.py +++ b/qase-behave/src/qase/behave/utils.py @@ -111,7 +111,17 @@ def parse_step(step: Step) -> QaseStep: ) current_time = time.time() - model.execution.set_status(step.status.name) + + # Map behave status to qase status + status_mapping = { + 'passed': 'passed', + 'failed': 'failed', # This will be updated in formatter based on error type + 'skipped': 'skipped', + 'undefined': 'skipped', + 'pending': 'skipped' + } + + model.execution.set_status(status_mapping.get(step.status.name, 'skipped')) model.execution.start_time = current_time - step.duration model.execution.duration = int(step.duration * 1000) model.execution.end_time = current_time diff --git a/qase-pytest/changelog.md b/qase-pytest/changelog.md index 0fa58397..8acc6621 100644 --- a/qase-pytest/changelog.md +++ b/qase-pytest/changelog.md @@ -1,3 +1,21 @@ +# qase-pytest 6.3.9 + +## What's new + +- Improved test failure status handling +- Enhanced error classification to distinguish assertion errors from other failures +- Assertion errors (AssertionError) now map to `failed` status +- Non-assertion errors (setup failures, exceptions, etc.) now map to `invalid` status +- Updated dependency on qase-python-commons to version 3.5.5 + +## Migration Guide + +The plugin now provides more accurate test result reporting by distinguishing between: +- `failed`: Test failed due to assertion error (test logic issue) +- `invalid`: Test failed due to non-assertion error (infrastructure/setup issue) + +This change provides better insights into test failures and helps identify whether issues are related to test logic or infrastructure problems. + # qase-pytest 6.3.8 ## What's new diff --git a/qase-pytest/pyproject.toml b/qase-pytest/pyproject.toml index 6867dc00..6f2ae03a 100644 --- a/qase-pytest/pyproject.toml +++ b/qase-pytest/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "qase-pytest" -version = "6.3.8" +version = "6.3.9" description = "Qase Pytest Plugin for Qase TestOps and Qase Report" readme = "README.md" keywords = ["qase", "pytest", "plugin", "testops", "report", "qase reporting", "test observability"] @@ -29,7 +29,7 @@ classifiers = [ ] requires-python = ">=3.7" dependencies = [ - "qase-python-commons~=3.5.4", + "qase-python-commons~=3.5.5", "pytest>=7.4.4", "filelock~=3.12.2", "more_itertools", diff --git a/qase-robotframework/changelog.md b/qase-robotframework/changelog.md index d495fb0e..485609fd 100644 --- a/qase-robotframework/changelog.md +++ b/qase-robotframework/changelog.md @@ -1,3 +1,21 @@ +# qase-robotframework 3.4.5 + +## What's new + +- Improved test failure status handling +- Enhanced error classification to distinguish assertion errors from other failures +- Assertion errors (containing keywords like 'assert', 'AssertionError', 'expect', 'should', 'must', 'equal', 'not equal') now map to `failed` status +- Non-assertion errors (setup failures, exceptions, etc.) now map to `invalid` status +- Updated dependency on qase-python-commons to version 3.5.5 + +## Migration Guide + +The listener now provides more accurate test result reporting by distinguishing between: +- `failed`: Test failed due to assertion error (test logic issue) +- `invalid`: Test failed due to non-assertion error (infrastructure/setup issue) + +This change provides better insights into test failures and helps identify whether issues are related to test logic or infrastructure problems. + # qase-robotframework 3.4.4 ## What's new diff --git a/qase-robotframework/changelog_new.md b/qase-robotframework/changelog_new.md new file mode 100644 index 00000000..91f66eff --- /dev/null +++ b/qase-robotframework/changelog_new.md @@ -0,0 +1,18 @@ +# qase-robotframework 3.4.5 + +## What's new + +- Improved test failure status handling +- Enhanced error classification to distinguish assertion errors from other failures +- Assertion errors (containing keywords like 'assert', 'AssertionError', 'expect', 'should', 'must', 'equal', 'not equal') now map to `failed` status +- Non-assertion errors (setup failures, exceptions, etc.) now map to `invalid` status +- Updated dependency on qase-python-commons to version 3.5.5 + +## Migration Guide + +The listener now provides more accurate test result reporting by distinguishing between: +- `failed`: Test failed due to assertion error (test logic issue) +- `invalid`: Test failed due to non-assertion error (infrastructure/setup issue) + +This change provides better insights into test failures and helps identify whether issues are related to test logic or infrastructure problems. + diff --git a/qase-robotframework/pyproject.toml b/qase-robotframework/pyproject.toml index d6a40300..ac5d31f3 100644 --- a/qase-robotframework/pyproject.toml +++ b/qase-robotframework/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "qase-robotframework" -version = "3.4.4" +version = "3.4.5" description = "Qase Robot Framework Plugin" readme = "README.md" authors = [{name = "Qase Team", email = "support@qase.io"}] @@ -17,7 +17,7 @@ classifiers = [ urls = {"Homepage" = "https://github.com/qase-tms/qase-python/tree/main/qase-robotframework"} requires-python = ">=3.7" dependencies = [ - "qase-python-commons~=3.5.3", + "qase-python-commons~=3.5.5", "filelock~=3.12.2", ] diff --git a/qase-robotframework/src/qase/robotframework/listener.py b/qase-robotframework/src/qase/robotframework/listener.py index 25ef2161..9a3d80d9 100644 --- a/qase-robotframework/src/qase/robotframework/listener.py +++ b/qase-robotframework/src/qase/robotframework/listener.py @@ -98,7 +98,16 @@ def end_test(self, test, result): self.runtime.result.testops_ids = test_metadata.qase_ids self.runtime.result.execution.complete() - self.runtime.result.execution.set_status(STATUSES[result.status]) + + # Determine if it's an assertion error or other error + status = STATUSES[result.status] + if status == "failed" and hasattr(result, 'message'): + # Check if the error message contains assertion-related keywords + assertion_keywords = ['assert', 'AssertionError', 'expect', 'should', 'must', 'equal', 'not equal'] + is_assertion_error = any(keyword in result.message for keyword in assertion_keywords) + status = "failed" if is_assertion_error else "invalid" + + self.runtime.result.execution.set_status(status) if hasattr(result, 'message'): self.runtime.result.execution.stacktrace = result.message @@ -197,7 +206,13 @@ def __parse_steps(self, result) -> List[Step]: data=StepGherkinData(keyword=step_name, name=step_name, line=0) ) - step.execution.set_status(STATUSES[result.body[i].status]) + # Determine step status + step_status = STATUSES[result.body[i].status] + if step_status == "failed": + # For steps, we'll use a simpler approach - check if it's a keyword failure + # Robot Framework doesn't provide detailed error info for individual steps + step_status = "failed" # Keep as failed for steps, main test status is handled above + step.execution.set_status(step_status) step.execution.start_time = result.body[i].start_time.timestamp() step.execution.duration = result.body[i].elapsed_time.microseconds step.execution.end_time = result.body[i].end_time.timestamp() diff --git a/qase-tavern/changelog.md b/qase-tavern/changelog.md index 16ae6d3d..fba7f28c 100644 --- a/qase-tavern/changelog.md +++ b/qase-tavern/changelog.md @@ -1,3 +1,21 @@ +# qase-tavern 1.1.4 + +## What's new + +- Improved test failure status handling +- Enhanced error classification to distinguish assertion errors from other failures +- Assertion errors (AssertionError) now map to `failed` status +- Non-assertion errors (setup failures, exceptions, etc.) now map to `invalid` status +- Updated dependency on qase-python-commons to version 3.5.5 + +## Migration Guide + +The plugin now provides more accurate test result reporting by distinguishing between: +- `failed`: Test failed due to assertion error (test logic issue) +- `invalid`: Test failed due to non-assertion error (infrastructure/setup issue) + +This change provides better insights into test failures and helps identify whether issues are related to test logic or infrastructure problems. + # qase-tavern 1.1.3 ## What's new diff --git a/qase-tavern/pyproject.toml b/qase-tavern/pyproject.toml index 22e6cbee..33602ea6 100644 --- a/qase-tavern/pyproject.toml +++ b/qase-tavern/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "qase-tavern" -version = "1.1.3" +version = "1.1.4" description = "Qase Tavern Plugin for Qase TestOps and Qase Report" readme = "README.md" keywords = ["qase", "tavern", "plugin", "testops", "report", "qase reporting", "test observability"] @@ -29,7 +29,7 @@ classifiers = [ ] requires-python = ">=3.7" dependencies = [ - "qase-python-commons~=3.5.4", + "qase-python-commons~=3.5.5", "pytest>=7,<7.3", "filelock~=3.12.2", "more_itertools", diff --git a/qase-tavern/src/qase/tavern/plugin.py b/qase-tavern/src/qase/tavern/plugin.py index 92ebbd66..d3ac871e 100644 --- a/qase-tavern/src/qase/tavern/plugin.py +++ b/qase-tavern/src/qase/tavern/plugin.py @@ -35,7 +35,10 @@ def pytest_runtest_protocol(self, item): def pytest_runtest_makereport(self, item, call): if call.when == "call": if call.excinfo: - self.runtime.result.execution.status = "failed" + # Determine if it's an assertion error or other error + is_assertion_error = call.excinfo.typename == "AssertionError" + status = "failed" if is_assertion_error else "invalid" + self.runtime.result.execution.status = status if hasattr(call.excinfo, "value"): self.runtime.result.execution.stacktrace = '\n'.join(call.excinfo.value.args) if hasattr(call.excinfo.value, "failures"):