feat: release version 1.1.6 with improved test failure status handling

qase-behave/changelog.md

@@ -1,3 +1,21 @@
+# qase-behave 1.1.6
+
+## What's new
+
+- Improved test failure status handling
+- Enhanced error classification to distinguish assertion errors from other failures
+- Assertion errors (containing keywords like 'assert', 'AssertionError', 'expect', 'should', 'must') now map to `failed` status
+- Non-assertion errors (setup failures, exceptions, etc.) now map to `invalid` status
+- Updated dependency on qase-python-commons to version 3.5.5
+
+## Migration Guide
+
+The formatter now provides more accurate test result reporting by distinguishing between:
+- `failed`: Test failed due to assertion error (test logic issue)
+- `invalid`: Test failed due to non-assertion error (infrastructure/setup issue)
+
+This change provides better insights into test failures and helps identify whether issues are related to test logic or infrastructure problems.
+
 # qase-behave 1.1.5
 
 ## What's new

qase-behave/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "qase-behave"
-version = "1.1.5"
+version = "1.1.6"
 description = "Qase Behave Plugin for Qase TestOps and Qase Report"
 readme = "README.md"
 keywords = ["qase", "behave", "plugin", "testops", "report", "qase reporting", "test observability"]
@@ -29,7 +29,7 @@ classifiers = [
 ]
 requires-python = ">=3.7"
 dependencies = [
-    "qase-python-commons~=3.5.4",
+    "qase-python-commons~=3.5.5",
     "behave>=1.2.6",
     "more_itertools",
 ]

qase-behave/src/qase/behave/formatter.py

@@ -46,7 +46,21 @@ def scenario(self, scenario: Scenario):
     def result(self, result: Step):
         step = parse_step(result)
         if step.execution.status != 'passed':
-            self.__current_scenario.execution.set_status(step.execution.status)
+            # Check if it's an assertion error or other error
+            is_assertion_error = False
+            if result.error_message:
+                # Check if the error message contains assertion-related keywords
+                assertion_keywords = ['assert', 'AssertionError', 'expect', 'should', 'must']
+                is_assertion_error = any(keyword in result.error_message for keyword in assertion_keywords)
+
+            # Set appropriate status
+            if step.execution.status == 'failed':
+                status = 'failed' if is_assertion_error else 'invalid'
+                step.execution.set_status(status)
+                self.__current_scenario.execution.set_status(status)
+            else:
+                self.__current_scenario.execution.set_status(step.execution.status)
+
             if result.error_message:
                 self.__current_scenario.execution.stacktrace = result.error_message
         self.__current_scenario.steps.append(step)

qase-behave/src/qase/behave/utils.py

@@ -111,7 +111,17 @@ def parse_step(step: Step) -> QaseStep:
     )
 
     current_time = time.time()
-    model.execution.set_status(step.status.name)
+
+    # Map behave status to qase status
+    status_mapping = {
+        'passed': 'passed',
+        'failed': 'failed',  # This will be updated in formatter based on error type
+        'skipped': 'skipped',
+        'undefined': 'skipped',
+        'pending': 'skipped'
+    }
+
+    model.execution.set_status(status_mapping.get(step.status.name, 'skipped'))
     model.execution.start_time = current_time - step.duration
     model.execution.duration = int(step.duration * 1000)
     model.execution.end_time = current_time

qase-pytest/changelog.md

@@ -1,3 +1,21 @@
+# qase-pytest 6.3.9
+
+## What's new
+
+- Improved test failure status handling
+- Enhanced error classification to distinguish assertion errors from other failures
+- Assertion errors (AssertionError) now map to `failed` status
+- Non-assertion errors (setup failures, exceptions, etc.) now map to `invalid` status
+- Updated dependency on qase-python-commons to version 3.5.5
+
+## Migration Guide
+
+The plugin now provides more accurate test result reporting by distinguishing between:
+- `failed`: Test failed due to assertion error (test logic issue)
+- `invalid`: Test failed due to non-assertion error (infrastructure/setup issue)
+
+This change provides better insights into test failures and helps identify whether issues are related to test logic or infrastructure problems.
+
 # qase-pytest 6.3.8
 
 ## What's new

qase-pytest/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "qase-pytest"
-version = "6.3.8"
+version = "6.3.9"
 description = "Qase Pytest Plugin for Qase TestOps and Qase Report"
 readme = "README.md"
 keywords = ["qase", "pytest", "plugin", "testops", "report", "qase reporting", "test observability"]
@@ -29,7 +29,7 @@ classifiers = [
 ]
 requires-python = ">=3.7"
 dependencies = [
-    "qase-python-commons~=3.5.4",
+    "qase-python-commons~=3.5.5",
     "pytest>=7.4.4",
     "filelock~=3.12.2",
     "more_itertools",

qase-robotframework/changelog.md

@@ -1,3 +1,21 @@
+# qase-robotframework 3.4.5
+
+## What's new
+
+- Improved test failure status handling
+- Enhanced error classification to distinguish assertion errors from other failures
+- Assertion errors (containing keywords like 'assert', 'AssertionError', 'expect', 'should', 'must', 'equal', 'not equal') now map to `failed` status
+- Non-assertion errors (setup failures, exceptions, etc.) now map to `invalid` status
+- Updated dependency on qase-python-commons to version 3.5.5
+
+## Migration Guide
+
+The listener now provides more accurate test result reporting by distinguishing between:
+- `failed`: Test failed due to assertion error (test logic issue)
+- `invalid`: Test failed due to non-assertion error (infrastructure/setup issue)
+
+This change provides better insights into test failures and helps identify whether issues are related to test logic or infrastructure problems.
+
 # qase-robotframework 3.4.4
 
 ## What's new

qase-robotframework/changelog_new.md

@@ -0,0 +1,18 @@
+# qase-robotframework 3.4.5
+
+## What's new
+
+- Improved test failure status handling
+- Enhanced error classification to distinguish assertion errors from other failures
+- Assertion errors (containing keywords like 'assert', 'AssertionError', 'expect', 'should', 'must', 'equal', 'not equal') now map to `failed` status
+- Non-assertion errors (setup failures, exceptions, etc.) now map to `invalid` status
+- Updated dependency on qase-python-commons to version 3.5.5
+
+## Migration Guide
+
+The listener now provides more accurate test result reporting by distinguishing between:
+- `failed`: Test failed due to assertion error (test logic issue)
+- `invalid`: Test failed due to non-assertion error (infrastructure/setup issue)
+
+This change provides better insights into test failures and helps identify whether issues are related to test logic or infrastructure problems.
+

qase-robotframework/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "qase-robotframework"
-version = "3.4.4"
+version = "3.4.5"
 description = "Qase Robot Framework Plugin"
 readme = "README.md"
 authors = [{name = "Qase Team", email = "support@qase.io"}]
@@ -17,7 +17,7 @@ classifiers = [
 urls = {"Homepage" = "https://github.com/qase-tms/qase-python/tree/main/qase-robotframework"}
 requires-python = ">=3.7"
 dependencies = [
-    "qase-python-commons~=3.5.3",
+    "qase-python-commons~=3.5.5",
     "filelock~=3.12.2",
 ]
 

qase-robotframework/src/qase/robotframework/listener.py

@@ -98,7 +98,16 @@ def end_test(self, test, result):
             self.runtime.result.testops_ids = test_metadata.qase_ids
 
         self.runtime.result.execution.complete()
-        self.runtime.result.execution.set_status(STATUSES[result.status])
+
+        # Determine if it's an assertion error or other error
+        status = STATUSES[result.status]
+        if status == "failed" and hasattr(result, 'message'):
+            # Check if the error message contains assertion-related keywords
+            assertion_keywords = ['assert', 'AssertionError', 'expect', 'should', 'must', 'equal', 'not equal']
+            is_assertion_error = any(keyword in result.message for keyword in assertion_keywords)
+            status = "failed" if is_assertion_error else "invalid"
+
+        self.runtime.result.execution.set_status(status)
         if hasattr(result, 'message'):
             self.runtime.result.execution.stacktrace = result.message
 
@@ -197,7 +206,13 @@ def __parse_steps(self, result) -> List[Step]:
                 data=StepGherkinData(keyword=step_name, name=step_name, line=0)
             )
 
-            step.execution.set_status(STATUSES[result.body[i].status])
+            # Determine step status
+            step_status = STATUSES[result.body[i].status]
+            if step_status == "failed":
+                # For steps, we'll use a simpler approach - check if it's a keyword failure
+                # Robot Framework doesn't provide detailed error info for individual steps
+                step_status = "failed"  # Keep as failed for steps, main test status is handled above
+            step.execution.set_status(step_status)
             step.execution.start_time = result.body[i].start_time.timestamp()
             step.execution.duration = result.body[i].elapsed_time.microseconds
             step.execution.end_time = result.body[i].end_time.timestamp()

qase-tavern/changelog.md

@@ -1,3 +1,21 @@
+# qase-tavern 1.1.4
+
+## What's new
+
+- Improved test failure status handling
+- Enhanced error classification to distinguish assertion errors from other failures
+- Assertion errors (AssertionError) now map to `failed` status
+- Non-assertion errors (setup failures, exceptions, etc.) now map to `invalid` status
+- Updated dependency on qase-python-commons to version 3.5.5
+
+## Migration Guide
+
+The plugin now provides more accurate test result reporting by distinguishing between:
+- `failed`: Test failed due to assertion error (test logic issue)
+- `invalid`: Test failed due to non-assertion error (infrastructure/setup issue)
+
+This change provides better insights into test failures and helps identify whether issues are related to test logic or infrastructure problems.
+
 # qase-tavern 1.1.3
 
 ## What's new

qase-tavern/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "qase-tavern"
-version = "1.1.3"
+version = "1.1.4"
 description = "Qase Tavern Plugin for Qase TestOps and Qase Report"
 readme = "README.md"
 keywords = ["qase", "tavern", "plugin", "testops", "report", "qase reporting", "test observability"]
@@ -29,7 +29,7 @@ classifiers = [
 ]
 requires-python = ">=3.7"
 dependencies = [
-    "qase-python-commons~=3.5.4",
+    "qase-python-commons~=3.5.5",
     "pytest>=7,<7.3",
     "filelock~=3.12.2",
     "more_itertools",

qase-tavern/src/qase/tavern/plugin.py

@@ -35,7 +35,10 @@ def pytest_runtest_protocol(self, item):
     def pytest_runtest_makereport(self, item, call):
         if call.when == "call":
             if call.excinfo:
-                self.runtime.result.execution.status = "failed"
+                # Determine if it's an assertion error or other error
+                is_assertion_error = call.excinfo.typename == "AssertionError"
+                status = "failed" if is_assertion_error else "invalid"
+                self.runtime.result.execution.status = status
                 if hasattr(call.excinfo, "value"):
                     self.runtime.result.execution.stacktrace = '\n'.join(call.excinfo.value.args)
                     if hasattr(call.excinfo.value, "failures"):