examples) {
/**
* Represents a single iteration for a "Manual" style test case in XRAY.
*
diff --git a/src/test/java/com/google/pdsl/xray/XrayIntegrationTest.java b/src/test/java/com/google/pdsl/xray/XrayIntegrationTest.java
index 21cd5e4..543da47 100644
--- a/src/test/java/com/google/pdsl/xray/XrayIntegrationTest.java
+++ b/src/test/java/com/google/pdsl/xray/XrayIntegrationTest.java
@@ -27,11 +27,15 @@
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Paths;
-import java.util.*;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
import java.util.function.Supplier;
import java.util.stream.Stream;
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
/*
Copyright 2025 Google LLC
@@ -47,122 +51,124 @@
See the License for the specific language governing permissions and
limitations under the License.
*/
+
/**
* This class contains JUnit Jupiter tests for Xray integration. It uses the Pdsl framework to
* execute Gherkin scenarios and integrates with Xray for test management and reporting.
*/
public class XrayIntegrationTest {
- private static final Properties properties = initProperties();
-
- private static final XrayAuth xrayAuth = XrayAuth.fromPropertiesFile("src/test/resources/xray.properties");
- private static final XrayTestResultUpdater updater = new XrayTestResultUpdater.Builder(
- "PDSL-XRAY Plugin E2E Tests",
- """
- End to end tests for the pdsl-xray plugin.
- These tests support the gherkin protocol both through special fields in
- the examples table or tags directly above scenarios:
- |xray-test-plan | xray-test-case | xray-test-env |
- """,
- () -> Map.of(
- "fields", Map.of(
- "project", Map.of("key", properties.get("xray.project.key")),
- "summary", "Automated test run by Polymorphic DSL Test Framework",
- "issuetype", Map.of("name", "Test Execution"),
- "assignee", Map.of("accountId", properties.get("xray.reporter.accountId")),
- "reporter", Map.of("accountId", properties.get("xray.reporter.accountId"))
- )
- )).withXrayAuth(xrayAuth)
- .build();
-
- private static final DefaultPolymorphicDslTestExecutor traceableTestRunExecutor = new DefaultPolymorphicDslTestExecutor();
- private static final PolymorphicDslPhraseFilter MY_CUSTOM_PDSL_PHRASE_FILTER = new MyCustomPDSLPhraseFilter();
- private static final PickleJarFactory PICKLE_JAR_FACTORY = init();
- private static final Supplier parseTreeListenerSupplier = AllGrammarsParserBaseListener::new;
-
- private static Properties initProperties() {
- Properties properties = new Properties();
- try {
- properties.load(new FileInputStream("src/test/resources/xray.properties"));
- return properties;
- } catch (IOException e) {
- throw new IllegalStateException(e);
- }
- }
-
- private static PickleJarFactory init() {
- traceableTestRunExecutor.registerObserver(updater);
- PickleJarFactory PICKLE_JAR_FACTORY = PickleJarFactory.getDefaultPickleJarFactory();
- PICKLE_JAR_FACTORY.registerObserver(updater);
- return PICKLE_JAR_FACTORY;
- }
-
- @TestTemplate
- @ExtendWith(IosExtension.class)
- public void iosTest(PdslExecutable executable) {
- executable.execute();
- }
-
- @TestTemplate
- @ExtendWith(AndroidExtension.class)
- public void androidTest(PdslExecutable executable) {
- executable.execute();
- }
-
- private static PdslConfigParameter createParameterWithTag(String tag) {
- return PdslConfigParameter.createGherkinPdslConfig(
- List.of(
- new PdslTestParameter.Builder(parseTreeListenerSupplier,
- AllGrammarsLexer.class, AllGrammarsParser.class)
- .withTagExpression(tag)
- .withIncludedResources(new String[]{"XRayIntegration.feature", "PdslXrayTabular.feature"})
- .build()
+ private static final Properties properties = initProperties();
+
+ private static final XrayAuth xrayAuth = XrayAuth.fromPropertiesFile("src/test/resources/xray.properties");
+ private static final XrayTestResultUpdater updater = new XrayTestResultUpdater.Builder(
+ "PDSL-XRAY Plugin E2E Tests",
+ """
+ End to end tests for the pdsl-xray plugin.
+ These tests support the gherkin protocol both through special fields in
+ the examples table or tags directly above scenarios:
+ |xray-test-plan | xray-test-case | xray-test-env |
+ """,
+ () -> Map.of(
+ "fields", Map.of(
+ "project", Map.of("key", properties.get("xray.project.key")),
+ "summary", "Automated test run by Polymorphic DSL Test Framework",
+ "issuetype", Map.of("name", "Test Execution"),
+ "assignee", Map.of("accountId", properties.get("xray.reporter.accountId")),
+ "reporter", Map.of("accountId", properties.get("xray.reporter.accountId"))
)
- )
- .withApplicationName("Polymorphic DSL Framework")
- .withContext("User Acceptance Test")
- .withResourceRoot(Paths.get("src/test/resources/features").toUri())
- .withRecognizerRule("polymorphicDslAllRules")
- .withTestRunExecutor(() -> traceableTestRunExecutor)
- .withTestSpecificationFactoryGenerator(
- () -> new DefaultGherkinTestSpecificationFactoryGenerator(
- new DefaultGherkinTestSpecificationFactory.Builder((MY_CUSTOM_PDSL_PHRASE_FILTER))
- .withPickleJarFactory(PICKLE_JAR_FACTORY)))
+ )).withXrayAuth(xrayAuth)
.build();
- }
- /**
- * A supplier that provides an instance of AllGrammarsParserBaseListener.
- */
- private static class IosExtension extends PdslGherkinInvocationContextProvider {
-
- @Override
- public Stream provideTestTemplateInvocationContexts(ExtensionContext context) {
- return getInvocationContext(createParameterWithTag("@ios")).stream();
+
+ private static final DefaultPolymorphicDslTestExecutor traceableTestRunExecutor = new DefaultPolymorphicDslTestExecutor();
+ private static final PolymorphicDslPhraseFilter MY_CUSTOM_PDSL_PHRASE_FILTER = new MyCustomPDSLPhraseFilter();
+ private static final PickleJarFactory PICKLE_JAR_FACTORY = init();
+ private static final Supplier parseTreeListenerSupplier = AllGrammarsParserBaseListener::new;
+
+ private static Properties initProperties() {
+ Properties properties = new Properties();
+ try {
+ properties.load(new FileInputStream("src/test/resources/xray.properties"));
+ return properties;
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private static PickleJarFactory init() {
+ traceableTestRunExecutor.registerObserver(updater);
+ PickleJarFactory PICKLE_JAR_FACTORY = PickleJarFactory.getDefaultPickleJarFactory();
+ PICKLE_JAR_FACTORY.registerObserver(updater);
+ return PICKLE_JAR_FACTORY;
+ }
+
+ private static PdslConfigParameter createParameterWithTag(String tag) {
+ return PdslConfigParameter.createGherkinPdslConfig(
+ List.of(
+ new PdslTestParameter.Builder(parseTreeListenerSupplier,
+ AllGrammarsLexer.class, AllGrammarsParser.class)
+ .withTagExpression(tag)
+ .withIncludedResources(new String[]{"XRayIntegration.feature", "PdslXrayTabular.feature"})
+ .build()
+ )
+ )
+ .withApplicationName("Polymorphic DSL Framework")
+ .withContext("User Acceptance Test")
+ .withResourceRoot(Paths.get("src/test/resources/features").toUri())
+ .withRecognizerRule("polymorphicDslAllRules")
+ .withTestRunExecutor(() -> traceableTestRunExecutor)
+ .withTestSpecificationFactoryGenerator(
+ () -> new DefaultGherkinTestSpecificationFactoryGenerator(
+ new DefaultGherkinTestSpecificationFactory.Builder((MY_CUSTOM_PDSL_PHRASE_FILTER))
+ .withPickleJarFactory(PICKLE_JAR_FACTORY)))
+ .build();
+ }
+
+ /**
+ * Publishes the test results to Xray after all tests have been executed.
+ */
+ @AfterAll
+ public static void publishReportsToXray() {
+ // Validation: Check if the updater has created a valid Xray payload.
+ assertNotNull(updater.getXrayPayload(), "Xray payload is null.");
+ List responses = updater.publishReportsToXray();
+ assertFalse(responses.isEmpty());
}
- }
- private static class AndroidExtension extends PdslGherkinInvocationContextProvider {
- @Override
- public Stream provideTestTemplateInvocationContexts(ExtensionContext context) {
- return getInvocationContext(createParameterWithTag("@wip")).stream();
+ @TestTemplate
+ @ExtendWith(IosExtension.class)
+ public void iosTest(PdslExecutable executable) {
+ executable.execute();
}
- }
-
- /**
- * Publishes the test results to Xray after all tests have been executed.
- */
- @AfterAll
- public static void publishReportsToXray() {
- // Validation: Check if the updater has created a valid Xray payload.
- assertNotNull(updater.getXrayPayload(), "Xray payload is null.");
- List responses = updater.publishReportsToXray();
- assertFalse(responses.isEmpty());
- }
-
- private static class MyCustomPDSLPhraseFilter implements PolymorphicDslPhraseFilter {
- @Override
- public Optional> filterPhrases(List testInput) {
- return Optional.empty();
+
+ @TestTemplate
+ @ExtendWith(AndroidExtension.class)
+ public void androidTest(PdslExecutable executable) {
+ executable.execute();
+ }
+
+ /**
+ * A supplier that provides an instance of AllGrammarsParserBaseListener.
+ */
+ private static class IosExtension extends PdslGherkinInvocationContextProvider {
+
+ @Override
+ public Stream provideTestTemplateInvocationContexts(ExtensionContext context) {
+ return getInvocationContext(createParameterWithTag("@ios")).stream();
+ }
+ }
+
+ private static class AndroidExtension extends PdslGherkinInvocationContextProvider {
+ @Override
+ public Stream provideTestTemplateInvocationContexts(ExtensionContext context) {
+ return getInvocationContext(createParameterWithTag("@wip")).stream();
+ }
+ }
+
+ private static class MyCustomPDSLPhraseFilter implements PolymorphicDslPhraseFilter {
+ @Override
+ public Optional> filterPhrases(List testInput) {
+ return Optional.empty();
+ }
}
- }
}
diff --git a/src/test/java/com/google/pdsl/xray/core/XrayTestResultUpdaterTest.java b/src/test/java/com/google/pdsl/xray/core/XrayTestResultUpdaterTest.java
index a947886..c17dceb 100644
--- a/src/test/java/com/google/pdsl/xray/core/XrayTestResultUpdaterTest.java
+++ b/src/test/java/com/google/pdsl/xray/core/XrayTestResultUpdaterTest.java
@@ -1,21 +1,37 @@
package com.google.pdsl.xray.core;
+import com.google.pdsl.xray.constants.StepStatus;
+import com.google.pdsl.xray.models.XrayTestExecution;
+import com.google.pdsl.xray.models.XrayTestResult;
+import com.pdsl.reports.TestResult;
+import com.pdsl.reports.proto.TechnicalReportData;
+import com.pdsl.specifications.Phrase;
+import com.pdsl.testcases.TaggedTestCase;
+import com.pdsl.testcases.TestCase;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
+import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import java.io.IOException;
+import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.Collection;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Properties;
+import java.util.Set;
import java.util.function.Supplier;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class XrayTestResultUpdaterTest {
@@ -59,4 +75,130 @@ void build_withXrayAuthProperties_doesNotThrowException() throws IOException {
assertDoesNotThrow(() -> xrayTestResultUpdaterBuilder.withPropertiesPath(propertiesFile).build());
}
+
+ @Test
+ void addResults_withStepLevelComments_allPassed() {
+ XrayTestResultUpdater updater = xrayTestResultUpdaterBuilder.withXrayAuth(xrayAuth).build();
+
+ Map> stepComments = new HashMap<>();
+ stepComments.put(1, List.of("@xray-test-case=STEP-KEY-1"));
+ stepComments.put(3, List.of("@xray-test-case=STEP-KEY-3"));
+ TaggedTestCase testCase = createMockTestCase(stepComments);
+
+ // Mock TestResult
+ TestResult result = Mockito.mock(TestResult.class);
+ when(result.getTestCase()).thenReturn(testCase);
+ when(result.getStatus()).thenReturn(TechnicalReportData.Status.PASSED);
+
+ updater.addResults(List.of(result));
+
+ XrayTestExecution testExecution = getTestExecution(updater, "EXEC-123");
+
+ assertEquals(3, testExecution.tests().size());
+ assertEquals(StepStatus.PASSED.name(), getTestStatus(testExecution, "STEP-KEY-1"));
+ assertEquals(StepStatus.PASSED.name(), getTestStatus(testExecution, "STEP-KEY-3"));
+ assertEquals(StepStatus.PASSED.name(), getTestStatus(testExecution, "SCENARIO-KEY"));
+ }
+
+ @Test
+ void addResults_withStepLevelComments_stepFailed() {
+ XrayTestResultUpdater updater = xrayTestResultUpdaterBuilder.withXrayAuth(xrayAuth).build();
+
+ // Setup step comments:
+ // Step 1 (index 1) has STEP-KEY-1 -> should be PASSED
+ // Step 2 (index 2) has STEP-KEY-2 -> should be FAILED
+ // Step 3 (index 3) has STEP-KEY-3 -> should be BLOCKED
+ Map> stepComments = new HashMap<>();
+ stepComments.put(1, List.of("@xray-test-case=STEP-KEY-1"));
+ stepComments.put(2, List.of("@xray-test-case=STEP-KEY-2"));
+ stepComments.put(3, List.of("@xray-test-case=STEP-KEY-3"));
+ TaggedTestCase testCase = createMockTestCase(stepComments);
+
+ Phrase failingPhrase = Mockito.mock(Phrase.class);
+ when(failingPhrase.getPrefilteredIndex()).thenReturn(2);
+
+ // Mock TestResult
+ TestResult result = Mockito.mock(TestResult.class);
+ when(result.getTestCase()).thenReturn(testCase);
+ when(result.getStatus()).thenReturn(TechnicalReportData.Status.FAILED);
+ when(result.getFailureReason()).thenReturn(Optional.of(new RuntimeException("Test Failure")));
+ when(result.getFailingPhrase()).thenReturn(Optional.of(failingPhrase));
+
+ updater.addResults(List.of(result));
+
+ XrayTestExecution testExecution = getTestExecution(updater, "EXEC-123");
+
+ // SCENARIO-KEY, STEP-KEY-1, STEP-KEY-2, STEP-KEY-3
+ assertEquals(4, testExecution.tests().size());
+ assertEquals(StepStatus.FAILED.name(), getTestStatus(testExecution, "SCENARIO-KEY"));
+ assertEquals(StepStatus.PASSED.name(), getTestStatus(testExecution, "STEP-KEY-1"));
+ assertEquals(StepStatus.FAILED.name(), getTestStatus(testExecution, "STEP-KEY-2"));
+ assertEquals(StepStatus.BLOCKED.name(), getTestStatus(testExecution, "STEP-KEY-3"));
+ }
+
+ @Test
+ void addResults_withDuplicateStepLevelComments_avoidsDuplicatesAndConsolidatesStatus() {
+ XrayTestResultUpdater updater = xrayTestResultUpdaterBuilder.withXrayAuth(xrayAuth).build();
+
+ // given
+ // Step 1 (index 1) has STEP-KEY-1 -> should be PASSED
+ // Step 2 (index 2) has STEP-KEY-1 (duplicate) -> should be FAILED
+ // Step 3 (index 3) has STEP-KEY-1 (duplicate) -> should be BLOCKED
+ Map> stepComments = new HashMap<>();
+ stepComments.put(1, List.of("@xray-test-case=STEP-KEY-1"));
+ stepComments.put(2, List.of("@xray-test-case=STEP-KEY-1"));
+ stepComments.put(3, List.of("@xray-test-case=STEP-KEY-1"));
+ TaggedTestCase testCase = createMockTestCase(stepComments);
+
+ Phrase failingPhrase = Mockito.mock(Phrase.class);
+ when(failingPhrase.getPrefilteredIndex()).thenReturn(1);
+
+ // when
+ TestResult result = Mockito.mock(TestResult.class);
+ when(result.getTestCase()).thenReturn(testCase);
+ when(result.getStatus()).thenReturn(TechnicalReportData.Status.FAILED);
+ when(result.getFailingPhrase()).thenReturn(Optional.of(failingPhrase));
+
+ updater.addResults(List.of(result));
+
+ XrayTestExecution testExecution = getTestExecution(updater, "EXEC-123");
+
+ // then
+ // Only 2 distinct tests: SCENARIO-KEY, STEP-KEY-1 (no duplicates)
+ assertEquals(2, testExecution.tests().size());
+ assertEquals(StepStatus.FAILED.name(), getTestStatus(testExecution, "SCENARIO-KEY"));
+ // Overall status of STEP-KEY-1 should be FAILED since one of the steps failed
+ assertEquals(StepStatus.FAILED.name(), getTestStatus(testExecution, "STEP-KEY-1"));
+ }
+
+ private TaggedTestCase createMockTestCase(Map> stepComments) {
+ TaggedTestCase testCase = Mockito.mock(TaggedTestCase.class);
+ when(testCase.getTags()).thenReturn(Set.of("@xray-test-plan=PLAN-123", "@xray-test-execution=EXEC-123", "@xray-test-case=SCENARIO-KEY"));
+ when(testCase.getOriginalSource()).thenReturn(URI.create("file:/some/path?ruleIndex=1&ordinal=2&tableIndex=3"));
+ when(testCase.getTestTitle()).thenReturn("My Scenario");
+ when(testCase.getUnfilteredPhraseBody()).thenReturn(List.of("Given step one", "When step two", "Then step three"));
+
+ Map metadata = new HashMap<>();
+ metadata.put(TestCase.STEP_COMMENTS, stepComments);
+ when(testCase.getMetadata()).thenReturn(metadata);
+ return testCase;
+ }
+
+ private XrayTestExecution getTestExecution(XrayTestResultUpdater updater, String testExecutionKey) {
+
+ Collection payloads = updater.getXrayPayload().stream().toList();
+ assertEquals(1, payloads.size());
+ XrayTestExecution testExecution = payloads.iterator().next();
+ assertEquals(testExecutionKey, testExecution.testExecutionKey());
+ return testExecution;
+ }
+
+ private String getTestStatus(XrayTestExecution testExecution, String testKey) {
+ XrayTestResult result = testExecution.tests().stream()
+ .filter(r -> r.testKey().equals(testKey))
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("Could not find test with key " + testKey));
+ return result.status();
+ }
+
}
\ No newline at end of file
diff --git a/src/test/resources/xray.properties b/src/test/resources/xray.properties
index 0b3f93e..a5b1897 100644
--- a/src/test/resources/xray.properties
+++ b/src/test/resources/xray.properties
@@ -5,4 +5,3 @@ xray.api.report.url=https://xray.cloud.getxray.app/api/v2/import/execution/multi
xray.environments=PRD
xray.reporter.accountId=
xray.project.key=
-