diff --git a/src/runners/baseRunner/BaseRunner.ts b/src/runners/baseRunner/BaseRunner.ts index b27c4390..7fb9e36e 100644 --- a/src/runners/baseRunner/BaseRunner.ts +++ b/src/runners/baseRunner/BaseRunner.ts @@ -2,9 +2,10 @@ // Licensed under the MIT license. import * as iconv from 'iconv-lite'; +import { randomUUID } from 'crypto'; import { AddressInfo, createServer, Server, Socket } from 'net'; import * as os from 'os'; -import { CancellationToken, debug, DebugConfiguration, DebugSession, Disposable } from 'vscode'; +import { CancellationToken, debug, DebugAdapterTracker, DebugConfiguration, DebugSession, Disposable, ProviderResult } from 'vscode'; import { sendError } from 'vscode-extension-telemetry-wrapper'; import { Configurations } from '../../constants'; import { IProgressReporter } from '../../debugger.api'; @@ -12,6 +13,8 @@ import { ITestRunnerInternal } from '../ITestRunner'; import { RunnerResultAnalyzer } from './RunnerResultAnalyzer'; import { IExecutionConfig, IRunTestContext } from '../../java-test-runner.api'; +const JAVA_TEST_RUN_ID: string = '__javaTestRunId'; + export abstract class BaseRunner implements ITestRunnerInternal { protected server: Server; protected socket: Socket; @@ -56,9 +59,26 @@ export abstract class BaseRunner implements ITestRunnerInternal { // So we force to use internal console here to make sure the session is still under debugger's control. launchConfiguration.console = 'internalConsole'; + const testRunId: string = randomUUID(); + launchConfiguration[JAVA_TEST_RUN_ID] = testRunId; + const isTestSession: (session: DebugSession) => boolean = (session: DebugSession): boolean => + session.configuration[JAVA_TEST_RUN_ID] === testRunId; + + // Mirror the test session's user-visible Debug Console output in Test Results. + this.disposables.push(debug.registerDebugAdapterTrackerFactory('java', { + createDebugAdapterTracker: (session: DebugSession): ProviderResult => { + if (!isTestSession(session)) { + return undefined; + } + return { + onDidSendMessage: (message: any): void => this.handleDebugAdapterMessage(message), + }; + }, + })); + let debugSession: DebugSession | undefined; this.disposables.push(debug.onDidStartDebugSession((session: DebugSession) => { - if (session.name === launchConfiguration.name) { + if (!debugSession && isTestSession(session)) { debugSession = session; } })); @@ -80,7 +100,7 @@ export abstract class BaseRunner implements ITestRunnerInternal { return await new Promise((resolve: () => void): void => { this.disposables.push( debug.onDidTerminateDebugSession((session: DebugSession): void => { - if (launchConfiguration.name === session.name) { + if (session.id === debugSession?.id) { debugSession = undefined; this.tearDown(); if (data.length > 0) { @@ -97,6 +117,19 @@ export abstract class BaseRunner implements ITestRunnerInternal { })); } + protected handleDebugAdapterMessage(message: any): void { + if (message?.type !== 'event' || message.event !== 'output' || message.body?.category === 'telemetry') { + return; + } + + const output: unknown = message.body?.output; + if (typeof output !== 'string' || output.length === 0) { + return; + } + + this.testContext.testRun.appendOutput(output.replace(/\r?\n/g, '\r\n')); + } + public async tearDown(): Promise { try { if (this.socket) { diff --git a/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts b/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts index 06f5719c..ac79b28a 100644 --- a/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts +++ b/src/runners/junitRunner/JUnitRunnerResultAnalyzer.ts @@ -52,8 +52,14 @@ export class JUnitRunnerResultAnalyzer extends RunnerResultAnalyzer { public analyzeData(data: string): void { const lines: string[] = data.split(/\r?\n/); for (const line of lines) { + // The socket stream carries only the JUnit runner's control protocol + // (`%`-prefixed frames plus stack-trace / expected-actual payloads). + // The control frames are noise, and the failure payloads are already + // surfaced structurally as TestMessages on the failed items, so nothing + // here is echoed to the Test Results output. The user-facing program + // output is instead forwarded from the debug session's DAP `output` + // events (see BaseRunner). this.processData(line); - this.testContext.testRun.appendOutput(line + '\r\n'); } } diff --git a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts index 39f46d75..0998fecb 100644 --- a/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts +++ b/src/runners/testngRunner/TestNGRunnerResultAnalyzer.ts @@ -10,6 +10,7 @@ import { IRunTestContext, TestLevel, TestResultState } from '../../java-test-run const TEST_START: string = 'testStarted'; const TEST_FAIL: string = 'testFailed'; const TEST_FINISH: string = 'testFinished'; +const TEST_ERROR: string = 'error'; export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { @@ -47,7 +48,7 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { try { this.processData(match[1]); } catch (error) { - this.testContext.testRun.appendOutput(`[ERROR] Failed to parse output data: ${match[1]}\n`); + this.testContext.testRun.appendOutput(`[ERROR] Failed to parse output data: ${match[1]}\r\n`); } } } @@ -55,9 +56,17 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { public processData(data: string): void { const outputData: ITestNGOutputData = JSON.parse(data) as ITestNGOutputData; - this.testContext.testRun.appendOutput(this.unescape(data).replace(/\r?\n/g, '\r\n')); + if (outputData.name === TEST_ERROR) { + this.processRunnerError(outputData.attributes); + return; + } + + const attributes: ITestNGAttributes | undefined = outputData.attributes; + if (!attributes?.name) { + return; + } - const id: string = `${this.projectName}@${outputData.attributes.name}`; + const id: string = `${this.projectName}@${attributes.name}`; if (outputData.name === TEST_START) { this.initializeCache(); const item: TestItem | undefined = this.getTestItem(id); @@ -76,11 +85,11 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { this.currentTestState = TestResultState.Failed; const testMessages: TestMessage[] = []; - if (outputData.attributes.trace) { + if (attributes.trace) { const markdownTrace: MarkdownString = new MarkdownString(); markdownTrace.isTrusted = true; markdownTrace.supportHtml = true; - for (const line of outputData.attributes.trace.split(/\r?\n/)) { + for (const line of attributes.trace.split(/\r?\n/)) { this.processStackTrace(line, markdownTrace, this.currentItem, this.projectName); } @@ -93,17 +102,17 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { } testMessages.push(testMessage); } - const duration: number = Number.parseInt(outputData.attributes.duration, 10); + const duration: number | undefined = this.parseDuration(attributes.duration); setTestState(this.testContext.testRun, item, this.currentTestState, testMessages, duration); } else if (outputData.name === TEST_FINISH) { - const item: TestItem | undefined = this.getTestItem(data); + const item: TestItem | undefined = this.getTestItem(id); if (!item) { return; } if (this.currentTestState === TestResultState.Running) { this.currentTestState = TestResultState.Passed; } - const duration: number = Number.parseInt(outputData.attributes.duration, 10); + const duration: number | undefined = this.parseDuration(attributes.duration); setTestState(this.testContext.testRun, item, this.currentTestState, undefined, duration); const itemData: ITestItemData | undefined = dataCache.get(item); if (itemData?.testLevel === TestLevel.Method) { @@ -121,20 +130,31 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { return this.currentItem; } - protected unescape(content: string): string { - return content.replace(/\\r/gm, '\r') - .replace(/\\f/gm, '\f') - .replace(/\\n/gm, '\n') - .replace(/\\t/gm, '\t') - .replace(/\\b/gm, '\b') - .replace(/\\"/gm, '"'); - } - protected initializeCache(): void { this.currentTestState = TestResultState.Queued; this.currentItem = undefined; } + private processRunnerError(attributes: ITestNGAttributes | undefined): void { + let message: string = attributes?.message || 'Failed to run TestNG tests.'; + if (attributes?.trace) { + message += `\n${attributes.trace}`; + } + const testMessage: TestMessage = new TestMessage(message); + for (const item of this.testContext.testItems) { + this.testContext.testRun.errored(item, testMessage); + } + } + + private parseDuration(duration: string | undefined): number | undefined { + if (!duration) { + return undefined; + } + + const parsed: number = Number.parseInt(duration, 10); + return Number.isNaN(parsed) ? undefined : parsed; + } + protected getStacktraceFilter(): string[] { return [ 'com.microsoft.java.test.runner.', @@ -151,20 +171,14 @@ export class TestNGRunnerResultAnalyzer extends RunnerResultAnalyzer { } interface ITestNGOutputData { - attributes: ITestNGAttributes; - type: TestOutputType; + attributes?: ITestNGAttributes; name: string; } -enum TestOutputType { - Info, - Error, -} - interface ITestNGAttributes { - name: string; - duration: string; - location: string; - message: string; - trace: string; + name?: string; + duration?: string; + location?: string; + message?: string; + trace?: string; } diff --git a/test/suite/TestNGAnalyzer.test.ts b/test/suite/TestNGAnalyzer.test.ts new file mode 100644 index 00000000..511eebc6 --- /dev/null +++ b/test/suite/TestNGAnalyzer.test.ts @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +'use strict'; + +import * as assert from 'assert'; +import * as sinon from 'sinon'; +import { TestController, TestMessage, TestRunRequest, tests, workspace } from 'vscode'; +import { TestNGRunnerResultAnalyzer } from '../../src/runners/testngRunner/TestNGRunnerResultAnalyzer'; +import { IRunTestContext, TestKind } from '../../src/java-test-runner.api'; +import { generateTestItem } from './utils'; + +// tslint:disable: only-arrow-functions +suite('TestNG Runner Analyzer Tests', () => { + + let testController: TestController; + + setup(() => { + testController = tests.createTestController('testngTestController', 'Mock TestNG'); + }); + + teardown(() => { + testController.dispose(); + }); + + test('surfaces runner errors as structured test errors', () => { + const testItem = generateTestItem(testController, 'testng@example.SampleTest#test', TestKind.TestNG); + const testRun = testController.createTestRun(new TestRunRequest([testItem], [])); + const erroredSpy = sinon.spy(testRun, 'errored'); + const runnerContext: IRunTestContext = { + isDebug: false, + kind: TestKind.TestNG, + projectName: 'testng', + testItems: [testItem], + testRun, + workspaceFolder: workspace.workspaceFolders?.[0]!, + }; + const analyzer = new TestNGRunnerResultAnalyzer(runnerContext); + const trace = 'java.lang.ClassNotFoundException: example.SampleTest'; + + analyzer.processData(JSON.stringify({ + name: 'error', + attributes: { + message: 'Failed to run TestNG tests', + trace, + }, + })); + + sinon.assert.calledOnce(erroredSpy); + sinon.assert.calledWith(erroredSpy, testItem, sinon.match.instanceOf(TestMessage)); + const testMessage = erroredSpy.firstCall.args[1] as TestMessage; + assert.strictEqual(testMessage.message, `Failed to run TestNG tests\n${trace}`); + }); + + test('ignores control messages without test attributes', () => { + const testItem = generateTestItem(testController, 'testng@example.SampleTest#test', TestKind.TestNG); + const testRun = testController.createTestRun(new TestRunRequest([testItem], [])); + const appendOutputSpy = sinon.spy(testRun, 'appendOutput'); + const runnerContext: IRunTestContext = { + isDebug: false, + kind: TestKind.TestNG, + projectName: 'testng', + testItems: [testItem], + testRun, + workspaceFolder: workspace.workspaceFolders?.[0]!, + }; + const analyzer = new TestNGRunnerResultAnalyzer(runnerContext); + + analyzer.analyzeData('@@'); + + sinon.assert.notCalled(appendOutputSpy); + }); +}); diff --git a/test/suite/baseRunner.test.ts b/test/suite/baseRunner.test.ts index 8b940846..960ee870 100644 --- a/test/suite/baseRunner.test.ts +++ b/test/suite/baseRunner.test.ts @@ -4,12 +4,17 @@ 'use strict'; import * as assert from 'assert'; +import * as sinon from 'sinon'; import { AddressInfo } from 'net'; import { BaseRunner } from '../../src/runners/baseRunner/BaseRunner'; import { RunnerResultAnalyzer } from '../../src/runners/baseRunner/RunnerResultAnalyzer'; import { IRunTestContext, TestKind } from '../../src/java-test-runner.api'; class TestableBaseRunner extends BaseRunner { + public handleMessage(message: any): void { + this.handleDebugAdapterMessage(message); + } + protected getAnalyzer(): RunnerResultAnalyzer { return {} as RunnerResultAnalyzer; } @@ -63,4 +68,52 @@ suite('BaseRunner Tests', () => { } }); }); + + suite('handleDebugAdapterMessage', () => { + test('forwards non-telemetry output at run level with CRLF newlines', () => { + const appendOutput = sinon.spy(); + const runner = new TestableBaseRunner({ + kind: TestKind.JUnit, + isDebug: false, + projectName: 'test-project', + testItems: [], + testRun: { appendOutput } as any, + workspaceFolder: {} as any, + } as IRunTestContext); + + runner.handleMessage({ + type: 'event', + event: 'output', + body: { + category: 'console', + output: 'first\nsecond\r\n', + }, + }); + + sinon.assert.calledOnceWithExactly(appendOutput, 'first\r\nsecond\r\n'); + }); + + test('ignores telemetry output', () => { + const appendOutput = sinon.spy(); + const runner = new TestableBaseRunner({ + kind: TestKind.JUnit, + isDebug: false, + projectName: 'test-project', + testItems: [], + testRun: { appendOutput } as any, + workspaceFolder: {} as any, + } as IRunTestContext); + + runner.handleMessage({ + type: 'event', + event: 'output', + body: { + category: 'telemetry', + output: 'internal event', + }, + }); + + sinon.assert.notCalled(appendOutput); + }); + }); });