diff --git a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AbstractModuleCompilationTest.java b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AbstractModuleCompilationTest.java index f07c5ebe06b..d9e6985d7d6 100644 --- a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AbstractModuleCompilationTest.java +++ b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AbstractModuleCompilationTest.java @@ -163,6 +163,7 @@ protected Set runConformModuleTest(String[] testFiles, String commandLin System.err.println("Skip testing javac in "+testName()); continue; } + numRunJavacTests++; StringBuilder log = new StringBuilder(); try { long compileResult = javacCompiler.compile( @@ -250,6 +251,7 @@ void runNegativeModuleTest(String[] testFiles, String commandLine, String expect if (javacCompiler.compliance < ClassFileConstants.JDK9) continue; JavacTestOptions.Excuse excuse = options.excuseFor(javacCompiler); + numRunJavacTests++; String javacCommandLine = " -d " + javacOutput + ' ' + adjustForJavac(commandLine, null); StringBuilder log = new StringBuilder(); diff --git a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AbstractRegressionTest.java b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AbstractRegressionTest.java index f91253ce97f..e9e3099ed96 100644 --- a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AbstractRegressionTest.java +++ b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/AbstractRegressionTest.java @@ -24,6 +24,8 @@ package org.eclipse.jdt.core.tests.compiler.regression; import java.io.*; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.net.URL; import java.nio.charset.Charset; import java.nio.file.Files; @@ -92,6 +94,11 @@ public abstract class AbstractRegressionTest extends AbstractCompilerTest implem protected static long PREVIEW_FEATURE_CLASS_FILE_CONST = ClassFileConstants.JDK25; protected static int PREVIEW_FEATURE_LEVEL = 25; + // statistics for run.javac mode: + static int numRunJavacTests = 0; + static Set javacBugs = new HashSet<>(); + static Set ecjBugs = new HashSet<>(); + protected class Runner { boolean shouldFlushOutputDirectory = true; // input: @@ -2781,6 +2788,7 @@ protected void runJavac( e.printStackTrace(); mismatch = JavacTestOptions.MismatchType.JavaNotLaunched; } + numRunJavacTests++; handleMismatch(compiler, testName, testFiles, expectedCompilerLog, expectedOutputString, expectedErrorString, compilerLog, output, err, excuse, mismatch); } @@ -2830,6 +2838,7 @@ void handleMismatch(JavacCompiler compiler, String testName, String[] testFiles, JavacTestOptions.Excuse excuse, int mismatch) { if (mismatch != 0) { if (excuse != null && excuse.clears(mismatch)) { + recordBugHit(excuse); excuse = null; } else { System.err.println("----------------------------------------"); @@ -4306,4 +4315,43 @@ protected Map setPresetPreviewOptions() { options.put(CompilerOptions.OPTION_ReportPreviewFeatures, CompilerOptions.IGNORE); return options; } + + private static void recordBugHit(JavacTestOptions.Excuse excuse) { + if (excuse instanceof JavacTestOptions.JavacHasABug) + javacBugs.add(excuse); + else if (excuse instanceof JavacTestOptions.EclipseHasABug) + ecjBugs.add(excuse); + } + + public static void printRunJavacStats() { + if (RUN_JAVAC || RUN_JAVAC_OPT_IN) { + System.out.println("Num tests with run.javac enabled: "+numRunJavacTests); + System.out.println("Num tests triggering a javac bug: "+javacBugs.size()); + printBugConstants(javacBugs, JavacTestOptions.JavacHasABug.class); + System.out.println("Num tests triggering an ecj bug : "+ecjBugs.size()); + printBugConstants(ecjBugs, JavacTestOptions.EclipseHasABug.class); + } + } + + private static void printBugConstants(Set bugs, Class clazz) { + List bugNames = new ArrayList<>(); + bugs: for (JavacTestOptions.Excuse bug : bugs) { + for (Class aClazz : new Class[] { clazz, JavacTestOptions.Excuse.class }) { + for (Field field : aClazz.getDeclaredFields()) { + if ((field.getModifiers() & Modifier.STATIC) != 0) { + try { + if (bug == field.get(null)) { + bugNames.add(field.getName()); + continue bugs; + } + } catch (IllegalArgumentException | IllegalAccessException e) { + // ignore + } + } + } + } + bugNames.add("\tunknown excuse "+bug); + } + bugNames.stream().sorted().forEach(s -> System.out.println("\t"+s)); + } } diff --git a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/JavadocTestForModule.java b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/JavadocTestForModule.java index 8a6caa18257..927eb297ee4 100644 --- a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/JavadocTestForModule.java +++ b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/JavadocTestForModule.java @@ -236,6 +236,7 @@ void runNegativeModuleTest(String[] testFiles, String commandLine, if (javacCompiler.compliance < ClassFileConstants.JDK9) continue; JavacTestOptions.Excuse excuse = options.excuseFor(javacCompiler); + numRunJavacTests++; commandLine = adjustForJavac(commandLine, null); StringBuilder log = new StringBuilder(); diff --git a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/PrintRunJavacStats.java b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/PrintRunJavacStats.java new file mode 100644 index 00000000000..c8e7a62c6bd --- /dev/null +++ b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/PrintRunJavacStats.java @@ -0,0 +1,32 @@ +/******************************************************************************* + * Copyright (c) 2026 GK Software and others. + * + * This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Stephan Herrmann - initial API and implementation + *******************************************************************************/ +package org.eclipse.jdt.core.tests.compiler.regression; + +import junit.framework.Test; + +/** + * This test class exists for the sole purpose of printing statistics about tests run as comparison of ecj vs javac. + * It is be integrated in {@link TestAll} as the very last test in the suite. + */ +public class PrintRunJavacStats extends AbstractRegressionTest { + public PrintRunJavacStats(String name) { + super(name); + } + public static Test suite() { + return buildMinimalComplianceTestSuite(PrintRunJavacStats.class, FIRST_SUPPORTED_JAVA_VERSION); + } + public void testPrint() { + printRunJavacStats(); + } +} \ No newline at end of file diff --git a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/TestAll.java b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/TestAll.java index b8875da2eae..192ecf29592 100644 --- a/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/TestAll.java +++ b/org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/TestAll.java @@ -294,6 +294,7 @@ public static Test suite() { } all.addTest(new TestSuite(Jsr14Test.class)); + all.addTest(new TestSuite(PrintRunJavacStats.class)); // the very last test to capture statistics of all tests return all; } }