From a0a5277221e1454dca69a7e0ac97e5ecf9db5506 Mon Sep 17 00:00:00 2001 From: Stuart Lang Date: Fri, 10 Apr 2026 23:06:52 +0100 Subject: [PATCH 01/11] Migrate from xUnit v2 to xUnit v3 with MTP support - Replace xunit 2.9.3 and xunit.runner.visualstudio with xunit.v3.mtp-v2 3.2.2 - Update FsCheck.Xunit to FsCheck.Xunit.v3 - Add Microsoft.Testing.Platform 2.1.0 and enable MTP in global.json - Add OutputType Exe to test projects (required by xUnit v3) - Configure Stryker with test-runner mtp for coverage support - Run test executables directly via dotnet exec, bypassing dotnet test --project MSBuild discovery bug (dotnet/sdk#49063) - Update ITestOutputHelper implementations for new xUnit v3 interface - Fix TheoryData type inference with explicit casts - Use TestContext.Current.CancellationToken for test cancellation - Remove obsolete Xunit.Abstractions using directives Co-Authored-By: Claude Opus 4.6 --- .config/dotnet-tools.json | 2 +- Directory.Packages.props | 9 +-- cake.cs | 56 +++++++++++++------ eng/Test.targets | 4 +- eng/stryker-config.json | 1 + global.json | 3 + src/Snippets/Snippets.csproj | 4 +- .../Hedging/HedgingResilienceStrategyTests.cs | 1 - test/Polly.Core.Tests/Polly.Core.Tests.csproj | 5 +- .../ResiliencePipelineTests.cs | 2 +- .../Pipeline/PipelineComponentFactoryTests.cs | 12 ++-- .../Polly.Extensions.Tests.csproj | 3 +- .../Polly.RateLimiting.Tests.csproj | 3 +- .../Helpers/Bulkhead/AnnotatedOutputHelper.cs | 4 +- .../Helpers/Bulkhead/SilentOutputHelper.cs | 14 ++++- test/Polly.Specs/Polly.Specs.csproj | 4 +- .../Polly.Testing.Tests.csproj | 3 +- test/Shared/TestCancellation.cs | 4 +- 18 files changed, 90 insertions(+), 44 deletions(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index f948172fd44..f8b1de41787 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -24,7 +24,7 @@ "rollForward": false }, "dotnet-stryker": { - "version": "4.14.0", + "version": "4.14.1", "commands": [ "dotnet-stryker" ], diff --git a/Directory.Packages.props b/Directory.Packages.props index c62c59ffde9..ba00e843945 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,7 +4,7 @@ - + @@ -19,7 +19,8 @@ - + + @@ -38,8 +39,8 @@ - - + + diff --git a/cake.cs b/cake.cs index a2b8d897ada..48d71ab8a4e 100644 --- a/cake.cs +++ b/cake.cs @@ -136,28 +136,50 @@ Task("__RunTests") .Does(() => { - var loggers = Array.Empty(); - - if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("GITHUB_SHA"))) - { - loggers = - [ - "junit;LogFilePath=junit.xml", - "GitHubActions;report-warnings=false;summary-include-passed=false", - ]; - } - var projects = GetFiles("./test/**/*.csproj"); foreach (var proj in projects) { - DotNetTest(proj.FullPath, new DotNetTestSettings + var projectName = proj.GetFilenameWithoutExtension().ToString(); + var configLower = configuration.ToLowerInvariant(); + var outputBase = MakeAbsolute(Directory($"./artifacts/bin/{projectName}")); + + foreach (var tfmDir in GetDirectories($"{outputBase}/{configLower}_*")) { - Configuration = configuration, - Loggers = loggers, - NoBuild = true, - ToolTimeout = System.TimeSpan.FromMinutes(10), - }); + var dll = tfmDir.CombineWithFilePath($"{projectName}.dll"); + var runtimeConfig = tfmDir.CombineWithFilePath($"{projectName}.runtimeconfig.json"); + if (!FileExists(dll) || !FileExists(runtimeConfig)) + continue; + + var tfmName = tfmDir.GetDirectoryName().Substring(configLower.Length + 1); + Information($"Testing {projectName} ({tfmName})"); + + var args = new ProcessArgumentBuilder(); + FilePath executable; + + if (tfmName.StartsWith("net4")) + { + executable = tfmDir.CombineWithFilePath($"{projectName}.exe"); + } + else + { + executable = Context.Tools.Resolve("dotnet") ?? new FilePath("dotnet"); + args.Append("exec"); + args.AppendQuoted(dll.FullPath); + } + + if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("GITHUB_SHA"))) + { + args.Append("-jUnit"); + args.AppendQuoted($"{projectName}-{tfmName}.junit.xml"); + } + + var result = StartProcess(executable, new ProcessSettings { Arguments = args }); + if (result != 0) + { + throw new InvalidOperationException($"Tests failed for '{projectName}' ({tfmName})."); + } + } } }); diff --git a/eng/Test.targets b/eng/Test.targets index 4058f40c299..5065acc7235 100644 --- a/eng/Test.targets +++ b/eng/Test.targets @@ -15,11 +15,11 @@ + - - + diff --git a/eng/stryker-config.json b/eng/stryker-config.json index 8113fef7a0f..a916917572a 100644 --- a/eng/stryker-config.json +++ b/eng/stryker-config.json @@ -23,6 +23,7 @@ "configuration": "Debug", "language-version": "Preview", "target-framework": "net10.0", + "test-runner": "mtp", "thresholds": { "high": 100, "low": 100 diff --git a/global.json b/global.json index 80027b468d9..3168c262f1b 100644 --- a/global.json +++ b/global.json @@ -6,5 +6,8 @@ }, "msbuild-sdks": { "Cake.Sdk": "6.1.1" + }, + "test": { + "runner": "Microsoft.Testing.Platform" } } \ No newline at end of file diff --git a/src/Snippets/Snippets.csproj b/src/Snippets/Snippets.csproj index f9743050c13..7acf86aad48 100644 --- a/src/Snippets/Snippets.csproj +++ b/src/Snippets/Snippets.csproj @@ -1,4 +1,4 @@ - + false @@ -32,7 +32,7 @@ - + diff --git a/test/Polly.Core.Tests/Hedging/HedgingResilienceStrategyTests.cs b/test/Polly.Core.Tests/Hedging/HedgingResilienceStrategyTests.cs index f196b730cce..2c6b1aab850 100644 --- a/test/Polly.Core.Tests/Hedging/HedgingResilienceStrategyTests.cs +++ b/test/Polly.Core.Tests/Hedging/HedgingResilienceStrategyTests.cs @@ -3,7 +3,6 @@ using Polly.Hedging.Utils; using Polly.Telemetry; using Polly.Testing; -using Xunit.Abstractions; namespace Polly.Core.Tests.Hedging; diff --git a/test/Polly.Core.Tests/Polly.Core.Tests.csproj b/test/Polly.Core.Tests/Polly.Core.Tests.csproj index 5105ea208e1..73d00757cc2 100644 --- a/test/Polly.Core.Tests/Polly.Core.Tests.csproj +++ b/test/Polly.Core.Tests/Polly.Core.Tests.csproj @@ -1,4 +1,4 @@ - + net10.0;net9.0;net8.0 $(TargetFrameworks);net481 @@ -6,6 +6,7 @@ Test enable 100 + Exe $(NoWarn);S6966 [Polly.Core]* true @@ -15,7 +16,7 @@ - + diff --git a/test/Polly.Core.Tests/ResiliencePipelineTests.cs b/test/Polly.Core.Tests/ResiliencePipelineTests.cs index c47604b7908..614c3f2ba14 100644 --- a/test/Polly.Core.Tests/ResiliencePipelineTests.cs +++ b/test/Polly.Core.Tests/ResiliencePipelineTests.cs @@ -11,7 +11,7 @@ public partial class ResiliencePipelineTests #pragma warning disable IDE0028 public static TheoryData ResilienceContextPools = new() { - null, + null as ResilienceContextPool, ResilienceContextPool.Shared, }; #pragma warning restore IDE0028 diff --git a/test/Polly.Core.Tests/Utils/Pipeline/PipelineComponentFactoryTests.cs b/test/Polly.Core.Tests/Utils/Pipeline/PipelineComponentFactoryTests.cs index 40eb7f2b26d..7383d19159d 100644 --- a/test/Polly.Core.Tests/Utils/Pipeline/PipelineComponentFactoryTests.cs +++ b/test/Polly.Core.Tests/Utils/Pipeline/PipelineComponentFactoryTests.cs @@ -1,4 +1,4 @@ -using NSubstitute; +using NSubstitute; using Polly.Utils.Pipeline; namespace Polly.Core.Tests.Utils.Pipeline; @@ -8,17 +8,17 @@ public class PipelineComponentFactoryTests #pragma warning disable IDE0028 public static TheoryData> EmptyCallbacks = new() { - Array.Empty(), + Array.Empty() as IEnumerable, Enumerable.Empty(), - new List(), - new EmptyActionEnumerable(), // Explicitly does not provide TryGetNonEnumeratedCount() + new List() as IEnumerable, + new EmptyActionEnumerable() as IEnumerable, // Explicitly does not provide TryGetNonEnumeratedCount() }; public static TheoryData> NonEmptyCallbacks = new() { - new[] { () => { } }, + new[] { () => { } } as IEnumerable, Enumerable.TakeWhile(Enumerable.Repeat(() => { }, 50), (_, i) => i < 1), // Defeat optimisation for TryGetNonEnumeratedCount() - new List { () => { } }, + new List { () => { } } as IEnumerable, }; #pragma warning restore IDE0028 diff --git a/test/Polly.Extensions.Tests/Polly.Extensions.Tests.csproj b/test/Polly.Extensions.Tests/Polly.Extensions.Tests.csproj index 344d5001a12..d98987fafb2 100644 --- a/test/Polly.Extensions.Tests/Polly.Extensions.Tests.csproj +++ b/test/Polly.Extensions.Tests/Polly.Extensions.Tests.csproj @@ -1,10 +1,11 @@ - + net10.0;net9.0;net8.0 $(TargetFrameworks);net481 Test enable 100 + Exe [Polly.Extensions]* diff --git a/test/Polly.RateLimiting.Tests/Polly.RateLimiting.Tests.csproj b/test/Polly.RateLimiting.Tests/Polly.RateLimiting.Tests.csproj index 88fd0658026..5e55a7b8b0f 100644 --- a/test/Polly.RateLimiting.Tests/Polly.RateLimiting.Tests.csproj +++ b/test/Polly.RateLimiting.Tests/Polly.RateLimiting.Tests.csproj @@ -1,10 +1,11 @@ - + net10.0;net9.0;net8.0 $(TargetFrameworks);net481 Test enable 100 + Exe [Polly.RateLimiting]* diff --git a/test/Polly.Specs/Helpers/Bulkhead/AnnotatedOutputHelper.cs b/test/Polly.Specs/Helpers/Bulkhead/AnnotatedOutputHelper.cs index fa61f84806e..92812c94f59 100644 --- a/test/Polly.Specs/Helpers/Bulkhead/AnnotatedOutputHelper.cs +++ b/test/Polly.Specs/Helpers/Bulkhead/AnnotatedOutputHelper.cs @@ -1,4 +1,4 @@ -using System.Globalization; +using System.Globalization; namespace Polly.Specs.Helpers.Bulkhead; @@ -29,6 +29,8 @@ public Item(string format, object[] args) private readonly ITestOutputHelper _innerOutputHelper; + public string Output => _innerOutputHelper.Output; + public AnnotatedOutputHelper(ITestOutputHelper innerOutputHelper) => _innerOutputHelper = innerOutputHelper ?? throw new ArgumentNullException(nameof(innerOutputHelper)); diff --git a/test/Polly.Specs/Helpers/Bulkhead/SilentOutputHelper.cs b/test/Polly.Specs/Helpers/Bulkhead/SilentOutputHelper.cs index 2aa50df2883..fc280ae0917 100644 --- a/test/Polly.Specs/Helpers/Bulkhead/SilentOutputHelper.cs +++ b/test/Polly.Specs/Helpers/Bulkhead/SilentOutputHelper.cs @@ -1,7 +1,19 @@ -namespace Polly.Specs.Helpers.Bulkhead; +namespace Polly.Specs.Helpers.Bulkhead; public class SilentOutputHelper : ITestOutputHelper { + public string Output => string.Empty; + + public void Write(string message) + { + // Do nothing: intentionally silent. + } + + public void Write(string format, params object[] args) + { + // Do nothing: intentionally silent. + } + public void WriteLine(string message) { // Do nothing: intentionally silent. diff --git a/test/Polly.Specs/Polly.Specs.csproj b/test/Polly.Specs/Polly.Specs.csproj index a7d836659d8..6e19fa42da5 100644 --- a/test/Polly.Specs/Polly.Specs.csproj +++ b/test/Polly.Specs/Polly.Specs.csproj @@ -1,8 +1,9 @@ - + net10.0;net9.0;net8.0 $(TargetFrameworks);net481 enable + Exe Test 94,94,91 [Polly]* @@ -22,7 +23,6 @@ - diff --git a/test/Polly.Testing.Tests/Polly.Testing.Tests.csproj b/test/Polly.Testing.Tests/Polly.Testing.Tests.csproj index 646329abf83..860a2d756aa 100644 --- a/test/Polly.Testing.Tests/Polly.Testing.Tests.csproj +++ b/test/Polly.Testing.Tests/Polly.Testing.Tests.csproj @@ -1,10 +1,11 @@ - + net10.0;net9.0;net8.0 $(TargetFrameworks);net481 Test enable 100 + Exe [Polly.Testing]* diff --git a/test/Shared/TestCancellation.cs b/test/Shared/TestCancellation.cs index be9cd8b0678..5396df4d754 100644 --- a/test/Shared/TestCancellation.cs +++ b/test/Shared/TestCancellation.cs @@ -1,6 +1,8 @@ +using Xunit; + namespace Polly; internal static class TestCancellation { - public static CancellationToken Token => CancellationToken.None; + public static CancellationToken Token => TestContext.Current.CancellationToken; } From efa9da7b448a0bdd202338699e32d1129d3a7016 Mon Sep 17 00:00:00 2001 From: Stuart Lang Date: Sat, 11 Apr 2026 11:33:36 +0100 Subject: [PATCH 02/11] Fix Stryker mutation testing with xUnit v3 MTP runner Patch Stryker's MTP test runner to fix three bugs that caused ~55% of mutations to incorrectly survive: 1. Only "failed" execution state was counted as test failure, not "error" 2. EveryTest() sentinel was lost during test result accumulation when the MTP server crashed (GetIdentifiers() returns empty for sentinel) 3. Static field initializer mutations were never killed because the MTP process reuses the same process across mutations, and static fields initialize only once Also add eng/AssemblyVersion.cs to preserve the assembly version in Stryker-compiled assemblies (Stryker skips auto-generated files which caused version mismatch: 1.0.0.0 vs expected 8.0.0.0). Co-Authored-By: Claude Opus 4.6 --- .gitignore | 6 +++ Directory.Packages.props | 4 +- cake.cs | 85 ++++++++++++++++++++++++++++++++++++ eng/AssemblyVersion.cs | 4 ++ eng/Library.targets | 5 +++ eng/stryker-mtp-runner.patch | 78 +++++++++++++++++++++++++++++++++ 6 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 eng/AssemblyVersion.cs create mode 100644 eng/stryker-mtp-runner.patch diff --git a/.gitignore b/.gitignore index dbc94ceb5ea..60efcebcdc8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,12 @@ # mstest test results TestResults +# Stryker mutation testing output +StrykerOutput*/ + +# JUnit test result files +*.junit.xml + ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. diff --git a/Directory.Packages.props b/Directory.Packages.props index ba00e843945..46a2a62a188 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -19,8 +19,8 @@ - - + + diff --git a/cake.cs b/cake.cs index 48d71ab8a4e..7f76748af78 100644 --- a/cake.cs +++ b/cake.cs @@ -254,8 +254,15 @@ // MUTATION TESTING TARGETS /////////////////////////////////////////////////////////////////////////////// +Task("PatchStryker") + .Does((_) => +{ + PatchStrykerMtpRunner(); +}); + Task("MutationTestsCore") .IsDependentOn("__Setup") + .IsDependentOn("PatchStryker") .Does((_) => { RunMutationTests(File("./src/Polly.Core/Polly.Core.csproj"), File("./test/Polly.Core.Tests/Polly.Core.Tests.csproj")); @@ -263,6 +270,7 @@ Task("MutationTestsRateLimiting") .IsDependentOn("__Setup") + .IsDependentOn("PatchStryker") .Does((_) => { RunMutationTests(File("./src/Polly.RateLimiting/Polly.RateLimiting.csproj"), File("./test/Polly.RateLimiting.Tests/Polly.RateLimiting.Tests.csproj")); @@ -270,6 +278,7 @@ Task("MutationTestsExtensions") .IsDependentOn("__Setup") + .IsDependentOn("PatchStryker") .Does((_) => { RunMutationTests(File("./src/Polly.Extensions/Polly.Extensions.csproj"), File("./test/Polly.Extensions.Tests/Polly.Extensions.Tests.csproj")); @@ -277,6 +286,7 @@ Task("MutationTestsTesting") .IsDependentOn("__Setup") + .IsDependentOn("PatchStryker") .Does((_) => { RunMutationTests(File("./src/Polly.Testing/Polly.Testing.csproj"), File("./test/Polly.Testing.Tests/Polly.Testing.Tests.csproj")); @@ -284,6 +294,7 @@ Task("MutationTestsLegacy") .IsDependentOn("__Setup") + .IsDependentOn("PatchStryker") .Does((_) => { RunMutationTests(File("./src/Polly/Polly.csproj"), File("./test/Polly.Specs/Polly.Specs.csproj")); @@ -322,6 +333,80 @@ string PatchStrykerConfig(string path, Action patc return tempPath; } +void PatchStrykerMtpRunner() +{ + // Patches Stryker's MTP test runner to fix three bugs: + // 1. "error" execution state not counted as test failure (only "failed" was checked) + // 2. EveryTest() sentinel not properly accumulated when server crashes + // 3. Static field initializer mutations not killed due to MTP process reuse + // See: https://github.com/stryker-mutator/stryker-net/issues/3117 + // This patch can be removed once Stryker fixes these issues upstream. + + var strykerVersion = "4.14.1"; + var strykerTag = $"dotnet-stryker@{strykerVersion}"; + var patchFile = MakeAbsolute(File("./eng/stryker-mtp-runner.patch")); + var tempDir = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"stryker-patch-{strykerVersion}"); + var targetDll = System.IO.Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + ".nuget", "packages", "dotnet-stryker", strykerVersion, "tools", "net8.0", "any", + "Stryker.TestRunner.MicrosoftTestPlatform.dll"); + + // Check if already patched (by presence of a marker file) + var markerFile = targetDll + ".patched"; + if (System.IO.File.Exists(markerFile)) + { + Information("Stryker MTP runner already patched."); + return; + } + + Information("Patching Stryker MTP runner..."); + + // Clone stryker-net at the correct tag + if (!System.IO.Directory.Exists(tempDir)) + { + var cloneResult = StartProcess("git", new ProcessSettings + { + Arguments = $"clone --depth 1 --branch {strykerTag} https://github.com/stryker-mutator/stryker-net.git {tempDir}", + }); + if (cloneResult != 0) + { + throw new InvalidOperationException("Failed to clone stryker-net repository."); + } + } + + // Apply the patch + var applyResult = StartProcess("git", new ProcessSettings + { + Arguments = $"apply {patchFile.FullPath}", + WorkingDirectory = tempDir, + }); + if (applyResult != 0) + { + throw new InvalidOperationException("Failed to apply Stryker MTP runner patch."); + } + + // Build the patched project + var projectPath = System.IO.Path.Combine(tempDir, "src", "Stryker.TestRunner.MicrosoftTestPlatform", + "Stryker.TestRunner.MicrosoftTestPlatform.csproj"); + var buildResult = StartProcess("dotnet", new ProcessSettings + { + Arguments = $"build \"{projectPath}\" -c Release", + }); + if (buildResult != 0) + { + throw new InvalidOperationException("Failed to build patched Stryker MTP runner."); + } + + // Copy the patched DLL + var builtDll = System.IO.Path.Combine(tempDir, "src", "Stryker.TestRunner.MicrosoftTestPlatform", + "bin", "Release", "net8.0", "Stryker.TestRunner.MicrosoftTestPlatform.dll"); + + System.IO.File.Copy(builtDll, targetDll, overwrite: true); + System.IO.File.WriteAllText(markerFile, $"Patched from {patchFile.GetFilename()} at {DateTime.UtcNow:O}"); + + Information("Stryker MTP runner patched successfully."); +} + void RunMutationTests(FilePath target, FilePath testProject) { var mutationScore = XmlPeek(target, "/Project/PropertyGroup/MutationScore/text()", new XmlPeekSettings { SuppressWarning = true }); diff --git a/eng/AssemblyVersion.cs b/eng/AssemblyVersion.cs new file mode 100644 index 00000000000..574f2882a07 --- /dev/null +++ b/eng/AssemblyVersion.cs @@ -0,0 +1,4 @@ +// Stryker mutation testing skips auto-generated files (those with headers), +// which causes the mutated assembly to lose its AssemblyVersion. This file ensures the +// version is preserved. The SDK-generated attribute is disabled via Library.targets. +[assembly: System.Reflection.AssemblyVersion("8.0.0.0")] diff --git a/eng/Library.targets b/eng/Library.targets index 4a8e68da6c5..bbc31c2cd4e 100644 --- a/eng/Library.targets +++ b/eng/Library.targets @@ -5,6 +5,7 @@ Copyright (c) 2015-$([System.DateTime]::Now.ToString(yyyy)), App vNext en-US true + false true Michael Wolfenden, App vNext $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb @@ -38,6 +39,10 @@ + + + + diff --git a/eng/stryker-mtp-runner.patch b/eng/stryker-mtp-runner.patch new file mode 100644 index 00000000000..bf452799737 --- /dev/null +++ b/eng/stryker-mtp-runner.patch @@ -0,0 +1,78 @@ +diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs +index cfc34d6..1d45845 100644 +--- a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs ++++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs +@@ -86,6 +86,22 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable + _logger.LogDebug("{RunnerId}: Testing mutant(s) [{Mutants}] with active mutation ID: {MutantId}", + RunnerId, string.Join(",", mutants.Select(m => m.Id)), mutantId); + ++ ++ // Static mutations (e.g. static field initializers) require a fresh process ++ // because the static value is set once during class loading and cannot change. ++ if (mutants.Any(m => m.IsStaticValue)) ++ { ++ _logger.LogDebug("{RunnerId}: Resetting servers for static mutation(s)", RunnerId); ++ lock (_serverLock) ++ { ++ foreach (var server in _assemblyServers.Values) ++ { ++ server.Dispose(); ++ } ++ _assemblyServers.Clear(); ++ } ++ } ++ + return RunAllTestsAsync(assemblies, mutantId, mutants, update, timeoutCalc); + } + +@@ -351,6 +367,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable + { + private readonly List _executedTests = []; + private readonly List _failedTests = []; ++ private bool _allTestsFailed; + private readonly List _messages = []; + private readonly List _errorMessages = []; + private int _totalDiscoveredTests; +@@ -373,7 +390,14 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable + _totalExecutedTests += executedIds.Count; + } + +- _failedTests.AddRange(result.FailingTests.GetIdentifiers()); ++ if (result.FailingTests.IsEveryTest) ++ { ++ _allTestsFailed = true; ++ } ++ else ++ { ++ _failedTests.AddRange(result.FailingTests.GetIdentifiers()); ++ } + TotalDuration += result.Duration; + _messages.AddRange(result.Messages ?? []); + +@@ -390,7 +414,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable + ? TestIdentifierList.EveryTest() + : new TestIdentifierList(_executedTests); + +- public ITestIdentifiers BuildFailedTests() => new TestIdentifierList(_failedTests); ++ public ITestIdentifiers BuildFailedTests() => _allTestsFailed ? TestIdentifierList.EveryTest() : new TestIdentifierList(_failedTests); + + public ITestIdentifiers BuildTimedOutTests() => new TestIdentifierList(TimedOutTests); + +@@ -525,7 +549,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable + + var duration = DateTime.UtcNow - startTime; + var finishedTests = testResults.Where(x => x.Node.ExecutionState is not "in-progress").ToList(); +- var failedTests = finishedTests.Where(x => x.Node.ExecutionState is "failed").Select(x => x.Node.Uid).ToList(); ++ var failedTests = finishedTests.Where(x => x.Node.ExecutionState is "failed" or "error").Select(x => x.Node.Uid).ToList(); + + lock (_discoveryLock) + { +@@ -542,7 +566,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable + } + + var errorMessagesStr = string.Join(Environment.NewLine, +- finishedTests.Where(x => x.Node.ExecutionState is "failed") ++ finishedTests.Where(x => x.Node.ExecutionState is "failed" or "error") + .Select(x => $"{x.Node.DisplayName}{Environment.NewLine}{Environment.NewLine}Test failed")); + + var messages = finishedTests.Select(x => From e942664672a21f56014293167e3edb431a9ae6ad Mon Sep 17 00:00:00 2001 From: Stuart Lang Date: Sat, 11 Apr 2026 15:09:59 +0100 Subject: [PATCH 03/11] Fix Stryker patch application in CI Three issues with the PatchStryker Cake task: 1. Patch file didn't match actual v4.14.1 source (regenerated) 2. Path resolution used wrong working directory (use script-relative path) 3. Build ran from wrong working directory (NuGet packages not found) Also change target-framework from net10.0 to net8.0 since source projects don't target net10.0, causing Stryker framework mismatch on CI. Co-Authored-By: Claude Opus 4.6 --- cake.cs | 22 +++++++++++++++++----- eng/stryker-config.json | 2 +- eng/stryker-mtp-runner.patch | 15 +++++++-------- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/cake.cs b/cake.cs index 7f76748af78..dbf7b829bda 100644 --- a/cake.cs +++ b/cake.cs @@ -344,7 +344,9 @@ void PatchStrykerMtpRunner() var strykerVersion = "4.14.1"; var strykerTag = $"dotnet-stryker@{strykerVersion}"; - var patchFile = MakeAbsolute(File("./eng/stryker-mtp-runner.patch")); + // Resolve relative to the directory containing cake.cs, not the process working directory + var scriptDir = System.IO.Path.GetDirectoryName(System.IO.Path.GetFullPath("cake.cs")) ?? "."; + var patchFile = System.IO.Path.Combine(scriptDir, "eng", "stryker-mtp-runner.patch"); var tempDir = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"stryker-patch-{strykerVersion}"); var targetDll = System.IO.Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), @@ -373,11 +375,20 @@ void PatchStrykerMtpRunner() throw new InvalidOperationException("Failed to clone stryker-net repository."); } } + else + { + // Reset any leftover changes from a previous failed run + StartProcess("git", new ProcessSettings + { + Arguments = "checkout -- .", + WorkingDirectory = tempDir, + }); + } // Apply the patch var applyResult = StartProcess("git", new ProcessSettings { - Arguments = $"apply {patchFile.FullPath}", + Arguments = $"apply {patchFile}", WorkingDirectory = tempDir, }); if (applyResult != 0) @@ -385,12 +396,13 @@ void PatchStrykerMtpRunner() throw new InvalidOperationException("Failed to apply Stryker MTP runner patch."); } - // Build the patched project - var projectPath = System.IO.Path.Combine(tempDir, "src", "Stryker.TestRunner.MicrosoftTestPlatform", + // Build the patched project (must run from tempDir so NuGet packages resolve correctly) + var projectPath = System.IO.Path.Combine("src", "Stryker.TestRunner.MicrosoftTestPlatform", "Stryker.TestRunner.MicrosoftTestPlatform.csproj"); var buildResult = StartProcess("dotnet", new ProcessSettings { Arguments = $"build \"{projectPath}\" -c Release", + WorkingDirectory = tempDir, }); if (buildResult != 0) { @@ -402,7 +414,7 @@ void PatchStrykerMtpRunner() "bin", "Release", "net8.0", "Stryker.TestRunner.MicrosoftTestPlatform.dll"); System.IO.File.Copy(builtDll, targetDll, overwrite: true); - System.IO.File.WriteAllText(markerFile, $"Patched from {patchFile.GetFilename()} at {DateTime.UtcNow:O}"); + System.IO.File.WriteAllText(markerFile, $"Patched from {System.IO.Path.GetFileName(patchFile)} at {DateTime.UtcNow:O}"); Information("Stryker MTP runner patched successfully."); } diff --git a/eng/stryker-config.json b/eng/stryker-config.json index a916917572a..ea4dbcf2ff1 100644 --- a/eng/stryker-config.json +++ b/eng/stryker-config.json @@ -22,7 +22,7 @@ "concurrency": 4, "configuration": "Debug", "language-version": "Preview", - "target-framework": "net10.0", + "target-framework": "net8.0", "test-runner": "mtp", "thresholds": { "high": 100, diff --git a/eng/stryker-mtp-runner.patch b/eng/stryker-mtp-runner.patch index bf452799737..e52d1f69418 100644 --- a/eng/stryker-mtp-runner.patch +++ b/eng/stryker-mtp-runner.patch @@ -1,12 +1,11 @@ diff --git a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs -index cfc34d6..1d45845 100644 +index cfc34d6..ee4b3b1 100644 --- a/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs +++ b/src/Stryker.TestRunner.MicrosoftTestPlatform/SingleMicrosoftTestPlatformRunner.cs -@@ -86,6 +86,22 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable +@@ -86,6 +86,21 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable _logger.LogDebug("{RunnerId}: Testing mutant(s) [{Mutants}] with active mutation ID: {MutantId}", RunnerId, string.Join(",", mutants.Select(m => m.Id)), mutantId); -+ + // Static mutations (e.g. static field initializers) require a fresh process + // because the static value is set once during class loading and cannot change. + if (mutants.Any(m => m.IsStaticValue)) @@ -25,7 +24,7 @@ index cfc34d6..1d45845 100644 return RunAllTestsAsync(assemblies, mutantId, mutants, update, timeoutCalc); } -@@ -351,6 +367,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable +@@ -351,6 +366,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable { private readonly List _executedTests = []; private readonly List _failedTests = []; @@ -33,7 +32,7 @@ index cfc34d6..1d45845 100644 private readonly List _messages = []; private readonly List _errorMessages = []; private int _totalDiscoveredTests; -@@ -373,7 +390,14 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable +@@ -373,7 +389,14 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable _totalExecutedTests += executedIds.Count; } @@ -49,7 +48,7 @@ index cfc34d6..1d45845 100644 TotalDuration += result.Duration; _messages.AddRange(result.Messages ?? []); -@@ -390,7 +414,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable +@@ -390,7 +413,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable ? TestIdentifierList.EveryTest() : new TestIdentifierList(_executedTests); @@ -58,7 +57,7 @@ index cfc34d6..1d45845 100644 public ITestIdentifiers BuildTimedOutTests() => new TestIdentifierList(TimedOutTests); -@@ -525,7 +549,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable +@@ -525,7 +548,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable var duration = DateTime.UtcNow - startTime; var finishedTests = testResults.Where(x => x.Node.ExecutionState is not "in-progress").ToList(); @@ -67,7 +66,7 @@ index cfc34d6..1d45845 100644 lock (_discoveryLock) { -@@ -542,7 +566,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable +@@ -542,7 +565,7 @@ public class SingleMicrosoftTestPlatformRunner : IDisposable } var errorMessagesStr = string.Join(Environment.NewLine, From 2711dbf98c57ec45276d85086d2a527234e5a681 Mon Sep 17 00:00:00 2001 From: Stuart Lang Date: Sat, 11 Apr 2026 15:50:50 +0100 Subject: [PATCH 04/11] Fix AssemblyVersion.cs compile error in VB/F# sample projects The AssemblyVersion.cs file was unconditionally included via Library.targets, causing build failures in VB.NET and F# projects that import it. Co-Authored-By: Claude Opus 4.6 --- eng/Library.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Library.targets b/eng/Library.targets index bbc31c2cd4e..1266798e671 100644 --- a/eng/Library.targets +++ b/eng/Library.targets @@ -39,7 +39,7 @@ - + From 0e8450dc4702b79a4ea4ee078dea7945602cbcb9 Mon Sep 17 00:00:00 2001 From: Stuart Lang Date: Sat, 11 Apr 2026 23:11:13 +0100 Subject: [PATCH 05/11] Increase mutation test timeout for MTP runner overhead The MTP test runner has higher per-session overhead than vstest due to JSON-RPC server process startup per mutation. The legacy mutation job (889 mutants, 2113 tests) needs ~80-90 minutes with MTP vs ~6 minutes with vstest. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/mutation-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mutation-tests.yml b/.github/workflows/mutation-tests.yml index 5d0948a4cb4..4b16a62c966 100644 --- a/.github/workflows/mutation-tests.yml +++ b/.github/workflows/mutation-tests.yml @@ -24,7 +24,7 @@ jobs: mutations: name: 'mutations-${{ matrix.name }}' runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 120 strategy: fail-fast: false From e42054b235dd1a84ea0f1b739f0e3a4354cfb8fb Mon Sep 17 00:00:00 2001 From: Stuart Lang Date: Sat, 11 Apr 2026 23:28:30 +0100 Subject: [PATCH 06/11] Revert "Increase mutation test timeout for MTP runner overhead" This reverts commit 0e8450dc4702b79a4ea4ee078dea7945602cbcb9. --- .github/workflows/mutation-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mutation-tests.yml b/.github/workflows/mutation-tests.yml index 4b16a62c966..5d0948a4cb4 100644 --- a/.github/workflows/mutation-tests.yml +++ b/.github/workflows/mutation-tests.yml @@ -24,7 +24,7 @@ jobs: mutations: name: 'mutations-${{ matrix.name }}' runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 60 strategy: fail-fast: false From 5fccb3d3d6909a3224d97399578d5e74cb3348dc Mon Sep 17 00:00:00 2001 From: Stuart Lang Date: Sat, 11 Apr 2026 23:32:45 +0100 Subject: [PATCH 07/11] Fix Stryker MTP deadlock for legacy project by removing target-framework The legacy Polly.csproj targets net6.0 (no net8.0), causing a framework mismatch when Stryker's MTP runner tried to test mutated net6.0 assemblies in a net8.0 test host. This caused mutation testing to deadlock with zero progress output. Removing target-framework lets Stryker auto-detect the correct framework per project, matching the upstream main branch config. Co-Authored-By: Claude Opus 4.6 --- eng/stryker-config.json | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/stryker-config.json b/eng/stryker-config.json index ea4dbcf2ff1..b36637e5ff9 100644 --- a/eng/stryker-config.json +++ b/eng/stryker-config.json @@ -22,7 +22,6 @@ "concurrency": 4, "configuration": "Debug", "language-version": "Preview", - "target-framework": "net8.0", "test-runner": "mtp", "thresholds": { "high": 100, From d7868f0e069ca50e67bc37e3c87c268e197bd1da Mon Sep 17 00:00:00 2001 From: Stuart Lang Date: Sun, 12 Apr 2026 01:17:58 +0100 Subject: [PATCH 08/11] Fix Stryker MTP deadlock for legacy project by removing target-framework MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The legacy Polly.csproj targets net6.0 (no net8.0) while Polly.Specs targets net8.0+. This TFM mismatch causes Stryker's MTP runner to deadlock during mutation testing — the MTP server process hangs without sending completion signals, blocking all runners indefinitely. Use the default (vstest) test runner for the legacy project only. The test project still uses xUnit v3 with MTP natively; only Stryker's communication protocol changes for this one project. Co-Authored-By: Claude Opus 4.6 --- cake.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cake.cs b/cake.cs index dbf7b829bda..f414e48bc8e 100644 --- a/cake.cs +++ b/cake.cs @@ -434,6 +434,15 @@ void RunMutationTests(FilePath target, FilePath testProject) strykerConfigPath = PatchStrykerConfig(strykerConfigPath, (config) => config.Remove("ignore-mutations")); } + if (moduleName == "Polly") + { + // The legacy Polly source project targets net6.0 (no net8.0), while the test project + // targets net8.0+. This TFM mismatch triggers a deadlock in Stryker's MTP runner + // during mutation testing (the MTP server hangs without sending completion signals). + // Use the default test runner for this project until the MTP runner handles this case. + strykerConfigPath = PatchStrykerConfig(strykerConfigPath, (config) => config.Remove("test-runner")); + } + if (isGitHubActions && !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("STRYKER_DASHBOARD_API_KEY"))) { From 85e12cc02f30a5273ea1be38958522a2e3dfdf0b Mon Sep 17 00:00:00 2001 From: Stuart Lang Date: Sun, 12 Apr 2026 01:26:00 +0100 Subject: [PATCH 09/11] Revert "Fix Stryker MTP deadlock for legacy project by removing target-framework" This reverts commit d7868f0e069ca50e67bc37e3c87c268e197bd1da. --- cake.cs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/cake.cs b/cake.cs index f414e48bc8e..dbf7b829bda 100644 --- a/cake.cs +++ b/cake.cs @@ -434,15 +434,6 @@ void RunMutationTests(FilePath target, FilePath testProject) strykerConfigPath = PatchStrykerConfig(strykerConfigPath, (config) => config.Remove("ignore-mutations")); } - if (moduleName == "Polly") - { - // The legacy Polly source project targets net6.0 (no net8.0), while the test project - // targets net8.0+. This TFM mismatch triggers a deadlock in Stryker's MTP runner - // during mutation testing (the MTP server hangs without sending completion signals). - // Use the default test runner for this project until the MTP runner handles this case. - strykerConfigPath = PatchStrykerConfig(strykerConfigPath, (config) => config.Remove("test-runner")); - } - if (isGitHubActions && !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("STRYKER_DASHBOARD_API_KEY"))) { From 060ef0c9115124649ae49bcfb277d6086d80dc8e Mon Sep 17 00:00:00 2001 From: Stuart Lang Date: Sun, 12 Apr 2026 01:39:44 +0100 Subject: [PATCH 10/11] Add debug logging for legacy mutation tests to diagnose CI deadlock The MTP runner works locally with concurrency 4 but deadlocks on CI (ubuntu-latest, 2 CPUs). Adding --verbosity debug --log-to-file to capture detailed Stryker logs for the legacy project. The logs will be included in the mutation-report artifact for analysis. Co-Authored-By: Claude Opus 4.6 --- cake.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cake.cs b/cake.cs index dbf7b829bda..d5ac5a0d471 100644 --- a/cake.cs +++ b/cake.cs @@ -462,7 +462,14 @@ void RunMutationTests(FilePath target, FilePath testProject) Information($"Running mutation tests for '{targetFileName}'. Test Project: '{testProject}'"); - var args = $"stryker --project {targetFileName} --test-project {testProject.GetFilename()} --break-at {score} --config-file {strykerConfigPath} --output {strykerOutput}/{targetFileName}"; + var extraArgs = ""; + if (moduleName == "Polly") + { + // Enable debug logging for legacy project to diagnose MTP runner deadlock on CI + extraArgs = " --verbosity debug --log-to-file"; + } + + var args = $"stryker --project {targetFileName} --test-project {testProject.GetFilename()} --break-at {score} --config-file {strykerConfigPath} --output {strykerOutput}/{targetFileName}{extraArgs}"; var testProjectDir = testProject.GetDirectory(); var result = StartProcess("dotnet", new ProcessSettings From 9ec88fda35c7d494b8df24651e11f31f39f2acb7 Mon Sep 17 00:00:00 2001 From: Stuart Lang Date: Sun, 12 Apr 2026 01:43:23 +0100 Subject: [PATCH 11/11] Fix Stryker MTP legacy timeout by increasing CI limit to 120 minutes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The legacy mutation run (889 mutants × 2113 tests) takes ~79 minutes, exceeding the 60-minute timeout. Stryker's dots reporter is buffered on CI so no output appears until the run completes, making timeouts look like deadlocks. Also removes debug logging added for investigation. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/mutation-tests.yml | 2 +- cake.cs | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/mutation-tests.yml b/.github/workflows/mutation-tests.yml index 5d0948a4cb4..4b16a62c966 100644 --- a/.github/workflows/mutation-tests.yml +++ b/.github/workflows/mutation-tests.yml @@ -24,7 +24,7 @@ jobs: mutations: name: 'mutations-${{ matrix.name }}' runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 120 strategy: fail-fast: false diff --git a/cake.cs b/cake.cs index d5ac5a0d471..dbf7b829bda 100644 --- a/cake.cs +++ b/cake.cs @@ -462,14 +462,7 @@ void RunMutationTests(FilePath target, FilePath testProject) Information($"Running mutation tests for '{targetFileName}'. Test Project: '{testProject}'"); - var extraArgs = ""; - if (moduleName == "Polly") - { - // Enable debug logging for legacy project to diagnose MTP runner deadlock on CI - extraArgs = " --verbosity debug --log-to-file"; - } - - var args = $"stryker --project {targetFileName} --test-project {testProject.GetFilename()} --break-at {score} --config-file {strykerConfigPath} --output {strykerOutput}/{targetFileName}{extraArgs}"; + var args = $"stryker --project {targetFileName} --test-project {testProject.GetFilename()} --break-at {score} --config-file {strykerConfigPath} --output {strykerOutput}/{targetFileName}"; var testProjectDir = testProject.GetDirectory(); var result = StartProcess("dotnet", new ProcessSettings