From 56fb4b4d2f2be4156d30158bab4afdb71ecfbdf7 Mon Sep 17 00:00:00 2001
From: Brennan <brecon@microsoft.com>
Date: Wed, 4 Mar 2020 08:10:18 -0800
Subject: [PATCH] Rename Flaky attribute (#19469)
---
.azure/pipelines/ci.yml | 24 ++---
Directory.Build.props | 4 +-
eng/FlakyTests.AfterArcade.props | 7 --
eng/FlakyTests.BeforeArcade.props | 18 ----
eng/QuarantinedTests.AfterArcade.props | 7 ++
eng/QuarantinedTests.BeforeArcade.props | 11 +++
eng/helix/content/runtests.cmd | 10 +-
eng/helix/content/runtests.sh | 6 +-
.../test/E2ETest/Tests/EventTest.cs | 2 +-
.../Identity.Test/IdentityUIScriptsTest.cs | 2 +-
...TempDataInCookiesUsingCookieConsentTest.cs | 4 +-
.../Listener/RequestBodyTests.cs | 4 +-
.../FunctionalTests/ResponseCachingTests.cs | 8 +-
.../test/FunctionalTests/ServerTests.cs | 2 +-
.../Common.FunctionalTests/AppOfflineTests.cs | 2 +-
.../ConfigurationChangeTests.cs | 2 +-
.../Inprocess/FrebTests.cs | 2 +-
.../Inprocess/SynchronousReadAndWriteTests.cs | 2 +-
.../Common.FunctionalTests/LogFileTests.cs | 2 +-
.../test/IIS.Tests/ClientDisconnectTests.cs | 2 +-
.../BindTests/AddressRegistrationTests.cs | 10 +-
.../MaxRequestBufferSizeTests.cs | 2 +-
.../FunctionalTests/UnixDomainSocketsTests.cs | 2 +-
.../ConnectionLimitTests.cs | 2 +-
.../Http2/Http2TimeoutTests.cs | 4 +-
.../Http3/Http3StreamTests.cs | 2 +-
.../Interop.FunctionalTests/H2SpecTests.cs | 2 +-
.../HttpClientHttp2InteropTests.cs | 4 +-
src/Testing/src/FlakyOn.cs | 41 --------
src/Testing/src/xunit/FlakyAttribute.cs | 92 -----------------
src/Testing/src/xunit/FlakyTraitDiscoverer.cs | 38 -------
.../src/xunit/QuarantinedTestAttribute.cs | 57 +++++++++++
.../xunit/QuarantinedTestTraitDiscoverer.cs | 26 +++++
src/Testing/test/FlakyAttributeTest.cs | 99 -------------------
.../test/QuarantinedTestAttributeTest.cs | 22 +++++
.../test/OpenApiAddURLTests.cs | 26 ++---
.../test/InitCommandTest.cs | 1 +
.../test/SecretManagerTests.cs | 20 ++--
.../dotnet-watch/test/AppWithDepsTests.cs | 2 +-
.../dotnet-watch/test/DotNetWatcherTests.cs | 4 +-
.../dotnet-watch/test/GlobbingAppTests.cs | 10 +-
src/Tools/dotnet-watch/test/NoDepsAppTests.cs | 4 +-
42 files changed, 209 insertions(+), 382 deletions(-)
delete mode 100644 eng/FlakyTests.AfterArcade.props
delete mode 100644 eng/FlakyTests.BeforeArcade.props
create mode 100644 eng/QuarantinedTests.AfterArcade.props
create mode 100644 eng/QuarantinedTests.BeforeArcade.props
delete mode 100644 src/Testing/src/FlakyOn.cs
delete mode 100644 src/Testing/src/xunit/FlakyAttribute.cs
delete mode 100644 src/Testing/src/xunit/FlakyTraitDiscoverer.cs
create mode 100644 src/Testing/src/xunit/QuarantinedTestAttribute.cs
create mode 100644 src/Testing/src/xunit/QuarantinedTestTraitDiscoverer.cs
delete mode 100644 src/Testing/test/FlakyAttributeTest.cs
create mode 100644 src/Testing/test/QuarantinedTestAttributeTest.cs
diff --git a/.azure/pipelines/ci.yml b/.azure/pipelines/ci.yml
index 82cb6cb5bce..eb38671015c 100644
--- a/.azure/pipelines/ci.yml
+++ b/.azure/pipelines/ci.yml
@@ -499,15 +499,15 @@ stages:
- powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1"
displayName: Setup IISExpress test certificates and schema
afterBuild:
- - powershell: "& ./build.ps1 -CI -NoBuild -Test /p:RunFlakyTests=true"
- displayName: Run Flaky Tests
+ - powershell: "& ./build.ps1 -CI -NoBuild -Test /p:RunQuarantinedTests=true"
+ displayName: Run Quarantined Tests
continueOnError: true
- task: PublishTestResults@2
- displayName: Publish Flaky Test Results
+ displayName: Publish Quarantined Test Results
inputs:
testResultsFormat: 'xUnit'
testResultsFiles: '*.xml'
- searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Flaky'
+ searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Quarantined'
artifacts:
- name: Windows_Test_Dumps
path: artifacts/dumps/
@@ -574,15 +574,15 @@ stages:
displayName: Pack Packages (for Template tests)
- bash: ./src/ProjectTemplates/build.sh --ci --pack --no-restore --no-build-deps "/bl:artifacts/log/template.pack.binlog"
displayName: Pack Templates (for Template tests)
- - bash: ./build.sh --no-build --ci --test -p:RunFlakyTests=true
- displayName: Run Flaky Tests
+ - bash: ./build.sh --no-build --ci --test -p:RunQuarantinedTests=true
+ displayName: Run Quarantined Tests
continueOnError: true
- task: PublishTestResults@2
- displayName: Publish Flaky Test Results
+ displayName: Publish Quarantined Test Results
inputs:
testResultsFormat: 'xUnit'
testResultsFiles: '*.xml'
- searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Flaky'
+ searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Quarantined'
artifacts:
- name: MacOS_Test_Logs
path: artifacts/log/
@@ -611,15 +611,15 @@ stages:
displayName: Pack Packages (for Template tests)
- bash: ./src/ProjectTemplates/build.sh --ci --pack --no-restore --no-build-deps "/bl:artifacts/log/template.pack.binlog"
displayName: Pack Templates (for Template tests)
- - bash: ./build.sh --no-build --ci --test -p:RunFlakyTests=true
- displayName: Run Flaky Tests
+ - bash: ./build.sh --no-build --ci --test -p:RunQuarantinedTests=true
+ displayName: Run Quarantined Tests
continueOnError: true
- task: PublishTestResults@2
- displayName: Publish Flaky Test Results
+ displayName: Publish Quarantined Test Results
inputs:
testResultsFormat: 'xUnit'
testResultsFiles: '*.xml'
- searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Flaky'
+ searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Quarantined'
artifacts:
- name: Linux_Test_Logs
path: artifacts/log/
diff --git a/Directory.Build.props b/Directory.Build.props
index 3ef40088907..a2765ff5607 100644
--- a/Directory.Build.props
+++ b/Directory.Build.props
@@ -35,9 +35,9 @@
<DisableImplicitComponentsAnalyzers>true</DisableImplicitComponentsAnalyzers>
</PropertyGroup>
- <Import Project="eng\FlakyTests.BeforeArcade.props" />
+ <Import Project="eng\QuarantinedTests.BeforeArcade.props" />
<Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />
- <Import Project="eng\FlakyTests.AfterArcade.props" />
+ <Import Project="eng\QuarantinedTests.AfterArcade.props" />
<PropertyGroup>
<Product>Microsoft ASP.NET Core</Product>
diff --git a/eng/FlakyTests.AfterArcade.props b/eng/FlakyTests.AfterArcade.props
deleted file mode 100644
index 12f631127e8..00000000000
--- a/eng/FlakyTests.AfterArcade.props
+++ /dev/null
@@ -1,7 +0,0 @@
-<Project>
- <!-- Override where xUnit logs and results go if we're doing the flaky run -->
- <PropertyGroup Condition="'$(RunFlakyTests)' == 'true'">
- <ArtifactsLogDir>$(ArtifactsDir)log\$(Configuration)\Flaky\</ArtifactsLogDir>
- <ArtifactsTestResultsDir>$(ArtifactsDir)TestResults\$(Configuration)\Flaky\</ArtifactsTestResultsDir>
- </PropertyGroup>
-</Project>
diff --git a/eng/FlakyTests.BeforeArcade.props b/eng/FlakyTests.BeforeArcade.props
deleted file mode 100644
index 02f9fb6c596..00000000000
--- a/eng/FlakyTests.BeforeArcade.props
+++ /dev/null
@@ -1,18 +0,0 @@
-<Project>
- <!-- Local Dev Flakiness -->
- <PropertyGroup>
- <_FlakyRunAdditionalArgs>-trait "Flaky:All=true"</_FlakyRunAdditionalArgs>
- <_NonFlakyRunAdditionalArgs>-notrait "Flaky:All=true"</_NonFlakyRunAdditionalArgs>
- </PropertyGroup>
-
- <!-- Azure Pipelines Flakiness -->
- <PropertyGroup Condition="'$(TF_BUILD)' != ''">
- <_FlakyRunAdditionalArgs>$(_FlakyRunAdditionalArgs) -trait "Flaky:AzP:All=true" -trait "Flaky:AzP:OS:$(AGENT_OS)=true"</_FlakyRunAdditionalArgs>
- <_NonFlakyRunAdditionalArgs>$(_NonFlakyRunAdditionalArgs) -notrait "Flaky:AzP:All=true" -notrait "Flaky:AzP:OS:$(AGENT_OS)=true"</_NonFlakyRunAdditionalArgs>
- </PropertyGroup>
-
- <PropertyGroup>
- <TestRunnerAdditionalArguments Condition="'$(RunFlakyTests)' == ''">$(_NonFlakyRunAdditionalArgs) $(TestRunnerAdditionalArguments)</TestRunnerAdditionalArguments>
- <TestRunnerAdditionalArguments Condition="'$(RunFlakyTests)' == 'true'">$(_FlakyRunAdditionalArgs) $(TestRunnerAdditionalArguments)</TestRunnerAdditionalArguments>
- </PropertyGroup>
-</Project>
diff --git a/eng/QuarantinedTests.AfterArcade.props b/eng/QuarantinedTests.AfterArcade.props
new file mode 100644
index 00000000000..4e0b5cde309
--- /dev/null
+++ b/eng/QuarantinedTests.AfterArcade.props
@@ -0,0 +1,7 @@
+<Project>
+ <!-- Override where xUnit logs and results go if we're doing the Quarantined run -->
+ <PropertyGroup Condition="'$(RunQuarantinedTests)' == 'true'">
+ <ArtifactsLogDir>$(ArtifactsDir)log\$(Configuration)\Quarantined\</ArtifactsLogDir>
+ <ArtifactsTestResultsDir>$(ArtifactsDir)TestResults\$(Configuration)\Quarantined\</ArtifactsTestResultsDir>
+ </PropertyGroup>
+</Project>
diff --git a/eng/QuarantinedTests.BeforeArcade.props b/eng/QuarantinedTests.BeforeArcade.props
new file mode 100644
index 00000000000..5fa5fdcaf65
--- /dev/null
+++ b/eng/QuarantinedTests.BeforeArcade.props
@@ -0,0 +1,11 @@
+<Project>
+ <PropertyGroup>
+ <_QuarantinedTestRunAdditionalArgs>-trait "Quarantined=true"</_QuarantinedTestRunAdditionalArgs>
+ <_NonQuarantinedTestRunAdditionalArgs>-notrait "Quarantined=true"</_NonQuarantinedTestRunAdditionalArgs>
+ </PropertyGroup>
+
+ <PropertyGroup>
+ <TestRunnerAdditionalArguments Condition="'$(RunQuarantinedTests)' == ''">$(_NonQuarantinedTestRunAdditionalArgs) $(TestRunnerAdditionalArguments)</TestRunnerAdditionalArguments>
+ <TestRunnerAdditionalArguments Condition="'$(RunQuarantinedTests)' == 'true'">$(_QuarantinedTestRunAdditionalArgs) $(TestRunnerAdditionalArguments)</TestRunnerAdditionalArguments>
+ </PropertyGroup>
+</Project>
diff --git a/eng/helix/content/runtests.cmd b/eng/helix/content/runtests.cmd
index 73cf658c18c..5fe1af7eb61 100644
--- a/eng/helix/content/runtests.cmd
+++ b/eng/helix/content/runtests.cmd
@@ -53,23 +53,21 @@ if %$quarantined%==True (
set %$quarantined=true
)
-set NONQUARANTINE_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:%HELIX%!=true"
-set QUARANTINE_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:%HELIX%=true"
+set NONQUARANTINE_FILTER="Quarantined!=true"
+set QUARANTINE_FILTER="Quarantined=true"
if %$quarantined%==true (
echo Running quarantined tests.
%DOTNET_ROOT%\dotnet vstest %$target% --logger:xunit --TestCaseFilter:%QUARANTINE_FILTER%
if errorlevel 1 (
- echo Failure in flaky test 1>&2
+ echo Failure in quarantined test 1>&2
REM DO NOT EXIT and DO NOT SET EXIT_CODE to 1
)
) else (
- REM We need to specify all possible Flaky filters that apply to this environment, because the flaky attribute
- REM only puts the explicit filter traits the user provided in
REM Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md
echo Running non-quarantined tests.
%DOTNET_ROOT%\dotnet vstest %$target% --logger:xunit --TestCaseFilter:%NONQUARANTINE_FILTER%
if errorlevel 1 (
- echo Failure in non-flaky test 1>&2
+ echo Failure in non-quarantined test 1>&2
set exit_code=1
REM DO NOT EXIT
)
diff --git a/eng/helix/content/runtests.sh b/eng/helix/content/runtests.sh
index c31297d587a..7d319e0a386 100644
--- a/eng/helix/content/runtests.sh
+++ b/eng/helix/content/runtests.sh
@@ -115,11 +115,9 @@ fi
exit_code=0
-# We need to specify all possible quarantined filters that apply to this environment, because the quarantine attribute
-# only puts the explicit filter traits the user provided in the flaky attribute
# Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md
-NONQUARANTINE_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:$helix_queue_name!=true"
-QUARANTINE_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:$helix_queue_name=true"
+NONQUARANTINE_FILTER="Quarantined!=true"
+QUARANTINE_FILTER="Quarantined=true"
if [ "$quarantined" == true ]; then
echo "Running all tests including quarantined."
$DOTNET_ROOT/dotnet vstest $test_binary_path --logger:xunit --TestCaseFilter:"$QUARANTINE_FILTER"
diff --git a/src/Components/test/E2ETest/Tests/EventTest.cs b/src/Components/test/E2ETest/Tests/EventTest.cs
index 06dbbb969f3..d2fd55b195a 100644
--- a/src/Components/test/E2ETest/Tests/EventTest.cs
+++ b/src/Components/test/E2ETest/Tests/EventTest.cs
@@ -167,7 +167,7 @@ namespace Microsoft.AspNetCore.Components.E2ETest.Tests
}
[Fact]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1987", FlakyOn.AzP.Windows)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1987")]
public void InputEvent_RespondsOnKeystrokes()
{
Browser.MountTestComponent<InputEventComponent>();
diff --git a/src/Identity/test/Identity.Test/IdentityUIScriptsTest.cs b/src/Identity/test/Identity.Test/IdentityUIScriptsTest.cs
index 96157ed4eaa..005895f9c07 100644
--- a/src/Identity/test/Identity.Test/IdentityUIScriptsTest.cs
+++ b/src/Identity/test/Identity.Test/IdentityUIScriptsTest.cs
@@ -79,7 +79,7 @@ namespace Microsoft.AspNetCore.Identity.Test
[Theory]
[MemberData(nameof(ScriptWithFallbackSrcData))]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2267", FlakyOn.AzP.macOS)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2267")]
public async Task IdentityUI_ScriptTags_FallbackSourceContent_Matches_CDNContent(ScriptTag scriptTag)
{
var wwwrootDir = Path.Combine(GetProjectBasePath(), "wwwroot", scriptTag.Version);
diff --git a/src/Mvc/test/Mvc.FunctionalTests/TempDataInCookiesUsingCookieConsentTest.cs b/src/Mvc/test/Mvc.FunctionalTests/TempDataInCookiesUsingCookieConsentTest.cs
index 14a46811807..546f862e212 100644
--- a/src/Mvc/test/Mvc.FunctionalTests/TempDataInCookiesUsingCookieConsentTest.cs
+++ b/src/Mvc/test/Mvc.FunctionalTests/TempDataInCookiesUsingCookieConsentTest.cs
@@ -30,7 +30,7 @@ namespace Microsoft.AspNetCore.Mvc.FunctionalTests
builder.UseStartup<BasicWebSite.StartupWithCookieTempDataProviderAndCookieConsent>();
[Fact]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1803", FlakyOn.AzP.Windows)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1803")]
public async Task CookieTempDataProviderCookie_SetInResponse_OnGrantingConsent()
{
// Arrange
@@ -64,7 +64,7 @@ namespace Microsoft.AspNetCore.Mvc.FunctionalTests
}
[Fact]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1803", FlakyOn.AzP.Windows)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1803")]
public async Task CookieTempDataProviderCookie_NotSetInResponse_OnNoConsent()
{
// Arrange
diff --git a/src/Servers/HttpSys/test/FunctionalTests/Listener/RequestBodyTests.cs b/src/Servers/HttpSys/test/FunctionalTests/Listener/RequestBodyTests.cs
index 2f0308af9be..8e2860cf69c 100644
--- a/src/Servers/HttpSys/test/FunctionalTests/Listener/RequestBodyTests.cs
+++ b/src/Servers/HttpSys/test/FunctionalTests/Listener/RequestBodyTests.cs
@@ -17,7 +17,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys.Listener
public class RequestBodyTests
{
[ConditionalFact]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1826", FlakyOn.All)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1826")]
public async Task RequestBody_SyncReadDisabledByDefault_WorksWhenEnabled()
{
string address;
@@ -142,7 +142,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys.Listener
}
[ConditionalFact]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2206", FlakyOn.All)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2206")]
public async Task RequestBody_ReadAsyncPartialBodyAndExpiredTimeout_Canceled()
{
StaggardContent content = new StaggardContent();
diff --git a/src/Servers/HttpSys/test/FunctionalTests/ResponseCachingTests.cs b/src/Servers/HttpSys/test/FunctionalTests/ResponseCachingTests.cs
index 2c45b352e32..1112420ed66 100644
--- a/src/Servers/HttpSys/test/FunctionalTests/ResponseCachingTests.cs
+++ b/src/Servers/HttpSys/test/FunctionalTests/ResponseCachingTests.cs
@@ -45,7 +45,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys.FunctionalTests
}
[ConditionalFact]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2135", FlakyOn.All)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2135")]
public async Task Caching_JustPublic_NotCached()
{
var requestCount = 1;
@@ -86,7 +86,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys.FunctionalTests
}
[ConditionalFact]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2207", FlakyOn.All)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2207")]
public async Task Caching_WithoutContentType_Cached_OnWin7AndWin2008R2()
{
if (Utilities.IsWin8orLater)
@@ -237,7 +237,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys.FunctionalTests
[ConditionalTheory]
[InlineData("0")]
[InlineData("-1")]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2208", FlakyOn.All)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2208")]
public async Task Caching_InvalidExpires_NotCached(string expiresValue)
{
var requestCount = 1;
@@ -378,7 +378,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys.FunctionalTests
}
[ConditionalFact]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2209", FlakyOn.All)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2209")]
public async Task Caching_VariousStatusCodes_Cached()
{
var requestCount = 1;
diff --git a/src/Servers/HttpSys/test/FunctionalTests/ServerTests.cs b/src/Servers/HttpSys/test/FunctionalTests/ServerTests.cs
index bf266025b47..fbea889de4a 100644
--- a/src/Servers/HttpSys/test/FunctionalTests/ServerTests.cs
+++ b/src/Servers/HttpSys/test/FunctionalTests/ServerTests.cs
@@ -131,7 +131,7 @@ namespace Microsoft.AspNetCore.Server.HttpSys
}
[ConditionalFact]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2267", FlakyOn.All)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2267")]
public async Task Server_ShutdownDuringRequest_Success()
{
Task<string> responseTask;
diff --git a/src/Servers/IIS/IIS/test/Common.FunctionalTests/AppOfflineTests.cs b/src/Servers/IIS/IIS/test/Common.FunctionalTests/AppOfflineTests.cs
index 13409c545d0..89c44e719f4 100644
--- a/src/Servers/IIS/IIS/test/Common.FunctionalTests/AppOfflineTests.cs
+++ b/src/Servers/IIS/IIS/test/Common.FunctionalTests/AppOfflineTests.cs
@@ -211,7 +211,7 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests
[ConditionalTheory]
[InlineData(HostingModel.InProcess)]
[InlineData(HostingModel.OutOfProcess)]
- [Flaky("https://github.com/dotnet/aspnetcore/issues/7075", FlakyOn.All)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore/issues/7075")]
public async Task AppOfflineAddedAndRemovedStress(HostingModel hostingModel)
{
var deploymentResult = await AssertStarts(hostingModel);
diff --git a/src/Servers/IIS/IIS/test/Common.FunctionalTests/ConfigurationChangeTests.cs b/src/Servers/IIS/IIS/test/Common.FunctionalTests/ConfigurationChangeTests.cs
index 92f581b1563..66ffde7f208 100644
--- a/src/Servers/IIS/IIS/test/Common.FunctionalTests/ConfigurationChangeTests.cs
+++ b/src/Servers/IIS/IIS/test/Common.FunctionalTests/ConfigurationChangeTests.cs
@@ -79,7 +79,7 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests
[ConditionalTheory]
[InlineData(HostingModel.InProcess)]
[InlineData(HostingModel.OutOfProcess)]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1794", FlakyOn.All)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1794")]
public async Task ConfigurationTouchedStress(HostingModel hostingModel)
{
var deploymentResult = await DeployAsync(Fixture.GetBaseDeploymentParameters(hostingModel));
diff --git a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/FrebTests.cs b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/FrebTests.cs
index 68c1762048e..7573fc021d5 100644
--- a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/FrebTests.cs
+++ b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/FrebTests.cs
@@ -80,7 +80,7 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests.InProcess
// I think this test is flaky due to freb file not being created quickly enough.
// Adding extra logging, marking as flaky, and repeating should help
[ConditionalFact]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2570", FlakyOn.Helix.All)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2570")]
[Repeat(10)]
[RequiresIIS(IISCapability.FailedRequestTracingModule)]
public async Task CheckFrebDisconnect()
diff --git a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/SynchronousReadAndWriteTests.cs b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/SynchronousReadAndWriteTests.cs
index 72fcdeefd5a..e896acc67c3 100644
--- a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/SynchronousReadAndWriteTests.cs
+++ b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Inprocess/SynchronousReadAndWriteTests.cs
@@ -22,7 +22,7 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests.InProcess
}
[ConditionalFact]
- [Flaky("https://github.com/dotnet/aspnetcore/issues/7341", FlakyOn.Helix.All)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore/issues/7341")]
public async Task ReadAndWriteSynchronously()
{
for (int i = 0; i < 100; i++)
diff --git a/src/Servers/IIS/IIS/test/Common.FunctionalTests/LogFileTests.cs b/src/Servers/IIS/IIS/test/Common.FunctionalTests/LogFileTests.cs
index 6a15e6f4b3b..0a601873150 100644
--- a/src/Servers/IIS/IIS/test/Common.FunctionalTests/LogFileTests.cs
+++ b/src/Servers/IIS/IIS/test/Common.FunctionalTests/LogFileTests.cs
@@ -171,7 +171,7 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests
[ConditionalTheory]
[MemberData(nameof(TestVariants))]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2200", FlakyOn.All)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2200")]
public async Task CheckUTF8File(TestVariant variant)
{
var path = "CheckConsoleFunctions";
diff --git a/src/Servers/IIS/IIS/test/IIS.Tests/ClientDisconnectTests.cs b/src/Servers/IIS/IIS/test/IIS.Tests/ClientDisconnectTests.cs
index 5fe5d75a870..354b8a2229c 100644
--- a/src/Servers/IIS/IIS/test/IIS.Tests/ClientDisconnectTests.cs
+++ b/src/Servers/IIS/IIS/test/IIS.Tests/ClientDisconnectTests.cs
@@ -225,7 +225,7 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests
}
[ConditionalFact]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1817", FlakyOn.AzP.Windows)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1817")]
public async Task ReaderThrowsResetExceptionOnInvalidBody()
{
var requestStartedCompletionSource = CreateTaskCompletionSource();
diff --git a/src/Servers/Kestrel/test/BindTests/AddressRegistrationTests.cs b/src/Servers/Kestrel/test/BindTests/AddressRegistrationTests.cs
index de4189a207e..787d3fe7c8f 100644
--- a/src/Servers/Kestrel/test/BindTests/AddressRegistrationTests.cs
+++ b/src/Servers/Kestrel/test/BindTests/AddressRegistrationTests.cs
@@ -100,7 +100,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
[ConditionalTheory]
[MemberData(nameof(IPEndPointRegistrationDataDynamicPort))]
[IPv6SupportedCondition]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2074", FlakyOn.AzP.macOS)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2074")]
public async Task RegisterIPEndPoint_DynamicPort_Success(IPEndPoint endPoint, string testUrl)
{
await RegisterIPEndPoint_Success(endPoint, testUrl);
@@ -109,7 +109,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
[ConditionalTheory]
[MemberData(nameof(IPEndPointRegistrationDataPort443))]
[IPv6SupportedCondition]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2711", FlakyOn.AzP.All)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2711")]
public async Task RegisterIPEndPoint_Port443_Success(IPEndPoint endpoint, string testUrl)
{
if (!CanBindToEndpoint(endpoint.Address, 443))
@@ -131,7 +131,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
[ConditionalTheory]
[MemberData(nameof(AddressRegistrationDataIPv6Port5000Default))]
[IPv6SupportedCondition]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2711", FlakyOn.AzP.All)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2711")]
public async Task RegisterAddresses_IPv6Port5000Default_Success(string addressInput, string[] testUrls)
{
if (!CanBindToEndpoint(IPAddress.Loopback, 5000) || !CanBindToEndpoint(IPAddress.IPv6Loopback, 5000))
@@ -145,7 +145,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
[ConditionalTheory]
[MemberData(nameof(AddressRegistrationDataIPv6Port80))]
[IPv6SupportedCondition]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2711", FlakyOn.AzP.All)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2711")]
public async Task RegisterAddresses_IPv6Port80_Success(string addressInput, string[] testUrls)
{
if (!CanBindToEndpoint(IPAddress.Loopback, 80) || !CanBindToEndpoint(IPAddress.IPv6Loopback, 80))
@@ -157,7 +157,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
}
[ConditionalTheory]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2179", FlakyOn.Helix.All)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2179")]
[MemberData(nameof(AddressRegistrationDataIPv6ScopeId))]
[IPv6SupportedCondition]
[IPv6ScopeIdPresentCondition]
diff --git a/src/Servers/Kestrel/test/FunctionalTests/MaxRequestBufferSizeTests.cs b/src/Servers/Kestrel/test/FunctionalTests/MaxRequestBufferSizeTests.cs
index b3c17c8ea34..59c3b9ce4cb 100644
--- a/src/Servers/Kestrel/test/FunctionalTests/MaxRequestBufferSizeTests.cs
+++ b/src/Servers/Kestrel/test/FunctionalTests/MaxRequestBufferSizeTests.cs
@@ -108,7 +108,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
}
}
[Theory]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2489", FlakyOn.AzP.All)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2489")]
[MemberData(nameof(LargeUploadData))]
public async Task LargeUpload(long? maxRequestBufferSize, bool connectionAdapter, bool expectPause)
{
diff --git a/src/Servers/Kestrel/test/FunctionalTests/UnixDomainSocketsTests.cs b/src/Servers/Kestrel/test/FunctionalTests/UnixDomainSocketsTests.cs
index cca069aeaeb..dc247c65437 100644
--- a/src/Servers/Kestrel/test/FunctionalTests/UnixDomainSocketsTests.cs
+++ b/src/Servers/Kestrel/test/FunctionalTests/UnixDomainSocketsTests.cs
@@ -30,7 +30,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.FunctionalTests
#endif
[ConditionalFact]
[CollectDump]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task TestUnixDomainSocket()
{
var path = Path.GetTempFileName();
diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/ConnectionLimitTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/ConnectionLimitTests.cs
index 43c92b01219..532629f0ddb 100644
--- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/ConnectionLimitTests.cs
+++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/ConnectionLimitTests.cs
@@ -142,7 +142,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests
}
[Fact]
- [Flaky("https://github.com/aspnet/KestrelHttpServer/issues/2282", FlakyOn.AzP.macOS)]
+ [QuarantinedTest("https://github.com/aspnet/KestrelHttpServer/issues/2282")]
public async Task ConnectionCountingReturnsToZero()
{
const int count = 100;
diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2TimeoutTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2TimeoutTests.cs
index 1d18ac20c06..5ae038fed11 100644
--- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2TimeoutTests.cs
+++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2TimeoutTests.cs
@@ -263,7 +263,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
}
[Fact]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/1323", FlakyOn.All)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/1323")]
public async Task DATA_Sent_TooSlowlyDueToSocketBackPressureOnSmallWrite_AbortsConnectionAfterGracePeriod()
{
var mockSystemClock = _serviceContext.MockSystemClock;
@@ -316,7 +316,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
}
[Fact]
- [Flaky("<no longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task DATA_Sent_TooSlowlyDueToSocketBackPressureOnLargeWrite_AbortsConnectionAfterRateTimeout()
{
var mockSystemClock = _serviceContext.MockSystemClock;
diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http3/Http3StreamTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http3/Http3StreamTests.cs
index ac206bcaa4f..a0445ab86ff 100644
--- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http3/Http3StreamTests.cs
+++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http3/Http3StreamTests.cs
@@ -264,7 +264,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task MissingAuthority_200Status()
{
var headers = new[]
diff --git a/src/Servers/Kestrel/test/Interop.FunctionalTests/H2SpecTests.cs b/src/Servers/Kestrel/test/Interop.FunctionalTests/H2SpecTests.cs
index d7b2066a3f3..862adb0bb48 100644
--- a/src/Servers/Kestrel/test/Interop.FunctionalTests/H2SpecTests.cs
+++ b/src/Servers/Kestrel/test/Interop.FunctionalTests/H2SpecTests.cs
@@ -21,7 +21,7 @@ namespace Interop.FunctionalTests
{
[ConditionalTheory]
[MemberData(nameof(H2SpecTestCases))]
- [Flaky("https://github.com/dotnet/aspnetcore-internal/issues/2225", FlakyOn.Helix.All)]
+ [QuarantinedTest("https://github.com/dotnet/aspnetcore-internal/issues/2225")]
public async Task RunIndividualTestCase(H2SpecTestCase testCase)
{
var hostBuilder = new WebHostBuilder()
diff --git a/src/Servers/Kestrel/test/Interop.FunctionalTests/HttpClientHttp2InteropTests.cs b/src/Servers/Kestrel/test/Interop.FunctionalTests/HttpClientHttp2InteropTests.cs
index 59c212d2eda..a3ac8e7656d 100644
--- a/src/Servers/Kestrel/test/Interop.FunctionalTests/HttpClientHttp2InteropTests.cs
+++ b/src/Servers/Kestrel/test/Interop.FunctionalTests/HttpClientHttp2InteropTests.cs
@@ -1024,7 +1024,7 @@ namespace Interop.FunctionalTests
}
[Theory]
- [Flaky("https://github.com/dotnet/runtime/issues/860", FlakyOn.All)]
+ [QuarantinedTest("https://github.com/dotnet/runtime/issues/860")]
[MemberData(nameof(SupportedSchemes))]
public async Task RequestHeaders_MultipleFrames_Accepted(string scheme)
{
@@ -1346,7 +1346,7 @@ namespace Interop.FunctionalTests
// Settings_MaxFrameSize_Larger_Client - Not configurable
[Theory]
- [Flaky("https://github.com/dotnet/runtime/issues/860", FlakyOn.All)]
+ [QuarantinedTest("https://github.com/dotnet/runtime/issues/860")]
[MemberData(nameof(SupportedSchemes))]
public async Task Settings_MaxHeaderListSize_Server(string scheme)
{
diff --git a/src/Testing/src/FlakyOn.cs b/src/Testing/src/FlakyOn.cs
deleted file mode 100644
index 81d92990439..00000000000
--- a/src/Testing/src/FlakyOn.cs
+++ /dev/null
@@ -1,41 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-
-namespace Microsoft.AspNetCore.Testing
-{
- public static class FlakyOn
- {
- public const string All = "All";
-
- public static class Helix
- {
- public const string All = QueuePrefix + "All";
-
- public const string Fedora28Amd64 = QueuePrefix + HelixQueues.Fedora28Amd64;
- public const string Fedora27Amd64 = QueuePrefix + HelixQueues.Fedora27Amd64;
- public const string Redhat7Amd64 = QueuePrefix + HelixQueues.Redhat7Amd64;
- public const string Debian9Amd64 = QueuePrefix + HelixQueues.Debian9Amd64;
- public const string Debian8Amd64 = QueuePrefix + HelixQueues.Debian8Amd64;
- public const string Centos7Amd64 = QueuePrefix + HelixQueues.Centos7Amd64;
- public const string Ubuntu1604Amd64 = QueuePrefix + HelixQueues.Ubuntu1604Amd64;
- public const string Ubuntu1810Amd64 = QueuePrefix + HelixQueues.Ubuntu1810Amd64;
- public const string macOS1012Amd64 = QueuePrefix + HelixQueues.macOS1012Amd64;
- public const string Windows10Amd64 = QueuePrefix + HelixQueues.Windows10Amd64;
-
- private const string Prefix = "Helix:";
- private const string QueuePrefix = Prefix + "Queue:";
- }
-
- public static class AzP
- {
- public const string All = Prefix + "All";
- public const string Windows = OsPrefix + "Windows_NT";
- public const string macOS = OsPrefix + "Darwin";
- public const string Linux = OsPrefix + "Linux";
-
- private const string Prefix = "AzP:";
- private const string OsPrefix = Prefix + "OS:";
- }
- }
-}
diff --git a/src/Testing/src/xunit/FlakyAttribute.cs b/src/Testing/src/xunit/FlakyAttribute.cs
deleted file mode 100644
index f77c56dfc0c..00000000000
--- a/src/Testing/src/xunit/FlakyAttribute.cs
+++ /dev/null
@@ -1,92 +0,0 @@
-using System;
-using System.Collections.Generic;
-using Xunit.Sdk;
-
-namespace Microsoft.AspNetCore.Testing
-{
- /// <summary>
- /// Marks a test as "Flaky" so that the build will sequester it and ignore failures.
- /// </summary>
- /// <remarks>
- /// <para>
- /// This attribute works by applying xUnit.net "Traits" based on the criteria specified in the attribute
- /// properties. Once these traits are applied, build scripts can include/exclude tests based on them.
- /// </para>
- /// <para>
- /// All flakiness-related traits start with <c>Flaky:</c> and are grouped first by the process running the tests: Azure Pipelines (AzP) or Helix.
- /// Then there is a segment specifying the "selector" which indicates where the test is flaky. Finally a segment specifying the value of that selector.
- /// The value of these traits is always either "true" or the trait is not present. We encode the entire selector in the name of the trait because xUnit.net only
- /// provides "==" and "!=" operators for traits, there is no way to check if a trait "contains" or "does not contain" a value. VSTest does support "contains" checks
- /// but does not appear to support "does not contain" checks. Using this pattern means we can use simple "==" and "!=" checks to either only run flaky tests, or exclude
- /// flaky tests.
- /// </para>
- /// </remarks>
- /// <example>
- /// <code>
- /// [Fact]
- /// [Flaky("...", HelixQueues.Fedora28Amd64, AzurePipelines.macOS)]
- /// public void FlakyTest()
- /// {
- /// // Flakiness
- /// }
- /// </code>
- ///
- /// <para>
- /// The above example generates the following facets:
- /// </para>
- ///
- /// <list type="bullet">
- /// <item>
- /// <description><c>Flaky:Helix:Queue:Fedora.28.Amd64.Open</c> = <c>true</c></description>
- /// </item>
- /// <item>
- /// <description><c>Flaky:AzP:OS:Darwin</c> = <c>true</c></description>
- /// </item>
- /// </list>
- ///
- /// <para>
- /// Given the above attribute, the Azure Pipelines macOS run can easily filter this test out by passing <c>-notrait "Flaky:AzP:OS:all=true" -notrait "Flaky:AzP:OS:Darwin=true"</c>
- /// to <c>xunit.console.exe</c>. Similarly, it can run only flaky tests using <c>-trait "Flaky:AzP:OS:all=true" -trait "Flaky:AzP:OS:Darwin=true"</c>
- /// </para>
- /// </example>
- [TraitDiscoverer("Microsoft.AspNetCore.Testing." + nameof(FlakyTraitDiscoverer), "Microsoft.AspNetCore.Testing")]
- [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly)]
- public sealed class FlakyAttribute : Attribute, ITraitAttribute
- {
- /// <summary>
- /// Gets a URL to a GitHub issue tracking this flaky test.
- /// </summary>
- public string GitHubIssueUrl { get; }
-
- public IReadOnlyList<string> Filters { get; }
-
- /// <summary>
- /// Initializes a new instance of the <see cref="FlakyAttribute"/> class with the specified <see cref="GitHubIssueUrl"/> and a list of <see cref="Filters"/>. If no
- /// filters are provided, the test is considered flaky in all environments.
- /// </summary>
- /// <remarks>
- /// At least one filter is required.
- /// </remarks>
- /// <param name="gitHubIssueUrl">The URL to a GitHub issue tracking this flaky test.</param>
- /// <param name="firstFilter">The first filter that indicates where the test is flaky. Use a value from <see cref="FlakyOn"/>.</param>
- /// <param name="additionalFilters">A list of additional filters that define where this test is flaky. Use values in <see cref="FlakyOn"/>.</param>
- public FlakyAttribute(string gitHubIssueUrl, string firstFilter, params string[] additionalFilters)
- {
- if (string.IsNullOrEmpty(gitHubIssueUrl))
- {
- throw new ArgumentNullException(nameof(gitHubIssueUrl));
- }
-
- if (string.IsNullOrEmpty(firstFilter))
- {
- throw new ArgumentNullException(nameof(firstFilter));
- }
-
- GitHubIssueUrl = gitHubIssueUrl;
- var filters = new List<string>();
- filters.Add(firstFilter);
- filters.AddRange(additionalFilters);
- Filters = filters;
- }
- }
-}
diff --git a/src/Testing/src/xunit/FlakyTraitDiscoverer.cs b/src/Testing/src/xunit/FlakyTraitDiscoverer.cs
deleted file mode 100644
index 4e6bc27b1bb..00000000000
--- a/src/Testing/src/xunit/FlakyTraitDiscoverer.cs
+++ /dev/null
@@ -1,38 +0,0 @@
-using System;
-using System.Collections.Generic;
-using Xunit.Abstractions;
-using Xunit.Sdk;
-
-// Do not change this namespace without changing the usage in FlakyAttribute
-namespace Microsoft.AspNetCore.Testing
-{
- public class FlakyTraitDiscoverer : ITraitDiscoverer
- {
- public IEnumerable<KeyValuePair<string, string>> GetTraits(IAttributeInfo traitAttribute)
- {
- if (traitAttribute is ReflectionAttributeInfo attribute && attribute.Attribute is FlakyAttribute flakyAttribute)
- {
- return GetTraitsCore(flakyAttribute);
- }
- else
- {
- throw new InvalidOperationException("The 'Flaky' attribute is only supported via reflection.");
- }
- }
-
- private IEnumerable<KeyValuePair<string, string>> GetTraitsCore(FlakyAttribute attribute)
- {
- if (attribute.Filters.Count > 0)
- {
- foreach (var filter in attribute.Filters)
- {
- yield return new KeyValuePair<string, string>($"Flaky:{filter}", "true");
- }
- }
- else
- {
- yield return new KeyValuePair<string, string>($"Flaky:All", "true");
- }
- }
- }
-}
diff --git a/src/Testing/src/xunit/QuarantinedTestAttribute.cs b/src/Testing/src/xunit/QuarantinedTestAttribute.cs
new file mode 100644
index 00000000000..350dcf1e129
--- /dev/null
+++ b/src/Testing/src/xunit/QuarantinedTestAttribute.cs
@@ -0,0 +1,57 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using Xunit.Sdk;
+
+namespace Microsoft.AspNetCore.Testing
+{
+ /// <summary>
+ /// Marks a test as "Quarantined" so that the build will sequester it and ignore failures.
+ /// </summary>
+ /// <remarks>
+ /// <para>
+ /// This attribute works by applying xUnit.net "Traits" based on the criteria specified in the attribute
+ /// properties. Once these traits are applied, build scripts can include/exclude tests based on them.
+ /// </para>
+ /// </remarks>
+ /// <example>
+ /// <code>
+ /// [Fact]
+ /// [QuarantinedTest]
+ /// public void FlakyTest()
+ /// {
+ /// // Flakiness
+ /// }
+ /// </code>
+ ///
+ /// <para>
+ /// The above example generates the following facet:
+ /// </para>
+ ///
+ /// <list type="bullet">
+ /// <item>
+ /// <description><c>Quarantined</c> = <c>true</c></description>
+ /// </item>
+ /// </list>
+ /// </example>
+ [TraitDiscoverer("Microsoft.AspNetCore.Testing." + nameof(QuarantinedTestTraitDiscoverer), "Microsoft.AspNetCore.Testing")]
+ [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class | AttributeTargets.Assembly)]
+ public sealed class QuarantinedTestAttribute : Attribute, ITraitAttribute
+ {
+ /// <summary>
+ /// Gets an optional reason for the quarantining, such as a link to a GitHub issue URL with more details as to why the test is quarantined.
+ /// </summary>
+ public string Reason { get; }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="QuarantinedTestAttribute"/> class with an optional <see cref="Reason"/>.
+ /// </summary>
+ /// <param name="reason">A reason that this test is quarantined.</param>
+ public QuarantinedTestAttribute(string reason = null)
+ {
+ Reason = reason;
+ }
+ }
+}
diff --git a/src/Testing/src/xunit/QuarantinedTestTraitDiscoverer.cs b/src/Testing/src/xunit/QuarantinedTestTraitDiscoverer.cs
new file mode 100644
index 00000000000..256e2b6cfcc
--- /dev/null
+++ b/src/Testing/src/xunit/QuarantinedTestTraitDiscoverer.cs
@@ -0,0 +1,26 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+// Do not change this namespace without changing the usage in QuarantinedTestAttribute
+namespace Microsoft.AspNetCore.Testing
+{
+ public class QuarantinedTestTraitDiscoverer : ITraitDiscoverer
+ {
+ public IEnumerable<KeyValuePair<string, string>> GetTraits(IAttributeInfo traitAttribute)
+ {
+ if (traitAttribute is ReflectionAttributeInfo attribute && attribute.Attribute is QuarantinedTestAttribute quarantinedTestAttribute)
+ {
+ yield return new KeyValuePair<string, string>("Quarantined", "true");
+ }
+ else
+ {
+ throw new InvalidOperationException("The 'QuarantinedTest' attribute is only supported via reflection.");
+ }
+ }
+ }
+}
diff --git a/src/Testing/test/FlakyAttributeTest.cs b/src/Testing/test/FlakyAttributeTest.cs
deleted file mode 100644
index ae06e5cf505..00000000000
--- a/src/Testing/test/FlakyAttributeTest.cs
+++ /dev/null
@@ -1,99 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-using System;
-using System.Collections.Generic;
-using Xunit;
-
-namespace Microsoft.AspNetCore.Testing.Tests
-{
- public class FlakyAttributeTest
- {
- [Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")]
- [Flaky("http://example.com", FlakyOn.All)]
- public void AlwaysFlakyInCI()
- {
- if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX")) || !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS")))
- {
- throw new Exception("Flaky!");
- }
- }
-
- [Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")]
- [Flaky("http://example.com", FlakyOn.Helix.All)]
- public void FlakyInHelixOnly()
- {
- if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX")))
- {
- throw new Exception("Flaky on Helix!");
- }
- }
-
- [Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")]
- [Flaky("http://example.com", FlakyOn.Helix.macOS1012Amd64, FlakyOn.Helix.Fedora28Amd64)]
- public void FlakyInSpecificHelixQueue()
- {
- // Today we don't run Extensions tests on Helix, but this test should light up when we do.
- var queueName = Environment.GetEnvironmentVariable("HELIX");
- if (!string.IsNullOrEmpty(queueName))
- {
- var failingQueues = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { HelixQueues.macOS1012Amd64, HelixQueues.Fedora28Amd64 };
- if (failingQueues.Contains(queueName))
- {
- throw new Exception($"Flaky on Helix Queue '{queueName}' !");
- }
- }
- }
-
- [Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")]
- [Flaky("http://example.com", FlakyOn.AzP.All)]
- public void FlakyInAzPOnly()
- {
- if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS")))
- {
- throw new Exception("Flaky on AzP!");
- }
- }
-
- [Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")]
- [Flaky("http://example.com", FlakyOn.AzP.Windows)]
- public void FlakyInAzPWindowsOnly()
- {
- if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), "Windows_NT"))
- {
- throw new Exception("Flaky on AzP Windows!");
- }
- }
-
- [Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")]
- [Flaky("http://example.com", FlakyOn.AzP.macOS)]
- public void FlakyInAzPmacOSOnly()
- {
- if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), "Darwin"))
- {
- throw new Exception("Flaky on AzP macOS!");
- }
- }
-
- [Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")]
- [Flaky("http://example.com", FlakyOn.AzP.Linux)]
- public void FlakyInAzPLinuxOnly()
- {
- if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), "Linux"))
- {
- throw new Exception("Flaky on AzP Linux!");
- }
- }
-
- [Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")]
- [Flaky("http://example.com", FlakyOn.AzP.Linux, FlakyOn.AzP.macOS)]
- public void FlakyInAzPNonWindowsOnly()
- {
- var agentOs = Environment.GetEnvironmentVariable("AGENT_OS");
- if (string.Equals(agentOs, "Linux") || string.Equals(agentOs, "Darwin"))
- {
- throw new Exception("Flaky on AzP non-Windows!");
- }
- }
- }
-}
diff --git a/src/Testing/test/QuarantinedTestAttributeTest.cs b/src/Testing/test/QuarantinedTestAttributeTest.cs
new file mode 100644
index 00000000000..5fc6c58041a
--- /dev/null
+++ b/src/Testing/test/QuarantinedTestAttributeTest.cs
@@ -0,0 +1,22 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using Xunit;
+
+namespace Microsoft.AspNetCore.Testing.Tests
+{
+ public class QuarantinedTestAttributeTest
+ {
+ [Fact(Skip = "These tests are nice when you need them but annoying when on all the time.")]
+ [QuarantinedTest]
+ public void AlwaysFlakyInCI()
+ {
+ if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX")) || !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS")))
+ {
+ throw new Exception("Flaky!");
+ }
+ }
+ }
+}
diff --git a/src/Tools/Microsoft.dotnet-openapi/test/OpenApiAddURLTests.cs b/src/Tools/Microsoft.dotnet-openapi/test/OpenApiAddURLTests.cs
index 26c9d3c1a1f..0336cfdef28 100644
--- a/src/Tools/Microsoft.dotnet-openapi/test/OpenApiAddURLTests.cs
+++ b/src/Tools/Microsoft.dotnet-openapi/test/OpenApiAddURLTests.cs
@@ -17,7 +17,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
public OpenApiAddURLTests(ITestOutputHelper output) : base(output){ }
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task OpenApi_Add_Url_WithContentDisposition()
{
var project = CreateBasicProject(withOpenApi: false);
@@ -50,7 +50,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task OpenAPI_Add_Url_NoContentDisposition()
{
var project = CreateBasicProject(withOpenApi: false);
@@ -84,7 +84,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task OpenAPI_Add_Url_NoExtension_AssumesJson()
{
var project = CreateBasicProject(withOpenApi: false);
@@ -118,7 +118,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task OpenApi_Add_Url_NoSegment()
{
var project = CreateBasicProject(withOpenApi: false);
@@ -152,7 +152,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task OpenApi_Add_Url()
{
var project = CreateBasicProject(withOpenApi: false);
@@ -185,7 +185,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task OpenApi_Add_Url_SameName_UniqueFile()
{
var project = CreateBasicProject(withOpenApi: false);
@@ -246,7 +246,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task OpenApi_Add_Url_NSwagCSharp()
{
var project = CreateBasicProject(withOpenApi: false);
@@ -279,7 +279,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task OpenApi_Add_Url_NSwagTypeScript()
{
var project = CreateBasicProject(withOpenApi: false);
@@ -312,7 +312,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task OpenApi_Add_Url_OutputFile()
{
var project = CreateBasicProject(withOpenApi: false);
@@ -345,7 +345,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task OpenApi_Add_URL_FileAlreadyExists_Fail()
{
var project = CreateBasicProject(withOpenApi: false);
@@ -404,7 +404,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public void OpenApi_Add_URL_MultipleTimes_OnlyOneReference()
{
var project = CreateBasicProject(withOpenApi: false);
@@ -431,7 +431,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task OpenAPi_Add_URL_InvalidUrl()
{
var project = CreateBasicProject(withOpenApi: false);
@@ -459,7 +459,7 @@ namespace Microsoft.DotNet.OpenApi.Add.Tests
Assert.False(File.Exists(jsonFile));
}
- [Flaky("<No longer needed, tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
[Fact]
public void OpenApi_Add_URL_ActualResponse()
{
diff --git a/src/Tools/dotnet-user-secrets/test/InitCommandTest.cs b/src/Tools/dotnet-user-secrets/test/InitCommandTest.cs
index 3ba82db5d5b..d299c208cab 100644
--- a/src/Tools/dotnet-user-secrets/test/InitCommandTest.cs
+++ b/src/Tools/dotnet-user-secrets/test/InitCommandTest.cs
@@ -57,6 +57,7 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests
}
[Fact]
+ [QuarantinedTest]
public void AddsEscapedSpecificSecretIdToProject()
{
const string SecretId = @"<lots of XML invalid values>&";
diff --git a/src/Tools/dotnet-user-secrets/test/SecretManagerTests.cs b/src/Tools/dotnet-user-secrets/test/SecretManagerTests.cs
index 6d79642a4c2..c16a89197f7 100644
--- a/src/Tools/dotnet-user-secrets/test/SecretManagerTests.cs
+++ b/src/Tools/dotnet-user-secrets/test/SecretManagerTests.cs
@@ -37,7 +37,7 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests
[Theory]
[InlineData(null)]
[InlineData("")]
- [Flaky("<No longer used; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public void Error_MissingId(string id)
{
var project = Path.Combine(_fixture.CreateProject(id), "TestProject.csproj");
@@ -83,7 +83,7 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests
[Theory]
[InlineData(true)]
[InlineData(false)]
- [Flaky("<No longer used; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public void SetSecrets(bool fromCurrentDirectory)
{
var secrets = new KeyValuePair<string, string>[]
@@ -149,19 +149,20 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests
}
[Fact]
+ [QuarantinedTest]
public void SetSecret_Update_Existing_Secret()
{
var projectPath = _fixture.GetTempSecretProject();
var secretManager = CreateProgram();
- secretManager.RunInternal("set", "secret1", "value1", "-p", projectPath);
+ secretManager.RunInternal("set", "secret1", "value1", "-p", projectPath, "--verbose");
Assert.Contains("Successfully saved secret1 = value1 to the secret store.", _console.GetOutput());
- secretManager.RunInternal("set", "secret1", "value2", "-p", projectPath);
+ secretManager.RunInternal("set", "secret1", "value2", "-p", projectPath, "--verbose");
Assert.Contains("Successfully saved secret1 = value2 to the secret store.", _console.GetOutput());
_console.ClearOutput();
- secretManager.RunInternal("list", "-p", projectPath);
+ secretManager.RunInternal("list", "-p", projectPath, "--verbose");
Assert.Contains("secret1 = value2", _console.GetOutput());
}
@@ -186,7 +187,7 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests
}
[Fact]
- [Flaky("<no longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public void Remove_Non_Existing_Secret()
{
var projectPath = _fixture.GetTempSecretProject();
@@ -212,7 +213,7 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests
}
[Fact]
- [Flaky("<No longer used; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public void List_Flattens_Nested_Objects()
{
string secretId;
@@ -262,15 +263,16 @@ namespace Microsoft.Extensions.SecretManager.Tools.Tests
}
[Fact]
+ [QuarantinedTest]
public void List_Empty_Secrets_File()
{
var projectPath = _fixture.GetTempSecretProject();
var secretManager = CreateProgram();
- secretManager.RunInternal("list", "-p", projectPath);
+ secretManager.RunInternal("list", "-p", projectPath, "--verbose");
Assert.Contains(Resources.Error_No_Secrets_Found, _console.GetOutput());
}
- [Flaky("<No longer needed, tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
[Theory]
[InlineData(true)]
[InlineData(false)]
diff --git a/src/Tools/dotnet-watch/test/AppWithDepsTests.cs b/src/Tools/dotnet-watch/test/AppWithDepsTests.cs
index 1ec8a59284c..5954f540b59 100644
--- a/src/Tools/dotnet-watch/test/AppWithDepsTests.cs
+++ b/src/Tools/dotnet-watch/test/AppWithDepsTests.cs
@@ -20,7 +20,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task ChangeFileInDependency()
{
await _app.StartWatcherAsync();
diff --git a/src/Tools/dotnet-watch/test/DotNetWatcherTests.cs b/src/Tools/dotnet-watch/test/DotNetWatcherTests.cs
index 3ef636b009e..d07f4e25a46 100644
--- a/src/Tools/dotnet-watch/test/DotNetWatcherTests.cs
+++ b/src/Tools/dotnet-watch/test/DotNetWatcherTests.cs
@@ -23,7 +23,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task RunsWithDotnetWatchEnvVariable()
{
Assert.True(string.IsNullOrEmpty(Environment.GetEnvironmentVariable("DOTNET_WATCH")), "DOTNET_WATCH cannot be set already when this test is running");
@@ -36,7 +36,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task RunsWithIterationEnvVariable()
{
await _app.StartWatcherAsync();
diff --git a/src/Tools/dotnet-watch/test/GlobbingAppTests.cs b/src/Tools/dotnet-watch/test/GlobbingAppTests.cs
index 25dc5b36164..9383b2d728a 100644
--- a/src/Tools/dotnet-watch/test/GlobbingAppTests.cs
+++ b/src/Tools/dotnet-watch/test/GlobbingAppTests.cs
@@ -24,7 +24,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
[Theory]
[InlineData(true)]
[InlineData(false)]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task ChangeCompiledFile(bool usePollingWatcher)
{
_app.UsePollingWatcher = usePollingWatcher;
@@ -43,7 +43,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task DeleteCompiledFile()
{
await _app.StartWatcherAsync();
@@ -60,7 +60,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task DeleteSourceFolder()
{
await _app.StartWatcherAsync();
@@ -77,7 +77,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task RenameCompiledFile()
{
await _app.StartWatcherAsync();
@@ -90,7 +90,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task ChangeExcludedFile()
{
await _app.StartWatcherAsync();
diff --git a/src/Tools/dotnet-watch/test/NoDepsAppTests.cs b/src/Tools/dotnet-watch/test/NoDepsAppTests.cs
index 5b1cc367885..b7171d5ad22 100644
--- a/src/Tools/dotnet-watch/test/NoDepsAppTests.cs
+++ b/src/Tools/dotnet-watch/test/NoDepsAppTests.cs
@@ -24,7 +24,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task RestartProcessOnFileChange()
{
await _app.StartWatcherAsync(new[] { "--no-exit" });
@@ -43,7 +43,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
}
[Fact]
- [Flaky("<No longer needed; tracked in Kusto>", FlakyOn.All)]
+ [QuarantinedTest]
public async Task RestartProcessThatTerminatesAfterFileChange()
{
await _app.StartWatcherAsync();
--
GitLab