[Helix] Split flaky helix tests into separate yml (#18597)

.azure/pipelines/helix-test.yml

+# Don't run CI for this config yet. We're not ready to move official builds on to Azure Pipelines
+trigger: none
+
+# Run PR validation on all branches
+pr:
+  branches:
+    include:
+    - '*'
+
+jobs:
+- template: jobs/default-build.yml
+  parameters:
+    jobName: Helix_quarantine_x64
+    jobDisplayName: 'Tests: Helix Quarantine x64'
+    agentOs: Windows
+    timeoutInMinutes: 240
+    steps:
+    - script: .\restore.cmd -ci
+      displayName: Restore
+    - script: .\build.cmd -ci -NoRestore -test -projects eng\helix\helix.proj /p:RunQuarantinedTests=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildAllProjects=true /p:BuildNative=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl
+      displayName: Run build.cmd helix target
+      env:
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops
+    artifacts:
+    - name: Helix_logs
+      path: artifacts/log/
+      publishOnError: true

eng/helix/content/runtests.cmd

@@ -8,6 +8,7 @@ set sdkVersion=%3
 set runtimeVersion=%4
 set helixQueue=%5
 set arch=%6
+set quarantined=%7
 
 set DOTNET_HOME=%HELIX_CORRELATION_PAYLOAD%\sdk
 set DOTNET_ROOT=%DOTNET_HOME%\%arch%
@@ -38,25 +39,25 @@ if not errorlevel 1 (
 
 set exit_code=0
 
-REM Run non-flaky tests first
-REM We need to specify all possible Flaky filters that apply to this environment, because the flaky attribute
-REM only puts the explicit filter traits the user provided in
-REM Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md
-set NONFLAKY_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:%HELIX%!=true"
-echo Running non-flaky tests.
-%DOTNET_ROOT%\dotnet vstest %target% --logger:xunit --TestCaseFilter:%NONFLAKY_FILTER%
-if errorlevel 1 (
-    echo Failure in non-flaky test 1>&2
-    set exit_code=1
-    REM DO NOT EXIT
-)
-
-set FLAKY_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:%HELIX%=true"
-echo Running known-flaky tests.
-%DOTNET_ROOT%\dotnet vstest %target% --TestCaseFilter:%FLAKY_FILTER%
-if errorlevel 1 (
-    echo Failure in flaky test 1>&2
-    REM DO NOT EXIT and DO NOT SET EXIT_CODE to 1
+set NONQUARANTINE_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:%HELIX%!=true"
+if (%quarantined%==true) (
+    echo Running all tests.
+    %DOTNET_ROOT%\dotnet vstest %target% --logger:xunit
+    if errorlevel 1 (
+        echo Failure in flaky test 1>&2
+        REM DO NOT EXIT and DO NOT SET EXIT_CODE to 1
+    )
+) else (
+    REM We need to specify all possible Flaky filters that apply to this environment, because the flaky attribute
+    REM only puts the explicit filter traits the user provided in
+    REM Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md
+    echo Running non-quarantined tests.
+    %DOTNET_ROOT%\dotnet vstest %target% --logger:xunit --TestCaseFilter:%NONQUARANTINE_FILTER%
+    if errorlevel 1 (
+        echo Failure in non-flaky test 1>&2
+        set exit_code=1
+        REM DO NOT EXIT
+    )
 )
 
 echo "Copying TestResults\TestResults.xml to ."

eng/helix/content/runtests.sh

@@ -4,6 +4,8 @@ test_binary_path="$1"
 dotnet_sdk_version="$2"
 dotnet_runtime_version="$3"
 helix_queue_name="$4"
+target_arch="$5"
+quarantined="$6"
 
 RESET="\033[0m"
 RED="\033[0;31m"
@@ -87,25 +89,27 @@ if grep -q "Exception thrown" discovered.txt; then
     exit 1
 fi
 
-# Run non-flaky tests first
-# We need to specify all possible Flaky filters that apply to this environment, because the flaky attribute
+exit_code=0
+
+# We need to specify all possible quarantined filters that apply to this environment, because the quarantine attribute
 # only puts the explicit filter traits the user provided in the flaky attribute
 # Filter syntax: https://github.com/Microsoft/vstest-docs/blob/master/docs/filter.md
-NONFLAKY_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:$helix_queue_name!=true"
-echo "Running non-flaky tests."
-$DOTNET_ROOT/dotnet vstest $test_binary_path --logger:xunit --TestCaseFilter:"$NONFLAKY_FILTER"
-nonflaky_exitcode=$?
-if [ $nonflaky_exitcode != 0 ]; then
-    echo "Non-flaky tests failed!" 1>&2
-    # DO NOT EXIT
-fi
-
-FLAKY_FILTER="Flaky:All=true|Flaky:Helix:All=true|Flaky:Helix:Queue:All=true|Flaky:Helix:Queue:$helix_queue_name=true"
-echo "Running known-flaky tests."
-$DOTNET_ROOT/dotnet vstest $test_binary_path --TestCaseFilter:"$FLAKY_FILTER"
-if [ $? != 0 ]; then
-    echo "Flaky tests failed!" 1>&2
-    # DO NOT EXIT
+NONQUARANTINE_FILTER="Flaky:All!=true&Flaky:Helix:All!=true&Flaky:Helix:Queue:All!=true&Flaky:Helix:Queue:$helix_queue_name!=true"
+if [ "$quarantined" == true ]; then
+    echo "Running all tests including quarantined."
+    $DOTNET_ROOT/dotnet vstest $test_binary_path --logger:xunit
+    if [ $? != 0 ]; then
+        echo "Quarantined tests failed!" 1>&2
+        # DO NOT EXIT
+    fi
+else
+    echo "Running non-quarantined tests."
+    $DOTNET_ROOT/dotnet vstest $test_binary_path --logger:xunit --TestCaseFilter:"$NONQUARANTINE_FILTER"
+    exit_code=$?
+    if [ $exit_code != 0 ]; then
+        echo "Non-quarantined tests failed!" 1>&2
+        # DO NOT EXIT
+    fi
 fi
 
 echo "Copying TestResults/TestResults to ."
@@ -114,4 +118,4 @@ echo "Copying artifacts/logs to $HELIX_WORKITEM_UPLOAD_ROOT/../"
 shopt -s globstar
 cp artifacts/log/**/*.log $HELIX_WORKITEM_UPLOAD_ROOT/../
 cp artifacts/log/**/*.log $HELIX_WORKITEM_UPLOAD_ROOT/
-exit $nonflaky_exitcode
+exit $exit_code

eng/targets/Helix.props

@@ -12,9 +12,12 @@
   <PropertyGroup>
     <CreateHelixPayload>true</CreateHelixPayload>
     <HelixTimeout>00:30:00</HelixTimeout>
+    <RunQuarantinedTests>false</RunQuarantinedTests>
     <IsWindowsHelixQueue>false</IsWindowsHelixQueue>
     <IsWindowsHelixQueue Condition="$(HelixTargetQueue.Contains('Windows')) or $(HelixTargetQueue.Contains('windows'))">true</IsWindowsHelixQueue>
-    <HelixTestName>$(MSBuildProjectName)-$(TargetFramework)</HelixTestName>
+    <_HelixRunType Condition="'$(RunQuarantinedTests)' == 'true'">quarantined</_HelixRunType>
+    <_HelixRunType Condition="'$(RunQuarantinedTests)' != 'true'">unquarantined</_HelixRunType>    
+    <HelixTestName>$(MSBuildProjectName)--$(TargetFramework)--$(_HelixRunType)</HelixTestName>
     <HelixUseArchive>false</HelixUseArchive>
     <LoggingTestingDisableFileLogging Condition="'$(IsHelixJob)' == 'true'">false</LoggingTestingDisableFileLogging>
     <NodeVersion>10.15.3</NodeVersion>

eng/targets/Helix.targets

@@ -83,8 +83,8 @@ Usage: dotnet msbuild /t:Helix src/MyTestProject.csproj
         <TestAssembly>$(TargetFileName)</TestAssembly>
         <PreCommands>@(HelixPreCommand)</PreCommands>
         <PostCommands>@(HelixPostCommand)</PostCommands>
-        <Command Condition="$(IsWindowsHelixQueue)">call runtests.cmd $(TargetFileName) $(TargetFrameworkIdentifier) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppRuntimeVersion) $(_HelixFriendlyNameTargetQueue) $(TargetArchitecture)</Command>
-        <Command Condition="!$(IsWindowsHelixQueue)">./runtests.sh $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppRuntimeVersion) $(_HelixFriendlyNameTargetQueue) $(TargetArchitecture)</Command>
+        <Command Condition="$(IsWindowsHelixQueue)">call runtests.cmd $(TargetFileName) $(TargetFrameworkIdentifier) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppRuntimeVersion) $(_HelixFriendlyNameTargetQueue) $(TargetArchitecture) $(RunQuarantinedTests)</Command>
+        <Command Condition="!$(IsWindowsHelixQueue)">./runtests.sh $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppRuntimeVersion) $(_HelixFriendlyNameTargetQueue) $(TargetArchitecture) $(RunQuarantinedTests)</Command>
         <Timeout>$(HelixTimeout)</Timeout>
       </HelixWorkItem>
     </ItemGroup>