diff --git a/.vsts-pipelines/templates/build-steps.yml b/.vsts-pipelines/templates/build-steps.yml
index 2d6544e107d96995a4ec4e72e67214cb9c19f4bc..36219f533b62fa88e7483dbf3a67d26df16032cb 100644
--- a/.vsts-pipelines/templates/build-steps.yml
+++ b/.vsts-pipelines/templates/build-steps.yml
@@ -16,13 +16,6 @@ phases:
artifactName: logs
artifactType: Container
pathtoPublish: artifacts/logs
- - task: PublishBuildArtifacts@1
- displayName: Upload dumps
- condition: eq(variables['system.pullrequest.isfork'], false)
- inputs:
- artifactName: dumps
- artifactType: Container
- pathtoPublish: artifacts/dumps
- template: .vsts-pipelines/templates/phases/default-build.yml@buildtools
parameters:
diff --git a/build/buildpipeline/pipeline.groovy b/build/buildpipeline/pipeline.groovy
index f3be6756b9b18524c6b99d9703e43f7867c99b1a..e1c8b612e9bf84ecc5790bb9b62d1232e36ba612 100644
--- a/build/buildpipeline/pipeline.groovy
+++ b/build/buildpipeline/pipeline.groovy
@@ -1,6 +1,7 @@
import org.dotnet.ci.pipelines.Pipeline
def windowsPipeline = Pipeline.createPipeline(this, 'build/buildpipeline/windows.groovy')
+def windowsAppverifPipeline = Pipeline.createPipeline(this, 'build/buildpipeline/windows-appverif.groovy')
def configurations = [
'Debug',
@@ -16,4 +17,5 @@ configurations.each { configuration ->
windowsPipeline.triggerPipelineOnEveryGithubPR("Windows ${configuration} x64 Build", params)
windowsPipeline.triggerPipelineOnGithubPush(params)
+ windowsAppverifPipeline.triggerPipelineOnEveryGithubPR("Windows AppVerifier ${configuration} x64 Build", params)
}
diff --git a/build/buildpipeline/windows-appverif.groovy b/build/buildpipeline/windows-appverif.groovy
new file mode 100644
index 0000000000000000000000000000000000000000..0c1a6affe815331256485e9cda9596d6e3e0d144
--- /dev/null
+++ b/build/buildpipeline/windows-appverif.groovy
@@ -0,0 +1,15 @@
+@Library('dotnet-ci') _
+
+// 'node' indicates to Jenkins that the enclosed block runs on a node that matches
+// the label 'windows-with-vs'
+simpleNode('Windows.10.Amd64.EnterpriseRS3.ASPNET.Open') {
+ stage ('Checking out source') {
+ checkout scm
+ bat 'git submodule update --init --recursive'
+ }
+ stage ('Build') {
+ def logFolder = getLogFolder()
+ def environment = "\$env:ASPNETCORE_TEST_LOG_DIR='${WORKSPACE}\\${logFolder}'"
+ bat "powershell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command \"${environment};&.\\tools\\SetupTestEnvironment.ps1 Setup;&.\\tools\\update_schema.ps1;&.\\tools\\UpdateIISExpressCertificate.ps1;&.\\run.cmd -CI default-build /p:Configuration=${params.Configuration}\";"
+ }
+}
diff --git a/build/buildpipeline/windows.groovy b/build/buildpipeline/windows.groovy
index eb1a95bca366945b7e202671d1938278746165a0..35bb4737ae1611397d57955ac8bb6afd4f057602 100644
--- a/build/buildpipeline/windows.groovy
+++ b/build/buildpipeline/windows.groovy
@@ -10,6 +10,6 @@ simpleNode('Windows.10.Amd64.EnterpriseRS3.ASPNET.Open') {
stage ('Build') {
def logFolder = getLogFolder()
def environment = "\$env:ASPNETCORE_TEST_LOG_DIR='${WORKSPACE}\\${logFolder}'"
- bat "powershell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command \"&.\\tools\\update_schema.ps1;&.\\tools\\UpdateIISExpressCertificate.ps1;${environment};&.\\run.cmd -CI default-build /p:Configuration=${params.Configuration}\""
+ bat "powershell -NoProfile -NoLogo -ExecutionPolicy unrestricted -Command \"${environment};&.\\tools\\SetupTestEnvironment.ps1 SetupDumps;&.\\tools\\update_schema.ps1;&.\\tools\\UpdateIISExpressCertificate.ps1;&.\\run.cmd -CI default-build /p:Configuration=${params.Configuration}\";"
}
}
diff --git a/test/Common.FunctionalTests/AppOfflineTests.cs b/test/Common.FunctionalTests/AppOfflineTests.cs
index c314a9dfab891e9db43b2cfd4caabed206c6b1d4..7d8d661ef043dc0adf5c3a5c119133c190d1e3a8 100644
--- a/test/Common.FunctionalTests/AppOfflineTests.cs
+++ b/test/Common.FunctionalTests/AppOfflineTests.cs
@@ -102,40 +102,46 @@ namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests
[RequiresIIS(IISCapability.ShutdownToken)]
public async Task AppOfflineDroppedWhileSiteStarting_SiteShutsDown_InProcess()
{
- var deploymentResult = await DeployApp(HostingModel.InProcess);
-
- for (int i = 0; i < 10; i++)
+ // This test often hits a race between debug logging and stdout redirection closing the handle
+ // we are fine having this race
+ using (AppVerifier.Disable(DeployerSelector.ServerType, 0x300))
{
- // send first request and add app_offline while app is starting
- var runningTask = AssertAppOffline(deploymentResult);
-
- // This test tries to hit a race where we drop app_offline file while
- // in process application is starting, application start takes at least 400ms
- // so we back off for 100ms to allow request to reach request handler
- // Test itself is racy and can result in two scenarios
- // 1. ANCM detects app_offline before it starts the request - if AssertAppOffline succeeds we've hit it
- // 2. Intended scenario where app starts and then shuts down
- // In first case we remove app_offline and try again
- await Task.Delay(RetryDelay);
-
- AddAppOffline(deploymentResult.ContentRoot);
-
- try
- {
- await runningTask.DefaultTimeout();
+ var deploymentResult = await DeployApp(HostingModel.InProcess);
- // if AssertAppOffline succeeded ANCM have picked up app_offline before starting the app
- // try again
- RemoveAppOffline(deploymentResult.ContentRoot);
- }
- catch
+ for (int i = 0; i < 10; i++)
{
- deploymentResult.AssertWorkerProcessStop();
- return;
+ // send first request and add app_offline while app is starting
+ var runningTask = AssertAppOffline(deploymentResult);
+
+ // This test tries to hit a race where we drop app_offline file while
+ // in process application is starting, application start takes at least 400ms
+ // so we back off for 100ms to allow request to reach request handler
+ // Test itself is racy and can result in two scenarios
+ // 1. ANCM detects app_offline before it starts the request - if AssertAppOffline succeeds we've hit it
+ // 2. Intended scenario where app starts and then shuts down
+ // In first case we remove app_offline and try again
+ await Task.Delay(RetryDelay);
+
+ AddAppOffline(deploymentResult.ContentRoot);
+
+ try
+ {
+ await runningTask.DefaultTimeout();
+
+ // if AssertAppOffline succeeded ANCM have picked up app_offline before starting the app
+ // try again
+ RemoveAppOffline(deploymentResult.ContentRoot);
+ }
+ catch
+ {
+ deploymentResult.AssertWorkerProcessStop();
+ return;
+ }
}
- }
- Assert.True(false);
+ Assert.True(false);
+
+ }
}
[ConditionalFact]
diff --git a/test/Common.FunctionalTests/Utilities/AppVerifier.cs b/test/Common.FunctionalTests/Utilities/AppVerifier.cs
new file mode 100644
index 0000000000000000000000000000000000000000..ab059d789dd8c3976a980fb3c9ef824da143e7f9
--- /dev/null
+++ b/test/Common.FunctionalTests/Utilities/AppVerifier.cs
@@ -0,0 +1,83 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Diagnostics;
+using System.Linq;
+
+namespace Microsoft.AspNetCore.Server.IntegrationTesting
+{
+ public class AppVerifier
+ {
+ private static readonly TimeSpan AppVerifierCommandTimeout = TimeSpan.FromSeconds(5);
+
+ public static IDisposable Disable(ServerType serverType, int code)
+ {
+ // Set in SetupTestEnvironment.ps1
+ var enabledCodes = (Environment.GetEnvironmentVariable("APPVERIFIER_ENABLED_CODES") ?? "").Split(' ');
+ string processName;
+ switch (serverType)
+ {
+ case ServerType.IISExpress:
+ processName = "iisexpress.exe";
+ break;
+ case ServerType.IIS:
+ processName = "w3wp.exe";
+ break;
+ default:
+ throw new ArgumentOutOfRangeException(nameof(serverType), serverType, null);
+ }
+
+ if (!enabledCodes.Contains(code.ToString()))
+ {
+ return null;
+ }
+
+ RunProcessAndWaitForExit("appverif.exe", $"-configure {code} -for {processName} -with ErrorReport=0", AppVerifierCommandTimeout);
+ return new AppVerifierToken(processName, code.ToString());
+ }
+
+ private static void RunProcessAndWaitForExit(string fileName, string arguments, TimeSpan timeout)
+ {
+ var startInfo = new ProcessStartInfo
+ {
+ FileName = fileName,
+ Arguments = arguments,
+ RedirectStandardOutput = true,
+ RedirectStandardError = true,
+ UseShellExecute = false,
+ };
+
+ var process = Process.Start(startInfo);
+
+ if (!process.WaitForExit((int)timeout.TotalMilliseconds))
+ {
+ process.Kill();
+ }
+
+ if (process.ExitCode != 0)
+ {
+ throw new InvalidOperationException($"Exit code {process.ExitCode} when running {fileName} {arguments}. Stdout: {process.StandardOutput.ReadToEnd()}");
+ }
+ }
+
+ public class AppVerifierToken : IDisposable
+ {
+ private readonly string _processName;
+
+ private readonly string _codes;
+
+ public AppVerifierToken(string processName, string codes)
+ {
+ _processName = processName;
+ _codes = codes;
+ }
+
+ public void Dispose()
+ {
+ //
+ RunProcessAndWaitForExit("appverif.exe", $"-configure {_codes} -for {_processName} -with ErrorReport={Environment.GetEnvironmentVariable("APPVERIFIER_LEVEL")}", AppVerifierCommandTimeout);
+ }
+ }
+ }
+}
diff --git a/test/IIS.FunctionalTests/ServicesTests.cs b/test/IIS.FunctionalTests/ServicesTests.cs
index b91fc8aa20cb4fb26ddc4c2f4087bb3cab0bd223..47e53b8d404b5a2770fb484554b52be9e0c3c080 100644
--- a/test/IIS.FunctionalTests/ServicesTests.cs
+++ b/test/IIS.FunctionalTests/ServicesTests.cs
@@ -32,15 +32,20 @@ namespace IIS.FunctionalTests
[InlineData(HostingModel.OutOfProcess)]
public async Task ApplicationPreloadStartsApp(HostingModel hostingModel)
{
- var baseDeploymentParameters = _fixture.GetBaseDeploymentParameters(hostingModel, publish: true);
- baseDeploymentParameters.TransformArguments((args, contentRoot)=> $"{args} CreateFile \"{Path.Combine(contentRoot, "Started.txt")}\"");
- EnablePreload(baseDeploymentParameters);
+ // This test often hits a memory leak in warmup.dll module, it has been reported to IIS team
+ using (AppVerifier.Disable(DeployerSelector.ServerType, 0x900))
+ {
+ var baseDeploymentParameters = _fixture.GetBaseDeploymentParameters(hostingModel, publish: true);
+ baseDeploymentParameters.TransformArguments(
+ (args, contentRoot) => $"{args} CreateFile \"{Path.Combine(contentRoot, "Started.txt")}\"");
+ EnablePreload(baseDeploymentParameters);
- var result = await DeployAsync(baseDeploymentParameters);
+ var result = await DeployAsync(baseDeploymentParameters);
- await Helpers.Retry(async () => await File.ReadAllTextAsync(Path.Combine(result.ContentRoot, "Started.txt")), 10, 200);
- StopServer();
- EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.Started(result));
+ await Helpers.Retry(async () => await File.ReadAllTextAsync(Path.Combine(result.ContentRoot, "Started.txt")), 10, 200);
+ StopServer();
+ EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.Started(result));
+ }
}
[ConditionalTheory]
@@ -49,22 +54,26 @@ namespace IIS.FunctionalTests
[InlineData(HostingModel.OutOfProcess)]
public async Task ApplicationInitializationPageIsRequested(HostingModel hostingModel)
{
- var baseDeploymentParameters = _fixture.GetBaseDeploymentParameters(hostingModel, publish: true);
- EnablePreload(baseDeploymentParameters);
+ // This test often hits a memory leak in warmup.dll module, it has been reported to IIS team
+ using (AppVerifier.Disable(DeployerSelector.ServerType, 0x900))
+ {
+ var baseDeploymentParameters = _fixture.GetBaseDeploymentParameters(hostingModel, publish: true);
+ EnablePreload(baseDeploymentParameters);
- baseDeploymentParameters.ServerConfigActionList.Add(
- (config, _) => {
- config
- .RequiredElement("system.webServer")
- .GetOrAdd("applicationInitialization")
- .GetOrAdd("add", "initializationPage", "/CreateFile");
- });
+ baseDeploymentParameters.ServerConfigActionList.Add(
+ (config, _) => {
+ config
+ .RequiredElement("system.webServer")
+ .GetOrAdd("applicationInitialization")
+ .GetOrAdd("add", "initializationPage", "/CreateFile");
+ });
- var result = await DeployAsync(baseDeploymentParameters);
+ var result = await DeployAsync(baseDeploymentParameters);
- await Helpers.Retry(async () => await File.ReadAllTextAsync(Path.Combine(result.ContentRoot, "Started.txt")), 10, 200);
- StopServer();
- EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.Started(result));
+ await Helpers.Retry(async () => await File.ReadAllTextAsync(Path.Combine(result.ContentRoot, "Started.txt")), 10, 200);
+ StopServer();
+ EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.Started(result));
+ }
}
private static void EnablePreload(IISDeploymentParameters baseDeploymentParameters)
diff --git a/tools/SetupTestEnvironment.ps1 b/tools/SetupTestEnvironment.ps1
index dcdaa84f57fbc28a12ad8cc7c587bcc7b16172ab..625b040e39f2daa0d7e0f5457b2f2b7f9a02abd1 100644
--- a/tools/SetupTestEnvironment.ps1
+++ b/tools/SetupTestEnvironment.ps1
@@ -1,14 +1,36 @@
param($Mode)
+$DumpFolder = "$env:ASPNETCORE_TEST_LOG_DIR\dumps"
+if (!($DumpFolder))
+{
+ $DumpFolder = "$PSScriptRoot\..\artifacts\dumps"
+}
+if (!(Test-Path $DumpFolder))
+{
+ New-Item $DumpFolder -ItemType Directory;
+}
+$DumpFolder = Resolve-Path $DumpFolder
+
+$LogsFolder = "$PSScriptRoot\..\artifacts\logs"
+if (!(Test-Path $LogsFolder))
+{
+ New-Item $LogsFolder -ItemType Directory;
+}
+$LogsFolder = Resolve-Path $LogsFolder
+
+$werHive = "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting";
+$ldHive = "$werHive\LocalDumps";
+
+
function Setup-appverif($application)
{
appverif.exe -enable Exceptions Handles Heaps Leak Locks Memory Threadpool TLS SRWLock -for $application
- $onlyLog = 0x1E1;
+ $level = 0x1E1;
$codes = @(
# Exceptions
0x650,
# Handles
- 0x300, 0x301, 0x302, 0x303, 0x304, 0x305,
+ 0x300, 0x301, 0x302, 0x303, 0x304, # 0x305,
# Heaps
0x001, 0x002, 0x003, 0x004, 0x005, 0x006, 0x007, 0x008, 0x009, 0x00A, 0x00B, 0x00C, 0x00D, 0x00E, 0x00F, 0x010, 0x011, 0x012, 0x013, 0x014,
# Leak
@@ -24,54 +46,32 @@ function Setup-appverif($application)
# ThreadPool
0x700, 0x701, 0x702, 0x703, 0x704, 0x705, 0x706, 0x707, 0x708, 0x709, 0x70A, 0x70B, 0x70C, 0x70D
);
- appverif.exe -configure $codes -for $application -with ErrorReport=$onlyLog
-}
-function Shutdown-appverif($application)
-{
- appverif.exe -export log -for $application -with To=$LogsFolder\$application.xml Log=0
- appverif.exe -disable * -for $application
-}
+ setx APPVERIFIER_ENABLED_CODES "$codes";
+ setx APPVERIFIER_LEVEL $level;
+ appverif.exe -configure $codes -for $application -with ErrorReport=$level
-$DumpFolder = "$PSScriptRoot\..\artifacts\dumps"
-if (!(Test-Path $DumpFolder))
-{
- New-Item $DumpFolder -ItemType Directory;
+ # 0x305, - disabled because coreclr.dll!SetThreadName(void *) ofthen passes invalid handle (0xffffff)
+ appverif.exe -configure 0x305 -for $application -with ErrorReport=0
}
-$DumpFolder = Resolve-Path $DumpFolder
-$LogsFolder = "$PSScriptRoot\..\artifacts\logs"
-if (!(Test-Path $LogsFolder))
+function Shutdown-appverif($application)
{
- New-Item $LogsFolder -ItemType Directory;
-}
-$LogsFolder = Resolve-Path $LogsFolder
-
-$werHive = "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting";
-$ldHive = "$werHive\LocalDumps";
+ setx APPVERIFIER_ENABLED_CODES "`"`"";
+ setx APPVERIFIER_LEVEL "`"`"";
-
-$cdb = "c:\Program Files (x86)\Windows Kits\10\Debuggers\x64\cdb.exe"
-if (!(Test-Path $cdb))
-{
- $downloadedFile = [System.IO.Path]::GetTempFileName();
- $downloadedFile = "$downloadedFile.exe";
- Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?linkid=870807" -OutFile $downloadedFile;
- & $downloadedFile /features OptionId.WindowsDesktopDebuggers /norestart /q;
+ appverif.exe -disable * -for $application
}
-if ($Mode -eq "Setup")
+function Setup-Dumps()
{
- Move-Item $env:windir\System32\vsjitdebugger.exe $env:windir\System32\_vsjitdebugger.exe;
-
- Setup-appverif w3wp.exe
- Setup-appverif iisexpress.exe
-
if (!(Test-Path $ldHive ))
{
New-Item -Path $werHive -Name LocalDumps
}
+ Move-Item $env:windir\System32\vsjitdebugger.exe $env:windir\System32\_vsjitdebugger.exe;
+
New-ItemProperty $werHive -Name "DontShowUI" -Value 1 -PropertyType "DWORD" -Force;
New-ItemProperty $ldHive -Name "DumpFolder" -Value $DumpFolder -PropertyType "ExpandString" -Force;
@@ -81,7 +81,7 @@ if ($Mode -eq "Setup")
Restart-Service WerSvc
}
-if ($Mode -eq "Shutdown")
+function Shutdown-Dumps()
{
Move-Item $env:windir\System32\_vsjitdebugger.exe $env:windir\System32\vsjitdebugger.exe;
@@ -89,8 +89,14 @@ if ($Mode -eq "Shutdown")
New-ItemProperty $werHive -Name "DontShowUI" -Value 0 -PropertyType "DWORD" -Force;
- Shutdown-appverif w3wp.exe
- Shutdown-appverif iisexpress.exe
+ $cdb = "c:\Program Files (x86)\Windows Kits\10\Debuggers\x64\cdb.exe"
+ if (!(Test-Path $cdb))
+ {
+ $downloadedFile = [System.IO.Path]::GetTempFileName();
+ $downloadedFile = "$downloadedFile.exe";
+ Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?linkid=870807" -OutFile $downloadedFile;
+ & $downloadedFile /features OptionId.WindowsDesktopDebuggers /norestart /q;
+ }
foreach ($dump in (Get-ChildItem -Path $DumpFolder -Filter "*.dmp"))
{
@@ -102,4 +108,28 @@ if ($Mode -eq "Shutdown")
}
}
+if ($Mode -eq "Setup")
+{
+ Setup-appverif w3wp.exe
+ Setup-appverif iisexpress.exe
+
+ Setup-Dumps;
+}
+
+if ($Mode -eq "SetupDumps")
+{
+ Shutdown-appverif w3wp.exe
+ Shutdown-appverif iisexpress.exe
+
+ Setup-Dumps;
+}
+
+if ($Mode -eq "Shutdown")
+{
+ Shutdown-appverif w3wp.exe
+ Shutdown-appverif iisexpress.exe
+
+ Shutdown-Dumps;
+}
+
Exit 0;
\ No newline at end of file