ci.yml

#
# See https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema for details on this file.
#

# Configure which branches trigger builds
trigger:
  batch: true
  branches:
    include:
    - master
    - release/*

# Run PR validation on all branches
pr:
  autoCancel: true
  branches:
    include:
    - '*'

variables:
- name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE
  value: true
- name: _TeamName
  value:  AspNetCore
- name: _DotNetPublishToBlobFeed
  value: true
- name: _PublishUsingPipelines
  value: true
- name: _DotNetArtifactsCategory
  value: .NETCORE
- name: _DotNetValidationArtifactsCategory
  value: .NETCORE
- ${{ if ne(variables['System.TeamProject'], 'internal') }}:
  - name: _BuildArgs
    value: ''
  - name: _PublishArgs
    value: ''
  - name: _SignType
    value: ''
- ${{ if eq(variables['System.TeamProject'], 'internal') }}:
  - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
    # DotNet-Blob-Feed provides: dotnetfeed-storage-access-key-1
    # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT
    - group: DotNet-Blob-Feed
    - group: Publish-Build-Assets

    - name: _BuildArgs
      value: /p:TeamName=$(_TeamName)
             /p:OfficialBuildId=$(Build.BuildNumber)
    - name: _SignType
      value: real

    # The following extra properties are not set when testing. Use with final build.[cmd,sh] of asset-producing jobs.
    - name: _PublishArgs
      value: /p:Publish=true
             /p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1)
             /p:DotNetPublishBlobFeedUrl=https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
             /p:DotNetPublishToBlobFeed=$(_DotNetPublishToBlobFeed)
             /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines)
             /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory)
  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
    - name: _BuildArgs
      value: ''
    - name: _SignType
      valule: test
    - name: _PublishArgs
      value: ''

stages:
- stage: build
  displayName: Build
  jobs:
  # Code check
  - template: jobs/default-build.yml
    parameters:
      jobName: Code_check
      jobDisplayName: Code check
      agentOs: Windows
      steps:
      - powershell: ./eng/scripts/CodeCheck.ps1 -ci
        displayName: Run eng/scripts/CodeCheck.ps1
      artifacts:
      - name: Code_Check_Logs
        path: artifacts/log/
        publishOnError: true

  # Build Windows (x64/x86)
  - template: jobs/default-build.yml
    parameters:
      codeSign: true
      jobName: Windows_build
      jobDisplayName: "Build: Windows x64/x86"
      agentOs: Windows
      steps:
      - script: "echo ##vso[build.addbuildtag]daily-build"
        condition: and(ne(variables['Build.Reason'], 'PullRequest'), notin(variables['DotNetFinalVersionKind'], 'release', 'prerelease'))
        displayName: 'Set CI tags'
      - script: "echo ##vso[build.addbuildtag]release-candidate"
        condition: and(ne(variables['Build.Reason'], 'PullRequest'), in(variables['DotNetFinalVersionKind'], 'release', 'prerelease'))
        displayName: 'Set CI tags'

      # !!! NOTE !!! Some of these steps have disabled code signing.
      # This is intentional to workaround https://github.com/dotnet/arcade/issues/1957 which always re-submits for code-signing, even
      # if they have already been signed. This results in slower builds due to re-submitting the same .nupkg many times for signing.
      # The sign settings have been configured to

      - script: ./build.cmd
                -ci
                -arch x64
                -pack
                -all
                -buildNative
                /bl:artifacts/log/build.x64.binlog
                $(_BuildArgs)
        displayName: Build x64

      # Build the x86 shared framework
      # TODO: make it possible to build for one Windows architecture at a time
      # This is going to actually build x86 native assets. See https://github.com/aspnet/AspNetCore/issues/7196
      - script: ./build.cmd
                -ci
                -arch x86
                -pack
                -all
                -buildNative
                -noBuildJava
                /p:OnlyPackPlatformSpecificPackages=true
                /bl:artifacts/log/build.x86.binlog
                $(_BuildArgs)
        displayName: Build x86

      # This is in a separate build step with -forceCoreMsbuild to workaround MAX_PATH limitations - https://github.com/Microsoft/msbuild/issues/53
      - script: .\src\SiteExtensions\build.cmd
                -ci
                -pack
                -noBuildDeps
                $(_BuildArgs)
        displayName: Build SiteExtension

      # This runs code-signing on all packages, zips, and jar files as defined in build/CodeSign.targets. If https://github.com/dotnet/arcade/issues/1957 is resolved,
      # consider running code-signing inline with the other previous steps.
      # Sign check is disabled because it is run in a separate step below, after installers are built.
      - script: ./build.cmd
                -ci
                -noBuild
                -noRestore
                -sign
                /bl:artifacts/log/build.codesign.binlog
                /p:DotNetSignType=$(_SignType)
                $(_BuildArgs)
        displayName: Code sign packages

      # Windows installers bundle both x86 and x64 assets
      - script: ./build.cmd
                -ci
                -sign
                -buildInstallers
                /bl:artifacts/log/installers.msbuild.binlog
                /p:DotNetSignType=$(_SignType)
                /p:AssetManifestFileName=aspnetcore-win-x64-x86.xml
                $(_BuildArgs)
                $(_PublishArgs)
                /p:PublishInstallerBaseVersion=true
        displayName: Build Installers

      artifacts:
      - name: Windows_Logs
        path: artifacts/log/
        publishOnError: true
      - name: Windows_Packages
        path: artifacts/packages/

  # Build Windows ARM
  - template: jobs/default-build.yml
    parameters:
      codeSign: true
      jobName: Windows_arm_build
      jobDisplayName: "Build: Windows ARM"
      agentOs: Windows
      buildArgs:
        -arch arm
        -sign
        -pack
        -noBuildNodeJS
        -noBuildJava
        /bl:artifacts/log/build.win-arm.binlog
        /p:DotNetSignType=$(_SignType)
        /p:OnlyPackPlatformSpecificPackages=true
        /p:AssetManifestFileName=aspnetcore-win-arm.xml
        $(_BuildArgs)
        $(_PublishArgs)
      installNodeJs: false
      installJdk: false
      artifacts:
      - name: Windows_arm_Logs
        path: artifacts/log/
        publishOnError: true
      - name: Windows_arm_Packages
        path: artifacts/packages/

  # Build MacOS
  - template: jobs/default-build.yml
    parameters:
      jobName: MacOs_x64_build
      jobDisplayName: "Build: macOS"
      agentOs: macOs
      buildArgs:
        --pack
        --all
        --no-build-nodejs
        --no-build-java
        -p:OnlyPackPlatformSpecificPackages=true
        -bl:artifacts/log/build.macos.binlog
        -p:AssetManifestFileName=aspnetcore-MacOS_x64.xml
        $(_BuildArgs)
        $(_PublishArgs)
      installNodeJs: false
      installJdk: false
      artifacts:
      - name: MacOS_x64_Logs
        path: artifacts/log/
        publishOnError: true
      - name: MacOS_x64_Packages
        path: artifacts/packages/
  - template: jobs/codesign-xplat.yml
    parameters:
      inputName: MacOS_x64

  # Build Linux x64
  - template: jobs/default-build.yml
    parameters:
      jobName: Linux_x64_build
      jobDisplayName: "Build: Linux x64"
      agentOs: Linux
      steps:
      - script: ./build.sh
            --ci
            --arch x64
            --pack
            --all
            --no-build-nodejs
            --no-build-java
            -p:OnlyPackPlatformSpecificPackages=true
            -bl:artifacts/log/build.linux-x64.binlog
            $(_BuildArgs)
        displayName: Run build.sh
      - script: |
          git clean -xfd src/**/obj/
          ./dockerbuild.sh bionic \
            --ci \
            --arch x64 \
            --build-installers \
            --no-build-deps \
            --no-build-nodejs \
            -p:OnlyPackPlatformSpecificPackages=true \
            -p:BuildRuntimeArchive=false \
            -p:LinuxInstallerType=deb \
            -bl:artifacts/log/build.deb.binlog \
            $(_BuildArgs)
        displayName: Build Debian installers
      - script: |
          git clean -xfd src/**/obj/
          ./dockerbuild.sh rhel \
            --ci \
            --arch x64 \
            --build-installers \
            --no-build-deps \
            --no-build-nodejs \
            -p:OnlyPackPlatformSpecificPackages=true \
            -p:BuildRuntimeArchive=false \
            -p:LinuxInstallerType=rpm \
            -bl:artifacts/log/build.rpm.binlog \
            -p:AssetManifestFileName=aspnetcore-Linux_x64.xml \
            $(_BuildArgs) \
            $(_PublishArgs)
        displayName: Build RPM installers
      installNodeJs: false
      installJdk: false
      artifacts:
      - name: Linux_x64_Logs
        path: artifacts/log/
        publishOnError: true
      - name: Linux_x64_Packages
        path: artifacts/packages/
  - template: jobs/codesign-xplat.yml
    parameters:
      inputName: Linux_x64

  # Build Linux ARM
  - template: jobs/default-build.yml
    parameters:
      jobName: Linux_arm_build
      jobDisplayName: "Build: Linux ARM"
      agentOs: Linux
      buildArgs:
        --arch arm
        --pack
        --all
        --no-build-nodejs
        --no-build-java
        -p:OnlyPackPlatformSpecificPackages=true
        -bl:artifacts/log/build.linux-arm.binlog
        -p:AssetManifestFileName=aspnetcore-Linux_arm.xml
        $(_BuildArgs)
        $(_PublishArgs)
      installNodeJs: false
      installJdk: false
      artifacts:
      - name: Linux_arm_Logs
        path: artifacts/log/
        publishOnError: true
      - name: Linux_arm_Packages
        path: artifacts/packages/
  - template: jobs/codesign-xplat.yml
    parameters:
      inputName: Linux_arm

  # Build Linux ARM64
  - template: jobs/default-build.yml
    parameters:
      jobName: Linux_arm64_build
      jobDisplayName: "Build: Linux ARM64"
      agentOs: Linux
      buildArgs:
        --arch arm64
        --all
        --pack
        --no-build-nodejs
        --no-build-java
        -p:OnlyPackPlatformSpecificPackages=true
        -bl:artifacts/log/build.arm64.binlog
        -p:AssetManifestFileName=aspnetcore-Linux_arm64.xml
        $(_BuildArgs)
        $(_PublishArgs)
      installNodeJs: false
      installJdk: false
      artifacts:
      - name: Linux_arm64_Logs
        path: artifacts/log/
        publishOnError: true
      - name: Linux_arm64_Packages
        path: artifacts/packages/
  - template: jobs/codesign-xplat.yml
    parameters:
      inputName: Linux_arm64

  # Build Linux Musl x64
  - template: jobs/default-build.yml
    parameters:
      jobName: Linux_musl_x64_build
      jobDisplayName: "Build: Linux Musl x64"
      agentOs: Linux
      buildScript: ./dockerbuild.sh alpine
      buildArgs:
        --ci
        --arch x64
        --os-name linux-musl
        --pack
        --all
        --no-build-nodejs
        --no-build-java
        -p:OnlyPackPlatformSpecificPackages=true
        -bl:artifacts/log/build.musl.binlog
        -p:AssetManifestFileName=aspnetcore-Linux_musl_x64.xml
        $(_BuildArgs)
        $(_PublishArgs)
      installNodeJs: false
      installJdk: false
      artifacts:
      - name: Linux_musl_x64_Logs
        path: artifacts/log/
        publishOnError: true
      - name: Linux_musl_x64_Packages
        path: artifacts/packages/
  - template: jobs/codesign-xplat.yml
    parameters:
      inputName: Linux_musl_x64

  # Build Linux Musl ARM64
  - template: jobs/default-build.yml
    parameters:
      jobName: Linux_musl_arm64_build
      jobDisplayName: "Build: Linux Musl ARM64"
      agentOs: Linux
      buildScript: ./dockerbuild.sh ubuntu-alpine37
      buildArgs:
        --ci
        --arch arm64
        --os-name linux-musl
        --pack
        --all
        --no-build-nodejs
        --no-build-java
        -p:OnlyPackPlatformSpecificPackages=true
        -bl:artifacts/log/build.musl.binlog
        -p:AssetManifestFileName=aspnetcore-Linux_musl_arm64.xml
        $(_BuildArgs)
        $(_PublishArgs)
      installNodeJs: false
      installJdk: false
      artifacts:
      - name: Linux_musl_arm64_Logs
        path: artifacts/log/
        publishOnError: true
      - name: Linux_musl_arm64_Packages
        path: artifacts/packages/
  - template: jobs/codesign-xplat.yml
    parameters:
      inputName: Linux_musl_arm64

  # Test jobs
  - template: jobs/default-build.yml
    parameters:
      condition: ne(variables['SkipTests'], 'true')
      jobName: Windows_Test
      jobDisplayName: "Test: Windows Server 2016 x64"
      agentOs: Windows
      isTestingJob: true
      buildArgs: -all -pack -test -BuildNative "/p:SkipIISNewHandlerTests=true /p:SkipIISTests=true /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false"
      beforeBuild:
      - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1"
        displayName: Setup IISExpress test certificates and schema
      afterBuild:
      - powershell: "& ./build.ps1 -CI -NoBuild -Test /p:RunFlakyTests=true"
        displayName: Run Flaky Tests
        continueOnError: true
      artifacts:
      - name: Windows_Test_Logs
        path: artifacts/log/
        publishOnError: true
      - name: Windows_Test_Results
        path: artifacts/TestResults/
        publishOnError: true

  - template: jobs/default-build.yml
    parameters:
      condition: ne(variables['SkipTests'], 'true')
      jobName: Windows_Templates_Test
      jobDisplayName: "Test: Templates - Windows Server 2016 x64"
      agentOs: Windows
      isTestingJob: true
      steps:
      - script: ./build.cmd -ci -all -pack
        displayName: Build Repo
      - script: ./src/ProjectTemplates/build.cmd -ci -pack -NoRestore -NoBuilddeps "/p:RunTemplateTests=true /bl:artifacts/log/template.pack.binlog"
        displayName: Pack Templates
      - script: ./src/ProjectTemplates/build.cmd -ci -test -NoRestore -NoBuild -NoBuilddeps "/p:RunTemplateTests=true /bl:artifacts/log/template.test.binlog"
        displayName: Test Templates
      artifacts:
      - name: Windows_Test_Templates_Logs
        path: artifacts/log/
        publishOnError: true
      - name: Windows_Test_Templates_Results
        path: artifacts/TestResults/
        publishOnError: true

  - template: jobs/default-build.yml
    parameters:
      condition: ne(variables['SkipTests'], 'true')
      jobName: MacOS_Test
      jobDisplayName: "Test: macOS 10.13"
      agentOs: macOS
      isTestingJob: true
      buildArgs: --all --test "/p:RunTemplateTests=false"
      beforeBuild:
      - bash: "./eng/scripts/install-nginx-mac.sh"
        displayName: Installing Nginx
      afterBuild:
      - bash: ./build.sh --ci --pack --no-build --no-restore --no-build-deps "/bl:artifacts/log/packages.pack.binlog"
        displayName: Pack Packages (for Template tests)
      - bash: ./src/ProjectTemplates/build.sh --ci --pack --no-restore --no-build-deps "/bl:artifacts/log/template.pack.binlog"
        displayName: Pack Templates (for Template tests)
      - bash: ./build.sh --no-build --ci --test -p:RunFlakyTests=true
        displayName: Run Flaky Tests
        continueOnError: true
      artifacts:
      - name: MacOS_Test_Logs
        path: artifacts/log/
        publishOnError: true
      - name: MacOS_Test_Results
        path: artifacts/TestResults/
        publishOnError: true

  - template: jobs/default-build.yml
    parameters:
      condition: ne(variables['SkipTests'], 'true')
      jobName: Linux_Test
      jobDisplayName: "Test: Ubuntu 16.04 x64"
      agentOs: Linux
      isTestingJob: true
      buildArgs: --all --test "/p:RunTemplateTests=false"
      beforeBuild:
      - bash: "./eng/scripts/install-nginx-linux.sh"
        displayName: Installing Nginx
      - bash: "echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p"
        displayName: Increase inotify limit
      afterBuild:
      - bash: ./build.sh --ci --pack --no-build --no-restore --no-build-deps "/bl:artifacts/log/packages.pack.binlog"
        displayName: Pack Packages (for Template tests)
      - bash: ./src/ProjectTemplates/build.sh --ci --pack --no-restore --no-build-deps "/bl:artifacts/log/template.pack.binlog"
        displayName: Pack Templates (for Template tests)
      - bash: ./build.sh --no-build --ci --test -p:RunFlakyTests=true
        displayName: Run Flaky Tests
        continueOnError: true
      artifacts:
      - name: Linux_Test_Logs
        path: artifacts/log/
        publishOnError: true
      - name: Linux_Test_Results
        path: artifacts/TestResults/
        publishOnError: true

  # Source build
  - job: Source_Build
    displayName: 'Test: Linux Source Build'
    container: centos:7
    pool:
      vmImage: 'ubuntu-16.04'
    variables:
      DotNetCoreSdkDir: $(Agent.ToolsDirectory)/dotnet
      DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: true
    steps:
    - script: |
        source eng/common/native/common-library.sh
        mkdir -p $HOME/bin
        GetFile https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 $HOME/bin/jq
        chmod +x $HOME/bin/jq
        echo "##vso[task.prependpath]$HOME/bin"
      displayName: Install jq
    - task: UseDotNet@2
      displayName: 'Use .NET Core sdk'
      inputs:
        packageType: sdk
        # The SDK version selected here is intentionally supposed to use the latest release
        # For the purpose of building Linux distros, we can't depend on features of the SDK
        # which may not exist in pre-built versions of the SDK
        # Pinning to preview 8 since preview 9 has breaking changes
        version: 3.0.100
        installationPath: $(DotNetCoreSdkDir)
        includePreviewVersions: true
    - script: ./eng/scripts/ci-source-build.sh --ci --configuration Release /p:BuildManaged=true /p:BuildNodeJs=false
      displayName: Run ci-source-build.sh
    - task: PublishBuildArtifacts@1
      displayName: Upload logs
      condition: always()
      continueOnError: true
      inputs:
        pathtoPublish: artifacts/log/
        artifactName: Source_Build_Logs
        artifactType: Container
        parallel: true
    - task: PublishBuildArtifacts@1
      displayName: Upload package artifacts
      # Only capture source build artifacts in PRs for the sake of inspecting
      # changes that impact source-build. The artifacts from this build pipeline are never actually used.
      condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
      inputs:
        pathtoPublish: artifacts/packages/
        artifactName: Source_Build_Packages
        artifactType: Container
        parallel: true

  # Publish to the BAR
  - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
    - template: /eng/common/templates/job/publish-build-assets.yml
      parameters:
        dependsOn:
          - Windows_build
          - Windows_arm_build
          - CodeSign_Xplat_MacOS_x64
          - CodeSign_Xplat_Linux_x64
          - CodeSign_Xplat_Linux_arm
          - CodeSign_Xplat_Linux_arm64
          - CodeSign_Xplat_Linux_musl_x64
          - CodeSign_Xplat_Linux_musl_arm64
          # In addition to the dependencies above, ensure the build was successful overall.
          - Code_check
          - Linux_Test
          - MacOS_Test
          - Source_Build
          - Windows_Templates_Test
          - Windows_Test
        pool:
          vmImage: vs2017-win2016
        publishUsingPipelines: ${{ variables._PublishUsingPipelines }}
        enablePublishBuildArtifacts: true # publish artifacts/log files

- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
  - template: /eng/common/templates/post-build/post-build.yml
    parameters:
      # See https://github.com/dotnet/arcade/issues/2871
      enableSymbolValidation: false
      publishInstallersAndChecksums: true