[release/6.0] Get `dotnet` cores on Linux / macOS (#37139)

- back port of #36824 and #37077

Get `dotnet` cores on Linux / macOS (#36824)

  - follow up to 0d0103b4
  - `COMPlus*` variables were uppercased and `dotnet` ignored them on Linux / macOS
  - only upload dumps in `failed()` jobs
    - use scripts to avoid more YAML complications and avoid empty artifacts

Remove unused parameter and add requested comments (#37077)

  - `parameters.buildScript` was only ever set to its default value
  - add comments explaining the `parameters.steps` expansion
    - see https://github.com/dotnet/aspnetcore/pull/36824#pullrequestreview-764795624

.azure/pipelines/ci.yml

@@ -541,7 +541,6 @@ stages:
       jobDisplayName: "Build: Linux Musl x64"
       agentOs: Linux
       container: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.9-WithNode-0fc54a3-20190918214015
-      buildScript: ./eng/build.sh
       buildArgs:
         --arch x64
         --os-name linux-musl
@@ -578,7 +577,6 @@ stages:
       agentOs: Linux
       useHostedUbuntu: false
       container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm-alpine-20210409142425-044d5b9
-      buildScript: ./eng/build.sh
       buildArgs:
         --arch arm
         --os-name linux-musl
@@ -614,7 +612,6 @@ stages:
       agentOs: Linux
       useHostedUbuntu: false
       container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-alpine-20210409142425-b2c2436
-      buildScript: ./eng/build.sh
       buildArgs:
         --arch arm64
         --os-name linux-musl

.azure/pipelines/jobs/default-build.yml

@@ -33,8 +33,6 @@
 #       This build definition is enabled for code signing. (Only applies to Windows)
 #   buildDirectory: string
 #       Specifies what directory to run build.sh/cmd
-#   buildScript: string
-#       Specifies the build script to run. Defaults to build.sh or build.cmd.
 #   skipComponentGovernanceDetection: boolean
 #       Determines if component governance detection can be skipped
 #
@@ -55,7 +53,6 @@ parameters:
   # jobDisplayName: '' - use agentOs by default.
   artifacts: []
   buildDirectory: $(System.DefaultWorkingDirectory)/eng/
-  buildScript: ''
   installTar: true
   installNodeJs: true
   installJdk: true
@@ -127,7 +124,6 @@ jobs:
     variables:
     - AgentOsName: ${{ parameters.agentOs }}
     - ASPNETCORE_TEST_LOG_MAXPATH: "200" # Keep test log file name length low enough for artifact zipping
-    - BuildScript: ${{ parameters.buildScript }}
     - BuildScriptArgs: ${{ parameters.buildArgs }}
     - _BuildConfig: ${{ parameters.configuration }}
     - BuildConfiguration: ${{ parameters.configuration }}
@@ -135,8 +131,6 @@ jobs:
       - BuildDirectory: $(System.DefaultWorkingDirectory)
     - ${{ if ne(parameters.buildDirectory, '') }}:
       - BuildDirectory: ${{ parameters.buildDirectory }}
-    - COMPlus_DbgEnableMiniDump: 1
-    - COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core"
     - DOTNET_CLI_HOME: $(System.DefaultWorkingDirectory)
     - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
     - TeamName: AspNetCore
@@ -214,19 +208,34 @@ jobs:
           env:
             Token: $(dn-bot-dnceng-artifact-feeds-rw)
 
+    # Add COMPlus_* environment variables to build steps.
     - ${{ if ne(parameters.steps, '')}}:
-      - ${{ parameters.steps }}
+      - ${{ each step in parameters.steps }}:
+          # Include all properties e.g. `task: CmdLine@2` or `displayName: Build x64` _except_ a provided `env:`.
+          # Aim here is to avoid having two `env:` properties in the expanded YAML.
+          - ${{ each pair in step }}:
+              ${{ if ne(pair.key, 'env') }}:
+                ${{ pair.key }}: ${{ pair.value }}
+            env:
+              # Include the variables we always want.
+              COMPlus_DbgEnableMiniDump: 1
+              COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core"
+              # Expand provided `env:` properties, if any.
+              ${{ if step.env }}:
+                ${{ step.env }}
     - ${{ if eq(parameters.steps, '')}}:
-      - ${{ if eq(parameters.buildScript, '') }}:
-        - ${{ if eq(parameters.agentOs, 'Windows') }}:
-          - script: $(BuildDirectory)\build.cmd -ci -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs) /p:DotNetSignType=$(_SignType)
-            displayName: Run build.cmd
-        - ${{ if ne(parameters.agentOs, 'Windows') }}:
-          - script: $(BuildDirectory)/build.sh --ci --nobl --configuration $(BuildConfiguration) $(BuildScriptArgs)
-            displayName: Run build.sh
-      - ${{ if ne(parameters.buildScript, '') }}:
-        - script: $(BuildScript) -ci -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs)
-          displayName: run $(BuildScript)
+      - ${{ if eq(parameters.agentOs, 'Windows') }}:
+        - script: $(BuildDirectory)\build.cmd -ci -nobl -Configuration $(BuildConfiguration) $(BuildScriptArgs) /p:DotNetSignType=$(_SignType)
+          displayName: Run build.cmd
+          env:
+            COMPlus_DbgEnableMiniDump: 1
+            COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core"
+      - ${{ if ne(parameters.agentOs, 'Windows') }}:
+        - script: $(BuildDirectory)/build.sh --ci --nobl --configuration $(BuildConfiguration) $(BuildScriptArgs)
+          displayName: Run build.sh
+          env:
+            COMPlus_DbgEnableMiniDump: 1
+            COMPlus_DbgMiniDumpName: "$(System.DefaultWorkingDirectory)/dotnet-%d.%t.core"
 
     - ${{ parameters.afterBuild }}
 
@@ -260,26 +269,6 @@ jobs:
       - script:  echo "##vso[task.setvariable variable=CG_RAN]true"
         displayName: 'Skip Component Detection'
 
-    - task: CopyFiles@2
-      displayName: Create dump directory
-      condition: always()
-      continueOnError: true
-      inputs:
-        contents: |
-          '*.core'
-          restore.sh
-        sourceFolder: $(System.DefaultWorkingDirectory)
-        targetFolder: artifacts/dumps/
-    - task: PublishBuildArtifacts@1
-      displayName: Upload dump files
-      condition: always()
-      continueOnError: true
-      inputs:
-        artifactName: ${{ coalesce(parameters.jobName, parameters.agentOs) }}_Dumps
-        artifactType: Container
-        parallel: true
-        pathtoPublish: artifacts/dumps/
-
     - ${{ each artifact in parameters.artifacts }}:
       - task: PublishBuildArtifacts@1
         displayName: Upload artifacts from ${{ artifact.path }}
@@ -294,6 +283,15 @@ jobs:
           artifactType: Container
           parallel: true
 
+    - ${{ if eq(parameters.agentOs, 'Windows') }}:
+      - powershell: $(Build.SourcesDirectory)/eng/scripts/UploadCores.ps1 -ProcDumpOutputPath artifacts/dumps/
+        condition: failed()
+        displayName: Upload cores
+    - ${{ if ne(parameters.agentOs, 'Windows') }}:
+      - script: $(Build.SourcesDirectory)/eng/scripts/upload-cores.sh
+        condition: failed()
+        displayName: Upload cores
+
     - ${{ if and(eq(parameters.isTestingJob, true), ne(parameters.jobName, 'Windows_Templates_Test')) }}:
       - task: PublishTestResults@2
         displayName: Publish js test results

eng/scripts/UploadCores.ps1

+param(
+  [Parameter(Mandatory = $true)]
+  [ValidateNotNullOrEmpty()]
+  [string]$ProcDumpOutputPath
+)
+
+Set-StrictMode -Version 2
+$ErrorActionPreference = 'Stop'
+
+function Warn {
+  [CmdletBinding()]
+  param (
+    [Parameter(Mandatory = $true)]
+    [ValidateNotNullOrEmpty()]
+    [string]$Message
+  )
+
+  Write-Warning "$Message"
+  if ((Test-Path env:TF_BUILD) -and $env:TF_BUILD) {
+    Write-Host "##vso[task.logissue type=warning]$Message"
+  }
+}
+
+$repoRoot = Resolve-Path "$PSScriptRoot\..\.."
+$procDumpOutputPath = Join-Path $repoRoot $ProcDumpOutputPath
+
+if ((Test-Path env:SYSTEM_DEFAULTWORKINGDIRECTORY) -and $env:SYSTEM_DEFAULTWORKINGDIRECTORY) {
+  if (Test-Path env:SYSTEM_PHASENAME) { $jobName = "$env:SYSTEM_PHASENAME" } else { $jobName = "$env:AGENT_OS" }
+  $artifactName = "${jobName}_Dumps"
+  $wd = $env:SYSTEM_DEFAULTWORKINGDIRECTORY
+} else {
+  $artifactName = "Artifacts_Dumps"
+  $wd = "$PWD"
+}
+
+[string[]]$files = Get-ChildItem "$wd\dotnet-*.core"
+if (Test-Path "$procDumpOutputPath") {
+  $files += Get-ChildItem "${procDumpOutputPath}*.dmp"
+}
+
+if ($null -eq $files -or 0 -eq $files.Count) {
+  Warn "No core files found."
+} else {
+  foreach ($file in $files) {
+    Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName]$file"
+  }
+}

eng/scripts/upload-cores.sh

+#!/usr/bin/env bash
+
+set -euo pipefail
+RESET="\033[0m"
+YELLOW="\033[0;33m"
+
+__warn() {
+  echo -e "${YELLOW}warning: $*${RESET}"
+  if [ -n "${TF_BUILD:-}" ]; then
+    echo "##vso[task.logissue type=warning]$*"
+  fi
+}
+
+if [ -n "${SYSTEM_DEFAULTWORKINGDIRECTORY:-}" ]; then
+  jobName="${SYSTEM_PHASENAME:-$AGENT_OS}"
+  artifactName="${jobName}_Dumps"
+  wd=$SYSTEM_DEFAULTWORKINGDIRECTORY
+else
+  artifactName=Artifacts_Dumps
+  wd=$(pwd -P)
+fi
+
+save_nullglob=$(shopt -p nullglob || true)
+shopt -s nullglob
+files=(
+  $wd/core*
+  $wd/dotnet-*.core
+)
+$save_nullglob
+
+if [ -z "${files:-}" ] || (( ${#files[@]} == 0 )); then
+  __warn "No core files found."
+else
+  for file in ${files[@]}; do
+    echo "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName]$file"
+  done
+fi