Merge pull request #7266 from dotnet/merges/master-to-release/dev16.3

Merge master to release/dev16.3

.vsts-signed.yaml

@@ -105,6 +105,14 @@ jobs:
     continueOnError: true
     condition: succeeded()
 
+  # Publish native PDBs for archiving
+  - task: PublishBuildArtifacts@1
+    displayName: Publish Artifact Symbols
+    inputs:
+      PathtoPublish: '$(Build.SourcesDirectory)/artifacts/SymStore/$(BuildConfiguration)'
+      ArtifactName: NativeSymbols
+    condition: succeeded()
+
   # Execute cleanup tasks
   - task: ms-vseng.MicroBuildTasks.521a94ea-9e68-468a-8167-6dcf361ea776.MicroBuildCleanup@1
     displayName: Execute cleanup tasks

FSharpBuild.Directory.Build.props

@@ -75,6 +75,7 @@
     <RepositoryUrl Condition="'$(RepositoryUrl)' == ''">https://github.com/Microsoft/visualfsharp</RepositoryUrl>
     <RepositoryType Condition="'$(RepositoryType)' == ''">git</RepositoryType>
   </PropertyGroup>
+
   <PropertyGroup Condition="'$(FSharpSourceBuild)' == 'true' AND '$(RepositoryCommit)' == ''">
     <_DotGitDir>$(RepoRoot).git</_DotGitDir>
     <_HeadFileContent Condition="Exists('$(_DotGitDir)/HEAD')">$([System.IO.File]::ReadAllText('$(_DotGitDir)/HEAD').Trim())</_HeadFileContent>
@@ -87,7 +88,7 @@
   <PropertyGroup>
     <NoWarn Condition="'$(Language)' == 'F#'">$(NoWarn);FS2003</NoWarn><!-- warning when AssemblyInformationalVersion looks like '1.2.3-dev' -->
     <NoCompilerStandardLib>true</NoCompilerStandardLib><!-- necessary for resource generation using csc.exe -->
-    <DebugType>portable</DebugType>
+    <DebugType>embedded</DebugType>
     <MicroBuildAssemblyFileLanguage>fs</MicroBuildAssemblyFileLanguage>
     <UseStandardResourceNames>false</UseStandardResourceNames>
     <GenerateDocumentationFile>true</GenerateDocumentationFile>

azure-pipelines.yml

@@ -113,6 +113,12 @@ jobs:
             PathtoPublish: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\VisualFSharpFull.vsix'
             ArtifactName: 'Nightly'
           condition: succeeded()
+        - task: PublishBuildArtifacts@1
+          displayName: Publish Artifact Symbols
+          inputs:
+            PathtoPublish: '$(Build.SourcesDirectory)\artifacts\SymStore\$(_BuildConfig)'
+            ArtifactName: 'NativeSymbols'
+          condition: succeeded()
 
 #---------------------------------------------------------------------------------------------------------------------#
 #                                                     PR builds                                                       #

eng/Build.ps1

@@ -225,13 +225,14 @@ function UpdatePath() {
     TestAndAddToPath "$ArtifactsDir\bin\fsiAnyCpu\$configuration\net472"
 }
 
-function VerifyAssemblyVersions() {
-    $fsiPath = Join-Path $ArtifactsDir "bin\fsi\Proto\net472\publish\fsi.exe"
+function VerifyAssemblyVersionsAndSymbols() {
+    $assemblyVerCheckPath = Join-Path $ArtifactsDir "Bootstrap\AssemblyCheck\AssemblyCheck.dll"
 
     # Only verify versions on CI or official build
     if ($ci -or $official) {
-        $asmVerCheckPath = "$RepoRoot\scripts"
-        Exec-Console $fsiPath """$asmVerCheckPath\AssemblyVersionCheck.fsx"" -- ""$ArtifactsDir"""
+        $dotnetPath = InitializeDotNetCli
+        $dotnetExe = Join-Path $dotnetPath "dotnet.exe"
+        Exec-Console $dotnetExe """$assemblyVerCheckPath"" ""$ArtifactsDir"""
     }
 }
 
@@ -308,7 +309,7 @@ try {
     }
 
     if ($build) {
-        VerifyAssemblyVersions
+        VerifyAssemblyVersionsAndSymbols
     }
 
     $desktopTargetFramework = "net472"

eng/Version.Details.xml

@@ -3,9 +3,9 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19364.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19375.15">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>0c81c2bbdc49749e9940bc8858ebd16026d51277</Sha>
+      <Sha>ef1c110152df0d500fffb87878a86f88d1ca5295</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/build-utils.ps1

@@ -236,10 +236,11 @@ function Make-BootstrapBuild() {
     Remove-Item -re $dir -ErrorAction SilentlyContinue
     Create-Directory $dir
 
-    # prepare FsLex and Fsyacc
+    # prepare FsLex and Fsyacc and AssemblyCheck
     Run-MSBuild "$RepoRoot\src\buildtools\buildtools.proj" "/restore /t:Publish" -logFileName "BuildTools" -configuration $bootstrapConfiguration
     Copy-Item "$ArtifactsDir\bin\fslex\$bootstrapConfiguration\netcoreapp2.1\publish" -Destination "$dir\fslex" -Force -Recurse
     Copy-Item "$ArtifactsDir\bin\fsyacc\$bootstrapConfiguration\netcoreapp2.1\publish" -Destination "$dir\fsyacc" -Force  -Recurse
+    Copy-Item "$ArtifactsDir\bin\AssemblyCheck\$bootstrapConfiguration\netcoreapp2.1\publish" -Destination "$dir\AssemblyCheck" -Force  -Recurse
 
     # prepare compiler
     $projectPath = "$RepoRoot\proto.proj"

eng/common/init-tools-native.ps1

@@ -98,12 +98,18 @@ try {
       }
 
       Write-Verbose "Installing $ToolName version $ToolVersion"
-      Write-Verbose "Executing '$InstallerPath $LocalInstallerArguments'"
+      Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'"
       & $InstallerPath @LocalInstallerArguments
       if ($LASTEXITCODE -Ne "0") {
         $errMsg = "$ToolName installation failed"
         if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) {
-            Write-Warning $errMsg
+            $showNativeToolsWarning = $true
+            if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) {
+                $showNativeToolsWarning = $false
+            }
+            if ($showNativeToolsWarning) {
+                Write-Warning $errMsg
+            }
             $toolInstallationFailure = $true
         } else {
             Write-Error $errMsg

eng/common/init-tools-native.sh

@@ -71,7 +71,7 @@ function ReadGlobalJsonNativeTools {
   local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}')
   native_tools_list=${native_tools_list//[\" ]/}
   native_tools_list=${native_tools_list//,/$'\n'}
-  native_tools_list="$(echo -e "${native_tools_list}" | tr -d '[:space:]')"
+  native_tools_list="$(echo -e "${native_tools_list}" | tr -d '[[:space:]]')"
 
   local old_IFS=$IFS
   while read -r line; do

eng/common/native/CommonLibrary.psm1

@@ -59,9 +59,38 @@ function DownloadAndExtract {
                                           -Verbose:$Verbose
 
   if ($UnzipStatus -Eq $False) {
-    Write-Error "Unzip failed"
-    return $False
+    # Retry Download one more time with Force=true
+    $DownloadRetryStatus = CommonLibrary\Get-File -Uri $Uri `
+                                             -Path $TempToolPath `
+                                             -DownloadRetries 1 `
+                                             -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds `
+                                             -Force:$True `
+                                             -Verbose:$Verbose
+
+    if ($DownloadRetryStatus -Eq $False) {
+      Write-Error "Last attempt of download failed as well"
+      return $False
+    }
+
+    # Retry unzip again one more time with Force=true
+    $UnzipRetryStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath `
+                                            -OutputDirectory $InstallDirectory `
+                                            -Force:$True `
+                                            -Verbose:$Verbose
+    if ($UnzipRetryStatus -Eq $False)
+    {
+      Write-Error "Last attempt of unzip failed as well"
+      # Clean up partial zips and extracts
+      if (Test-Path $TempToolPath) {
+        Remove-Item $TempToolPath -Force
+      }
+      if (Test-Path $InstallDirectory) {
+        Remove-Item $InstallDirectory -Force -Recurse
+      }
+      return $False
+    }
   }
+
   return $True
 }
 

eng/common/pipeline-logging-functions.sh

@@ -39,11 +39,11 @@ function Write-PipelineTaskError {
     return
   fi
 
-  message_type="error"
-  sourcepath=''
-  linenumber=''
-  columnnumber=''
-  error_code=''
+  local message_type="error"
+  local sourcepath=''
+  local linenumber=''
+  local columnnumber=''
+  local error_code=''
 
   while [[ $# -gt 0 ]]; do
     opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
@@ -76,7 +76,7 @@ function Write-PipelineTaskError {
     shift
   done
 
-  message="##vso[task.logissue"
+  local message="##vso[task.logissue"
 
   message="$message type=$message_type"
 
@@ -100,3 +100,73 @@ function Write-PipelineTaskError {
   echo "$message"
 }
 
+function Write-PipelineSetVariable {
+  if [[ "$ci" != true ]]; then
+    return
+  fi
+
+  local name=''
+  local value=''
+  local secret=false
+  local as_output=false
+  local is_multi_job_variable=true
+
+  while [[ $# -gt 0 ]]; do
+    opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+    case "$opt" in
+      -name|-n)
+        name=$2
+        shift
+        ;;
+      -value|-v)
+        value=$2
+        shift
+        ;;
+      -secret|-s)
+        secret=true
+        ;;
+      -as_output|-a)
+        as_output=true
+        ;;
+      -is_multi_job_variable|-i)
+        is_multi_job_variable=$2
+        shift
+        ;;
+    esac
+    shift
+  done
+
+  value=${value/;/%3B}
+  value=${value/\\r/%0D}
+  value=${value/\\n/%0A}
+  value=${value/]/%5D}
+
+  local message="##vso[task.setvariable variable=$name;isSecret=$secret;isOutput=$is_multi_job_variable]$value"
+
+  if [[ "$as_output" == true ]]; then
+    $message
+  else
+    echo "$message"
+  fi
+}
+
+function Write-PipelinePrependPath {
+  local prepend_path=''
+
+  while [[ $# -gt 0 ]]; do
+    opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+    case "$opt" in
+      -path|-p)
+        prepend_path=$2
+        shift
+        ;;
+    esac
+    shift
+  done
+
+  export PATH="$prepend_path:$PATH"
+
+  if [[ "$ci" == true ]]; then
+    echo "##vso[task.prependpath]$prepend_path"
+  fi
+}

eng/common/post-build/darc-gather-drop.ps1

@@ -0,0 +1,36 @@
+param(
+  [Parameter(Mandatory=$true)][string] $BarBuildId,             # ID of the build which assets should be downloaded
+  [Parameter(Mandatory=$true)][string] $MaestroAccessToken,     # Token used to access Maestro API
+  [Parameter(Mandatory=$true)][string] $DropLocation            # Where the assets should be downloaded to
+)
+
+$ErrorActionPreference = "Stop"
+Set-StrictMode -Version 2.0
+
+. $PSScriptRoot\..\tools.ps1
+
+try {
+  Write-Host "Installing DARC ..."
+
+  . $PSScriptRoot\..\darc-init.ps1
+  $exitCode = $LASTEXITCODE
+
+  if ($exitCode -ne 0) {
+    Write-PipelineTaskError "Something failed while running 'darc-init.ps1'. Check for errors above. Exiting now..."
+    ExitWithExitCode $exitCode
+  }
+
+  darc gather-drop --non-shipping `
+    --continue-on-error `
+    --id $BarBuildId `
+    --output-dir $DropLocation `
+    --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ `
+    --password $MaestroAccessToken `
+    --latest-location
+}
+catch {
+  Write-Host $_
+  Write-Host $_.Exception
+  Write-Host $_.ScriptStackTrace
+  ExitWithExitCode 1
+}

eng/common/templates/post-build/channels/internal-servicing.yml

@@ -84,6 +84,7 @@ stages:
             /p:AzureStorageAccountName=$(ProxyBackedFeedsAccountName)
             /p:AzureStorageAccountKey=$(dotnetfeed-storage-access-key-1)
             /p:AzureDevOpsFeedsBaseUrl=$(dotnetfeed-internal-private-feed-url)
+            /p:StaticInternalFeed=$(dotnetfeed-internal-nonstable-feed-url)
             /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe
             /p:BARBuildId=$(BARBuildId) 
             /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com'
@@ -142,29 +143,6 @@ stages:
             filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1
             arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion)
 
-  - job:
-    displayName: Gather Drop
-    dependsOn: setupMaestroVars
-    variables:
-      BARBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
-    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.InternalServicing_30_Channel_Id)
-    pool:
-      vmImage: 'windows-2019'
-    steps:
-      - task: PowerShell@2
-        displayName: Setup Darc CLI
-        inputs:
-          targetType: filePath
-          filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1'
-
-      - task: PowerShell@2
-        displayName: Run Darc gather-drop
-        inputs:
-          targetType: inline
-          script: |
-            darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) --latest-location
-        enabled: false
-
   - template: ../promote-build.yml
     parameters:
       ChannelId: ${{ variables.InternalServicing_30_Channel_Id }}

eng/common/templates/post-build/channels/public-dev-release.yml

@@ -77,7 +77,7 @@ stages:
           filePath: eng\common\sdk-task.ps1
           arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet 
             /p:ChannelId=$(PublicDevRelease_30_Channel_Id)
-            /p:ArtifactsCategory=.NetCore
+            /p:ArtifactsCategory=$(_DotNetArtifactsCategory)
             /p:IsStableBuild=$(IsStableBuild)
             /p:IsInternalBuild=$(IsInternalBuild)
             /p:RepositoryName=$(Build.Repository.Name)
@@ -139,27 +139,9 @@ stages:
             filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1
             arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion)
 
-  - job:
-    displayName: Gather Drop
-    dependsOn: setupMaestroVars
-    variables:
-      BARBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
-    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id)
-    pool:
-      vmImage: 'windows-2019'
-    steps:
-      - task: PowerShell@2
-        displayName: Setup Darc CLI
-        inputs:
-          targetType: filePath
-          filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1'
-
-      - task: PowerShell@2
-        displayName: Run Darc gather-drop
-        inputs:
-          targetType: inline
-          script: |
-            darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) --latest-location
+  - template: ../darc-gather-drop.yml
+    parameters:
+      ChannelId: ${{ variables.PublicDevRelease_30_Channel_Id }}
 
   - template: ../promote-build.yml	
     parameters: