Add support for source-build (#620)

.editorconfig

@@ -7,6 +7,7 @@ root = true
 [*]
 indent_style = space
 # (Please don't specify an indent_size here; that has too many unintended consequences.)
+trim_trailing_whitespace = true
 
 # Code files
 [*.{cs,csx,vb,vbx}]

Directory.Build.props

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project>
   <!--
-    We don't follow Arcade conventions for project naming. 
+    We don't follow Arcade conventions for project naming.
   -->
   <PropertyGroup Condition="'$(IsUnitTestProject)' == ''">
     <IsUnitTestProject>false</IsUnitTestProject>
@@ -14,12 +14,7 @@
   <Import
     Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), AspNetCoreSettings.props))\AspNetCoreSettings.props"
     Condition=" '$(CI)' != 'true' AND '$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), AspNetCoreSettings.props))' != '' " />
-
-  <PropertyGroup Condition="'$(CopyrightMicrosoft)' != ''">
-    <Copyright>$(CopyrightMicrosoft)</Copyright>
-    <PackageLicenseExpression>Apache-2.0</PackageLicenseExpression>
-  </PropertyGroup>
-
+
   <PropertyGroup Label="Build Settings">
     <LangVersion>8.0</LangVersion>
     <StrongNameKeyId>MicrosoftAspNetCore</StrongNameKeyId>
@@ -28,22 +23,19 @@
     <PublishWindowsPdb>true</PublishWindowsPdb>
     <GenerateResxSource>true</GenerateResxSource>
     <GenerateResxSourceEmitFormatMethods>true</GenerateResxSourceEmitFormatMethods>
+    <ExcludeFromSourceBuild Condition="'$(IsUnitTestProject)' == 'true'">true</ExcludeFromSourceBuild>
   </PropertyGroup>
 
   <PropertyGroup Label="Package and Assembly Metadata">
+    <PackageLicenseExpression>Apache-2.0</PackageLicenseExpression>
     <Product>Microsoft ASP.NET Core</Product>
     <PackageIconUrl>https://go.microsoft.com/fwlink/?LinkID=288859</PackageIconUrl>
   </PropertyGroup>
 
   <PropertyGroup Label="Warning Suppressions">
-    <!-- 
-      Suppress a warning about upcoming deprecation of PackageLicenseUrl. When embedding licenses are supported, replace PackageLicenseUrl with PackageLicenseExpression.
-    -->
-    <NoWarn>$(NoWarn);NU5125</NoWarn>
-
-    <!-- 
+    <!--
       Suppress warnings about assembly conflicts. This happens for assemblies that ship in VS so it's irrelevant.
     -->
     <NoWarn>$(NoWarn);MSB3277</NoWarn>
   </PropertyGroup>
-</Project>
+</Project>

Directory.Build.targets

@@ -3,13 +3,8 @@
   <Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
   <Import Project="eng\MPack.targets" />
 
-  <ItemGroup>
-    <!-- Workaround https://github.com/dotnet/sdk/issues/2976 -->
-    <PackageReference Update="Microsoft.NETCore.Platforms" PrivateAssets="All" />
-  </ItemGroup>
-
   <!-- Workaround https://github.com/dotnet/cli/issues/10528 -->
   <PropertyGroup>
     <BundledNETCorePlatformsPackageVersion>$(MicrosoftNETCorePlatformsPackageVersion)</BundledNETCorePlatformsPackageVersion>
   </PropertyGroup>
-</Project>
+</Project>

NuGet.config

@@ -2,8 +2,7 @@
 <configuration>
   <packageSources>
     <clear />
-    <add key="arcade" value="https://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json" />
     <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
     <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
   </packageSources>
-</configuration>
+</configuration>

azure-pipelines.yml

@@ -1,3 +1,7 @@
+#
+# See https://docs.microsoft.com/azure/devops/pipelines/yaml-schema for reference.
+#
+
 variables:
 - name: Build.Repository.Clean
   value: true
@@ -10,18 +14,18 @@ variables:
 - name: _DotNetArtifactsCategory
   value: ASPNETCORETOOLING
 
-resources:
-  containers:
-  - container: LinuxContainer
-    image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-14.04-cross-0cd4667-20170319080304
-    options: --init # This ensures all the stray defunct processes are reaped.
-
 trigger:
-- master
-- release/*
+  batch: true
+  branches:
+    include:
+      - master
+      - release/*
 
 pr:
-- "*"
+  autoCancel: true
+  branches:
+    include:
+      - '*'
 
 jobs:
 # This job checks for code quality
@@ -50,6 +54,60 @@ jobs:
       - powershell: ./restore.cmd -ci; ./eng/scripts/CodeCheck.ps1 -ci
         displayName: Run eng/scripts/CodeCheck.ps1
 
+    - job: Source_Build
+      jobName: Source_Build
+      displayName: 'Test: Linux Source Build'
+      container:
+        image: centos:7
+        options: --init # This ensures all the stray defunct processes are reaped.
+      pool:
+        vmImage: ubuntu-16.04
+      variables:
+      - DotNetCoreSdkDir: $(Agent.ToolsDirectory)/dotnet
+      - DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: true
+      - _BuildConfig: Release
+      steps:
+      - script: |
+          source eng/common/native/common-library.sh
+          mkdir -p $HOME/bin
+          GetFile https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 $HOME/bin/jq
+          chmod +x $HOME/bin/jq
+          echo "##vso[task.prependpath]$HOME/bin"
+        displayName: Install jq
+      - task: UseDotNet@2
+        displayName: 'Use .NET Core sdk'
+        inputs:
+          packageType: sdk
+          # The SDK version selected here is intentionally supposed to use the latest release
+          # For the purpose of building Linux distros, we can't depend on features of the SDK
+          # which may not exist in pre-built versions of the SDK
+          version: 3.0.x
+          installationPath: $(DotNetCoreSdkDir)
+          includePreviewVersions: true
+
+      - script: ./eng/scripts/ci-source-build.sh --ci --configuration $(_BuildConfig)
+        displayName: Run ci-source-build.sh
+
+      - task: PublishBuildArtifacts@1
+        displayName: Upload package artifacts
+        # Only capture source build artifacts in PRs for the sake of inspecting
+        # changes that impact source-build. The artifacts from this build pipeline are never actually used.
+        condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
+        inputs:
+          pathtoPublish: artifacts/packages/
+          artifactName: Source_Build_Packages
+          artifactType: Container
+          parallel: true
+
+      - task: PublishBuildArtifacts@1
+        displayName: Publish Logs
+        inputs:
+          PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)'
+          PublishLocation: Container
+          ArtifactName: Source_Build_Logs
+        continueOnError: true
+        condition: always()
+
 # Three jobs for each of the three OSes we want to run on
 - template: /eng/common/templates/jobs/jobs.yml
   parameters:
@@ -165,8 +223,10 @@ jobs:
         condition: succeeded()
 
     - job: Linux
+      container:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-14.04-cross-0cd4667-20170319080304
+        options: --init # This ensures all the stray defunct processes are reaped.
       pool:
-        container: LinuxContainer
         vmImage: ubuntu-16.04
       strategy:
         matrix:
@@ -181,3 +241,4 @@ jobs:
         name: Build
         displayName: Build
         condition: succeeded()
+

build.cmd

@@ -1,3 +1,3 @@
 @echo off
-powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -build -restore -pack -test %*"
-exit /b %ErrorLevel%
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -restore -build -pack %*"
+exit /b %ErrorLevel%

build.sh

@@ -13,4 +13,4 @@ while [[ -h $source ]]; do
 done
 
 scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
-"$scriptroot/eng/common/build.sh" --pack --build --restore --test $@
+"$scriptroot/eng/common/build.sh" --restore --build --pack $@

eng/AfterSigning.targets

@@ -1,11 +1,11 @@
 <Project>
-  <Target Name="_EnsureVSIXWasGenerated" AfterTargets="GenerateVisualStudioInsertionManifests" Condition="'$(OS)'=='WINDOWS_NT'">
+  <Target Name="_EnsureVSIXWasGenerated" AfterTargets="GenerateVisualStudioInsertionManifests" Condition="'$(OS)'=='WINDOWS_NT' AND '$(DotNetBuildFromSource)' != 'true'">
     <PropertyGroup>
       <VSSetupDir>$(ArtifactsDir)VSSetup\</VSSetupDir>
       <RazorExtensionVSIXName>Microsoft.VisualStudio.RazorExtension.vsix</RazorExtensionVSIXName>
     </PropertyGroup>
-    <Error 
+    <Error
       Text="$(RazorExtensionVSIXName) was not generated."
       Condition="!Exists('$(VSSetupDir)$(Configuration)\$(RazorExtensionVSIXName)')" />
   </Target>
-</Project>
+</Project>

eng/MPack.targets

@@ -1,16 +1,16 @@
 <Project>
-  <Target 
-    Name="GenerateMPack" 
-    AfterTargets="Build" 
-    DependsOnTargets="_MPackWriteAddinInfo" 
-    Condition="'$(IsMPackProject)'=='true' and '$(TargetFramework)'!=''">
+  <Target
+    Name="GenerateMPack"
+    AfterTargets="Build"
+    DependsOnTargets="_MPackWriteAddinInfo"
+    Condition="'$(IsMPackProject)'=='true' and '$(TargetFramework)'!='' and ('$(DotNetBuildFromSource)' != 'true' or '$(ExcludeFromSourceBuild)' != 'true')">
 
     <!--
       In our case the mpack archive requires the following:
       1. An addin.info
       2. An addin binary (Microsoft.VisualStudio.Mac.RazorAddin.dll)
         a. _Manifest.addin.xml embedded
-        b. Addin assembly attributes for metadata 
+        b. Addin assembly attributes for metadata
       3. All language service binaries
     -->
 
@@ -27,11 +27,11 @@
     <RemoveDir Directories="$(IntermediateOutputPath)MPack\" />
     <MakeDir Directories="$(IntermediateOutputPath);$(IntermediateOutputPath)MPack\" />
     <Copy SourceFiles="@(MPackFile)" DestinationFolder="$(IntermediateOutputPath)MPack\" />
-    
+
     <MakeDir Directories="$(ArtifactsMPackDir)" />
-    <ZipDirectory 
-      SourceDirectory="$(IntermediateOutputPath)MPack\" 
-      DestinationFile="$(MPackOutputPath)" 
+    <ZipDirectory
+      SourceDirectory="$(IntermediateOutputPath)MPack\"
+      DestinationFile="$(MPackOutputPath)"
       Overwrite="true" />
 
     <ItemGroup>
@@ -41,7 +41,7 @@
 
   <Target Name="_MPackWriteAddinInfo">
     <PropertyGroup>
-      <MPackManifest>$(AddinDirectory)Properties\_Manifest.addin.xml</MPackManifest>  
+      <MPackManifest>$(AddinDirectory)Properties\_Manifest.addin.xml</MPackManifest>
       <AddinInfoFilePath>$(IntermediateOutputPath)addin.info</AddinInfoFilePath>
     </PropertyGroup>
 

eng/Versions.props

@@ -33,7 +33,7 @@
     <AddinVersion Condition="'$(OfficialBuildId)' == ''">$(AddinVersion).42424242.42</AddinVersion>
   </PropertyGroup>
   <PropertyGroup>
-    <RestoreSources>
+    <RestoreSources Condition="'$(DotNetBuildOffline)' != 'true'">
       $(RestoreSources);
       https://dotnetfeed.blob.core.windows.net/aspnet-extensions/index.json;
       https://dotnet.myget.org/F/roslyn/api/v3/index.json;

eng/configure-toolset.sh

@@ -2,4 +2,6 @@
 # We could potentially try to find an existing installation that has all the required runtimes,
 # but it's unlikely one will be available.
 
-use_installed_dotnet_cli="false"
+if [ "${DotNetBuildFromSource:-false}" = false ]; then
+    use_installed_dotnet_cli="false"
+fi

eng/scripts/ci-source-build.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+#
+# This script is meant for testing source build by imitating some of the input parameters and conditions.
+#
+
+set -euo pipefail
+
+scriptroot="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+reporoot="$(dirname "$(dirname "$scriptroot")")"
+
+# For local development, make a backup copy of this file first
+if [ ! -f "$reporoot/global.bak.json" ]; then
+    mv "$reporoot/global.json" "$reporoot/global.bak.json"
+fi
+
+# Detect the current version of .NET Core installed
+export SDK_VERSION=$(dotnet --version)
+echo "The ambient version of .NET Core SDK version = $SDK_VERSION"
+
+# Update the global.json file to match the current .NET environment
+cat "$reporoot/global.bak.json" | \
+    jq '.sdk.version=env.SDK_VERSION' | \
+    jq '.tools.dotnet=env.SDK_VERSION' | \
+    jq 'del(.tools.runtimes)' \
+    > "$reporoot/global.json"
+
+# Restore the original global.json file
+trap "{
+    mv "$reporoot/global.bak.json" "$reporoot/global.json"
+}" EXIT
+
+export DotNetBuildFromSource='true'
+"$reporoot/build.sh" "$@"

src/Razor/benchmarks/Microsoft.AspNetCore.Razor.Performance/Microsoft.AspNetCore.Razor.Performance.csproj

@@ -6,6 +6,7 @@
     <ServerGarbageCollection>true</ServerGarbageCollection>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <IsPackable>false</IsPackable>
+    <ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
   </PropertyGroup>
 
   <ItemGroup>

src/Razor/src/Microsoft.AspNetCore.Razor.Language/Components/Microsoft.AspNetCore.Components.Razor.Extensions.csproj

@@ -6,6 +6,8 @@
     <RootNamespace>Microsoft.AspNetCore.Components.Razor</RootNamespace>
     <Description>Extensions to the Razor compiler to support building Razor Components.</Description>
     <IsProductPackage>true</IsProductPackage>
+    <!-- Excluded because this assembly has a 3rd party dependency. -->
+    <ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
 
     <!-- Copy package references to output, needed so the build project can find them -->
     <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>

src/Razor/src/Microsoft.CodeAnalysis.Razor.Workspaces/Microsoft.CodeAnalysis.Razor.Workspaces.csproj

@@ -5,6 +5,7 @@
     <TargetFramework>netstandard2.0</TargetFramework>
     <EnableApiCheck>false</EnableApiCheck>
     <IsShipping>false</IsShipping>
+    <ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
   </PropertyGroup>
 
   <ItemGroup>

src/Razor/src/Microsoft.CodeAnalysis.Remote.Razor/Microsoft.CodeAnalysis.Remote.Razor.csproj

@@ -5,6 +5,7 @@
     <TargetFramework>net472</TargetFramework>
     <EnableApiCheck>false</EnableApiCheck>
     <IsShipping>false</IsShipping>
+    <ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
   </PropertyGroup>
 
   <ItemGroup>

src/Razor/src/Microsoft.VisualStudio.Editor.Razor/Microsoft.VisualStudio.Editor.Razor.csproj

@@ -5,6 +5,7 @@
     <Description>Razor is a markup syntax for adding server-side logic to web pages. This package contains the Visual Studio agnostic Razor design-time infrastructure.</Description>
     <EnableApiCheck>false</EnableApiCheck>
     <IsShipping>false</IsShipping>
+    <ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
     <NoWarn>$(NoWarn);VSTHRD200</NoWarn>
   </PropertyGroup>
 

src/Razor/src/Microsoft.VisualStudio.LanguageServices.Razor/Microsoft.VisualStudio.LanguageServices.Razor.csproj

@@ -7,6 +7,7 @@
     <RulesDirectory>..\Microsoft.NET.Sdk.Razor\build\netstandard2.0\Rules\</RulesDirectory>
     <CodeAnalysisRuleSet>Microsoft.VisualStudio.LanguageServices.Razor.ruleset</CodeAnalysisRuleSet>
     <IsShipping>false</IsShipping>
+    <ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
   </PropertyGroup>
 
   <ItemGroup>
@@ -48,7 +49,7 @@
   <ItemGroup>
     <Reference Include="System.Xaml" />
   </ItemGroup>
-  
+
   <!-- Workaround for Microsoft.VisualStudio.SDK.EmbedInteropTypes not working correctly-->
   <Target Name="_EmbeddedAssemblyWorkaround" DependsOnTargets="ResolveReferences" BeforeTargets="FindReferenceAssembliesForReferences">
     <ItemGroup>
@@ -153,7 +154,7 @@
   </ItemGroup>
 
   <!--
-    Despite us specifying %(XamlPropertyRule.OutputPath), the ProjectSystem.SDK targets still add files in the 
+    Despite us specifying %(XamlPropertyRule.OutputPath), the ProjectSystem.SDK targets still add files in the
     obj folder to the Compile group. This is a workaround to remove them.
   -->
   <Target Name="GrossProjectSystemSDKWorkaround" AfterTargets="GenerateRulePrep">