Rebase on llvm 16 (#421)

* Rebase our changes on top of llvm release/16.x branch

* Don't enable zstd, we can't use it from Brew on arm64 builds on x64

* Build libcxx, and use it instead of libstdc++, on Linux

* Remove Windows arm32 packages.

Co-authored-by: Adeel Mujahid <3840695+am11@users.noreply.github.com>
Co-authored-by: Alexander Köplinger <alex.koeplinger@outlook.com>
Co-authored-by: Zoltan Varga <vargaz@gmail.com>

.config/tsaoptions.json

@@ -0,0 +1,10 @@
+{
+    "instanceUrl": "https://devdiv.visualstudio.com/",
+    "template": "TFSDEVDIV",
+    "projectName": "DEVDIV",
+    "areaPath": "DevDiv\\mono",
+    "iterationPath": "DevDiv",
+    "notificationAliases": [ "runtimerepo-infra@microsoft.com" ],
+    "repositoryName": "llvm-project",
+    "codebaseName": "llvm-project"
+}

.gitignore

@@ -70,3 +70,7 @@ pythonenv*
 /clang/utils/analyzer/projects/*/RefScanBuildResults
 # automodapi puts generated documentation files here.
 /lldb/docs/python_api/
+
+/artifacts
+/.dotnet
+/.packages

Directory.Build.props

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project>
+  <Import Project="dir.common.props" />
+  <Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />
+</Project>

Directory.Build.targets

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project>
+  <Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
+</Project>

NuGet.config

@@ -0,0 +1,16 @@
+<configuration>
+  <!-- Don't use any higher level config files.
+       Our builds need to be isolated from user/machine state -->
+  <fallbackPackageFolders>
+    <clear />
+  </fallbackPackageFolders>
+  <packageSources>
+    <clear />
+    <add key="dotnet-public" value="https://dnceng.pkgs.visualstudio.com/public/_packaging/dotnet-public/nuget/v3/index.json" />
+    <add key="dotnet-tools" value="https://dnceng.pkgs.visualstudio.com/public/_packaging/dotnet-tools/nuget/v3/index.json" />
+    <add key="dotnet-eng" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" />
+  </packageSources>
+  <disabledPackageSources>
+    <clear />
+  </disabledPackageSources>
+</configuration>

build.cmd

@@ -0,0 +1,8 @@
+@echo off
+setlocal
+
+set _args=%*
+if "%~1"=="-?" set _args=-help
+
+powershell -ExecutionPolicy ByPass -NoProfile -File "%~dp0eng\build.ps1" %_args%
+exit /b %ERRORLEVEL%

build.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+
+function is_cygwin_or_mingw()
+{
+  case $(uname -s) in
+    CYGWIN*)    return 0;;
+    MINGW*)     return 0;;
+    *)          return 1;;
+  esac
+}
+
+# resolve $SOURCE until the file is no longer a symlink
+while [[ -h $source ]]; do
+  scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+  source="$(readlink "$source")"
+
+  # if $source was a relative symlink, we need to resolve it relative to the path where the
+  # symlink file was located
+  [[ $source != /* ]] && source="$scriptroot/$source"
+done
+
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+if is_cygwin_or_mingw; then
+  # if bash shell running on Windows (not WSL),
+  # pass control to powershell build script.
+  scriptroot=$(cygpath -d "$scriptroot")
+  powershell -c "$scriptroot\\build.cmd" $@
+else
+  "$scriptroot/eng/build.sh" $@
+fi

clang/cmake/modules/AddClang.cmake

@@ -177,6 +177,56 @@ macro(add_clang_tool name)
                                  DEPENDS ${name}
                                  COMPONENT ${name})
       endif()
+
+      if(APPLE)
+        if(LLVM_EXTERNALIZE_DEBUGINFO_INSTALL)
+          if(LLVM_EXTERNALIZE_DEBUGINFO_EXTENSION)
+            set(file_ext ${LLVM_EXTERNALIZE_DEBUGINFO_EXTENSION})
+          elseif(LLVM_EXTERNALIZE_DEBUGINFO_FLATTEN)
+            set(file_ext dwarf)
+          else()
+            set(file_ext dSYM)
+          endif()
+          set(output_name "$<TARGET_FILE_NAME:${name}>.${file_ext}")
+          if(LLVM_EXTERNALIZE_DEBUGINFO_OUTPUT_DIR)
+            set(output_path "${LLVM_EXTERNALIZE_DEBUGINFO_OUTPUT_DIR}/${output_name}")
+          else()
+            set(output_path "${output_name}")
+          endif()
+          get_filename_component(debuginfo_absolute_path ${output_path} REALPATH BASE_DIR $<TARGET_FILE_DIR:${name}>)
+          install(FILES ${debuginfo_absolute_path} DESTINATION bin OPTIONAL COMPONENT ${name})
+        endif()
+      elseif(WIN32)
+        if(LLVM_EXTERNALIZE_DEBUGINFO_INSTALL)
+          install(FILES $<TARGET_PDB_FILE:${name}> DESTINATION bin OPTIONAL COMPONENT ${name})
+        endif()
+      else()
+        if(LLVM_EXTERNALIZE_DEBUGINFO_INSTALL)
+          if(LLVM_EXTERNALIZE_DEBUGINFO_EXTENSION)
+            set(file_ext ${LLVM_EXTERNALIZE_DEBUGINFO_EXTENSION})
+          else()
+            set(file_ext debug)
+          endif()
+
+          set(output_name "$<TARGET_FILE_NAME:${name}>.${file_ext}")
+
+          if(LLVM_EXTERNALIZE_DEBUGINFO_OUTPUT_DIR)
+            set(output_path "${LLVM_EXTERNALIZE_DEBUGINFO_OUTPUT_DIR}/${output_name}")
+            # If an output dir is specified, it must be manually mkdir'd on Linux,
+            # as that directory needs to exist before we can pipe to a file in it.
+            add_custom_command(TARGET ${name} POST_BUILD
+              WORKING_DIRECTORY ${LLVM_RUNTIME_OUTPUT_INTDIR}
+              COMMAND ${CMAKE_COMMAND} -E make_directory ${LLVM_EXTERNALIZE_DEBUGINFO_OUTPUT_DIR}
+              )
+          else()
+            set(output_path "${output_name}")
+          endif()
+
+          get_filename_component(debuginfo_absolute_path ${output_path} REALPATH BASE_DIR $<TARGET_FILE_DIR:${name}>)
+          install(FILES ${debuginfo_absolute_path} DESTINATION bin OPTIONAL COMPONENT ${name})
+        endif()
+      endif()
+
       set_property(GLOBAL APPEND PROPERTY CLANG_EXPORTS ${name})
     endif()
   endif()

dir.common.props

@@ -0,0 +1,85 @@
+<Project>
+
+  <!-- Set default Configuration and Platform -->
+  <PropertyGroup>
+    <BuildArch>$(__BuildArch)</BuildArch>
+    <BuildArch Condition="'$(__BuildArch)'==''">x64</BuildArch>
+    <BuildArch Condition="'$(__BuildArch)' == 'amd64'">x64</BuildArch>
+
+    <BuildType>$(__BuildType)</BuildType>
+    <BuildType Condition="'$(__BuildType)'==''">Debug</BuildType>
+    <BuildType Condition="'$(__BuildType)' == 'debug'">Debug</BuildType>
+    <BuildType Condition="'$(__BuildType)' == 'release'">Release</BuildType>
+    <BuildType Condition="'$(__BuildType)' == 'checked'">Checked</BuildType>
+
+    <BuildOS>$(__BuildOS)</BuildOS>
+    <BuildOS Condition="'$(__BuildOS)' == '' And '$([MSBuild]::IsOSPlatform(Windows))' == 'true'">Windows_NT</BuildOS>
+    <BuildOS Condition="'$(__BuildOS)' == '' And '$([MSBuild]::IsOSPlatform(Linux))' == 'true'">Linux</BuildOS>
+    <BuildOS Condition="'$(__BuildOS)' == '' And '$([MSBuild]::IsOSPlatform(OSX))' == 'true'">OSX</BuildOS>
+
+    <!-- TODO: converge on one property for BuildOS and __BuildOS (and similar), and remove these extra lines. -->
+    <__BuildOS>$(BuildOS)</__BuildOS>
+    <__BuildArch>$(BuildArch)</__BuildArch>
+
+    <Configuration Condition="'$(Configuration)' == ''">$(BuildType)</Configuration>
+    <Platform Condition="'$(Platform)' == ''">$(BuildArch)</Platform>
+
+    <PlatformConfigPathPart>$(BuildOS).$(BuildArch).$(BuildType)</PlatformConfigPathPart>
+
+  </PropertyGroup>
+
+  <!-- Common properties -->
+  <PropertyGroup>
+
+    <__ProjectDir Condition="'$(__ProjectDir)'==''">$(MSBuildThisFileDirectory)</__ProjectDir>
+    <ProjectDir>$(__ProjectDir)\</ProjectDir>
+    <RootRepoDir>$(ProjectDir)\..\..\</RootRepoDir>
+    <ProjectDir Condition="'$(__ProjectDir)'==''">$(MSBuildThisFileDirectory)</ProjectDir>
+
+    <BaseIntermediateOutputPath>$(RootRepoDir)artifacts\obj\llvm\$(MSBuildProjectName)\</BaseIntermediateOutputPath>
+
+    <SourceDir>$(__SourceDir)\</SourceDir>
+    <SourceDir Condition="'$(__SourceDir)'==''">$(ProjectDir)src\</SourceDir>
+
+    <RootBinDir>$(__RootBinDir)\</RootBinDir>
+    <RootBinDir Condition="'$(__RootBinDir)'==''">$(RootRepoDir)artifacts\</RootBinDir>
+
+    <BinDir>$(__BinDir)\</BinDir>
+    <BinDir Condition="'$(__BinDir)'==''">$(RootBinDir)bin\llvm\$(PlatformConfigPathPart)\</BinDir>
+
+    <_LLVMSourceDir Condition="'$(_LLVMSourceDir)'==''">$(MSBuildThisFileDirectory)/llvm</_LLVMSourceDir>
+    <_LLVMBuildDir Condition="'$(_LLVMBuildDir)'==''">$(MSBuildThisFileDirectory)/artifacts/obj/BuildRoot-$(TargetArchitecture)</_LLVMBuildDir>
+    <_LLVMInstallDir Condition="'$(_LLVMInstallDir)'==''">$(MSBuildThisFileDirectory)/artifacts/obj/InstallRoot-$(TargetArchitecture)</_LLVMInstallDir>
+
+    <_LibCxxSourceDir Condition="'$(_LibCxxSourceDir)'==''">$(MSBuildThisFileDirectory)/runtimes</_LibCxxSourceDir>
+    <_LibCxxBuildDir Condition="'$(_LibCxxBuildDir)'==''">$(MSBuildThisFileDirectory)/artifacts/obj/libcxx/BuildRoot-$(TargetArchitecture)</_LibCxxBuildDir>
+    <_LibCxxInstallDir Condition="'$(_LibCxxInstallDir)'==''">$(MSBuildThisFileDirectory)/artifacts/obj/libcxx/InstallRoot-$(TargetArchitecture)</_LibCxxInstallDir>
+
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <!-- Central place to set the versions of all nuget packages produced in the repo -->
+    <LLVMVersion Condition="'$(LLVMVersion)' == '' AND !Exists('$(_LLVMBuildDir)\version.txt')">1.0.0</LLVMVersion>
+    <LLVMVersion Condition="'$(LLVMVersion)' == '' AND Exists('$(_LLVMBuildDir)\version.txt')">$([System.IO.File]::ReadAllText('$(_LLVMBuildDir)\version.txt'))</LLVMVersion>
+    <PackageVersion Condition="'$(LLVMVersion)' != '' and '$(PackageVersion)' == ''">$(LLVMVersion)</PackageVersion>
+
+    <!-- Set the boolean below to true to generate packages with stabilized versions -->
+    <StabilizePackageVersion Condition="'$(StabilizePackageVersion)' == ''">false</StabilizePackageVersion>
+    <StableVersion Condition="'$(StabilizePackageVersion)' == 'true' and '$(StableVersion)' == ''">$(PackageVersion)</StableVersion>
+
+    <PreReleaseLabel>preview8</PreReleaseLabel>
+  </PropertyGroup>
+
+  <!-- Set up common target properties that we use to conditionally include sources -->
+  <PropertyGroup>
+    <TargetsFreeBSD Condition="'$(BuildOS)' == 'FreeBSD'">true</TargetsFreeBSD>
+    <TargetsLinux Condition="'$(BuildOS)' == 'Linux'">true</TargetsLinux>
+    <TargetsNetBSD Condition="'$(BuildOS)' == 'NetBSD'">true</TargetsNetBSD>
+    <TargetsOSX Condition="'$(BuildOS)' == 'OSX'">true</TargetsOSX>
+    <TargetsWindows Condition="'$(BuildOS)' == 'Windows_NT'">true</TargetsWindows>
+
+    <TargetsUnix Condition="'$(TargetsFreeBSD)' == 'true' or '$(TargetsLinux)' == 'true' or '$(TargetsNetBSD)' == 'true' or '$(TargetsOSX)' == 'true'">true</TargetsUnix>
+
+  </PropertyGroup>
+
+</Project>

eng/Build.props

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <ProjectToBuild Include="$(MSBuildThisFileDirectory)..\llvm.proj" />
+  </ItemGroup>
+</Project>

eng/Publishing.props

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project>
+   <PropertyGroup>
+      <PublishingVersion>3</PublishingVersion>
+   </PropertyGroup>
+</Project>