More tries to support .dSYM

Treat .dSYM bundle as set of files. There are places where this doesn't
work very well, such as in src/installer/pkg/projects/Directory.Build.targets,
where the `GetSymbolPackageFiles` target is trying to construct a set of
debug files to package, while also giving them a TargetPath property.
I haven't been able to figure out how to get MSBuild to do my bidding
in creating the right paths.

eng/liveBuilds.targets

@@ -24,7 +24,9 @@
 
   <!-- Set up artifact subpaths. -->
   <PropertyGroup>
+    <CoreCLRArtifactsPdbDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'PDB'))</CoreCLRArtifactsPdbDir>
     <CoreCLRSharedFrameworkDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'sharedFramework'))</CoreCLRSharedFrameworkDir>
+    <CoreCLRSharedFrameworkPdbDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRSharedFrameworkDir)', 'PDB'))</CoreCLRSharedFrameworkPdbDir>
     <CoreCLRILCompilerDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'ilc-published'))</CoreCLRILCompilerDir>
     <CoreCLRCrossILCompilerDir Condition="'$(CrossBuild)' == 'true' or '$(BuildArchitecture)' != '$(TargetArchitecture)' or '$(HostOS)' != '$(TargetOS)' or '$(EnableNativeSanitizers)' != ''">$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', '$(BuildArchitecture)', 'ilc'))</CoreCLRCrossILCompilerDir>
     <CoreCLRAotSdkDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'aotsdk'))</CoreCLRAotSdkDir>
@@ -67,22 +69,10 @@
            Text="The Mono artifacts path does not exist '$(MonoArtifactsPath)'. The 'mono' subset must be built before building this project. Configuration: '$(MonoConfiguration)'. To use a different configuration, specify the 'RuntimeConfiguration' property." />
 
     <PropertyGroup Condition="'$(RuntimeFlavor)' == 'CoreCLR'">
-      <CoreCLRArtifactsPath>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)'))</CoreCLRArtifactsPath>
-      <CoreCLRArtifactsPdbDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)','PDB'))</CoreCLRArtifactsPdbDir>
-      <!--
-        Even though CoreCLRSharedFrameworkDir is statically initialized, set it again in case the
-        value is different after CoreCLRArtifactsPath is normalized.
-      -->
-      <CoreCLRSharedFrameworkDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)','sharedFramework'))</CoreCLRSharedFrameworkDir>
-      <CoreCLRSharedFrameworkPdbDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRSharedFrameworkDir)','PDB'))</CoreCLRSharedFrameworkPdbDir>
       <CoreCLRCrossTargetComponentDir
         Condition="'$(CoreCLRCrossTargetComponentDirName)' != ''">$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)','$(CoreCLRCrossTargetComponentDirName)','sharedFramework'))</CoreCLRCrossTargetComponentDir>
     </PropertyGroup>
 
-    <PropertyGroup Condition="'$(RuntimeFlavor)' == 'Mono'">
-      <MonoArtifactsPath>$([MSBuild]::NormalizeDirectory('$(MonoArtifactsPath)'))</MonoArtifactsPath>
-    </PropertyGroup>
-
     <ItemGroup Condition="'$(BuildNativeAOTRuntimePack)' == 'true'">
       <_nativeAotRuntimeFiles Include="$(CoreCLRAotSdkDir)*" />
       <RuntimeFiles Include="@(_nativeAotRuntimeFiles)" Condition="'%(Extension)' != '.xml'">
@@ -97,18 +87,22 @@
       <RuntimeFiles>
         <IsNative>true</IsNative>
       </RuntimeFiles>
+
+      <!-- Include System.Private.CoreLib.dll if it exists; get the one in the 'IL' subdirectory if the one in the root directory is missing. -->
       <_systemPrivateCoreLib Include="$(CoreCLRArtifactsPath)System.Private.CoreLib.dll"
                              Condition="Exists('$(CoreCLRArtifactsPath)System.Private.CoreLib.dll')" />
       <_systemPrivateCoreLib Include="$(CoreCLRArtifactsPath)IL/System.Private.CoreLib.dll"
                              Condition="Exists('$(CoreCLRArtifactsPath)IL/System.Private.CoreLib.dll') and '@(_systemPrivateCoreLib)' == ''" />
       <RuntimeFiles Include="@(_systemPrivateCoreLib)" />
+
+      <!-- Include debugging symbols -->
       <RuntimeFiles
         Include="
           $(CoreCLRSharedFrameworkPdbDir)*.pdb;
           $(CoreCLRSharedFrameworkPdbDir)*.dbg;
           $(CoreCLRSharedFrameworkPdbDir)*.dwarf;
-          $(CoreCLRSharedFrameworkPdbDir)*.dSYM"
-          IsNative="true" />
+          $(CoreCLRSharedFrameworkPdbDir)*.dSYM/**"
+        IsNative="true" />
       <RuntimeFiles Condition="Exists('$(CoreCLRArtifactsPdbDir)System.Private.CoreLib.pdb')"
         Include="$(CoreCLRArtifactsPdbDir)System.Private.CoreLib.pdb" />
       <RuntimeFiles Condition="Exists('$(CoreCLRArtifactsPdbDir)System.Private.CoreLib.ni.pdb')"
@@ -119,7 +113,7 @@
           $(CoreCLRSharedFrameworkPdbDir)*.pdb;
           $(CoreCLRSharedFrameworkPdbDir)*.dbg;
           $(CoreCLRSharedFrameworkPdbDir)*.dwarf;
-          $(CoreCLRSharedFrameworkPdbDir)*.dSYM" />
+          $(CoreCLRSharedFrameworkPdbDir)*.dSYM/**" />
 
       <CoreCLRCrossTargetFiles>
         <TargetPath>runtime/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture)/native</TargetPath>
@@ -195,7 +189,7 @@
         $(LibrariesNativeArtifactsPath)*.so;
         $(LibrariesNativeArtifactsPath)*.dbg;
         $(LibrariesNativeArtifactsPath)*.dwarf;
-        $(LibrariesNativeArtifactsPath)*.dSYM;
+        $(LibrariesNativeArtifactsPath)*.dSYM/**;
         $(LibrariesNativeArtifactsPath)*.pdb"
         IsNative="true"
         Exclude="@(ExcludeNativeLibrariesRuntimeFiles)" />

eng/native/functions.cmake

@@ -369,7 +369,6 @@ endfunction()
 function (get_symbol_file_name targetName outputSymbolFilename)
   if (CLR_CMAKE_HOST_UNIX)
     if (CLR_CMAKE_TARGET_APPLE)
-      # set(strip_destination_file $<TARGET_FILE:${targetName}>.dwarf)
       set(strip_destination_file $<TARGET_FILE:${targetName}>.dSYM)
     else ()
       set(strip_destination_file $<TARGET_FILE:${targetName}>.dbg)
@@ -417,7 +416,6 @@ function(strip_symbols targetName outputFilename)
         OUTPUT_VARIABLE DSYMUTIL_HELP_OUTPUT
       )
 
-      # set(DSYMUTIL_OPTS "--flat")
       set(DSYMUTIL_OPTS "")
       if ("${DSYMUTIL_HELP_OUTPUT}" MATCHES "--minimize")
         list(APPEND DSYMUTIL_OPTS "--minimize")

eng/native/naming.props

@@ -11,15 +11,18 @@
         <LibSuffix>.dll</LibSuffix>
         <StaticLibSuffix>.lib</StaticLibSuffix>
         <SymbolsSuffix>.pdb</SymbolsSuffix>
+        <SymbolsSuffixPath>.pdb</SymbolsSuffixPath>
+        <SymbolsIsDirectory>false</SymbolsIsDirectory>
       </PropertyGroup>
     </When>
     <When Condition="$(PackageRID.StartsWith('osx')) or $(PackageRID.StartsWith('maccatalyst')) or $(PackageRID.StartsWith('ios')) or $(PackageRID.StartsWith('tvos'))">
       <PropertyGroup>
         <LibPrefix>lib</LibPrefix>
         <LibSuffix>.dylib</LibSuffix>
         <StaticLibSuffix>.a</StaticLibSuffix>
-        <!-- <SymbolsSuffix>.dwarf</SymbolsSuffix> -->
         <SymbolsSuffix>.dSYM</SymbolsSuffix>
+        <SymbolsSuffixPath>.dSYM/**</SymbolsSuffixPath>
+        <SymbolsIsDirectory>true</SymbolsIsDirectory>
       </PropertyGroup>
     </When>
     <Otherwise>
@@ -28,12 +31,15 @@
         <LibSuffix>.so</LibSuffix>
         <StaticLibSuffix>.a</StaticLibSuffix>
         <SymbolsSuffix>.dbg</SymbolsSuffix>
+        <SymbolsSuffixPath>.dbg</SymbolsSuffixPath>
+        <SymbolsIsDirectory>false</SymbolsIsDirectory>
       </PropertyGroup>
     </Otherwise>
   </Choose>
 
   <ItemGroup>
-    <AdditionalLibPackageExcludes Condition="'$(SymbolsSuffix)' != ''" Include="%2A%2A\%2A$(SymbolsSuffix)" />
+    <!-- %2A is * so below is "**\*.pdb" for example -->
+    <AdditionalLibPackageExcludes Condition="'$(SymbolsSuffixPath)' != ''" Include="%2A%2A\%2A$(SymbolsSuffixPath)" />
     <AdditionalSymbolPackageExcludes Condition="'$(LibSuffix)' != ''" Include="%2A%2A\%2A.a;%2A%2A\%2A$(LibSuffix)" />
   </ItemGroup>
 

src/coreclr/tools/aot/crossgen2/crossgen2_publish.csproj

@@ -51,9 +51,6 @@
     <IlcFrameworkPath>$(MicrosoftNetCoreAppRuntimePackRidLibTfmDir)</IlcFrameworkPath>
     <IlcFrameworkNativePath>$(MicrosoftNetCoreAppRuntimePackNativeDir)</IlcFrameworkNativePath>
     <TrimmerSingleWarn>false</TrimmerSingleWarn>
-    <!-- Use .dwarf files instead of .dsym files since our symbol exporting may not safely handle folders. -->
-    <!-- <NativeSymbolExt Condition="'$(_IsApplePlatform)' == 'true'">.dwarf</NativeSymbolExt> -->
-    <!-- <DsymUtilOptions Condition="'$(_IsApplePlatform)' == 'true'">--flat</DsymUtilOptions> -->
   </PropertyGroup>
 
   <ItemGroup Condition="'$(NativeAotSupported)' == 'true'">

src/installer/pkg/archives/dotnet-nethost.proj

@@ -34,7 +34,15 @@
           Condition="'$(TargetsWindows)' == 'true'"
           DestinationFolder="$(SymbolsOutputPath)" />
     <Copy SourceFiles="$(DotNetHostBinDir)\$(LibPrefix)nethost$(LibSuffix)$(SymbolsSuffix)"
-          Condition="'$(TargetsWindows)' != 'true'"
+          Condition="('$(TargetsWindows)' != 'true') and ('$(SymbolsIsDirectory)' != 'true')"
           DestinationFolder="$(SymbolsOutputPath)" />
+
+    <!-- Do some magic to tree copy the symbols if they represent a directory instead of a file (e.g., .dSYM) -->
+    <ItemGroup Condition="('$(TargetsWindows)' != 'true') and ('$(SymbolsIsDirectory)' == 'true')">
+        <RecursiveSourceFiles Include="$(DotNetHostBinDir)\$(LibPrefix)nethost$(LibSuffix)$(SymbolsSuffix)/**/*"/>
+    </ItemGroup>
+    <Copy SourceFiles="@(RecursiveSourceFiles)"
+          Condition="('$(TargetsWindows)' != 'true') and ('$(SymbolsIsDirectory)' == 'true')"
+          DestinationFolder="$(SymbolsOutputPath)/%(RecursiveDir)" />
   </Target>
 </Project>

src/installer/pkg/projects/Directory.Build.targets

@@ -9,8 +9,8 @@
 
     <!--
       Explicitly re-set the symbol package output path. The pkgproj files here end up importing the targets from
-      Microsoft.DotNet.Build.Tasks.Packaging (based on a PackageReference) before importing the Arcade targets hat
-      set defaults for project output paths. This means the value set by the packaging targets does not accountfor
+      Microsoft.DotNet.Build.Tasks.Packaging (based on a PackageReference) before importing the Arcade targets that
+      set defaults for project output paths. This means the value set by the packaging targets does not account for
       the updated defaults from the Arcade targets.
     -->
     <SymbolPackageOutputPath>$(PackageOutputPath)</SymbolPackageOutputPath>
@@ -95,8 +95,7 @@
       -->
       <NonWindowsNativeFile Include="@(NativeWithSymbolFile)"
                             Exclude="@(WindowsNativeFile)" />
-
-      <NonWindowsSymbolFile Include="@(NonWindowsNativeFile -> '%(Identity)$(SymbolsSuffix)')" />
+      <NonWindowsSymbolFile Include="@(NonWindowsNativeFile -> '%(Identity)$(SymbolsSuffix)')</NonWindowsSymbolFile>
 
       <ExistingWindowsSymbolFile Include="@(WindowsSymbolFile)" Condition="Exists('%(Identity)')" />
       <ExistingNonWindowsSymbolFile Include="@(NonWindowsSymbolFile)" Condition="Exists('%(Identity)') AND '$(SkipPackagingXplatSymbols)'!='true'" />

src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj

@@ -73,6 +73,7 @@
       <_Crossgen2SymbolFilesToPackage Include="@(_CrossgenPublishFiles)" Condition="'%(Extension)' == '.pdb'" />
       <!-- Symbol files for JIT libraries are placed in a different location for Windows builds -->
       <_Crossgen2SymbolFilesToPackage Include="@(NativeRuntimeAsset->'$(CoreCLRArtifactsPdbDir)%(FileName).pdb')" Condition="'$(TargetOS)' == 'windows' and '%(FileName)' != 'crossgen2'" />
+      <!-- TODO: handle .dSYM -->
       <_Crossgen2SymbolFilesToPackage Include="@(NativeRuntimeAsset->'$(CoreCLRArtifactsPath)%(FileName)%(Extension)$(SymbolsSuffix)')" Condition="'$(TargetOS)' != 'windows' and '%(FileName)' != 'crossgen2'" />
       <_Crossgen2SymbolFilesToPackage Remove="@(_Crossgen2SymbolFilesToPackage)" Condition="!Exists('%(Identity)')" />
 

src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Host.sfxproj

@@ -47,25 +47,23 @@
     <NativeRuntimeAsset Include="@(_SymbolsToIncludeAlways)" IncludeAlways="true" />
   </ItemGroup>
 
-  <ItemGroup>
-    <_SymbolFiles Condition="'$(TargetOS)' == 'windows'"
-                    Include="@(NativeRuntimeAsset->'%(RootDir)%(Directory)PDB/%(Filename).pdb')"
-                    IsSymbolFile="true"
-                    IsNative="true" />
-    <_SymbolFiles Condition="'$(TargetOS)' != 'windows'"
-                    Include="@(NativeRuntimeAsset->'%(RootDir)%(Directory)%(Filename)%(Extension)$(SymbolsSuffix)')"
-                    IsSymbolFile="true"
-                    IsNative="true" />
-
+  <ItemGroup Condition="'$(TargetOS)' == 'windows'">
+    <_SymbolFilesBase Condition="'$(TargetOS)' == 'windows'"
+                      Include="@(NativeRuntimeAsset->'%(RootDir)%(Directory)PDB/%(Filename).pdb')" />
+    <_SymbolFilesBase Condition="'$(TargetOS)' != 'windows'"
+                      Include="@(NativeRuntimeAsset->'%(RootDir)%(Directory)%(Filename)%(Extension)$(SymbolsSuffixPath)')" />
+
     <!-- dotnet doesn't have its own package, so it uses this one as a means to publish to symbol servers -->
-    <_SymbolFiles Condition="'$(TargetOS)' == 'windows'"
-                    Include="$(DotNetHostBinDir)PDB/dotnet.pdb"
-                    IsSymbolFile="true"
-                    IsNative="true" />
-    <_SymbolFiles Condition="'$(TargetOS)' != 'windows'"
-                    Include="$(DotNetHostBinDir)dotnet$(ExeSuffix)$(SymbolsSuffix)"
-                    IsSymbolFile="true"
-                    IsNative="true" />
+    <_SymbolFilesBase Condition="'$(TargetOS)' == 'windows'"
+                      Include="$(DotNetHostBinDir)PDB/dotnet.pdb" />
+    <_SymbolFilesBase Condition="'$(TargetOS)' != 'windows'"
+                      Include="$(DotNetHostBinDir)dotnet$(ExeSuffix)$(SymbolsSuffixPath)" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <_SymbolFiles Include="@(_SymbolFilesBase)"
+                  IsSymbolFile="true"
+                  IsNative="true" />
 
     <!-- Remove symbols that are explicitly shipped with implementation package -->
     <_SymbolFiles Remove="@(_SymbolsToIncludeAlways)" />

src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.props

@@ -72,6 +72,7 @@
     <_HostFiles Include="$(DotNetHostBinDir)/$(LibPrefix)hostfxr$(LibSuffix)" PackOnly="true" />
     <NativeRuntimeAsset Include="@(_HostFiles)" />
 
+    <!-- TODO: handle .dSYM -->
     <_HostSymbolFiles Include="@(_HostFiles->'%(RootDir)%(Directory)PDB/%(Filename)$(SymbolsSuffix)')" Condition="'$(TargetOS)' == 'windows'" />
     <_HostSymbolFiles Include="@(_HostFiles->'%(RootDir)%(Directory)%(Filename)%(Extension)$(SymbolsSuffix)')" Condition="'$(TargetOS)' != 'windows'" />
     <_SymbolFilesToPackage Include="@(_HostSymbolFiles->Exists())" IsNative="true" />
@@ -80,9 +81,12 @@
   <Target Name="AddRuntimeFilesToPackage" DependsOnTargets="ResolveRuntimeFilesFromLocalBuild">
     <ItemGroup>
       <RuntimeFiles Condition="'%(RuntimeFiles.IsNative)' == 'true'">
-        <TargetPath>runtimes/$(RuntimeIdentifier)/native</TargetPath>
+        <TargetPath>runtimes/$(RuntimeIdentifier)/native/%(RecursiveDir)</TargetPath>
       </RuntimeFiles>
     </ItemGroup>
+    <Message Text="RuntimeFiles START =====================================================================================" Importance="High"/>
+    <Message Text="RuntimeFiles %(Identity) RecursiveDir: %(RuntimeFiles.RecursiveDir) TargetPath: %(RuntimeFiles.TargetPath)" Importance="High"/>
+    <Message Text="RuntimeFiles END =====================================================================================" Importance="High"/>
     <ItemGroup Condition="'$(BuildNativeAOTRuntimePack)' != 'true'">
       <RuntimeFiles Condition="'%(FileName)' == 'crossgen'">
         <TargetPath>tools</TargetPath>
@@ -121,11 +125,25 @@
       <CoreCLRCrossTargetFiles Condition="'%(FileName)%(Extension)' == 'mscordbi.dll' and '$(TargetsWindows)' == 'true'">
         <TargetPath>tools/$(CoreCLRCrossTargetComponentDirName)_$(TargetArchitecture)</TargetPath>
       </CoreCLRCrossTargetFiles>
+
       <CoreCLROptimizationFiles Include="$(CoreCLRArtifactsPath)StandardOptimizationData.mibc"
                                 Condition="Exists('$(CoreCLRArtifactsPath)StandardOptimizationData.mibc')">
         <TargetPath>tools</TargetPath>
       </CoreCLROptimizationFiles>
     </ItemGroup>
+
+    <Message Text="RuntimeFiles START =====================================================================================" Importance="High"/>
+    <Message Text="RuntimeFiles %(Identity) RecursiveDir: %(RuntimeFiles.RecursiveDir) TargetPath: %(RuntimeFiles.TargetPath)" Importance="High"/>
+    <Message Text="RuntimeFiles END =====================================================================================" Importance="High"/>
+
+    <Message Text="CoreCLRCrossTargetFiles START =====================================================================================" Importance="High"/>
+    <Message Text="CoreCLRCrossTargetFiles %(Identity) RecursiveDir: %(CoreCLRCrossTargetFiles.RecursiveDir) TargetPath: %(CoreCLRCrossTargetFiles.TargetPath)" Importance="High"/>
+    <Message Text="CoreCLRCrossTargetFiles END =====================================================================================" Importance="High"/>
+
+    <Message Text="CoreCLROptimizationFiles START =====================================================================================" Importance="High"/>
+    <Message Text="CoreCLROptimizationFiles %(Identity) RecursiveDir: %(CoreCLROptimizationFiles.RecursiveDir) TargetPath: %(CoreCLROptimizationFiles.TargetPath)" Importance="High"/>
+    <Message Text="CoreCLROptimizationFiles END =====================================================================================" Importance="High"/>
+
     <ItemGroup>
       <ReferenceCopyLocalPaths Include="@(RuntimeFiles);@(CoreCLRCrossTargetFiles);@(CoreCLROptimizationFiles)" />
     </ItemGroup>

src/installer/pkg/sfx/installers/dotnet-host.proj

@@ -75,6 +75,7 @@
           Condition="'$(TargetOS)' == 'windows'"
           DestinationFolder="$(SymbolsOutputPath)" />
 
+    <!-- TODO: handle .dSYM -->
     <Copy SourceFiles="$(DotNetHostBinDir)\dotnet$(SymbolsSuffix)"
           Condition="'$(TargetOS)' != 'windows'"
           DestinationFolder="$(SymbolsOutputPath)" />

src/installer/pkg/sfx/installers/dotnet-hostfxr.proj

@@ -34,6 +34,7 @@
           Condition="'$(TargetOS)' == 'windows'"
           DestinationFolder="$(SymbolsOutputPath)" />
 
+    <!-- TODO: handle .dSYM -->
     <Copy SourceFiles="$(DotNetHostBinDir)\$(LibPrefix)hostfxr$(LibSuffix)$(SymbolsSuffix)"
           Condition="'$(TargetOS)' != 'windows'"
           DestinationFolder="$(SymbolsOutputPath)" />