Enable generation of aka.ms links

This change enables automatic generate of 'latest' aka.ms links for builds that publish installers and checksums and are applied to a channel
What this does is generate an aka.ms for the blob by stripping away version numbers.  So
Runtime/5.0.0-alpha.1.20062.3/dotnet-apphost-pack-5.0.0-alpha.1.20062.3-osx-x64.pkg becomes Runtime/dotnet-apphost-pack-osx-x64.pkg. We then generate the aka.ms link as such:
dotnet/<channel>/Runtime/dotnet-apphost-pack-osx-x64.pkg -> https://dotnetfeed.blob.core.windows.net/akamslinkstest/Runtime/5.0.0-alpha.1.20062.3/dotnet-apphost-pack-5.0.0-alpha.1.20062.3-osx-x64.pkg

eng/common/templates/post-build/channels/generic-public-channel.yml

@@ -10,6 +10,8 @@ parameters:
   transportFeed: ''
   shippingFeed: ''
   symbolsFeed: ''
+  # If the channel name is empty, no links will be generated
+  akaMSChannelName: ''
 
 stages:
 - stage: ${{ parameters.stageName }}
@@ -161,6 +163,12 @@ stages:
             /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
             /p:AzureDevOpsStaticSymbolsFeed='${{ parameters.symbolsFeed }}'
             /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+            /p:LatestLinkShortUrlPrefix=dotnet/'${{ parameters.akaMSChannelName }}'
+            /p:AkaMSClientId=$(akams-client-id)
+            /p:AkaMSClientSecret=$(akams-client-secret)
+            /p:AkaMSTenant=ncd
+            /p:AkaMSOwners=dn-bot
+            /p:AkaMSCreatedBy=dn-bot
             ${{ parameters.artifactsPublishingAdditionalParameters }}
 
       - template: ../../steps/publish-logs.yml

eng/common/templates/post-build/post-build.yml

@@ -199,6 +199,7 @@ stages:
     symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
     stageName: 'NetCore_Dev5_Publish'
     channelName: '.NET Core 5 Dev'
+    akaMSChannelName: 'net5/daily'
     channelId: ${{ parameters.NetCoreDev5ChannelId }}
     transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
     shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
@@ -212,6 +213,7 @@ stages:
     symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
     stageName: 'Net5_Preview1_Publish'
     channelName: '.NET 5 Preview 1'
+    akaMSChannelName: 'net5/preview1'
     channelId: ${{ parameters.Net5Preview1ChannelId }}
     transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
     shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
@@ -225,6 +227,7 @@ stages:
     symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
     stageName: 'Net5_Preview2_Publish'
     channelName: '.NET 5 Preview 2'
+    akaMSChannelName: 'net5/preview2'
     channelId: ${{ parameters.Net5Preview2ChannelId }}
     transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
     shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
@@ -238,6 +241,7 @@ stages:
     symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
     stageName: 'Net_Eng_Latest_Publish'
     channelName: '.NET Eng - Latest'
+    akaMSChannelName: 'eng/daily'
     channelId: ${{ parameters.NetEngLatestChannelId }}
     transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
     shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
@@ -251,6 +255,7 @@ stages:
     symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
     stageName: 'Net_Eng_Validation_Publish'
     channelName: '.NET Eng - Validation'
+    akaMSChannelName: 'eng/validation'
     channelId: ${{ parameters.NetEngValidationChannelId }}
     transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
     shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
@@ -264,6 +269,7 @@ stages:
     symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
     stageName: 'General_Testing_Publish'
     channelName: 'General Testing'
+    akaMSChannelName: 'generaltesting'
     channelId: ${{ parameters.GeneralTestingChannelId }}
     transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json'
     shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json'

src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj

@@ -72,7 +72,12 @@
       AssetManifestPath="%(ManifestFiles.Identity)"
       BlobAssetsBasePath="$(BlobBasePath)"
       PackageAssetsBasePath="$(PackageBasePath)"
-      NugetPath="$(NugetPath)"/>
+      NugetPath="$(NugetPath)"
+      AkaMSClientId="$(AkaMSClientId)"
+      AkaMSClientSecret="$(AkaMSClientSecret)"
+      AkaMSTenant="$(AkaMSTenant)"
+      AkaMsOwners="$(AkaMsOwners)"
+      AkaMSCreatedBy="$(AkaMSCreatedBy)" />
   </Target>
 
   <ItemGroup>

src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/SetupTargetFeeds.proj

@@ -48,6 +48,7 @@
       - AzureDevOpsStaticTransportFeedKey     : Key of the Azure DevOps NuGet feed to publish to transport packages to.
       - AzureDevOpsStaticSymbolsFeed          : URL of the Azure DevOps NuGet feed to publish symbol packages to.
       - AzureDevOpsStaticSymbolsFeedKey       : Key of the Azure DevOps NuGet feed to publish to symbol packages to.
+      - LatestLinkShortUrlPrefix              : Prefix of aka.ms url's to set up latest links for each published blob.
 
       - CreateTestConfig                      : If set to true the user will be able to test a TargetFeedConfig
                                                 constructed using parameters below:
@@ -197,7 +198,7 @@
       <TargetFeedConfig
         Condition="'$(IsInternalBuild)' == 'false'"
         AssetSelection="NonShippingOnly"
-        Include="PACKAGE;SYMBOLS"
+        Include="Package;Symbols"
         TargetURL="$(TargetStaticFeed)"
         Isolated="false"
         Type="AzureStorageFeed"
@@ -317,14 +318,16 @@
         TargetURL="$(InstallersTargetStaticFeed)"
         Isolated="false"
         Type="AzureStorageFeed"
-        Token="$(InstallersAzureAccountKey)" />
+        Token="$(InstallersAzureAccountKey)"
+        LatestLinkShortUrlPrefix="$(LatestLinkShortUrlPrefix)" />
 
       <TargetFeedConfig Condition="'$(PublishInstallersAndChecksums)' == 'true'"
         Include="Checksum"
         TargetURL="$(ChecksumsTargetStaticFeed)"
         Isolated="false"
         Type="AzureStorageFeed"
-        Token="$(ChecksumsAzureAccountKey)" />
+        Token="$(ChecksumsAzureAccountKey)"
+        LatestLinkShortUrlPrefix="$(LatestLinkShortUrlPrefix)" />
 
       <TargetFeedConfig
         Include="Package"

src/Microsoft.DotNet.Build.Tasks.Feed/Microsoft.DotNet.Build.Tasks.Feed.csproj

@@ -21,6 +21,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\Microsoft.DotNet.Deployment.Tasks.Links\Microsoft.DotNet.Deployment.Tasks.Links.csproj" />
     <ProjectReference Include="..\Microsoft.DotNet.VersionTools\lib\Microsoft.DotNet.VersionTools.csproj" />
   </ItemGroup>
 

src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs

@@ -4,8 +4,10 @@
 
 using Microsoft.Build.Framework;
 using Microsoft.DotNet.Build.CloudTestTasks;
+using Microsoft.DotNet.Deployment.Tasks.Links.src;
 using Microsoft.DotNet.Maestro.Client;
 using Microsoft.DotNet.Maestro.Client.Models;
+using Microsoft.DotNet.VersionTools.BuildManifest;
 using Microsoft.DotNet.VersionTools.BuildManifest.Model;
 using Microsoft.DotNet.VersionTools.Util;
 using NuGet.Packaging.Core;
@@ -30,6 +32,8 @@ namespace Microsoft.DotNet.Build.Tasks.Feed
     /// </summary>
     public class PublishArtifactsInManifest : MSBuild.Task
     {
+        const string FeedExpectedSuffix = "index.json";
+
         // Matches package feeds like
         // https://dotnet-feed-internal.azurewebsites.net/container/dotnet-core-internal/sig/dsdfasdfasdf234234s/se/2020-02-02/darc-int-dotnet-arcade-services-babababababe-08/index.json
         const string AzureStorageProxyFeedPattern =
@@ -60,6 +64,23 @@ public class PublishArtifactsInManifest : MSBuild.Task
         /// Metadata TargetURL: target URL where assets of this category should be published to.
         /// Metadata Type: type of the target feed.
         /// Metadata Token: token to be used for publishing to target feed.
+        /// Metadata AssetSelection (optional): Can be "All", "ShippingOnly" or "NonShippingOnly"
+        ///                                     Determines which assets are pushed to this feed config
+        /// Metadata Internal (optional): If true, the feed is assumed to be only internally visible.
+        ///                               If false, the feed is public.
+        ///                               If not provided, then this task will attempt to determine whether the feed URL is publicly visible or not.
+        /// Metadata Isolated (optional): If true, the feed is assumed to be isolated, and stable packages can be pushed to it.
+        ///                               If false, the feed is assumed to be non-isolated, and stable packages will be rejected.
+        ///                               If not provided then defaults to false.
+        /// Metadata AllowOverwrite (optional): If true, existing azure blob storage assets can be overwritten
+        ///                                     If false, an error is thrown if an asset already exists
+        ///                                     If not provided then defaults to false.
+        ///                                     Azure DevOps feeds can never be overwritten.
+        /// Metadata LatestLinkShortUrlPrefix (optional): If provided, AKA ms links are generated (for artifacts blobs only)
+        ///                                               that target this short url path. The link is construct as such:
+        ///                                               aka.ms/AkaShortUrlPath/BlobArtifactPath -> Target blob url
+        ///                                               If specified, then AkaMSClientId, AkaMSClientSecret and AkaMSTenant must be provided.
+        ///                                               The version information is stripped away from the file and blob artifact path.
         /// </summary>
         [Required]
         public ITaskItem[] TargetFeedConfig { get; set; }
@@ -126,6 +147,16 @@ public class PublishArtifactsInManifest : MSBuild.Task
         /// </summary>
         public bool SkipSafetyChecks { get; set; } = false;
 
+        #region Information for AKA MS link generation
+
+        public string AkaMSClientId { get; set; }
+        public string AkaMSClientSecret { get; set; }
+        public string AkaMSTenant { get; set; }
+        public string AkaMsOwners { get; set; }
+        public string AkaMSCreatedBy { get; set; }
+        public string AkaMSGroupOwner { get; set; }
+        #endregion
+
         public readonly Dictionary<string, List<FeedConfig>> FeedConfigs = new Dictionary<string, List<FeedConfig>>();
 
         private readonly Dictionary<string, List<PackageArtifactModel>> PackagesByCategory = new Dictionary<string, List<PackageArtifactModel>>();
@@ -211,6 +242,12 @@ public async Task ParseTargetFeedConfigAsync()
                         continue;
                     }
 
+                    if (!targetFeedUrl.EndsWith(FeedExpectedSuffix))
+                    {
+                        Log.LogError($"Exepcted that feed '{targetFeedUrl}' would end in {FeedExpectedSuffix}");
+                        continue;
+                    }
+
                     if (!Enum.TryParse<FeedType>(type, true, out FeedType feedType))
                     {
                         Log.LogError($"Invalid feed config type '{type}'. Possible values are: {string.Join(", ", Enum.GetNames(typeof(FeedType)))}");
@@ -284,6 +321,23 @@ public async Task ParseTargetFeedConfigAsync()
                         feedConfig.AllowOverwrite = feedSetting;
                     }
 
+                    string latestLinkShortUrlPrefix = fc.GetMetadata(nameof(FeedConfig.LatestLinkShortUrlPrefix));
+                    if (!string.IsNullOrEmpty(latestLinkShortUrlPrefix))
+                    {
+                        // Verify other inputs are provided
+                        if (string.IsNullOrEmpty(AkaMSClientId) ||
+                            string.IsNullOrEmpty(AkaMSClientSecret) ||
+                            string.IsNullOrEmpty(AkaMSTenant) ||
+                            string.IsNullOrEmpty(AkaMsOwners))
+                        {
+                            Log.LogError($"If a short url path is provided, please provide {nameof(AkaMSClientId)}, {nameof(AkaMSClientSecret)}, " +
+                                $"{nameof(AkaMSTenant)}, {nameof(AkaMsOwners)}, {nameof(AkaMSCreatedBy)}");
+                            continue;
+                        }
+                        feedConfig.LatestLinkShortUrlPrefix = latestLinkShortUrlPrefix;
+                    }
+
+
                     string categoryKey = fc.ItemSpec.Trim().ToUpper();
                     if (!FeedConfigs.TryGetValue(categoryKey, out var feedsList))
                     {
@@ -1073,9 +1127,9 @@ private async Task PublishBlobsToAzureStorageNugetFeedAsync(
                 PassIfExistingItemIdentical = true
             };
 
-            foreach (var blob in blobsToPublish)
+            foreach (BlobArtifactModel blob in blobsToPublish)
             {
-                var assetRecord = buildInformation.Assets
+                Asset assetRecord = buildInformation.Assets
                     .Where(a => a.Name.Equals(blob.Id))
                     .SingleOrDefault();
 
@@ -1089,6 +1143,48 @@ private async Task PublishBlobsToAzureStorageNugetFeedAsync(
             }
 
             await blobFeedAction.PublishToFlatContainerAsync(blobs, maxClients: MaxClients, pushOptions);
+            await CreateOrUpdateLatestLinksAsync(blobsToPublish, feedConfig);
+        }
+
+        private async Task CreateOrUpdateLatestLinksAsync(List<BlobArtifactModel> blobsToPublish, FeedConfig feedConfig)
+        {
+            // Update or create aka ms links if desired.
+            if (!string.IsNullOrEmpty(feedConfig.LatestLinkShortUrlPrefix))
+            {
+                Log.LogMessage(MessageImportance.High, "The following aka.ms links for blobs will be created:");
+                List<AkaMSLink> linksToCreate = blobsToPublish.Select(blob =>
+                {
+                    // Strip away the feed expected suffix (index.json) and append on the
+                    // blob path.
+                    string actualTargetUrl = feedConfig.TargetURL.Substring(0,
+                        feedConfig.TargetURL.Length - FeedExpectedSuffix.Length) + blob.Id;
+
+                    AkaMSLink newLink = new AkaMSLink
+                    {
+                        ShortUrl = GetLatestShortUrlForBlob(feedConfig, blob),
+                        TargetUrl = actualTargetUrl
+                    };
+                    Log.LogMessage(MessageImportance.High, $"  {newLink.ShortUrl} -> {newLink.TargetUrl}");
+                    return newLink;
+                }).ToList();
+
+                AkaMSLinkManager linkManager = new AkaMSLinkManager(AkaMSClientId, AkaMSClientSecret, AkaMSTenant, Log);
+                await linkManager.CreateOrUpateLinksAsync(linksToCreate, AkaMsOwners, AkaMSCreatedBy, AkaMSGroupOwner, true);
+            }
+        }
+
+        /// <summary>
+        ///     Get the short url for a blob.
+        /// </summary>
+        /// <param name="feedConfig">Feed configuration</param>
+        /// <param name="blob">Blob</param>
+        /// <returns>Short url prefix for the blob.</returns>
+        /// <remarks>
+        public string GetLatestShortUrlForBlob(FeedConfig feedConfig, BlobArtifactModel blob)
+        {
+            string blobIdWithoutVersions = VersionIdentifier.RemoveVersions(blob.Id);
+
+            return Path.Combine(feedConfig.LatestLinkShortUrlPrefix, blobIdWithoutVersions).Replace("\\", "/");
         }
 
         private BlobFeedAction CreateBlobFeedAction(FeedConfig feedConfig)
@@ -1230,5 +1326,13 @@ public class FeedConfig
         /// valid for azure blob storage feeds.
         /// </summary>
         public bool AllowOverwrite { get; set; } = false;
+        /// <summary>
+        /// Prefix of aka.ms links that should be generated for blobs.
+        /// Not applicable to packages
+        /// Generates a link the blob, stripping away any version information in the file or blob path.
+        /// E.g. 
+        ///      [LatestLinkShortUrl]/aspnetcore/Runtime/dotnet-hosting-win.exe -> aspnetcore/Runtime/3.1.0-preview2.19511.6/dotnet-hosting-3.1.0-preview2.19511.6-win.exe
+        /// </summary>
+        public string LatestLinkShortUrlPrefix { get; set; }
     }
 }

src/Microsoft.DotNet.Deployment.Tasks.Links/Microsoft.DotNet.Deployment.Tasks.Links.csproj

@@ -24,6 +24,9 @@
   <ItemGroup>
     <Content Include="build\*" PackagePath="build\" />
   </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Microsoft.DotNet.VersionTools\lib\Microsoft.DotNet.VersionTools.csproj" />
+  </ItemGroup>
 
   <!-- 
     Specify binaries to include in NuGet package 

src/Microsoft.DotNet.Deployment.Tasks.Links/src/AkaMSLinksBase.cs

@@ -0,0 +1,24 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Net.Http;
+using System.Threading.Tasks;
+using Microsoft.Build.Framework;
+using Microsoft.IdentityModel.Clients.ActiveDirectory;
+
+namespace Microsoft.DotNet.Deployment.Tasks.Links
+{
+    public abstract class AkaMSLinksBase : Microsoft.Build.Utilities.Task
+    {
+        [Required]
+        // Authentication data
+        public string ClientId { get; set; }
+        [Required]
+        // Authentication data
+        public string ClientSecret { get; set; }
+        [Required]
+        public string Tenant { get; set; }
+    }
+}