Skip to content

Commit

Permalink
Add support for dotnetbuilds-published dotnet cli packages (#8162)
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexperovich
alexperovich authored Nov 10, 2021
1 parent 6818f9b commit 0750fc9
Show file tree
Hide file tree
Showing 3 changed files with 145 additions and 90 deletions.
55 changes: 34 additions & 21 deletions eng/common/tools.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,31 +301,44 @@ function InstallDotNet([string] $dotnetRoot,
if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles }
if ($noPath) { $installParameters.NoPath = $True }

try {
& $installScript @installParameters
}
catch {
if ($runtimeSourceFeed -or $runtimeSourceFeedKey) {
Write-Host "Failed to install dotnet from public location. Trying from '$runtimeSourceFeed'"
if ($runtimeSourceFeed) { $installParameters.AzureFeed = $runtimeSourceFeed }
$variations = @()
$variations += @($installParameters)

if ($runtimeSourceFeedKey) {
$decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey)
$decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes)
$installParameters.FeedCredential = $decodedString
}
$dotnetBuilds = $installParameters.Clone()
$dotnetbuilds.AzureFeed = "https://dotnetbuilds.azureedge.net/public"
$variations += @($dotnetBuilds)

try {
& $installScript @installParameters
}
catch {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from custom location '$runtimeSourceFeed'."
ExitWithExitCode 1
}
if ($runtimeSourceFeed) {
$runtimeSource = $installParameters.Clone()
$runtimeSource.AzureFeed = $runtimeSourceFeed
if ($runtimeSourceFeedKey) {
$decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey)
$decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes)
$runtimeSource.FeedCredential = $decodedString
}
$variations += @($runtimeSource)
}

$installSuccess = $false
foreach ($variation in $variations) {
if ($variation | Get-Member AzureFeed) {
$location = $variation.AzureFeed
} else {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from public location."
ExitWithExitCode 1
$location = "public location";
}
Write-Host "Attempting to install dotnet from $location."
try {
& $installScript @variation
$installSuccess = $true
break
}
catch {
Write-Host "Failed to install dotnet from $location."
}
}
if (-not $installSuccess) {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from any of the specified locations."
ExitWithExitCode 1
}
}

Expand Down
60 changes: 33 additions & 27 deletions eng/common/tools.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,28 +188,29 @@ function InstallDotNet {
GetDotNetInstallScript "$root"
local install_script=$_GetDotNetInstallScript

local archArg=''
local installParameters=(--version $version --install-dir "$root")

if [[ -n "${3:-}" ]] && [ "$3" != 'unset' ]; then
archArg="--architecture $3"
installParameters+=(--architecture $3)
fi
local runtimeArg=''
if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then
runtimeArg="--runtime $4"
installParameters+=(--runtime $4)
fi
local skipNonVersionedFilesArg=""
if [[ "$#" -ge "5" ]] && [[ "$5" != 'false' ]]; then
skipNonVersionedFilesArg="--skip-non-versioned-files"
installParameters+=(--skip-non-versioned-files)
fi
bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || {
local exit_code=$?
echo "Failed to install dotnet SDK from public location (exit code '$exit_code')."

local runtimeSourceFeed=''
if [[ -n "${6:-}" ]]; then
runtimeSourceFeed="--azure-feed $6"
fi
local variations=() # list of variable names with parameter arrays in them

local public_location=("${installParameters[@]}")
variations+=(public_location)

local dotnetbuilds=("${installParameters[@]}" --azure-feed "https://dotnetbuilds.azureedge.net/public")
variations+=(dotnetbuilds)

local runtimeSourceFeedKey=''
if [[ -n "${6:-}" ]]; then
variations+=(private_feed)
local private_feed=("${installParameters[@]}" --azure-feed $6)
if [[ -n "${7:-}" ]]; then
# The 'base64' binary on alpine uses '-d' and doesn't support '--decode'
# '-d'. To work around this, do a simple detection and switch the parameter
Expand All @@ -219,22 +220,27 @@ function InstallDotNet {
decodeArg="-d"
fi
decodedFeedKey=`echo $7 | base64 $decodeArg`
runtimeSourceFeedKey="--feed-credential $decodedFeedKey"
private_feed+=(--feed-credential $decodedFeedKey)
fi
fi

if [[ -n "$runtimeSourceFeed" || -n "$runtimeSourceFeedKey" ]]; then
bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg $runtimeSourceFeed $runtimeSourceFeedKey || {
local exit_code=$?
Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from custom location '$runtimeSourceFeed' (exit code '$exit_code')."
ExitWithExitCode $exit_code
}
else
if [[ $exit_code != 0 ]]; then
Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from public location (exit code '$exit_code')."
fi
ExitWithExitCode $exit_code
local installSuccess=0
for variationName in "${variations[@]}"; do
local name="$variationName[@]"
local variation=("${!name}")
echo "Attempting to install dotnet from $variationName."
bash "$install_script" "${variation[@]}" && installSuccess=1
if [[ "$installSuccess" -eq 1 ]]; then
break
fi
}

echo "Failed to install dotnet from $variationName."
done

if [[ "$installSuccess" -eq 0 ]]; then
Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from any of the specified locations."
ExitWithExitCode 1
fi
}

function with_retries {
Expand Down
120 changes: 78 additions & 42 deletions src/Microsoft.DotNet.Helix/Sdk/FindDotNetCliPackage.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
Expand All @@ -17,7 +19,6 @@ public class FindDotNetCliPackage : BaseTask
DelayBase = 3.0
};
private static readonly HttpClient _client = new HttpClient(new HttpClientHandler { CheckCertificateRevocationList = true });
private const string DotNetCliAzureFeed = "https://dotnetcli.blob.core.windows.net/dotnet";

/// <summary>
/// 'LTS' or 'Current'
Expand Down Expand Up @@ -55,82 +56,114 @@ public override bool Execute()
private async Task ExecuteAsync()
{
NormalizeParameters();
await ResolveVersionAsync();

string downloadUrl = await GetDownloadUrlAsync();

Log.LogMessage($"Retrieved dotnet cli {PackageType} version {Version} package uri {downloadUrl}, testing...");

try
var feeds = new List<string>
{
using HttpResponseMessage res = await HeadRequestWithRetry(downloadUrl);
"https://dotnetcli.azureedge.net/dotnet",
"https://dotnetbuilds.azureedge.net/public",
};

if (res.StatusCode == HttpStatusCode.NotFound)
string finalDownloadUrl = null;
foreach (var feed in feeds)
{
string downloadUrl = await GetDownloadUrlAsync(feed);
if (downloadUrl == null)
{
// 404 means that we successfully hit the server, and it returned 404. This cannot be a network hiccup
Log.LogError(FailureCategory.Build, $"Unable to find dotnet cli {PackageType} version {Version}, tried {downloadUrl}");
Log.LogMessage($"Could not retrieve dotnet cli {PackageType} version {Version} package uri from feed {feed}");
continue;
}
else

Log.LogMessage($"Retrieved dotnet cli {PackageType} version {Version} package uri {downloadUrl} from feed {feed}, testing...");

try
{
using HttpResponseMessage res = await HeadRequestWithRetry(downloadUrl);

if (res.StatusCode == HttpStatusCode.NotFound)
{
// 404 means that we successfully hit the server, and it returned 404. This cannot be a network hiccup
Log.LogMessage($"Unable to find dotnet cli {PackageType} version {Version} from feed {feed}");
continue;
}

res.EnsureSuccessStatusCode();
finalDownloadUrl = downloadUrl;
}
catch (Exception ex)
{
Log.LogMessage($"Unable to access dotnet cli {PackageType} version {Version} from feed {feed}, {ex.Message}");
}
}
catch (Exception ex)

if (finalDownloadUrl == null)
{
Log.LogError(FailureCategory.Build, $"Unable to access dotnet cli {PackageType} version {Version} at {downloadUrl}, {ex.Message}");
Log.LogError(FailureCategory.Build, $"Unable to find dotnet cli {PackageType} version {Version} from any of the specified feeds.");
}


if (!Log.HasLoggedErrors)
{
Log.LogMessage($"Url {downloadUrl} is valid.");
PackageUri = downloadUrl;
Log.LogMessage($"Url {finalDownloadUrl} is valid.");
PackageUri = finalDownloadUrl;
}
}

private async Task<string> GetDownloadUrlAsync()
private async Task<string> GetDownloadUrlAsync(string feed)
{
string extension = Runtime.StartsWith("win") ? "zip" : "tar.gz";
string effectiveVersion = await GetEffectiveVersion();
var oldVersion = Version; // ResolveVersionAsync will adjust the Version property, but we need it set back for other feeds to see the same initial Version
try
{
var version = await ResolveVersionAsync(feed);
string extension = Runtime.StartsWith("win") ? "zip" : "tar.gz";
string effectiveVersion = await GetEffectiveVersion(feed, version);

return PackageType switch
return PackageType switch
{
"sdk" => $"{feed}/Sdk/{version}/dotnet-sdk-{effectiveVersion}-{Runtime}.{extension}",
"aspnetcore-runtime" =>
$"{feed}/aspnetcore/Runtime/{version}/aspnetcore-runtime-{effectiveVersion}-{Runtime}.{extension}",
_ => $"{feed}/Runtime/{version}/dotnet-runtime-{effectiveVersion}-{Runtime}.{extension}"
};
}
catch (Exception ex)
{
"sdk" => $"{DotNetCliAzureFeed}/Sdk/{Version}/dotnet-sdk-{effectiveVersion}-{Runtime}.{extension}",
"aspnetcore-runtime" => $"{DotNetCliAzureFeed}/aspnetcore/Runtime/{Version}/aspnetcore-runtime-{effectiveVersion}-{Runtime}.{extension}",
_ => $"{DotNetCliAzureFeed}/Runtime/{Version}/dotnet-runtime-{effectiveVersion}-{Runtime}.{extension}"
};
Log.LogWarning($"Unable to resolve download link from feed {feed}; {ex.Message}");
return null;
}
finally
{
Version = oldVersion;
}
}

private async Task<string> GetEffectiveVersion()
private async Task<string> GetEffectiveVersion(string feed, string version)
{
if (NuGetVersion.TryParse(Version, out NuGetVersion semanticVersion))
if (NuGetVersion.TryParse(version, out NuGetVersion semanticVersion))
{
// Pared down version of the logic from https://github.com/dotnet/install-scripts/blob/main/src/dotnet-install.ps1
// If this functionality stops working, review changes made there.
// Current strategy is to start with a runtime-specific name then fall back to 'productVersion.txt'
string effectiveVersion = Version;
string effectiveVersion = version;

// Do nothing for older runtimes; the file won't exist
if (semanticVersion >= new NuGetVersion("5.0.0"))
{
var productVersionText = PackageType switch
{
"sdk" => await GetMatchingProductVersionTxtContents($"{DotNetCliAzureFeed}/Sdk/{Version}", "sdk-productVersion.txt"),
"aspnetcore-runtime" => await GetMatchingProductVersionTxtContents($"{DotNetCliAzureFeed}/aspnetcore/Runtime/{Version}", "aspnetcore-productVersion.txt"),
_ => await GetMatchingProductVersionTxtContents($"{DotNetCliAzureFeed}/Runtime/{Version}", "runtime-productVersion.txt")
"sdk" => await GetMatchingProductVersionTxtContents($"{feed}/Sdk/{version}", "sdk-productVersion.txt"),
"aspnetcore-runtime" => await GetMatchingProductVersionTxtContents($"{feed}/aspnetcore/Runtime/{version}", "aspnetcore-productVersion.txt"),
_ => await GetMatchingProductVersionTxtContents($"{feed}/Runtime/{version}", "runtime-productVersion.txt")
};

if (!productVersionText.Equals(Version))
if (!productVersionText.Equals(version))
{
effectiveVersion = productVersionText;
Log.LogMessage($"Switched to effective .NET Core version '{productVersionText}' from matching productVersion.txt");
}
}
return effectiveVersion;
}
else
{
throw new ArgumentException($"'{Version}' is not a valid semantic version.");
}

throw new ArgumentException($"'{version}' is not a valid semantic version.");
}
private async Task<string> GetMatchingProductVersionTxtContents(string baseUri, string customVersionTextFileName)
{
Expand Down Expand Up @@ -243,16 +276,17 @@ private void NormalizeParameters()
}
}

private async Task ResolveVersionAsync()
private async Task<string> ResolveVersionAsync(string feed)
{
string version = Version;
if (Version == "latest")
{
Log.LogMessage(MessageImportance.Low, "Resolving latest dotnet cli version.");
string latestVersionUrl = PackageType switch
{
"sdk" => $"{DotNetCliAzureFeed}/Sdk/{Channel}/latest.version",
"aspnetcore-runtime" => $"{DotNetCliAzureFeed}/aspnetcore/Runtime/{Channel}/latest.version",
_ => $"{DotNetCliAzureFeed}/Runtime/{Channel}/latest.version"
"sdk" => $"{feed}/Sdk/{Channel}/latest.version",
"aspnetcore-runtime" => $"{feed}/aspnetcore/Runtime/{Channel}/latest.version",
_ => $"{feed}/Runtime/{Channel}/latest.version"
};

Log.LogMessage(MessageImportance.Low, $"Resolving latest version from url {latestVersionUrl}");
Expand All @@ -261,9 +295,11 @@ private async Task ResolveVersionAsync()
versionResponse.EnsureSuccessStatusCode();
string latestVersionContent = await versionResponse.Content.ReadAsStringAsync();
string[] versionData = latestVersionContent.Split(Array.Empty<char>(), StringSplitOptions.RemoveEmptyEntries);
Version = versionData[1];
Log.LogMessage(MessageImportance.Low, $"Got latest dotnet cli version {Version}");
version = versionData[1];
Log.LogMessage(MessageImportance.Low, $"Got latest dotnet cli version {version}");
}

return version;
}
}
}

0 comments on commit 0750fc9

Please sign in to comment.