From 90aec41b58006a6a21a3b276885cc03cd48b007c Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 25 May 2022 22:09:01 +0200 Subject: [PATCH] [xaprepare] cache .NET install artifacts (#7026) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Context: https://github.com/dotnet/install-scripts/issues/15 Context: https://dot.net/v1/dotnet-install.sh Context: https://dot.net/v1/dotnet-install.ps1 We've been installing dotnet versions using the [`dotnet-install`][0] scripts for Unix & Windows. However, they do not cache the downloaded archive, and therefore we end up re-downloading the same archive over and over again. Additionally, if one finds themselves without an internet connection, there's no way to easily install the required version of dotnet. The installation scripts don't provide a way to cache the payloads and they appear to be in maintenance mode (dotnet/install-scripts#15), so there doesn't appear to be a chance to add caching support to them. Fortunately, we can "ask" the scripts what they're downloading: % curl -o dotnet-install.sh 'https://dot.net/v1/dotnet-install.sh' % ./dotnet-install.sh --version 7.0.100-preview.5.22273.1 --verbose --dry-run \ | grep 'dotnet-install: URL' This returns a list of URLs, which may or may not exist: dotnet-install: URL #0 - primary: https://dotnetcli.azureedge.net/dotnet/Sdk/7.0.100-preview.5.22273.1/dotnet-sdk-7.0.100-preview.5.22273.1-osx-x64.tar.gz dotnet-install: URL #1 - legacy: https://dotnetcli.azureedge.net/dotnet/Sdk/7.0.100-preview.5.22273.1/dotnet-dev-osx-x64.7.0.100-preview.5.22273.1.tar.gz dotnet-install: URL #2 - primary: https://dotnetbuilds.azureedge.net/public/Sdk/7.0.100-preview.5.22273.1/dotnet-sdk-7.0.100-preview.5.22273.1-osx-x64.tar.gz dotnet-install: URL #3 - legacy: https://dotnetbuilds.azureedge.net/public/Sdk/7.0.100-preview.5.22273.1/dotnet-dev-osx-x64.7.0.100-preview.5.22273.1.tar.gz We now parse this output, extract the URLs, then download and cache the URL contents into `$(AndroidToolchainCacheDirectory)`. When we need to install .NET, we just extract the cached archive into the appropriate directory. If no `dotnet-install: URL…` messages are generated, then we run the `dotnet-install` script as we previously did. This process lets us take a first step towards fully "offline" builds, along with smaller downloads on CI servers. [0]: https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script --- .../Steps/Step_InstallDotNetPreview.cs | 198 ++++++++++++++---- 1 file changed, 158 insertions(+), 40 deletions(-) diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_InstallDotNetPreview.cs b/build-tools/xaprepare/xaprepare/Steps/Step_InstallDotNetPreview.cs index 83ee68c7c4e..18789598971 100644 --- a/build-tools/xaprepare/xaprepare/Steps/Step_InstallDotNetPreview.cs +++ b/build-tools/xaprepare/xaprepare/Steps/Step_InstallDotNetPreview.cs @@ -22,9 +22,6 @@ protected override async Task Execute (Context context) dotnetPath = dotnetPath.TrimEnd (new char [] { Path.DirectorySeparatorChar }); var dotnetTool = Path.Combine (dotnetPath, "dotnet"); - // Always delete the bin/$(Configuration)/dotnet/ directory - Utilities.DeleteDirectory (dotnetPath); - if (!await InstallDotNetAsync (context, dotnetPath, BuildToolVersion)) { Log.ErrorLine ($"Installation of dotnet SDK {BuildToolVersion} failed."); return false; @@ -65,68 +62,189 @@ protected override async Task Execute (Context context) return true; } - async Task InstallDotNetAsync (Context context, string dotnetPath, string version, bool runtimeOnly = false) + async Task DownloadDotNetInstallScript (Context context, string dotnetScriptPath, Uri dotnetScriptUrl) { - if (Directory.Exists (Path.Combine (dotnetPath, "sdk", version)) && !runtimeOnly) { - Log.Status ($"dotnet SDK version "); - Log.Status (version, ConsoleColor.Yellow); - Log.StatusLine (" already installed in: ", Path.Combine (dotnetPath, "sdk", version), tailColor: ConsoleColor.Cyan); - return true; + string tempDotnetScriptPath = dotnetScriptPath + "-tmp"; + Utilities.DeleteFile (tempDotnetScriptPath); + + Log.StatusLine ("Downloading dotnet-install..."); + + (bool success, ulong size, HttpStatusCode status) = await Utilities.GetDownloadSizeWithStatus (dotnetScriptUrl); + if (!success) { + string message; + if (status == HttpStatusCode.NotFound) { + message = "dotnet-install URL not found"; + } else { + message = $"Failed to obtain dotnet-install size. HTTP status code: {status} ({(int)status})"; + } + + return ReportAndCheckCached (message, quietOnError: true); } - if (Directory.Exists (Path.Combine (dotnetPath, "shared", "Microsoft.NETCore.App", version)) && runtimeOnly) { - Log.Status ($"dotnet runtime version "); - Log.Status (version, ConsoleColor.Yellow); - Log.StatusLine (" already installed in: ", Path.Combine (dotnetPath, "shared", "Microsoft.NETCore.App", version), tailColor: ConsoleColor.Cyan); - return true; + DownloadStatus downloadStatus = Utilities.SetupDownloadStatus (context, size, context.InteractiveSession); + Log.StatusLine ($" {context.Characters.Link} {dotnetScriptUrl}", ConsoleColor.White); + await Download (context, dotnetScriptUrl, tempDotnetScriptPath, "dotnet-install", Path.GetFileName (dotnetScriptUrl.LocalPath), downloadStatus); + + if (!File.Exists (tempDotnetScriptPath)) { + return ReportAndCheckCached ($"Download of dotnet-install from {dotnetScriptUrl} failed"); } - Uri dotnetScriptUrl = Configurables.Urls.DotNetInstallScript; - string dotnetScriptPath = Path.Combine (dotnetPath, Path.GetFileName (dotnetScriptUrl.LocalPath)); - if (File.Exists (dotnetScriptPath)) - Utilities.DeleteFile (dotnetScriptPath); + Utilities.CopyFile (tempDotnetScriptPath, dotnetScriptPath); + Utilities.DeleteFile (tempDotnetScriptPath); + return true; - Log.StatusLine ("Downloading dotnet-install..."); + bool ReportAndCheckCached (string message, bool quietOnError = false) + { + if (File.Exists (dotnetScriptPath)) { + Log.WarningLine (message); + Log.WarningLine ($"Using cached installation script found in {dotnetScriptPath}"); + return true; + } - (bool success, ulong size, HttpStatusCode status) = await Utilities.GetDownloadSizeWithStatus (dotnetScriptUrl); + if (!quietOnError) { + Log.ErrorLine (message); + Log.ErrorLine ($"Cached installation script not found in {dotnetScriptPath}"); + } + return false; + } + } + + async Task DownloadDotNetArchive (Context context, string archiveDestinationPath, Uri archiveUrl) + { + Log.StatusLine ("Downloading dotnet archive..."); + + (bool success, ulong size, HttpStatusCode status) = await Utilities.GetDownloadSizeWithStatus (archiveUrl); if (!success) { - if (status == HttpStatusCode.NotFound) - Log.ErrorLine ("dotnet-install URL not found"); - else - Log.ErrorLine ("Failed to obtain dotnet-install size. HTTP status code: {status} ({(int)status})"); + if (status == HttpStatusCode.NotFound) { + Log.ErrorLine ($"dotnet archive URL {archiveUrl} not found"); + return false; + } else { + Log.WarningLine ($"Failed to obtain dotnet archive size. HTTP status code: {status} ({(int)status})"); + } + return false; } + string tempArchiveDestinationPath = archiveDestinationPath + "-tmp"; + Utilities.DeleteFile (tempArchiveDestinationPath); + DownloadStatus downloadStatus = Utilities.SetupDownloadStatus (context, size, context.InteractiveSession); - Log.StatusLine ($" {context.Characters.Link} {dotnetScriptUrl}", ConsoleColor.White); - await Download (context, dotnetScriptUrl, dotnetScriptPath, "dotnet-install", Path.GetFileName (dotnetScriptUrl.LocalPath), downloadStatus); + Log.StatusLine ($" {context.Characters.Link} {archiveUrl}", ConsoleColor.White); + await Download (context, archiveUrl, tempArchiveDestinationPath, "dotnet archive", Path.GetFileName (archiveUrl.LocalPath), downloadStatus); - if (!File.Exists (dotnetScriptPath)) { - Log.ErrorLine ($"Download of dotnet-install from {dotnetScriptUrl} failed"); + if (!File.Exists (tempArchiveDestinationPath)) { return false; } - var type = runtimeOnly ? "runtime" : "SDK"; - Log.StatusLine ($"Installing dotnet {type} '{version}'..."); + Utilities.CopyFile (tempArchiveDestinationPath, archiveDestinationPath); + Utilities.DeleteFile (tempArchiveDestinationPath); + + return true; + } + string[] GetInstallationScriptArgs (string version, string dotnetPath, string dotnetScriptPath, bool onlyGetUrls, bool runtimeOnly) + { + List args; if (Context.IsWindows) { - var args = new List { + args = new List { "-NoProfile", "-ExecutionPolicy", "unrestricted", "-file", dotnetScriptPath, "-Version", version, "-InstallDir", dotnetPath, "-Verbose" }; - if (runtimeOnly) + if (runtimeOnly) { args.AddRange (new string [] { "-Runtime", "dotnet" }); + } + if (onlyGetUrls) { + args.Add ("-DryRun"); + } - return Utilities.RunCommand ("powershell.exe", args.ToArray ()); - } else { - var args = new List { - dotnetScriptPath, "--version", version, "--install-dir", dotnetPath, "--verbose" - }; - if (runtimeOnly) - args.AddRange (new string [] { "-Runtime", "dotnet" }); + return args.ToArray (); + } + + args = new List { + dotnetScriptPath, "--version", version, "--install-dir", dotnetPath, "--verbose" + }; + + if (runtimeOnly) { + args.AddRange (new string [] { "-Runtime", "dotnet" }); + } + if (onlyGetUrls) { + args.Add ("--dry-run"); + } + + return args.ToArray (); + } + + async Task InstallDotNetAsync (Context context, string dotnetPath, string version, bool runtimeOnly = false) + { + string cacheDir = context.Properties.GetRequiredValue (KnownProperties.AndroidToolchainCacheDirectory); - return Utilities.RunCommand ("bash", args.ToArray ()); + // Always delete the bin/$(Configuration)/dotnet/ directory + Utilities.DeleteDirectory (dotnetPath); + + Uri dotnetScriptUrl = Configurables.Urls.DotNetInstallScript; + string scriptFileName = Path.GetFileName (dotnetScriptUrl.LocalPath); + string cachedDotnetScriptPath = Path.Combine (cacheDir, scriptFileName); + if (!await DownloadDotNetInstallScript (context, cachedDotnetScriptPath, dotnetScriptUrl)) { + return false; } + + string dotnetScriptPath = Path.Combine (dotnetPath, scriptFileName); + Utilities.CopyFile (cachedDotnetScriptPath, dotnetScriptPath); + + var type = runtimeOnly ? "runtime" : "SDK"; + + Log.StatusLine ($"Discovering download URLs for dotnet {type} '{version}'..."); + string scriptCommand = Context.IsWindows ? "powershell.exe" : "bash"; + string[] scriptArgs = GetInstallationScriptArgs (version, dotnetPath, dotnetScriptPath, onlyGetUrls: true, runtimeOnly: runtimeOnly); + string scriptReply = Utilities.GetStringFromStdout (scriptCommand, scriptArgs); + var archiveUrls = new List (); + + char[] fieldSplitChars = new char[] { ':' }; + foreach (string l in scriptReply.Split (new char[] { '\n' })) { + string line = l.Trim (); + + if (!line.StartsWith ("dotnet-install: URL #", StringComparison.OrdinalIgnoreCase)) { + continue; + } + + string[] parts = line.Split (fieldSplitChars, 3); + if (parts.Length < 3) { + Log.WarningLine ($"dotnet-install URL line has unexpected number of parts. Expected 3, got {parts.Length}"); + Log.WarningLine ($"Line: {line}"); + continue; + } + + archiveUrls.Add (parts[2].Trim ()); + } + + if (archiveUrls.Count == 0) { + Log.WarningLine ("No dotnet archive URLs discovered, attempting to run the installation script"); + scriptArgs = GetInstallationScriptArgs (version, dotnetPath, dotnetScriptPath, onlyGetUrls: false, runtimeOnly: runtimeOnly); + return Utilities.RunCommand (scriptCommand, scriptArgs); + } + + string? archivePath = null; + foreach (string url in archiveUrls) { + var archiveUrl = new Uri (url); + string archiveDestinationPath = Path.Combine (cacheDir, Path.GetFileName (archiveUrl.LocalPath)); + + if (File.Exists (archiveDestinationPath)) { + archivePath = archiveDestinationPath; + break; + } + + if (await DownloadDotNetArchive (context, archiveDestinationPath, archiveUrl)) { + archivePath = archiveDestinationPath; + break; + } + } + + if (String.IsNullOrEmpty (archivePath)) { + return false; + } + + Log.StatusLine ($"Installing dotnet {type} '{version}'..."); + return await Utilities.Unpack (archivePath, dotnetPath); } bool TestDotNetSdk (string dotnetTool)