diff --git a/Cmdline/Action/Install.cs b/Cmdline/Action/Install.cs index 0cb7157c28..d9ac81f410 100644 --- a/Cmdline/Action/Install.cs +++ b/Cmdline/Action/Install.cs @@ -52,8 +52,8 @@ public int RunCommand(CKAN.GameInstance instance, object raw_options) try { var targets = options.ckan_files - .Select(arg => new NetAsyncDownloader.DownloadTarget(getUri(arg))) - .ToList(); + .Select(arg => new NetAsyncDownloader.DownloadTargetFile(getUri(arg))) + .ToArray(); log.DebugFormat("Urls: {0}", targets.SelectMany(t => t.urls)); new NetAsyncDownloader(new NullUser()).DownloadAndWait(targets); log.DebugFormat("Files: {0}", targets.Select(t => t.filename)); diff --git a/Core/AutoUpdate/GithubReleaseCkanUpdate.cs b/Core/AutoUpdate/GithubReleaseCkanUpdate.cs index eb0b76dbc0..11bd037d78 100644 --- a/Core/AutoUpdate/GithubReleaseCkanUpdate.cs +++ b/Core/AutoUpdate/GithubReleaseCkanUpdate.cs @@ -54,9 +54,9 @@ public GitHubReleaseCkanUpdate(GitHubReleaseInfo releaseJson = null) public override IList Targets => new[] { - new NetAsyncDownloader.DownloadTarget( + new NetAsyncDownloader.DownloadTargetFile( UpdaterDownload, updaterFilename, UpdaterSize), - new NetAsyncDownloader.DownloadTarget( + new NetAsyncDownloader.DownloadTargetFile( ReleaseDownload, ckanFilename, ReleaseSize), }; diff --git a/Core/AutoUpdate/S3BuildCkanUpdate.cs b/Core/AutoUpdate/S3BuildCkanUpdate.cs index 19def760e3..7307790b96 100644 --- a/Core/AutoUpdate/S3BuildCkanUpdate.cs +++ b/Core/AutoUpdate/S3BuildCkanUpdate.cs @@ -29,9 +29,9 @@ public S3BuildCkanUpdate(S3BuildVersionInfo versionJson = null) public override IList Targets => new[] { - new NetAsyncDownloader.DownloadTarget( + new NetAsyncDownloader.DownloadTargetFile( UpdaterDownload, updaterFilename), - new NetAsyncDownloader.DownloadTarget( + new NetAsyncDownloader.DownloadTargetFile( ReleaseDownload, ckanFilename), }; diff --git a/Core/FileIdentifier.cs b/Core/FileIdentifier.cs index 626dfe8811..c10f717465 100644 --- a/Core/FileIdentifier.cs +++ b/Core/FileIdentifier.cs @@ -1,6 +1,8 @@ using System.IO; using System.Linq; +using ICSharpCode.SharpZipLib.GZip; + using CKAN.Extensions; namespace CKAN @@ -173,11 +175,10 @@ public static FileType IdentifyFile(Stream stream) if (CheckGZip(stream)) { // This may contain a tar file inside, create a new stream and check. - stream.Seek (0, SeekOrigin.Begin); - using (ICSharpCode.SharpZipLib.GZip.GZipInputStream gz_stream = new ICSharpCode.SharpZipLib.GZip.GZipInputStream (stream)) - { - type = CheckTar(gz_stream) ? FileType.TarGz : FileType.GZip; - } + stream.Seek(0, SeekOrigin.Begin); + type = CheckTar(new GZipInputStream(stream)) + ? FileType.TarGz + : FileType.GZip; } else if (CheckTar(stream)) { diff --git a/Core/Net/NetAsyncDownloader.DownloadPart.cs b/Core/Net/NetAsyncDownloader.DownloadPart.cs index b49f6434e7..088a39da24 100644 --- a/Core/Net/NetAsyncDownloader.DownloadPart.cs +++ b/Core/Net/NetAsyncDownloader.DownloadPart.cs @@ -1,5 +1,4 @@ using System; -using System.IO; using System.ComponentModel; using Autofac; @@ -14,7 +13,6 @@ public partial class NetAsyncDownloader private class DownloadPart { public readonly DownloadTarget target; - public readonly string path; public DateTime lastProgressUpdateTime; public long lastProgressUpdateSize; @@ -38,14 +36,14 @@ private class DownloadPart public DownloadPart(DownloadTarget target) { this.target = target; - path = target.filename ?? Path.GetTempFileName(); size = bytesLeft = target.size; lastProgressUpdateTime = DateTime.Now; triedDownloads = 0; } - public void Download(Uri url, string path) + public void Download() { + var url = CurrentUri; ResetAgent(); // Check whether to use an auth token for this host if (url.IsAbsoluteUri @@ -56,7 +54,7 @@ public void Download(Uri url, string path) // Send our auth token to the GitHub API (or whoever else needs one) agent.Headers.Add("Authorization", $"token {token}"); } - agent.DownloadFileAsyncWithResume(url, path); + target.DownloadWith(agent, url); } public Uri CurrentUri => target.urls[triedDownloads]; diff --git a/Core/Net/NetAsyncDownloader.DownloadTarget.cs b/Core/Net/NetAsyncDownloader.DownloadTarget.cs index b00787be30..241f34357a 100644 --- a/Core/Net/NetAsyncDownloader.DownloadTarget.cs +++ b/Core/Net/NetAsyncDownloader.DownloadTarget.cs @@ -1,42 +1,99 @@ using System; +using System.IO; using System.Collections.Generic; -using ChinhDo.Transactions.FileManager; - namespace CKAN { public partial class NetAsyncDownloader { - public class DownloadTarget + public abstract class DownloadTarget { - public List urls { get; private set; } - public string filename { get; private set; } - public long size { get; set; } - public string mimeType { get; private set; } + public List urls { get; protected set; } + public long size { get; protected set; } + public string mimeType { get; protected set; } - public DownloadTarget(List urls, - string filename = null, - long size = 0, - string mimeType = "") + protected DownloadTarget(List urls, + long size = 0, + string mimeType = "") { - var FileTransaction = new TxFileManager(); - this.urls = urls; - this.filename = string.IsNullOrEmpty(filename) - ? FileTransaction.GetTempFileName() - : filename; this.size = size; this.mimeType = mimeType; } - public DownloadTarget(Uri url, - string filename = null, - long size = 0, - string mimeType = "") - : this(new List { url }, - filename, size, mimeType) + public abstract long CalculateSize(); + public abstract void DownloadWith(ResumingWebClient wc, Uri url); + } + + public sealed class DownloadTargetFile : DownloadTarget + { + public string filename { get; private set; } + + public DownloadTargetFile(List urls, + string filename = null, + long size = 0, + string mimeType = "") + : base(urls, size, mimeType) + { + this.filename = filename ?? Path.GetTempFileName(); + } + + public DownloadTargetFile(Uri url, + string filename = null, + long size = 0, + string mimeType = "") + : this(new List { url }, filename, size, mimeType) + { + } + + public override long CalculateSize() + { + size = new FileInfo(filename).Length; + return size; + } + + public override void DownloadWith(ResumingWebClient wc, Uri url) + { + wc.DownloadFileAsyncWithResume(url, filename); + } + } + + public sealed class DownloadTargetStream : DownloadTarget, IDisposable + { + public Stream contents { get; private set; } + + public DownloadTargetStream(List urls, + long size = 0, + string mimeType = "") + : base(urls, size, mimeType) { + contents = new MemoryStream(); + } + + public DownloadTargetStream(Uri url, + long size = 0, + string mimeType = "") + : this(new List { url }, size, mimeType) + { + } + + public override long CalculateSize() + { + size = contents.Length; + return size; + } + + public override void DownloadWith(ResumingWebClient wc, Uri url) + { + wc.DownloadFileAsyncWithResume(url, contents); + } + + public void Dispose() + { + // Close the stream + contents.Dispose(); } } + } } diff --git a/Core/Net/NetAsyncDownloader.cs b/Core/Net/NetAsyncDownloader.cs index 88c3d107fd..99c7a5d675 100644 --- a/Core/Net/NetAsyncDownloader.cs +++ b/Core/Net/NetAsyncDownloader.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Net; using System.Text.RegularExpressions; @@ -36,7 +35,13 @@ public partial class NetAsyncDownloader private volatile bool download_canceled; private readonly ManualResetEvent complete_or_canceled; - public event Action onOneCompleted; + /// + /// Invoked when a download completes or fails. + /// + /// The download that is done + /// Exception thrown if failed + /// ETag of the URL + public event Action onOneCompleted; /// /// Returns a perfectly boring NetAsyncDownloader @@ -54,7 +59,7 @@ public static string DownloadWithProgress(Uri url, string filename = null, IUser { var targets = new[] { - new DownloadTarget(url, filename) + new DownloadTargetFile(url, filename) }; DownloadWithProgress(targets, user); return targets.First().filename; @@ -63,16 +68,12 @@ public static string DownloadWithProgress(Uri url, string filename = null, IUser public static void DownloadWithProgress(IList downloadTargets, IUser user = null) { var downloader = new NetAsyncDownloader(user ?? new NullUser()); - downloader.onOneCompleted += (url, filename, error, etag) => + downloader.onOneCompleted += (target, error, etag) => { if (error != null) { user?.RaiseError(error.ToString()); } - else - { - File.Move(filename, downloadTargets.First(p => p.urls.Contains(url)).filename); - } }; downloader.DownloadAndWait(downloadTargets); } @@ -88,7 +89,7 @@ public void DownloadAndWait(IList targets) if (downloads.Count + queuedDownloads.Count > completed_downloads) { // Some downloads are still in progress, add to the current batch - foreach (DownloadTarget target in targets) + foreach (var target in targets) { DownloadModule(new DownloadPart(target)); } @@ -259,7 +260,7 @@ private void DownloadModule(DownloadPart dl) dl.CurrentUri.ToString().Replace(" ", "%20")); // Start the download! - dl.Download(dl.CurrentUri, dl.path); + dl.Download(); } } @@ -403,7 +404,7 @@ private void FileDownloadComplete(int index, Exception error, bool canceled, str log.InfoFormat("Finished downloading {0}", string.Join(", ", dl.target.urls)); dl.bytesLeft = 0; // Let calling code find out how big this file is - dl.target.size = new FileInfo(dl.target.filename).Length; + dl.target.CalculateSize(); } PopFromQueue(doneUri.Host); @@ -411,7 +412,7 @@ private void FileDownloadComplete(int index, Exception error, bool canceled, str try { // Tell calling code that this file is ready - onOneCompleted?.Invoke(dl.target.urls.First(), dl.path, dl.error, etag); + onOneCompleted?.Invoke(dl.target, dl.error, etag); } catch (Exception exc) { diff --git a/Core/Net/NetAsyncModulesDownloader.cs b/Core/Net/NetAsyncModulesDownloader.cs index 0093876b2a..29020be520 100644 --- a/Core/Net/NetAsyncModulesDownloader.cs +++ b/Core/Net/NetAsyncModulesDownloader.cs @@ -42,16 +42,16 @@ public NetAsyncModulesDownloader(IUser user, NetModuleCache cache) this.cache = cache; } - internal NetAsyncDownloader.DownloadTarget TargetFromModuleGroup( + internal NetAsyncDownloader.DownloadTargetFile TargetFromModuleGroup( HashSet group, string[] preferredHosts) => TargetFromModuleGroup(group, group.OrderBy(m => m.identifier).First(), preferredHosts); - private NetAsyncDownloader.DownloadTarget TargetFromModuleGroup( + private NetAsyncDownloader.DownloadTargetFile TargetFromModuleGroup( HashSet group, CkanModule first, string[] preferredHosts) - => new NetAsyncDownloader.DownloadTarget( + => new NetAsyncDownloader.DownloadTargetFile( group.SelectMany(mod => mod.download) .Concat(group.Select(mod => mod.InternetArchiveDownload) .Where(uri => uri != null) @@ -90,7 +90,7 @@ public void DownloadModules(IEnumerable modules) .Where(grp => grp.All(mod => mod.download.All(dlUri => !activeURLs.Contains(dlUri)))) // Each group gets one target containing all the URLs .Select(grp => TargetFromModuleGroup(grp, preferredHosts)) - .ToList()); + .ToArray()); this.modules.Clear(); AllComplete?.Invoke(); } @@ -125,8 +125,13 @@ public void CancelDownload() private readonly NetModuleCache cache; private CancellationTokenSource cancelTokenSrc; - private void ModuleDownloadComplete(Uri url, string filename, Exception error, string etag) + private void ModuleDownloadComplete(NetAsyncDownloader.DownloadTarget target, + Exception error, + string etag) { + var url = target.urls.First(); + var filename = (target as NetAsyncDownloader.DownloadTargetFile)?.filename; + log.DebugFormat("Received download completion: {0}, {1}, {2}", url, filename, error?.Message); if (error != null) diff --git a/Core/Net/ResumingWebClient.cs b/Core/Net/ResumingWebClient.cs index 10ddec921f..5772873ccf 100644 --- a/Core/Net/ResumingWebClient.cs +++ b/Core/Net/ResumingWebClient.cs @@ -14,7 +14,7 @@ namespace CKAN { - internal class ResumingWebClient : WebClient + public class ResumingWebClient : WebClient { /// /// A version of DownloadFileAsync that appends to its destination @@ -39,12 +39,20 @@ public void DownloadFileAsyncWithResume(Uri url, string path) // Reset in case we try multiple with same webclient bytesToSkip = 0; } - // Ideally the bytes to skip would be passed in the userToken param, - // but GetWebRequest can't access it!! OpenReadAsync(url, path); }); } + public void DownloadFileAsyncWithResume(Uri url, Stream stream) + { + contentLength = 0; + Task.Factory.StartNew(() => + { + bytesToSkip = 0; + OpenReadAsync(url, stream); + }); + } + /// /// Same as DownloadProgressChanged, but usable by us. /// Called with percent, bytes received, total bytes to receive. @@ -115,36 +123,41 @@ protected override void OnOpenReadCompleted(OpenReadCompletedEventArgs e) { if (!netStream.CanRead || contentLength == 0) { - log.DebugFormat("OnOpenReadCompleted got closed stream or zero contentLength, skipping download to {0}", destination); + log.DebugFormat("OnOpenReadCompleted got closed stream or zero contentLength, skipping download to {0}", + destination ?? "stream"); // Synthesize a progress update for 100% completion - var fi = new FileInfo(destination); - DownloadProgress?.Invoke(100, fi.Length, fi.Length); + if (!string.IsNullOrEmpty(destination) + && File.Exists(destination)) + { + var fi = new FileInfo(destination); + DownloadProgress?.Invoke(100, fi.Length, fi.Length); + } } else { try { - log.DebugFormat("OnOpenReadCompleted got open stream, appending to {0}", destination); - using (var fileStream = new FileStream(destination, FileMode.Append, FileAccess.Write)) + log.DebugFormat("OnOpenReadCompleted got open stream, appending to {0}", + destination ?? "stream"); + // file:// URLs don't support timeouts + if (netStream.CanTimeout) + { + log.DebugFormat("Default stream read timeout is {0}", netStream.ReadTimeout); + netStream.ReadTimeout = timeoutMs; + } + cancelTokenSrc = new CancellationTokenSource(); + switch (e.UserState) { - // file:// URLs don't support timeouts - if (netStream.CanTimeout) - { - log.DebugFormat("Default stream read timeout is {0}", netStream.ReadTimeout); - netStream.ReadTimeout = timeoutMs; - } - cancelTokenSrc = new CancellationTokenSource(); - netStream.CopyTo(fileStream, new Progress(bytesDownloaded => - { - DownloadProgress?.Invoke((int)(100 * bytesDownloaded / contentLength), - bytesDownloaded, contentLength); - }), - TimeSpan.FromSeconds(5), - cancelTokenSrc.Token); - // Make sure caller knows we've finished - DownloadProgress?.Invoke(100, contentLength, contentLength); - cancelTokenSrc = null; + case string path: + ToFile(netStream, path); + break; + case Stream stream: + ToStream(netStream, stream); + break; } + // Make sure caller knows we've finished + DownloadProgress?.Invoke(100, contentLength, contentLength); + cancelTokenSrc = null; } catch (OperationCanceledException exc) { @@ -163,6 +176,29 @@ protected override void OnOpenReadCompleted(OpenReadCompletedEventArgs e) OnDownloadFileCompleted(new AsyncCompletedEventArgs(e.Error, e.Cancelled, e.UserState)); } + private void ToFile(Stream netStream, string path) + { + using (var outStream = new FileStream(path, FileMode.Append, FileAccess.Write)) + { + ToStream(netStream, outStream); + } + } + + private void ToStream(Stream netStream, Stream outStream) + { + netStream.CopyTo(outStream, new Progress(bytesDownloaded => + { + DownloadProgress?.Invoke((int)(100 * bytesDownloaded / contentLength), + bytesDownloaded, contentLength); + }), + TimeSpan.FromSeconds(5), + cancelTokenSrc.Token); + } + + /// + /// Ideally the bytes to skip would be passed in the userToken param of OpenReadAsync, + /// but GetWebRequest can't access it, so we store it here. + /// private long bytesToSkip = 0; private long contentLength = 0; private CancellationTokenSource cancelTokenSrc; diff --git a/Core/Properties/Resources.resx b/Core/Properties/Resources.resx index 8194994192..fdf9759d90 100644 --- a/Core/Properties/Resources.resx +++ b/Core/Properties/Resources.resx @@ -152,6 +152,7 @@ Install the `mono-complete` package or equivalent for your operating system.Loading modules from downloaded {0} repository...Should contain UserProgressDownloadSubstring from CmdLine Loaded download counts from {0} repository Not a .tar.gz or .zip, cannot process: {0} + Input stream is not a .tar.gz or .zip, cannot process `any_of` should not be combined with `{0}` {0} wishes to install {1}, but this file is registered to {2} Error unregistering {1}, file not removed: {0} diff --git a/Core/Repositories/RepositoryData.cs b/Core/Repositories/RepositoryData.cs index 3b4e11ef75..62e007f9b6 100644 --- a/Core/Repositories/RepositoryData.cs +++ b/Core/Repositories/RepositoryData.cs @@ -191,9 +191,27 @@ public static RepositoryData FromDownload(string path, IGame game, IProgress progress) + { + switch (FileIdentifier.IdentifyFile(stream)) + { + case FileType.TarGz: return FromTarGzStream(stream, game, progress); + case FileType.Zip: return FromZipStream(stream, game, progress); + default: throw new UnsupportedKraken(Properties.Resources.NetRepoNotATarGzStream); + } + } + private static RepositoryData FromTarGz(string path, IGame game, IProgress progress) { - using (var inputStream = File.OpenRead(path)) + using (var inputStream = File.OpenRead(path)) + { + return FromTarGzStream(inputStream, game, progress); + } + } + + private static RepositoryData FromTarGzStream(Stream inputStream, IGame game, IProgress progress) + { + inputStream.Seek(0, SeekOrigin.Begin); using (var progressStream = new ReadProgressStream(inputStream, progress)) using (var gzipStream = new GZipInputStream(progressStream)) using (var tarStream = new TarInputStream(gzipStream, Encoding.UTF8)) @@ -253,6 +271,14 @@ private static string tarStreamString(TarInputStream stream, TarEntry entry) private static RepositoryData FromZip(string path, IGame game, IProgress progress) { using (var inputStream = File.OpenRead(path)) + { + return FromZipStream(inputStream, game, progress); + } + } + + private static RepositoryData FromZipStream(Stream inputStream, IGame game, IProgress progress) + { + inputStream.Seek(0, SeekOrigin.Begin); using (var progressStream = new ReadProgressStream(inputStream, progress)) using (var zipfile = new ZipFile(progressStream)) { diff --git a/Core/Repositories/RepositoryDataManager.cs b/Core/Repositories/RepositoryDataManager.cs index 3baef358ca..b264c2a936 100644 --- a/Core/Repositories/RepositoryDataManager.cs +++ b/Core/Repositories/RepositoryDataManager.cs @@ -159,7 +159,7 @@ public UpdateResult Update(Repository[] repos, try { // Download metadata - var targets = toUpdate.Select(r => new NetAsyncDownloader.DownloadTarget(r.uri)) + var targets = toUpdate.Select(r => new NetAsyncDownloader.DownloadTargetStream(r.uri)) .ToArray(); downloader.DownloadAndWait(targets); @@ -168,24 +168,23 @@ public UpdateResult Update(Repository[] repos, string msg = ""; var progress = new ProgressFilesOffsetsToPercent( new Progress(p => user.RaiseProgress(msg, p)), - targets.Select(t => new FileInfo(t.filename).Length)); + targets.Select(t => t.size)); foreach ((var repo, var target) in toUpdate.Zip(targets)) { - var file = target.filename; msg = string.Format(Properties.Resources.NetRepoLoadingModulesFromRepo, repo.name); - log.InfoFormat("Loading {0}...", file); + log.InfoFormat("Loading repo stream..."); try { - // Load the file, save to in memory cache - var repoData = repositoriesData[repo] = - RepositoryData.FromDownload(file, game, progress); - // Save parsed data to disk - log.DebugFormat("Saving data for {0} repo...", repo.name); - repoData.SaveTo(GetRepoDataPath(repo)); - // Delete downloaded archive - log.DebugFormat("Deleting {0}...", file); - File.Delete(file); + using (target) + { + // Load the stream, save to in memory cache + var repoData = repositoriesData[repo] = + RepositoryData.FromStream(target.contents, game, progress); + // Save parsed data to disk + log.DebugFormat("Saving data for {0} repo...", repo.name); + repoData.SaveTo(GetRepoDataPath(repo)); + } } catch (UnsupportedKraken kraken) { @@ -260,8 +259,9 @@ private void saveETags() file_transaction.WriteAllText(etagsPath, JsonConvert.SerializeObject(etags, Formatting.Indented)); } - private void setETag(Uri url, string filename, Exception error, string etag) + private void setETag(NetAsyncDownloader.DownloadTarget target, Exception error, string etag) { + var url = target.urls.First(); if (etag != null) { etags[url] = etag; diff --git a/Tests/Core/Net/NetAsyncDownloaderTests.cs b/Tests/Core/Net/NetAsyncDownloaderTests.cs index fab3e2d76a..7d7ec51db8 100644 --- a/Tests/Core/Net/NetAsyncDownloaderTests.cs +++ b/Tests/Core/Net/NetAsyncDownloaderTests.cs @@ -24,8 +24,8 @@ public void DownloadAndWait_WithValidFileUrl_SetsTargetSize(string pathWithinTes // Arrange var downloader = new NetAsyncDownloader(new NullUser()); var fromPath = TestData.DataDir(pathWithinTestData); - var target = new NetAsyncDownloader.DownloadTarget(new Uri(fromPath), - Path.GetTempFileName()); + var target = new NetAsyncDownloader.DownloadTargetFile(new Uri(fromPath), + Path.GetTempFileName()); var targets = new NetAsyncDownloader.DownloadTarget[] { target }; var origSize = new FileInfo(fromPath).Length; @@ -70,8 +70,8 @@ public void DownloadAndWait_WithValidFileUrls_SetsTargetsSize(params string[] pa // Arrange var downloader = new NetAsyncDownloader(new NullUser()); var fromPaths = pathsWithinTestData.Select(p => TestData.DataDir(p)).ToArray(); - var targets = fromPaths.Select(p => new NetAsyncDownloader.DownloadTarget(new Uri(p), - Path.GetTempFileName())) + var targets = fromPaths.Select(p => new NetAsyncDownloader.DownloadTargetFile(new Uri(p), + Path.GetTempFileName())) .ToArray(); var origSizes = fromPaths.Select(p => new FileInfo(p).Length).ToArray(); @@ -186,8 +186,8 @@ public void DownloadAndWait_WithSomeInvalidUrls_ThrowsDownloadErrorsKraken( // Arrange var downloader = new NetAsyncDownloader(new NullUser()); var fromPaths = pathsWithinTestData.Select(p => Path.GetFullPath(TestData.DataDir(p))).ToArray(); - var targets = fromPaths.Select(p => new NetAsyncDownloader.DownloadTarget(new Uri(p), - Path.GetTempFileName())) + var targets = fromPaths.Select(p => new NetAsyncDownloader.DownloadTargetFile(new Uri(p), + Path.GetTempFileName())) .ToArray(); var badIndices = fromPaths.Select((p, i) => new Tuple(i, File.Exists(p))) .Where(tuple => !tuple.Item2)