diff --git a/Core/ModuleInstaller.cs b/Core/ModuleInstaller.cs index d11e3c5f1f..7101427d1c 100644 --- a/Core/ModuleInstaller.cs +++ b/Core/ModuleInstaller.cs @@ -102,7 +102,7 @@ public static string CachedOrDownload(CkanModule module, NetModuleCache cache, s filename = CkanModule.StandardName(module.identifier, module.version); } - string full_path = cache.GetCachedZip(module); + string full_path = cache.GetCachedFilename(module); if (full_path == null) { return Download(module, filename, cache); @@ -264,7 +264,7 @@ private void Install(CkanModule module, bool autoInstalled, Registry registry, r } // Find ZIP in the cache if we don't already have it. - filename = filename ?? Cache.GetCachedZip(module); + filename = filename ?? Cache.GetCachedFilename(module); // If we *still* don't have a file, then kraken bitterly. if (filename == null) @@ -1284,7 +1284,7 @@ public static IEnumerable PrioritizedHosts(IEnumerable urls) /// private void DownloadModules(IEnumerable mods, IDownloader downloader) { - List downloads = mods.Where(module => !Cache.IsCachedZip(module)).ToList(); + List downloads = mods.Where(module => !Cache.IsCached(module)).ToList(); if (downloads.Count > 0) { diff --git a/Core/Net/NetFileCache.cs b/Core/Net/NetFileCache.cs index 41f8399485..d582619c52 100644 --- a/Core/Net/NetFileCache.cs +++ b/Core/Net/NetFileCache.cs @@ -11,7 +11,6 @@ using log4net; using ChinhDo.Transactions.FileManager; -using ICSharpCode.SharpZipLib.Zip; using CKAN.Extensions; using CKAN.Versioning; @@ -36,13 +35,6 @@ public class NetFileCache : IDisposable private static readonly Regex cacheFileRegex = new Regex("^[0-9A-F]{8}-", RegexOptions.Compiled); private static readonly ILog log = LogManager.GetLogger(typeof (NetFileCache)); - static NetFileCache() - { - // SharpZibLib 1.1.0 changed this to default to false, but we depend on it for international mods. - // https://github.com/icsharpcode/SharpZipLib/issues/591 - ZipStrings.UseUnicode = true; - } - /// /// Initialize a cache given a GameInstanceManager /// @@ -163,13 +155,6 @@ public bool IsCached(Uri url, out string outFilename) return outFilename != null; } - /// - /// Returns true if our given URL is cached, *and* it passes zip - /// validation tests. Prefer this over IsCached when working with - /// zip files. - /// - public bool IsCachedZip(Uri url) => GetCachedZip(url) != null; - /// /// Returns true if a file matching the given URL is cached, but makes no /// attempts to check if it's even valid. This is very fast. @@ -260,38 +245,6 @@ private string scanDirectory(Dictionary files, string findHash, return null; } - /// - /// Returns the filename for a cached URL, if and only if it - /// passes zipfile validation tests. Prefer this to GetCachedFilename - /// when working with zip files. Returns null if not available, or - /// validation failed. - /// - /// Low level CRC (cyclic redundancy check) checks will be done. - /// This can take time on order of seconds for larger zip files. - /// - public string GetCachedZip(Uri url) - { - string filename = GetCachedFilename(url); - if (string.IsNullOrEmpty(filename)) - { - return null; - } - else - { - string invalidReason; - if (ZipValid(filename, out invalidReason, null)) - { - return filename; - } - else - { - // Purge invalid cache entries - File.Delete(filename); - return null; - } - } - } - /// /// Count the files and bytes in the cache /// @@ -442,87 +395,6 @@ private List allFiles(bool includeInProgress = false) ).ToList(); } - /// - /// Check whether a ZIP file is valid - /// - /// Path to zip file to check - /// Description of problem with the file - /// Callback to notify as we traverse the input, called with percentages from 0 to 100 - /// - /// True if valid, false otherwise. See invalidReason param for explanation. - /// - public static bool ZipValid(string filename, out string invalidReason, IProgress progress) - { - try - { - if (filename != null) - { - using (ZipFile zip = new ZipFile(filename)) - { - string zipErr = null; - // Limit progress updates to 100 per ZIP file - long highestPercent = -1; - // Perform CRC and other checks - if (zip.TestArchive(true, TestStrategy.FindFirstError, - (TestStatus st, string msg) => - { - // This delegate is called as TestArchive proceeds through its - // steps, both routine and abnormal. - // The second parameter is non-null if an error occurred. - if (st != null && !st.EntryValid && !string.IsNullOrEmpty(msg)) - { - // Capture the error string so we can return it - zipErr = string.Format( - Properties.Resources.NetFileCacheZipError, - st.Operation, st.Entry?.Name, msg); - } - else if (st.Entry != null && progress != null) - { - // Report progress - var percent = 100 * st.Entry.ZipFileIndex / zip.Count; - if (percent > highestPercent) - { - progress.Report(percent); - highestPercent = percent; - } - } - })) - { - invalidReason = ""; - return true; - } - else - { - invalidReason = zipErr ?? Properties.Resources.NetFileCacheZipTestArchiveFalse; - return false; - } - } - } - else - { - invalidReason = Properties.Resources.NetFileCacheNullFileName; - return false; - } - } - catch (ZipException ze) - { - // Save the errors someplace useful - invalidReason = ze.Message; - return false; - } - catch (ArgumentException ex) - { - invalidReason = ex.Message; - return false; - } - catch (NotSupportedException nse) when (Platform.IsMono) - { - // SharpZipLib throws this if your locale isn't installed on Mono - invalidReason = string.Format(Properties.Resources.NetFileCacheMonoNotSupported, nse.Message); - return false; - } - } - /// /// Stores the results of a given URL in the cache. /// Description is adjusted to be filesystem-safe and then appended to the file hash when saving. diff --git a/Core/Net/NetModuleCache.cs b/Core/Net/NetModuleCache.cs index dca351a359..b3db6165cb 100644 --- a/Core/Net/NetModuleCache.cs +++ b/Core/Net/NetModuleCache.cs @@ -4,6 +4,8 @@ using System.Threading; using System.Security.Cryptography; +using ICSharpCode.SharpZipLib.Zip; + namespace CKAN { /// @@ -16,6 +18,12 @@ namespace CKAN /// public class NetModuleCache : IDisposable { + static NetModuleCache() + { + // SharpZibLib 1.1.0 changed this to default to false, but we depend on it for international mods. + // https://github.com/icsharpcode/SharpZipLib/issues/591 + ZipStrings.UseUnicode = true; + } /// /// Initialize the cache @@ -66,9 +74,6 @@ public bool IsCached(CkanModule m, out string outFilename) outFilename = null; return false; } - public bool IsCachedZip(CkanModule m) - => m.download?.Any(dlUri => cache.IsCachedZip(dlUri)) - ?? false; public bool IsMaybeCachedZip(CkanModule m) => m.download?.Any(dlUri => cache.IsMaybeCachedZip(dlUri, m.release_date)) ?? false; @@ -76,10 +81,6 @@ public string GetCachedFilename(CkanModule m) => m.download?.Select(dlUri => cache.GetCachedFilename(dlUri, m.release_date)) .Where(filename => filename != null) .FirstOrDefault(); - public string GetCachedZip(CkanModule m) - => m.download?.Select(dlUri => cache.GetCachedZip(dlUri)) - .Where(filename => filename != null) - .FirstOrDefault(); public void GetSizeInfo(out int numFiles, out long numBytes, out long bytesFree) { cache.GetSizeInfo(out numFiles, out numBytes, out bytesFree); @@ -172,8 +173,7 @@ public string DescribeAvailability(CkanModule m) cancelToken.ThrowIfCancellationRequested(); // Check valid CRC - string invalidReason; - if (!NetFileCache.ZipValid(path, out invalidReason, new Progress(percent => + if (!ZipValid(path, out string invalidReason, new Progress(percent => progress?.Report(percent * zipValidPercent / 100)))) { throw new InvalidModuleFileKraken(module, path, string.Format( @@ -216,6 +216,87 @@ public string DescribeAvailability(CkanModule m) return success; } + /// + /// Check whether a ZIP file is valid + /// + /// Path to zip file to check + /// Description of problem with the file + /// Callback to notify as we traverse the input, called with percentages from 0 to 100 + /// + /// True if valid, false otherwise. See invalidReason param for explanation. + /// + public static bool ZipValid(string filename, out string invalidReason, IProgress progress) + { + try + { + if (filename != null) + { + using (ZipFile zip = new ZipFile(filename)) + { + string zipErr = null; + // Limit progress updates to 100 per ZIP file + long highestPercent = -1; + // Perform CRC and other checks + if (zip.TestArchive(true, TestStrategy.FindFirstError, + (TestStatus st, string msg) => + { + // This delegate is called as TestArchive proceeds through its + // steps, both routine and abnormal. + // The second parameter is non-null if an error occurred. + if (st != null && !st.EntryValid && !string.IsNullOrEmpty(msg)) + { + // Capture the error string so we can return it + zipErr = string.Format( + Properties.Resources.NetFileCacheZipError, + st.Operation, st.Entry?.Name, msg); + } + else if (st.Entry != null && progress != null) + { + // Report progress + var percent = 100 * st.Entry.ZipFileIndex / zip.Count; + if (percent > highestPercent) + { + progress.Report(percent); + highestPercent = percent; + } + } + })) + { + invalidReason = ""; + return true; + } + else + { + invalidReason = zipErr ?? Properties.Resources.NetFileCacheZipTestArchiveFalse; + return false; + } + } + } + else + { + invalidReason = Properties.Resources.NetFileCacheNullFileName; + return false; + } + } + catch (ZipException ze) + { + // Save the errors someplace useful + invalidReason = ze.Message; + return false; + } + catch (ArgumentException ex) + { + invalidReason = ex.Message; + return false; + } + catch (NotSupportedException nse) when (Platform.IsMono) + { + // SharpZipLib throws this if your locale isn't installed on Mono + invalidReason = string.Format(Properties.Resources.NetFileCacheMonoNotSupported, nse.Message); + return false; + } + } + /// /// Remove a module's download files from the cache /// diff --git a/Netkan/Services/CachingHttpService.cs b/Netkan/Services/CachingHttpService.cs index 76d7e30da7..ab3c12bc03 100644 --- a/Netkan/Services/CachingHttpService.cs +++ b/Netkan/Services/CachingHttpService.cs @@ -87,7 +87,7 @@ private string DownloadPackage(Uri url, string identifier, DateTime? updated, Ur case FileType.Zip: extension = "zip"; string invalidReason; - if (!NetFileCache.ZipValid(downloadedFile, out invalidReason, null)) + if (!NetModuleCache.ZipValid(downloadedFile, out invalidReason, null)) { log.Debug($"{url} is not a valid ZIP file: {invalidReason}"); File.Delete(downloadedFile); diff --git a/Tests/Core/Cache.cs b/Tests/Core/Cache.cs index 09d42a4a2f..4e66bcaa9f 100644 --- a/Tests/Core/Cache.cs +++ b/Tests/Core/Cache.cs @@ -161,21 +161,21 @@ public void ZipValidation() // We could use any URL, but this one is awesome. <3 Uri url = new Uri("http://kitte.nz/"); - Assert.IsFalse(cache.IsCachedZip(url)); + Assert.IsFalse(cache.IsCached(url)); // Store a bad zip. cache.Store(url, TestData.DogeCoinFlagZipCorrupt()); - // Make sure it's stored, but not valid as a zip + // Make sure it's stored Assert.IsTrue(cache.IsCached(url)); - Assert.IsFalse(cache.IsCachedZip(url)); + // Make sure it's not valid as a zip + Assert.IsFalse(NetModuleCache.ZipValid(cache.GetCachedFilename(url), out string invalidReason, null)); // Store a good zip. cache.Store(url, TestData.DogeCoinFlagZip()); // Make sure it's stored, and valid. Assert.IsTrue(cache.IsCached(url)); - Assert.IsTrue(cache.IsCachedZip(url)); } [Test] @@ -189,7 +189,7 @@ public void ZipValid_ContainsFilenameWithBadChars_NoException() bool valid = false; string reason = ""; Assert.DoesNotThrow(() => - valid = NetFileCache.ZipValid(TestData.ZipWithBadChars, out reason, null)); + valid = NetModuleCache.ZipValid(TestData.ZipWithBadChars, out reason, null)); // The file is considered valid on Linux; // only check the reason if found invalid @@ -212,7 +212,7 @@ public void ZipValid_ContainsFilenameWithUnicodeChars_Valid() string reason = null; Assert.DoesNotThrow(() => - valid = NetFileCache.ZipValid(TestData.ZipWithUnicodeChars, out reason, null)); + valid = NetModuleCache.ZipValid(TestData.ZipWithUnicodeChars, out reason, null)); Assert.IsTrue(valid, reason); } diff --git a/Tests/Core/ModuleInstallerTests.cs b/Tests/Core/ModuleInstallerTests.cs index 7262c3fe3a..3d021014d7 100644 --- a/Tests/Core/ModuleInstallerTests.cs +++ b/Tests/Core/ModuleInstallerTests.cs @@ -407,13 +407,13 @@ public void CanInstallMod() Assert.IsFalse(File.Exists(mod_file_path)); // Copy the zip file to the cache directory. - Assert.IsFalse(manager.Cache.IsCachedZip(TestData.DogeCoinFlag_101_module())); + Assert.IsFalse(manager.Cache.IsCached(TestData.DogeCoinFlag_101_module())); string cache_path = manager.Cache.Store(TestData.DogeCoinFlag_101_module(), TestData.DogeCoinFlagZip(), new Progress(bytes => {})); - Assert.IsTrue(manager.Cache.IsCachedZip(TestData.DogeCoinFlag_101_module())); + Assert.IsTrue(manager.Cache.IsCached(TestData.DogeCoinFlag_101_module())); Assert.IsTrue(File.Exists(cache_path)); var registry = CKAN.RegistryManager.Instance(manager.CurrentInstance, repoData.Manager).registry; diff --git a/Tests/Core/Net/NetAsyncModulesDownloaderTests.cs b/Tests/Core/Net/NetAsyncModulesDownloaderTests.cs index 219a8b9081..0bc51bef3f 100644 --- a/Tests/Core/Net/NetAsyncModulesDownloaderTests.cs +++ b/Tests/Core/Net/NetAsyncModulesDownloaderTests.cs @@ -171,8 +171,8 @@ public void SingleDownload() // Download our module. async.DownloadModules(modules); - // Assert that we have it, and it passes zip validation. - Assert.IsTrue(cache.IsCachedZip(kOS)); + // Assert that we have it (which meansit passed zip validation) + Assert.IsTrue(cache.IsCached(kOS)); } [Test] @@ -189,13 +189,13 @@ public void MultiDownload() modules.Add(kOS); modules.Add(quick_revert); - Assert.IsFalse(cache.IsCachedZip(kOS)); - Assert.IsFalse(cache.IsCachedZip(quick_revert)); + Assert.IsFalse(cache.IsCached(kOS)); + Assert.IsFalse(cache.IsCached(quick_revert)); async.DownloadModules(modules); - Assert.IsTrue(cache.IsCachedZip(kOS)); - Assert.IsTrue(cache.IsCachedZip(quick_revert)); + Assert.IsTrue(cache.IsCached(kOS)); + Assert.IsTrue(cache.IsCached(quick_revert)); } [Test] @@ -210,11 +210,11 @@ public void RandSdownload() modules.Add(rAndS); - Assert.IsFalse(cache.IsCachedZip(rAndS), "Module not yet downloaded"); + Assert.IsFalse(cache.IsCached(rAndS), "Module not yet downloaded"); async.DownloadModules(modules); - Assert.IsTrue(cache.IsCachedZip(rAndS), "Module download successful"); + Assert.IsTrue(cache.IsCached(rAndS), "Module download successful"); } }