Configurable cache size limit

Cmdline/Action/Cache.cs

@@ -25,6 +25,12 @@ private class CacheSubOptions : VerbCommandOptions
             [VerbOption("reset", HelpText = "Set the download cache path to the default")]
             public CommonOptions ResetOptions { get; set; }
 
+            [VerbOption("showlimit", HelpText = "Show the cache size limit")]
+            public CommonOptions ShowLimitOptions { get; set; }
+
+            [VerbOption("setlimit", HelpText = "Set the cache size limit")]
+            public SetLimitOptions SetLimitOptions { get; set; }
+
             [HelpVerbOption]
             public string GetUsage(string verb)
             {
@@ -45,11 +51,15 @@ public string GetUsage(string verb)
                         case "set":
                             ht.AddPreOptionsLine($"Usage: ckan cache {verb} [options] path");
                             break;
+                        case "setlimit":
+                            ht.AddPreOptionsLine($"Usage: ckan cache {verb} [options] megabytes");
+                            break;
 
                         // Now the commands with only --flag type options
                         case "list":
                         case "clear":
                         case "reset":
+                        case "showlimit":
                         default:
                             ht.AddPreOptionsLine($"Usage: ckan cache {verb} [options]");
                             break;
@@ -65,6 +75,12 @@ private class SetOptions : CommonOptions
             public string Path { get; set; }
         }
 
+        private class SetLimitOptions : CommonOptions
+        {
+            [ValueOption(0)]
+            public long Megabytes { get; set; } = -1;
+        }
+
         /// <summary>
         /// Execute a cache subcommand
         /// </summary>
@@ -111,6 +127,14 @@ public int RunSubCommand(KSPManager mgr, CommonOptions opts, SubCommandOptions u
                             exitCode = ResetCacheDirectory((CommonOptions)suboptions);
                             break;
 
+                        case "showlimit":
+                            exitCode = ShowCacheSizeLimit((CommonOptions)suboptions);
+                            break;
+
+                        case "setlimit":
+                            exitCode = SetCacheSizeLimit((SetLimitOptions)suboptions);
+                            break;
+
                         default:
                             user.RaiseMessage("Unknown command: cache {0}", option);
                             exitCode = Exit.BADOPT;
@@ -176,6 +200,34 @@ private int ResetCacheDirectory(CommonOptions options)
             return Exit.OK;
         }
 
+        private int ShowCacheSizeLimit(CommonOptions options)
+        {
+            IWin32Registry winReg = new Win32Registry();
+            if (winReg.CacheSizeLimit.HasValue)
+            {
+                user.RaiseMessage(CkanModule.FmtSize(winReg.CacheSizeLimit.Value));
+            }
+            else
+            {
+                user.RaiseMessage("Unlimited");
+            }
+            return Exit.OK;
+        }
+
+        private int SetCacheSizeLimit(SetLimitOptions options)
+        {
+            IWin32Registry winReg = new Win32Registry();
+            if (options.Megabytes < 0)
+            {
+                winReg.CacheSizeLimit = null;
+            }
+            else
+            {
+                winReg.CacheSizeLimit = options.Megabytes * (long)1024 * (long)1024;
+            }
+            return ShowCacheSizeLimit(null);
+        }
+
         private void printCacheInfo()
         {
             int fileCount;
@@ -187,7 +239,7 @@ private void printCacheInfo()
         private KSPManager manager;
         private IUser      user;
 
-        private static readonly ILog       log = LogManager.GetLogger(typeof(Cache));
+        private static readonly ILog log = LogManager.GetLogger(typeof(Cache));
     }
 
 }

Core/ModuleInstaller.cs

@@ -219,6 +219,8 @@ public void InstallList(ICollection<CkanModule> modules, RelationshipResolverOpt
 
             }
 
+            EnforceCacheSizeLimit();
+
             // We can scan GameData as a separate transaction. Installing the mods
             // leaves everything consistent, and this is just gravy. (And ScanGameData
             // acts as a Tx, anyway, so we don't need to provide our own.)
@@ -253,6 +255,8 @@ public void InstallList(ModuleResolution modules, RelationshipResolverOptions op
                 User.RaiseProgress("Committing filesystem changes", 80);
 
                 transaction.Complete();
+
+                EnforceCacheSizeLimit();
             }
         }
 
@@ -996,6 +1000,8 @@ public void AddRemove(IEnumerable<CkanModule> add = null, IEnumerable<string> re
                 registry_manager.Save(enforceConsistency);
 
                 tx.Complete();
+
+                EnforceCacheSizeLimit();
             }
         }
 
@@ -1157,6 +1163,18 @@ public void ImportFiles(HashSet<FileInfo> files, IUser user, Action<CkanModule>
                     f.Delete();
                 }
             }
+
+            EnforceCacheSizeLimit();
+        }
+
+        private void EnforceCacheSizeLimit()
+        {
+            // Purge old downloads if we're over the limit
+            Win32Registry winReg = new Win32Registry();
+            if (winReg.CacheSizeLimit.HasValue)
+            {
+                Cache.EnforceSizeLimit(winReg.CacheSizeLimit.Value, registry_manager.registry);
+            }
         }
 
         /// <summary>

Core/Net/NetFileCache.cs

@@ -11,6 +11,7 @@
 using ICSharpCode.SharpZipLib.Zip;
 using log4net;
 using CKAN.Extensions;
+using CKAN.Versioning;
 
 namespace CKAN
 {
@@ -290,6 +291,103 @@ private HashSet<string> legacyDirs()
                 .ToHashSet();
         }
 
+        public void EnforceSizeLimit(long bytes, Registry registry)
+        {
+            int numFiles;
+            long curBytes;
+            GetSizeInfo(out numFiles, out curBytes);
+            if (curBytes > bytes)
+            {
+                // This object will let us determine whether a module is compatible with any of our instances
+                KspVersionCriteria aggregateCriteria = manager?.Instances.Values
+                    .Where(ksp => ksp.Valid)
+                    .Select(ksp => ksp.VersionCriteria())
+                    .Aggregate((a, b) => a.Union(b));
+
+                // This object lets us find the modules associated with a cached file
+                Dictionary<string, List<CkanModule>> hashMap = registry.GetDownloadHashIndex();
+
+                // Prune the module lists to only those that are compatible
+                foreach (var kvp in hashMap)
+                {
+                    kvp.Value.RemoveAll(mod => !mod.IsCompatibleKSP(aggregateCriteria));
+                }
+
+                // Now get all the files in all the caches...
+                List<FileInfo> files = allFiles();
+                // ... and sort them by compatibilty and timestamp...
+                files.Sort((a, b) => compareFiles(
+                    hashMap, aggregateCriteria, a, b
+                ));
+
+                // ... and delete them till we're under the limit
+                foreach (FileInfo fi in files)
+                {
+                    curBytes -= fi.Length;
+                    fi.Delete();
+                    if (curBytes <= bytes)
+                    {
+                        // Limit met, all done!
+                        break;
+                    }
+                }
+                OnCacheChanged();
+            }
+        }
+
+        private int compareFiles(Dictionary<string, List<CkanModule>> hashMap, KspVersionCriteria crit, FileInfo a, FileInfo b)
+        {
+            // Compatible modules for file A
+            List<CkanModule> modulesA;
+            hashMap.TryGetValue(a.Name.Substring(0, 8), out modulesA);
+            bool compatA = modulesA?.Any() ?? false;
+
+            // Compatible modules for file B
+            List<CkanModule> modulesB;
+            hashMap.TryGetValue(b.Name.Substring(0, 8), out modulesB);
+            bool compatB = modulesB?.Any() ?? false;
+
+            if (modulesA == null && modulesB != null)
+            {
+                // A isn't indexed but B is, delete A first
+                return -1;
+            }
+            else if (modulesA != null && modulesB == null)
+            {
+                // A is indexed but B isn't, delete B first
+                return 1;
+            }
+            else if (!compatA && compatB)
+            {
+                // A isn't compatible but B is, delete A first
+                return -1;
+            }
+            else if (compatA && !compatB)
+            {
+                // A is compatible but B isn't, delete B first
+                return 1;
+            }
+            else
+            {
+                // Both are either compatible or incompatible
+                // Go by file age, oldest first
+                return (int)(a.CreationTime - b.CreationTime).TotalSeconds;
+            }
+            return 0;
+        }
+
+        private List<FileInfo> allFiles()
+        {
+            DirectoryInfo mainDir = new DirectoryInfo(cachePath);
+            var files = mainDir.EnumerateFiles();
+            foreach (string legacyDir in legacyDirs())
+            {
+                DirectoryInfo legDir = new DirectoryInfo(legacyDir);
+                files = files.Union(legDir.EnumerateFiles());
+            }
+            return files.ToList();
+        }
+
         /// <summary>
         /// Check whether a ZIP file is valid
         /// </summary>

Core/Net/NetModuleCache.cs

@@ -74,6 +74,10 @@ public void GetSizeInfo(out int numFiles, out long numBytes)
         {
             cache.GetSizeInfo(out numFiles, out numBytes);
         }
+        public void EnforceSizeLimit(long bytes, Registry registry)
+        {
+            cache.EnforceSizeLimit(bytes, registry);
+        }
 
         /// <summary>
         /// Calculate the SHA1 hash of a file

Core/Registry/Registry.cs

@@ -1097,14 +1097,20 @@ public HashSet<string> FindReverseDependencies(IEnumerable<string> modules_to_re
         public Dictionary<string, List<CkanModule>> GetSha1Index()
         {
             var index = new Dictionary<string, List<CkanModule>>();
-            foreach (var kvp in available_modules) {
+            foreach (var kvp in available_modules)
+            {
                 AvailableModule am = kvp.Value;
-                foreach (var kvp2 in am.module_version) {
+                foreach (var kvp2 in am.module_version)
+                {
                     CkanModule mod = kvp2.Value;
-                    if (mod.download_hash != null) {
-                        if (index.ContainsKey(mod.download_hash.sha1)) {
+                    if (mod.download_hash != null)
+                    {
+                        if (index.ContainsKey(mod.download_hash.sha1))
+                        {
                             index[mod.download_hash.sha1].Add(mod);
-                        } else {
+                        }
+                        else
+                        {
                             index.Add(mod.download_hash.sha1, new List<CkanModule>() {mod});
                         }
                     }
@@ -1113,5 +1119,38 @@ public Dictionary<string, List<CkanModule>> GetSha1Index()
             return index;
         }
 
+        /// <summary>
+        /// Get a dictionary of all mod versions indexed by their download URLs' hash.
+        /// Useful for finding the mods for a group of URLs without repeatedly searching the entire registry.
+        /// </summary>
+        /// <returns>
+        /// dictionary[urlHash] = {mod1, mod2, mod3};
+        /// </returns>
+        public Dictionary<string, List<CkanModule>> GetDownloadHashIndex()
+        {
+            var index = new Dictionary<string, List<CkanModule>>();
+            foreach (var kvp in available_modules)
+            {
+                AvailableModule am = kvp.Value;
+                foreach (var kvp2 in am.module_version)
+                {
+                    CkanModule mod = kvp2.Value;
+                    if (mod.download != null)
+                    {
+                        string hash = NetFileCache.CreateURLHash(mod.download);
+                        if (index.ContainsKey(hash))
+                        {
+                            index[hash].Add(mod);
+                        }
+                        else
+                        {
+                            index.Add(hash, new List<CkanModule>() {mod});
+                        }
+                    }
+                }
+            }
+            return index;
+        }
+
     }
 }

Core/Versioning/KspVersionCriteria.cs

@@ -6,7 +6,7 @@ namespace CKAN.Versioning
 {
     public class KspVersionCriteria
     {
-        private List<KspVersion> _versions = new List<KspVersion> ();
+        private List<KspVersion> _versions = new List<KspVersion>();
 
         public KspVersionCriteria (KspVersion v)
         {
@@ -34,6 +34,14 @@ public IList<KspVersion> Versions
             }
         }
 
+        public KspVersionCriteria Union(KspVersionCriteria other)
+        {
+            return new KspVersionCriteria(
+                null,
+                _versions.Union(other.Versions).ToList()
+            );
+        }
+
         public override String ToString()
         {
             return "[Versions: " + _versions.ToString() + "]";

Core/Win32Registry.cs

@@ -14,6 +14,7 @@ public interface IWin32Registry
         string GetKSPBuilds();
         void SetKSPBuilds(string buildMap);
         string DownloadCacheDir { get; set; }
+        long? CacheSizeLimit { get; set; }
     }
 
     public class Win32Registry : IWin32Registry
@@ -58,6 +59,30 @@ public string DownloadCacheDir
             }
         }
 
+        /// <summary>
+        /// Get and set the maximum number of bytes allowed in the cache.
+        /// Unlimited if null.
+        /// </summary>
+        public long? CacheSizeLimit
+        {
+            get
+            {
+                string val = GetRegistryValue<string>(@"CacheSizeLimit", null);
+                return string.IsNullOrEmpty(val) ? null : (long?)Convert.ToInt64(val);
+            }
+            set
+            {
+                if (!value.HasValue)
+                {
+                    DeleteRegistryValue(@"CacheSizeLimit");
+                }
+                else
+                {
+                    SetRegistryValue(@"CacheSizeLimit", value.Value);
+                }
+            }
+        }
+
         private int InstanceCount
         {
             get { return GetRegistryValue(@"KSPInstanceCount", 0); }