Merge KSP-CKAN#4273 Don't raise inflation errors for single-hosted pr…

…ereleases of multi-hosted mods

CHANGELOG.md

@@ -33,6 +33,7 @@ All notable changes to this project will be documented in this file.
 - [Netkan] Create netkan schema (#4254, #4258 by: HebaruSan, Danny2462; reviewed: HebaruSan)
 - [Netkan] Handle raw GitHub URLs containing refs/heads/ (#4263 by: HebaruSan)
 - [Netkan] Restore default version support for SpaceDock (#4272 by: HebaruSan)
+- [Netkan] Don't raise inflation errors for single-hosted prereleases of multi-hosted mods (#4273 by: HebaruSan)
 
 ## v1.35.2 (Penrose)
 

Netkan/CmdLineOptions.cs

@@ -46,9 +46,12 @@ internal class CmdLineOptions
         [Option("queues", HelpText = "Input,Output queue names for Queue Inflator mode")]
         public string? Queues { get; set; }
 
-        [Option("highest-version", HelpText = "Highest known version for auto-epoching")]
+        [Option("highest-version", HelpText = "Highest known non-prerelease version for auto-epoching")]
         public string? HighestVersion { get; set; }
 
+        [Option("highest-version-prerelease", HelpText = "Highest known prerelease version for auto-epoching")]
+        public string? HighestVersionPrerelease { get; set; }
+
         [Option("validate-ckan", HelpText = "Name of .ckan file to check for errors")]
         public string? ValidateCkan { get; set; }
 

Netkan/Processors/QueueHandler.cs

@@ -152,13 +152,20 @@ private IEnumerable<SendMessageBatchRequestEntry> Inflate(Message msg)
                 highVer = new ModuleVersion(highVerAttr.StringValue);
             }
 
+            ModuleVersion? highVerPre = null;
+            if (msg.MessageAttributes.TryGetValue("HighestVersionPrerelease", out MessageAttributeValue? highVerPreAttr))
+            {
+                highVerPre = new ModuleVersion(highVerPreAttr.StringValue);
+            }
+
             log.InfoFormat("Inflating {0}", netkans.First().Identifier);
             IEnumerable<Metadata>? ckans = null;
             bool    caught        = false;
             string? caughtMessage = null;
             var     opts          = new TransformOptions(releases,
                                                          null,
                                                          highVer,
+                                                         highVerPre,
                                                          netkans.First().Staged,
                                                          netkans.First().StagingReason);
             try

Netkan/Program.cs

@@ -109,6 +109,7 @@ is string[] array
                                 ParseReleases(Options.Releases),
                                 ParseSkipReleases(Options.SkipReleases),
                                 ParseHighestVersion(Options.HighestVersion),
+                                ParseHighestVersion(Options.HighestVersionPrerelease),
                                 netkans.First().Staged,
                                 netkans.First().StagingReason))
                         .ToArray();

Netkan/Transformers/EpochTransformer.cs

@@ -1,7 +1,10 @@
 using System;
+using System.Linq;
 using System.Collections.Generic;
+
 using log4net;
 using Newtonsoft.Json.Linq;
+
 using CKAN.Versioning;
 using CKAN.NetKAN.Model;
 
@@ -29,7 +32,7 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions? opts
 
                 if (uint.TryParse(epoch.ToString(), out uint epochNumber))
                 {
-                    //Implicit if zero. No need to add
+                    // Implicit if zero. No need to add
                     if (epochNumber != 0)
                     {
                         json["version"] = epochNumber + ":" + json["version"];
@@ -43,40 +46,62 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions? opts
                 }
             }
 
-            if (json.TryGetValue("x_netkan_allow_out_of_order", out JToken? allowOOO) && (bool)allowOOO)
+            if (json.TryGetValue("x_netkan_allow_out_of_order", out JToken? allowOOO)
+                && (bool)allowOOO)
             {
                 Log.Debug("Out of order versions enabled in netkan, skipping OOO check");
             }
-            else if (opts?.HighestVersion != null && (string?)json["version"] is string v)
+            else if (opts != null
+                     && (string?)json["version"] is string v
+                     && (metadata.Prerelease
+                             ? new ModuleVersion?[]
+                               {
+                                   opts.HighestVersionPrerelease,
+                                   opts.HighestVersion,
+                               }
+                             : new ModuleVersion?[]
+                               {
+                                   opts.HighestVersion,
+                               })
+                            .OfType<ModuleVersion>()
+                            .Max()
+                        is ModuleVersion highest)
+            {
+                json["version"] = CheckOutOfOrder(opts, highest,
+                                                  new ModuleVersion(v))
+                                      .ToString();
+            }
+
+            yield return new Metadata(json);
+        }
+
+        private static ModuleVersion CheckOutOfOrder(TransformOptions opts,
+                                                     ModuleVersion    highest,
+                                                     ModuleVersion    start)
+        {
+            // Ensure we are greater or equal to the previous max
+            ModuleVersion current = start;
+            while (current < highest)
+            {
+                Log.DebugFormat("Auto-epoching out of order version: {0} < {1}",
+                                current, highest);
+                // Increment epoch if too small
+                current = current.IncrementEpoch();
+            }
+            if (!highest.EpochEquals(current) && start < highest && highest < current)
             {
-                // Ensure we are greater or equal to the previous max
-                ModuleVersion startV = new ModuleVersion(v);
-                ModuleVersion currentV = startV;
-                while (currentV < opts.HighestVersion)
+                if (opts.FlakyAPI)
                 {
-                    Log.DebugFormat("Auto-epoching out of order version: {0} < {1}",
-                        currentV, opts.HighestVersion);
-                    // Increment epoch if too small
-                    currentV = currentV.IncrementEpoch();
+                    throw new Kraken($"Out-of-order version found on unreliable server: {start} < {highest} < {current}");
                 }
-                if (!opts.HighestVersion.EpochEquals(currentV)
-                    && startV < opts.HighestVersion && opts.HighestVersion < currentV)
+                else
                 {
-                    if (opts.FlakyAPI)
-                    {
-                        throw new Kraken($"Out-of-order version found on unreliable server: {startV} < {opts.HighestVersion} < {currentV}");
-                    }
-                    else
-                    {
-                        // New file, tell the Indexer to be careful
-                        opts.Staged = true;
-                        opts.StagingReasons.Add($"Auto-epoching out of order version: {startV} < {opts.HighestVersion} < {currentV}");
-                    }
+                    // New file, tell the Indexer to be careful
+                    opts.Staged = true;
+                    opts.StagingReasons.Add($"Auto-epoching out of order version: {start} < {highest} < {current}");
                 }
-                json["version"] = currentV.ToString();
             }
-
-            yield return new Metadata(json);
+            return current;
         }
     }
 }

Netkan/Transformers/ITransformer.cs

@@ -9,14 +9,16 @@ internal class TransformOptions
         public TransformOptions(int?           releases,
                                 int?           skipReleases,
                                 ModuleVersion? highVer,
+                                ModuleVersion? highVerPre,
                                 bool           staged,
                                 string?        stagingReason)
         {
-            Releases       = releases;
-            SkipReleases   = skipReleases;
-            HighestVersion = highVer;
-            Staged         = staged;
-            StagingReasons = new List<string>();
+            Releases                 = releases;
+            SkipReleases             = skipReleases;
+            HighestVersion           = highVer;
+            HighestVersionPrerelease = highVerPre;
+            Staged                   = staged;
+            StagingReasons           = new List<string>();
             if (stagingReason != null && !string.IsNullOrEmpty(stagingReason))
             {
                 StagingReasons.Add(stagingReason);
@@ -26,6 +28,7 @@ public TransformOptions(int?           releases,
         public readonly int?           Releases;
         public readonly int?           SkipReleases;
         public readonly ModuleVersion? HighestVersion;
+        public readonly ModuleVersion? HighestVersionPrerelease;
         public          bool           Staged;
         public readonly List<string>   StagingReasons;
         public          bool           FlakyAPI = false;

Tests/NetKAN/MainClass.cs

@@ -11,7 +11,7 @@ namespace Tests.NetKAN
     [TestFixture]
     public class MainClassTests
     {
-        private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null);
+        private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null);
 
         [Test]
         public void FixVersionStringsUnharmed()

Tests/NetKAN/NetkanOverride.cs

@@ -13,7 +13,7 @@ namespace Tests.NetKAN
     public class NetkanOverride
     {
         private JObject? such_metadata;
-        private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null);
+        private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null);
 
         [SetUp]
         public void Setup()

Tests/NetKAN/Transformers/AvcKrefTransformerTests.cs

@@ -14,7 +14,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class AvcKrefTransformerTests
     {
-        private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null);
+        private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null);
 
         [Test,
             TestCase(

Tests/NetKAN/Transformers/AvcTransformerTests.cs

@@ -18,7 +18,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class AvcTransformerTests
     {
-        private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null);
+        private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null);
         private readonly IGame game = new KerbalSpaceProgram();
 
         [Test]

Tests/NetKAN/Transformers/CurseTransformerTests.cs

@@ -12,7 +12,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class CurseTransformerTests
     {
-        private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null);
+        private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null);
 
         // GH #199: Don't pre-fill KSP version fields if we see a ksp_min/max
         [Test]

Tests/NetKAN/Transformers/DownloadAttributeTransformerTests.cs

@@ -11,7 +11,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class DownloadAttributeTransformerTests
     {
-        private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null);
+        private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null);
 
         [Test]
         [Category("Cache")]

Tests/NetKAN/Transformers/EpochTransformerTests.cs

@@ -10,33 +10,56 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class EpochTransformerTests
     {
-        [Test,
-            TestCase("1.1",     null,    "1.1"),
-            TestCase("1.1",    "1.1",    "1.1"),
-            TestCase("1.1",    "1.2",  "1:1.1"),
-            TestCase("1.1",  "5:1.1",  "5:1.1"),
-            TestCase("1.1",  "5:1.2",  "6:1.1"),
-            TestCase("0.7",   "0.65",  "1:0.7"),
-            TestCase("2.5",   "v2.4",  "1:2.5"),
-            TestCase("2.5",   "V2.4",  "1:2.5"),
-            TestCase("v2.5", "vV2.4", "1:v2.5"),
+        [
+            TestCase("1.1",     null,   null, false,    "1.1", false),
+            TestCase("1.1",     null,   null,  true,    "1.1", false),
+            TestCase("1.1",    "1.1",   null, false,    "1.1", false),
+            TestCase("1.1",    "1.2",   null, false,  "1:1.1", true),
+            TestCase("1.1",  "5:1.1",   null, false,  "5:1.1", false),
+            TestCase("1.1",  "5:1.2",   null, false,  "6:1.1", true),
+            TestCase("0.7",   "0.65",   null, false,  "1:0.7", true),
+            TestCase("2.5",   "v2.4",   null, false,  "1:2.5", true),
+            TestCase("2.5",   "V2.4",   null, false,  "1:2.5", true),
+            TestCase("v2.5", "vV2.4",   null, false, "1:v2.5", true),
+
+            // Non-prerelease with a prerelease on another host
+            TestCase("1.0",    "1.0",  "2.0", false,    "1.0", false),
+            // Prerelease with no normal releases
+            TestCase("1.0",     null,  "1.0",  true,    "1.0", false),
+            // Updating a prerelease to a normal release
+            TestCase("2.0",    "1.0",  "2.0", false,    "2.0", false),
+            // Prerelease needing an epoch boost
+            TestCase("2.0",    "1.0",  "3.0",  true,  "1:2.0", true),
+            // The previous prerelease is old
+            TestCase("1.5",    "2.0",  "1.0",  true,  "1:1.5", true),
         ]
-        public void Transform_WithHighVersionParam_MatchesExpected(string version, string highVer, string expected)
+        public void Transform_WithHighVersionParam_MatchesExpected(string  version,
+                                                                   string? highVer,
+                                                                   string? highVerPre,
+                                                                   bool    prerelease,
+                                                                   string  expected,
+                                                                   bool    staged)
         {
             // Arrange
             var json = new JObject()
             {
-                { "spec_version", "v1.4"       },
-                { "identifier",   "AwesomeMod" },
-                { "version",      version      },
+                { "spec_version",   "v1.4"          },
+                { "identifier",     "AwesomeMod"    },
+                { "version",        version         },
+                { "release_status", prerelease
+                                        ? "testing"
+                                        : "stable"  },
             };
             ITransformer     sut  = new EpochTransformer();
             TransformOptions opts = new TransformOptions(
                 1,
                 null,
-                string.IsNullOrEmpty(highVer)
+                highVer is null or ""
                     ? null
                     : new ModuleVersion(highVer),
+                highVerPre is null or ""
+                    ? null
+                    : new ModuleVersion(highVerPre),
                 false,
                 null
             );
@@ -47,6 +70,7 @@ public void Transform_WithHighVersionParam_MatchesExpected(string version, strin
 
             // Assert
             Assert.AreEqual(expected, (string?)transformedJson["version"]);
+            Assert.AreEqual(staged,   opts.Staged);
         }
     }
 }

Tests/NetKAN/Transformers/GeneratedByTransformerTests.cs

@@ -9,7 +9,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class GeneratedByTransformerTests
     {
-        private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null);
+        private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null);
 
         [Test]
         public void AddsGeneratedByProperty()

Tests/NetKAN/Transformers/GithubTransformerTests.cs

@@ -14,7 +14,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class GithubTransformerTests
     {
-        private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null);
+        private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null);
 
         private Mock<IGithubApi>? apiMockUp;
 
@@ -192,7 +192,7 @@ public void Transform_MultipleReleases_TransformsAll()
             // Act
             var results = sut.Transform(
                 new Metadata(json),
-                new TransformOptions(2, null, null, false, null)
+                new TransformOptions(2, null, null, null, false, null)
             );
             var transformedJsons = results.Select(result => result.Json()).ToArray();
 
@@ -230,7 +230,7 @@ public void Transform_MultipleAssets_TransformsAll()
             // Act
             var results = sut.Transform(
                 new Metadata(json),
-                new TransformOptions(1, 3, null, false, null)
+                new TransformOptions(1, 3, null, null, false, null)
             );
             var transformedJsons = results.Select(result => result.Json()).ToArray();
 
@@ -268,7 +268,7 @@ public void Transform_SkipReleases_SkipsCorrectly()
             // Act
             var results = sut.Transform(
                 new Metadata(json),
-                new TransformOptions(2, 1, null, false, null)
+                new TransformOptions(2, 1, null, null, false, null)
             ).ToArray();
 
             // Assert

Tests/NetKAN/Transformers/HttpTransformerTests.cs

@@ -9,7 +9,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class HttpTransformerTests
     {
-        private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null);
+        private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null);
 
         [TestCase("#/ckan/github/foo/bar")]
         [TestCase("#/ckan/netkan/http://awesomemod.example/awesomemod.netkan")]

Tests/NetKAN/Transformers/InstallSizeTransformerTests.cs

@@ -38,6 +38,6 @@ public void Transform_NormalModule_CorrectInstallSize()
             Assert.AreEqual(52291, (int?)transformedJson["install_size"]);
         }
 
-        private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null);
+        private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null);
     }
 }

Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs

@@ -15,7 +15,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class InternalCkanTransformerTests
     {
-        private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null);
+        private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null);
 
         [Test]
         public void AddsMissingProperties()