Merge #3526 Get licenses from embedded schema, skip bad modules in de…

…serialize

CHANGELOG.md

@@ -38,6 +38,7 @@ All notable changes to this project will be documented in this file.
 - [Core] Sort exported modpack relationships by identifier (#3499 by: HebaruSan; reviewed: DasSkelett)
 - [Core] Disable tx timeouts, add tx debug logging, static DLL pattern (#3512 by: HebaruSan; reviewed: DasSkelett)
 - [Core] Only delete diversely capitalized directories once on Windows (#3528 by: HebaruSan; reviewed: DasSkelett)
+- [Core] Get licenses from embedded schema, skip bad modules in deserialize (#3526 by: HebaruSan; reviewed: DasSkelett)
 
 ### Internal
 

CKAN.schema

@@ -477,6 +477,41 @@
                 "open-source", "restricted", "unrestricted", "unknown"
             ]
         },
+        "redistributable_license": {
+            "description" : "A license that grants us the right to redistribute.",
+            "enum" : [
+                "public-domain",
+                "Apache", "Apache-1.0", "Apache-2.0",
+                "Artistic", "Artistic-1.0", "Artistic-2.0",
+                "BSD-2-clause", "BSD-3-clause", "BSD-4-clause",
+                "ISC",
+                "CC-BY", "CC-BY-1.0", "CC-BY-2.0", "CC-BY-2.5", "CC-BY-3.0", "CC-BY-4.0",
+                "CC-BY-SA", "CC-BY-SA-1.0", "CC-BY-SA-2.0", "CC-BY-SA-2.5", "CC-BY-SA-3.0", "CC-BY-SA-4.0",
+                "CC-BY-NC", "CC-BY-NC-1.0", "CC-BY-NC-2.0", "CC-BY-NC-2.5", "CC-BY-NC-3.0", "CC-BY-NC-4.0",
+                "CC-BY-NC-SA", "CC-BY-NC-SA-1.0", "CC-BY-NC-SA-2.0", "CC-BY-NC-SA-2.5", "CC-BY-NC-SA-3.0", "CC-BY-NC-SA-4.0",
+                "CC-BY-NC-ND", "CC-BY-NC-ND-1.0", "CC-BY-NC-ND-2.0", "CC-BY-NC-ND-2.5", "CC-BY-NC-ND-3.0", "CC-BY-NC-ND-4.0",
+                "CC-BY-ND", "CC-BY-ND-1.0", "CC-BY-ND-2.0", "CC-BY-ND-2.5", "CC-BY-ND-3.0", "CC-BY-ND-4.0",
+                "CC0",
+                "CDDL", "CPL",
+                "EFL-1.0", "EFL-2.0",
+                "Expat", "MIT",
+                "GPL-1.0", "GPL-2.0", "GPL-3.0",
+                "LGPL-2.0", "LGPL-2.1", "LGPL-3.0",
+                "GFDL-1.0", "GFDL-1.1", "GFDL-1.2", "GFDL-1.3",
+                "GFDL-NIV-1.0", "GFDL-NIV-1.1", "GFDL-NIV-1.2", "GFDL-NIV-1.3",
+                "LPPL-1.0", "LPPL-1.1", "LPPL-1.2", "LPPL-1.3c",
+                "MPL-1.1", "MPL-2.0",
+                "Perl",
+                "Python-2.0",
+                "QPL-1.0",
+                "W3C",
+                "Zlib",
+                "Zope",
+                "WTFPL",
+                "Unlicense",
+                "open-source", "unrestricted"
+            ]
+        },
         "licenses" : {
             "description" : "A license, or array of licenses",
             "anyOf" : [

Core/CKAN-core.csproj

@@ -41,6 +41,7 @@
     <PackageReference Include="SharpZipLib" Version="1.3.3" />
     <PackageReference Include="log4net" Version="2.0.10" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+    <PackageReference Include="NJsonSchema" Version="10.0.27" />
   </ItemGroup>
   <ItemGroup Condition=" '$(TargetFramework)' == 'net45' ">
     <PackageReference Include="ChinhDo.Transactions.FileManager" Version="1.2.0" />
@@ -64,6 +65,9 @@
   </ItemGroup>
   <ItemGroup>
     <EmbeddedResource Include="builds.json" />
+    <EmbeddedResource Include="..\CKAN.schema">
+      <LogicalName>CKAN.Core.CKAN.schema</LogicalName>
+    </EmbeddedResource>
   </ItemGroup>
   <ItemGroup>
     <Content Include="log4net.xml">

Core/Converters/JsonLeakySortedDictionaryConverter.cs

@@ -0,0 +1,64 @@
+using System;
+using System.Collections.Generic;
+
+using log4net;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+namespace CKAN
+{
+    /// <summary>
+    /// [De]serializes a dictionary that might have some questionably
+    /// valid data in it.
+    /// If exceptions are thrown for any key/value pair, leave it out.
+    /// Removes CkanModule objects from AvailableModule.module_version
+    /// if License throws BadMetadataKraken.
+    /// </summary>
+    public class JsonLeakySortedDictionaryConverter<K, V> : JsonConverter
+    {
+        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+        {
+            var dict = new SortedDictionary<K, V>();
+            foreach (var kvp in JObject.Load(reader))
+            {
+                try
+                {
+                    dict.Add(
+                        (K)Activator.CreateInstance(typeof(K), kvp.Key),
+                        kvp.Value.ToObject<V>());
+                }
+                catch (Exception exc)
+                {
+                    log.Warn($"Failed to deserialize {kvp.Key}: {kvp.Value}", exc);
+                }
+            }
+            return dict;
+        }
+
+        /// <summary>
+        /// Use default serializer for writing
+        /// </summary>
+        public override bool CanWrite => false;
+
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        {
+            throw new NotImplementedException();
+        }
+
+        /// <summary>
+        /// We *only* want to be triggered for types that have explicitly
+        /// set an attribute in their class saying they can be converted.
+        /// By returning false here, we declare we're not interested in participating
+        /// in any other conversions.
+        /// </summary>
+        /// <returns>
+        /// false
+        /// </returns>
+        public override bool CanConvert(Type objectType)
+        {
+            return false;
+        }
+
+        private static readonly ILog log = LogManager.GetLogger(typeof(JsonLeakySortedDictionaryConverter<K, V>));
+    }
+}

Core/Converters/JsonSimpleStringConverter.cs

@@ -1,32 +1,38 @@
 using System;
-using Newtonsoft.Json;
-
-namespace CKAN {
-    // A lovely class for serialising things that can be converted
-    // to simple strings and back.
 
-    public class JsonSimpleStringConverter : JsonConverter {
+using Newtonsoft.Json;
 
+namespace CKAN
+{
+    /// <summary>
+    /// Serialises things that can be converted
+    /// to simple strings and back.
+    /// </summary>
+    public class JsonSimpleStringConverter : JsonConverter
+    {
         public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
         {
-            writer.WriteValue (value.ToString ());
+            writer.WriteValue(value.ToString());
         }
 
         public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
         {
-
             // If we find a null, then that might be okay, so we pass it down to our
             // activator. Otherwise we convert to string, since that's our job.
-            string value = reader.Value == null ? null : reader.Value.ToString();
-            return Activator.CreateInstance (objectType, value);
+            return Activator.CreateInstance(objectType, reader.Value?.ToString());
         }
 
+        /// <summary>
+        /// We *only* want to be triggered for types that have explicitly
+        /// set an attribute in their class saying they can be converted.
+        /// By returning false here, we declare we're not interested in participating
+        /// in any other conversions.
+        /// </summary>
+        /// <returns>
+        /// false
+        /// </returns>
         public override bool CanConvert(Type objectType)
         {
-            // We *only* want to be triggered for types that have explicitly
-            // set an attribute in their class saying they can be converted.
-            // By returning false here, we declare we're not interested in participating
-            // in any other conversions.
             return false;
         }
     }

Core/Converters/JsonSingleOrArrayConverter.cs

@@ -1,25 +1,17 @@
 using System;
 using System.Collections.Generic;
+
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 
 namespace CKAN
 {
-
-    // With thanks to 
-    // https://stackoverflow.com/questions/18994685/how-to-handle-both-a-single-item-and-an-array-for-the-same-property-using-json-n
-
+    /// <summary>
+    /// With thanks to 
+    /// https://stackoverflow.com/questions/18994685/how-to-handle-both-a-single-item-and-an-array-for-the-same-property-using-json-n
+    /// </summary>
     public class JsonSingleOrArrayConverter<T> : JsonConverter
     {
-        public override bool CanConvert(Type object_type)
-        {
-            // We *only* want to be triggered for types that have explicitly
-            // set an attribute in their class saying they can be converted.
-            // By returning false here, we declare we're not interested in participating
-            // in any other conversions.
-            return false;
-        }
-
         public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
         {
             JToken token = JToken.Load(reader);
@@ -32,15 +24,25 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
             return token.ToObject<T>() == null ? null : new List<T> { token.ToObject<T>() };
         }
 
-        public override bool CanWrite
-        {
-            get { return false; }
-        }
+        public override bool CanWrite => false;
 
         public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
         {
             throw new NotImplementedException();
         }
+
+        /// <summary>
+        /// We *only* want to be triggered for types that have explicitly
+        /// set an attribute in their class saying they can be converted.
+        /// By returning false here, we declare we're not interested in participating
+        /// in any other conversions.
+        /// </summary>
+        /// <returns>
+        /// false
+        /// </returns>
+        public override bool CanConvert(Type object_type)
+        {
+            return false;
+        }
     }
 }
-

Core/Registry/AvailableModule.cs

@@ -5,12 +5,14 @@
 using System.Diagnostics;
 using System.Linq;
 using System.Runtime.Serialization;
-using CKAN.Extensions;
-using CKAN.Versioning;
+
 using log4net;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Serialization;
 
+using CKAN.Extensions;
+using CKAN.Versioning;
+
 namespace CKAN
 {
     /// <summary>
@@ -32,10 +34,8 @@ private AvailableModule()
         [OnDeserialized]
         internal void DeserialisationFixes(StreamingContext context)
         {
-            // Set identifier
-            var mod = module_version.Values.LastOrDefault();
-            identifier = mod.identifier;
-            Debug.Assert(module_version.Values.All(m=>identifier.Equals(m.identifier)));
+            identifier = module_version.Values.LastOrDefault()?.identifier;
+            Debug.Assert(module_version.Values.All(m => identifier.Equals(m.identifier)));
         }
 
         /// <param name="identifier">The module to keep track of</param>
@@ -49,6 +49,7 @@ public AvailableModule(string identifier)
         // The map of versions -> modules, that's what we're about!
         // First element is the oldest version, last is the newest.
         [JsonProperty]
+        [JsonConverter(typeof(JsonLeakySortedDictionaryConverter<ModuleVersion, CkanModule>))]
         internal SortedDictionary<ModuleVersion, CkanModule> module_version =
             new SortedDictionary<ModuleVersion, CkanModule>();
 

Core/Registry/Registry.cs

@@ -25,12 +25,13 @@ namespace CKAN
     public class Registry : IEnlistmentNotification, IRegistryQuerier
     {
         [JsonIgnore] private const int LATEST_REGISTRY_VERSION = 3;
-        [JsonIgnore] private static readonly ILog log = LogManager.GetLogger(typeof (Registry));
+        [JsonIgnore] private static readonly ILog log = LogManager.GetLogger(typeof(Registry));
 
         [JsonProperty] private int registry_version;
 
+        // name => Repository
         [JsonProperty("sorted_repositories")]
-        private SortedDictionary<string, Repository> repositories; // name => Repository
+        private SortedDictionary<string, Repository> repositories;
 
         // TODO: These may be good as custom types, especially those which process
         // paths (and flip from absolute to relative, and vice-versa).