From 98b0b4152311405de7319016dd3b5223675096b5 Mon Sep 17 00:00:00 2001 From: Paul Hebble Date: Wed, 26 Jan 2022 23:28:13 +0000 Subject: [PATCH] Get licenses from embedded schema, skip bad modules in deserialize --- CKAN.schema | 35 ++++++ Core/CKAN-core.csproj | 4 + .../JsonLeakySortedDictionaryConverter.cs | 64 +++++++++++ Core/Converters/JsonSimpleStringConverter.cs | 34 +++--- Core/Converters/JsonSingleOrArrayConverter.cs | 38 +++---- Core/Registry/AvailableModule.cs | 13 +-- Core/Registry/Registry.cs | 5 +- Core/Types/License.cs | 101 ++++-------------- Core/Types/Schema.cs | 29 +++++ Netkan/CKAN-netkan.csproj | 6 -- Netkan/Validators/ObeysCKANSchemaValidator.cs | 18 +--- 11 files changed, 203 insertions(+), 144 deletions(-) create mode 100644 Core/Converters/JsonLeakySortedDictionaryConverter.cs create mode 100644 Core/Types/Schema.cs diff --git a/CKAN.schema b/CKAN.schema index 474db9dac4..7dc54fcc03 100644 --- a/CKAN.schema +++ b/CKAN.schema @@ -477,6 +477,41 @@ "open-source", "restricted", "unrestricted", "unknown" ] }, + "redistributable_license": { + "description" : "A license that grants us the right to redistribute.", + "enum" : [ + "public-domain", + "Apache", "Apache-1.0", "Apache-2.0", + "Artistic", "Artistic-1.0", "Artistic-2.0", + "BSD-2-clause", "BSD-3-clause", "BSD-4-clause", + "ISC", + "CC-BY", "CC-BY-1.0", "CC-BY-2.0", "CC-BY-2.5", "CC-BY-3.0", "CC-BY-4.0", + "CC-BY-SA", "CC-BY-SA-1.0", "CC-BY-SA-2.0", "CC-BY-SA-2.5", "CC-BY-SA-3.0", "CC-BY-SA-4.0", + "CC-BY-NC", "CC-BY-NC-1.0", "CC-BY-NC-2.0", "CC-BY-NC-2.5", "CC-BY-NC-3.0", "CC-BY-NC-4.0", + "CC-BY-NC-SA", "CC-BY-NC-SA-1.0", "CC-BY-NC-SA-2.0", "CC-BY-NC-SA-2.5", "CC-BY-NC-SA-3.0", "CC-BY-NC-SA-4.0", + "CC-BY-NC-ND", "CC-BY-NC-ND-1.0", "CC-BY-NC-ND-2.0", "CC-BY-NC-ND-2.5", "CC-BY-NC-ND-3.0", "CC-BY-NC-ND-4.0", + "CC-BY-ND", "CC-BY-ND-1.0", "CC-BY-ND-2.0", "CC-BY-ND-2.5", "CC-BY-ND-3.0", "CC-BY-ND-4.0", + "CC0", + "CDDL", "CPL", + "EFL-1.0", "EFL-2.0", + "Expat", "MIT", + "GPL-1.0", "GPL-2.0", "GPL-3.0", + "LGPL-2.0", "LGPL-2.1", "LGPL-3.0", + "GFDL-1.0", "GFDL-1.1", "GFDL-1.2", "GFDL-1.3", + "GFDL-NIV-1.0", "GFDL-NIV-1.1", "GFDL-NIV-1.2", "GFDL-NIV-1.3", + "LPPL-1.0", "LPPL-1.1", "LPPL-1.2", "LPPL-1.3c", + "MPL-1.1", "MPL-2.0", + "Perl", + "Python-2.0", + "QPL-1.0", + "W3C", + "Zlib", + "Zope", + "WTFPL", + "Unlicense", + "open-source", "unrestricted" + ] + }, "licenses" : { "description" : "A license, or array of licenses", "anyOf" : [ diff --git a/Core/CKAN-core.csproj b/Core/CKAN-core.csproj index c276c2df61..445cc6ffe2 100644 --- a/Core/CKAN-core.csproj +++ b/Core/CKAN-core.csproj @@ -41,6 +41,7 @@ + @@ -64,6 +65,9 @@ + + CKAN.Core.CKAN.schema + diff --git a/Core/Converters/JsonLeakySortedDictionaryConverter.cs b/Core/Converters/JsonLeakySortedDictionaryConverter.cs new file mode 100644 index 0000000000..a0c017382c --- /dev/null +++ b/Core/Converters/JsonLeakySortedDictionaryConverter.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; + +using log4net; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace CKAN +{ + /// + /// [De]serializes a dictionary that might have some questionably + /// valid data in it. + /// If exceptions are thrown for any key/value pair, leave it out. + /// Removes CkanModule objects from AvailableModule.module_version + /// if License throws BadMetadataKraken. + /// + public class JsonLeakySortedDictionaryConverter : JsonConverter + { + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + var dict = new SortedDictionary(); + foreach (var kvp in JObject.Load(reader)) + { + try + { + dict.Add( + (K)Activator.CreateInstance(typeof(K), kvp.Key), + kvp.Value.ToObject()); + } + catch (Exception exc) + { + log.Warn($"Failed to deserialize {kvp.Key}: {kvp.Value}", exc); + } + } + return dict; + } + + /// + /// Use default serializer for writing + /// + public override bool CanWrite => false; + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + throw new NotImplementedException(); + } + + /// + /// We *only* want to be triggered for types that have explicitly + /// set an attribute in their class saying they can be converted. + /// By returning false here, we declare we're not interested in participating + /// in any other conversions. + /// + /// + /// false + /// + public override bool CanConvert(Type objectType) + { + return false; + } + + private static readonly ILog log = LogManager.GetLogger(typeof(JsonLeakySortedDictionaryConverter)); + } +} diff --git a/Core/Converters/JsonSimpleStringConverter.cs b/Core/Converters/JsonSimpleStringConverter.cs index 3698c190b2..624baf0708 100644 --- a/Core/Converters/JsonSimpleStringConverter.cs +++ b/Core/Converters/JsonSimpleStringConverter.cs @@ -1,32 +1,38 @@ using System; -using Newtonsoft.Json; - -namespace CKAN { - // A lovely class for serialising things that can be converted - // to simple strings and back. - public class JsonSimpleStringConverter : JsonConverter { +using Newtonsoft.Json; +namespace CKAN +{ + /// + /// Serialises things that can be converted + /// to simple strings and back. + /// + public class JsonSimpleStringConverter : JsonConverter + { public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) { - writer.WriteValue (value.ToString ()); + writer.WriteValue(value.ToString()); } public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) { - // If we find a null, then that might be okay, so we pass it down to our // activator. Otherwise we convert to string, since that's our job. - string value = reader.Value == null ? null : reader.Value.ToString(); - return Activator.CreateInstance (objectType, value); + return Activator.CreateInstance(objectType, reader.Value?.ToString()); } + /// + /// We *only* want to be triggered for types that have explicitly + /// set an attribute in their class saying they can be converted. + /// By returning false here, we declare we're not interested in participating + /// in any other conversions. + /// + /// + /// false + /// public override bool CanConvert(Type objectType) { - // We *only* want to be triggered for types that have explicitly - // set an attribute in their class saying they can be converted. - // By returning false here, we declare we're not interested in participating - // in any other conversions. return false; } } diff --git a/Core/Converters/JsonSingleOrArrayConverter.cs b/Core/Converters/JsonSingleOrArrayConverter.cs index 64613b0868..9f5fa883ff 100644 --- a/Core/Converters/JsonSingleOrArrayConverter.cs +++ b/Core/Converters/JsonSingleOrArrayConverter.cs @@ -1,25 +1,17 @@ using System; using System.Collections.Generic; + using Newtonsoft.Json; using Newtonsoft.Json.Linq; namespace CKAN { - - // With thanks to - // https://stackoverflow.com/questions/18994685/how-to-handle-both-a-single-item-and-an-array-for-the-same-property-using-json-n - + /// + /// With thanks to + /// https://stackoverflow.com/questions/18994685/how-to-handle-both-a-single-item-and-an-array-for-the-same-property-using-json-n + /// public class JsonSingleOrArrayConverter : JsonConverter { - public override bool CanConvert(Type object_type) - { - // We *only* want to be triggered for types that have explicitly - // set an attribute in their class saying they can be converted. - // By returning false here, we declare we're not interested in participating - // in any other conversions. - return false; - } - public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) { JToken token = JToken.Load(reader); @@ -32,15 +24,25 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist return token.ToObject() == null ? null : new List { token.ToObject() }; } - public override bool CanWrite - { - get { return false; } - } + public override bool CanWrite => false; public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) { throw new NotImplementedException(); } + + /// + /// We *only* want to be triggered for types that have explicitly + /// set an attribute in their class saying they can be converted. + /// By returning false here, we declare we're not interested in participating + /// in any other conversions. + /// + /// + /// false + /// + public override bool CanConvert(Type object_type) + { + return false; + } } } - diff --git a/Core/Registry/AvailableModule.cs b/Core/Registry/AvailableModule.cs index 51210617e5..3c1e7f1881 100644 --- a/Core/Registry/AvailableModule.cs +++ b/Core/Registry/AvailableModule.cs @@ -5,12 +5,14 @@ using System.Diagnostics; using System.Linq; using System.Runtime.Serialization; -using CKAN.Extensions; -using CKAN.Versioning; + using log4net; using Newtonsoft.Json; using Newtonsoft.Json.Serialization; +using CKAN.Extensions; +using CKAN.Versioning; + namespace CKAN { /// @@ -32,10 +34,8 @@ private AvailableModule() [OnDeserialized] internal void DeserialisationFixes(StreamingContext context) { - // Set identifier - var mod = module_version.Values.LastOrDefault(); - identifier = mod.identifier; - Debug.Assert(module_version.Values.All(m=>identifier.Equals(m.identifier))); + identifier = module_version.Values.LastOrDefault()?.identifier; + Debug.Assert(module_version.Values.All(m => identifier.Equals(m.identifier))); } /// The module to keep track of @@ -49,6 +49,7 @@ public AvailableModule(string identifier) // The map of versions -> modules, that's what we're about! // First element is the oldest version, last is the newest. [JsonProperty] + [JsonConverter(typeof(JsonLeakySortedDictionaryConverter))] internal SortedDictionary module_version = new SortedDictionary(); diff --git a/Core/Registry/Registry.cs b/Core/Registry/Registry.cs index 5a080825dd..2aa9d53fc0 100644 --- a/Core/Registry/Registry.cs +++ b/Core/Registry/Registry.cs @@ -25,12 +25,13 @@ namespace CKAN public class Registry : IEnlistmentNotification, IRegistryQuerier { [JsonIgnore] private const int LATEST_REGISTRY_VERSION = 3; - [JsonIgnore] private static readonly ILog log = LogManager.GetLogger(typeof (Registry)); + [JsonIgnore] private static readonly ILog log = LogManager.GetLogger(typeof(Registry)); [JsonProperty] private int registry_version; + // name => Repository [JsonProperty("sorted_repositories")] - private SortedDictionary repositories; // name => Repository + private SortedDictionary repositories; // TODO: These may be good as custom types, especially those which process // paths (and flip from absolute to relative, and vice-versa). diff --git a/Core/Types/License.cs b/Core/Types/License.cs index 9ac4bb1f03..f1eb38853d 100644 --- a/Core/Types/License.cs +++ b/Core/Types/License.cs @@ -1,89 +1,32 @@ +using System.Linq; using System.Collections.Generic; + using Newtonsoft.Json; +using CKAN.Extensions; + namespace CKAN { /// - /// A Spec compliment license string. + /// A spec complement license string /// [JsonConverter(typeof(JsonSimpleStringConverter))] public class License { - static License _unknownLicense; - public static License UnknownLicense => _unknownLicense ?? (_unknownLicense = new License("unknown")); + public static readonly HashSet valid_licenses = + CKANSchema.schema.Definitions["license"] + .Enumeration + .Select(obj => obj.ToString()) + .ToHashSet(); - // TODO: It would be lovely for our build system to write these for us. - public static readonly HashSet valid_licenses = new HashSet() - { - "public-domain", - "AFL-3.0", - "AGPL-3.0", - "Apache", "Apache-1.0", "Apache-2.0", - "APSL-2.0", - "Artistic", "Artistic-1.0", "Artistic-2.0", - "BSD-2-clause", "BSD-3-clause", "BSD-4-clause", - "ISC", - "CC-BY", "CC-BY-1.0", "CC-BY-2.0", "CC-BY-2.5", "CC-BY-3.0", "CC-BY-4.0", - "CC-BY-SA", "CC-BY-SA-1.0", "CC-BY-SA-2.0", "CC-BY-SA-2.5", "CC-BY-SA-3.0", "CC-BY-SA-4.0", - "CC-BY-NC", "CC-BY-NC-1.0", "CC-BY-NC-2.0", "CC-BY-NC-2.5", "CC-BY-NC-3.0", "CC-BY-NC-4.0", - "CC-BY-NC-SA", "CC-BY-NC-SA-1.0", "CC-BY-NC-SA-2.0", "CC-BY-NC-SA-2.5", "CC-BY-NC-SA-3.0", "CC-BY-NC-SA-4.0", - "CC-BY-NC-ND", "CC-BY-NC-ND-1.0", "CC-BY-NC-ND-2.0", "CC-BY-NC-ND-2.5", "CC-BY-NC-ND-3.0", "CC-BY-NC-ND-4.0", - "CC-BY-ND", "CC-BY-ND-1.0", "CC-BY-ND-2.0", "CC-BY-ND-2.5", "CC-BY-ND-3.0", "CC-BY-ND-4.0", - "CC0", - "CDDL", "CPL", - "EFL-1.0", "EFL-2.0", - "Expat", "MIT", - "GPL-1.0", "GPL-2.0", "GPL-3.0", - "LGPL-2.0", "LGPL-2.1", "LGPL-3.0", - "GFDL-1.0", "GFDL-1.1", "GFDL-1.2", "GFDL-1.3", - "GFDL-NIV-1.0", "GFDL-NIV-1.1", "GFDL-NIV-1.2", "GFDL-NIV-1.3", - "LPPL-1.0", "LPPL-1.1", "LPPL-1.2", "LPPL-1.3c", - "MPL-1.0", "MPL-1.1", "MPL-2.0", - "Ms-PL", "Ms-RL", - "Perl", - "Python-2.0", - "QPL-1.0", - "Unlicense", - "W3C", - "WTFPL", - "Zlib", - "Zope", - "open-source", "restricted", "unrestricted", "unknown" - }; + private static readonly HashSet redistributable_licenses = + CKANSchema.schema.Definitions["redistributable_license"] + .Enumeration + .Select(obj => obj.ToString()) + .ToHashSet(); - private static readonly HashSet redistributable_licenses = new HashSet() - { - "public-domain", - "Apache", "Apache-1.0", "Apache-2.0", - "Artistic", "Artistic-1.0", "Artistic-2.0", - "BSD-2-clause", "BSD-3-clause", "BSD-4-clause", - "ISC", - "CC-BY", "CC-BY-1.0", "CC-BY-2.0", "CC-BY-2.5", "CC-BY-3.0", "CC-BY-4.0", - "CC-BY-SA", "CC-BY-SA-1.0", "CC-BY-SA-2.0", "CC-BY-SA-2.5", "CC-BY-SA-3.0", "CC-BY-SA-4.0", - "CC-BY-NC", "CC-BY-NC-1.0", "CC-BY-NC-2.0", "CC-BY-NC-2.5", "CC-BY-NC-3.0", "CC-BY-NC-4.0", - "CC-BY-NC-SA", "CC-BY-NC-SA-1.0", "CC-BY-NC-SA-2.0", "CC-BY-NC-SA-2.5", "CC-BY-NC-SA-3.0", "CC-BY-NC-SA-4.0", - "CC-BY-NC-ND", "CC-BY-NC-ND-1.0", "CC-BY-NC-ND-2.0", "CC-BY-NC-ND-2.5", "CC-BY-NC-ND-3.0", "CC-BY-NC-ND-4.0", - "CC-BY-ND", "CC-BY-ND-1.0", "CC-BY-ND-2.0", "CC-BY-ND-2.5", "CC-BY-ND-3.0", "CC-BY-ND-4.0", - "CC0", - "CDDL", "CPL", - "EFL-1.0", "EFL-2.0", - "Expat", "MIT", - "GPL-1.0", "GPL-2.0", "GPL-3.0", - "LGPL-2.0", "LGPL-2.1", "LGPL-3.0", - "GFDL-1.0", "GFDL-1.1", "GFDL-1.2", "GFDL-1.3", - "GFDL-NIV-1.0", "GFDL-NIV-1.1", "GFDL-NIV-1.2", "GFDL-NIV-1.3", - "LPPL-1.0", "LPPL-1.1", "LPPL-1.2", "LPPL-1.3c", - "MPL-1.1", "MPL-2.0", - "Perl", - "Python-2.0", - "QPL-1.0", - "W3C", - "Zlib", - "Zope", - "WTFPL", - "Unlicense", - "open-source", "unrestricted" - }; + // Make sure this is the last static field so the others will be ready for the instance constructor! + public static readonly License UnknownLicense = new License("unknown"); private string license; @@ -94,7 +37,7 @@ public class License /// License. public License(string license) { - if (! valid_licenses.Contains(license)) + if (!valid_licenses.Contains(license)) { throw new BadMetadataKraken( null, @@ -112,13 +55,7 @@ public License(string license) /// /// True if redistributable, false otherwise. /// - public bool Redistributable - { - get - { - return redistributable_licenses.Contains(license); - } - } + public bool Redistributable => redistributable_licenses.Contains(license); /// /// Returns the license as a string. diff --git a/Core/Types/Schema.cs b/Core/Types/Schema.cs new file mode 100644 index 0000000000..96a6ec8e7b --- /dev/null +++ b/Core/Types/Schema.cs @@ -0,0 +1,29 @@ +using System.IO; +using System.Reflection; + +using Newtonsoft.Json; +using NJsonSchema; + +namespace CKAN +{ + /// + /// A static container for a JsonSchema object representing CKAN.schema + /// + public static class CKANSchema + { + /// + /// Parsed representation of our embedded CKAN.schema resource + /// + public static readonly JsonSchema schema = + JsonSchema.FromJsonAsync( + // ♫ Ooh-ooh StreamReader, I believe you can get me all the lines ♫ + // ♫ Ooh-ooh StreamReader, I believe we can reach the end of file ♫ + new StreamReader( + Assembly.GetExecutingAssembly() + .GetManifestResourceStream(embeddedSchema) + ).ReadToEnd() + ).Result; + + private const string embeddedSchema = "CKAN.Core.CKAN.schema"; + } +} diff --git a/Netkan/CKAN-netkan.csproj b/Netkan/CKAN-netkan.csproj index 8cf6ec50e3..e1bdbd10ff 100644 --- a/Netkan/CKAN-netkan.csproj +++ b/Netkan/CKAN-netkan.csproj @@ -46,7 +46,6 @@ - @@ -154,11 +153,6 @@ - - - CKAN.NetKAN.CKAN.schema - - diff --git a/Netkan/Validators/ObeysCKANSchemaValidator.cs b/Netkan/Validators/ObeysCKANSchemaValidator.cs index 5bad70eafe..b7a4513780 100644 --- a/Netkan/Validators/ObeysCKANSchemaValidator.cs +++ b/Netkan/Validators/ObeysCKANSchemaValidator.cs @@ -1,25 +1,14 @@ -using System.IO; using System.Linq; -using System.Reflection; -using NJsonSchema; + using CKAN.NetKAN.Model; namespace CKAN.NetKAN.Validators { internal sealed class ObeysCKANSchemaValidator : IValidator { - static ObeysCKANSchemaValidator() - { - var resourceStream = Assembly.GetExecutingAssembly().GetManifestResourceStream(embeddedSchema); - using (var reader = new StreamReader(resourceStream)) - { - schema = JsonSchema.FromJsonAsync(reader.ReadToEnd()).Result; - } - } - public void Validate(Metadata metadata) { - var errors = schema.Validate(metadata.Json()); + var errors = CKANSchema.schema.Validate(metadata.Json()); if (errors.Any()) { string msg = errors @@ -28,8 +17,5 @@ public void Validate(Metadata metadata) throw new Kraken($"Schema validation failed: {msg}"); } } - - private static readonly JsonSchema schema; - private const string embeddedSchema = "CKAN.NetKAN.CKAN.schema"; } }