Skip to content

Commit

Permalink
Allow YAML for human-edited metadata
Browse files Browse the repository at this point in the history
  • Loading branch information
HebaruSan committed May 14, 2021
1 parent 5a2cd8e commit b65698c
Show file tree
Hide file tree
Showing 10 changed files with 261 additions and 7 deletions.
2 changes: 2 additions & 0 deletions Netkan/CKAN-netkan.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
<PackageReference Include="log4net" Version="2.0.10" />
<PackageReference Include="Namotion.Reflection" Version="1.0.7" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="YamlDotNet" Version="9.1.0" />
<PackageReference Include="NJsonSchema" Version="10.0.27" />
</ItemGroup>
<ItemGroup>
Expand All @@ -62,6 +63,7 @@
<Compile Include="Constants.cs" />
<Compile Include="Extensions\JObjectExtensions.cs" />
<Compile Include="Extensions\VersionExtensions.cs" />
<Compile Include="Extensions\YamlExtensions.cs" />
<Compile Include="Model\Metadata.cs" />
<Compile Include="Model\RemoteRef.cs" />
<Compile Include="Processors\Inflator.cs" />
Expand Down
85 changes: 85 additions & 0 deletions Netkan/Extensions/YamlExtensions.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
using System.IO;
using System.Linq;
using log4net;
using YamlDotNet.RepresentationModel;
using Newtonsoft.Json.Linq;

namespace CKAN.NetKAN.Extensions
{
internal static class YamlExtensions
{
public static YamlMappingNode Parse(string input)
{
return Parse(new StringReader(input));
}

public static YamlMappingNode Parse(TextReader input)
{
var stream = new YamlStream();
stream.Load(input);
return stream.Documents.FirstOrDefault()?.RootNode as YamlMappingNode;
}

/// <summary>
/// Convert a YAML object to a JSON object
/// </summary>
/// <param name="yaml">The input object</param>
/// <returns>
/// A JObject representation of the input data
/// </returns>
public static JObject ToJObject(this YamlMappingNode yaml)
{
var jobj = new JObject();
foreach (var kvp in yaml)
{
switch (kvp.Value.NodeType)
{
case YamlNodeType.Mapping:
jobj.Add((string)kvp.Key, (kvp.Value as YamlMappingNode).ToJObject());
break;
case YamlNodeType.Sequence:
jobj.Add((string)kvp.Key, (kvp.Value as YamlSequenceNode).ToJarray());
break;
case YamlNodeType.Scalar:
jobj.Add((string)kvp.Key, (kvp.Value as YamlScalarNode).ToJValue());
break;
}
}
return jobj;
}

private static JArray ToJarray(this YamlSequenceNode yaml)
{
var jarr = new JArray();
foreach (var elt in yaml)
{
switch (elt.NodeType)
{
case YamlNodeType.Mapping:
jarr.Add((elt as YamlMappingNode).ToJObject());
break;
case YamlNodeType.Sequence:
jarr.Add((elt as YamlSequenceNode).ToJarray());
break;
case YamlNodeType.Scalar:
jarr.Add((elt as YamlScalarNode).ToJValue());
break;
}
}
return jarr;
}

private static JValue ToJValue(this YamlScalarNode yaml)
{
switch (yaml.Value)
{
case "null": return JValue.CreateNull();
case "true": return new JValue(true);
case "false": return new JValue(false);
default: return new JValue(yaml.Value);
}
}

private static readonly ILog log = LogManager.GetLogger(typeof(YamlExtensions));
}
}
6 changes: 6 additions & 0 deletions Netkan/Model/Metadata.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
using System.Linq;
using CKAN.Versioning;
using Newtonsoft.Json.Linq;
using YamlDotNet.RepresentationModel;
using CKAN.NetKAN.Extensions;

namespace CKAN.NetKAN.Model
{
Expand Down Expand Up @@ -117,6 +119,10 @@ public Metadata(JObject json)
}
}

public Metadata(YamlMappingNode yaml) : this(yaml?.ToJObject())
{
}

public string[] Licenses
{
get
Expand Down
3 changes: 2 additions & 1 deletion Netkan/Processors/QueueHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
using CKAN.Versioning;
using CKAN.NetKAN.Transformers;
using CKAN.NetKAN.Model;
using CKAN.NetKAN.Extensions;

namespace CKAN.NetKAN.Processors
{
Expand Down Expand Up @@ -127,7 +128,7 @@ private void handleMessages(string url, int howMany, int timeoutMinutes)
private IEnumerable<SendMessageBatchRequestEntry> Inflate(Message msg)
{
log.DebugFormat("Metadata returned: {0}", msg.Body);
var netkan = new Metadata(JObject.Parse(msg.Body));
var netkan = new Metadata(YamlExtensions.Parse(msg.Body));

int releases = 1;
MessageAttributeValue releasesAttr;
Expand Down
3 changes: 2 additions & 1 deletion Netkan/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
using CKAN.NetKAN.Model;
using CKAN.NetKAN.Processors;
using CKAN.NetKAN.Transformers;
using CKAN.NetKAN.Extensions;

namespace CKAN.NetKAN
{
Expand Down Expand Up @@ -174,7 +175,7 @@ private static Metadata ReadNetkan()
Log.WarnFormat("Input is not a .netkan file");
}

return new Metadata(JObject.Parse(File.ReadAllText(Options.File)));
return new Metadata(YamlExtensions.Parse(File.OpenText(Options.File)));
}

internal static string CkanFileName(Metadata metadata)
Expand Down
6 changes: 2 additions & 4 deletions Netkan/Services/ModuleService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;

using CKAN.NetKAN.Extensions;
using CKAN.Versioning;
using CKAN.Extensions;
using CKAN.NetKAN.Sources.Avc;
Expand Down Expand Up @@ -200,10 +201,7 @@ private static JObject DeserializeFromStream(Stream stream)
{
using (var sr = new StreamReader(stream))
{
using (var jsonTextReader = new JsonTextReader(sr))
{
return (JObject)JToken.ReadFrom(jsonTextReader);
}
return YamlExtensions.Parse(sr).ToJObject();
}
}

Expand Down
3 changes: 2 additions & 1 deletion Netkan/Transformers/MetaNetkanTransformer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Collections.Generic;
using log4net;
using Newtonsoft.Json.Linq;

using CKAN.Versioning;
using CKAN.NetKAN.Extensions;
using CKAN.NetKAN.Model;
Expand Down Expand Up @@ -51,7 +52,7 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)

Log.DebugFormat("Target netkan:{0}{1}", Environment.NewLine, targetFileText);

var targetJson = JObject.Parse(targetFileText);
var targetJson = YamlExtensions.Parse(targetFileText).ToJObject();
var targetMetadata = new Metadata(targetJson);

if (targetMetadata.Kref == null || targetMetadata.Kref.Source != "netkan")
Expand Down
19 changes: 19 additions & 0 deletions Spec.md
Original file line number Diff line number Diff line change
Expand Up @@ -688,6 +688,23 @@ NetKAN is the name the tool which is used to automatically generate CKAN files f
consumes `.netkan` files to produce `.ckan` files. `.netkan` files are a *strict superset* of `.ckan` files. Every
`.ckan` file is a valid `.netkan` file but not vice versa. NetKAN uses the following fields to produce `.ckan` files.

##### YAML Option

A `.netkan` file may be in either JSON or YAML format. All examples shown below assume JSON, but the YAML equivalents will work the same way.

Note that `#` is the comment character in YAML, so even if you choose YAML syntax, you still can't omit the quotes around a value that includes `#`, such as typical values of `$kref` and `$vref`:

```yaml
$kref: "#/ckan/spacedock/1234"
$vref: "#/ckan/ksp-avc"
```
##### Internal `.ckan` files

If a module's download contains a file with a `.ckan` extension, this file will be parsed and its contents added to the module's metadata. This can be a convenient way to handle metadata values that can change from one version to the next, such as dependencies.

An internal `.ckan` file may be in either JSON or YAML format.

##### `$kref`

The `$kref` field indicates that data should be filled in from an external service provider. The following `$kref`
Expand Down Expand Up @@ -857,6 +874,8 @@ The remote `.netkan` file is downloaded and used as if it were the original. `.n
reference are known as *recursive netkans* or *metanetkans*. They are primarily used so that mod authors can provide
authoritative metadata.

A metanetkan may be in either JSON or YAML format.

The following conditions apply:
- A metanekan may not reference another metanetkan, otherwise an error is produced.
- Any fields specified in the metanetkan will override any fields in the target netkan file.
Expand Down
140 changes: 140 additions & 0 deletions Tests/NetKAN/Extensions/YamlExtensionsTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
using System.Linq;
using NUnit.Framework;
using YamlDotNet.RepresentationModel;
using Newtonsoft.Json.Linq;

using CKAN.NetKAN.Extensions;

namespace Tests.NetKAN.Extensions
{
[TestFixture]
public sealed class YamlExtensionsTests
{
[Test]
public void Parse_ValidInput_Works()
{
// Arrange
string input = string.Join("\r\n", new string[]
{
"spec_version: v1.4",
"identifier: Astrogator",
"$kref: \"#/ckan/github/HebaruSan/Astrogator\"",
"$vref: \"#/ckan/ksp-avc\"",
"license: GPL-3.0",
"tags:",
" - plugin",
" - information",
" - control",
"resources:",
" homepage: https://forum.kerbalspaceprogram.com/index.php?/topic/155998-*",
" bugtracker: https://github.com/HebaruSan/Astrogator/issues",
" repository: https://github.com/HebaruSan/Astrogator",
"recommends:",
" - name: ModuleManager",
" - name: LoadingTipsPlus",
});

// Act
YamlMappingNode yaml = YamlExtensions.Parse(input);

// Assert
Assert.AreEqual("v1.4", (string)yaml["spec_version"]);
Assert.AreEqual("Astrogator", (string)yaml["identifier"]);
Assert.AreEqual("#/ckan/github/HebaruSan/Astrogator", (string)yaml["$kref"]);
Assert.AreEqual("#/ckan/ksp-avc", (string)yaml["$vref"]);
Assert.AreEqual("GPL-3.0", (string)yaml["license"]);

CollectionAssert.AreEqual(
new string[] { "plugin", "information", "control" },
(yaml["tags"] as YamlSequenceNode).Children.Select(yn => (string)yn)
);
Assert.AreEqual(
"https://forum.kerbalspaceprogram.com/index.php?/topic/155998-*",
(string)yaml["resources"]["homepage"]
);
Assert.AreEqual(
"https://github.com/HebaruSan/Astrogator/issues",
(string)yaml["resources"]["bugtracker"]
);
Assert.AreEqual(
"https://github.com/HebaruSan/Astrogator",
(string)yaml["resources"]["repository"]
);
Assert.AreEqual("ModuleManager", (string)yaml["recommends"][0]["name"]);
Assert.AreEqual("LoadingTipsPlus", (string)yaml["recommends"][1]["name"]);
}

[Test]
public void ToJObject_ValidInput_Works()
{
// Arrange
var yaml = new YamlMappingNode()
{
{ "spec_version", "v1.4" },
{ "identifier", "Astrogator" },
{ "$kref", "#/ckan/github/HebaruSan/Astrogator" },
{ "$vref", "#/ckan/ksp-avc" },
{ "license", "GPL-3.0" },
{
"tags",
new YamlSequenceNode(
"plugin",
"information",
"control"
)
},
{
"resources",
new YamlMappingNode()
{
{ "homepage", "https://forum.kerbalspaceprogram.com/index.php?/topic/155998-*" },
{ "bugtracker", "https://github.com/HebaruSan/Astrogator/issues" },
{ "repository", "https://github.com/HebaruSan/Astrogator" },
}
},
{
"recommends",
new YamlSequenceNode(
new YamlMappingNode()
{
{ "name", "ModuleManager" }
},
new YamlMappingNode()
{
{ "name", "LoadingTipsPlus" }
}
)
}
};

// Act
JObject json = yaml.ToJObject();

// Assert
Assert.AreEqual("v1.4", (string)json["spec_version"]);
Assert.AreEqual("Astrogator", (string)json["identifier"]);
Assert.AreEqual("#/ckan/github/HebaruSan/Astrogator", (string)json["$kref"]);
Assert.AreEqual("#/ckan/ksp-avc", (string)json["$vref"]);
Assert.AreEqual("GPL-3.0", (string)json["license"]);

CollectionAssert.AreEqual(
new string[] { "plugin", "information", "control" },
(json["tags"] as JArray).Select(elt => (string)elt)
);
Assert.AreEqual(
"https://forum.kerbalspaceprogram.com/index.php?/topic/155998-*",
(string)json["resources"]["homepage"]
);
Assert.AreEqual(
"https://github.com/HebaruSan/Astrogator/issues",
(string)json["resources"]["bugtracker"]
);
Assert.AreEqual(
"https://github.com/HebaruSan/Astrogator",
(string)json["resources"]["repository"]
);
Assert.AreEqual("ModuleManager", (string)json["recommends"][0]["name"]);
Assert.AreEqual("LoadingTipsPlus", (string)json["recommends"][1]["name"]);
}
}
}
1 change: 1 addition & 0 deletions Tests/Tests.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
<PackageReference Include="log4net" Version="2.0.10" />
<PackageReference Include="Moq" Version="4.14.5" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="YamlDotNet" Version="9.1.0" />
<PackageReference Include="NUnit" Version="3.12.0" />
<PackageReference Include="NUnit3TestAdapter" Version="3.16.1" />
</ItemGroup>
Expand Down

0 comments on commit b65698c

Please sign in to comment.