From ef8fc9aec4bdb7173b9cd3691635e0902b0a8ed5 Mon Sep 17 00:00:00 2001 From: Paul Hebble Date: Tue, 3 Sep 2024 16:20:31 -0500 Subject: [PATCH] SourceForge kref --- Core/Extensions/RegexExtensions.cs | 2 +- Core/Net/Net.cs | 7 +- Core/Net/RedirectWebClient.cs | 4 +- Netkan/CKAN-netkan.csproj | 2 + Netkan/Sources/SourceForge/ISourceForgeApi.cs | 7 ++ Netkan/Sources/SourceForge/SourceForgeApi.cs | 26 ++++++ Netkan/Sources/SourceForge/SourceForgeMod.cs | 29 ++++++ Netkan/Sources/SourceForge/SourceForgeRef.cs | 40 +++++++++ .../Sources/SourceForge/SourceForgeVersion.cs | 21 +++++ .../{ISpaceDock.cs => ISpacedockApi.cs} | 0 .../{SpaceDockApi.cs => SpacedockApi.cs} | 0 Netkan/Transformers/NetkanTransformer.cs | 3 + Netkan/Transformers/SourceForgeTransformer.cs | 90 +++++++++++++++++++ Netkan/Validators/KrefValidator.cs | 1 + Spec.md | 23 +++++ 15 files changed, 249 insertions(+), 6 deletions(-) create mode 100644 Netkan/Sources/SourceForge/ISourceForgeApi.cs create mode 100644 Netkan/Sources/SourceForge/SourceForgeApi.cs create mode 100644 Netkan/Sources/SourceForge/SourceForgeMod.cs create mode 100644 Netkan/Sources/SourceForge/SourceForgeRef.cs create mode 100644 Netkan/Sources/SourceForge/SourceForgeVersion.cs rename Netkan/Sources/Spacedock/{ISpaceDock.cs => ISpacedockApi.cs} (100%) rename Netkan/Sources/Spacedock/{SpaceDockApi.cs => SpacedockApi.cs} (100%) create mode 100644 Netkan/Transformers/SourceForgeTransformer.cs diff --git a/Core/Extensions/RegexExtensions.cs b/Core/Extensions/RegexExtensions.cs index d4f14101b1..136ec4ae2f 100644 --- a/Core/Extensions/RegexExtensions.cs +++ b/Core/Extensions/RegexExtensions.cs @@ -12,7 +12,7 @@ public static class RegexExtensions /// The string to check /// Object representing the match, if any /// True if the regex matched the value, false otherwise - public static bool TryMatch(this Regex regex, string value, + public static bool TryMatch(this Regex regex, string? value, [NotNullWhen(returnValue: true)] out Match? match) { if (value == null) diff --git a/Core/Net/Net.cs b/Core/Net/Net.cs index 9776124865..113cc20004 100644 --- a/Core/Net/Net.cs +++ b/Core/Net/Net.cs @@ -197,12 +197,13 @@ public static string Download(string url, out string? etag, string? filename = n return null; } - public static Uri? ResolveRedirect(Uri url) + public static Uri? ResolveRedirect(Uri url, + string? userAgent = "") { const int maxRedirects = 6; for (int redirects = 0; redirects <= maxRedirects; ++redirects) { - var rwClient = new RedirectWebClient(); + var rwClient = new RedirectWebClient(userAgent); using (rwClient.OpenRead(url)) { } var location = rwClient.ResponseHeaders?["Location"]; if (location == null) @@ -239,7 +240,7 @@ public static string Download(string url, out string? etag, string? filename = n // Is it supposed to turn a "&" into part of the content of a form field, // or is it supposed to assume that it separates different form fields? // https://github.com/dotnet/runtime/issues/31387 - // So now we have to just substitude certain characters ourselves one by one. + // So now we have to just substitute certain characters ourselves one by one. // Square brackets are "reserved characters" that should not appear // in strings to begin with, so C# doesn't try to escape them in case diff --git a/Core/Net/RedirectWebClient.cs b/Core/Net/RedirectWebClient.cs index 3c279ce8e9..9eeeb8b891 100644 --- a/Core/Net/RedirectWebClient.cs +++ b/Core/Net/RedirectWebClient.cs @@ -8,9 +8,9 @@ namespace CKAN // HttpClient doesn't handle redirects well on Mono, but net7.0 considers WebClient obsolete internal sealed class RedirectWebClient : WebClient { - public RedirectWebClient() + public RedirectWebClient(string? userAgent = null) { - Headers.Add("User-Agent", Net.UserAgentString); + Headers.Add("User-Agent", userAgent ?? Net.UserAgentString); } protected override WebRequest GetWebRequest(Uri address) diff --git a/Netkan/CKAN-netkan.csproj b/Netkan/CKAN-netkan.csproj index 3c0b1b7b71..ef0a2801aa 100644 --- a/Netkan/CKAN-netkan.csproj +++ b/Netkan/CKAN-netkan.csproj @@ -37,11 +37,13 @@ + + diff --git a/Netkan/Sources/SourceForge/ISourceForgeApi.cs b/Netkan/Sources/SourceForge/ISourceForgeApi.cs new file mode 100644 index 0000000000..b8424e9807 --- /dev/null +++ b/Netkan/Sources/SourceForge/ISourceForgeApi.cs @@ -0,0 +1,7 @@ +namespace CKAN.NetKAN.Sources.SourceForge +{ + internal interface ISourceForgeApi + { + SourceForgeMod GetMod(SourceForgeRef sfRef); + } +} diff --git a/Netkan/Sources/SourceForge/SourceForgeApi.cs b/Netkan/Sources/SourceForge/SourceForgeApi.cs new file mode 100644 index 0000000000..662b5a51a3 --- /dev/null +++ b/Netkan/Sources/SourceForge/SourceForgeApi.cs @@ -0,0 +1,26 @@ +using System; +using System.IO; +using System.Xml; +using System.ServiceModel.Syndication; + +using CKAN.NetKAN.Services; + +namespace CKAN.NetKAN.Sources.SourceForge +{ + internal sealed class SourceForgeApi : ISourceForgeApi + { + public SourceForgeApi(IHttpService httpSvc) + { + this.httpSvc = httpSvc; + } + + public SourceForgeMod GetMod(SourceForgeRef sfRef) + => new SourceForgeMod(sfRef, + SyndicationFeed.Load(XmlReader.Create(new StringReader( + httpSvc.DownloadText(new Uri( + $"https://sourceforge.net/projects/{sfRef.Name}/rss")) + ?? "")))); + + private readonly IHttpService httpSvc; + } +} diff --git a/Netkan/Sources/SourceForge/SourceForgeMod.cs b/Netkan/Sources/SourceForge/SourceForgeMod.cs new file mode 100644 index 0000000000..a1a7b09658 --- /dev/null +++ b/Netkan/Sources/SourceForge/SourceForgeMod.cs @@ -0,0 +1,29 @@ +using System; +using System.Linq; +using System.ServiceModel.Syndication; + +namespace CKAN.NetKAN.Sources.SourceForge +{ + internal class SourceForgeMod + { + public SourceForgeMod(SourceForgeRef sfRef, + SyndicationFeed feed) + { + Title = feed.Title.Text; + Description = feed.Description.Text; + HomepageLink = $"https://sourceforge.net/projects/{sfRef.Name}/"; + RepositoryLink = $"https://sourceforge.net/p/{sfRef.Name}/code/"; + BugTrackerLink = $"https://sourceforge.net/p/{sfRef.Name}/bugs/"; + Versions = feed.Items.Where(item => item.Title.Text.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) + .Select(item => new SourceForgeVersion(item)) + .ToArray(); + } + + public readonly string Title; + public readonly string Description; + public readonly string HomepageLink; + public readonly string RepositoryLink; + public readonly string BugTrackerLink; + public readonly SourceForgeVersion[] Versions; + } +} diff --git a/Netkan/Sources/SourceForge/SourceForgeRef.cs b/Netkan/Sources/SourceForge/SourceForgeRef.cs new file mode 100644 index 0000000000..fb2d089bb8 --- /dev/null +++ b/Netkan/Sources/SourceForge/SourceForgeRef.cs @@ -0,0 +1,40 @@ +using System.Text.RegularExpressions; + +using CKAN.Extensions; +using CKAN.NetKAN.Model; + +namespace CKAN.NetKAN.Sources.SourceForge +{ + /// + /// Represents a SourceForge $kref + /// + internal sealed class SourceForgeRef : RemoteRef + { + /// + /// Initialize the SourceForge reference + /// + /// The base $kref object from a netkan + public SourceForgeRef(RemoteRef reference) + : base(reference) + { + if (Pattern.TryMatch(reference.Id, out Match? match)) + { + Name = match.Groups["name"].Value; + } + else + { + throw new Kraken(string.Format(@"Could not parse reference: ""{0}""", + reference)); + } + } + + /// + /// The name of the project on SourceForge + /// + public readonly string Name; + + private static readonly Regex Pattern = + new Regex(@"^(?[^/]+)$", + RegexOptions.Compiled); + } +} diff --git a/Netkan/Sources/SourceForge/SourceForgeVersion.cs b/Netkan/Sources/SourceForge/SourceForgeVersion.cs new file mode 100644 index 0000000000..d536ff1122 --- /dev/null +++ b/Netkan/Sources/SourceForge/SourceForgeVersion.cs @@ -0,0 +1,21 @@ +using System; +using System.Linq; +using System.ServiceModel.Syndication; + +namespace CKAN.NetKAN.Sources.SourceForge +{ + internal class SourceForgeVersion + { + public SourceForgeVersion(SyndicationItem item) + { + Title = item.Title.Text.TrimStart('/'); + // Throw an exception on missing or multiple s + Link = item.Links.Single().Uri; + Timestamp = item.PublishDate; + } + + public readonly string Title; + public readonly Uri Link; + public readonly DateTimeOffset Timestamp; + } +} diff --git a/Netkan/Sources/Spacedock/ISpaceDock.cs b/Netkan/Sources/Spacedock/ISpacedockApi.cs similarity index 100% rename from Netkan/Sources/Spacedock/ISpaceDock.cs rename to Netkan/Sources/Spacedock/ISpacedockApi.cs diff --git a/Netkan/Sources/Spacedock/SpaceDockApi.cs b/Netkan/Sources/Spacedock/SpacedockApi.cs similarity index 100% rename from Netkan/Sources/Spacedock/SpaceDockApi.cs rename to Netkan/Sources/Spacedock/SpacedockApi.cs diff --git a/Netkan/Transformers/NetkanTransformer.cs b/Netkan/Transformers/NetkanTransformer.cs index 267c6aa6e8..d9306f126b 100644 --- a/Netkan/Transformers/NetkanTransformer.cs +++ b/Netkan/Transformers/NetkanTransformer.cs @@ -10,6 +10,7 @@ using CKAN.NetKAN.Sources.Jenkins; using CKAN.NetKAN.Sources.Spacedock; using CKAN.Games; +using CKAN.NetKAN.Sources.SourceForge; namespace CKAN.NetKAN.Transformers { @@ -35,6 +36,7 @@ public NetkanTransformer(IHttpService http, _validator = validator; var ghApi = new GithubApi(http, githubToken); var glApi = new GitlabApi(http, gitlabToken); + var sfApi = new SourceForgeApi(http); _transformers = InjectVersionedOverrideTransformers(new List { new StagingTransformer(game), @@ -43,6 +45,7 @@ public NetkanTransformer(IHttpService http, new CurseTransformer(new CurseApi(http)), new GithubTransformer(ghApi, prerelease), new GitlabTransformer(glApi), + new SourceForgeTransformer(sfApi), new HttpTransformer(), new JenkinsTransformer(new JenkinsApi(http)), new AvcKrefTransformer(http, ghApi), diff --git a/Netkan/Transformers/SourceForgeTransformer.cs b/Netkan/Transformers/SourceForgeTransformer.cs new file mode 100644 index 0000000000..6b76c70f87 --- /dev/null +++ b/Netkan/Transformers/SourceForgeTransformer.cs @@ -0,0 +1,90 @@ +using System; +using System.Linq; +using System.Collections.Generic; + +using Newtonsoft.Json.Linq; +using log4net; + +using CKAN.NetKAN.Model; +using CKAN.NetKAN.Extensions; +using CKAN.NetKAN.Sources.SourceForge; + +namespace CKAN.NetKAN.Transformers +{ + /// + /// An that looks up data from GitLab. + /// + internal sealed class SourceForgeTransformer : ITransformer + { + /// + /// Initialize the transformer + /// + /// Object to use for accessing the SourceForge API + public SourceForgeTransformer(ISourceForgeApi api) + { + this.api = api; + } + + /// + /// Defines the name of this transformer + /// + public string Name => "sourceforge"; + + /// + /// If input metadata has a GitLab kref, inflate it with whatever info we can get, + /// otherwise return it unchanged + /// + /// Input netkan + /// Inflation options from command line + /// + public IEnumerable Transform(Metadata metadata, TransformOptions? opts) + { + if (metadata.Kref?.Source == Name) + { + log.InfoFormat("Executing SourceForge transformation with {0}", metadata.Kref); + var reference = new SourceForgeRef(metadata.Kref); + var mod = api.GetMod(reference); + var releases = mod.Versions + .Skip(opts?.SkipReleases ?? 0) + .Take(opts?.Releases ?? 1) + .ToArray(); + if (releases.Length < 1) + { + log.WarnFormat("No releases found for {0}", reference); + return Enumerable.Repeat(metadata, 1); + } + return releases.Select(ver => TransformOne(metadata.Json(), mod, ver)); + } + else + { + // Passthrough for non-GitLab mods + return Enumerable.Repeat(metadata, 1); + } + } + + private static Metadata TransformOne(JObject json, + SourceForgeMod mod, + SourceForgeVersion version) + { + json.SafeAdd("name", mod.Title); + json.SafeMerge("resources", JObject.FromObject(new Dictionary() + { + { "homepage", mod.HomepageLink }, + { "repository", mod.RepositoryLink }, + { "bugtracker", mod.BugTrackerLink }, + })); + // SourceForge doesn't send redirects to user agents it considers browser-like + json.SafeAdd("download", Net.ResolveRedirect(version.Link, "Wget") + ?.OriginalString); + json.SafeAdd(Metadata.UpdatedPropertyName, version.Timestamp); + + json.Remove("$kref"); + + log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json); + return new Metadata(json); + } + + private readonly ISourceForgeApi api; + private static readonly ILog log = LogManager.GetLogger(typeof(GitlabTransformer)); + } +} diff --git a/Netkan/Validators/KrefValidator.cs b/Netkan/Validators/KrefValidator.cs index 2f989bdad8..5ddea11404 100644 --- a/Netkan/Validators/KrefValidator.cs +++ b/Netkan/Validators/KrefValidator.cs @@ -22,6 +22,7 @@ public void Validate(Metadata metadata) case "jenkins": case "netkan": case "spacedock": + case "sourceforge": // We know this $kref, looks good break; diff --git a/Spec.md b/Spec.md index 2e2797d433..126d4a32a9 100644 --- a/Spec.md +++ b/Spec.md @@ -882,6 +882,29 @@ An `x_netkan_gitlab` field must be provided to customize how the metadata is fet Specifies that the source ZIP of the release will be used instead of any discrete assets.
Note that this must be `true`! GitLab only offers source ZIP assets, so we can only index mods that use them. If at some point in the future GitLab adds support for non-source assets, we will be able to add support for setting this property to `false` or omitting it. +###### `#/ckan/sourceforge/:repo` + +Indicates that data should be fetched from SourceForge using the `:repo` provided. +For example: `'#/ckan/sourceforge/ksre` + +When used, the following fields will be auto-filled if not already present: + +- `name` +- `resources.homepage` +- `resources.repository` +- `resources.bugtracker` +- `download` +- `download_size` +- `download_hash` +- `download_content_type` +- `release_date` + +An example `.netkan` excerpt: + +```yaml +$kref: '#/ckan/sourceforge/ksre' +``` + ###### `#/ckan/jenkins/:joburl` Indicates data should be fetched from a [Jenkins CI server](https://jenkins-ci.org/) using the `:joburl` provided. For