Migrate most of the logic that doesn't depend on CLI to a separate shared library

.github/workflows/codeql-analysis.yml

@@ -25,9 +25,9 @@ jobs:
         fetch-depth: 0
 
     - name: Setup .NET Core SDK
-      uses: actions/setup-dotnet@v1.7.2
+      uses: actions/setup-dotnet@v1
       with:
-        dotnet-version: 5.0.x
+        dotnet-version: 6.0.x
 
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v1

src/OSSGadget.sln

@@ -1,9 +1,9 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 16
-VisualStudioVersion = 16.0.29806.167
+# Visual Studio Version 17
+VisualStudioVersion = 17.0.32014.148
 MinimumVisualStudioVersion = 10.0.40219.1
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Shared", "Shared\Shared.csproj", "{292BEE61-1C30-4EB5-A2DE-810E7118433A}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Shared.CLI", "Shared\Shared.CLI.csproj", "{292BEE61-1C30-4EB5-A2DE-810E7118433A}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "oss-download", "oss-download\oss-download.csproj", "{DEBC05E9-A5A9-4FFE-9023-1248F2DA58D9}"
 EndProject
@@ -38,7 +38,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "oss-find-domain-squats", "o
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "oss-reproducible", "oss-reproducible\oss-reproducible.csproj", "{B79FCA22-5A3B-47F5-8FB1-692E3175162D}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "oss-find-squats-lib", "oss-find-squats-lib\oss-find-squats-lib.csproj", "{182CA72B-6BB1-400E-83FC-B35558928BAB}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "oss-find-squats-lib", "oss-find-squats-lib\oss-find-squats-lib.csproj", "{182CA72B-6BB1-400E-83FC-B35558928BAB}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Shared.Lib", "Shared-lib\Shared.Lib.csproj", "{E87DAFE2-A060-458F-8924-C946585BF746}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -109,6 +111,10 @@ Global
 		{182CA72B-6BB1-400E-83FC-B35558928BAB}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{182CA72B-6BB1-400E-83FC-B35558928BAB}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{182CA72B-6BB1-400E-83FC-B35558928BAB}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E87DAFE2-A060-458F-8924-C946585BF746}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E87DAFE2-A060-458F-8924-C946585BF746}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E87DAFE2-A060-458F-8924-C946585BF746}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E87DAFE2-A060-458F-8924-C946585BF746}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/Shared-lib/DefaultHttpClientFactory.cs

@@ -0,0 +1,48 @@
+// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
+
+namespace Microsoft.CST.OpenSource
+{
+    using Microsoft.CST.OpenSource.Helpers;
+    using System;
+    using System.Net.Http;
+
+    /// <summary>
+    /// This is the HttpClientFactory that will be used if one is not specified.  This factory lazily constructs a single HttpClient and presents it for reuse to reduce resource usage.
+    /// </summary>
+    public sealed class DefaultHttpClientFactory : IHttpClientFactory
+    {
+        [System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "IDE0044:Add readonly modifier", Justification = "Modified through reflection.")]
+        private string ENV_HTTPCLIENT_USER_AGENT = "OSSDL";
+
+        public DefaultHttpClientFactory(string? userAgent = null)
+        {
+            EnvironmentHelper.OverrideEnvironmentVariables(this);
+
+            _httpClientLazy = new(() =>
+            {
+                HttpClient cli = new(handler);
+                cli.DefaultRequestHeaders.UserAgent.ParseAdd(userAgent ?? ENV_HTTPCLIENT_USER_AGENT);
+                return cli;
+            });
+        }
+
+        private static readonly SocketsHttpHandler handler = new()
+        {
+            AllowAutoRedirect = true,
+            UseCookies = false,
+            MaxAutomaticRedirections = 5,
+            PooledConnectionIdleTimeout = TimeSpan.FromSeconds(30),
+            PooledConnectionLifetime = TimeSpan.FromSeconds(30),
+            AutomaticDecompression = System.Net.DecompressionMethods.All,
+        };
+
+        private readonly Lazy<HttpClient> _httpClientLazy;
+
+        /// <summary>
+        /// Returns the singleton HttpClient for this factory.
+        /// </summary>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public HttpClient CreateClient(string name) => _httpClientLazy.Value;
+    }
+}

src/Shared-lib/Exceptions/InvalidProjectManagerException.cs

@@ -0,0 +1,20 @@
+// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
+
+namespace Microsoft.CST.OpenSource.Exceptions
+{
+    using System;
+
+    /// <summary>
+    /// Exception thrown when the PackageURL has an invalid manager..
+    /// </summary>
+    public class InvalidProjectManagerException : Exception
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="InvalidProjectManagerException"/> class.
+        /// </summary>
+        public InvalidProjectManagerException(PackageURL packageUrl)
+            : base($"The package URL: {packageUrl} has an invalid Project Manager.")
+        {
+        }
+    }
+}

src/Shared-lib/Helpers/Check.cs

@@ -0,0 +1,127 @@
+// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
+
+namespace Microsoft.CST.OpenSource.Helpers
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Linq;
+
+    /// <summary>
+    /// Static helper methods for validating parameters.
+    /// </summary>
+    public static class Check
+    {
+        /// <summary>
+        /// Checks that the given parameter is not null, empty, or whitespace.
+        /// </summary>
+        /// <param name="paramName">The name of the parameter. Can be retrieved by using nameof(parameter).</param>
+        /// <param name="value">The value of the parameter.</param>
+        /// <returns>The value of the parameter if it passes the validation.</returns>
+        /// <exception cref="ArgumentNullException">The parameter value is null, empty, or whitespace.</exception>
+        public static string NotEmptyOrWhitespace(string paramName, string? value)
+        {
+            // The parameterName can't be null so explicitly check (can't call EnsureIsNotNullOrWhitespace in this case!)
+            if (string.IsNullOrWhiteSpace(paramName))
+            {
+                throw new ArgumentNullException(nameof(paramName));
+            }
+
+            if (string.IsNullOrWhiteSpace(value))
+            {
+                throw new ArgumentNullException(value);
+            }
+
+            return value;
+        }
+
+        /// <summary>
+        /// Checks that the given parameter is not null.
+        /// </summary>
+        /// <typeparam name="T">The type of the parameter.</typeparam>
+        /// <param name="paramName">The name of the parameter. Can be retrieved by using nameof(parameter).</param>
+        /// <param name="value">The value of the parameter.</param>
+        /// <param name="message">An optional message that will be included as apart of the exception if the value is null.</param>
+        /// <returns>The value of the parameter if it passes the validation.</returns>
+        /// <exception cref="ArgumentNullException">The parameter value is null.</exception>
+        public static T NotNull<T>(string paramName, T? value, string? message = null)
+        {
+            NotEmptyOrWhitespace(nameof(paramName), paramName);
+            message = message ?? $"Value cannot be null. (Parameter '{paramName}')";
+
+            if (value == null)
+            {
+                throw new ArgumentNullException(paramName, message);
+            }
+
+            return value;
+        }
+
+        /// <summary>
+        /// Check that the given parameter is in the enum.
+        /// </summary>
+        /// <typeparam name="T">The enum type of the parameter.</typeparam>
+        /// <param name="paramName">The name of the parameter. Can be retrieved by using nameof(parameter).</param>
+        /// <param name="value">The value of the parameter.</param>
+        /// <returns>The value of the parameter if it passes the validation.</returns>
+        /// <exception cref="ArgumentOutOfRangeException">The parameter value is not apart of the enum.</exception>
+        public static T IsInEnum<T>(string paramName, T value)
+            where T : Enum
+        {
+            NotEmptyOrWhitespace(nameof(paramName), paramName);
+            NotNull(paramName, value);
+
+            if (!Enum.IsDefined(value.GetType(), value))
+            {
+                throw new ArgumentOutOfRangeException(
+                    paramName,
+                    $"The value is {value} is not a valid value of {value.GetType()}.");
+            }
+
+            return value;
+        }
+
+        /// <summary>
+        /// Ensures that the provided enumerable is not null and contains at least one item.
+        /// </summary>
+        /// <typeparam name="T">The type of object contained in the enumerable.</typeparam>
+        /// <param name="parameterName">Parameter name (embedded in exception if thrown).</param>
+        /// <param name="parameterValue">The value to validate.</param>
+        /// <exception cref="ArgumentNullException"> Enumerable object is null.</exception>
+        /// <exception cref="ArgumentOutOfRangeException">Enumerable does not contain at least 1 object.</exception>
+        public static void NotNullOrEmpty<T>(
+            string parameterName,
+            IEnumerable<T>? parameterValue)
+        {
+            NotNullAndHasMinimumCount(parameterName, parameterValue, 1);
+        }
+
+        /// <summary>
+        /// Ensures that the provided enumerable is not null and contains at least minimumCount elements.
+        /// </summary>
+        /// <typeparam name="T">The type of object contained in the enumerable.</typeparam>
+        /// <param name="parameterName">Parameter name (embedded in exception if thrown).</param>
+        /// <param name="parameterValue">The value to validate.</param>
+        /// <param name="minimumCount">The minimum number of items required in the enumerable.</param>
+        /// <exception cref="ArgumentNullException">Enumerable object is null.</exception>
+        /// <exception cref="ArgumentOutOfRangeException">Enumerable does not contain at least <paramref name="minimumCount"/> number of objects.</exception>
+        public static void NotNullAndHasMinimumCount<T>(
+            string parameterName,
+            IEnumerable<T>? parameterValue,
+            int minimumCount)
+        {
+            NotEmptyOrWhitespace(nameof(parameterName), parameterName);
+
+            if (parameterValue == null)
+            {
+                throw new ArgumentNullException(parameterName);
+            }
+
+            if (parameterValue.Count() < minimumCount)
+            {
+                throw new ArgumentOutOfRangeException(
+                    parameterName,
+                    $"{parameterName} does not contain at least {minimumCount} elements.");
+            }
+        }
+    }
+}

src/Shared-lib/Helpers/EnvironmentHelper.cs

@@ -0,0 +1,39 @@
+// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
+
+namespace Microsoft.CST.OpenSource.Helpers
+{
+    using System;
+    using System.Reflection;
+
+    public class EnvironmentHelper
+    {
+        public static NLog.ILogger Logger { get; set; } = NLog.LogManager.GetCurrentClassLogger();
+
+        /// <summary>
+        ///     Overrides static members starting with ENV_ with the respective environment variables.Allows
+        ///     users to easily override fields like API endpoint roots. Only strings are supported.
+        /// </summary>
+        /// <param name="targetObject"> Examine this object (using reflection) </param>
+        public static void OverrideEnvironmentVariables(object targetObject)
+        {
+            foreach (FieldInfo fieldInfo in targetObject.GetType().GetFields(BindingFlags.Static |
+                                                                       BindingFlags.Public |
+                                                                       BindingFlags.NonPublic))
+            {
+                if (fieldInfo.FieldType.FullName == "System.String" &&
+                    fieldInfo.Name.StartsWith("ENV_") &&
+                    fieldInfo.Name.Length > 4)
+                {
+                    string? bareName = fieldInfo.Name[4..];
+
+                    string? value = Environment.GetEnvironmentVariable(bareName);
+                    if (value != null)
+                    {
+                        Logger.Debug("Assiging value of {0} to {1}", bareName, fieldInfo.Name);
+                        fieldInfo.SetValue(null, value);
+                    }
+                }
+            }
+        }
+    }
+}

src/Shared/Helpers/StringExtensions.cs → src/Shared-lib/Helpers/StringExtensions.cs

@@ -1,6 +1,6 @@
 // Copyright (c) Microsoft Corporation. Licensed under the MIT License.
 
-namespace Microsoft.CST.OpenSource.Shared.Extensions
+namespace Microsoft.CST.OpenSource.Helpers
 {
     /// <summary>
     /// A utilities class for string extension functions.

src/Shared/Model/Repository.cs → src/Shared-lib/Model/Repository.cs

@@ -2,13 +2,13 @@
 
 namespace Microsoft.CST.OpenSource.Model
 {
-    using Microsoft.CST.OpenSource.Shared;
     using Newtonsoft.Json;
     using Octokit;
     using System;
     using System.Collections.Generic;
     using System.Threading.Tasks;
-    using CSTRepository = Microsoft.CST.OpenSource.Model.Repository;
+    using Utilities;
+    using CSTRepository = Model.Repository;
     using GHRepository = Octokit.Repository;
 
     public class Repository
@@ -30,7 +30,7 @@ public class Repository
         public bool? Archived { get; set; }
 
         [JsonProperty(PropertyName = "contributors", NullValueHandling = NullValueHandling.Ignore)]
-        public List<User>? Contributors { get; set; }
+        public List<Model.User>? Contributors { get; set; }
 
         [JsonProperty(PropertyName = "created_at", NullValueHandling = NullValueHandling.Ignore)]
         public DateTimeOffset? CreatedAt { get; set; }
@@ -54,19 +54,19 @@ public class Repository
         public string? Language { get; set; }
 
         [JsonProperty(PropertyName = "licenses", NullValueHandling = NullValueHandling.Ignore)]
-        public List<License>? Licenses { get; set; }
+        public List<Model.License>? Licenses { get; set; }
 
         [JsonProperty(PropertyName = "linked_data", NullValueHandling = NullValueHandling.Ignore)]
         public LinkedData? LinkedData { get; set; }
 
         [JsonProperty(PropertyName = "maintainers", NullValueHandling = NullValueHandling.Ignore)]
-        public List<User>? Maintainers { get; set; }
+        public List<Model.User>? Maintainers { get; set; }
 
         [JsonProperty(PropertyName = "openissues_count", NullValueHandling = NullValueHandling.Ignore)]
         public int? OpenIssuesCount { get; set; }
 
         [JsonProperty(PropertyName = "owner", NullValueHandling = NullValueHandling.Ignore)]
-        public User? Owner { get; set; }
+        public Model.User? Owner { get; set; }
 
         [JsonProperty(PropertyName = "parent", NullValueHandling = NullValueHandling.Ignore)]
         public string? Parent { get; set; }
@@ -121,33 +121,33 @@ public class Repository
                 Archived = ghRepository.Archived;
                 CreatedAt = ghRepository.CreatedAt;
                 UpdatedAt = ghRepository.UpdatedAt;
-                Description = Utilities.GetMaxClippedLength(ghRepository.Description);
+                Description = OssUtilities.GetMaxClippedLength(ghRepository.Description);
                 IsFork = ghRepository.Fork;
                 Forks = ghRepository.ForksCount;
-                Homepage = Utilities.GetMaxClippedLength(ghRepository.Homepage);
+                Homepage = OssUtilities.GetMaxClippedLength(ghRepository.Homepage);
                 Id = ghRepository.Id;
-                Language = Utilities.GetMaxClippedLength(ghRepository.Language);
+                Language = OssUtilities.GetMaxClippedLength(ghRepository.Language);
                 Name = ghRepository.Name;
                 OpenIssuesCount = ghRepository.OpenIssuesCount;
                 Parent = ghRepository.Parent?.Url;
                 PushedAt = ghRepository.PushedAt;
                 Size = ghRepository.Size;
                 FollowersCount = ghRepository.StargazersCount;
-                Uri = Utilities.GetMaxClippedLength(ghRepository.Url);
+                Uri = OssUtilities.GetMaxClippedLength(ghRepository.Url);
                 StakeholdersCount = ghRepository.WatchersCount;
 
                 if (ghRepository.License is not null)
                 {
-                    Licenses ??= new List<License>();
-                    Licenses.Add(new License()
+                    Licenses ??= new List<Model.License>();
+                    Licenses.Add(new Model.License()
                     {
                         Name = ghRepository.License.Name,
                         Url = ghRepository.License.Url,
                         SPIX_ID = ghRepository.License.SpdxId
                     });
                 }
 
-                Owner ??= new User()
+                Owner ??= new Model.User()
                 {
                     Id = ghRepository.Owner.Id,
                     Name = ghRepository.Owner.Name,