Fix oss-detect-backdoor output, add add'l patterns

Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/dotnet/core/sdk:3.1
+FROM mcr.microsoft.com/dotnet/sdk:5.0
 COPY . /app/
 WORKDIR /app/src
 

src/Shared/PackageManagers/NPMProjectManager.cs

@@ -4,6 +4,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text.Encodings.Web;
 using System.Text.Json;
 using System.Threading.Tasks;
 using Version = SemVer.Version;
@@ -184,7 +185,7 @@ protected async override Task<Dictionary<PackageURL, double>> PackageMetadataSea
             string metadata)
         {
             var mapping = new Dictionary<PackageURL, double>();
-            if (purl?.Name is string purlName && (purlName.StartsWith('_') || npm_internal_modules.Contains(purlName)))
+            if (purl?.Name is string purlName && (purlName.StartsWith('_') || NODEJS_INTERNAL_MODULES.Contains(purlName)))
             {
                 // url = 'https://github.com/nodejs/node/tree/master/lib' + package.name,
 
@@ -233,7 +234,10 @@ protected async override Task<Dictionary<PackageURL, double>> PackageMetadataSea
             return mapping;
         }
 
-        private static readonly List<string> npm_internal_modules = new List<string>()
+        /// <summary>
+        /// Internal Node.js modules that should be ignored when searching metadata.
+        /// </summary>
+        private static readonly List<string> NODEJS_INTERNAL_MODULES = new List<string>()
         {
             "assert",
             "async_hooks",

src/Shared/PackageManagers/VSMProjectManager.cs

@@ -190,6 +190,7 @@ public override async Task<IEnumerable<string>> EnumerateVersions(PackageURL pur
                     SetCache(packageName, resultStreamReader.ReadToEnd());
                     resultStream.Seek(0, SeekOrigin.Begin);
                 }
+
                 var doc = await JsonDocument.ParseAsync(resultStream);
                 await resultStream.DisposeAsync();
 

src/oss-characteristics/CharacteristicTool.cs

@@ -83,8 +83,7 @@ public CharacteristicTool() : base()
                 LogFileLevel = "Off",
                 SourcePath = directory,
                 IgnoreDefaultRules = options.DisableDefaultRules == true,
-                CustomRulesPath = options.CustomRuleDirectory,
-
+                CustomRulesPath = options.CustomRuleDirectory
             };
 
             try
@@ -235,7 +234,7 @@ private void AppendOutput(IOutputBuilder outputBuilder, PackageURL purl, Diction
             }
         }
 
-        private async Task RunAsync(Options options)
+        public async Task RunAsync(Options options)
         {
             // select output destination and format
             SelectOutput(options.OutputFile);

src/oss-detect-backdoor/DetectBackdoorTool.cs

@@ -29,6 +29,14 @@ public static IEnumerable<Example> Examples
                             HelpText = "the directory to download the package to.")]
             public string DownloadDirectory { get; set; } = ".";
 
+            [Option('f', "format", Required = false, Default = "text",
+                HelpText = "selct the output format(text|sarifv1|sarifv2)")]
+            public string Format { get; set; } = "text";
+
+            [Option('o', "output-file", Required = false, Default = "",
+                HelpText = "send the command output to a file instead of stdout")]
+            public string OutputFile { get; set; } = "";
+
             [Value(0, Required = true,
                 HelpText = "PackgeURL(s) specifier to analyze (required, repeats OK)", Hidden = true)] // capture all targets to analyze
             public IEnumerable<string>? Targets { get; set; }
@@ -69,22 +77,10 @@ private async Task RunAsync(Options options)
                 cOptions.CustomRuleDirectory = RULE_DIRECTORY;
                 cOptions.DownloadDirectory = options.DownloadDirectory;
                 cOptions.UseCache = options.UseCache;
+                cOptions.Format = options.Format;
+                cOptions.OutputFile = options.OutputFile;
 
-                foreach (var target in targetList)
-                {
-                    try
-                    {
-                        var purl = new PackageURL(target);
-                        string downloadDirectory = options.DownloadDirectory == "." ? Directory.GetCurrentDirectory() : options.DownloadDirectory;
-                        characteristicTool.AnalyzePackage(cOptions, purl,
-                            downloadDirectory,
-                            options.UseCache == true).Wait();
-                    }
-                    catch (Exception ex)
-                    {
-                        Logger.Warn(ex, "Error processing {0}: {1}", target, ex.Message);
-                    }
-                }
+                characteristicTool.RunAsync(cOptions).Wait();
             }
         }
     }

src/oss-detect-backdoor/Resources/BackdoorRules/obfuscation.json

@@ -15,5 +15,34 @@
         "modifiers": [ "i" ]
       }
     ]
+  },
+  {
+    "name": "Backdoor: Executing Obfuscated Code",
+    "id": "BD000801",
+    "description": "Backdoor: Executing Obfuscated Code",
+    "tags": [
+      "Security.Backdoor.ExecuteObfuscatedCode"
+    ],
+    "severity": "important",
+    "patterns": [
+      {
+        "pattern": "fromhex.+decode",
+        "type": "regex",
+        "scopes": [ "code" ],
+        "modifiers": [ "i" ]
+      },
+      {
+        "pattern": "\\.get\\(.*(fromhex|decode)",
+        "type": "regex",
+        "scopes": [ "code" ],
+        "modifiers": [ "i" ]
+      },
+      {
+        "pattern": "65786563",
+        "type": "string",
+        "scopes": [ "code" ],
+        "modifiers": [ "i" ]
+      }
+    ]
   }
 ]

src/oss-tests/DownloadTests.cs

@@ -111,6 +111,13 @@ public async Task NPM_Download_Version_Succeeds(string purl, string targetFilena
             await TestDownload(purl, targetFilename, expectedDirectoryCount);
         }
 
+        [DataTestMethod]
+        [DataRow("pkg:npm/%40angular%2Fanimation@4.0.0-beta.8", "package.json", 1)]
+        public async Task NPM_Download_ScopedVersion_Succeeds(string purl, string targetFilename, int expectedDirectoryCount)
+        {
+            await TestDownload(purl, targetFilename, expectedDirectoryCount);
+        }
+
         [DataTestMethod]
         [DataRow("pkg:nuget/RandomType@2.0.0", "RandomType.nuspec", 1)]
         [DataRow("pkg:nuget/d3.TypeScript.DefinitelyTyped", "d3.TypeScript.DefinitelyTyped.nuspec", 1)]
@@ -154,7 +161,7 @@ await Assert.ThrowsExceptionAsync<InternalTestFailureException>(async () =>
         }
 
         [DataTestMethod]
-        [DataRow("pkg:vsm/ms-vscode/Theme-1337", "extension.vsixmanifest", 1)]
+        [DataRow("pkg:vsm/ms-vscode/PowerShell", "extension.vsixmanifest", 1)]
         public async Task VSM_Download_Version_Succeeds(string purl, string targetFilename, int expectedDirectoryCount)
         {
             await TestDownload(purl, targetFilename, expectedDirectoryCount);
@@ -192,7 +199,7 @@ private void deleteTempDirs(PackageDownloader? packageDownloader, string tempDir
         /// <param name="packageUrl"> </param>
         /// <param name="tempDirectoryName"> </param>
         /// <returns> </returns>
-        private async Task<PackageDownloader?> DownloadPackage(PackageURL packageUrl, string tempDirectoryName, bool doCache = false)
+        private PackageDownloader? DownloadPackage(PackageURL packageUrl, string tempDirectoryName, bool doCache = false)
         {
             int numAttempts = 3;
             int numSecondsWait = 10;
@@ -231,7 +238,7 @@ private async Task TestDownload(string purl, string targetFilename, int expected
             try
             {
                 var packageUrl = new PackageURL(purl);
-                var packageDownloader = DownloadPackage(packageUrl, tempDirectoryName).Result;
+                var packageDownloader = DownloadPackage(packageUrl, tempDirectoryName);
 
                 var targetFileWasDownloaded = Directory.EnumerateFiles(tempDirectoryName, targetFilename, SearchOption.AllDirectories).Any();
                 if (!targetFileWasDownloaded)
@@ -242,18 +249,16 @@ private async Task TestDownload(string purl, string targetFilename, int expected
                 var topLevelDirectoryCount = Directory.GetDirectories(tempDirectoryName).Length;
                 if (expectedDirectoryCount != topLevelDirectoryCount)
                 {
-                    Console.WriteLine(string.Join(";", Directory.GetDirectories(tempDirectoryName)));
                     errorString = string.Format("Directory count {0} does not match expected {1}", topLevelDirectoryCount, expectedDirectoryCount);
                 }
 
                 // Download again (with caching) - TODO, move this to a separate test.
-                await DownloadPackage(packageUrl, tempDirectoryName, true);
+                packageDownloader = DownloadPackage(packageUrl, tempDirectoryName, true);
 
                 // Re-calculate the top level directories, since in might have changed (it shouldn't).
                 topLevelDirectoryCount = Directory.GetDirectories(tempDirectoryName).Length;
                 if (expectedDirectoryCount != topLevelDirectoryCount)
                 {
-                    Console.WriteLine(string.Join(";", Directory.GetDirectories(tempDirectoryName)));
                     errorString = string.Format("Directory count {0} does not match expected {1}", topLevelDirectoryCount, expectedDirectoryCount);
                 }