From 19909b7e10f5e0c19842c61fbeee384be4f02451 Mon Sep 17 00:00:00 2001 From: Michael Scovetta Date: Tue, 17 Nov 2020 12:28:19 -0800 Subject: [PATCH 1/2] Fix broken tests, add NPM scoped test. --- Dockerfile | 2 +- src/Shared/PackageManagers/NPMProjectManager.cs | 8 ++++++-- src/Shared/PackageManagers/VSMProjectManager.cs | 1 + src/oss-tests/DownloadTests.cs | 17 +++++++++++------ 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0f653bd1..3c394f2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/dotnet/core/sdk:3.1 +FROM mcr.microsoft.com/dotnet/sdk:5.0 COPY . /app/ WORKDIR /app/src diff --git a/src/Shared/PackageManagers/NPMProjectManager.cs b/src/Shared/PackageManagers/NPMProjectManager.cs index efee6eb5..f36f69a2 100644 --- a/src/Shared/PackageManagers/NPMProjectManager.cs +++ b/src/Shared/PackageManagers/NPMProjectManager.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text.Encodings.Web; using System.Text.Json; using System.Threading.Tasks; using Version = SemVer.Version; @@ -184,7 +185,7 @@ protected async override Task> PackageMetadataSea string metadata) { var mapping = new Dictionary(); - if (purl?.Name is string purlName && (purlName.StartsWith('_') || npm_internal_modules.Contains(purlName))) + if (purl?.Name is string purlName && (purlName.StartsWith('_') || NODEJS_INTERNAL_MODULES.Contains(purlName))) { // url = 'https://github.com/nodejs/node/tree/master/lib' + package.name, @@ -233,7 +234,10 @@ protected async override Task> PackageMetadataSea return mapping; } - private static readonly List npm_internal_modules = new List() + /// + /// Internal Node.js modules that should be ignored when searching metadata. + /// + private static readonly List NODEJS_INTERNAL_MODULES = new List() { "assert", "async_hooks", diff --git a/src/Shared/PackageManagers/VSMProjectManager.cs b/src/Shared/PackageManagers/VSMProjectManager.cs index 2f3f3ec8..f54c4bb5 100644 --- a/src/Shared/PackageManagers/VSMProjectManager.cs +++ b/src/Shared/PackageManagers/VSMProjectManager.cs @@ -190,6 +190,7 @@ public override async Task> EnumerateVersions(PackageURL pur SetCache(packageName, resultStreamReader.ReadToEnd()); resultStream.Seek(0, SeekOrigin.Begin); } + var doc = await JsonDocument.ParseAsync(resultStream); await resultStream.DisposeAsync(); diff --git a/src/oss-tests/DownloadTests.cs b/src/oss-tests/DownloadTests.cs index 23c55864..3cc8c105 100644 --- a/src/oss-tests/DownloadTests.cs +++ b/src/oss-tests/DownloadTests.cs @@ -111,6 +111,13 @@ public async Task NPM_Download_Version_Succeeds(string purl, string targetFilena await TestDownload(purl, targetFilename, expectedDirectoryCount); } + [DataTestMethod] + [DataRow("pkg:npm/%40angular%2Fanimation@4.0.0-beta.8", "package.json", 1)] + public async Task NPM_Download_ScopedVersion_Succeeds(string purl, string targetFilename, int expectedDirectoryCount) + { + await TestDownload(purl, targetFilename, expectedDirectoryCount); + } + [DataTestMethod] [DataRow("pkg:nuget/RandomType@2.0.0", "RandomType.nuspec", 1)] [DataRow("pkg:nuget/d3.TypeScript.DefinitelyTyped", "d3.TypeScript.DefinitelyTyped.nuspec", 1)] @@ -154,7 +161,7 @@ await Assert.ThrowsExceptionAsync(async () => } [DataTestMethod] - [DataRow("pkg:vsm/ms-vscode/Theme-1337", "extension.vsixmanifest", 1)] + [DataRow("pkg:vsm/ms-vscode/PowerShell", "extension.vsixmanifest", 1)] public async Task VSM_Download_Version_Succeeds(string purl, string targetFilename, int expectedDirectoryCount) { await TestDownload(purl, targetFilename, expectedDirectoryCount); @@ -192,7 +199,7 @@ private void deleteTempDirs(PackageDownloader? packageDownloader, string tempDir /// /// /// - private async Task DownloadPackage(PackageURL packageUrl, string tempDirectoryName, bool doCache = false) + private PackageDownloader? DownloadPackage(PackageURL packageUrl, string tempDirectoryName, bool doCache = false) { int numAttempts = 3; int numSecondsWait = 10; @@ -231,7 +238,7 @@ private async Task TestDownload(string purl, string targetFilename, int expected try { var packageUrl = new PackageURL(purl); - var packageDownloader = DownloadPackage(packageUrl, tempDirectoryName).Result; + var packageDownloader = DownloadPackage(packageUrl, tempDirectoryName); var targetFileWasDownloaded = Directory.EnumerateFiles(tempDirectoryName, targetFilename, SearchOption.AllDirectories).Any(); if (!targetFileWasDownloaded) @@ -242,18 +249,16 @@ private async Task TestDownload(string purl, string targetFilename, int expected var topLevelDirectoryCount = Directory.GetDirectories(tempDirectoryName).Length; if (expectedDirectoryCount != topLevelDirectoryCount) { - Console.WriteLine(string.Join(";", Directory.GetDirectories(tempDirectoryName))); errorString = string.Format("Directory count {0} does not match expected {1}", topLevelDirectoryCount, expectedDirectoryCount); } // Download again (with caching) - TODO, move this to a separate test. - await DownloadPackage(packageUrl, tempDirectoryName, true); + packageDownloader = DownloadPackage(packageUrl, tempDirectoryName, true); // Re-calculate the top level directories, since in might have changed (it shouldn't). topLevelDirectoryCount = Directory.GetDirectories(tempDirectoryName).Length; if (expectedDirectoryCount != topLevelDirectoryCount) { - Console.WriteLine(string.Join(";", Directory.GetDirectories(tempDirectoryName))); errorString = string.Format("Directory count {0} does not match expected {1}", topLevelDirectoryCount, expectedDirectoryCount); } From 306b4884313502404c6599b583ae408228c7da8e Mon Sep 17 00:00:00 2001 From: Michael Scovetta Date: Thu, 19 Nov 2020 16:29:28 -0800 Subject: [PATCH 2/2] Improve output of oss-detect-backdoor. --- src/oss-characteristics/CharacteristicTool.cs | 5 ++-- src/oss-detect-backdoor/DetectBackdoorTool.cs | 26 +++++++---------- .../Resources/BackdoorRules/obfuscation.json | 29 +++++++++++++++++++ 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/src/oss-characteristics/CharacteristicTool.cs b/src/oss-characteristics/CharacteristicTool.cs index f0adaa74..d92fa342 100644 --- a/src/oss-characteristics/CharacteristicTool.cs +++ b/src/oss-characteristics/CharacteristicTool.cs @@ -83,8 +83,7 @@ public CharacteristicTool() : base() LogFileLevel = "Off", SourcePath = directory, IgnoreDefaultRules = options.DisableDefaultRules == true, - CustomRulesPath = options.CustomRuleDirectory, - + CustomRulesPath = options.CustomRuleDirectory }; try @@ -235,7 +234,7 @@ private void AppendOutput(IOutputBuilder outputBuilder, PackageURL purl, Diction } } - private async Task RunAsync(Options options) + public async Task RunAsync(Options options) { // select output destination and format SelectOutput(options.OutputFile); diff --git a/src/oss-detect-backdoor/DetectBackdoorTool.cs b/src/oss-detect-backdoor/DetectBackdoorTool.cs index 1a182cf3..a0c66259 100644 --- a/src/oss-detect-backdoor/DetectBackdoorTool.cs +++ b/src/oss-detect-backdoor/DetectBackdoorTool.cs @@ -29,6 +29,14 @@ public static IEnumerable Examples HelpText = "the directory to download the package to.")] public string DownloadDirectory { get; set; } = "."; + [Option('f', "format", Required = false, Default = "text", + HelpText = "selct the output format(text|sarifv1|sarifv2)")] + public string Format { get; set; } = "text"; + + [Option('o', "output-file", Required = false, Default = "", + HelpText = "send the command output to a file instead of stdout")] + public string OutputFile { get; set; } = ""; + [Value(0, Required = true, HelpText = "PackgeURL(s) specifier to analyze (required, repeats OK)", Hidden = true)] // capture all targets to analyze public IEnumerable? Targets { get; set; } @@ -69,22 +77,10 @@ private async Task RunAsync(Options options) cOptions.CustomRuleDirectory = RULE_DIRECTORY; cOptions.DownloadDirectory = options.DownloadDirectory; cOptions.UseCache = options.UseCache; + cOptions.Format = options.Format; + cOptions.OutputFile = options.OutputFile; - foreach (var target in targetList) - { - try - { - var purl = new PackageURL(target); - string downloadDirectory = options.DownloadDirectory == "." ? Directory.GetCurrentDirectory() : options.DownloadDirectory; - characteristicTool.AnalyzePackage(cOptions, purl, - downloadDirectory, - options.UseCache == true).Wait(); - } - catch (Exception ex) - { - Logger.Warn(ex, "Error processing {0}: {1}", target, ex.Message); - } - } + characteristicTool.RunAsync(cOptions).Wait(); } } } diff --git a/src/oss-detect-backdoor/Resources/BackdoorRules/obfuscation.json b/src/oss-detect-backdoor/Resources/BackdoorRules/obfuscation.json index decc27a4..5a3a7b55 100644 --- a/src/oss-detect-backdoor/Resources/BackdoorRules/obfuscation.json +++ b/src/oss-detect-backdoor/Resources/BackdoorRules/obfuscation.json @@ -15,5 +15,34 @@ "modifiers": [ "i" ] } ] + }, + { + "name": "Backdoor: Executing Obfuscated Code", + "id": "BD000801", + "description": "Backdoor: Executing Obfuscated Code", + "tags": [ + "Security.Backdoor.ExecuteObfuscatedCode" + ], + "severity": "important", + "patterns": [ + { + "pattern": "fromhex.+decode", + "type": "regex", + "scopes": [ "code" ], + "modifiers": [ "i" ] + }, + { + "pattern": "\\.get\\(.*(fromhex|decode)", + "type": "regex", + "scopes": [ "code" ], + "modifiers": [ "i" ] + }, + { + "pattern": "65786563", + "type": "string", + "scopes": [ "code" ], + "modifiers": [ "i" ] + } + ] } ] \ No newline at end of file