Skip to content

Commit

Permalink
Add support for NuGet Curation Audit (#159)
Browse files Browse the repository at this point in the history
  • Loading branch information
@igorz-jf
igorz-jf authored Sep 5, 2024
1 parent 64585a6 commit 2243ace
Show file tree
Hide file tree
Showing 13 changed files with 478 additions and 54 deletions.
114 changes: 85 additions & 29 deletions commands/audit/sca/nuget/nuget.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,34 +9,39 @@ import (
"path/filepath"
"strings"

"github.com/jfrog/gofrog/datastructures"
"github.com/jfrog/jfrog-client-go/utils/errorutils"
"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
"github.com/jfrog/jfrog-client-go/utils/log"
xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
"golang.org/x/exp/maps"

bidotnet "github.com/jfrog/build-info-go/build/utils/dotnet"
"github.com/jfrog/build-info-go/build/utils/dotnet/solution"
"github.com/jfrog/build-info-go/entities"
biutils "github.com/jfrog/build-info-go/utils"
"github.com/jfrog/gofrog/datastructures"

"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/dotnet"
"github.com/jfrog/jfrog-cli-core/v2/utils/config"

"github.com/jfrog/jfrog-cli-security/commands/audit/sca"
"github.com/jfrog/jfrog-cli-security/utils"
"github.com/jfrog/jfrog-cli-security/utils/xray"
"github.com/jfrog/jfrog-client-go/utils/errorutils"
"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
"github.com/jfrog/jfrog-client-go/utils/log"
xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
"golang.org/x/exp/maps"
)

const (
nugetPackageTypeIdentifier = "nuget://"
csprojFileSuffix = ".csproj"
packageReferenceSyntax = "PackageReference"
packageReferenceSyntax = "PackageReference Include"
packagesConfigFileName = "packages.config"
installCommandName = "restore"
dotnetToolType = "dotnet"
nugetToolType = "nuget"
globalPackagesNotFoundErrorMessage = "could not find global packages path at:"
)

// BuildDependencyTree generates a temporary duplicate of the project to execute the 'install' command without impacting the original directory and establishing the JFrog configuration file for Artifactory resolution
// Additionally, re-loads the project's Solution so the dependencies sources will be identified
func BuildDependencyTree(params utils.AuditParams) (dependencyTree []*xrayUtils.GraphNode, uniqueDeps []string, err error) {
wd, err := os.Getwd()
if err != nil {
Expand All @@ -50,9 +55,25 @@ func BuildDependencyTree(params utils.AuditParams) (dependencyTree []*xrayUtils.
return
}

// Creating a temporary copy of the project in order to run 'install' command without effecting the original directory + creating the jfrog config for artifactory resolution
tmpWd, err := fileutils.CreateTempDir()
if err != nil {
err = fmt.Errorf("failed to create a temporary dir: %w", err)
return
}
defer func() {
err = errors.Join(err, fileutils.RemoveTempDir(tmpWd))
}()

err = biutils.CopyDir(wd, tmpWd, true, nil)
if err != nil {
err = fmt.Errorf("failed copying project to temp dir: %w", err)
return
}

if isInstallRequired(params, sol) {
log.Info("Dependencies sources were not detected nor 'install' command provided. Running 'restore' command")
sol, err = runDotnetRestoreAndLoadSolution(params, wd, exclusionPattern)
sol, err = runDotnetRestoreAndLoadSolution(params, tmpWd, exclusionPattern)
if err != nil {
return
}
Expand All @@ -72,32 +93,14 @@ func isInstallRequired(params utils.AuditParams, sol solution.Solution) bool {
// Additionally, if dependency sources were not identified during the construction of the Solution struct, the project will necessitate an 'install'
solDependencySourcesExists := len(sol.GetDependenciesSources()) > 0
solProjectsExists := len(sol.GetProjects()) > 0
return len(params.InstallCommandArgs()) > 0 || !solDependencySourcesExists || !solProjectsExists
return len(params.InstallCommandArgs()) > 0 || !solDependencySourcesExists || !solProjectsExists || params.IsCurationCmd()
}

// Generates a temporary duplicate of the project to execute the 'install' command without impacting the original directory and establishing the JFrog configuration file for Artifactory resolution
// Additionally, re-loads the project's Solution so the dependencies sources will be identified
func runDotnetRestoreAndLoadSolution(params utils.AuditParams, originalWd, exclusionPattern string) (sol solution.Solution, err error) {
// Creating a temporary copy of the project in order to run 'install' command without effecting the original directory + creating the jfrog config for artifactory resolution
tmpWd, err := fileutils.CreateTempDir()
if err != nil {
err = fmt.Errorf("failed to create a temporary dir: %w", err)
return
}
defer func() {
err = errors.Join(err, fileutils.RemoveTempDir(tmpWd))
}()

err = biutils.CopyDir(originalWd, tmpWd, true, nil)
if err != nil {
err = fmt.Errorf("failed copying project to temp dir: %w", err)
return
}

func runDotnetRestoreAndLoadSolution(params utils.AuditParams, tmpWd, exclusionPattern string) (sol solution.Solution, err error) {
toolName := params.InstallCommandName()
if toolName == "" {
// Determine if the project is a NuGet or .NET project
toolName, err = getProjectToolName(originalWd)
toolName, err = getProjectToolName(tmpWd)
if err != nil {
err = fmt.Errorf("failed while checking for the porject's tool type: %s", err.Error())
return
Expand All @@ -112,6 +115,11 @@ func runDotnetRestoreAndLoadSolution(params utils.AuditParams, originalWd, exclu
if depsRepo != "" {
var serverDetails *config.ServerDetails
serverDetails, err = params.ServerDetails()

// Use the pass-through URL if the project is being restored as part of Curation Audit
if params.IsCurationCmd() {
serverDetails.ArtifactoryUrl += "api/curation/audit"
}
if err != nil {
err = fmt.Errorf("failed to get server details: %s", err.Error())
return
Expand Down Expand Up @@ -195,6 +203,42 @@ func getProjectConfigurationFilesPaths(wd string) (projectConfigFilesPaths []str
return
}

func getEnvVariablesForCurationAudit() ([]string, error) {
curationCache, err := utils.GetCurationNugetCacheFolder()
if err != nil {
return nil, err
}

// Create Curation cache folders to avoid polluting the default cache
if err := os.MkdirAll(filepath.Join(curationCache, "packages"), os.ModePerm); err != nil {
return nil, err
}
if err := os.MkdirAll(filepath.Join(curationCache, "cache"), os.ModePerm); err != nil {
return nil, err
}
if err := os.MkdirAll(filepath.Join(curationCache, "scratch"), os.ModePerm); err != nil {
return nil, err
}
if err := os.MkdirAll(filepath.Join(curationCache, "cache"), os.ModePerm); err != nil {
return nil, err
}

// Configure NuGet to use the Curation cache folders
if err := os.Setenv("NUGET_PACKAGES", filepath.Join(curationCache, "packages")); err != nil {
return nil, err
}
if err := os.Setenv("NUGET_SCRATCH", filepath.Join(curationCache, "scratch")); err != nil {
return nil, err
}
if err := os.Setenv("NUGET_PLUGINS_CACHE", filepath.Join(curationCache, "plugins")); err != nil {
return nil, err
}
if err := os.Setenv("NUGET_HTTP_CACHE", filepath.Join(curationCache, "cache")); err != nil {
return nil, err
}
return os.Environ(), nil
}

func runDotnetRestore(wd string, params utils.AuditParams, toolType bidotnet.ToolchainType, commandExtraArgs []string) (err error) {
var completeCommandArgs []string
if len(params.InstallCommandArgs()) > 0 {
Expand All @@ -209,6 +253,18 @@ func runDotnetRestore(wd string, params utils.AuditParams, toolType bidotnet.Too
completeCommandArgs = append(completeCommandArgs, commandExtraArgs...)
command := exec.Command(completeCommandArgs[0], completeCommandArgs[1:]...)
command.Dir = wd
if params.IsCurationCmd() {
command.Env, err = getEnvVariablesForCurationAudit()
if err != nil {
return err
}

// Specify a custom output directory to force NuGet to rebuild all the dependencies
if toolType.String() == nugetToolType {
command.Args = append(command.Args, "-OutputDirectory", "cur_output")
}
}
log.Info(command.String())
output, err := command.CombinedOutput()
if err != nil {
err = errorutils.CheckErrorf("'dotnet restore' command failed: %s - %s", err.Error(), output)
Expand Down
72 changes: 59 additions & 13 deletions commands/curation/curationaudit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,26 +16,32 @@ import (

"github.com/jfrog/gofrog/datastructures"
"github.com/jfrog/gofrog/parallel"

"github.com/jfrog/jfrog-client-go/artifactory"
"github.com/jfrog/jfrog-client-go/auth"
clientutils "github.com/jfrog/jfrog-client-go/utils"
"github.com/jfrog/jfrog-client-go/utils/errorutils"
"github.com/jfrog/jfrog-client-go/utils/io/httputils"
"github.com/jfrog/jfrog-client-go/utils/log"
xrayClient "github.com/jfrog/jfrog-client-go/xray"
xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"

"github.com/jfrog/build-info-go/build/utils/dotnet/dependencies"

rtUtils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
"github.com/jfrog/jfrog-cli-core/v2/common/cliutils"
outFormat "github.com/jfrog/jfrog-cli-core/v2/common/format"
"github.com/jfrog/jfrog-cli-core/v2/common/project"

"github.com/jfrog/jfrog-cli-core/v2/utils/config"
"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"

"github.com/jfrog/jfrog-cli-security/commands/audit"
"github.com/jfrog/jfrog-cli-security/commands/audit/sca/python"
"github.com/jfrog/jfrog-cli-security/formats"
"github.com/jfrog/jfrog-cli-security/utils"
"github.com/jfrog/jfrog-cli-security/utils/techutils"
"github.com/jfrog/jfrog-cli-security/utils/xray"
"github.com/jfrog/jfrog-client-go/artifactory"
"github.com/jfrog/jfrog-client-go/auth"
clientutils "github.com/jfrog/jfrog-client-go/utils"
"github.com/jfrog/jfrog-client-go/utils/errorutils"
"github.com/jfrog/jfrog-client-go/utils/io/httputils"
"github.com/jfrog/jfrog-client-go/utils/log"
xrayClient "github.com/jfrog/jfrog-client-go/xray"
xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
)

const (
Expand All @@ -61,6 +67,7 @@ const (

MinArtiPassThroughSupport = "7.82.0"
MinArtiGolangSupport = "7.87.0"
MinArtiNuGetSupport = "7.93.0"
MinXrayPassTHroughSupport = "3.92.0"
)

Expand All @@ -69,18 +76,21 @@ var CurationOutputFormats = []string{string(outFormat.Table), string(outFormat.J
var supportedTech = map[techutils.Technology]func(ca *CurationAuditCommand) (bool, error){
techutils.Npm: func(ca *CurationAuditCommand) (bool, error) { return true, nil },
techutils.Pip: func(ca *CurationAuditCommand) (bool, error) {
return ca.checkSupportByVersionOrEnv(techutils.Pip, utils.CurationSupportFlag, MinArtiPassThroughSupport)
return ca.checkSupportByVersionOrEnv(techutils.Pip, MinArtiPassThroughSupport)
},
techutils.Maven: func(ca *CurationAuditCommand) (bool, error) {
return ca.checkSupportByVersionOrEnv(techutils.Maven, utils.CurationSupportFlag, MinArtiPassThroughSupport)
return ca.checkSupportByVersionOrEnv(techutils.Maven, MinArtiPassThroughSupport)
},
techutils.Go: func(ca *CurationAuditCommand) (bool, error) {
return ca.checkSupportByVersionOrEnv(techutils.Go, utils.CurationSupportFlag, MinArtiGolangSupport)
return ca.checkSupportByVersionOrEnv(techutils.Go, MinArtiGolangSupport)
},
techutils.Nuget: func(ca *CurationAuditCommand) (bool, error) {
return ca.checkSupportByVersionOrEnv(techutils.Nuget, MinArtiNuGetSupport)
},
}

func (ca *CurationAuditCommand) checkSupportByVersionOrEnv(tech techutils.Technology, envName string, minArtiVersion string) (bool, error) {
if flag, err := clientutils.GetBoolEnvValue(envName, false); flag {
func (ca *CurationAuditCommand) checkSupportByVersionOrEnv(tech techutils.Technology, minArtiVersion string) (bool, error) {
if flag, err := clientutils.GetBoolEnvValue(utils.CurationSupportFlag, false); flag {
return true, nil
} else if err != nil {
log.Error(err)
Expand Down Expand Up @@ -611,6 +621,10 @@ func (nc *treeAnalyzer) fetchNodeStatus(node xrayUtils.GraphNode, p *sync.Map) e
return err
}
}
// Due to CreateAlternativeVersionForms, it's expected that for NuGet some of the URLs will be missing
if resp.StatusCode == http.StatusNotFound && nc.tech == techutils.Nuget {
continue
}
if resp != nil && resp.StatusCode >= 400 && resp.StatusCode != http.StatusForbidden {
return errorutils.CheckErrorf(errorTemplateHeadRequest, packageUrl, name, version, resp.StatusCode, err)
}
Expand All @@ -623,6 +637,12 @@ func (nc *treeAnalyzer) fetchNodeStatus(node xrayUtils.GraphNode, p *sync.Map) e
p.Store(pkStatus.BlockedPackageUrl, pkStatus)
}
}
if nc.tech == techutils.Nuget {
// DotNet can have multiple URLs only due to CreateAlternativeVersionForms.
// Once the matching version was found, we can stop iterating.
// See CreateAlternativeVersionForms for more details.
return nil
}
}
return nil
}
Expand Down Expand Up @@ -712,6 +732,9 @@ func getUrlNameAndVersionByTech(tech techutils.Technology, node *xrayUtils.Graph
return
case techutils.Go:
return getGoNameScopeAndVersion(node.Id, artiUrl, repo)
case techutils.Nuget:
downloadUrls, name, version = getNugetNameScopeAndVersion(node.Id, artiUrl, repo)
return
}
return
}
Expand All @@ -733,6 +756,29 @@ func getPythonNameVersion(id string, downloadUrlsMap map[string]string) (downloa
return
}

func toNugetDownloadUrl(artifactoryUrl, repo, compName, compVersion string) string {
return fmt.Sprintf("%s/api/nuget/v3/%s/registration-semver2/Download/%s/%s",
strings.TrimSuffix(artifactoryUrl, "/"),
repo,
strings.ToLower(compName),
compVersion,
)
}

// input- id: gav://org.apache.tomcat.embed:tomcat-embed-jasper:8.0.33
// input - repo: libs-release
// output - downloadUrl: <arti-url>/libs-release/org/apache/tomcat/embed/tomcat-embed-jasper/8.0.33/tomcat-embed-jasper-8.0.33.jar
func getNugetNameScopeAndVersion(id, artiUrl, repo string) (downloadUrls []string, name, version string) {
name, version, _ = utils.SplitComponentId(id)

downloadUrls = append(downloadUrls, toNugetDownloadUrl(artiUrl, repo, name, version))
for _, versionVariant := range dependencies.CreateAlternativeVersionForms(version) {
downloadUrls = append(downloadUrls, toNugetDownloadUrl(artiUrl, repo, name, versionVariant))
}

return downloadUrls, name, version
}

// input - id: go://github.com/kennygrant/sanitize:v1.2.4
// input - repo: go
// output: downloadUrl: <artiUrl>/api/go/go/github.com/kennygrant/sanitize/@v/v1.2.4.zip
Expand Down
Loading

0 comments on commit 2243ace

Please sign in to comment.