Skip to content

Commit

Permalink
feat: add SPDX support to ecosystems enrich
Browse files Browse the repository at this point in the history
  • Loading branch information
mcombuechen authored and garethr committed Jun 29, 2023
1 parent f543a3b commit 52a3757
Show file tree
Hide file tree
Showing 5 changed files with 225 additions and 41 deletions.
3 changes: 3 additions & 0 deletions lib/ecosystems/enrich.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package ecosystems

import (
cdx "github.com/CycloneDX/cyclonedx-go"
"github.com/spdx/tools-golang/spdx"

"github.com/snyk/parlay/lib/sbom"
)
Expand All @@ -26,6 +27,8 @@ func EnrichSBOM(doc *sbom.SBOMDocument) *sbom.SBOMDocument {
switch bom := doc.BOM.(type) {
case *cdx.BOM:
enrichCDX(bom)
case *spdx.Document:
enrichSPDX(bom)
}
return doc
}
72 changes: 44 additions & 28 deletions lib/ecosystems/enrich_cyclonedx.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
/*
* © 2023 Snyk Limited All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package ecosystems

import (
Expand All @@ -13,29 +29,29 @@ import (
type cdxEnricher = func(cdx.Component, packages.Package) cdx.Component

var cdxEnrichers = []cdxEnricher{
enrichDescription,
enrichLicense,
enrichHomepage,
enrichRegistryURL,
enrichRepositoryURL,
enrichDocumentationURL,
enrichFirstReleasePublishedAt,
enrichLatestReleasePublishedAt,
enrichRepoArchived,
enrichLocation,
enrichTopics,
enrichAuthor,
enrichSupplier,
}

func enrichDescription(component cdx.Component, packageData packages.Package) cdx.Component {
enrichCDXDescription,
enrichCDXLicense,
enrichCDXHomepage,
enrichCDXRegistryURL,
enrichCDXRepositoryURL,
enrichCDXDocumentationURL,
enrichCDXFirstReleasePublishedAt,
enrichCDXLatestReleasePublishedAt,
enrichCDXRepoArchived,
enrichCDXLocation,
enrichCDXTopics,
enrichCDXAuthor,
enrichCDXSupplier,
}

func enrichCDXDescription(component cdx.Component, packageData packages.Package) cdx.Component {
if packageData.Description != nil {
component.Description = *packageData.Description
}
return component
}

func enrichLicense(component cdx.Component, packageData packages.Package) cdx.Component {
func enrichCDXLicense(component cdx.Component, packageData packages.Package) cdx.Component {
if packageData.NormalizedLicenses != nil {
if len(packageData.NormalizedLicenses) > 0 {
expression := packageData.NormalizedLicenses[0]
Expand Down Expand Up @@ -75,39 +91,39 @@ func enrichProperty(component cdx.Component, name string, value string) cdx.Comp
return component
}

func enrichHomepage(component cdx.Component, packageData packages.Package) cdx.Component {
func enrichCDXHomepage(component cdx.Component, packageData packages.Package) cdx.Component {
return enrichExternalReference(component, packageData, packageData.Homepage, cdx.ERTypeWebsite)
}

func enrichRegistryURL(component cdx.Component, packageData packages.Package) cdx.Component {
func enrichCDXRegistryURL(component cdx.Component, packageData packages.Package) cdx.Component {
return enrichExternalReference(component, packageData, packageData.RegistryUrl, cdx.ERTypeDistribution)
}

func enrichRepositoryURL(component cdx.Component, packageData packages.Package) cdx.Component {
func enrichCDXRepositoryURL(component cdx.Component, packageData packages.Package) cdx.Component {
return enrichExternalReference(component, packageData, packageData.RepositoryUrl, cdx.ERTypeVCS)
}

func enrichDocumentationURL(component cdx.Component, packageData packages.Package) cdx.Component {
func enrichCDXDocumentationURL(component cdx.Component, packageData packages.Package) cdx.Component {
return enrichExternalReference(component, packageData, packageData.DocumentationUrl, cdx.ERTypeDocumentation)
}

func enrichFirstReleasePublishedAt(component cdx.Component, packageData packages.Package) cdx.Component {
func enrichCDXFirstReleasePublishedAt(component cdx.Component, packageData packages.Package) cdx.Component {
if packageData.FirstReleasePublishedAt == nil {
return component
}
timestamp := packageData.FirstReleasePublishedAt.UTC().Format(time.RFC3339)
return enrichProperty(component, "ecosystems:first_release_published_at", timestamp)
}

func enrichLatestReleasePublishedAt(component cdx.Component, packageData packages.Package) cdx.Component {
func enrichCDXLatestReleasePublishedAt(component cdx.Component, packageData packages.Package) cdx.Component {
if packageData.LatestReleasePublishedAt == nil {
return component
}
timestamp := packageData.LatestReleasePublishedAt.UTC().Format(time.RFC3339)
return enrichProperty(component, "ecosystems:latest_release_published_at", timestamp)
}

func enrichRepoArchived(component cdx.Component, packageData packages.Package) cdx.Component {
func enrichCDXRepoArchived(component cdx.Component, packageData packages.Package) cdx.Component {
if packageData.RepoMetadata != nil {
if archived, ok := (*packageData.RepoMetadata)["archived"].(bool); ok && archived {
return enrichProperty(component, "ecosystems:repository_archived", "true")
Expand All @@ -116,7 +132,7 @@ func enrichRepoArchived(component cdx.Component, packageData packages.Package) c
return component
}

func enrichLocation(component cdx.Component, packageData packages.Package) cdx.Component {
func enrichCDXLocation(component cdx.Component, packageData packages.Package) cdx.Component {
if packageData.RepoMetadata != nil {
meta := *packageData.RepoMetadata
if ownerRecord, ok := meta["owner_record"].(map[string]interface{}); ok {
Expand All @@ -128,7 +144,7 @@ func enrichLocation(component cdx.Component, packageData packages.Package) cdx.C
return component
}

func enrichAuthor(component cdx.Component, packageData packages.Package) cdx.Component {
func enrichCDXAuthor(component cdx.Component, packageData packages.Package) cdx.Component {
if packageData.RepoMetadata != nil {
meta := *packageData.RepoMetadata
if ownerRecord, ok := meta["owner_record"].(map[string]interface{}); ok {
Expand All @@ -141,7 +157,7 @@ func enrichAuthor(component cdx.Component, packageData packages.Package) cdx.Com
return component
}

func enrichSupplier(component cdx.Component, packageData packages.Package) cdx.Component {
func enrichCDXSupplier(component cdx.Component, packageData packages.Package) cdx.Component {
if packageData.RepoMetadata != nil {
meta := *packageData.RepoMetadata
if ownerRecord, ok := meta["owner_record"].(map[string]interface{}); ok {
Expand All @@ -161,7 +177,7 @@ func enrichSupplier(component cdx.Component, packageData packages.Package) cdx.C
return component
}

func enrichTopics(component cdx.Component, packageData packages.Package) cdx.Component {
func enrichCDXTopics(component cdx.Component, packageData packages.Package) cdx.Component {
if packageData.RepoMetadata != nil {
meta := *packageData.RepoMetadata

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import (
"github.com/snyk/parlay/lib/sbom"
)

func TestEnrichSBOM(t *testing.T) {
func TestEnrichSBOM_CycloneDX(t *testing.T) {
httpmock.Activate()
defer httpmock.DeactivateAndReset()

Expand Down Expand Up @@ -118,7 +118,7 @@ func TestEnrichDescription(t *testing.T) {
pack := packages.Package{
Description: &desc,
}
component = enrichDescription(component, pack)
component = enrichCDXDescription(component, pack)
assert.Equal(t, "description", component.Description)
}

Expand All @@ -131,7 +131,7 @@ func TestEnrichLicense(t *testing.T) {
pack := packages.Package{
NormalizedLicenses: []string{"BSD-3-Clause"},
}
component = enrichLicense(component, pack)
component = enrichCDXLicense(component, pack)
licenses := *component.Licenses

comp := cdx.LicenseChoice(cdx.LicenseChoice{Expression: "BSD-3-Clause"})
Expand Down Expand Up @@ -172,7 +172,7 @@ func TestEnrichHomepageWithNilHomepage(t *testing.T) {
component := cdx.Component{}
packageData := packages.Package{Homepage: nil}

result := enrichHomepage(component, packageData)
result := enrichCDXHomepage(component, packageData)

assert.Equal(t, component, result)
}
Expand All @@ -181,7 +181,7 @@ func TestEnrichHomepageWithNonNullHomepage(t *testing.T) {
component := cdx.Component{}
packageData := packages.Package{Homepage: pointerToString("https://example.com")}

result := enrichHomepage(component, packageData)
result := enrichCDXHomepage(component, packageData)

expected := cdx.Component{
ExternalReferences: &[]cdx.ExternalReference{
Expand All @@ -195,7 +195,7 @@ func TestEnrichRegistryURLWithNilRegistryURL(t *testing.T) {
component := cdx.Component{}
packageData := packages.Package{RegistryUrl: nil}

result := enrichRegistryURL(component, packageData)
result := enrichCDXRegistryURL(component, packageData)

assert.Equal(t, component, result)
}
Expand All @@ -204,7 +204,7 @@ func TestEnrichRegistryURLWithNonNullRegistryURL(t *testing.T) {
component := cdx.Component{}
packageData := packages.Package{RegistryUrl: pointerToString("https://example.com")}

result := enrichRegistryURL(component, packageData)
result := enrichCDXRegistryURL(component, packageData)

expected := cdx.Component{
ExternalReferences: &[]cdx.ExternalReference{
Expand All @@ -224,13 +224,13 @@ func TestEnrichLatestReleasePublishedAt(t *testing.T) {
LatestReleasePublishedAt: nil,
}

result := enrichLatestReleasePublishedAt(component, packageData)
result := enrichCDXLatestReleasePublishedAt(component, packageData)
assert.Equal(t, component, result)

latestReleasePublishedAt := time.Date(2023, time.May, 1, 0, 0, 0, 0, time.UTC)
packageData.LatestReleasePublishedAt = &latestReleasePublishedAt
expectedTimestamp := latestReleasePublishedAt.UTC().Format(time.RFC3339)
result = enrichLatestReleasePublishedAt(component, packageData)
result = enrichCDXLatestReleasePublishedAt(component, packageData)

prop := (*result.Properties)[0]
assert.Equal(t, "ecosystems:latest_release_published_at", prop.Name)
Expand All @@ -243,15 +243,15 @@ func TestEnrichLocation(t *testing.T) {
// Test case 1: packageData.RepoMetadata is nil
component := cdx.Component{Name: "test"}
packageData := packages.Package{}
result := enrichLocation(component, packageData)
result := enrichCDXLocation(component, packageData)
assert.Equal(component, result)

// Test case 2: packageData.RepoMetadata is not nil, but "owner_record" is missing
component = cdx.Component{Name: "test"}
packageData = packages.Package{RepoMetadata: &map[string]interface{}{
"not_owner_record": map[string]interface{}{},
}}
result = enrichLocation(component, packageData)
result = enrichCDXLocation(component, packageData)
assert.Equal(component, result)

// Test case 3: "location" field is missing in "owner_record"
Expand All @@ -261,7 +261,7 @@ func TestEnrichLocation(t *testing.T) {
"not_location": "test",
},
}}
result = enrichLocation(component, packageData)
result = enrichCDXLocation(component, packageData)
assert.Equal(component, result)

// Test case 4: "location" field is present in "owner_record"
Expand All @@ -277,6 +277,6 @@ func TestEnrichLocation(t *testing.T) {
{Name: "ecosystems:owner_location", Value: "test_location"},
},
}
result = enrichLocation(component, packageData)
result = enrichCDXLocation(component, packageData)
assert.Equal(expectedComponent, result)
}
90 changes: 90 additions & 0 deletions lib/ecosystems/enrich_spdx.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* © 2023 Snyk Limited All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package ecosystems

import (
"errors"
"fmt"
"strings"

"github.com/package-url/packageurl-go"
"github.com/spdx/tools-golang/spdx"
"github.com/spdx/tools-golang/spdx/v2/v2_3"

"github.com/snyk/parlay/ecosystems/packages"
)

func enrichSPDX(bom *spdx.Document) {
packages := bom.Packages

for _, pkg := range packages {
purl, err := extractPurl(pkg)
if err != nil {
continue
}

resp, err := GetPackageData(*purl)
if err != nil {
continue
}

pkgData := resp.JSON200
if pkgData == nil {
continue
}

enrichSPDXDescription(pkg, pkgData)
enrichSPDXLicense(pkg, pkgData)
enrichSPDXHomepage(pkg, pkgData)
}
}

func extractPurl(pkg *v2_3.Package) (*packageurl.PackageURL, error) {
for _, ref := range pkg.PackageExternalReferences {
if ref.RefType != "purl" {
continue
}
purl, err := packageurl.FromString(ref.Locator)
if err != nil {
return nil, err
}
return &purl, nil
}
return nil, errors.New("no purl found on SPDX package")
}

func enrichSPDXLicense(pkg *v2_3.Package, data *packages.Package) {
if len(data.NormalizedLicenses) == 1 {
pkg.PackageLicenseConcluded = data.NormalizedLicenses[0]
} else if len(data.NormalizedLicenses) > 1 {
pkg.PackageLicenseConcluded = fmt.Sprintf("(%s)", strings.Join(data.NormalizedLicenses, " OR "))
}
}

func enrichSPDXHomepage(pkg *v2_3.Package, data *packages.Package) {
if data.Homepage == nil {
return
}
pkg.PackageHomePage = *data.Homepage
}

func enrichSPDXDescription(pkg *v2_3.Package, data *packages.Package) {
if data.Description == nil {
return
}
pkg.PackageDescription = *data.Description
}
Loading

0 comments on commit 52a3757

Please sign in to comment.