Merge pull request #2064 from puerco/sbom-spdx

BOM: Feature complete SPDX and license packages

pkg/license/catalog.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package license
+
+import (
+	"path/filepath"
+	"sync"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// CatalogOptions are the spdx settings
+type CatalogOptions struct {
+	CacheDir string // Directrory to catch the license we download from SPDX.org
+}
+
+// DefaultCatalogOpts are the predetermined settings. License and cache directories
+// are in the temporary OS directory and are created if the do not exist
+var DefaultCatalogOpts = &CatalogOptions{}
+
+// NewSPDXWithOptions returns a SPDX object with the specified options
+func NewCatalogWithOptions(opts *CatalogOptions) (catalog *Catalog, err error) {
+	// Create the license downloader
+	doptions := DefaultDownloaderOpts
+	doptions.CacheDir = opts.CacheDir
+	downloader, err := NewDownloaderWithOptions(doptions)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating downloader")
+	}
+	catalog = &Catalog{
+		Downloader: downloader,
+		opts:       opts,
+	}
+
+	return catalog, nil
+}
+
+// Options returns  a pointer to the catlog options
+func (catalog *Catalog) Options() *CatalogOptions {
+	return catalog.opts
+}
+
+// LoadLicenses reads the license data from the downloader
+func (catalog *Catalog) LoadLicenses() error {
+	logrus.Info("Loading license data from downloader")
+	licenses, err := catalog.Downloader.GetLicenses()
+	if err != nil {
+		return errors.Wrap(err, "getting licenses from downloader")
+	}
+	catalog.List = licenses
+	logrus.Infof("Got %d licenses from downloader", len(licenses.Licenses))
+	return nil
+}
+
+// Catalog is an objec to interact with licenses and manifest creation
+type Catalog struct {
+	Downloader *Downloader     // License Downloader
+	List       *List           // List of licenses
+	opts       *CatalogOptions // SPDX Options
+}
+
+// WriteLicensesAsText writes the SPDX license collection to text files
+func (catalog *Catalog) WriteLicensesAsText(targetDir string) error {
+	logrus.Info("Writing SPDX licenses to " + targetDir)
+	if catalog.List.Licenses == nil {
+		return errors.New("unable to write licenses, they have not been loaded yet")
+	}
+	wg := sync.WaitGroup{}
+	var err error
+	for _, l := range catalog.List.Licenses {
+		wg.Add(1)
+		go func(l *License) {
+			defer wg.Done()
+			if lerr := l.WriteText(filepath.Join(targetDir, l.LicenseID+".txt")); err != nil {
+				if err == nil {
+					err = lerr
+				} else {
+					err = errors.Wrap(err, lerr.Error())
+				}
+			}
+		}(l)
+	}
+	wg.Wait()
+	return errors.Wrap(err, "caught errors while writing license files")
+}
+
+// GetLicense returns a license struct from its SPDX ID label
+func (catalog *Catalog) GetLicense(label string) *License {
+	if lic, ok := catalog.List.Licenses[label]; ok {
+		return lic
+	}
+	logrus.Warn("Label %s is not an identifier of a known license " + label)
+	return nil
+}

pkg/license/download.go

@@ -20,19 +20,22 @@ import (
 	"crypto/sha1"
 	"encoding/json"
 	"fmt"
-	"io"
-	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/nozzle/throttler"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"sigs.k8s.io/release-utils/http"
 	"sigs.k8s.io/release-utils/util"
 )
 
 // ListURL is the json list of all spdx licenses
-const ListURL = "https://spdx.org/licenses/licenses.json"
+const (
+	LicenseDataURL      = "https://spdx.org/licenses/"
+	LicenseListFilename = "licenses.json"
+)
 
 // NewDownloader returns a downloader with the default options
 func NewDownloader() (*Downloader, error) {
@@ -72,7 +75,7 @@ func (do *DownloaderOptions) Validate() error {
 		}
 		// And no cache dir was specified
 		if do.CacheDir == "" {
-			dir, err := os.MkdirTemp("", "license-cache-")
+			dir, err := os.MkdirTemp(os.TempDir(), "license-cache-")
 			if err != nil {
 				return errors.Wrap(err, "creating temporary directory")
 			}
@@ -94,15 +97,15 @@ func (d *Downloader) SetImplementation(di DownloaderImplementation) {
 
 // GetLicenses is the mina function of the downloader. Returns a license list
 // or an error if could get them
-func (d *Downloader) GetLicenses() (*SPDXLicenseList, error) {
+func (d *Downloader) GetLicenses() (*List, error) {
 	return d.impl.GetLicenses()
 }
 
 //counterfeiter:generate . DownloaderImplementation
 
 // DownloaderImplementation has only one method
 type DownloaderImplementation interface {
-	GetLicenses() (*SPDXLicenseList, error)
+	GetLicenses() (*List, error)
 	SetOptions(*DownloaderOptions)
 }
 
@@ -124,23 +127,17 @@ func (ddi *DefaultDownloaderImpl) SetOptions(opts *DownloaderOptions) {
 }
 
 // GetLicenses downloads the main json file listing all SPDX supported licenses
-func (ddi *DefaultDownloaderImpl) GetLicenses() (licenses *SPDXLicenseList, err error) {
+func (ddi *DefaultDownloaderImpl) GetLicenses() (licenses *List, err error) {
 	// TODO: Cache licenselist
-	logrus.Info("Downloading main SPDX license data")
+	logrus.Info("Downloading main SPDX license data from " + LicenseDataURL)
 
 	// Get the list of licenses
-	resp, err := http.Get(ListURL)
+	licensesJSON, err := http.NewAgent().Get(LicenseDataURL + LicenseListFilename)
 	if err != nil {
 		return nil, errors.Wrap(err, "fetching licenses list")
 	}
-	defer resp.Body.Close()
 
-	licensesJSON, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, errors.Wrap(err, "reading license list response body")
-	}
-
-	licenseList := &SPDXLicenseList{}
+	licenseList := &List{}
 	if err := json.Unmarshal(licensesJSON, licenseList); err != nil {
 		return nil, errors.Wrap(err, "parsing SPDX licence list")
 	}
@@ -150,6 +147,11 @@ func (ddi *DefaultDownloaderImpl) GetLicenses() (licenses *SPDXLicenseList, err
 	// Create a new Throttler that will get `parallelDownloads` urls at a time
 	t := throttler.New(ddi.Options.parallelDownloads, len(licenseList.LicenseData))
 	for _, l := range licenseList.LicenseData {
+		licURL := l.Reference
+		// If the license URLs have a local reference
+		if strings.HasPrefix(licURL, "./") {
+			licURL = LicenseDataURL + strings.TrimPrefix(licURL, "./")
+		}
 		// Launch a goroutine to fetch the URL.
 		go func(url string) {
 			var err error
@@ -158,8 +160,9 @@ func (ddi *DefaultDownloaderImpl) GetLicenses() (licenses *SPDXLicenseList, err
 			if err != nil {
 				return
 			}
+			logrus.Debugf("Got license: %s from %s", l.LicenseID, url)
 			licenseList.Add(l)
-		}(l.DetailsURL)
+		}(licURL)
 		t.Throttle()
 	}
 
@@ -214,7 +217,7 @@ func (ddi *DefaultDownloaderImpl) getCachedData(url string) ([]byte, error) {
 }
 
 // getLicenseFromURL downloads a license in json and returns it parsed into a struct
-func (ddi *DefaultDownloaderImpl) getLicenseFromURL(url string) (license *SPDXLicense, err error) {
+func (ddi *DefaultDownloaderImpl) getLicenseFromURL(url string) (license *License, err error) {
 	licenseJSON := []byte{}
 	// Determine the cache file name
 	if ddi.Options.EnableCache {
@@ -230,15 +233,10 @@ func (ddi *DefaultDownloaderImpl) getLicenseFromURL(url string) (license *SPDXLi
 	// If we still don't have json data, download it
 	if len(licenseJSON) == 0 {
 		logrus.Infof("Downloading license data from %s", url)
-		resp, err := http.Get(url)
+		licenseJSON, err = http.NewAgent().Get(url)
 		if err != nil {
 			return nil, errors.Wrapf(err, "getting %s", url)
 		}
-		defer resp.Body.Close()
-		licenseJSON, err = io.ReadAll(resp.Body)
-		if err != nil {
-			return nil, errors.Wrap(err, "reading response body")
-		}
 
 		logrus.Infof("Downloaded %d bytes from %s", len(licenseJSON), url)
 
@@ -249,5 +247,10 @@ func (ddi *DefaultDownloaderImpl) getLicenseFromURL(url string) (license *SPDXLi
 		}
 	}
 
-	return ParseSPDXLicense(licenseJSON)
+	// Parse the SPDX license from the JSON data
+	l, err := ParseLicense(licenseJSON)
+	if err != nil {
+		return nil, errors.Wrap(err, "parsing license json data")
+	}
+	return l, err
 }