From ce67927a989ee3cf1c82f49c0de09541964ff05a Mon Sep 17 00:00:00 2001 From: William Murphy Date: Wed, 7 Feb 2024 11:26:23 -0500 Subject: [PATCH] Fix: unmarshal key values in Java, Go, and Conan metadata (#2603) Previously, Syft represented several metadata fields as map[string]string, however this representation erased ordering, so Syft now represents these values as []KeyValue. Add custom unmarshaling so that JSON that was written by older versions of Syft using the map[string]string representation can be parsed into the new []KeyValue representation. Signed-off-by: Will Murphy --- syft/file/cataloger/executable/cataloger.go | 5 +- syft/format/syftjson/model/package_test.go | 137 ++++++++++++++++++++ syft/pkg/java.go | 56 ++++++++ syft/pkg/java_test.go | 101 +++++++++++++++ syft/pkg/key_value.go | 42 ++++++ 5 files changed, 340 insertions(+), 1 deletion(-) diff --git a/syft/file/cataloger/executable/cataloger.go b/syft/file/cataloger/executable/cataloger.go index 48872deadc1..b6510e74c5d 100644 --- a/syft/file/cataloger/executable/cataloger.go +++ b/syft/file/cataloger/executable/cataloger.go @@ -6,6 +6,7 @@ import ( "debug/macho" "encoding/binary" "fmt" + "sort" "github.com/bmatcuk/doublestar/v4" "github.com/dustin/go-humanize" @@ -28,8 +29,10 @@ type Cataloger struct { } func DefaultConfig() Config { + m := mimetype.ExecutableMIMETypeSet.List() + sort.Strings(m) return Config{ - MIMETypes: mimetype.ExecutableMIMETypeSet.List(), + MIMETypes: m, Globs: nil, } } diff --git a/syft/format/syftjson/model/package_test.go b/syft/format/syftjson/model/package_test.go index 3c67041d08c..778b7b237ed 100644 --- a/syft/format/syftjson/model/package_test.go +++ b/syft/format/syftjson/model/package_test.go @@ -386,6 +386,143 @@ func Test_UnmarshalJSON(t *testing.T) { assert.Equal(t, reflect.TypeOf(pkg.RustBinaryAuditEntry{}).Name(), reflect.TypeOf(p.Metadata).Name()) }, }, + { + name: "map-based java metadata", + packageData: []byte(`{ + "id": "e6f845bdaa69ddb2", + "name": "SparseBitSet", + "version": "1.2", + "type": "java-archive", + "foundBy": "java-archive-cataloger", + "locations": [], + "licenses": [], + "language": "java", + "cpes": [], + "purl": "pkg:maven/com.zaxxer/SparseBitSet@1.2", + "metadataType": "java-archive", + "metadata": { + "virtualPath": "/opt/solr-9.4.1/modules/extraction/lib/SparseBitSet-1.2.jar", + "manifest": { + "main": { + "Archiver-Version": "Plexus Archiver", + "Build-Jdk": "1.8.0_73", + "Built-By": "lbayer", + "Created-By": "Apache Maven 3.5.0", + "Manifest-Version": "1.0" + }, + "namedSections": { + "META-INF/mailcap": { + "SHA-256-Digest": "kXN4VupOQOJhduMGwxumj4ijmD/YAlz97a9Mp7CVXtk=" + }, + "META-INF/versions/9/module-info.class": { + "SHA-256-Digest": "cMeIRa5l8DWPgrVWavr/6TKVBUGixVKGcu6yOTZMlKk=" + } + } + } + } +}`), + assert: func(p *Package) { + meta := p.Metadata.(pkg.JavaArchive) + manifest := meta.Manifest + assert.Equal(t, "1.8.0_73", manifest.Main.MustGet("Build-Jdk")) + require.Equal(t, 2, len(manifest.Sections)) + assert.Equal(t, "META-INF/mailcap", manifest.Sections[0].MustGet("Name")) + assert.Equal(t, "kXN4VupOQOJhduMGwxumj4ijmD/YAlz97a9Mp7CVXtk=", manifest.Sections[0].MustGet("SHA-256-Digest")) + assert.Equal(t, "META-INF/versions/9/module-info.class", manifest.Sections[1].MustGet("Name")) + assert.Equal(t, "cMeIRa5l8DWPgrVWavr/6TKVBUGixVKGcu6yOTZMlKk=", manifest.Sections[1].MustGet("SHA-256-Digest")) + }, + }, + { + name: "pre key-value golang metadata", + packageData: []byte(`{ + "id": "e348ed25484a94c9", + "name": "github.com/anchore/syft", + "version": "v0.101.1-SNAPSHOT-4c777834", + "type": "go-module", + "foundBy": "go-module-binary-cataloger", + "locations": [ + { + "path": "/syft", + "layerID": "sha256:2274947a5f3527e48d8725a96646aefdcce3d99340c1eefb1e7c894043863c92", + "accessPath": "/syft", + "annotations": { + "evidence": "primary" + } + } + ], + "licenses": [], + "language": "go", + "cpes": [ + "cpe:2.3:a:anchore:syft:v0.101.1-SNAPSHOT-4c777834:*:*:*:*:*:*:*" + ], + "purl": "pkg:golang/github.com/anchore/syft@v0.101.1-SNAPSHOT-4c777834", + "metadataType": "go-module-buildinfo-entry", + "metadata": { + "goBuildSettings": { + "-buildmode": "exe", + "-compiler": "gc", + "-ldflags": "-w -s -extldflags '-static' -X main.version=0.101.1-SNAPSHOT-4c777834 -X main.gitCommit=4c777834618b2ad8ad94cd200a45d6670bc1c013 -X main.buildDate=2024-01-22T16:43:49Z -X main.gitDescription=v0.101.1-4-g4c777834 ", + "CGO_ENABLED": "0", + "GOAMD64": "v1", + "GOARCH": "amd64", + "GOOS": "linux", + "vcs": "git", + "vcs.modified": "false", + "vcs.revision": "4c777834618b2ad8ad94cd200a45d6670bc1c013", + "vcs.time": "2024-01-22T16:31:41Z" + }, + "goCompiledVersion": "go1.21.2", + "architecture": "amd64", + "mainModule": "github.com/anchore/syft" + } +} +`), + assert: func(p *Package) { + buildInfo := p.Metadata.(pkg.GolangBinaryBuildinfoEntry) + assert.Equal(t, "exe", buildInfo.BuildSettings.MustGet("-buildmode")) + }, + }, + { + name: "conan lock with legacy options", + packageData: []byte(`{ + "id": "75eb35307226c921", + "name": "boost", + "version": "1.75.0", + "type": "conan", + "foundBy": "conan-cataloger", + "locations": [ + { + "path": "/conan.lock", + "accessPath": "/conan.lock", + "annotations": { + "evidence": "primary" + } + } + ], + "licenses": [], + "language": "c++", + "cpes": [ + "cpe:2.3:a:boost:boost:1.75.0:*:*:*:*:*:*:*" + ], + "purl": "pkg:conan/boost@1.75.0", + "metadataType": "c-conan-lock-entry", + "metadata": { + "ref": "boost/1.75.0#a9c318f067216f900900e044e7af4ab1", + "package_id": "dc8aedd23a0f0a773a5fcdcfe1ae3e89c4205978", + "prev": "b9d7912e6131dfa453c725593b36c808", + "options": { + "addr2line_location": "/usr/bin/addr2line", + "asio_no_deprecated": "False", + "zstd": "False" + }, + "context": "host" + } +}`), + assert: func(p *Package) { + metadata := p.Metadata.(pkg.ConanV1LockEntry) + assert.Equal(t, "False", metadata.Options.MustGet("asio_no_deprecated")) + }, + }, } for _, test := range tests { diff --git a/syft/pkg/java.go b/syft/pkg/java.go index 1fb5a3b5b40..2a990b5d497 100644 --- a/syft/pkg/java.go +++ b/syft/pkg/java.go @@ -1,6 +1,9 @@ package pkg import ( + "encoding/json" + "fmt" + "sort" "strings" "github.com/anchore/syft/internal" @@ -70,6 +73,59 @@ type JavaManifest struct { Sections []KeyValues `json:"sections,omitempty"` } +type unmarshalJavaManifest JavaManifest + +type legacyJavaManifest struct { + Main map[string]string `json:"main"` + NamedSections map[string]map[string]string `json:"namedSections"` +} + +func (m *JavaManifest) UnmarshalJSON(b []byte) error { + var either map[string]any + err := json.Unmarshal(b, &either) + if err != nil { + return fmt.Errorf("could not unmarshal java manifest: %w", err) + } + if _, ok := either["namedSections"]; ok { + var lm legacyJavaManifest + if err = json.Unmarshal(b, &lm); err != nil { + return fmt.Errorf("could not unmarshal java manifest: %w", err) + } + *m = lm.toNewManifest() + return nil + } + var jm unmarshalJavaManifest + err = json.Unmarshal(b, &jm) + if err != nil { + return fmt.Errorf("could not unmarshal java manifest: %w", err) + } + *m = JavaManifest(jm) + return nil +} + +func (lm legacyJavaManifest) toNewManifest() JavaManifest { + var result JavaManifest + result.Main = keyValuesFromMap(lm.Main) + var sectionNames []string + for k := range lm.NamedSections { + sectionNames = append(sectionNames, k) + } + sort.Strings(sectionNames) + var sections []KeyValues + for _, name := range sectionNames { + section := KeyValues{ + KeyValue{ + Key: "Name", + Value: name, + }, + } + section = append(section, keyValuesFromMap(lm.NamedSections[name])...) + sections = append(sections, section) + } + result.Sections = sections + return result +} + func (m JavaManifest) Section(name string) KeyValues { for _, section := range m.Sections { if sectionName, ok := section.Get("Name"); ok && sectionName == name { diff --git a/syft/pkg/java_test.go b/syft/pkg/java_test.go index dd8679fef86..e0c4248573d 100644 --- a/syft/pkg/java_test.go +++ b/syft/pkg/java_test.go @@ -3,6 +3,7 @@ package pkg import ( "testing" + "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" ) @@ -109,3 +110,103 @@ func TestPomProperties_PkgTypeIndicated(t *testing.T) { }) } } + +func Test_legacyJavaManifest_toNewManifest(t *testing.T) { + tests := []struct { + name string + lm legacyJavaManifest + want JavaManifest + }{ + { + name: "empty", + lm: legacyJavaManifest{}, + want: JavaManifest{}, + }, + { + name: "main sections are sorted", + lm: legacyJavaManifest{ + Main: map[string]string{ + "a key": "a value", + "b key": "b value", + "c key": "c value", + }, + }, + want: JavaManifest{Main: KeyValues{ + { + Key: "a key", + Value: "a value", + }, + { + Key: "b key", + Value: "b value", + }, + { + Key: "c key", + Value: "c value", + }, + }}, + }, + { + name: "named sections have their name in the result", + lm: legacyJavaManifest{ + NamedSections: map[string]map[string]string{ + "a section": { + "a key": "a value", + "b key": "b value", + "c key": "c value", + }, + "b section": { + "d key": "d value", + "e key": "e value", + "f key": "f value", + }, + }, + }, + want: JavaManifest{Sections: []KeyValues{ + { + { + Key: "Name", + Value: "a section", + }, + { + Key: "a key", + Value: "a value", + }, + { + Key: "b key", + Value: "b value", + }, + { + Key: "c key", + Value: "c value", + }, + }, + { + { + Key: "Name", + Value: "b section", + }, + { + Key: "d key", + Value: "d value", + }, + { + Key: "e key", + Value: "e value", + }, + { + Key: "f key", + Value: "f value", + }, + }, + }}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if diff := cmp.Diff(tt.want, tt.lm.toNewManifest()); diff != "" { + t.Errorf("unexpected diff in manifest (-want +got):\n%s", diff) + } + }) + } +} diff --git a/syft/pkg/key_value.go b/syft/pkg/key_value.go index 97cce0193de..52337be75b9 100644 --- a/syft/pkg/key_value.go +++ b/syft/pkg/key_value.go @@ -1,5 +1,11 @@ package pkg +import ( + "encoding/json" + "fmt" + "sort" +) + type KeyValue struct { Key string `json:"key"` Value string `json:"value"` @@ -26,3 +32,39 @@ func (k KeyValues) MustGet(key string) string { return "" } + +func keyValuesFromMap(m map[string]string) KeyValues { + var result KeyValues + var mapKeys []string + for k := range m { + mapKeys = append(mapKeys, k) + } + sort.Strings(mapKeys) + for _, k := range mapKeys { + result = append(result, KeyValue{ + Key: k, + Value: m[k], + }) + } + return result +} + +func (k *KeyValues) UnmarshalJSON(b []byte) error { + var kvs []KeyValue + if err := json.Unmarshal(b, &kvs); err != nil { + var legacyMap map[string]string + if err := json.Unmarshal(b, &legacyMap); err != nil { + return fmt.Errorf("unable to unmarshal KeyValues: %w", err) + } + var keys []string + for k := range legacyMap { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + kvs = append(kvs, KeyValue{Key: k, Value: legacyMap[k]}) + } + } + *k = kvs + return nil +}