Skip to content

Commit

Permalink
Fix: unmarshal key values in Java, Go, and Conan metadata (#2603)
Browse files Browse the repository at this point in the history 
Previously, Syft represented several metadata fields as map[string]string,
however this representation erased ordering, so Syft now represents these values
as []KeyValue. Add custom unmarshaling so that JSON that was written by
older versions of Syft using the map[string]string representation can be parsed
into the new []KeyValue representation.

Signed-off-by: Will Murphy <will.murphy@anchore.com>
  • Loading branch information
@willmurphyscode
willmurphyscode authored Feb 7, 2024
1 parent bbd34f6 commit ce67927
Show file tree
Hide file tree
Showing 5 changed files with 340 additions and 1 deletion.
5 changes: 4 additions & 1 deletion syft/file/cataloger/executable/cataloger.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"debug/macho"
"encoding/binary"
"fmt"
"sort"

"github.com/bmatcuk/doublestar/v4"
"github.com/dustin/go-humanize"
Expand All @@ -28,8 +29,10 @@ type Cataloger struct {
}

func DefaultConfig() Config {
m := mimetype.ExecutableMIMETypeSet.List()
sort.Strings(m)
return Config{
MIMETypes: mimetype.ExecutableMIMETypeSet.List(),
MIMETypes: m,
Globs: nil,
}
}
Expand Down
137 changes: 137 additions & 0 deletions syft/format/syftjson/model/package_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -386,6 +386,143 @@ func Test_UnmarshalJSON(t *testing.T) {
assert.Equal(t, reflect.TypeOf(pkg.RustBinaryAuditEntry{}).Name(), reflect.TypeOf(p.Metadata).Name())
},
},
{
name: "map-based java metadata",
packageData: []byte(`{
"id": "e6f845bdaa69ddb2",
"name": "SparseBitSet",
"version": "1.2",
"type": "java-archive",
"foundBy": "java-archive-cataloger",
"locations": [],
"licenses": [],
"language": "java",
"cpes": [],
"purl": "pkg:maven/com.zaxxer/SparseBitSet@1.2",
"metadataType": "java-archive",
"metadata": {
"virtualPath": "/opt/solr-9.4.1/modules/extraction/lib/SparseBitSet-1.2.jar",
"manifest": {
"main": {
"Archiver-Version": "Plexus Archiver",
"Build-Jdk": "1.8.0_73",
"Built-By": "lbayer",
"Created-By": "Apache Maven 3.5.0",
"Manifest-Version": "1.0"
},
"namedSections": {
"META-INF/mailcap": {
"SHA-256-Digest": "kXN4VupOQOJhduMGwxumj4ijmD/YAlz97a9Mp7CVXtk="
},
"META-INF/versions/9/module-info.class": {
"SHA-256-Digest": "cMeIRa5l8DWPgrVWavr/6TKVBUGixVKGcu6yOTZMlKk="
}
}
}
}
}`),
assert: func(p *Package) {
meta := p.Metadata.(pkg.JavaArchive)
manifest := meta.Manifest
assert.Equal(t, "1.8.0_73", manifest.Main.MustGet("Build-Jdk"))
require.Equal(t, 2, len(manifest.Sections))
assert.Equal(t, "META-INF/mailcap", manifest.Sections[0].MustGet("Name"))
assert.Equal(t, "kXN4VupOQOJhduMGwxumj4ijmD/YAlz97a9Mp7CVXtk=", manifest.Sections[0].MustGet("SHA-256-Digest"))
assert.Equal(t, "META-INF/versions/9/module-info.class", manifest.Sections[1].MustGet("Name"))
assert.Equal(t, "cMeIRa5l8DWPgrVWavr/6TKVBUGixVKGcu6yOTZMlKk=", manifest.Sections[1].MustGet("SHA-256-Digest"))
},
},
{
name: "pre key-value golang metadata",
packageData: []byte(`{
"id": "e348ed25484a94c9",
"name": "github.com/anchore/syft",
"version": "v0.101.1-SNAPSHOT-4c777834",
"type": "go-module",
"foundBy": "go-module-binary-cataloger",
"locations": [
{
"path": "/syft",
"layerID": "sha256:2274947a5f3527e48d8725a96646aefdcce3d99340c1eefb1e7c894043863c92",
"accessPath": "/syft",
"annotations": {
"evidence": "primary"
}
}
],
"licenses": [],
"language": "go",
"cpes": [
"cpe:2.3:a:anchore:syft:v0.101.1-SNAPSHOT-4c777834:*:*:*:*:*:*:*"
],
"purl": "pkg:golang/github.com/anchore/syft@v0.101.1-SNAPSHOT-4c777834",
"metadataType": "go-module-buildinfo-entry",
"metadata": {
"goBuildSettings": {
"-buildmode": "exe",
"-compiler": "gc",
"-ldflags": "-w -s -extldflags '-static' -X main.version=0.101.1-SNAPSHOT-4c777834 -X main.gitCommit=4c777834618b2ad8ad94cd200a45d6670bc1c013 -X main.buildDate=2024-01-22T16:43:49Z -X main.gitDescription=v0.101.1-4-g4c777834 ",
"CGO_ENABLED": "0",
"GOAMD64": "v1",
"GOARCH": "amd64",
"GOOS": "linux",
"vcs": "git",
"vcs.modified": "false",
"vcs.revision": "4c777834618b2ad8ad94cd200a45d6670bc1c013",
"vcs.time": "2024-01-22T16:31:41Z"
},
"goCompiledVersion": "go1.21.2",
"architecture": "amd64",
"mainModule": "github.com/anchore/syft"
}
}
`),
assert: func(p *Package) {
buildInfo := p.Metadata.(pkg.GolangBinaryBuildinfoEntry)
assert.Equal(t, "exe", buildInfo.BuildSettings.MustGet("-buildmode"))
},
},
{
name: "conan lock with legacy options",
packageData: []byte(`{
"id": "75eb35307226c921",
"name": "boost",
"version": "1.75.0",
"type": "conan",
"foundBy": "conan-cataloger",
"locations": [
{
"path": "/conan.lock",
"accessPath": "/conan.lock",
"annotations": {
"evidence": "primary"
}
}
],
"licenses": [],
"language": "c++",
"cpes": [
"cpe:2.3:a:boost:boost:1.75.0:*:*:*:*:*:*:*"
],
"purl": "pkg:conan/boost@1.75.0",
"metadataType": "c-conan-lock-entry",
"metadata": {
"ref": "boost/1.75.0#a9c318f067216f900900e044e7af4ab1",
"package_id": "dc8aedd23a0f0a773a5fcdcfe1ae3e89c4205978",
"prev": "b9d7912e6131dfa453c725593b36c808",
"options": {
"addr2line_location": "/usr/bin/addr2line",
"asio_no_deprecated": "False",
"zstd": "False"
},
"context": "host"
}
}`),
assert: func(p *Package) {
metadata := p.Metadata.(pkg.ConanV1LockEntry)
assert.Equal(t, "False", metadata.Options.MustGet("asio_no_deprecated"))
},
},
}

for _, test := range tests {
Expand Down
56 changes: 56 additions & 0 deletions syft/pkg/java.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package pkg

import (
"encoding/json"
"fmt"
"sort"
"strings"

"github.com/anchore/syft/internal"
Expand Down Expand Up @@ -70,6 +73,59 @@ type JavaManifest struct {
Sections []KeyValues `json:"sections,omitempty"`
}

type unmarshalJavaManifest JavaManifest

type legacyJavaManifest struct {
Main map[string]string `json:"main"`
NamedSections map[string]map[string]string `json:"namedSections"`
}

func (m *JavaManifest) UnmarshalJSON(b []byte) error {
var either map[string]any
err := json.Unmarshal(b, &either)
if err != nil {
return fmt.Errorf("could not unmarshal java manifest: %w", err)
}
if _, ok := either["namedSections"]; ok {
var lm legacyJavaManifest
if err = json.Unmarshal(b, &lm); err != nil {
return fmt.Errorf("could not unmarshal java manifest: %w", err)
}
*m = lm.toNewManifest()
return nil
}
var jm unmarshalJavaManifest
err = json.Unmarshal(b, &jm)
if err != nil {
return fmt.Errorf("could not unmarshal java manifest: %w", err)
}
*m = JavaManifest(jm)
return nil
}

func (lm legacyJavaManifest) toNewManifest() JavaManifest {
var result JavaManifest
result.Main = keyValuesFromMap(lm.Main)
var sectionNames []string
for k := range lm.NamedSections {
sectionNames = append(sectionNames, k)
}
sort.Strings(sectionNames)
var sections []KeyValues
for _, name := range sectionNames {
section := KeyValues{
KeyValue{
Key: "Name",
Value: name,
},
}
section = append(section, keyValuesFromMap(lm.NamedSections[name])...)
sections = append(sections, section)
}
result.Sections = sections
return result
}

func (m JavaManifest) Section(name string) KeyValues {
for _, section := range m.Sections {
if sectionName, ok := section.Get("Name"); ok && sectionName == name {
Expand Down
101 changes: 101 additions & 0 deletions syft/pkg/java_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package pkg
import (
"testing"

"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
)

Expand Down Expand Up @@ -109,3 +110,103 @@ func TestPomProperties_PkgTypeIndicated(t *testing.T) {
})
}
}

func Test_legacyJavaManifest_toNewManifest(t *testing.T) {
tests := []struct {
name string
lm legacyJavaManifest
want JavaManifest
}{
{
name: "empty",
lm: legacyJavaManifest{},
want: JavaManifest{},
},
{
name: "main sections are sorted",
lm: legacyJavaManifest{
Main: map[string]string{
"a key": "a value",
"b key": "b value",
"c key": "c value",
},
},
want: JavaManifest{Main: KeyValues{
{
Key: "a key",
Value: "a value",
},
{
Key: "b key",
Value: "b value",
},
{
Key: "c key",
Value: "c value",
},
}},
},
{
name: "named sections have their name in the result",
lm: legacyJavaManifest{
NamedSections: map[string]map[string]string{
"a section": {
"a key": "a value",
"b key": "b value",
"c key": "c value",
},
"b section": {
"d key": "d value",
"e key": "e value",
"f key": "f value",
},
},
},
want: JavaManifest{Sections: []KeyValues{
{
{
Key: "Name",
Value: "a section",
},
{
Key: "a key",
Value: "a value",
},
{
Key: "b key",
Value: "b value",
},
{
Key: "c key",
Value: "c value",
},
},
{
{
Key: "Name",
Value: "b section",
},
{
Key: "d key",
Value: "d value",
},
{
Key: "e key",
Value: "e value",
},
{
Key: "f key",
Value: "f value",
},
},
}},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if diff := cmp.Diff(tt.want, tt.lm.toNewManifest()); diff != "" {
t.Errorf("unexpected diff in manifest (-want +got):\n%s", diff)
}
})
}
}
42 changes: 42 additions & 0 deletions syft/pkg/key_value.go
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
package pkg

import (
"encoding/json"
"fmt"
"sort"
)

type KeyValue struct {
Key string `json:"key"`
Value string `json:"value"`
Expand All @@ -26,3 +32,39 @@ func (k KeyValues) MustGet(key string) string {

return ""
}

func keyValuesFromMap(m map[string]string) KeyValues {
var result KeyValues
var mapKeys []string
for k := range m {
mapKeys = append(mapKeys, k)
}
sort.Strings(mapKeys)
for _, k := range mapKeys {
result = append(result, KeyValue{
Key: k,
Value: m[k],
})
}
return result
}

func (k *KeyValues) UnmarshalJSON(b []byte) error {
var kvs []KeyValue
if err := json.Unmarshal(b, &kvs); err != nil {
var legacyMap map[string]string
if err := json.Unmarshal(b, &legacyMap); err != nil {
return fmt.Errorf("unable to unmarshal KeyValues: %w", err)
}
var keys []string
for k := range legacyMap {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
kvs = append(kvs, KeyValue{Key: k, Value: legacyMap[k]})
}
}
*k = kvs
return nil
}

0 comments on commit ce67927

Please sign in to comment.