diff --git a/cmd/syft/internal/test/integration/package_deduplication_test.go b/cmd/syft/internal/test/integration/package_deduplication_test.go index 12fa9dcf207..898586f2030 100644 --- a/cmd/syft/internal/test/integration/package_deduplication_test.go +++ b/cmd/syft/internal/test/integration/package_deduplication_test.go @@ -1,5 +1,3 @@ -//go:build !arm64 - package integration import ( @@ -7,7 +5,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" @@ -22,41 +19,39 @@ func TestPackageDeduplication(t *testing.T) { }{ { scope: source.AllLayersScope, - packageCount: 172, // without deduplication this would be 618 + packageCount: 178, // without deduplication this would be ~600 instanceCount: map[string]int{ - "basesystem": 1, - "wget": 1, - "curl": 2, // upgraded in the image - "vsftpd": 1, - "httpd": 1, // rpm, - we exclude binary + "basesystem": 1, + "wget": 1, + "curl-minimal": 2, // upgraded in the image + "vsftpd": 1, + "httpd": 1, // rpm, - we exclude binary }, locationCount: map[string]int{ - "basesystem-10.0-7.el7.centos": 4, - "curl-7.29.0-59.el7": 1, // from base image - "curl-7.29.0-59.el7_9.1": 3, // upgrade - "wget-1.14-18.el7_6.1": 3, - "vsftpd-3.0.2-29.el7_9": 2, - "httpd-2.4.6-97.el7.centos.5": 1, - // "httpd-2.4.6": 1, // binary + "basesystem-11-13.el9": 5, // in all layers + "curl-minimal-7.76.1-26.el9_3.2.0.1": 2, // base + wget layer + "curl-minimal-7.76.1-29.el9_4.1": 3, // curl upgrade layer + all above layers + "wget-1.21.1-7.el9": 4, // wget + all above layers + "vsftpd-3.0.5-5.el9": 2, // vsftpd + all above layers + "httpd-2.4.57-11.el9_4.1": 1, // last layer }, }, { scope: source.SquashedScope, - packageCount: 170, + packageCount: 172, instanceCount: map[string]int{ - "basesystem": 1, - "wget": 1, - "curl": 1, // upgraded, but the most recent - "vsftpd": 1, - "httpd": 1, // rpm, binary is now excluded by overlap + "basesystem": 1, + "wget": 1, + "curl-minimal": 1, // upgraded, but the most recent + "vsftpd": 1, + "httpd": 1, // rpm, binary is now excluded by overlap }, locationCount: map[string]int{ - "basesystem-10.0-7.el7.centos": 1, - "curl-7.29.0-59.el7_9.1": 1, // upgrade - "wget-1.14-18.el7_6.1": 1, - "vsftpd-3.0.2-29.el7_9": 1, - "httpd-2.4.6-97.el7.centos.5": 1, - // "httpd-2.4.6": 1, // binary (excluded) + "basesystem-11-13.el9": 1, + "curl-minimal-7.76.1-29.el9_4.1": 1, // upgrade + "wget-1.21.1-7.el9": 1, + "vsftpd-3.0.5-5.el9": 1, + "httpd-2.4.57-11.el9_4.1": 1, }, }, } @@ -75,20 +70,21 @@ func TestPackageDeduplication(t *testing.T) { pkgs := sbom.Artifacts.Packages.PackagesByName(name) // with multiple packages with the same name, something is wrong (or this is the wrong fixture) - require.Len(t, pkgs, expectedInstanceCount) - - for _, p := range pkgs { - nameVersion := fmt.Sprintf("%s-%s", name, p.Version) - expectedLocationCount, ok := tt.locationCount[nameVersion] - if !ok { - t.Fatalf("missing name-version: %s", nameVersion) - } + if assert.Len(t, pkgs, expectedInstanceCount, "unexpected package count for %s", name) { + for _, p := range pkgs { + nameVersion := fmt.Sprintf("%s-%s", name, p.Version) + expectedLocationCount, ok := tt.locationCount[nameVersion] + if !ok { + t.Errorf("missing name-version: %s", nameVersion) + continue + } - // we should see merged locations (assumption, there was 1 location for each package) - assert.Len(t, p.Locations.ToSlice(), expectedLocationCount) + // we should see merged locations (assumption, there was 1 location for each package) + assert.Len(t, p.Locations.ToSlice(), expectedLocationCount, "unexpected location count for %s", nameVersion) - // all paths should match - assert.Len(t, p.Locations.CoordinateSet().Paths(), 1) + // all paths should match + assert.Len(t, p.Locations.CoordinateSet().Paths(), 1, "unexpected location count for %s", nameVersion) + } } } diff --git a/cmd/syft/internal/test/integration/test-fixtures/image-test-java-purls/extract.py b/cmd/syft/internal/test/integration/test-fixtures/image-test-java-purls/extract.py index 656da8a2b6d..e0f005b4ce9 100644 --- a/cmd/syft/internal/test/integration/test-fixtures/image-test-java-purls/extract.py +++ b/cmd/syft/internal/test/integration/test-fixtures/image-test-java-purls/extract.py @@ -2,7 +2,6 @@ import zipfile import io -# define constants for file extensions and metadata files ARCHIVE_EXTENSIONS = ('.jar', '.war', '.ear', '.hpi', '.war', '.sar', '.nar', '.par') METADATA_FILES = ('pom.xml', 'pom.properties', 'MANIFEST.MF') @@ -61,8 +60,10 @@ def walk_directory_and_slim_jars(base_dir, output_dir): slim_archive(archive_path, output_dir, os.path.relpath(dirpath, base_dir), filename) +# a helper script for slimming down JAR files by keeping only metadata files but still keeping the jar packaging, +# including nested JARs! Useful for testing purposes. if __name__ == "__main__": - BASE_DIR = "." # replace with your base directory to search + BASE_DIR = "." OUTPUT_DIR = "./slim" os.makedirs(OUTPUT_DIR, exist_ok=True) walk_directory_and_slim_jars(BASE_DIR, OUTPUT_DIR) diff --git a/cmd/syft/internal/test/integration/test-fixtures/image-vertical-package-dups/Dockerfile b/cmd/syft/internal/test/integration/test-fixtures/image-vertical-package-dups/Dockerfile index cd0e69b5de8..28f95ba5949 100644 --- a/cmd/syft/internal/test/integration/test-fixtures/image-vertical-package-dups/Dockerfile +++ b/cmd/syft/internal/test/integration/test-fixtures/image-vertical-package-dups/Dockerfile @@ -1,6 +1,27 @@ -FROM centos:7.9.2009@sha256:be65f488b7764ad3638f236b7b515b3678369a5124c47b8d32916d6487418ea4 +FROM --platform=linux/amd64 rockylinux:9.3.20231119@sha256:d644d203142cd5b54ad2a83a203e1dee68af2229f8fe32f52a30c6e1d3c3a9e0 AS base + # modifying the RPM DB multiple times will result in duplicate packages when using all-layers (if there was no de-dup logic) # curl is tricky, it already exists in the image and is being upgraded -RUN yum install -y wget-1.14-18.el7_6.1 curl-7.29.0-59.el7_9.1 -RUN yum install -y vsftpd-3.0.2-29.el7_9 -RUN yum install -y httpd-2.4.6-97.el7.centos.5 + +# but... we want to make the test image as small as possible, so we are making the changes in stages and then +# copying the RPM DB from each stage to a final stage in separate layers. This will result in a much smaller image. + +FROM base AS stage1 +RUN dnf install -y wget + +FROM stage1 AS stage2 +RUN dnf update -y curl-minimal + +FROM stage2 AS stage3 +RUN dnf install -y vsftpd + +FROM stage3 AS stage4 +RUN dnf install -y httpd + +FROM scratch + +COPY --from=base /var/lib/rpm /var/lib/rpm +COPY --from=stage1 /var/lib/rpm /var/lib/rpm +COPY --from=stage2 /var/lib/rpm /var/lib/rpm +COPY --from=stage3 /var/lib/rpm /var/lib/rpm +COPY --from=stage4 /var/lib/rpm /var/lib/rpm diff --git a/test/cli/scan_cmd_test.go b/test/cli/scan_cmd_test.go index 555f0856b61..ed84933ffad 100644 --- a/test/cli/scan_cmd_test.go +++ b/test/cli/scan_cmd_test.go @@ -143,8 +143,22 @@ func TestPackagesCmdFlags(t *testing.T) { name: "squashed-scope-flag-hidden-packages", args: []string{"scan", "-o", "json", "-s", "squashed", hiddenPackagesImage}, assertions: []traitAssertion{ - assertPackageCount(162), - assertNotInOutput("vsftpd"), // hidden package + assertPackageCount(14), + // package 1: alpine-baselayout-data@3.6.5-r0 (apk) + // package 2: alpine-baselayout@3.6.5-r0 (apk) + // package 3: alpine-keys@2.4-r1 (apk) + // package 4: apk-tools@2.14.4-r0 (apk) + // package 5: busybox-binsh@1.36.1-r29 (apk) + // package 6: busybox@1.36.1-r29 (apk) + // package 7: ca-certificates-bundle@20240705-r0 (apk) + // package 8: libcrypto3@3.3.1-r3 (apk) + // package 9: libssl3@3.3.1-r3 (apk) + // package 10: musl-utils@1.2.5-r0 (apk) + // package 11: musl@1.2.5-r0 (apk) + // package 12: scanelf@1.3.7-r2 (apk) + // package 13: ssl_client@1.36.1-r29 (apk) + // package 14: zlib@1.3.1-r1 (apk) + assertNotInOutput(`"name":"curl"`), // hidden package assertSuccessfulReturnCode, }, }, @@ -152,9 +166,33 @@ func TestPackagesCmdFlags(t *testing.T) { name: "all-layers-scope-flag", args: []string{"scan", "-o", "json", "-s", "all-layers", hiddenPackagesImage}, assertions: []traitAssertion{ - assertPackageCount(163), // packages are now deduplicated for this case + assertPackageCount(24), + // package 1: alpine-baselayout-data@3.6.5-r0 (apk) + // package 2: alpine-baselayout@3.6.5-r0 (apk) + // package 3: alpine-keys@2.4-r1 (apk) + // package 4: apk-tools@2.14.4-r0 (apk) + // package 5: brotli-libs@1.1.0-r2 (apk) + // package 6: busybox-binsh@1.36.1-r29 (apk) + // package 7: busybox@1.36.1-r29 (apk) + // package 8: c-ares@1.28.1-r0 (apk) + // package 9: ca-certificates-bundle@20240705-r0 (apk) + // package 10: ca-certificates@20240705-r0 (apk) + // package 11: curl@8.9.1-r1 (apk) + // package 12: libcrypto3@3.3.1-r3 (apk) + // package 13: libcurl@8.9.1-r1 (apk) + // package 14: libidn2@2.3.7-r0 (apk) + // package 15: libpsl@0.21.5-r1 (apk) + // package 16: libssl3@3.3.1-r3 (apk) + // package 17: libunistring@1.2-r0 (apk) + // package 18: musl-utils@1.2.5-r0 (apk) + // package 19: musl@1.2.5-r0 (apk) + // package 20: nghttp2-libs@1.62.1-r0 (apk) + // package 21: scanelf@1.3.7-r2 (apk) + // package 22: ssl_client@1.36.1-r29 (apk) + // package 23: zlib@1.3.1-r1 (apk) + // package 24: zstd-libs@1.5.6-r0 (apk) assertInOutput("all-layers"), - assertInOutput("vsftpd"), // hidden package + assertInOutput(`"name":"curl"`), // hidden package assertSuccessfulReturnCode, }, }, @@ -165,9 +203,9 @@ func TestPackagesCmdFlags(t *testing.T) { "SYFT_SCOPE": "all-layers", }, assertions: []traitAssertion{ - assertPackageCount(163), // packages are now deduplicated for this case + assertPackageCount(24), // packages are now deduplicated for this case assertInOutput("all-layers"), - assertInOutput("vsftpd"), // hidden package + assertInOutput(`"name":"curl"`), // hidden package assertSuccessfulReturnCode, }, }, diff --git a/test/cli/test-fixtures/image-hidden-packages/Dockerfile b/test/cli/test-fixtures/image-hidden-packages/Dockerfile index 1150209e8b2..07b7f32754e 100644 --- a/test/cli/test-fixtures/image-hidden-packages/Dockerfile +++ b/test/cli/test-fixtures/image-hidden-packages/Dockerfile @@ -1,4 +1,4 @@ -FROM centos:7.9.2009@sha256:dead07b4d8ed7e29e98de0f4504d87e8880d4347859d839686a31da35a3b532f -# all-layers scope should pickup on vsftpd -RUN yum install -y vsftpd -RUN yum remove -y vsftpd +FROM --platform=linux/amd64 alpine:3.20.2@sha256:eddacbc7e24bf8799a4ed3cdcfa50d4b88a323695ad80f317b6629883b2c2a78 + +RUN apk add --no-cache curl +RUN apk del curl