Skip to content

Commit

Permalink
remove old centos images
Browse files Browse the repository at this point in the history 
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
  • Loading branch information
@wagoodman
wagoodman committed Sep 5, 2024
1 parent e08df0f commit 0ecfe1a
Show file tree
Hide file tree
Showing 5 changed files with 112 additions and 56 deletions.
76 changes: 36 additions & 40 deletions cmd/syft/internal/test/integration/package_deduplication_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,10 @@
//go:build !arm64

package integration

import (
"fmt"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
Expand All @@ -22,41 +19,39 @@ func TestPackageDeduplication(t *testing.T) {
}{
{
scope: source.AllLayersScope,
packageCount: 172, // without deduplication this would be 618
packageCount: 178, // without deduplication this would be ~600
instanceCount: map[string]int{
"basesystem": 1,
"wget": 1,
"curl": 2, // upgraded in the image
"vsftpd": 1,
"httpd": 1, // rpm, - we exclude binary
"basesystem": 1,
"wget": 1,
"curl-minimal": 2, // upgraded in the image
"vsftpd": 1,
"httpd": 1, // rpm, - we exclude binary
},
locationCount: map[string]int{
"basesystem-10.0-7.el7.centos": 4,
"curl-7.29.0-59.el7": 1, // from base image
"curl-7.29.0-59.el7_9.1": 3, // upgrade
"wget-1.14-18.el7_6.1": 3,
"vsftpd-3.0.2-29.el7_9": 2,
"httpd-2.4.6-97.el7.centos.5": 1,
// "httpd-2.4.6": 1, // binary
"basesystem-11-13.el9": 5, // in all layers
"curl-minimal-7.76.1-26.el9_3.2.0.1": 2, // base + wget layer
"curl-minimal-7.76.1-29.el9_4.1": 3, // curl upgrade layer + all above layers
"wget-1.21.1-7.el9": 4, // wget + all above layers
"vsftpd-3.0.5-5.el9": 2, // vsftpd + all above layers
"httpd-2.4.57-11.el9_4.1": 1, // last layer
},
},
{
scope: source.SquashedScope,
packageCount: 170,
packageCount: 172,
instanceCount: map[string]int{
"basesystem": 1,
"wget": 1,
"curl": 1, // upgraded, but the most recent
"vsftpd": 1,
"httpd": 1, // rpm, binary is now excluded by overlap
"basesystem": 1,
"wget": 1,
"curl-minimal": 1, // upgraded, but the most recent
"vsftpd": 1,
"httpd": 1, // rpm, binary is now excluded by overlap
},
locationCount: map[string]int{
"basesystem-10.0-7.el7.centos": 1,
"curl-7.29.0-59.el7_9.1": 1, // upgrade
"wget-1.14-18.el7_6.1": 1,
"vsftpd-3.0.2-29.el7_9": 1,
"httpd-2.4.6-97.el7.centos.5": 1,
// "httpd-2.4.6": 1, // binary (excluded)
"basesystem-11-13.el9": 1,
"curl-minimal-7.76.1-29.el9_4.1": 1, // upgrade
"wget-1.21.1-7.el9": 1,
"vsftpd-3.0.5-5.el9": 1,
"httpd-2.4.57-11.el9_4.1": 1,
},
},
}
Expand All @@ -75,20 +70,21 @@ func TestPackageDeduplication(t *testing.T) {
pkgs := sbom.Artifacts.Packages.PackagesByName(name)

// with multiple packages with the same name, something is wrong (or this is the wrong fixture)
require.Len(t, pkgs, expectedInstanceCount)

for _, p := range pkgs {
nameVersion := fmt.Sprintf("%s-%s", name, p.Version)
expectedLocationCount, ok := tt.locationCount[nameVersion]
if !ok {
t.Fatalf("missing name-version: %s", nameVersion)
}
if assert.Len(t, pkgs, expectedInstanceCount, "unexpected package count for %s", name) {
for _, p := range pkgs {
nameVersion := fmt.Sprintf("%s-%s", name, p.Version)
expectedLocationCount, ok := tt.locationCount[nameVersion]
if !ok {
t.Errorf("missing name-version: %s", nameVersion)
continue
}

// we should see merged locations (assumption, there was 1 location for each package)
assert.Len(t, p.Locations.ToSlice(), expectedLocationCount)
// we should see merged locations (assumption, there was 1 location for each package)
assert.Len(t, p.Locations.ToSlice(), expectedLocationCount, "unexpected location count for %s", nameVersion)

// all paths should match
assert.Len(t, p.Locations.CoordinateSet().Paths(), 1)
// all paths should match
assert.Len(t, p.Locations.CoordinateSet().Paths(), 1, "unexpected location count for %s", nameVersion)
}
}
}

Expand Down
5 changes: 3 additions & 2 deletions cmd/syft/internal/test/integration/test-fixtures/image-test-java-purls/extract.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
import zipfile
import io

# define constants for file extensions and metadata files
ARCHIVE_EXTENSIONS = ('.jar', '.war', '.ear', '.hpi', '.war', '.sar', '.nar', '.par')
METADATA_FILES = ('pom.xml', 'pom.properties', 'MANIFEST.MF')

Expand Down Expand Up @@ -61,8 +60,10 @@ def walk_directory_and_slim_jars(base_dir, output_dir):
slim_archive(archive_path, output_dir, os.path.relpath(dirpath, base_dir), filename)


# a helper script for slimming down JAR files by keeping only metadata files but still keeping the jar packaging,
# including nested JARs! Useful for testing purposes.
if __name__ == "__main__":
BASE_DIR = "." # replace with your base directory to search
BASE_DIR = "."
OUTPUT_DIR = "./slim"
os.makedirs(OUTPUT_DIR, exist_ok=True)
walk_directory_and_slim_jars(BASE_DIR, OUTPUT_DIR)
29 changes: 25 additions & 4 deletions cmd/syft/internal/test/integration/test-fixtures/image-vertical-package-dups/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,27 @@
FROM centos:7.9.2009@sha256:be65f488b7764ad3638f236b7b515b3678369a5124c47b8d32916d6487418ea4
FROM --platform=linux/amd64 rockylinux:9.3.20231119@sha256:d644d203142cd5b54ad2a83a203e1dee68af2229f8fe32f52a30c6e1d3c3a9e0 AS base

# modifying the RPM DB multiple times will result in duplicate packages when using all-layers (if there was no de-dup logic)
# curl is tricky, it already exists in the image and is being upgraded
RUN yum install -y wget-1.14-18.el7_6.1 curl-7.29.0-59.el7_9.1
RUN yum install -y vsftpd-3.0.2-29.el7_9
RUN yum install -y httpd-2.4.6-97.el7.centos.5

# but... we want to make the test image as small as possible, so we are making the changes in stages and then
# copying the RPM DB from each stage to a final stage in separate layers. This will result in a much smaller image.

FROM base AS stage1
RUN dnf install -y wget

FROM stage1 AS stage2
RUN dnf update -y curl-minimal

FROM stage2 AS stage3
RUN dnf install -y vsftpd

FROM stage3 AS stage4
RUN dnf install -y httpd

FROM scratch

COPY --from=base /var/lib/rpm /var/lib/rpm
COPY --from=stage1 /var/lib/rpm /var/lib/rpm
COPY --from=stage2 /var/lib/rpm /var/lib/rpm
COPY --from=stage3 /var/lib/rpm /var/lib/rpm
COPY --from=stage4 /var/lib/rpm /var/lib/rpm
50 changes: 44 additions & 6 deletions test/cli/scan_cmd_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,18 +143,56 @@ func TestPackagesCmdFlags(t *testing.T) {
name: "squashed-scope-flag-hidden-packages",
args: []string{"scan", "-o", "json", "-s", "squashed", hiddenPackagesImage},
assertions: []traitAssertion{
assertPackageCount(162),
assertNotInOutput("vsftpd"), // hidden package
assertPackageCount(14),
// package 1: alpine-baselayout-data@3.6.5-r0 (apk)
// package 2: alpine-baselayout@3.6.5-r0 (apk)
// package 3: alpine-keys@2.4-r1 (apk)
// package 4: apk-tools@2.14.4-r0 (apk)
// package 5: busybox-binsh@1.36.1-r29 (apk)
// package 6: busybox@1.36.1-r29 (apk)
// package 7: ca-certificates-bundle@20240705-r0 (apk)
// package 8: libcrypto3@3.3.1-r3 (apk)
// package 9: libssl3@3.3.1-r3 (apk)
// package 10: musl-utils@1.2.5-r0 (apk)
// package 11: musl@1.2.5-r0 (apk)
// package 12: scanelf@1.3.7-r2 (apk)
// package 13: ssl_client@1.36.1-r29 (apk)
// package 14: zlib@1.3.1-r1 (apk)
assertNotInOutput(`"name":"curl"`), // hidden package
assertSuccessfulReturnCode,
},
},
{
name: "all-layers-scope-flag",
args: []string{"scan", "-o", "json", "-s", "all-layers", hiddenPackagesImage},
assertions: []traitAssertion{
assertPackageCount(163), // packages are now deduplicated for this case
assertPackageCount(24),
// package 1: alpine-baselayout-data@3.6.5-r0 (apk)
// package 2: alpine-baselayout@3.6.5-r0 (apk)
// package 3: alpine-keys@2.4-r1 (apk)
// package 4: apk-tools@2.14.4-r0 (apk)
// package 5: brotli-libs@1.1.0-r2 (apk)
// package 6: busybox-binsh@1.36.1-r29 (apk)
// package 7: busybox@1.36.1-r29 (apk)
// package 8: c-ares@1.28.1-r0 (apk)
// package 9: ca-certificates-bundle@20240705-r0 (apk)
// package 10: ca-certificates@20240705-r0 (apk)
// package 11: curl@8.9.1-r1 (apk)
// package 12: libcrypto3@3.3.1-r3 (apk)
// package 13: libcurl@8.9.1-r1 (apk)
// package 14: libidn2@2.3.7-r0 (apk)
// package 15: libpsl@0.21.5-r1 (apk)
// package 16: libssl3@3.3.1-r3 (apk)
// package 17: libunistring@1.2-r0 (apk)
// package 18: musl-utils@1.2.5-r0 (apk)
// package 19: musl@1.2.5-r0 (apk)
// package 20: nghttp2-libs@1.62.1-r0 (apk)
// package 21: scanelf@1.3.7-r2 (apk)
// package 22: ssl_client@1.36.1-r29 (apk)
// package 23: zlib@1.3.1-r1 (apk)
// package 24: zstd-libs@1.5.6-r0 (apk)
assertInOutput("all-layers"),
assertInOutput("vsftpd"), // hidden package
assertInOutput(`"name":"curl"`), // hidden package
assertSuccessfulReturnCode,
},
},
Expand All @@ -165,9 +203,9 @@ func TestPackagesCmdFlags(t *testing.T) {
"SYFT_SCOPE": "all-layers",
},
assertions: []traitAssertion{
assertPackageCount(163), // packages are now deduplicated for this case
assertPackageCount(24), // packages are now deduplicated for this case
assertInOutput("all-layers"),
assertInOutput("vsftpd"), // hidden package
assertInOutput(`"name":"curl"`), // hidden package
assertSuccessfulReturnCode,
},
},
Expand Down
8 changes: 4 additions & 4 deletions test/cli/test-fixtures/image-hidden-packages/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM centos:7.9.2009@sha256:dead07b4d8ed7e29e98de0f4504d87e8880d4347859d839686a31da35a3b532f
# all-layers scope should pickup on vsftpd
RUN yum install -y vsftpd
RUN yum remove -y vsftpd
FROM --platform=linux/amd64 alpine:3.20.2@sha256:eddacbc7e24bf8799a4ed3cdcfa50d4b88a323695ad80f317b6629883b2c2a78

RUN apk add --no-cache curl
RUN apk del curl

0 comments on commit 0ecfe1a

Please sign in to comment.