From 59fd5667a22d6b3baa4c71c96f469a0cd1375413 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Thu, 18 Jan 2024 17:21:17 +0100 Subject: [PATCH 1/3] Fill the `repo_digests` field in the SBOM payloads --- go.mod | 2 +- go.sum | 4 ++-- pkg/collector/corechecks/sbom/processor.go | 19 ++++++++++++++----- test/fakeintake/go.mod | 2 +- test/fakeintake/go.sum | 4 ++-- test/new-e2e/go.mod | 2 +- test/new-e2e/go.sum | 4 ++-- 7 files changed, 23 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index db3d000c29d70..127e7c2ff8d32 100644 --- a/go.mod +++ b/go.mod @@ -592,7 +592,7 @@ require github.com/lorenzosaino/go-sysctl v0.3.1 require ( github.com/DATA-DOG/go-sqlmock v1.5.0 - github.com/DataDog/agent-payload/v5 v5.0.100 + github.com/DataDog/agent-payload/v5 v5.0.103-0.20240118142331-3069f58aa284 github.com/DataDog/datadog-agent/cmd/agent/common/path v0.51.0-rc.2 github.com/DataDog/datadog-agent/comp/core/config v0.51.0-rc.2 github.com/DataDog/datadog-agent/comp/core/flare/types v0.51.0-rc.2 diff --git a/go.sum b/go.sum index 4b3d3951e8782..e59045f06735f 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ github.com/CycloneDX/cyclonedx-go v0.7.2 h1:kKQ0t1dPOlugSIYVOMiMtFqeXI2wp/f5DBId github.com/CycloneDX/cyclonedx-go v0.7.2/go.mod h1:K2bA+324+Og0X84fA8HhN2X066K7Bxz4rpMQ4ZhjtSk= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/DataDog/agent-payload/v5 v5.0.100 h1:mslyqwtKyTikNyeqITv6ynd1kZ1uMemEMx5TXlPQ/i8= -github.com/DataDog/agent-payload/v5 v5.0.100/go.mod h1:COngtbYYCncpIPiE5D93QlXDH/3VAKk10jDNwGHcMRE= +github.com/DataDog/agent-payload/v5 v5.0.103-0.20240118142331-3069f58aa284 h1:XPoynqoZuVbVNxgP79s93RJA0QYbmvUQ97dMmLS2WXc= +github.com/DataDog/agent-payload/v5 v5.0.103-0.20240118142331-3069f58aa284/go.mod h1:COngtbYYCncpIPiE5D93QlXDH/3VAKk10jDNwGHcMRE= github.com/DataDog/appsec-internal-go v1.3.0 h1:bqD/pnXpJl116ne/0kCtL6ZPBeeGJ1zX/DADKfA0LLQ= github.com/DataDog/appsec-internal-go v1.3.0/go.mod h1:ONW8aV6R7Thgb4g0bB9ZQCm+oRgyz5eWiW7XoQ19wIc= github.com/DataDog/aptly v1.5.3 h1:oLsRvjuXSVM4ia0N83dU3KiQeiJ6BaszYbTZOkSfDlw= diff --git a/pkg/collector/corechecks/sbom/processor.go b/pkg/collector/corechecks/sbom/processor.go index 3416610db0cad..b8c6be517500f 100644 --- a/pkg/collector/corechecks/sbom/processor.go +++ b/pkg/collector/corechecks/sbom/processor.go @@ -17,6 +17,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/workloadmeta" "github.com/DataDog/datadog-agent/pkg/aggregator/sender" "github.com/DataDog/datadog-agent/pkg/config" + //nolint:revive // TODO(CINT) Fix revive linter ddConfig "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/epforwarder" @@ -295,6 +296,13 @@ func (p *processor) processImageSBOM(img *workloadmeta.ContainerImageMetadata) { } } + repoDigests := make([]string, 0, len(img.RepoDigests)) + for _, repoDigest := range img.RepoDigests { + if strings.HasPrefix(repoDigest, repo+"@sha256:") { + repoDigests = append(repoDigests, strings.SplitN(repoDigest, "@sha256:", 2)[1]) + } + } + // Because we split a single image entity into different payloads if it has several repo digests, // me must re-compute `image_id`, `image_name`, `short_image` and `image_tag` tags. ddTags2 := make([]string, 0, len(ddTags)) @@ -316,11 +324,12 @@ func (p *processor) processImageSBOM(img *workloadmeta.ContainerImageMetadata) { } sbom := &model.SBOMEntity{ - Type: model.SBOMSourceType_CONTAINER_IMAGE_LAYERS, - Id: id, - DdTags: ddTags2, - RepoTags: repoTags, - InUse: inUse, + Type: model.SBOMSourceType_CONTAINER_IMAGE_LAYERS, + Id: id, + DdTags: ddTags2, + RepoTags: repoTags, + RepoDigests: repoDigests, + InUse: inUse, } switch img.SBOM.Status { diff --git a/test/fakeintake/go.mod b/test/fakeintake/go.mod index f1a25eb494298..cfa08e429dca4 100644 --- a/test/fakeintake/go.mod +++ b/test/fakeintake/go.mod @@ -3,7 +3,7 @@ module github.com/DataDog/datadog-agent/test/fakeintake go 1.21 require ( - github.com/DataDog/agent-payload/v5 v5.0.100 + github.com/DataDog/agent-payload/v5 v5.0.103-0.20240118142331-3069f58aa284 github.com/DataDog/datadog-agent/pkg/proto v0.50.2 github.com/benbjohnson/clock v1.3.5 github.com/cenkalti/backoff v2.2.1+incompatible diff --git a/test/fakeintake/go.sum b/test/fakeintake/go.sum index eaad9952235c9..e1b83650b2063 100644 --- a/test/fakeintake/go.sum +++ b/test/fakeintake/go.sum @@ -1,5 +1,5 @@ -github.com/DataDog/agent-payload/v5 v5.0.100 h1:mslyqwtKyTikNyeqITv6ynd1kZ1uMemEMx5TXlPQ/i8= -github.com/DataDog/agent-payload/v5 v5.0.100/go.mod h1:COngtbYYCncpIPiE5D93QlXDH/3VAKk10jDNwGHcMRE= +github.com/DataDog/agent-payload/v5 v5.0.103-0.20240118142331-3069f58aa284 h1:XPoynqoZuVbVNxgP79s93RJA0QYbmvUQ97dMmLS2WXc= +github.com/DataDog/agent-payload/v5 v5.0.103-0.20240118142331-3069f58aa284/go.mod h1:COngtbYYCncpIPiE5D93QlXDH/3VAKk10jDNwGHcMRE= github.com/DataDog/datadog-agent/pkg/proto v0.50.2 h1:dDg+U7ERLvir5D/TkJ6xA/RvU1hXcujTjCJoZG03KZE= github.com/DataDog/datadog-agent/pkg/proto v0.50.2/go.mod h1:HvK26YCxg6MfYvPJgpHC7nSjw6DTpNhd75Wlb0GAvis= github.com/DataDog/mmh3 v0.0.0-20200805151601-30884ca2197a h1:m9REhmyaWD5YJ0P53ygRHxKKo+KM+nw+zz0hEdKztMo= diff --git a/test/new-e2e/go.mod b/test/new-e2e/go.mod index 18bf24f9c68fe..1e0e362871769 100644 --- a/test/new-e2e/go.mod +++ b/test/new-e2e/go.mod @@ -12,7 +12,7 @@ replace ( ) require ( - github.com/DataDog/agent-payload/v5 v5.0.100 + github.com/DataDog/agent-payload/v5 v5.0.103-0.20240118142331-3069f58aa284 github.com/DataDog/datadog-agent/pkg/util/pointer v0.51.0-rc.2 github.com/DataDog/datadog-agent/test/fakeintake v0.51.0-rc.2 github.com/DataDog/datadog-api-client-go v1.16.0 diff --git a/test/new-e2e/go.sum b/test/new-e2e/go.sum index 82d8833f8f2c3..1818d93670343 100644 --- a/test/new-e2e/go.sum +++ b/test/new-e2e/go.sum @@ -4,8 +4,8 @@ dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/agent-payload/v5 v5.0.100 h1:mslyqwtKyTikNyeqITv6ynd1kZ1uMemEMx5TXlPQ/i8= -github.com/DataDog/agent-payload/v5 v5.0.100/go.mod h1:COngtbYYCncpIPiE5D93QlXDH/3VAKk10jDNwGHcMRE= +github.com/DataDog/agent-payload/v5 v5.0.103-0.20240118142331-3069f58aa284 h1:XPoynqoZuVbVNxgP79s93RJA0QYbmvUQ97dMmLS2WXc= +github.com/DataDog/agent-payload/v5 v5.0.103-0.20240118142331-3069f58aa284/go.mod h1:COngtbYYCncpIPiE5D93QlXDH/3VAKk10jDNwGHcMRE= github.com/DataDog/datadog-agent/pkg/proto v0.50.2 h1:dDg+U7ERLvir5D/TkJ6xA/RvU1hXcujTjCJoZG03KZE= github.com/DataDog/datadog-agent/pkg/proto v0.50.2/go.mod h1:HvK26YCxg6MfYvPJgpHC7nSjw6DTpNhd75Wlb0GAvis= github.com/DataDog/datadog-api-client-go v1.16.0 h1:5jOZv1m98criCvYTa3qpW8Hzv301nbZX3K9yJtwGyWY= From 71cb208e2bc26718ad1ec710ee0bbe099435eeb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Fri, 19 Jan 2024 14:19:48 +0100 Subject: [PATCH 2/3] Do not truncate the repo digests and do not send payloads with no digest --- pkg/collector/corechecks/sbom/processor.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/collector/corechecks/sbom/processor.go b/pkg/collector/corechecks/sbom/processor.go index b8c6be517500f..ae8f214b7a26a 100644 --- a/pkg/collector/corechecks/sbom/processor.go +++ b/pkg/collector/corechecks/sbom/processor.go @@ -299,10 +299,15 @@ func (p *processor) processImageSBOM(img *workloadmeta.ContainerImageMetadata) { repoDigests := make([]string, 0, len(img.RepoDigests)) for _, repoDigest := range img.RepoDigests { if strings.HasPrefix(repoDigest, repo+"@sha256:") { - repoDigests = append(repoDigests, strings.SplitN(repoDigest, "@sha256:", 2)[1]) + repoDigests = append(repoDigests, repoDigest) } } + if len(repoDigests) == 0 { + log.Errorf("The image %s has no repo digest for repo %s", img.ID, repo) + continue + } + // Because we split a single image entity into different payloads if it has several repo digests, // me must re-compute `image_id`, `image_name`, `short_image` and `image_tag` tags. ddTags2 := make([]string, 0, len(ddTags)) From beaef5214d3da0d5faab6ac5a1e14b4ba592e838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Wed, 24 Jan 2024 15:58:03 +0100 Subject: [PATCH 3/3] Fix tests --- .../corechecks/sbom/processor_test.go | 71 ++++++++++--------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/pkg/collector/corechecks/sbom/processor_test.go b/pkg/collector/corechecks/sbom/processor_test.go index 214e1e34cc060..1d9ecfd336a83 100644 --- a/pkg/collector/corechecks/sbom/processor_test.go +++ b/pkg/collector/corechecks/sbom/processor_test.go @@ -106,6 +106,9 @@ func TestProcessEvents(t *testing.T) { "7-rc", "7.41.1-rc.1", }, + RepoDigests: []string{ + "datadog/agent@sha256:052f1fdf4f9a7117d36a1838ab60782829947683007c34b69d4991576375c409", + }, InUse: false, GeneratedAt: timestamppb.New(sbomGenerationTime), GenerationDuration: durationpb.New(10 * time.Second), @@ -142,6 +145,9 @@ func TestProcessEvents(t *testing.T) { "7-rc", "7.41.1-rc.1", }, + RepoDigests: []string{ + "gcr.io/datadoghq/agent@sha256:052f1fdf4f9a7117d36a1838ab60782829947683007c34b69d4991576375c409", + }, InUse: false, GeneratedAt: timestamppb.New(sbomGenerationTime), GenerationDuration: durationpb.New(10 * time.Second), @@ -178,6 +184,9 @@ func TestProcessEvents(t *testing.T) { "7-rc", "7.41.1-rc.1", }, + RepoDigests: []string{ + "public.ecr.aws/datadog/agent@sha256:052f1fdf4f9a7117d36a1838ab60782829947683007c34b69d4991576375c409", + }, InUse: false, GeneratedAt: timestamppb.New(sbomGenerationTime), GenerationDuration: durationpb.New(10 * time.Second), @@ -208,7 +217,8 @@ func TestProcessEvents(t *testing.T) { // this test, we define an image with 2 repo tags: one for the gcr.io // registry and another for the public.ecr.aws registry, but there's only // one repo digest. - // We expect to send 2 events, one for each registry. + // We expect to send only one event as the backend-end will drop + // sbom without any repo digest anyhow. name: "repo tag with no matching repo digest", inputEvents: []workloadmeta.Event{ { @@ -262,39 +272,8 @@ func TestProcessEvents(t *testing.T) { RepoTags: []string{ "7-rc", }, - InUse: false, - GeneratedAt: timestamppb.New(sbomGenerationTime), - GenerationDuration: durationpb.New(10 * time.Second), - Sbom: &model.SBOMEntity_Cyclonedx{ - Cyclonedx: &cyclonedx_v1_4.Bom{ - SpecVersion: "1.4", - Version: pointer.Ptr(int32(42)), - Components: []*cyclonedx_v1_4.Component{ - { - Name: "Foo", - }, - { - Name: "Bar", - }, - { - Name: "Baz", - }, - }, - }, - }, - Status: model.SBOMStatus_SUCCESS, - }, - { - Type: model.SBOMSourceType_CONTAINER_IMAGE_LAYERS, - Id: "gcr.io/datadoghq/agent@sha256:9634b84c45c6ad220c3d0d2305aaa5523e47d6d43649c9bbeda46ff010b4aacd", - DdTags: []string{ - "image_id:gcr.io/datadoghq/agent@sha256:9634b84c45c6ad220c3d0d2305aaa5523e47d6d43649c9bbeda46ff010b4aacd", - "image_name:gcr.io/datadoghq/agent", - "short_image:agent", - "image_tag:7-rc", - }, - RepoTags: []string{ - "7-rc", + RepoDigests: []string{ + "public.ecr.aws/datadog/agent@sha256:052f1fdf4f9a7117d36a1838ab60782829947683007c34b69d4991576375c409", }, InUse: false, GeneratedAt: timestamppb.New(sbomGenerationTime), @@ -372,6 +351,9 @@ func TestProcessEvents(t *testing.T) { RepoTags: []string{ "7-rc", }, + RepoDigests: []string{ + "datadog/agent@sha256:052f1fdf4f9a7117d36a1838ab60782829947683007c34b69d4991576375c409", + }, InUse: false, GeneratedAt: timestamppb.New(sbomGenerationTime), GenerationDuration: durationpb.New(10 * time.Second), @@ -395,6 +377,9 @@ func TestProcessEvents(t *testing.T) { RepoTags: []string{ "7-rc", }, + RepoDigests: []string{ + "datadog/agent@sha256:052f1fdf4f9a7117d36a1838ab60782829947683007c34b69d4991576375c409", + }, InUse: true, GeneratedAt: timestamppb.New(sbomGenerationTime), GenerationDuration: durationpb.New(10 * time.Second), @@ -452,6 +437,9 @@ func TestProcessEvents(t *testing.T) { "7-rc", "7.41.1-rc.1", }, + RepoDigests: []string{ + "datadog/agent@sha256:052f1fdf4f9a7117d36a1838ab60782829947683007c34b69d4991576375c409", + }, InUse: false, Status: model.SBOMStatus_PENDING, }, @@ -469,6 +457,9 @@ func TestProcessEvents(t *testing.T) { "7-rc", "7.41.1-rc.1", }, + RepoDigests: []string{ + "gcr.io/datadoghq/agent@sha256:052f1fdf4f9a7117d36a1838ab60782829947683007c34b69d4991576375c409", + }, InUse: false, Status: model.SBOMStatus_PENDING, }, @@ -486,6 +477,9 @@ func TestProcessEvents(t *testing.T) { "7-rc", "7.41.1-rc.1", }, + RepoDigests: []string{ + "public.ecr.aws/datadog/agent@sha256:052f1fdf4f9a7117d36a1838ab60782829947683007c34b69d4991576375c409", + }, InUse: false, Status: model.SBOMStatus_PENDING, }, @@ -536,6 +530,9 @@ func TestProcessEvents(t *testing.T) { "7-rc", "7.41.1-rc.1", }, + RepoDigests: []string{ + "datadog/agent@sha256:052f1fdf4f9a7117d36a1838ab60782829947683007c34b69d4991576375c409", + }, InUse: false, Sbom: &model.SBOMEntity_Error{ Error: "error", @@ -556,6 +553,9 @@ func TestProcessEvents(t *testing.T) { "7-rc", "7.41.1-rc.1", }, + RepoDigests: []string{ + "gcr.io/datadoghq/agent@sha256:052f1fdf4f9a7117d36a1838ab60782829947683007c34b69d4991576375c409", + }, InUse: false, Sbom: &model.SBOMEntity_Error{ Error: "error", @@ -576,6 +576,9 @@ func TestProcessEvents(t *testing.T) { "7-rc", "7.41.1-rc.1", }, + RepoDigests: []string{ + "public.ecr.aws/datadog/agent@sha256:052f1fdf4f9a7117d36a1838ab60782829947683007c34b69d4991576375c409", + }, InUse: false, Sbom: &model.SBOMEntity_Error{ Error: "error",