Add exclusionList to enable excluding tags based on regex.

.gitignore

@@ -25,3 +25,4 @@ bin
 *~
 
 build/
+data

Makefile

@@ -32,7 +32,7 @@ manager: generate fmt vet
 
 # Run against the configured Kubernetes cluster in ~/.kube/config
 run: generate fmt vet manifests
-	go run ./main.go
+	go run ./main.go --storage-path=./data
 
 # Install CRDs into a cluster
 install: manifests

api/v1beta1/imagerepository_types.go

@@ -79,6 +79,11 @@ type ImageRepositorySpec struct {
 	// to the ImageRepository object based on the caller's namespace labels.
 	// +optional
 	AccessFrom *acl.AccessFrom `json:"accessFrom,omitempty"`
+
+	// ExclusionList is a list of regex strings used to exclude certain tags
+	// from being stored in the database.
+	// +optional
+	ExclusionList []string `json:"exclusionList,omitempty"`
 }
 
 type ScanResult struct {

config/crd/bases/image.toolkit.fluxcd.io_imagerepositories.yaml

@@ -433,6 +433,12 @@ spec:
                 required:
                 - name
                 type: object
+              exclusionList:
+                description: ExclusionList is a list of regex strings used to exclude
+                  certain tags from being stored in the database.
+                items:
+                  type: string
+                type: array
               image:
                 description: Image is the name of the image repository
                 type: string

controllers/imagerepository_controller.go

@@ -69,9 +69,10 @@ import (
 // for consistency (and perhaps this will have its own flux create
 // secret subcommand at some point).
 const (
-	ClientCert = "certFile"
-	ClientKey  = "keyFile"
-	CACert     = "caFile"
+	ClientCert        = "certFile"
+	ClientKey         = "keyFile"
+	CACert            = "caFile"
+	CosignObjectRegex = "^.*\\.sig$"
 )
 
 // ImageRepositoryReconciler reconciles a ImageRepository object
@@ -469,7 +470,9 @@ func (r *ImageRepositoryReconciler) scan(ctx context.Context, imageRepo *imagev1
 		}
 	}
 
-	tags, err := remote.ListWithContext(ctx, ref.Context(), options...)
+	options = append(options, remote.WithContext(ctx))
+
+	tags, err := remote.List(ref.Context(), options...)
 	if err != nil {
 		imagev1.SetImageRepositoryReadiness(
 			imageRepo,
@@ -480,14 +483,33 @@ func (r *ImageRepositoryReconciler) scan(ctx context.Context, imageRepo *imagev1
 		return err
 	}
 
+	// If no exclusion list has been defined, we make sure to always skip tags ending with
+	// ".sig", since that tag does not point to a valid image.
+	if len(imageRepo.Spec.ExclusionList) == 0 {
+		imageRepo.Spec.ExclusionList = append(imageRepo.Spec.ExclusionList, CosignObjectRegex)
+	}
+
+	filteredTags := []string{}
+	for _, regex := range imageRepo.Spec.ExclusionList {
+		r, err := regexp.Compile(regex)
+		if err != nil {
+			return fmt.Errorf("failed to compile regex %s: %w", regex, err)
+		}
+		for _, tag := range tags {
+			if !r.MatchString(tag) {
+				filteredTags = append(filteredTags, tag)
+			}
+		}
+	}
+
 	canonicalName := ref.Context().String()
-	if err := r.Database.SetTags(canonicalName, tags); err != nil {
+	if err := r.Database.SetTags(canonicalName, filteredTags); err != nil {
 		return fmt.Errorf("failed to set tags for %q: %w", canonicalName, err)
 	}
 
 	scanTime := metav1.Now()
 	imageRepo.Status.LastScanResult = &imagev1.ScanResult{
-		TagCount: len(tags),
+		TagCount: len(filteredTags),
 		ScanTime: scanTime,
 	}
 
@@ -502,7 +524,7 @@ func (r *ImageRepositoryReconciler) scan(ctx context.Context, imageRepo *imagev1
 		imageRepo,
 		metav1.ConditionTrue,
 		imagev1.ReconciliationSucceededReason,
-		fmt.Sprintf("successful scan, found %v tags", len(tags)),
+		fmt.Sprintf("successful scan, found %v tags", len(filteredTags)),
 	)
 
 	return nil

controllers/scan_test.go

@@ -81,37 +81,64 @@ func TestImageRepositoryReconciler_fetchImageTags(t *testing.T) {
 
 	registryServer := test.NewRegistryServer()
 	defer registryServer.Close()
-
-	versions := []string{"0.1.0", "0.1.1", "0.2.0", "1.0.0", "1.0.1", "1.0.2", "1.1.0-alpha"}
-	imgRepo, err := test.LoadImages(registryServer, "test-fetch-"+randStringRunes(5), versions)
-	g.Expect(err).ToNot(HaveOccurred())
-
-	repo := imagev1.ImageRepository{
-		Spec: imagev1.ImageRepositorySpec{
-			Interval: metav1.Duration{Duration: reconciliationInterval},
-			Image:    imgRepo,
+	tests := []struct {
+		name          string
+		versions      []string
+		wantVersions  []string
+		exclusionList []string
+	}{
+		{
+			name:         "fetch image tags",
+			versions:     []string{"0.1.0", "0.1.1", "0.2.0", "1.0.0", "1.1.0", "1.1.0-alpha"},
+			wantVersions: []string{"0.1.0", "0.1.1", "0.2.0", "1.0.0", "1.1.0", "1.1.0-alpha"},
+		},
+		{
+			name:         "fetch image tags - .sig is excluded",
+			versions:     []string{"0.1.0", "0.1.1", "0.1.1.sig", "1.0.0-alpha", "1.0.0", "1.0.0.sig"},
+			wantVersions: []string{"0.1.0", "0.1.1", "1.0.0-alpha", "1.0.0"},
+		},
+		{
+			name:          "fetch image tags - tags in exclusionList are excluded",
+			versions:      []string{"0.1.0", "0.1.1-alpha", "0.1.1", "0.1.1.sig", "1.0.0-alpha", "1.0.0"},
+			wantVersions:  []string{"0.1.0", "0.1.1", "0.1.1.sig", "1.0.0"},
+			exclusionList: []string{"^.*\\-alpha$"},
 		},
 	}
-	objectName := types.NamespacedName{
-		Name:      "test-fetch-img-tags-" + randStringRunes(5),
-		Namespace: "default",
-	}
-
-	repo.Name = objectName.Name
-	repo.Namespace = objectName.Namespace
 
-	ctx, cancel := context.WithTimeout(context.Background(), contextTimeout)
-	defer cancel()
-	g.Expect(testEnv.Create(ctx, &repo)).To(Succeed())
-
-	g.Eventually(func() bool {
-		err := testEnv.Get(context.Background(), objectName, &repo)
-		return err == nil && repo.Status.LastScanResult != nil
-	}, timeout, interval).Should(BeTrue())
-	g.Expect(repo.Status.CanonicalImageName).To(Equal(imgRepo))
-	g.Expect(repo.Status.LastScanResult.TagCount).To(Equal(len(versions)))
-	// Cleanup.
-	g.Expect(testEnv.Delete(ctx, &repo)).To(Succeed())
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			imgRepo, err := test.LoadImages(registryServer, "test-fetch-"+randStringRunes(5), tt.versions)
+			g.Expect(err).ToNot(HaveOccurred())
+
+			repo := imagev1.ImageRepository{
+				Spec: imagev1.ImageRepositorySpec{
+					Interval:      metav1.Duration{Duration: reconciliationInterval},
+					Image:         imgRepo,
+					ExclusionList: tt.exclusionList,
+				},
+			}
+			objectName := types.NamespacedName{
+				Name:      "test-fetch-img-tags-" + randStringRunes(5),
+				Namespace: "default",
+			}
+
+			repo.Name = objectName.Name
+			repo.Namespace = objectName.Namespace
+
+			ctx, cancel := context.WithTimeout(context.Background(), contextTimeout)
+			defer cancel()
+			g.Expect(testEnv.Create(ctx, &repo)).To(Succeed())
+
+			g.Eventually(func() bool {
+				err := testEnv.Get(context.Background(), objectName, &repo)
+				return err == nil && repo.Status.LastScanResult != nil
+			}, timeout, interval).Should(BeTrue())
+			g.Expect(repo.Status.CanonicalImageName).To(Equal(imgRepo))
+			g.Expect(repo.Status.LastScanResult.TagCount).To(Equal(len(tt.wantVersions)))
+			// Cleanup.
+			g.Expect(testEnv.Delete(ctx, &repo)).To(Succeed())
+		})
+	}
 }
 
 func TestImageRepositoryReconciler_repositorySuspended(t *testing.T) {

docs/api/image-reflector.md

@@ -499,6 +499,19 @@ github.com/fluxcd/pkg/apis/acl.AccessFrom
 to the ImageRepository object based on the caller&rsquo;s namespace labels.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>exclusionList</code><br>
+<em>
+[]string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>ExclusionList is a list of regex strings used to exclude certain tags
+from being stored in the database.</p>
+</td>
+</tr>
 </table>
 </td>
 </tr>
@@ -656,6 +669,19 @@ github.com/fluxcd/pkg/apis/acl.AccessFrom
 to the ImageRepository object based on the caller&rsquo;s namespace labels.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>exclusionList</code><br>
+<em>
+[]string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>ExclusionList is a list of regex strings used to exclude certain tags
+from being stored in the database.</p>
+</td>
+</tr>
 </tbody>
 </table>
 </div>

docs/spec/v1beta1/imagerepositories.md

@@ -63,6 +63,11 @@ type ImageRepositorySpec struct {
 	// to the ImageRepository object based on the caller's namespace labels.
 	// +optional
 	AccessFrom *AccessFrom `json:"accessFrom,omitempty"`
+
+	// ExclusionList is a list of regex strings used to exclude certain tags
+	// from being stored in the database.
+	// +optional
+	ExclusionList []string `json:"exclusionList,omitempty"`
 }
 ```
 
@@ -200,6 +205,14 @@ To grant access to all namespaces, an empty `matchLabels` must be provided:
       - matchLabels: {}
 ```
 
+### Exclude Tags
+
+To exclude certain tags, the `spec.exclusionList` field can be used to specify a list of regex expressions.
+Any tags that match any of the regex expressions, will be excluded from the final tag list.
+If `spec.exclusionList` is empty, by default the regex `"^.*\\.sig$"` is used to exclude all tags ending with
+`.sig`, since these are [Cosign](https://github.com/sigstore/cosign) generated objects and not container images
+which can be deployed on a Kubernetes cluster. 
+
 ## Status
 
 ```go