Add support for OCIRepository sources

- allow `OCIRepository` to be specified in `sourceRef.kind`
- react to `OCIRepository` artifacts events
- add end-to-end test for OCI repositories

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>

.github/workflows/e2e.yaml

@@ -144,6 +144,13 @@ jobs:
           kubectl -n impersonation wait kustomizations/podinfo --for=condition=ready --timeout=4m
           kubectl -n impersonation delete kustomizations/podinfo
           until kubectl -n impersonation get deploy/podinfo 2>&1 | grep NotFound ; do sleep 2; done
+      - name: Run OCI tests
+        run: |
+          kubectl create ns oci
+          kubectl -n oci apply -f ./config/testdata/oci
+          kubectl -n oci wait kustomizations/oci --for=condition=ready --timeout=2m
+          kubectl -n oci get ocirepository/oci -oyaml
+          kubectl -n oci get kustomization/oci -oyaml
       - name: Run CRDs + CRs tests
         run: |
           kubectl -n kustomize-system apply -f ./config/testdata/crds-crs
@@ -162,5 +169,7 @@ jobs:
           kubectl -n kustomize-system get gitrepositories -oyaml
           kubectl -n kustomize-system get kustomizations -oyaml
           kubectl -n kustomize-system get all
+          kubectl -n oci get ocirepository/oci -oyaml
+          kubectl -n oci get kustomization/oci -oyaml
           kubectl -n kustomize-system logs deploy/source-controller
           kubectl -n kustomize-system logs deploy/kustomize-controller

.gitignore

@@ -15,6 +15,7 @@
 # vendor/
 bin/
 config/release/
+config/crd/bases/ocirepositories.yaml
 config/crd/bases/gitrepositories.yaml
 config/crd/bases/buckets.yaml
 

Makefile

@@ -57,10 +57,11 @@ run: generate fmt vet manifests
 	go run ./main.go --metrics-addr=:8089
 
 # Download the CRDs the controller depends on
+# TODO: remove the hardcoded branch
 download-crd-deps:
-	curl -s https://raw.githubusercontent.com/fluxcd/source-controller/${SOURCE_VER}/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml > config/crd/bases/gitrepositories.yaml
-	curl -s https://raw.githubusercontent.com/fluxcd/source-controller/${SOURCE_VER}/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml > config/crd/bases/buckets.yaml
-
+	curl -s https://raw.githubusercontent.com/fluxcd/source-controller/oci/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml > config/crd/bases/gitrepositories.yaml
+	curl -s https://raw.githubusercontent.com/fluxcd/source-controller/oci/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml > config/crd/bases/buckets.yaml
+	curl -s https://raw.githubusercontent.com/fluxcd/source-controller/oci/config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml > config/crd/bases/ocirepositories.yaml
 # Install CRDs into a cluster
 install: manifests
 	kustomize build config/crd | kubectl apply -f -

api/v1beta2/reference_types.go

@@ -26,7 +26,7 @@ type CrossNamespaceSourceReference struct {
 	APIVersion string `json:"apiVersion,omitempty"`
 
 	// Kind of the referent.
-	// +kubebuilder:validation:Enum=GitRepository;Bucket
+	// +kubebuilder:validation:Enum=OCIRepository;GitRepository;Bucket
 	// +required
 	Kind string `json:"kind"`
 

config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml

@@ -942,6 +942,7 @@ spec:
                   kind:
                     description: Kind of the referent.
                     enum:
+                    - OCIRepository
                     - GitRepository
                     - Bucket
                     type: string

config/default/kustomization.yaml

@@ -1,10 +1,15 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: kustomize-system
+# TODO: remove the hardcoded branch
 resources:
-- https://github.com/fluxcd/source-controller/releases/download/v0.25.5/source-controller.crds.yaml
+- https://github.com/fluxcd/source-controller/config/crd?ref=oci
 - https://github.com/fluxcd/source-controller/releases/download/v0.25.5/source-controller.deployment.yaml
 - ../crd
 - ../rbac
 - ../manager
 - namespace.yaml
+# TODO: remove the hardcoded image
+images:
+  - name: fluxcd/source-controller
+    newTag: oci-778f6d5d

config/rbac/role.yaml

@@ -58,6 +58,7 @@ rules:
   resources:
   - buckets
   - gitrepositories
+  - ocirepositories
   verbs:
   - get
   - list
@@ -67,5 +68,6 @@ rules:
   resources:
   - buckets/status
   - gitrepositories/status
+  - ocirepositories/status
   verbs:
   - get

config/testdata/oci/podinfo.yaml

@@ -0,0 +1,37 @@
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: OCIRepository
+metadata:
+  name: oci
+  namespace: oci
+spec:
+  interval: 10m
+  url: ghcr.io/stefanprodan/manifests/podinfo
+  ref:
+    tag: "6.1.6"
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: oci
+  namespace: oci
+spec:
+  targetNamespace: oci
+  interval: 10m
+  path: "./kustomize"
+  prune: true
+  sourceRef:
+    kind: OCIRepository
+    name: oci
+  wait: true
+  timeout: 2m
+  patches:
+    - patch: |-
+        apiVersion: autoscaling/v2beta2
+        kind: HorizontalPodAutoscaler
+        metadata:
+          name: podinfo
+        spec:
+          minReplicas: 1
+      target:
+        name: podinfo
+        kind: HorizontalPodAutoscaler

controllers/kustomization_controller.go

@@ -71,8 +71,8 @@ import (
 // +kubebuilder:rbac:groups=kustomize.toolkit.fluxcd.io,resources=kustomizations,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=kustomize.toolkit.fluxcd.io,resources=kustomizations/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=kustomize.toolkit.fluxcd.io,resources=kustomizations/finalizers,verbs=get;create;update;patch;delete
-// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=buckets;gitrepositories,verbs=get;list;watch
-// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=buckets/status;gitrepositories/status,verbs=get
+// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=buckets;ocirepositories;gitrepositories,verbs=get;list;watch
+// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=buckets/status;ocirepositories/status;gitrepositories/status,verbs=get
 // +kubebuilder:rbac:groups="",resources=configmaps;secrets;serviceaccounts,verbs=get;list;watch
 // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
 
@@ -103,10 +103,17 @@ type KustomizationReconcilerOptions struct {
 
 func (r *KustomizationReconciler) SetupWithManager(mgr ctrl.Manager, opts KustomizationReconcilerOptions) error {
 	const (
+		ociRepositoryIndexKey string = ".metadata.ociRepository"
 		gitRepositoryIndexKey string = ".metadata.gitRepository"
 		bucketIndexKey        string = ".metadata.bucket"
 	)
 
+	// Index the Kustomizations by the OCIRepository references they (may) point at.
+	if err := mgr.GetCache().IndexField(context.TODO(), &kustomizev1.Kustomization{}, ociRepositoryIndexKey,
+		r.indexBy(sourcev1.OCIRepositoryKind)); err != nil {
+		return fmt.Errorf("failed setting index fields: %w", err)
+	}
+
 	// Index the Kustomizations by the GitRepository references they (may) point at.
 	if err := mgr.GetCache().IndexField(context.TODO(), &kustomizev1.Kustomization{}, gitRepositoryIndexKey,
 		r.indexBy(sourcev1.GitRepositoryKind)); err != nil {
@@ -135,6 +142,11 @@ func (r *KustomizationReconciler) SetupWithManager(mgr ctrl.Manager, opts Kustom
 		For(&kustomizev1.Kustomization{}, builder.WithPredicates(
 			predicate.Or(predicate.GenerationChangedPredicate{}, predicates.ReconcileRequestedPredicate{}),
 		)).
+		Watches(
+			&source.Kind{Type: &sourcev1.OCIRepository{}},
+			handler.EnqueueRequestsFromMapFunc(r.requestsForRevisionChangeOf(ociRepositoryIndexKey)),
+			builder.WithPredicates(SourceRevisionChangePredicate{}),
+		).
 		Watches(
 			&source.Kind{Type: &sourcev1.GitRepository{}},
 			handler.EnqueueRequestsFromMapFunc(r.requestsForRevisionChangeOf(gitRepositoryIndexKey)),
@@ -607,6 +619,16 @@ func (r *KustomizationReconciler) getSource(ctx context.Context, kustomization k
 	}
 
 	switch kustomization.Spec.SourceRef.Kind {
+	case sourcev1.OCIRepositoryKind:
+		var repository sourcev1.OCIRepository
+		err := r.Client.Get(ctx, namespacedName, &repository)
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				return source, err
+			}
+			return source, fmt.Errorf("unable to get source '%s': %w", namespacedName, err)
+		}
+		source = &repository
 	case sourcev1.GitRepositoryKind:
 		var repository sourcev1.GitRepository
 		err := r.Client.Get(ctx, namespacedName, &repository)

go.mod

@@ -28,7 +28,8 @@ require (
 	github.com/fluxcd/pkg/ssa v0.17.0
 	github.com/fluxcd/pkg/testserver v0.2.0
 	github.com/fluxcd/pkg/untar v0.1.0
-	github.com/fluxcd/source-controller/api v0.25.5
+	// TODO: update SC API when OCIRepository is released
+	github.com/fluxcd/source-controller/api v0.25.7-0.20220621143350-2ff461b191a8
 	github.com/hashicorp/go-retryablehttp v0.7.1
 	github.com/hashicorp/vault/api v1.6.0
 	github.com/onsi/gomega v1.19.0

go.sum

@@ -304,8 +304,8 @@ github.com/fluxcd/pkg/testserver v0.2.0 h1:Mj0TapmKaywI6Fi5wvt1LAZpakUHmtzWQpJNK
 github.com/fluxcd/pkg/testserver v0.2.0/go.mod h1:bgjjydkXsZTeFzjz9Cr4heGANr41uTB1Aj1Q5qzuYVk=
 github.com/fluxcd/pkg/untar v0.1.0 h1:k97V/xV5hFrAkIkVPuv5AVhyxh1ZzzAKba/lbDfGo6o=
 github.com/fluxcd/pkg/untar v0.1.0/go.mod h1:aGswNyzB1mlz/T/kpOS58mITBMxMKc9tlJBH037A2HY=
-github.com/fluxcd/source-controller/api v0.25.5 h1:64rLb5cuHhZ3LcRIxkp+/oAVCyVtjOhQ9kbphdFfR/s=
-github.com/fluxcd/source-controller/api v0.25.5/go.mod h1:/e7YRDOqb8z8I3N8ifbDF1mknf8zFsoADtS/Q93iWPs=
+github.com/fluxcd/source-controller/api v0.25.7-0.20220621143350-2ff461b191a8 h1:luR+na6gEdstH0b0AbRfF37Y3Ioq6AjrLUaXF64JIcA=
+github.com/fluxcd/source-controller/api v0.25.7-0.20220621143350-2ff461b191a8/go.mod h1:/e7YRDOqb8z8I3N8ifbDF1mknf8zFsoADtS/Q93iWPs=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y=