cloud-provider-e2e: Use test image-reflector build

Update the test suite to take the image-reflector-controller build to be
tested using the variable TEST_IMG. This test image must be built
locally before running the e2e tests. In the e2e tests, this local image
is retagged and pushed into a remote registry and is used in flux
deployment.

For Google Artifact Registry and Azure Container Registry, the
previously created test registries are reused to push the
image-reflector image. But for ECR, a new repository is created due to
lack of support for dynamic repositories.

Signed-off-by: Sunny <darkowlzz@protonmail.com>

tests/integration/Makefile

@@ -5,25 +5,21 @@ TEST_TIMEOUT ?= 30m
 FLUX_MANIFEST_URL ?= https://github.com/fluxcd/flux2/releases/latest/download/install.yaml
 INSTALL_MANIFEST_PATH ?= build/flux/install.yaml
 
-IMG ?= fluxcd/image-reflector-controller
+# image-reflector-controller test image.
+TEST_IMG ?=
 
 $(INSTALL_MANIFEST_PATH):
 	mkdir -p build/flux
 	curl -Lo $(INSTALL_MANIFEST_PATH) $(FLUX_MANIFEST_URL)
 
-# Build the manifests required in the test.
-build-manifests: $(INSTALL_MANIFEST_PATH)
-	cp kustomization.yaml build/flux
-	cd build/flux && kustomize edit set image fluxcd/image-reflector-controller=${IMG}
-	kustomize build build/flux > build/flux.yaml
-
 # Delete all the build files.
 distclean:
 	rm -r build/
 
 # Builds manifests and run the tests.
-test: build-manifests
-	go test -timeout $(TEST_TIMEOUT) -v $(GO_TEST_PATH) $(GO_TEST_ARGS) $(PROVIDER_ARG)
+test: $(INSTALL_MANIFEST_PATH)
+	cp kustomization.yaml build/flux
+	TEST_IMG=$(TEST_IMG) go test -timeout $(TEST_TIMEOUT) -v $(GO_TEST_PATH) $(GO_TEST_ARGS) $(PROVIDER_ARG)
 
 test-aws:
 	$(MAKE) test PROVIDER_ARG="-provider aws"

tests/integration/README.md

@@ -38,10 +38,14 @@ deleted manually.
 Copy `.env.sample` to `.env`, put the respective provider configurations in the
 environment variables and source it, `source .env`.
 
-Run the test with `make test-*`:
+Ensure the image-reflector-controller container image to be tested is built and
+ready for testing.
+
+Run the test with `make test-*`, setting the image-reflector image with variable
+`TEST_IMG`:
 
 ```console
-$ make test-aws
+$ make test-aws TEST_IMG=foo/image-reflector-controller:dev
 mkdir -p build/flux
 curl -Lo build/flux/install.yaml https://github.com/fluxcd/flux2/releases/latest/download/install.yaml
   % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
@@ -54,10 +58,12 @@ go test -timeout 20m -v ./... -existing
 2022/06/15 01:55:09 Terraform binary:  /go/src/github.com/fluxcd/image-reflector-controller/tests/integration/build/terraform
 2022/06/15 01:55:09 Init Terraform
 2022/06/15 01:55:14 Applying Terraform
+2022/06/15 01:55:21 pushing flux test image foo111.dkr.ecr.us-east-2.amazonaws.com/flux-test-image-reflector-direct-elephant:test
 2022/06/15 01:55:41 pushing test image foo111.dkr.ecr.us-east-2.amazonaws.com/flux-image-automation-test:v0.1.0
 2022/06/15 01:55:45 pushing test image foo111.dkr.ecr.us-east-2.amazonaws.com/flux-image-automation-test:v0.1.2
 2022/06/15 01:55:48 pushing test image foo111.dkr.ecr.us-east-2.amazonaws.com/flux-image-automation-test:v0.1.3
 2022/06/15 01:55:51 pushing test image foo111.dkr.ecr.us-east-2.amazonaws.com/flux-image-automation-test:v0.1.4
+2022/06/15 01:55:54 setting images: [fluxcd/image-reflector-controller=457472006214.dkr.ecr.us-east-2.amazonaws.com/flux-test-image-reflector-direct-elephant:test]
 2022/06/15 01:55:54 Installing flux
 === RUN   TestImageRepositoryScan
 === RUN   TestImageRepositoryScan/ecr
@@ -78,9 +84,9 @@ Then the `kustomization.yaml` is copied to `build/flux/`. This kustomization
 contains configurations to configure the flux installation by patching the
 downloaded `install.yaml`. It can also be used to set any custom images for any
 of the flux components. The image-reflector-controller image can be configured
-by setting the `IMG` variable when running the test. The kustomization is built
-and the resulting flux installation manifest is written to `build/flux.yaml`.
-This is used by the test to install flux.
+by setting the `TEST_IMG` variable when running the test. The kustomization is
+built and the resulting flux installation manifest is written to
+`build/flux.yaml`.  This is used by the test to install flux.
 
 The go test is started with a long timeout because the infrastructure set up
 can take a long time. It can also be configured by setting the variable
@@ -110,25 +116,25 @@ For debugging environment provisioning, enable verbose output with `-verbose`
 test flag.
 
 ```console
-$ make test-aws GO_TEST_ARGS="-verbose"
+$ make test-aws GO_TEST_ARGS="-verbose" TEST_IMG=foo/image-reflector-controller:dev
 ```
 
 The test environment is destroyed at the end by default. Run the tests with
 `-retain` flag to retain the created test infrastructure.
 
 ```console
-$ make test-aws GO_TEST_ARGS="-retain"
+$ make test-aws GO_TEST_ARGS="-retain" TEST_IMG=foo/image-reflector-controller:dev
 ```
 
 The tests require the infrastructure state to be clean. For re-running the tests
 with a retained infrastructure, set `-existing` flag.
 
 ```console
-$ make test-aws GO_TEST_ARGS="-retain -existing"
+$ make test-aws GO_TEST_ARGS="-retain -existing" TEST_IMG=foo/image-reflector-controller:dev
 ```
 
 To delete an existing infrastructure created with `-retain` flag:
 
 ```console
-$ make test-aws GO_TEST_ARGS="-existing"
+$ make test-aws GO_TEST_ARGS="-existing" TEST_IMG=foo/image-reflector-controller:dev
 ```

tests/integration/aws_test.go

@@ -19,6 +19,7 @@ package integration
 import (
 	"context"
 	"fmt"
+	"log"
 	"os"
 
 	tfjson "github.com/hashicorp/terraform-json"
@@ -94,15 +95,67 @@ func registryLoginECR(ctx context.Context, output map[string]*tfjson.StateOutput
 	// NOTE: ECR provides pre-existing registry per account. It requires
 	// repositories to be created explicitly using their API before pushing
 	// image.
-	repoURL := output["ecr_repository_url"].Value.(string)
+	testRepoURL := output["ecr_repository_url"].Value.(string)
+	ircRepoURL := output["ecr_image_reflector_controller_repo_url"].Value.(string)
 	region := output["region"].Value.(string)
 
 	if err := tftestenv.RunCommand(ctx, "./",
-		fmt.Sprintf("aws ecr get-login-password --region %s | docker login --username AWS --password-stdin %s", region, repoURL),
+		fmt.Sprintf("aws ecr get-login-password --region %s | docker login --username AWS --password-stdin %s", region, testRepoURL),
 		tftestenv.RunCommandOptions{},
 	); err != nil {
 		return nil, err
 	}
 
-	return map[string]string{"ecr": repoURL}, nil
+	if err := tftestenv.RunCommand(ctx, "./",
+		fmt.Sprintf("aws ecr get-login-password --region %s | docker login --username AWS --password-stdin %s", region, ircRepoURL),
+		tftestenv.RunCommandOptions{},
+	); err != nil {
+		return nil, err
+	}
+
+	return map[string]string{"ecr": testRepoURL}, nil
+}
+
+// pushFluxTestImagesECR pushes flux image that is being tested. It must be
+// called after registryLoginECR to ensure the local docker client is already
+// logged in and is capable of pushing the test images.
+func pushFluxTestImagesECR(ctx context.Context, localImgs map[string]string, output map[string]*tfjson.StateOutput) (map[string]string, error) {
+	// NOTE: Unlike Azure Container Registry and Google Artifact Registry, ECR
+	// does not support dynamic image repositories. A new repository for a new
+	// image has to be explicitly created. Therefore, the single local image
+	// is retagged and pushed in the already created repository.
+	if len(localImgs) != 1 {
+		return nil, fmt.Errorf("ECR repository supports pushing one image only, got: %v", localImgs)
+	}
+
+	// Get the registry name and construct the image names accordingly.
+	repo := output["ecr_image_reflector_controller_repo_url"].Value.(string)
+
+	remoteImage := repo + ":test"
+
+	// Extract the component name and local image.
+	var name, localImage string
+	for n, i := range localImgs {
+		name, localImage = n, i
+	}
+
+	if err := tftestenv.RunCommand(ctx, "./",
+		fmt.Sprintf("docker tag %s %s", localImage, remoteImage),
+		tftestenv.RunCommandOptions{},
+	); err != nil {
+		return nil, err
+	}
+
+	log.Printf("pushing flux test image %s\n", remoteImage)
+
+	if err := tftestenv.RunCommand(ctx, "./",
+		fmt.Sprintf("docker push %s", remoteImage),
+		tftestenv.RunCommandOptions{},
+	); err != nil {
+		return nil, err
+	}
+
+	return map[string]string{
+		name: remoteImage,
+	}, nil
 }

tests/integration/azure_test.go

@@ -59,3 +59,12 @@ func registryLoginACR(ctx context.Context, output map[string]*tfjson.StateOutput
 	}
 	return map[string]string{"acr": registryURL + "/" + randStringRunes(5)}, nil
 }
+
+// pushFluxTestImagesACR pushes flux images that are being tested. It must be
+// called after registryLoginACR to ensure the local docker client is already
+// logged in and is capable of pushing the test images.
+func pushFluxTestImagesACR(ctx context.Context, localImgs map[string]string, output map[string]*tfjson.StateOutput) (map[string]string, error) {
+	// Get the registry name and construct the image names accordingly.
+	repo := output["acr_registry_url"].Value.(string)
+	return retagAndPush(ctx, repo, localImgs)
+}

tests/integration/gcp_test.go

@@ -58,6 +58,21 @@ func registryLoginGCR(ctx context.Context, output map[string]*tfjson.StateOutput
 		return nil, err
 	}
 
+	artifactRegistry, artifactURL := getGoogleArtifactRegistryAndRepository(output)
+	if err := tftestenv.RunCommand(ctx, "./",
+		fmt.Sprintf("gcloud auth configure-docker %s", artifactRegistry),
+		tftestenv.RunCommandOptions{},
+	); err != nil {
+		return nil, err
+	}
+
+	return map[string]string{
+		"gcr":               repoURL + "/" + randStringRunes(5),
+		"artifact_registry": artifactURL + "/" + randStringRunes(5),
+	}, nil
+}
+
+func getGoogleArtifactRegistryAndRepository(output map[string]*tfjson.StateOutput) (string, string) {
 	// NOTE: Artifact Registry calls a registry a "repository". A repository can
 	// contain multiple different images, unlike ECR or ACR where a repository
 	// can contain multiple tags of only a single image.
@@ -70,16 +85,15 @@ func registryLoginGCR(ctx context.Context, output map[string]*tfjson.StateOutput
 	// Use the fixed docker formatted repository suffix with the location to
 	// create the registry address.
 	artifactRegistry := fmt.Sprintf("%s-docker.pkg.dev", location)
-	artifactURL := fmt.Sprintf("%s/%s/%s", artifactRegistry, project, repository)
-	if err := tftestenv.RunCommand(ctx, "./",
-		fmt.Sprintf("gcloud auth configure-docker %s", artifactRegistry),
-		tftestenv.RunCommandOptions{},
-	); err != nil {
-		return nil, err
-	}
+	artifactRepository := fmt.Sprintf("%s/%s/%s", artifactRegistry, project, repository)
+	return artifactRegistry, artifactRepository
+}
 
-	return map[string]string{
-		"gcr":               repoURL + "/" + randStringRunes(5),
-		"artifact_registry": artifactURL + "/" + randStringRunes(5),
-	}, nil
+// pushFluxTestImagesGCR pushes flux images that are being tested. It must be
+// called after registryLoginGCR to ensure the local docker client is already
+// logged in and is capable of pushing the test images.
+func pushFluxTestImagesGCR(ctx context.Context, localImgs map[string]string, output map[string]*tfjson.StateOutput) (map[string]string, error) {
+	// Get the repository name and construct the image names accordingly.
+	_, repo := getGoogleArtifactRegistryAndRepository(output)
+	return retagAndPush(ctx, repo, localImgs)
 }

tests/integration/suite_test.go

@@ -92,6 +92,12 @@ var (
 // output.
 type registryLoginFunc func(ctx context.Context, output map[string]*tfjson.StateOutput) (map[string]string, error)
 
+// pushTestImages is used to push local flux test images to a remote registry
+// after logging in using registryLoginFunc. It takes a map of image name and
+// local images and terraform state output. The local images are retagged and
+// pushed to a corresponding registry repository for the image.
+type pushTestImages func(ctx context.Context, localImgs map[string]string, output map[string]*tfjson.StateOutput) (map[string]string, error)
+
 // ProviderConfig is the test configuration of a supported cloud provider to run
 // the tests against.
 type ProviderConfig struct {
@@ -102,6 +108,8 @@ type ProviderConfig struct {
 	registryLogin registryLoginFunc
 	// createKubeconfig is used to create kubeconfig of a cluster.
 	createKubeconfig tftestenv.CreateKubeconfig
+	// pushFluxTestImages is used to push flux test images to a remote registry.
+	pushFluxTestImages pushTestImages
 }
 
 func init() {
@@ -122,6 +130,15 @@ func TestMain(m *testing.M) {
 	flag.Parse()
 	ctx := context.TODO()
 
+	ircImg := os.Getenv("TEST_IMG")
+	if ircImg == "" {
+		log.Fatal("TEST_IMG must be set to the test image-reflector-controller image, cannot be empty")
+	}
+
+	localImgs := map[string]string{
+		"image-reflector-controller": ircImg,
+	}
+
 	// Validate the provider.
 	if *targetProvider == "" {
 		log.Fatalf("-provider flag must be set to one of %v", supportedProviders)
@@ -171,11 +188,21 @@ func TestMain(m *testing.M) {
 		panic(fmt.Sprintf("Failed to log into registry: %v", err))
 	}
 
+	pushedImages, err := providerCfg.pushFluxTestImages(ctx, localImgs, output)
+	if err != nil {
+		panic(fmt.Sprintf("Failed to push test images: %v", err))
+	}
+
 	// Create and push test images.
 	if err := createAndPushImages(testRepos, testImageTags); err != nil {
 		panic(fmt.Sprintf("Failed to create and push images: %v", err))
 	}
 
+	// Update flux install manifests with the pushed test images.
+	if err := updateAndBuildFluxInstallManifests(ctx, pushedImages); err != nil {
+		panic(fmt.Sprintf("Failed to update and build flux install manifests: %v", err))
+	}
+
 	log.Println("Installing flux")
 	if err := installFlux(ctx, kubeconfigPath, fluxInstallManifestPath); err != nil {
 		log.Printf("Failed to install flux: %v", err)
@@ -197,21 +224,24 @@ func getProviderConfig(provider string) *ProviderConfig {
 	switch provider {
 	case "aws":
 		return &ProviderConfig{
-			terraformPath:    terraformPathAWS,
-			registryLogin:    registryLoginECR,
-			createKubeconfig: createKubeconfigEKS,
+			terraformPath:      terraformPathAWS,
+			registryLogin:      registryLoginECR,
+			pushFluxTestImages: pushFluxTestImagesECR,
+			createKubeconfig:   createKubeconfigEKS,
 		}
 	case "azure":
 		return &ProviderConfig{
-			terraformPath:    terraformPathAzure,
-			registryLogin:    registryLoginACR,
-			createKubeconfig: createKubeConfigAKS,
+			terraformPath:      terraformPathAzure,
+			registryLogin:      registryLoginACR,
+			pushFluxTestImages: pushFluxTestImagesACR,
+			createKubeconfig:   createKubeConfigAKS,
 		}
 	case "gcp":
 		return &ProviderConfig{
-			terraformPath:    terraformPathGCP,
-			registryLogin:    registryLoginGCR,
-			createKubeconfig: createKubeconfigGKE,
+			terraformPath:      terraformPathGCP,
+			registryLogin:      registryLoginGCR,
+			pushFluxTestImages: pushFluxTestImagesGCR,
+			createKubeconfig:   createKubeconfigGKE,
 		}
 	}
 	return nil

tests/integration/terraform/aws/ecr.tf

@@ -3,3 +3,9 @@ resource "aws_ecr_repository" "testrepo" {
   image_tag_mutability = "MUTABLE"
   force_delete         = true
 }
+
+resource "aws_ecr_repository" "image_reflector_controller" {
+  name                 = "flux-test-image-reflector-${random_pet.suffix.id}"
+  image_tag_mutability = "MUTABLE"
+  force_delete         = true
+}

tests/integration/terraform/aws/outputs.tf

@@ -30,3 +30,7 @@ output "ecr_repository_url" {
 output "ecr_registry_id" {
   value = aws_ecr_repository.testrepo.registry_id
 }
+
+output "ecr_image_reflector_controller_repo_url" {
+  value = aws_ecr_repository.image_reflector_controller.repository_url
+}