Skip to content

Commit

Permalink
feat: Automation of the expected images to be used by ImagePuller (#1822
Browse files Browse the repository at this point in the history 
)

* feat: Retrieve default images from plugin & devfile registries

Signed-off-by: Anatolii Bazko <abazko@redhat.com>
  • Loading branch information
@tolusha
tolusha authored Mar 29, 2024
1 parent 822bda0 commit a824926
Show file tree
Hide file tree
Showing 3 changed files with 144 additions and 134 deletions.
4 changes: 2 additions & 2 deletions bundle/next/eclipse-che/manifests/che-operator.clusterserviceversion.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ metadata:
operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
repository: https://github.com/eclipse-che/che-operator
support: Eclipse Foundation
name: eclipse-che.v7.84.0-861.next
name: eclipse-che.v7.84.0-862.next
namespace: placeholder
spec:
apiservicedefinitions: {}
Expand Down Expand Up @@ -1032,7 +1032,7 @@ spec:
minKubeVersion: 1.19.0
provider:
name: Eclipse Foundation
version: 7.84.0-861.next
version: 7.84.0-862.next
webhookdefinitions:
- admissionReviewVersions:
- v1
Expand Down
192 changes: 140 additions & 52 deletions pkg/deploy/image-puller/imagepuller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,38 +13,32 @@
package imagepuller

import (
goerror "errors"
"fmt"
"io"
"net/http"
"sort"
"strings"
"time"

"github.com/google/go-cmp/cmp/cmpopts"

"github.com/google/go-cmp/cmp"
ctrl "sigs.k8s.io/controller-runtime"
"k8s.io/apimachinery/pkg/util/validation"

chev1alpha1 "github.com/che-incubator/kubernetes-image-puller-operator/api/v1alpha1"
"github.com/eclipse-che/che-operator/pkg/common/chetypes"
"github.com/eclipse-che/che-operator/pkg/common/constants"
"github.com/eclipse-che/che-operator/pkg/common/utils"
"github.com/eclipse-che/che-operator/pkg/deploy"
"github.com/google/go-cmp/cmp/cmpopts"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/validation"
"sigs.k8s.io/controller-runtime/pkg/reconcile"

"github.com/google/go-cmp/cmp"
ctrl "sigs.k8s.io/controller-runtime"

chev1alpha1 "github.com/che-incubator/kubernetes-image-puller-operator/api/v1alpha1"
"github.com/eclipse-che/che-operator/pkg/deploy"
)

var (
log = ctrl.Log.WithName("image-puller")
defaultImagePatterns = [...]string{
"^RELATED_IMAGE_.*_theia.*",
"^RELATED_IMAGE_.*_code.*",
"^RELATED_IMAGE_.*_idea.*",
"^RELATED_IMAGE_.*_machine(_)?exec(_.*)?_plugin_registry_image.*",
"^RELATED_IMAGE_.*_kubernetes(_.*)?_plugin_registry_image.*",
"^RELATED_IMAGE_.*_openshift(_.*)?_plugin_registry_image.*",
"^RELATED_IMAGE_universal(_)?developer(_)?image(_.*)?_devfile_registry_image.*",
}
log = ctrl.Log.WithName("image-puller")
kubernetesImagePullerDiffOpts = cmp.Options{
cmpopts.IgnoreFields(chev1alpha1.KubernetesImagePuller{}, "TypeMeta", "ObjectMeta", "Status"),
}
Expand All @@ -59,8 +53,6 @@ const (
defaultImagePullerImage = "quay.io/eclipse/kubernetes-image-puller:next"
)

type Images2Pull = map[string]string

type ImagePuller struct {
deploy.Reconcilable
}
Expand All @@ -77,11 +69,11 @@ func (ip *ImagePuller) Reconcile(ctx *chetypes.DeployContext) (reconcile.Result,
}

if done, err := ip.syncKubernetesImagePuller(ctx); !done {
return reconcile.Result{}, false, err
return reconcile.Result{Requeue: true}, false, err
}
} else {
if done, err := ip.uninstallImagePuller(ctx); !done {
return reconcile.Result{}, false, err
return reconcile.Result{Requeue: true}, false, err
}
}
return reconcile.Result{}, true, nil
Expand Down Expand Up @@ -139,7 +131,9 @@ func (ip *ImagePuller) syncKubernetesImagePuller(ctx *chetypes.DeployContext) (b
imagePuller.Spec.ConfigMapName = utils.GetValue(imagePuller.Spec.ConfigMapName, defaultConfigMapName)
imagePuller.Spec.DeploymentName = utils.GetValue(imagePuller.Spec.DeploymentName, defaultDeploymentName)
imagePuller.Spec.ImagePullerImage = utils.GetValue(imagePuller.Spec.ImagePullerImage, defaultImagePullerImage)
imagePuller.Spec.Images = utils.GetValue(imagePuller.Spec.Images, getDefaultImages())
if strings.TrimSpace(imagePuller.Spec.Images) == "" {
imagePuller.Spec.Images = getDefaultImages(ctx)
}

return deploy.SyncWithClient(ctx.ClusterAPI.NonCachingClient, ctx, imagePuller, kubernetesImagePullerDiffOpts)
}
Expand All @@ -148,22 +142,132 @@ func getImagePullerCustomResourceName(ctx *chetypes.DeployContext) string {
return ctx.CheCluster.Name + "-image-puller"
}

// imagesToString returns a string representation of the provided image slice,
// suitable for the imagePuller.spec.images field
func imagesToString(images Images2Pull) string {
imageNames := make([]string, 0, len(images))
for k := range images {
imageNames = append(imageNames, k)
func getDefaultImages(ctx *chetypes.DeployContext) string {
urls := collectRegistriesUrls(ctx)
allImages := fetchImagesFromRegistries(urls, ctx)

// having them sorted, prevents from constant changing CR spec
sortedImages := sortImages(allImages)
return convertToSpecField(sortedImages)
}

func collectRegistriesUrls(ctx *chetypes.DeployContext) []string {
urls := make([]string, 0)

if ctx.CheCluster.Status.PluginRegistryURL != "" {
urls = append(
urls,
fmt.Sprintf(
"http://%s.%s.svc:8080/v3/%s",
constants.PluginRegistryName,
ctx.CheCluster.Namespace,
"external_images.txt",
),
)
}

if ctx.CheCluster.Status.DevfileRegistryURL != "" {
urls = append(
urls,
fmt.Sprintf(
"http://%s.%s.svc:8080/%s",
constants.DevfileRegistryName,
ctx.CheCluster.Namespace,
"devfiles/external_images.txt",
),
)
}
sort.Strings(imageNames)

imagesAsString := ""
for _, imageName := range imageNames {
if name, err := convertToRFC1123(imageName); err == nil {
imagesAsString += name + "=" + images[imageName] + ";"
return urls
}

func fetchImagesFromRegistries(urls []string, ctx *chetypes.DeployContext) map[string]bool {
// return as map to make the list unique
allImages := make(map[string]bool)

for _, url := range urls {
images, err := fetchImagesFromUrl(url, ctx)
if err != nil {
log.Error(err, fmt.Sprintf("Failed to fetch images from %s", url))
} else {
for image := range images {
allImages[image] = true
}
}
}
return imagesAsString

return allImages
}

func sortImages(images map[string]bool) []string {
sortedImages := make([]string, len(images))

i := 0
for image := range images {
sortedImages[i] = image
i++
}

sort.Strings(sortedImages)
return sortedImages
}

func convertToSpecField(images []string) string {
specField := ""
for index, image := range images {
imageName, _ := utils.GetImageNameAndTag(image)
imageNameEntries := strings.Split(imageName, "/")
name, err := convertToRFC1123(imageNameEntries[len(imageNameEntries)-1])
if err != nil {
name = "image"
}

// Adding index make the name unique
specField += fmt.Sprintf("%s-%d=%s;", name, index, image)
}

return specField
}

func fetchImagesFromUrl(url string, ctx *chetypes.DeployContext) (map[string]bool, error) {
transport := &http.Transport{}
if ctx.Proxy.HttpProxy != "" {
deploy.ConfigureProxy(ctx, transport)
}

client := &http.Client{
Transport: transport,
Timeout: time.Second * 3,
}

req, err := http.NewRequest("GET", url, nil)
if err != nil {
return map[string]bool{}, err
}

resp, err := client.Do(req)
if err != nil {
return map[string]bool{}, err
}

data, err := io.ReadAll(resp.Body)
if err != nil {
return map[string]bool{}, err
}

images := make(map[string]bool)
for _, image := range strings.Split(string(data), "\n") {
image = strings.TrimSpace(image)
if image != "" {
images[image] = true
}
}

if err = resp.Body.Close(); err != nil {
log.Error(err, "Failed to close a body response")
}

return images, nil
}

// convertToRFC1123 converts input string to RFC 1123 format ([a-z0-9]([-a-z0-9]*[a-z0-9])?) max 63 characters, if possible
Expand All @@ -183,7 +287,7 @@ func convertToRFC1123(str string) (string, error) {
result = strings.ReplaceAll(result, "_", "-")

if errs := validation.IsDNS1123Label(result); len(errs) > 0 {
return "", goerror.New("Cannot convert the following string to RFC 1123 format: " + str)
return "", fmt.Errorf("cannot convert the following string to RFC 1123 format: %s", str)
}
return result, nil
}
Expand All @@ -192,19 +296,3 @@ func isRFC1123Char(ch byte) bool {
errs := validation.IsDNS1123Label(string(ch))
return len(errs) == 0
}

// GetDefaultImages returns the current default images from the environment variables
func getDefaultImages() string {
images := map[string]string{}
for _, pattern := range defaultImagePatterns {
matches := utils.GetGetArchitectureDependentEnvsByRegExp(pattern)
sort.SliceStable(matches, func(i, j int) bool {
return strings.Compare(matches[i].Name, matches[j].Name) < 0
})

for _, match := range matches {
images[match.Name[len("RELATED_IMAGE_"):]] = match.Value
}
}
return imagesToString(images)
}
Loading

0 comments on commit a824926

Please sign in to comment.