From 68536a41adc31571d1fbaee6ab13a9e56e4eb9c6 Mon Sep 17 00:00:00 2001 From: Gidi233 Date: Mon, 2 Sep 2024 20:17:57 +0800 Subject: [PATCH 01/10] Install the Gateway plugin in fleet Signed-off-by: Gidi233 --- pkg/apis/fleet/v1alpha1/types.go | 21 +++++- pkg/fleet-manager/fleet_plugin.go | 4 + pkg/fleet-manager/fleet_plugin_provider.go | 75 +++++++++++++++++++ pkg/fleet-manager/manifests/plugins/kuma.yaml | 5 ++ .../manifests/plugins/nginx.yaml | 12 +++ pkg/fleet-manager/plugin/plugin.go | 45 +++++++++++ 6 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 pkg/fleet-manager/fleet_plugin_provider.go create mode 100644 pkg/fleet-manager/manifests/plugins/kuma.yaml create mode 100644 pkg/fleet-manager/manifests/plugins/nginx.yaml diff --git a/pkg/apis/fleet/v1alpha1/types.go b/pkg/apis/fleet/v1alpha1/types.go index 80ae44a2..3bcb0604 100644 --- a/pkg/apis/fleet/v1alpha1/types.go +++ b/pkg/apis/fleet/v1alpha1/types.go @@ -562,13 +562,28 @@ type FlaggerConfig struct { // TrafficRoutingProvider defines traffic routing provider. // And Kurator will install flagger in trafficRoutingProvider's namespace // For example, If you use `istio` as a provider, flager will be installed in istio's namespace `istio-system`. + // And if you use `istio` as a provider, you need to install it manually. + // Otherwise, you can configure it in ProviderConfig (or use the default configuration) and Kurator will automatically deploy it. // Other provider will be added later. - // +optional - TrafficRoutingProvider Provider `json:"trafficRoutingProvider,omitempty"` + TrafficRoutingProvider Provider `json:"trafficRoutingProvider"` // PublicTestloader defines whether to install the publictestloader or not. // In addition to the public testloader you can configure here, // you can also specify a private testloader in the Application.Spec.SyncPolicies.Rollout.TestLoader PublicTestloader bool `json:"publicTestloader,omitempty"` + // ProviderConfig defines the configuration for the TrafficRoutingProvider. + // +optional + ProviderConfig *Config `json:"Config,omitempty"` +} + +type Config struct { + // Chart defines the helm chart config of the TrafficRoutingProvider. + // default value is in ./pkg/fleet-manager/manifests/plugins/ + // +optional + Chart *ChartConfig `json:"chart,omitempty"` + // ExtraArgs is the set of extra arguments for TrafficRoutingProvider's chart. + // You can pass in values according to your needs. + // +optional + ExtraArgs apiextensionsv1.JSON `json:"extraArgs,omitempty"` } type SubMarinerOperatorConfig struct { @@ -624,6 +639,8 @@ type Provider string const ( Istio Provider = "istio" + Kuma Provider = "kuma" + Nginx Provider = "nginx" ) // FleetStatus defines the observed state of the fleet diff --git a/pkg/fleet-manager/fleet_plugin.go b/pkg/fleet-manager/fleet_plugin.go index f1833d52..f8ca1395 100644 --- a/pkg/fleet-manager/fleet_plugin.go +++ b/pkg/fleet-manager/fleet_plugin.go @@ -40,6 +40,8 @@ const ( ) func (f *FleetManager) reconcilePlugins(ctx context.Context, fleet *fleetapi.Fleet, fleetClusters map[ClusterKey]*FleetCluster) (ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + log = log.WithValues("fleet", types.NamespacedName{Name: fleet.Name, Namespace: fleet.Namespace}) var resources kube.ResourceList if fleet.Spec.Plugin != nil { @@ -59,6 +61,7 @@ func (f *FleetManager) reconcilePlugins(ctx context.Context, fleet *fleetapi.Fle f.reconcileDistributedStoragePlugin, f.reconcileFlaggerPlugin, f.reconcileSubmarinerPlugin, + f.reconcileProviderPlugin, } resultsChannel := make(chan reconcileResult, len(funcs)) @@ -102,6 +105,7 @@ func (f *FleetManager) reconcilePlugins(ctx context.Context, fleet *fleetapi.Fle } } + log.Info("All plugin Resources succeed") return f.reconcilePluginResources(ctx, fleet, resources) } diff --git a/pkg/fleet-manager/fleet_plugin_provider.go b/pkg/fleet-manager/fleet_plugin_provider.go new file mode 100644 index 00000000..1b944a73 --- /dev/null +++ b/pkg/fleet-manager/fleet_plugin_provider.go @@ -0,0 +1,75 @@ +/* +Copyright Kurator Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fleet + +import ( + "context" + "time" + + "helm.sh/helm/v3/pkg/kube" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + + fleetapi "kurator.dev/kurator/pkg/apis/fleet/v1alpha1" + "kurator.dev/kurator/pkg/fleet-manager/plugin" + "kurator.dev/kurator/pkg/infra/util" +) + +// reconcileProviderPlugin reconciles the Provider plugin. +func (f *FleetManager) reconcileProviderPlugin(ctx context.Context, fleet *fleetapi.Fleet, fleetClusters map[ClusterKey]*FleetCluster) (kube.ResourceList, ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + + flaggerCfg := fleet.Spec.Plugin.Flagger + if flaggerCfg == nil || flaggerCfg.TrafficRoutingProvider == fleetapi.Istio { + // reconcilePluginResources will delete all resources if plugin is nil + return nil, ctrl.Result{}, nil + } + + fleetNN := types.NamespacedName{ + Namespace: fleet.Namespace, + Name: fleet.Name, + } + + fleetOwnerRef := ownerReference(fleet) + var resources kube.ResourceList + + for key, cluster := range fleetClusters { + b, err := plugin.RenderProvider(f.Manifests, fleetNN, fleetOwnerRef, plugin.KubeConfigSecretRef{ + Name: key.Name, + SecretName: cluster.Secret, + SecretKey: cluster.SecretKey, + }, flaggerCfg) + if err != nil { + return nil, ctrl.Result{}, err + } + + // apply provider helm resources + providerResources, err := util.PatchResources(b) + if err != nil { + return nil, ctrl.Result{}, err + } + resources = append(resources, providerResources...) + } + + log.V(4).Info("wait for provider helm release to be reconciled") + if !f.helmReleaseReady(ctx, fleet, resources) { + // wait for HelmRelease to be ready + return nil, ctrl.Result{ + // HelmRelease check interval is 1m, so we set 30s here + RequeueAfter: 30 * time.Second, + }, nil + } + + return resources, ctrl.Result{}, nil +} diff --git a/pkg/fleet-manager/manifests/plugins/kuma.yaml b/pkg/fleet-manager/manifests/plugins/kuma.yaml new file mode 100644 index 00000000..75c9d95c --- /dev/null +++ b/pkg/fleet-manager/manifests/plugins/kuma.yaml @@ -0,0 +1,5 @@ +type: default +repo: https://kumahq.github.io/charts +name: kuma +version: 2.7.3 +targetNamespace: kuma-system diff --git a/pkg/fleet-manager/manifests/plugins/nginx.yaml b/pkg/fleet-manager/manifests/plugins/nginx.yaml new file mode 100644 index 00000000..b50aac0e --- /dev/null +++ b/pkg/fleet-manager/manifests/plugins/nginx.yaml @@ -0,0 +1,12 @@ +type: default +repo: https://kubernetes.github.io/ingress-nginx +name: ingress-nginx +version: 4.x +targetNamespace: ingress-nginx +values: + controller: + metrics: + enabled: true + podAnnotations: + prometheus.io/scrape: true + prometheus.io/port: 10254 \ No newline at end of file diff --git a/pkg/fleet-manager/plugin/plugin.go b/pkg/fleet-manager/plugin/plugin.go index 8b8627c1..86d96ff0 100644 --- a/pkg/fleet-manager/plugin/plugin.go +++ b/pkg/fleet-manager/plugin/plugin.go @@ -61,6 +61,8 @@ const ( var ProviderNamespace = map[fleetv1a1.Provider]string{ "istio": "istio-system", + "kuma": "kuma-system", + "nginx": "ingress-nginx", } type GrafanaDataSource struct { @@ -401,6 +403,14 @@ func RenderFlagger( c.TargetNamespace = ProviderNamespace[flaggerConfig.TrafficRoutingProvider] values, err := toMap(flaggerConfig.ExtraArgs) + if flaggerConfig.TrafficRoutingProvider == fleetv1a1.Nginx { + values = transform.MergeMaps(values, map[string]interface{}{ + "prometheus": map[string]interface{}{ + "install": true, + }, + "meshProvider": "nginx", + }) + } if err != nil { return nil, err } @@ -416,6 +426,41 @@ func RenderFlagger( }) } +func RenderProvider( + fsys fs.FS, + fleetNN types.NamespacedName, + fleetRef *metav1.OwnerReference, + cluster KubeConfigSecretRef, + flaggerConfig *fleetv1a1.FlaggerConfig, +) ([]byte, error) { + name := string(flaggerConfig.TrafficRoutingProvider) + // get and merge the chart config + c, err := getFleetPluginChart(fsys, name) + if err != nil { + return nil, err + } + + values := map[string]interface{}{} + if providerConfig := flaggerConfig.ProviderConfig; providerConfig != nil { + mergeChartConfig(c, providerConfig.Chart) + values, err = toMap(providerConfig.ExtraArgs) + if err != nil { + return nil, err + } + } + c.TargetNamespace = ProviderNamespace[flaggerConfig.TrafficRoutingProvider] + + return renderFleetPlugin(fsys, FleetPluginConfig{ + Name: name, + Component: name, + Fleet: fleetNN, + Cluster: &cluster, + OwnerReference: fleetRef, + Chart: *c, + Values: values, + }) +} + func RenderRolloutTestloader( fsys fs.FS, fleetNN types.NamespacedName, From 9365d8d5550527b89aa1f161d2421bead9f7747f Mon Sep 17 00:00:00 2001 From: Gidi233 Date: Mon, 2 Sep 2024 20:21:14 +0800 Subject: [PATCH 02/10] Support the Gateway plugin in application Signed-off-by: Gidi233 --- .../rollout/Add_Gateway_Plugin_Support.md | 21 +- .../fleet-manager/templates/webhooks.yaml | 31 +++ pkg/apis/apps/v1alpha1/types.go | 39 +++- pkg/client/client.go | 4 + .../application/rollout_helper.go | 211 +++++++++++++++--- .../application/rollout_helper_test.go | 3 +- pkg/webhooks/application_webhook.go | 52 ++++- 7 files changed, 321 insertions(+), 40 deletions(-) diff --git a/docs/proposals/rollout/Add_Gateway_Plugin_Support.md b/docs/proposals/rollout/Add_Gateway_Plugin_Support.md index b6d7a34f..5fcb0002 100644 --- a/docs/proposals/rollout/Add_Gateway_Plugin_Support.md +++ b/docs/proposals/rollout/Add_Gateway_Plugin_Support.md @@ -42,31 +42,34 @@ We will delve into the API design required to support these configurations. The type TrafficRoutingConfig struct { ... // for NGINX - // The default created ingress is as follows, (replace app.example.com with your own domain, and change the path matching rules as needed) + // The default created ingress is as follows, (Fill in `host` with your own domain) + // ```yaml // apiVersion: networking.k8s.io/v1 // kind: Ingress // metadata: - // name: application.syncPolicies.rollout.name - // namespace: application.syncPolicies.rollout.namespace + // name: nginx + // namespace: application.syncPolicies.rollout.workload.namespace // labels: - // app: application.syncPolicies.rollout.name + // app: application.syncPolicies.rollout.ServiceName // annotations: // kubernetes.io/ingress.class: "nginx" // spec: // rules: - // - host: "app.example.com" + // - host: "" // http: // paths: // - pathType: Prefix // path: "/" // backend: // service: - // name: application.syncPolicies.rollout.name + // name: application.syncPolicies.rollout.ServiceName // port: - // number: 80 + // number: application.syncPolicies.rollout.port + // ``` // +optional - Ingress *ingressv1.IngressRule `json:"ingress,omitempty"` - // for Kuma + Host string `json:"host,omitempty"` + + // for kuma // Defaults to http // +optional Protocol string `json:"protocol,omitempty"` diff --git a/manifests/charts/fleet-manager/templates/webhooks.yaml b/manifests/charts/fleet-manager/templates/webhooks.yaml index b2c4fc76..1fc15926 100644 --- a/manifests/charts/fleet-manager/templates/webhooks.yaml +++ b/manifests/charts/fleet-manager/templates/webhooks.yaml @@ -28,3 +28,34 @@ webhooks: resources: - applications sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/kurator-serving-cert + creationTimestamp: null + name: fleet-manager-mutating-webhook-configuration +webhooks: + - admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: kurator-webhook-service-fleet + namespace: {{ .Release.Namespace }} + path: /mutate-apps-kurator-dev-v1alpha1-application # do not change this + failurePolicy: Fail + matchPolicy: Equivalent + name: mutation.application.apps.kurator.dev + rules: + - apiGroups: + - apps.kurator.dev + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - applications + sideEffects: None diff --git a/pkg/apis/apps/v1alpha1/types.go b/pkg/apis/apps/v1alpha1/types.go index 43bbbaf9..a1738b56 100644 --- a/pkg/apis/apps/v1alpha1/types.go +++ b/pkg/apis/apps/v1alpha1/types.go @@ -23,6 +23,7 @@ import ( kustomizev1beta2 "github.com/fluxcd/kustomize-controller/api/v1beta2" sourcev1beta2 "github.com/fluxcd/source-controller/api/v1beta2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + fleetapi "kurator.dev/kurator/pkg/apis/fleet/v1alpha1" ) // +genclient @@ -117,10 +118,9 @@ type RolloutConfig struct { TestLoader *bool `json:"testLoader,omitempty"` // TrafficRoutingProvider defines traffic routing provider. - // Kurator only supports istio for now. + // Kurator supports istio,kuma,nginx for now. // Other provider will be added later. - // +optional - TrafficRoutingProvider string `json:"trafficRoutingProvider,omitempty"` + TrafficRoutingProvider fleetapi.Provider `json:"trafficRoutingProvider"` // Workload specifies what workload to deploy the test to. // Workload of type Deployment or DaemonSet. @@ -239,6 +239,39 @@ type TrafficRoutingConfig struct { // +optional CorsPolicy *istiov1alpha3.CorsPolicy `json:"corsPolicy,omitempty"` + // for NGINX + // The default created ingress is as follows, (Fill in `host` with your own domain) + // ```yaml + // apiVersion: networking.k8s.io/v1 + // kind: Ingress + // metadata: + // name: nginx + // namespace: application.syncPolicies.rollout.workload.namespace + // labels: + // app: application.syncPolicies.rollout.ServiceName + // annotations: + // kubernetes.io/ingress.class: "nginx" + // spec: + // rules: + // - host: "" + // http: + // paths: + // - pathType: Prefix + // path: "/" + // backend: + // service: + // name: application.syncPolicies.rollout.ServiceName + // port: + // number: application.syncPolicies.rollout.port + // ``` + // +optional + Host string `json:"host,omitempty"` + + // for kuma + // Defaults to http + // +optional + Protocol string `json:"protocol,omitempty"` + // CanaryStrategy defines parameters for Canary Deployment. // Note: Kurator determines A/B Testing, Blue/Green Deployment, or Canary Deployment // based on the presence of content in the canaryStrategy field. diff --git a/pkg/client/client.go b/pkg/client/client.go index 2b445831..f68f41fb 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -33,6 +33,7 @@ import ( helmclient "helm.sh/helm/v3/pkg/kube" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + ingressv1 "k8s.io/api/networking/v1" crdclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -87,6 +88,9 @@ func NewClient(rest genericclioptions.RESTClientGetter) (*Client, error) { if err := appsv1.AddToScheme(scheme); err != nil { return nil, fmt.Errorf("failed to add appv1 api to scheme: %v", err) } + if err := ingressv1.AddToScheme(scheme); err != nil { + return nil, fmt.Errorf("failed to add ingress api to scheme: %v", err) + } // create controller-runtime client with scheme ctrlRuntimeClient, err := client.New(c, client.Options{Scheme: scheme}) if err != nil { diff --git a/pkg/fleet-manager/application/rollout_helper.go b/pkg/fleet-manager/application/rollout_helper.go index cbdff8fe..0c3d9376 100644 --- a/pkg/fleet-manager/application/rollout_helper.go +++ b/pkg/fleet-manager/application/rollout_helper.go @@ -19,12 +19,15 @@ package application import ( "context" "fmt" + "strings" "time" flaggerv1b1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" "github.com/pkg/errors" + "istio.io/istio/pkg/util/sets" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + ingressv1 "k8s.io/api/networking/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -43,13 +46,22 @@ import ( const ( // kurator rollout labels RolloutIdentifier = "kurator.dev/rollout" - sidecarInject = "istio-injection" - + istioInject = "istio-injection" + kumaInject = "kuma.io/sidecar-injection" // StatusSyncInterval specifies the interval for requeueing when synchronizing status. It determines how frequently the status should be checked and updated. StatusSyncInterval = 30 * time.Second currentClusterKind = "currentCluster" currentClusterName = "host" + // resources config + ingressAPIVersion = "networking.k8s.io/v1" + ingressKind = "Ingress" + ingressName = "nginx" + ingressLabelKey = "app" + ingressAnnotationKey = "kubernetes.io/ingress.class" + ingressAnnotationValue = "nginx" + + kumaAnnotation = "9898.service.kuma.io/protocol" ) func (a *ApplicationManager) fetchRolloutClusters(ctx context.Context, @@ -115,18 +127,39 @@ func (a *ApplicationManager) syncRolloutPolicyForCluster(ctx context.Context, annotation := map[string]string{ RolloutIdentifier: policyName, } + provider := rolloutPolicy.TrafficRoutingProvider for clusterKey, fleetCluster := range destinationClusters { fleetClusterClient := fleetCluster.Client.CtrlRuntimeClient() - - // If the trafficRoutingProvider is Istio, add the sidecar injection label to the workload's namespace. - if rolloutPolicy.TrafficRoutingProvider == "istio" { - err := enableIstioSidecarInjection(ctx, fleetClusterClient, rolloutPolicy.Workload.Namespace) + switch provider { + // If the trafficRoutingProvider is Istio or Kuma, add the sidecar injection label/Annotations to the workload's namespace. + case fleetapi.Istio, fleetapi.Kuma: + err := enableSidecarInjection(ctx, fleetClusterClient, rolloutPolicy.Workload.Namespace, provider, annotation) if err != nil { - return ctrl.Result{}, errors.Wrapf(err, "failed to set namespace %s istio-injection enable", rolloutPolicy.Workload.Namespace) + return ctrl.Result{}, errors.Wrapf(err, "failed to set namespace %s %s's Inject enable", rolloutPolicy.Workload.Namespace, provider) } - } + case fleetapi.Nginx: + // Canaries in the same namespace reference the same ingress + ingress := &ingressv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: ingressName, + Namespace: rolloutPolicy.Workload.Namespace, + }, + } + result, err := controllerutil.CreateOrUpdate(ctx, fleetClusterClient, ingress, func() error { + ingress.SetAnnotations(annotation) + renderIngress(ingress, rolloutPolicy) + return nil + }) + if err != nil { + return ctrl.Result{}, errors.Wrapf(err, "failed to operate ingress") + } + log.Info("sync nginx", "result:", result) + default: + return ctrl.Result{}, errors.Errorf("unknown provider type %s", provider) + } + log.Info("pre-operation of operating canary success") // if delete private testloader when rollout polity has changed if rolloutPolicy.TestLoader == nil || !*rolloutPolicy.TestLoader { testloaderDeploy := &appsv1.Deployment{} @@ -248,6 +281,10 @@ func (a *ApplicationManager) deleteResourcesInMemberClusters(ctx context.Context return errors.Wrapf(err, "failed to fetch destination clusters when delete rollout resource") } + namespacedName := types.NamespacedName{ + Namespace: rolloutPolicy.Workload.Namespace, + Name: rolloutPolicy.Workload.Namespace, + } serviceNamespaceName := types.NamespacedName{ Namespace: rolloutPolicy.Workload.Namespace, Name: rolloutPolicy.ServiceName, @@ -270,6 +307,14 @@ func (a *ApplicationManager) deleteResourcesInMemberClusters(ctx context.Context } for _, cluster := range destinationClusters { newClient := cluster.Client.CtrlRuntimeClient() + + ns := &corev1.Namespace{} + if err := deleteResourceCreatedByKurator(ctx, namespacedName, newClient, ns); err != nil { + return errors.Wrapf(err, "failed to delete namespace") + } + if err := deleteIngressCreatedByKurator(ctx, newClient, rolloutPolicy); err != nil { + return errors.Wrapf(err, "failed to delete ingress") + } testloaderDeploy := &appsv1.Deployment{} if err := deleteResourceCreatedByKurator(ctx, testloaderNamespaceName, newClient, testloaderDeploy); err != nil { return errors.Wrapf(err, "failed to delete testloader deployment") @@ -291,7 +336,7 @@ func (a *ApplicationManager) deleteResourcesInMemberClusters(ctx context.Context return nil } -func enableIstioSidecarInjection(ctx context.Context, kubeClient client.Client, namespace string) error { +func enableSidecarInjection(ctx context.Context, kubeClient client.Client, namespace string, provider fleetapi.Provider, annotation map[string]string) error { log := ctrl.LoggerFrom(ctx) ns := &corev1.Namespace{} @@ -303,9 +348,13 @@ func enableIstioSidecarInjection(ctx context.Context, kubeClient client.Client, // if no found, create a namespace if apierrors.IsNotFound(err) { ns.SetName(namespace) - ns.SetLabels(map[string]string{ - sidecarInject: "enabled", - }) + switch provider { + case fleetapi.Kuma: + annotation[kumaInject] = "enabled" + case fleetapi.Istio: + ns.SetLabels(map[string]string{istioInject: "enabled"}) + } + ns.SetAnnotations(annotation) if createErr := kubeClient.Create(ctx, ns); createErr != nil { return errors.Wrapf(createErr, "failed to create namespace %s", namespacedName.Namespace) } @@ -314,8 +363,14 @@ func enableIstioSidecarInjection(ctx context.Context, kubeClient client.Client, return err } } else { - ns := addLabels(ns, sidecarInject, "enabled") - if updateErr := kubeClient.Update(ctx, ns); updateErr != nil { + var newNs client.Object + switch provider { + case fleetapi.Kuma: + newNs = addAnnotations(ns, kumaInject, "enabled") + case fleetapi.Istio: + newNs = addLabels(ns, istioInject, "enabled") + } + if updateErr := kubeClient.Update(ctx, newNs); updateErr != nil { return errors.Wrapf(updateErr, "failed to update namespace %s", namespacedName.Namespace) } } @@ -397,6 +452,79 @@ func deleteResourceCreatedByKurator(ctx context.Context, namespaceName types.Nam return nil } +func deleteIngressCreatedByKurator(ctx context.Context, kubeClient client.Client, rollout *applicationapi.RolloutConfig) error { + ingress := &ingressv1.Ingress{} + namespaceName := types.NamespacedName{ + Namespace: rollout.Workload.Namespace, + Name: ingressName, + } + if err := kubeClient.Get(ctx, namespaceName, ingress); err != nil { + if !apierrors.IsNotFound(err) { + return errors.Wrapf(err, "failed to get ingress %s in %s", namespaceName.Name, namespaceName.Namespace) + } + } else { + // verify if the ingress were created by kurator + annotations := ingress.GetAnnotations() + if _, exist := annotations[RolloutIdentifier]; exist { + labels := ingress.GetLabels() + if set := sets.New(strings.Split(labels[ingressLabelKey], ",")...); set.Contains(rollout.ServiceName) { + if set.Len() == 1 { + if deleteErr := kubeClient.Delete(ctx, ingress); deleteErr != nil && !apierrors.IsNotFound(deleteErr) { + return errors.Wrapf(deleteErr, "failed to Delete ingress %s in %s", namespaceName.Name, namespaceName.Namespace) + } + } else { + newRules := make([]ingressv1.IngressRule, 0) + for _, rule := range ingress.Spec.Rules { + if rule.Host != rollout.RolloutPolicy.TrafficRouting.Host { + newRules = append(newRules, rule) + } + } + ingress.Spec.Rules = newRules + labels[ingressLabelKey] = strings.Join(set.Delete(rollout.ServiceName).UnsortedList(), ",") + ingress.SetLabels(labels) + if err := kubeClient.Update(ctx, ingress); err != nil { + return errors.Wrapf(err, "failed to Update ingress %s in %s", namespaceName.Name, namespaceName.Namespace) + } + } + } + } + } + return nil +} + +// create/update ingress configuration +func renderIngress(ingress *ingressv1.Ingress, rollout *applicationapi.RolloutConfig) { + + if labels := ingress.GetLabels(); labels == nil || labels[ingressLabelKey] == "" { + ingress.SetLabels(map[string]string{ingressLabelKey: rollout.ServiceName}) + } else { + labels[ingressLabelKey] = strings.Join(sets.New(strings.Split(labels[ingressLabelKey], ",")...).Insert(rollout.ServiceName).UnsortedList(), ",") + ingress.SetLabels(labels) + } + addAnnotations(ingress, ingressAnnotationKey, ingressAnnotationValue) + Prefix := ingressv1.PathTypePrefix + rule := ingressv1.IngressRule{ + Host: rollout.RolloutPolicy.TrafficRouting.Host, + IngressRuleValue: ingressv1.IngressRuleValue{ + HTTP: &ingressv1.HTTPIngressRuleValue{ + Paths: []ingressv1.HTTPIngressPath{{ + PathType: &Prefix, + Path: "/", + Backend: ingressv1.IngressBackend{ + Service: &ingressv1.IngressServiceBackend{ + Name: rollout.ServiceName, + Port: ingressv1.ServiceBackendPort{ + Number: rollout.Port, + }, + }, + }, + }}, + }, + }, + } + ingress.Spec.Rules = append(ingress.Spec.Rules, rule) +} + // create/update canary configuration func renderCanary(rolloutPolicy applicationapi.RolloutConfig, canaryInCluster *flaggerv1b1.Canary) *flaggerv1b1.Canary { canaryInCluster.ObjectMeta.Namespace = rolloutPolicy.Workload.Namespace @@ -404,7 +532,7 @@ func renderCanary(rolloutPolicy applicationapi.RolloutConfig, canaryInCluster *f canaryInCluster.TypeMeta.Kind = "Canary" canaryInCluster.TypeMeta.APIVersion = "flagger.app/v1beta1" canaryInCluster.Spec = flaggerv1b1.CanarySpec{ - Provider: rolloutPolicy.TrafficRoutingProvider, + Provider: string(rolloutPolicy.TrafficRoutingProvider), TargetRef: flaggerv1b1.LocalObjectReference{ APIVersion: rolloutPolicy.Workload.APIVersion, Kind: rolloutPolicy.Workload.Kind, @@ -415,7 +543,16 @@ func renderCanary(rolloutPolicy applicationapi.RolloutConfig, canaryInCluster *f RevertOnDeletion: rolloutPolicy.RolloutPolicy.RevertOnDeletion, Suspend: rolloutPolicy.RolloutPolicy.Suspend, } - + switch rolloutPolicy.TrafficRoutingProvider { + case fleetapi.Nginx: + canaryInCluster.Spec.IngressRef = &flaggerv1b1.LocalObjectReference{ + APIVersion: ingressAPIVersion, + Kind: ingressKind, + Name: ingressName, + } + case fleetapi.Kuma: + canaryInCluster.SetAnnotations(map[string]string{"kuma.io/mesh": "default"}) + } return canaryInCluster } @@ -425,17 +562,22 @@ func renderCanaryService(rolloutPolicy applicationapi.RolloutConfig, service *co } ports := service.Spec.Ports canaryService := &flaggerv1b1.CanaryService{ - Name: rolloutPolicy.ServiceName, - Port: rolloutPolicy.Port, - Gateways: rolloutPolicy.RolloutPolicy.TrafficRouting.Gateways, - Hosts: rolloutPolicy.RolloutPolicy.TrafficRouting.Hosts, - Retries: rolloutPolicy.RolloutPolicy.TrafficRouting.Retries, - Headers: rolloutPolicy.RolloutPolicy.TrafficRouting.Headers, - CorsPolicy: rolloutPolicy.RolloutPolicy.TrafficRouting.CorsPolicy, - Primary: (*flaggerv1b1.CustomMetadata)(rolloutPolicy.Primary), - Canary: (*flaggerv1b1.CustomMetadata)(rolloutPolicy.Preview), + Name: rolloutPolicy.ServiceName, + Port: rolloutPolicy.Port, + } + switch rolloutPolicy.TrafficRoutingProvider { + case fleetapi.Istio: + canaryService.Gateways = rolloutPolicy.RolloutPolicy.TrafficRouting.Gateways + canaryService.Hosts = rolloutPolicy.RolloutPolicy.TrafficRouting.Hosts + canaryService.Retries = rolloutPolicy.RolloutPolicy.TrafficRouting.Retries + canaryService.Headers = rolloutPolicy.RolloutPolicy.TrafficRouting.Headers + canaryService.CorsPolicy = rolloutPolicy.RolloutPolicy.TrafficRouting.CorsPolicy + case fleetapi.Kuma: + annotations := &flaggerv1b1.CustomMetadata{Annotations: map[string]string{kumaAnnotation: rolloutPolicy.RolloutPolicy.TrafficRouting.Protocol}} + canaryService.Apex = annotations + canaryService.Canary = annotations + canaryService.Primary = annotations } - Timeout := fmt.Sprintf("%d", rolloutPolicy.RolloutPolicy.TrafficRouting.TimeoutSeconds) + "s" canaryService.Timeout = Timeout @@ -569,6 +711,23 @@ func addLabels(obj client.Object, key, value string) client.Object { return obj } +func addAnnotations(obj client.Object, keysAndValues ...string) client.Object { + annotations := obj.GetAnnotations() + // prevent nil pointer panic + if annotations == nil { + annotations = make(map[string]string) + } + for i := 0; i < len(keysAndValues); i += 2 { + if i+1 < len(keysAndValues) { + annotations[keysAndValues[i]] = keysAndValues[i+1] + } else { + annotations[keysAndValues[i]] = "" + } + } + obj.SetAnnotations(annotations) + return obj +} + func mergeMap(map1, map2 map[string]*applicationapi.RolloutStatus) map[string]*applicationapi.RolloutStatus { for name, rolloutStatus := range map1 { if _, exist := map2[name]; !exist { diff --git a/pkg/fleet-manager/application/rollout_helper_test.go b/pkg/fleet-manager/application/rollout_helper_test.go index 43bd83e5..e24a078f 100644 --- a/pkg/fleet-manager/application/rollout_helper_test.go +++ b/pkg/fleet-manager/application/rollout_helper_test.go @@ -29,6 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" applicationapi "kurator.dev/kurator/pkg/apis/apps/v1alpha1" + fleetapi "kurator.dev/kurator/pkg/apis/fleet/v1alpha1" ) func generateRolloutPolicy(installPrivateTestloader *bool) applicationapi.RolloutConfig { @@ -39,7 +40,7 @@ func generateRolloutPolicy(installPrivateTestloader *bool) applicationapi.Rollou rolloutPolicy := applicationapi.RolloutConfig{ TestLoader: installPrivateTestloader, - TrafficRoutingProvider: "istio", + TrafficRoutingProvider: fleetapi.Istio, Workload: &applicationapi.CrossNamespaceObjectReference{ APIVersion: "appv1/deployment", Kind: "Deployment", diff --git a/pkg/webhooks/application_webhook.go b/pkg/webhooks/application_webhook.go index af01abb3..889361dd 100644 --- a/pkg/webhooks/application_webhook.go +++ b/pkg/webhooks/application_webhook.go @@ -22,6 +22,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -29,9 +30,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "kurator.dev/kurator/pkg/apis/apps/v1alpha1" + fleetapi "kurator.dev/kurator/pkg/apis/fleet/v1alpha1" ) var _ webhook.CustomValidator = &ApplicationWebhook{} +var _ webhook.CustomDefaulter = &ApplicationWebhook{} type ApplicationWebhook struct { Client client.Reader @@ -41,11 +44,14 @@ func (wh *ApplicationWebhook) SetupWebhookWithManager(mgr ctrl.Manager) error { return ctrl.NewWebhookManagedBy(mgr). For(&v1alpha1.Application{}). WithValidator(wh). + WithDefaulter(wh). Complete() } -func (wh *ApplicationWebhook) ValidateCreate(_ context.Context, obj runtime.Object) (admission.Warnings, error) { +func (wh *ApplicationWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { in, ok := obj.(*v1alpha1.Application) + log := ctrl.LoggerFrom(ctx) + log.Info("All field Validate succeed") if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected a Application but got a %T", obj)) } @@ -131,3 +137,47 @@ func (wh *ApplicationWebhook) ValidateUpdate(_ context.Context, oldObj, newObj r func (wh *ApplicationWebhook) ValidateDelete(_ context.Context, obj runtime.Object) (admission.Warnings, error) { return nil, nil } + +func (wh *ApplicationWebhook) Default(ctx context.Context, obj runtime.Object) error { + log := ctrl.LoggerFrom(ctx) + log.Info("setting default") + app, ok := obj.(*v1alpha1.Application) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a Application but got a %T", obj)) + } + if err := defaultApp(ctx, app); err != nil { + return err + } + return nil +} + +func defaultApp(ctx context.Context, app *v1alpha1.Application) error { + log := ctrl.LoggerFrom(ctx) + log = log.WithValues("application", types.NamespacedName{Name: app.Name, Namespace: app.Namespace}) + if err := defaultSyncPolicies(ctx, app.Spec.SyncPolicies); err != nil { + return err + } + log.Info("All field set default") + return nil +} + +func defaultSyncPolicies(ctx context.Context, SyncPolicies []*v1alpha1.ApplicationSyncPolicy) error { + log := ctrl.LoggerFrom(ctx) + log = log.WithValues("application") + for i, syncPoliciy := range SyncPolicies { + if syncPoliciy.Rollout == nil { + continue + } + switch syncPoliciy.Rollout.TrafficRoutingProvider { + case fleetapi.Nginx: + if syncPoliciy.Rollout.RolloutPolicy.TrafficRouting.Host == "" { + return apierrors.NewBadRequest(fmt.Sprintf("expected application.syncPolicies.Rollout.RolloutPolicy.TrafficRouting.Host in index %d", i)) + } + case fleetapi.Kuma: + if syncPoliciy.Rollout.RolloutPolicy.TrafficRouting.Protocol == "" { + syncPoliciy.Rollout.RolloutPolicy.TrafficRouting.Protocol = "http" + } + } + } + return nil +} From a5cd58aa7c3d3ebb91afaa6e2f757d4276d71ea1 Mon Sep 17 00:00:00 2001 From: Gidi233 Date: Mon, 2 Sep 2024 20:21:44 +0800 Subject: [PATCH 03/10] Add example Signed-off-by: Gidi233 --- examples/rollout/ab-testingNginx.yaml | 82 ++++++++++++++++++++ examples/rollout/blue_greenNginx.yaml | 75 ++++++++++++++++++ examples/rollout/canaryKuma.yaml | 76 ++++++++++++++++++ examples/rollout/canaryNginx.yaml | 77 ++++++++++++++++++ examples/rollout/canaryWithCustomMetric.yaml | 2 +- 5 files changed, 311 insertions(+), 1 deletion(-) create mode 100644 examples/rollout/ab-testingNginx.yaml create mode 100644 examples/rollout/blue_greenNginx.yaml create mode 100644 examples/rollout/canaryKuma.yaml create mode 100644 examples/rollout/canaryNginx.yaml diff --git a/examples/rollout/ab-testingNginx.yaml b/examples/rollout/ab-testingNginx.yaml new file mode 100644 index 00000000..744ccb0a --- /dev/null +++ b/examples/rollout/ab-testingNginx.yaml @@ -0,0 +1,82 @@ +apiVersion: apps.kurator.dev/v1alpha1 +kind: Application +metadata: + name: abtesting-nginx-demo + namespace: default +spec: + source: + gitRepository: + interval: 3m0s + ref: + branch: master + timeout: 1m0s + url: https://github.com/stefanprodan/podinfo + syncPolicies: + - destination: + fleet: quickstart + kustomization: + interval: 0s + path: ./deploy/webapp + prune: true + timeout: 2m0s + rollout: + testLoader: true + trafficRoutingProvider: nginx + workload: + apiVersion: apps/v1 + name: backend + kind: Deployment + namespace: webapp + serviceName: backend + port: 9898 + rolloutPolicy: + trafficRouting: + analysisTimes: 3 + timeoutSeconds: 60 + host: "app.example.com" + match: + - headers: + x-canary: + exact: "insider" + - headers: + cookie: + exact: "canary" + trafficAnalysis: + checkIntervalSeconds: 90 + checkFailedTimes: 2 + metrics: + - name: my-metric + intervalSeconds: 90 + thresholdRange: + min: 99 + customMetric: + provider: + type: prometheus + address: http://ingress-nginx-flagger-kurator-member-prometheus.ingress-nginx:9090 + query: | + sum( + rate( + http_requests_total{ + status!~"5.*" + }[{{ interval }}] + ) + ) + / + sum( + rate( + http_requests_total[{{ interval }}] + ) + ) * 100 + webhooks: + timeoutSeconds: 60 + command: + - "hey -z 1m -q 10 -c 2 http://app.example.com/" + rolloutTimeoutSeconds: 600 + - destination: + fleet: quickstart + kustomization: + targetNamespace: default + interval: 5m0s + path: ./kustomize + prune: true + timeout: 2m0s diff --git a/examples/rollout/blue_greenNginx.yaml b/examples/rollout/blue_greenNginx.yaml new file mode 100644 index 00000000..8cb75610 --- /dev/null +++ b/examples/rollout/blue_greenNginx.yaml @@ -0,0 +1,75 @@ +apiVersion: apps.kurator.dev/v1alpha1 +kind: Application +metadata: + name: abtesting-nginx-demo + namespace: default +spec: + source: + gitRepository: + interval: 3m0s + ref: + branch: master + timeout: 1m0s + url: https://github.com/stefanprodan/podinfo + syncPolicies: + - destination: + fleet: quickstart + kustomization: + interval: 0s + path: ./deploy/webapp + prune: true + timeout: 2m0s + rollout: + testLoader: true + trafficRoutingProvider: nginx + workload: + apiVersion: apps/v1 + name: backend + kind: Deployment + namespace: webapp + serviceName: backend + port: 9898 + rolloutPolicy: + trafficRouting: + analysisTimes: 3 + timeoutSeconds: 60 + host: "app.example.com" + trafficAnalysis: + checkIntervalSeconds: 90 + checkFailedTimes: 2 + metrics: + - name: my-metric + intervalSeconds: 90 + thresholdRange: + min: 99 + customMetric: + provider: + type: prometheus + address: http://ingress-nginx-flagger-kurator-member-prometheus.ingress-nginx:9090 + query: | + sum( + rate( + http_requests_total{ + status!~"5.*" + }[{{ interval }}] + ) + ) + / + sum( + rate( + http_requests_total[{{ interval }}] + ) + ) * 100 + webhooks: + timeoutSeconds: 60 + command: + - "hey -z 1m -q 10 -c 2 http://app.example.com/" + rolloutTimeoutSeconds: 600 + - destination: + fleet: quickstart + kustomization: + targetNamespace: default + interval: 5m0s + path: ./kustomize + prune: true + timeout: 2m0s diff --git a/examples/rollout/canaryKuma.yaml b/examples/rollout/canaryKuma.yaml new file mode 100644 index 00000000..1fb50867 --- /dev/null +++ b/examples/rollout/canaryKuma.yaml @@ -0,0 +1,76 @@ +apiVersion: apps.kurator.dev/v1alpha1 +kind: Application +metadata: + name: rollout-demo + namespace: default +spec: + source: + gitRepository: + interval: 3m0s + ref: + branch: master + timeout: 1m0s + url: https://github.com/stefanprodan/podinfo + syncPolicies: + - destination: + fleet: quickstart + kustomization: + interval: 0s + path: ./deploy/webapp + prune: true + timeout: 2m0s + rollout: + testLoader: true + trafficRoutingProvider: kuma + workload: + apiVersion: apps/v1 + name: backend + kind: Deployment + namespace: webapp + serviceName: backend + port: 9898 + rolloutPolicy: + trafficRouting: + timeoutSeconds: 60 + canaryStrategy: + maxWeight: 50 + stepWeight: 10 + trafficAnalysis: + checkIntervalSeconds: 90 + checkFailedTimes: 2 + metrics: + - name: my-metric + intervalSeconds: 90 + thresholdRange: + min: 99 + customMetric: + provider: + type: prometheus + address: http://prometheus-server.mesh-observability:80 + query: | + sum( + rate( + http_requests_total{ + status!~"5.*" + }[{{ interval }}] + ) + ) + / + sum( + rate( + http_requests_total[{{ interval }}] + ) + ) * 100 + webhooks: + timeoutSeconds: 60 + command: + - "hey -z 1m -q 10 -c 2 http://app.example.com/" + rolloutTimeoutSeconds: 600 + - destination: + fleet: quickstart + kustomization: + targetNamespace: default + interval: 5m0s + path: ./kustomize + prune: true + timeout: 2m0s \ No newline at end of file diff --git a/examples/rollout/canaryNginx.yaml b/examples/rollout/canaryNginx.yaml new file mode 100644 index 00000000..a3f45685 --- /dev/null +++ b/examples/rollout/canaryNginx.yaml @@ -0,0 +1,77 @@ +apiVersion: apps.kurator.dev/v1alpha1 +kind: Application +metadata: + name: rollout-demo + namespace: default +spec: + source: + gitRepository: + interval: 3m0s + ref: + branch: master + timeout: 1m0s + url: https://github.com/stefanprodan/podinfo + syncPolicies: + - destination: + fleet: quickstart + kustomization: + interval: 0s + path: ./deploy/webapp + prune: true + timeout: 2m0s + rollout: + testLoader: true + trafficRoutingProvider: nginx + workload: + apiVersion: apps/v1 + name: backend + kind: Deployment + namespace: webapp + serviceName: backend + port: 9898 + rolloutPolicy: + trafficRouting: + timeoutSeconds: 60 + canaryStrategy: + maxWeight: 50 + stepWeight: 10 + host: "app.example.com" + trafficAnalysis: + checkIntervalSeconds: 90 + checkFailedTimes: 2 + metrics: + - name: my-metric + intervalSeconds: 90 + thresholdRange: + min: 99 + customMetric: + provider: + type: prometheus + address: http://ingress-nginx-flagger-kurator-member-prometheus.ingress-nginx:9090 + query: | + sum( + rate( + http_requests_total{ + status!~"5.*" + }[{{ interval }}] + ) + ) + / + sum( + rate( + http_requests_total[{{ interval }}] + ) + ) * 100 + webhooks: + timeoutSeconds: 60 + command: + - "hey -z 1m -q 10 -c 2 http://app.example.com/" + rolloutTimeoutSeconds: 600 + - destination: + fleet: quickstart + kustomization: + targetNamespace: default + interval: 5m0s + path: ./kustomize + prune: true + timeout: 2m0s \ No newline at end of file diff --git a/examples/rollout/canaryWithCustomMetric.yaml b/examples/rollout/canaryWithCustomMetric.yaml index e30ba67d..4d3f87c7 100644 --- a/examples/rollout/canaryWithCustomMetric.yaml +++ b/examples/rollout/canaryWithCustomMetric.yaml @@ -54,7 +54,7 @@ spec: customMetric: provider: type: prometheus - address: http://flagger-prometheus.ingress-nginx:9090 + address: http://prometheus.istio-system:9090 query: | sum( rate( From 54f70cce6662b975b2046c4eb9f503ba7aace88d Mon Sep 17 00:00:00 2001 From: Gidi233 Date: Mon, 2 Sep 2024 21:01:28 +0800 Subject: [PATCH 04/10] Add gen Signed-off-by: Gidi233 --- .../en/references/apps_v1alpha1_types.html | 53 ++++++++++++++- .../en/references/fleet_v1alpha1_types.html | 68 ++++++++++++++++++- .../crds/apps.kurator.dev_applications.yaml | 36 +++++++++- .../crds/fleet.kurator.dev_fleets.yaml | 35 ++++++++++ .../fleet/v1alpha1/zz_generated.deepcopy.go | 27 ++++++++ 5 files changed, 214 insertions(+), 5 deletions(-) diff --git a/docs/content/en/references/apps_v1alpha1_types.html b/docs/content/en/references/apps_v1alpha1_types.html index f8596a8c..1d309e91 100644 --- a/docs/content/en/references/apps_v1alpha1_types.html +++ b/docs/content/en/references/apps_v1alpha1_types.html @@ -1707,13 +1707,12 @@

RolloutConfig trafficRoutingProvider
-string +Kurator fleet/v1alpha1.Provider -(Optional)

TrafficRoutingProvider defines traffic routing provider. -Kurator only supports istio for now. +Kurator supports istio,kuma,nginx for now. Other provider will be added later.

@@ -2213,6 +2212,54 @@

TrafficRoutingConfig +host
+ +string + + + +(Optional) +

for NGINX +The default created ingress is as follows, (Fill in host with your own domain)

+
apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+name: nginx
+namespace: application.syncPolicies.rollout.workload.namespace
+labels:
+app: application.syncPolicies.rollout.ServiceName
+annotations:
+kubernetes.io/ingress.class: "nginx"
+spec:
+rules:
+- host: ""
+http:
+paths:
+- pathType: Prefix
+path: "/"
+backend:
+service:
+name: application.syncPolicies.rollout.ServiceName
+port:
+number: application.syncPolicies.rollout.port
+
+ + + + +protocol
+ +string + + + +(Optional) +

for kuma +Defaults to http

+ + + + canaryStrategy
diff --git a/docs/content/en/references/fleet_v1alpha1_types.html b/docs/content/en/references/fleet_v1alpha1_types.html index 83843a4b..51b54163 100644 --- a/docs/content/en/references/fleet_v1alpha1_types.html +++ b/docs/content/en/references/fleet_v1alpha1_types.html @@ -338,6 +338,7 @@

ChartConfig

(Appears on: BackupConfig, +Config, DistributedStorageConfig, FlaggerConfig, GrafanaConfig, @@ -399,6 +400,56 @@

ChartConfig +

Config +

+

+(Appears on: +FlaggerConfig) +

+
+
+ + + + + + + + + + + + + + + + + +
FieldDescription
+chart
+ + +ChartConfig + + +		 +(Optional) +

Chart defines the helm chart config of the TrafficRoutingProvider. +default value is in ./pkg/fleet-manager/manifests/plugins/

+
+extraArgs
+ + +Kubernetes /apiextensions/v1.JSON + + +		 +(Optional) +

ExtraArgs is the set of extra arguments for TrafficRoutingProvider’s chart. +You can pass in values according to your needs.

+
+
+

Device

@@ -665,10 +716,11 @@

FlaggerConfig -(Optional)

TrafficRoutingProvider defines traffic routing provider. And Kurator will install flagger in trafficRoutingProvider’s namespace For example, If you use istio as a provider, flager will be installed in istio’s namespace istio-system. +And if you use istio as a provider, you need to install it manually. +Otherwise, you can configure it in ProviderConfig (or use the default configuration) and Kurator will automatically deploy it. Other provider will be added later.

@@ -685,6 +737,20 @@

FlaggerConfig you can also specify a private testloader in the Application.Spec.SyncPolicies.Rollout.TestLoader

+ + +Config
+ + +Config + + + + +(Optional) +

ProviderConfig defines the configuration for the TrafficRoutingProvider.

+ + diff --git a/manifests/charts/fleet-manager/crds/apps.kurator.dev_applications.yaml b/manifests/charts/fleet-manager/crds/apps.kurator.dev_applications.yaml index ee74280a..7496ff53 100644 --- a/manifests/charts/fleet-manager/crds/apps.kurator.dev_applications.yaml +++ b/manifests/charts/fleet-manager/crds/apps.kurator.dev_applications.yaml @@ -1618,6 +1618,34 @@ spec: type: object type: object type: object + host: + description: |- + for NGINX + The default created ingress is as follows, (Fill in `host` with your own domain) + ```yaml + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: nginx + namespace: application.syncPolicies.rollout.workload.namespace + labels: + app: application.syncPolicies.rollout.ServiceName + annotations: + kubernetes.io/ingress.class: "nginx" + spec: + rules: + - host: "" + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: application.syncPolicies.rollout.ServiceName + port: + number: application.syncPolicies.rollout.port + ``` + type: string hosts: description: Defaults to the RolloutConfig.ServiceName items: @@ -1923,6 +1951,11 @@ spec: type: object type: object type: array + protocol: + description: |- + for kuma + Defaults to http + type: string retries: description: |- Retries describes the retry policy to use when a HTTP request fails. @@ -1978,7 +2011,7 @@ spec: trafficRoutingProvider: description: |- TrafficRoutingProvider defines traffic routing provider. - Kurator only supports istio for now. + Kurator supports istio,kuma,nginx for now. Other provider will be added later. type: string workload: @@ -2005,6 +2038,7 @@ spec: - port - rolloutPolicy - serviceName + - trafficRoutingProvider - workload type: object type: object diff --git a/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml b/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml index a9d39296..2ec945f6 100644 --- a/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml +++ b/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml @@ -2465,6 +2465,37 @@ spec: description: Flagger defines the configuration for the kurator rollout engine. properties: + Config: + description: ProviderConfig defines the configuration for + the TrafficRoutingProvider. + properties: + chart: + description: |- + Chart defines the helm chart config of the TrafficRoutingProvider. + default value is in ./pkg/fleet-manager/manifests/plugins/ + properties: + name: + description: |- + Name defines the name of the chart. + Default value depends on the kind of the component. + type: string + repository: + description: |- + Repository defines the repository of chart. + Default value depends on the kind of the component. + type: string + version: + description: |- + Version defines the version of the chart. + Default value depends on the kind of the component. + type: string + type: object + extraArgs: + description: |- + ExtraArgs is the set of extra arguments for TrafficRoutingProvider's chart. + You can pass in values according to your needs. + x-kubernetes-preserve-unknown-fields: true + type: object chart: description: |- Chart defines the helm chart config of the flagger. @@ -2517,8 +2548,12 @@ spec: TrafficRoutingProvider defines traffic routing provider. And Kurator will install flagger in trafficRoutingProvider's namespace For example, If you use `istio` as a provider, flager will be installed in istio's namespace `istio-system`. + And if you use `istio` as a provider, you need to install it manually. + Otherwise, you can configure it in ProviderConfig (or use the default configuration) and Kurator will automatically deploy it. Other provider will be added later. type: string + required: + - trafficRoutingProvider type: object grafana: description: Grafana defines the configuration for the grafana diff --git a/pkg/apis/fleet/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/fleet/v1alpha1/zz_generated.deepcopy.go index e3c14874..a25d4022 100644 --- a/pkg/apis/fleet/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/fleet/v1alpha1/zz_generated.deepcopy.go @@ -105,6 +105,28 @@ func (in *ChartConfig) DeepCopy() *ChartConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Config) DeepCopyInto(out *Config) { + *out = *in + if in.Chart != nil { + in, out := &in.Chart, &out.Chart + *out = new(ChartConfig) + **out = **in + } + in.ExtraArgs.DeepCopyInto(&out.ExtraArgs) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Config. +func (in *Config) DeepCopy() *Config { + if in == nil { + return nil + } + out := new(Config) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Device) DeepCopyInto(out *Device) { *out = *in @@ -220,6 +242,11 @@ func (in *FlaggerConfig) DeepCopyInto(out *FlaggerConfig) { **out = **in } in.ExtraArgs.DeepCopyInto(&out.ExtraArgs) + if in.ProviderConfig != nil { + in, out := &in.ProviderConfig, &out.ProviderConfig + *out = new(Config) + (*in).DeepCopyInto(*out) + } return } From 2897a444a0a70ec48b1280a34085612bb6633d30 Mon Sep 17 00:00:00 2001 From: Gidi233 Date: Mon, 2 Sep 2024 22:21:23 +0800 Subject: [PATCH 05/10] fix warning and add log Signed-off-by: Gidi233 --- .../en/references/fleet_v1alpha1_types.html | 2 +- .../crds/fleet.kurator.dev_fleets.yaml | 60 +++++++++---------- pkg/apis/fleet/v1alpha1/types.go | 2 +- .../application/rollout_helper.go | 1 - pkg/webhooks/application_webhook.go | 1 + 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/docs/content/en/references/fleet_v1alpha1_types.html b/docs/content/en/references/fleet_v1alpha1_types.html index 51b54163..8b49ea92 100644 --- a/docs/content/en/references/fleet_v1alpha1_types.html +++ b/docs/content/en/references/fleet_v1alpha1_types.html @@ -739,7 +739,7 @@

FlaggerConfig -Config
+config
Config diff --git a/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml b/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml index 2ec945f6..b20c86ae 100644 --- a/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml +++ b/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml @@ -2465,7 +2465,36 @@ spec: description: Flagger defines the configuration for the kurator rollout engine. properties: - Config: + chart: + description: |- + Chart defines the helm chart config of the flagger. + default value is + + + ```yaml + chart: + repository: oci://ghcr.io/fluxcd/charts + name: flagger + version: 1.x + ``` + properties: + name: + description: |- + Name defines the name of the chart. + Default value depends on the kind of the component. + type: string + repository: + description: |- + Repository defines the repository of chart. + Default value depends on the kind of the component. + type: string + version: + description: |- + Version defines the version of the chart. + Default value depends on the kind of the component. + type: string + type: object + config: description: ProviderConfig defines the configuration for the TrafficRoutingProvider. properties: @@ -2496,35 +2525,6 @@ spec: You can pass in values according to your needs. x-kubernetes-preserve-unknown-fields: true type: object - chart: - description: |- - Chart defines the helm chart config of the flagger. - default value is - - - ```yaml - chart: - repository: oci://ghcr.io/fluxcd/charts - name: flagger - version: 1.x - ``` - properties: - name: - description: |- - Name defines the name of the chart. - Default value depends on the kind of the component. - type: string - repository: - description: |- - Repository defines the repository of chart. - Default value depends on the kind of the component. - type: string - version: - description: |- - Version defines the version of the chart. - Default value depends on the kind of the component. - type: string - type: object extraArgs: description: |- ExtraArgs is the set of extra arguments for flagger chart. diff --git a/pkg/apis/fleet/v1alpha1/types.go b/pkg/apis/fleet/v1alpha1/types.go index 3bcb0604..921d698f 100644 --- a/pkg/apis/fleet/v1alpha1/types.go +++ b/pkg/apis/fleet/v1alpha1/types.go @@ -572,7 +572,7 @@ type FlaggerConfig struct { PublicTestloader bool `json:"publicTestloader,omitempty"` // ProviderConfig defines the configuration for the TrafficRoutingProvider. // +optional - ProviderConfig *Config `json:"Config,omitempty"` + ProviderConfig *Config `json:"config,omitempty"` } type Config struct { diff --git a/pkg/fleet-manager/application/rollout_helper.go b/pkg/fleet-manager/application/rollout_helper.go index 0c3d9376..e7b58bd7 100644 --- a/pkg/fleet-manager/application/rollout_helper.go +++ b/pkg/fleet-manager/application/rollout_helper.go @@ -494,7 +494,6 @@ func deleteIngressCreatedByKurator(ctx context.Context, kubeClient client.Client // create/update ingress configuration func renderIngress(ingress *ingressv1.Ingress, rollout *applicationapi.RolloutConfig) { - if labels := ingress.GetLabels(); labels == nil || labels[ingressLabelKey] == "" { ingress.SetLabels(map[string]string{ingressLabelKey: rollout.ServiceName}) } else { diff --git a/pkg/webhooks/application_webhook.go b/pkg/webhooks/application_webhook.go index 889361dd..c4e000de 100644 --- a/pkg/webhooks/application_webhook.go +++ b/pkg/webhooks/application_webhook.go @@ -179,5 +179,6 @@ func defaultSyncPolicies(ctx context.Context, SyncPolicies []*v1alpha1.Applicati } } } + log.Info("set SyncPolicies default success") return nil } From f59ff6d9339ca8b3b1b9699db80837a2456325d1 Mon Sep 17 00:00:00 2001 From: Gidi233 Date: Tue, 3 Sep 2024 12:29:20 +0800 Subject: [PATCH 06/10] Change comments and redundant logic Signed-off-by: Gidi233 --- pkg/apis/apps/v1alpha1/types.go | 9 +++-- pkg/apis/fleet/v1alpha1/types.go | 18 ++++++++- .../application/rollout_helper.go | 37 +++++++------------ 3 files changed, 36 insertions(+), 28 deletions(-) diff --git a/pkg/apis/apps/v1alpha1/types.go b/pkg/apis/apps/v1alpha1/types.go index a1738b56..a65ccf48 100644 --- a/pkg/apis/apps/v1alpha1/types.go +++ b/pkg/apis/apps/v1alpha1/types.go @@ -239,8 +239,10 @@ type TrafficRoutingConfig struct { // +optional CorsPolicy *istiov1alpha3.CorsPolicy `json:"corsPolicy,omitempty"` - // for NGINX - // The default created ingress is as follows, (Fill in `host` with your own domain) + // Host defines the domain name you specify. + // Only for NGINX + // Fill in the host field with your own domain as the host of the Ingress. + // The actual created Ingress is as follows: // ```yaml // apiVersion: networking.k8s.io/v1 // kind: Ingress @@ -267,7 +269,8 @@ type TrafficRoutingConfig struct { // +optional Host string `json:"host,omitempty"` - // for kuma + // Protocol defines the protocol used by Kuma + // Only for kuma // Defaults to http // +optional Protocol string `json:"protocol,omitempty"` diff --git a/pkg/apis/fleet/v1alpha1/types.go b/pkg/apis/fleet/v1alpha1/types.go index 921d698f..3cb9f022 100644 --- a/pkg/apis/fleet/v1alpha1/types.go +++ b/pkg/apis/fleet/v1alpha1/types.go @@ -570,18 +570,32 @@ type FlaggerConfig struct { // In addition to the public testloader you can configure here, // you can also specify a private testloader in the Application.Spec.SyncPolicies.Rollout.TestLoader PublicTestloader bool `json:"publicTestloader,omitempty"` - // ProviderConfig defines the configuration for the TrafficRoutingProvider. + // ProviderConfig defines the helm configuration for the TrafficRoutingProvider. + // You can pass in a custom helm configuration to install the TrafficRoutingProvider + // And default value is in `./pkg/fleet-manager/manifests/plugins/` + // Currently only used for Kuma and Nginx // +optional ProviderConfig *Config `json:"config,omitempty"` } type Config struct { // Chart defines the helm chart config of the TrafficRoutingProvider. - // default value is in ./pkg/fleet-manager/manifests/plugins/ + // For Example, using the following configuration to change the version of nginx installed. + // ```yaml + // repo: https://kubernetes.github.io/ingress-nginx + // version: 4.10.0 + // ``` // +optional Chart *ChartConfig `json:"chart,omitempty"` // ExtraArgs is the set of extra arguments for TrafficRoutingProvider's chart. // You can pass in values according to your needs. + // For Example, using the following configuration to change the port that Prometheus uses to scrape metrics. + // ```yaml + // values: + // controller: + // podAnnotations: + // prometheus.io/port: 10378 + // ``` // +optional ExtraArgs apiextensionsv1.JSON `json:"extraArgs,omitempty"` } diff --git a/pkg/fleet-manager/application/rollout_helper.go b/pkg/fleet-manager/application/rollout_helper.go index e7b58bd7..a322d2da 100644 --- a/pkg/fleet-manager/application/rollout_helper.go +++ b/pkg/fleet-manager/application/rollout_helper.go @@ -136,7 +136,7 @@ func (a *ApplicationManager) syncRolloutPolicyForCluster(ctx context.Context, case fleetapi.Istio, fleetapi.Kuma: err := enableSidecarInjection(ctx, fleetClusterClient, rolloutPolicy.Workload.Namespace, provider, annotation) if err != nil { - return ctrl.Result{}, errors.Wrapf(err, "failed to set namespace %s %s's Inject enable", rolloutPolicy.Workload.Namespace, provider) + return ctrl.Result{}, errors.Wrapf(err, "failed to set namespace %s %s's sidecar inject enable", rolloutPolicy.Workload.Namespace, provider) } case fleetapi.Nginx: // Canaries in the same namespace reference the same ingress @@ -159,7 +159,6 @@ func (a *ApplicationManager) syncRolloutPolicyForCluster(ctx context.Context, default: return ctrl.Result{}, errors.Errorf("unknown provider type %s", provider) } - log.Info("pre-operation of operating canary success") // if delete private testloader when rollout polity has changed if rolloutPolicy.TestLoader == nil || !*rolloutPolicy.TestLoader { testloaderDeploy := &appsv1.Deployment{} @@ -281,10 +280,6 @@ func (a *ApplicationManager) deleteResourcesInMemberClusters(ctx context.Context return errors.Wrapf(err, "failed to fetch destination clusters when delete rollout resource") } - namespacedName := types.NamespacedName{ - Namespace: rolloutPolicy.Workload.Namespace, - Name: rolloutPolicy.Workload.Namespace, - } serviceNamespaceName := types.NamespacedName{ Namespace: rolloutPolicy.Workload.Namespace, Name: rolloutPolicy.ServiceName, @@ -308,12 +303,8 @@ func (a *ApplicationManager) deleteResourcesInMemberClusters(ctx context.Context for _, cluster := range destinationClusters { newClient := cluster.Client.CtrlRuntimeClient() - ns := &corev1.Namespace{} - if err := deleteResourceCreatedByKurator(ctx, namespacedName, newClient, ns); err != nil { - return errors.Wrapf(err, "failed to delete namespace") - } if err := deleteIngressCreatedByKurator(ctx, newClient, rolloutPolicy); err != nil { - return errors.Wrapf(err, "failed to delete ingress") + return err } testloaderDeploy := &appsv1.Deployment{} if err := deleteResourceCreatedByKurator(ctx, testloaderNamespaceName, newClient, testloaderDeploy); err != nil { @@ -366,7 +357,9 @@ func enableSidecarInjection(ctx context.Context, kubeClient client.Client, names var newNs client.Object switch provider { case fleetapi.Kuma: - newNs = addAnnotations(ns, kumaInject, "enabled") + newNs = addAnnotations(ns, map[string]string{ + kumaInject: "enabled", + }) case fleetapi.Istio: newNs = addLabels(ns, istioInject, "enabled") } @@ -500,7 +493,9 @@ func renderIngress(ingress *ingressv1.Ingress, rollout *applicationapi.RolloutCo labels[ingressLabelKey] = strings.Join(sets.New(strings.Split(labels[ingressLabelKey], ",")...).Insert(rollout.ServiceName).UnsortedList(), ",") ingress.SetLabels(labels) } - addAnnotations(ingress, ingressAnnotationKey, ingressAnnotationValue) + addAnnotations(ingress, map[string]string{ + ingressAnnotationKey: ingressAnnotationValue, + }) Prefix := ingressv1.PathTypePrefix rule := ingressv1.IngressRule{ Host: rollout.RolloutPolicy.TrafficRouting.Host, @@ -710,20 +705,16 @@ func addLabels(obj client.Object, key, value string) client.Object { return obj } -func addAnnotations(obj client.Object, keysAndValues ...string) client.Object { +func addAnnotations(obj client.Object, ann map[string]string) client.Object { annotations := obj.GetAnnotations() - // prevent nil pointer panic if annotations == nil { - annotations = make(map[string]string) - } - for i := 0; i < len(keysAndValues); i += 2 { - if i+1 < len(keysAndValues) { - annotations[keysAndValues[i]] = keysAndValues[i+1] - } else { - annotations[keysAndValues[i]] = "" + obj.SetAnnotations(ann) + } else { + for k, v := range ann { + annotations[k] = v } + obj.SetAnnotations(annotations) } - obj.SetAnnotations(annotations) return obj } From efbb89f87f17ebe52e94068047eea322a2e86084 Mon Sep 17 00:00:00 2001 From: Gidi233 Date: Tue, 3 Sep 2024 12:29:29 +0800 Subject: [PATCH 07/10] Change example Signed-off-by: Gidi233 --- examples/rollout/ab-testingNginx.yaml | 2 +- examples/rollout/blue_greenNginx.yaml | 2 +- examples/rollout/canaryKuma.yaml | 2 +- examples/rollout/canaryNginx.yaml | 2 +- examples/rollout/canaryWithCustomMetric.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/rollout/ab-testingNginx.yaml b/examples/rollout/ab-testingNginx.yaml index 744ccb0a..f9d29e65 100644 --- a/examples/rollout/ab-testingNginx.yaml +++ b/examples/rollout/ab-testingNginx.yaml @@ -45,7 +45,7 @@ spec: checkIntervalSeconds: 90 checkFailedTimes: 2 metrics: - - name: my-metric + - name: nginx-request-success-rate intervalSeconds: 90 thresholdRange: min: 99 diff --git a/examples/rollout/blue_greenNginx.yaml b/examples/rollout/blue_greenNginx.yaml index 8cb75610..75f07ecd 100644 --- a/examples/rollout/blue_greenNginx.yaml +++ b/examples/rollout/blue_greenNginx.yaml @@ -38,7 +38,7 @@ spec: checkIntervalSeconds: 90 checkFailedTimes: 2 metrics: - - name: my-metric + - name: nginx-request-success-rate intervalSeconds: 90 thresholdRange: min: 99 diff --git a/examples/rollout/canaryKuma.yaml b/examples/rollout/canaryKuma.yaml index 1fb50867..1e102fbd 100644 --- a/examples/rollout/canaryKuma.yaml +++ b/examples/rollout/canaryKuma.yaml @@ -39,7 +39,7 @@ spec: checkIntervalSeconds: 90 checkFailedTimes: 2 metrics: - - name: my-metric + - name: kuma-request-success-rate intervalSeconds: 90 thresholdRange: min: 99 diff --git a/examples/rollout/canaryNginx.yaml b/examples/rollout/canaryNginx.yaml index a3f45685..e8bb2080 100644 --- a/examples/rollout/canaryNginx.yaml +++ b/examples/rollout/canaryNginx.yaml @@ -40,7 +40,7 @@ spec: checkIntervalSeconds: 90 checkFailedTimes: 2 metrics: - - name: my-metric + - name: nginx-request-success-rate intervalSeconds: 90 thresholdRange: min: 99 diff --git a/examples/rollout/canaryWithCustomMetric.yaml b/examples/rollout/canaryWithCustomMetric.yaml index 4d3f87c7..f7b51fe0 100644 --- a/examples/rollout/canaryWithCustomMetric.yaml +++ b/examples/rollout/canaryWithCustomMetric.yaml @@ -47,7 +47,7 @@ spec: intervalSeconds: 90 thresholdRange: min: 99 - - name: my-metric + - name: istio-request-success-rate intervalSeconds: 90 thresholdRange: max: 99 From 0a31983020fb435ff3b22e1febd8a82ef718e1a9 Mon Sep 17 00:00:00 2001 From: Gidi233 Date: Wed, 4 Sep 2024 21:28:41 +0800 Subject: [PATCH 08/10] add gen Signed-off-by: Gidi233 --- .../en/references/apps_v1alpha1_types.html | 9 +++++--- .../en/references/fleet_v1alpha1_types.html | 18 ++++++++++++--- .../crds/apps.kurator.dev_applications.yaml | 9 +++++--- .../crds/fleet.kurator.dev_fleets.yaml | 22 ++++++++++++++----- 4 files changed, 43 insertions(+), 15 deletions(-) diff --git a/docs/content/en/references/apps_v1alpha1_types.html b/docs/content/en/references/apps_v1alpha1_types.html index 1d309e91..ebefed34 100644 --- a/docs/content/en/references/apps_v1alpha1_types.html +++ b/docs/content/en/references/apps_v1alpha1_types.html @@ -2219,8 +2219,10 @@

TrafficRoutingConfig (Optional) -

for NGINX -The default created ingress is as follows, (Fill in host with your own domain)

+

Host defines the domain name you specify. +Only for NGINX +Fill in the host field with your own domain as the host of the Ingress. +The actual created Ingress is as follows:

apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -2254,7 +2256,8 @@ 
TrafficRoutingConfig
 
 
 (Optional)
-
for kuma
+
Protocol defines the protocol used by Kuma
+Only for kuma
 Defaults to http

 
 
diff --git a/docs/content/en/references/fleet_v1alpha1_types.html b/docs/content/en/references/fleet_v1alpha1_types.html
index 8b49ea92..5694cf08 100644
--- a/docs/content/en/references/fleet_v1alpha1_types.html
+++ b/docs/content/en/references/fleet_v1alpha1_types.html
@@ -428,7 +428,10 @@ 
Config
 
 (Optional)
 
Chart defines the helm chart config of the TrafficRoutingProvider.
-default value is in ./pkg/fleet-manager/manifests/plugins/

+For Example, using the following configuration to change the version of nginx installed.

+
repo: https://kubernetes.github.io/ingress-nginx
+version: 4.10.0
+

 
 
 
@@ -443,7 +446,13 @@ 
Config
 
 (Optional)
 
ExtraArgs is the set of extra arguments for TrafficRoutingProvider’s chart.
-You can pass in values according to your needs.

+You can pass in values according to your needs.
+For Example, using the following configuration to change the port that Prometheus uses to scrape metrics.

+
values:
+controller:
+podAnnotations:
+prometheus.io/port: 10378
+

 
 
 
@@ -748,7 +757,10 @@ 
FlaggerConfig
 
 
 (Optional)
-
ProviderConfig defines the configuration for the TrafficRoutingProvider.

+
ProviderConfig defines the helm configuration for the TrafficRoutingProvider.
+You can pass in a custom helm configuration to install the TrafficRoutingProvider
+And default value is in ./pkg/fleet-manager/manifests/plugins/
+Currently only used for Kuma and Nginx

 
 
 
diff --git a/manifests/charts/fleet-manager/crds/apps.kurator.dev_applications.yaml b/manifests/charts/fleet-manager/crds/apps.kurator.dev_applications.yaml
index 7496ff53..ced86e24 100644
--- a/manifests/charts/fleet-manager/crds/apps.kurator.dev_applications.yaml
+++ b/manifests/charts/fleet-manager/crds/apps.kurator.dev_applications.yaml
@@ -1620,8 +1620,10 @@ spec:
                                   type: object
                                 host:
                                   description: |-
-                                    for NGINX
-                                    The default created ingress is as follows, (Fill in `host` with your own domain)
+                                    Host defines the domain name you specify.
+                                    Only for NGINX
+                                    Fill in the host field with your own domain as the host of the Ingress.
+                                    The actual created Ingress is as follows:
                                     ```yaml
                                     apiVersion: networking.k8s.io/v1
                                     kind: Ingress
@@ -1953,7 +1955,8 @@ spec:
                                   type: array
                                 protocol:
                                   description: |-
-                                    for kuma
+                                    Protocol defines the protocol used by Kuma
+                                    Only for kuma
                                     Defaults to http
                                   type: string
                                 retries:
diff --git a/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml b/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml
index b20c86ae..26ca859d 100644
--- a/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml
+++ b/manifests/charts/fleet-manager/crds/fleet.kurator.dev_fleets.yaml
@@ -2495,13 +2495,20 @@ spec:
                             type: string
                         type: object
                       config:
-                        description: ProviderConfig defines the configuration for
-                          the TrafficRoutingProvider.
+                        description: |-
+                          ProviderConfig defines the helm configuration for the TrafficRoutingProvider.
+                          You can pass in a custom helm configuration to install the TrafficRoutingProvider
+                          And default value is in `./pkg/fleet-manager/manifests/plugins/`
+                          Currently only used for Kuma and Nginx
                         properties:
                           chart:
                             description: |-
                               Chart defines the helm chart config of the TrafficRoutingProvider.
-                              default value is in ./pkg/fleet-manager/manifests/plugins/
+                              For Example, using the following configuration to change the version of nginx installed.
+                              ```yaml
+                              repo: https://kubernetes.github.io/ingress-nginx
+                              version: 4.10.0
+                              ```
                             properties:
                               name:
                                 description: |-
@@ -2520,9 +2527,12 @@ spec:
                                 type: string
                             type: object
                           extraArgs:
-                            description: |-
-                              ExtraArgs is the set of extra arguments for TrafficRoutingProvider's chart.
-                              You can pass in values according to your needs.
+                            description: "ExtraArgs is the set of extra arguments
+                              for TrafficRoutingProvider's chart.\nYou can pass in
+                              values according to your needs.\nFor Example, using
+                              the following configuration to change the port that
+                              Prometheus uses to scrape metrics.\n```yaml\nvalues:\n\tcontroller:\n\t
+                              \ podAnnotations:\n\t\tprometheus.io/port: 10378\n```"
                             x-kubernetes-preserve-unknown-fields: true
                         type: object
                       extraArgs:

From a7a9cf956072e016924fd9742077a659b36c3002 Mon Sep 17 00:00:00 2001
From: Gidi233 
Date: Wed, 11 Sep 2024 10:59:26 +0800
Subject: [PATCH 09/10] fix

Signed-off-by: Gidi233 
---
 .../application/rollout_helper.go             | 98 ++++++++++++-------
 .../application/rollout_helper_test.go        | 15 ++-
 2 files changed, 70 insertions(+), 43 deletions(-)

diff --git a/pkg/fleet-manager/application/rollout_helper.go b/pkg/fleet-manager/application/rollout_helper.go
index a322d2da..ec197dfb 100644
--- a/pkg/fleet-manager/application/rollout_helper.go
+++ b/pkg/fleet-manager/application/rollout_helper.go
@@ -148,8 +148,7 @@ func (a *ApplicationManager) syncRolloutPolicyForCluster(ctx context.Context,
 			}
 			result, err := controllerutil.CreateOrUpdate(ctx, fleetClusterClient, ingress, func() error {
 				ingress.SetAnnotations(annotation)
-				renderIngress(ingress, rolloutPolicy)
-				return nil
+				return renderNginxIngress(ingress, rolloutPolicy)
 			})
 			if err != nil {
 				return ctrl.Result{}, errors.Wrapf(err, "failed to operate ingress")
@@ -357,11 +356,16 @@ func enableSidecarInjection(ctx context.Context, kubeClient client.Client, names
 		var newNs client.Object
 		switch provider {
 		case fleetapi.Kuma:
-			newNs = addAnnotations(ns, map[string]string{
+			newNs, err = addAnnotations(ns, map[string]string{
 				kumaInject: "enabled",
 			})
+			if err != nil {
+				return err
+			}
 		case fleetapi.Istio:
-			newNs = addLabels(ns, istioInject, "enabled")
+			if newNs, err = addLabels(ns, map[string]string{istioInject: "enabled"}); err != nil {
+				return err
+			}
 		}
 		if updateErr := kubeClient.Update(ctx, newNs); updateErr != nil {
 			return errors.Wrapf(updateErr, "failed to update namespace %s", namespacedName.Namespace)
@@ -462,21 +466,13 @@ func deleteIngressCreatedByKurator(ctx context.Context, kubeClient client.Client
 			labels := ingress.GetLabels()
 			if set := sets.New(strings.Split(labels[ingressLabelKey], ",")...); set.Contains(rollout.ServiceName) {
 				if set.Len() == 1 {
-					if deleteErr := kubeClient.Delete(ctx, ingress); deleteErr != nil && !apierrors.IsNotFound(deleteErr) {
+					if deleteErr := kubeClient.Delete(ctx, ingress); deleteErr != nil {
 						return errors.Wrapf(deleteErr, "failed to Delete ingress %s in %s", namespaceName.Name, namespaceName.Namespace)
 					}
 				} else {
-					newRules := make([]ingressv1.IngressRule, 0)
-					for _, rule := range ingress.Spec.Rules {
-						if rule.Host != rollout.RolloutPolicy.TrafficRouting.Host {
-							newRules = append(newRules, rule)
-						}
-					}
-					ingress.Spec.Rules = newRules
-					labels[ingressLabelKey] = strings.Join(set.Delete(rollout.ServiceName).UnsortedList(), ",")
-					ingress.SetLabels(labels)
-					if err := kubeClient.Update(ctx, ingress); err != nil {
-						return errors.Wrapf(err, "failed to Update ingress %s in %s", namespaceName.Name, namespaceName.Namespace)
+					// There are still other canaries using this ingress
+					if err := updateIngressRulesAndLabels(ctx, kubeClient, ingress, rollout, namespaceName, set); err != nil {
+						return err
 					}
 				}
 			}
@@ -485,17 +481,38 @@ func deleteIngressCreatedByKurator(ctx context.Context, kubeClient client.Client
 	return nil
 }
 
+func updateIngressRulesAndLabels(ctx context.Context, kubeClient client.Client, ingress *ingressv1.Ingress, rollout *applicationapi.RolloutConfig, namespaceName types.NamespacedName, set sets.Set[string]) error {
+	newRules := make([]ingressv1.IngressRule, 0)
+	for _, rule := range ingress.Spec.Rules {
+		if rule.Host != rollout.RolloutPolicy.TrafficRouting.Host {
+			newRules = append(newRules, rule)
+		}
+	}
+	ingress.Spec.Rules = newRules
+
+	labels := ingress.GetLabels()
+	labels[ingressLabelKey] = strings.Join(set.Delete(rollout.ServiceName).UnsortedList(), ",")
+	ingress.SetLabels(labels)
+
+	if err := kubeClient.Update(ctx, ingress); err != nil {
+		return errors.Wrapf(err, "failed to Update ingress %s in %s", namespaceName.Name, namespaceName.Namespace)
+	}
+	return nil
+}
+
 // create/update ingress configuration
-func renderIngress(ingress *ingressv1.Ingress, rollout *applicationapi.RolloutConfig) {
+func renderNginxIngress(ingress *ingressv1.Ingress, rollout *applicationapi.RolloutConfig) error {
 	if labels := ingress.GetLabels(); labels == nil || labels[ingressLabelKey] == "" {
 		ingress.SetLabels(map[string]string{ingressLabelKey: rollout.ServiceName})
 	} else {
 		labels[ingressLabelKey] = strings.Join(sets.New(strings.Split(labels[ingressLabelKey], ",")...).Insert(rollout.ServiceName).UnsortedList(), ",")
 		ingress.SetLabels(labels)
 	}
-	addAnnotations(ingress, map[string]string{
+	if _, err := addAnnotations(ingress, map[string]string{
 		ingressAnnotationKey: ingressAnnotationValue,
-	})
+	}); err != nil {
+		return err
+	}
 	Prefix := ingressv1.PathTypePrefix
 	rule := ingressv1.IngressRule{
 		Host: rollout.RolloutPolicy.TrafficRouting.Host,
@@ -517,6 +534,7 @@ func renderIngress(ingress *ingressv1.Ingress, rollout *applicationapi.RolloutCo
 		},
 	}
 	ingress.Spec.Rules = append(ingress.Spec.Rules, rule)
+	return nil
 }
 
 // create/update canary configuration
@@ -691,31 +709,41 @@ func generateWebhookUrl(name, namespace string) string {
 	return url
 }
 
-func addLabels(obj client.Object, key, value string) client.Object {
-	labels := obj.GetLabels()
-	// prevent nil pointer panic
-	if labels == nil {
-		obj.SetLabels(map[string]string{
-			key: value,
-		})
-		return obj
-	}
-	labels[key] = value
-	obj.SetLabels(labels)
-	return obj
+func addLabels(obj client.Object, labels map[string]string) (client.Object, error) {
+	existingLabels := obj.GetLabels()
+	if existingLabels == nil {
+		obj.SetLabels(labels)
+	} else {
+		for k, v := range labels {
+			if value, exists := existingLabels[k]; exists {
+				if value != v {
+					return nil, errors.New("label key conflict")
+				}
+			} else {
+				existingLabels[k] = v
+			}
+		}
+		obj.SetLabels(existingLabels)
+	}
+	return obj, nil
 }
-
-func addAnnotations(obj client.Object, ann map[string]string) client.Object {
+func addAnnotations(obj client.Object, ann map[string]string) (client.Object, error) {
 	annotations := obj.GetAnnotations()
 	if annotations == nil {
 		obj.SetAnnotations(ann)
 	} else {
 		for k, v := range ann {
-			annotations[k] = v
+			if value, exists := annotations[k]; exists {
+				if value != v {
+					return nil, errors.New("annotation key conflict")
+				}
+			} else {
+				annotations[k] = v
+			}
 		}
 		obj.SetAnnotations(annotations)
 	}
-	return obj
+	return obj, nil
 }
 
 func mergeMap(map1, map2 map[string]*applicationapi.RolloutStatus) map[string]*applicationapi.RolloutStatus {
diff --git a/pkg/fleet-manager/application/rollout_helper_test.go b/pkg/fleet-manager/application/rollout_helper_test.go
index e24a078f..640a78a1 100644
--- a/pkg/fleet-manager/application/rollout_helper_test.go
+++ b/pkg/fleet-manager/application/rollout_helper_test.go
@@ -595,8 +595,7 @@ func Test_renderCanaryAnalysis(t *testing.T) {
 func Test_addLables(t *testing.T) {
 	type args struct {
 		obj   client.Object
-		key   string
-		value string
+		label map[string]string
 	}
 	tests := []struct {
 		name string
@@ -618,8 +617,9 @@ func Test_addLables(t *testing.T) {
 						},
 					},
 				},
-				key:   "istio-injection",
-				value: "ebabled",
+				label: map[string]string{
+					"istio-injection": "enabled",
+				},
 			},
 			want: &corev1.Namespace{
 				TypeMeta: metav1.TypeMeta{
@@ -630,7 +630,7 @@ func Test_addLables(t *testing.T) {
 					Name: "webapp",
 					Labels: map[string]string{
 						"xxx":             "abc",
-						"istio-injection": "ebabled",
+						"istio-injection": "enabled",
 					},
 				},
 			},
@@ -647,8 +647,7 @@ func Test_addLables(t *testing.T) {
 						Name: "webapp",
 					},
 				},
-				key:   "XXX",
-				value: "abc",
+				label: map[string]string{"XXX": "abc"},
 			},
 			want: &corev1.Namespace{
 				TypeMeta: metav1.TypeMeta{
@@ -666,7 +665,7 @@ func Test_addLables(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := addLabels(tt.args.obj, tt.args.key, tt.args.value); !reflect.DeepEqual(got, tt.want) {
+			if got, _ := addLabels(tt.args.obj, tt.args.label); !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("addLablesOrAnnotaions() = %v, want %v", got, tt.want)
 			}
 		})

From 7a5c768b5fe540e2f693198dfe86c7d4b417ecb9 Mon Sep 17 00:00:00 2001
From: Gidi233 
Date: Fri, 13 Sep 2024 09:24:09 +0800
Subject: [PATCH 10/10] fix example

Signed-off-by: Gidi233 
---
 examples/rollout/blue_greenNginx.yaml | 2 +-
 examples/rollout/canaryKuma.yaml      | 4 ++--
 examples/rollout/canaryNginx.yaml     | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/examples/rollout/blue_greenNginx.yaml b/examples/rollout/blue_greenNginx.yaml
index 75f07ecd..05f23642 100644
--- a/examples/rollout/blue_greenNginx.yaml
+++ b/examples/rollout/blue_greenNginx.yaml
@@ -1,7 +1,7 @@
 apiVersion: apps.kurator.dev/v1alpha1
 kind: Application
 metadata:
-  name: abtesting-nginx-demo
+  name: blue-green-nginx-demo
   namespace: default
 spec:
   source:
diff --git a/examples/rollout/canaryKuma.yaml b/examples/rollout/canaryKuma.yaml
index 1e102fbd..30c3a0c6 100644
--- a/examples/rollout/canaryKuma.yaml
+++ b/examples/rollout/canaryKuma.yaml
@@ -1,7 +1,7 @@
 apiVersion: apps.kurator.dev/v1alpha1
 kind: Application
 metadata:
-  name: rollout-demo
+  name: rollout-kuma-demo
   namespace: default
 spec:
   source:
@@ -64,7 +64,7 @@ spec:
              webhooks:
                  timeoutSeconds: 60
                  command:
-                 - "hey -z 1m -q 10 -c 2 http://app.example.com/"
+                 - "hey -z 1m -q 10 -c 2 http://podinfo-canary.test:9898/"
           rolloutTimeoutSeconds: 600
     - destination:
         fleet: quickstart
diff --git a/examples/rollout/canaryNginx.yaml b/examples/rollout/canaryNginx.yaml
index e8bb2080..e1f39e49 100644
--- a/examples/rollout/canaryNginx.yaml
+++ b/examples/rollout/canaryNginx.yaml
@@ -1,7 +1,7 @@
 apiVersion: apps.kurator.dev/v1alpha1
 kind: Application
 metadata:
-  name: rollout-demo
+  name: rollout-nginx-demo
   namespace: default
 spec:
   source: