diff --git a/deploy/manifests/oci-native-ingress-controller/templates/deployment.yaml b/deploy/manifests/oci-native-ingress-controller/templates/deployment.yaml index fef67e19..1a8e23d3 100644 --- a/deploy/manifests/oci-native-ingress-controller/templates/deployment.yaml +++ b/deploy/manifests/oci-native-ingress-controller/templates/deployment.yaml @@ -61,7 +61,6 @@ spec: - --controller-class=oci.oraclecloud.com/native-ingress-controller - --compartment-id= - --subnet-id= - - --cluster-id= - --metrics-backend=prometheus - --metrics-port=2223 - --cniType=OCI_VCN_IP_NATIVE diff --git a/helm/oci-native-ingress-controller/templates/deployment.yaml b/helm/oci-native-ingress-controller/templates/deployment.yaml index b34fd0e5..3647b0d5 100644 --- a/helm/oci-native-ingress-controller/templates/deployment.yaml +++ b/helm/oci-native-ingress-controller/templates/deployment.yaml @@ -61,7 +61,6 @@ spec: - --controller-class={{ .Values.controller_class }} - --compartment-id={{ .Values.compartment_id }} - --subnet-id={{ .Values.subnet_id }} - - --cluster-id={{ .Values.cluster_id }} - --metrics-backend={{.Values.metrics.backend}} - --metrics-port={{.Values.metrics.port}} - --cniType={{.Values.cniType}} diff --git a/helm/oci-native-ingress-controller/values.yaml b/helm/oci-native-ingress-controller/values.yaml index 215a43f7..3a45f43e 100644 --- a/helm/oci-native-ingress-controller/values.yaml +++ b/helm/oci-native-ingress-controller/values.yaml @@ -13,7 +13,6 @@ controller_class: oci.oraclecloud.com/native-ingress-controller lease_lock_name: oci-native-ingress-controller compartment_id: "" subnet_id: "" -cluster_id: "" deploymentNamespace : native-ingress-controller-system replicaCount: 1 diff --git a/pkg/server/server.go b/pkg/server/server.go index 458189e6..d55d394b 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -10,187 +10,187 @@ package server import ( - "context" - "net/http" - "os" - - ctrcache "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/webhook" - - ociwaf "github.com/oracle/oci-go-sdk/v65/waf" - "github.com/oracle/oci-native-ingress-controller/pkg/client" - "github.com/oracle/oci-native-ingress-controller/pkg/controllers/nodeBackend" - "k8s.io/client-go/kubernetes" - - "github.com/oracle/oci-go-sdk/v65/certificates" - "github.com/oracle/oci-go-sdk/v65/certificatesmanagement" - ociloadbalancer "github.com/oracle/oci-go-sdk/v65/loadbalancer" - clientset "k8s.io/client-go/kubernetes" - - "github.com/oracle/oci-native-ingress-controller/pkg/auth" - "github.com/oracle/oci-native-ingress-controller/pkg/certificate" - "github.com/oracle/oci-native-ingress-controller/pkg/controllers/backend" - "github.com/oracle/oci-native-ingress-controller/pkg/controllers/ingress" - "github.com/oracle/oci-native-ingress-controller/pkg/controllers/ingressclass" - "github.com/oracle/oci-native-ingress-controller/pkg/controllers/routingpolicy" - "github.com/oracle/oci-native-ingress-controller/pkg/loadbalancer" - "github.com/oracle/oci-native-ingress-controller/pkg/metric" - ociclient "github.com/oracle/oci-native-ingress-controller/pkg/oci/client" - "github.com/oracle/oci-native-ingress-controller/pkg/types" - "github.com/oracle/oci-native-ingress-controller/pkg/waf" - "github.com/prometheus/client_golang/prometheus" - - v1 "k8s.io/client-go/informers/core/v1" - networkinginformers "k8s.io/client-go/informers/networking/v1" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/klog/v2" - - "github.com/oracle/oci-native-ingress-controller/pkg/podreadiness" + "context" + "net/http" + "os" + + ctrcache "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + ociwaf "github.com/oracle/oci-go-sdk/v65/waf" + "github.com/oracle/oci-native-ingress-controller/pkg/client" + "github.com/oracle/oci-native-ingress-controller/pkg/controllers/nodeBackend" + "k8s.io/client-go/kubernetes" + + "github.com/oracle/oci-go-sdk/v65/certificates" + "github.com/oracle/oci-go-sdk/v65/certificatesmanagement" + ociloadbalancer "github.com/oracle/oci-go-sdk/v65/loadbalancer" + clientset "k8s.io/client-go/kubernetes" + + "github.com/oracle/oci-native-ingress-controller/pkg/auth" + "github.com/oracle/oci-native-ingress-controller/pkg/certificate" + "github.com/oracle/oci-native-ingress-controller/pkg/controllers/backend" + "github.com/oracle/oci-native-ingress-controller/pkg/controllers/ingress" + "github.com/oracle/oci-native-ingress-controller/pkg/controllers/ingressclass" + "github.com/oracle/oci-native-ingress-controller/pkg/controllers/routingpolicy" + "github.com/oracle/oci-native-ingress-controller/pkg/loadbalancer" + "github.com/oracle/oci-native-ingress-controller/pkg/metric" + ociclient "github.com/oracle/oci-native-ingress-controller/pkg/oci/client" + "github.com/oracle/oci-native-ingress-controller/pkg/types" + "github.com/oracle/oci-native-ingress-controller/pkg/waf" + "github.com/prometheus/client_golang/prometheus" + + v1 "k8s.io/client-go/informers/core/v1" + networkinginformers "k8s.io/client-go/informers/networking/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + + "github.com/oracle/oci-native-ingress-controller/pkg/podreadiness" ) func BuildConfig(kubeconfig string) (*rest.Config, error) { - if kubeconfig != "" { - cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig) - if err != nil { - return nil, err - } - return cfg, nil - } - - cfg, err := rest.InClusterConfig() - if err != nil { - return nil, err - } - return cfg, nil + if kubeconfig != "" { + cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return nil, err + } + return cfg, nil + } + + cfg, err := rest.InClusterConfig() + if err != nil { + return nil, err + } + return cfg, nil } func SetUpControllers(opts types.IngressOpts, ingressClassInformer networkinginformers.IngressClassInformer, - ingressInformer networkinginformers.IngressInformer, k8client kubernetes.Interface, - serviceInformer v1.ServiceInformer, endpointInformer v1.EndpointsInformer, podInformer v1.PodInformer, nodeInformer v1.NodeInformer, c ctrcache.Cache, - reg *prometheus.Registry) func(ctx context.Context) { - return func(ctx context.Context) { - klog.Info("Controller loop...") - - client := setupClient(ctx, opts, k8client) - - ingressController := ingress.NewController( - opts.ControllerClass, - opts.CompartmentId, - ingressClassInformer, - ingressInformer, - serviceInformer.Lister(), - client, - reg, - ) - - routingPolicyController := routingpolicy.NewController( - opts.ControllerClass, - ingressClassInformer, - ingressInformer, - serviceInformer.Lister(), - client, - ) - - ingressClassController := ingressclass.NewController( - opts.CompartmentId, - opts.SubnetId, - opts.ControllerClass, - ingressClassInformer, - client, - c, - ) - - go ingressClassController.Run(3, ctx.Done()) - go ingressController.Run(3, ctx.Done()) - go routingPolicyController.Run(3, ctx.Done()) - - if opts.CniType == "FLANNEL_OVERLAY" { - backendController := nodeBackend.NewController( - opts.ControllerClass, - ingressClassInformer, - ingressInformer, - serviceInformer.Lister(), - endpointInformer.Lister(), - podInformer.Lister(), - nodeInformer.Lister(), - client, - ) - go backendController.Run(3, ctx.Done()) - } else { - backendController := backend.NewController( - opts.ControllerClass, - ingressClassInformer, - ingressInformer, - serviceInformer.Lister(), - endpointInformer.Lister(), - podInformer.Lister(), - client, - ) - go backendController.Run(3, ctx.Done()) - } - } + ingressInformer networkinginformers.IngressInformer, k8client kubernetes.Interface, + serviceInformer v1.ServiceInformer, endpointInformer v1.EndpointsInformer, podInformer v1.PodInformer, nodeInformer v1.NodeInformer, c ctrcache.Cache, + reg *prometheus.Registry) func(ctx context.Context) { + return func(ctx context.Context) { + klog.Info("Controller loop...") + + client := setupClient(ctx, opts, k8client) + + ingressController := ingress.NewController( + opts.ControllerClass, + opts.CompartmentId, + ingressClassInformer, + ingressInformer, + serviceInformer.Lister(), + client, + reg, + ) + + routingPolicyController := routingpolicy.NewController( + opts.ControllerClass, + ingressClassInformer, + ingressInformer, + serviceInformer.Lister(), + client, + ) + + ingressClassController := ingressclass.NewController( + opts.CompartmentId, + opts.SubnetId, + opts.ControllerClass, + ingressClassInformer, + client, + c, + ) + + go ingressClassController.Run(3, ctx.Done()) + go ingressController.Run(3, ctx.Done()) + go routingPolicyController.Run(3, ctx.Done()) + + if opts.CniType == "FLANNEL_OVERLAY" { + backendController := nodeBackend.NewController( + opts.ControllerClass, + ingressClassInformer, + ingressInformer, + serviceInformer.Lister(), + endpointInformer.Lister(), + podInformer.Lister(), + nodeInformer.Lister(), + client, + ) + go backendController.Run(3, ctx.Done()) + } else { + backendController := backend.NewController( + opts.ControllerClass, + ingressClassInformer, + ingressInformer, + serviceInformer.Lister(), + endpointInformer.Lister(), + podInformer.Lister(), + client, + ) + go backendController.Run(3, ctx.Done()) + } + } } func setupClient(ctx context.Context, opts types.IngressOpts, k8client clientset.Interface) *client.ClientProvider { - configProvider, err := auth.GetConfigurationProvider(ctx, opts, k8client) - if err != nil { - klog.Fatalf("failed to load authentication configuration provider: %v", err) - } - - ociLBClient, err := ociloadbalancer.NewLoadBalancerClientWithConfigurationProvider(configProvider) - if err != nil { - klog.Fatalf("unable to construct oci load balancer client: %v", err) - } - - ociCertificatesClient, err := certificates.NewCertificatesClientWithConfigurationProvider(configProvider) - if err != nil { - klog.Fatalf("unable to construct oci certificate client: %v", err) - } - - ociCertificatesMgmtClient, err := certificatesmanagement.NewCertificatesManagementClientWithConfigurationProvider(configProvider) - if err != nil { - klog.Fatalf("unable to construct oci certificate management client: %v", err) - } - - ociWafClient, err := ociwaf.NewWafClientWithConfigurationProvider(configProvider) - if err != nil { - klog.Fatalf("unable to construct oci web application firewall client: %v", err) - } - - lbClient := loadbalancer.New(&ociLBClient) - - certificatesClient := certificate.New(&ociCertificatesMgmtClient, ociclient.NewCertificateClient(&ociCertificatesClient)) - - wafClient := waf.New(&ociWafClient) - - return client.NewWrapperClient(k8client, wafClient, lbClient, certificatesClient) + configProvider, err := auth.GetConfigurationProvider(ctx, opts, k8client) + if err != nil { + klog.Fatalf("failed to load authentication configuration provider: %v", err) + } + + ociLBClient, err := ociloadbalancer.NewLoadBalancerClientWithConfigurationProvider(configProvider) + if err != nil { + klog.Fatalf("unable to construct oci load balancer client: %v", err) + } + + ociCertificatesClient, err := certificates.NewCertificatesClientWithConfigurationProvider(configProvider) + if err != nil { + klog.Fatalf("unable to construct oci certificate client: %v", err) + } + + ociCertificatesMgmtClient, err := certificatesmanagement.NewCertificatesManagementClientWithConfigurationProvider(configProvider) + if err != nil { + klog.Fatalf("unable to construct oci certificate management client: %v", err) + } + + ociWafClient, err := ociwaf.NewWafClientWithConfigurationProvider(configProvider) + if err != nil { + klog.Fatalf("unable to construct oci web application firewall client: %v", err) + } + + lbClient := loadbalancer.New(&ociLBClient) + + certificatesClient := certificate.New(&ociCertificatesMgmtClient, ociclient.NewCertificateClient(&ociCertificatesClient)) + + wafClient := waf.New(&ociWafClient) + + return client.NewWrapperClient(k8client, wafClient, lbClient, certificatesClient) } func SetupWebhookServer(ingressInformer networkinginformers.IngressInformer, serviceInformer v1.ServiceInformer, client *clientset.Clientset, ctx context.Context) { - klog.Info("setting up webhook server") - - server := &webhook.Server{} - server.Register("/mutate-v1-pod", &webhook.Admission{Handler: podreadiness.NewWebhook(ingressInformer.Lister(), serviceInformer.Lister(), client)}) - - go func() { - klog.Infof("starting webhook server...") - err := server.StartStandalone(ctx, nil) - if err != nil { - klog.Errorf("failed to run webhook server: %v", err) - os.Exit(1) - } - }() + klog.Info("setting up webhook server") + + server := &webhook.Server{} + server.Register("/mutate-v1-pod", &webhook.Admission{Handler: podreadiness.NewWebhook(ingressInformer.Lister(), serviceInformer.Lister(), client)}) + + go func() { + klog.Infof("starting webhook server...") + err := server.StartStandalone(ctx, nil) + if err != nil { + klog.Errorf("failed to run webhook server: %v", err) + os.Exit(1) + } + }() } func SetupMetricsServer(metricsBackend string, metricsPort int, mux *http.ServeMux, ctx context.Context) (*prometheus.Registry, error) { - // initialize metrics exporter before creating measurements - reg, err := metric.InitMetricsExporter(metricsBackend) - if err != nil { - klog.Error("failed to initialize metrics exporter: %s", err.Error()) - return nil, err - } - metric.RegisterMetrics(reg, mux) - - return reg, nil + // initialize metrics exporter before creating measurements + reg, err := metric.InitMetricsExporter(metricsBackend) + if err != nil { + klog.Error("failed to initialize metrics exporter: %s", err.Error()) + return nil, err + } + metric.RegisterMetrics(reg, mux) + + return reg, nil } diff --git a/pkg/util/testutil.go b/pkg/util/testutil.go index 496cf31e..f5a18db6 100644 --- a/pkg/util/testutil.go +++ b/pkg/util/testutil.go @@ -20,6 +20,10 @@ import ( k8stesting "k8s.io/client-go/testing" ) +const ( + ToBeDeletedTaint = "ToBeDeletedByClusterAutoscaler" +) + func ReadResourceAsIngressList(fileName string) *networkingv1.IngressList { data, err := os.ReadFile(fileName) if err != nil { @@ -511,22 +515,42 @@ func GetNodesList() *v1.NodeList { } conditions = append(conditions, cond) - node := v1.Node{ + nodeA := v1.Node{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", - Kind: "Pod", + Kind: "Node", }, ObjectMeta: metav1.ObjectMeta{ Namespace: "default", - Name: "nodename", + Name: "nodeA", }, Status: v1.NodeStatus{ Conditions: conditions, }, } + var taints []v1.Taint + taint := v1.Taint{ + Key: ToBeDeletedTaint, + } + taints = append(taints, taint) + + nodeB := v1.Node{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "nodeB", + }, + Spec: v1.NodeSpec{ + Taints: taints, + }, + } var nodes []v1.Node - nodes = append(nodes, node) + nodes = append(nodes, nodeA) + nodes = append(nodes, nodeB) return &v1.NodeList{ TypeMeta: metav1.TypeMeta{