diff --git a/GettingStarted.md b/GettingStarted.md index 1cbf433f..b9b1e39a 100644 --- a/GettingStarted.md +++ b/GettingStarted.md @@ -36,6 +36,8 @@ The native ingress controller itself is lightweight process and pushes all the r + [Web Firewall Integration](#web-firewall-integration) + [Ingress Level HTTP(S) Listener Ports](#ingress-level-https-listener-ports) + [TCP Listener Support](#tcp-listener-support) + + [Network Security Groups Support](#network-security-groups-support) + + [Load Balancer Preservation on `IngressClass` delete](#load-balancer-preservation-on-ingressclass-delete) * [Dependency management](#dependency-management) + [How to introduce new modules or upgrade existing ones?](#how-to-introduce-new-modules-or-upgrade-existing-ones) * [Known Issues](#known-issues) @@ -50,6 +52,7 @@ Currently supported kubernetes versions are: - 1.27 - 1.28 - 1.29 +- 1.30 We set up the cluster with either native pod networking or flannel CNI and update the security rules. The documentation for NPN : [Doc Ref](https://docs.oracle.com/en-us/iaas/Content/ContEng/Concepts/contengpodnetworking_topic-OCI_CNI_plugin.htm). @@ -603,6 +606,38 @@ spec: number: 8081 ``` +### Network Security Groups Support +Users can use the `IngressClass` resource annotation `oci-native-ingress.oraclecloud.com/network-security-group-ids` to supply +a comma separated list of Network Security Group OCIDs. +The supplied NSGs will be associated with the LB associated with the `IngressClass`. + +Example: +```yaml +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + annotations: + oci-native-ingress.oraclecloud.com/network-security-group-ids: ocid1.networksecuritygroup.oc1.abc,ocid1.networksecuritygroup.oc1.xyz +``` + +### Load Balancer Preservation on `IngressClass` delete +If you want the Load Balancer associated with an `IngressClass` resource to be preserved after `IngressClass` is deleted, +set the annotation `oci-native-ingress.oraclecloud.com/delete-protection-enabled` annotation to `"true"`. +This annotation defaults to `"false"` when not specified or empty. + +OCI Native Ingress Controller will aim to leave the LB in a 'blank' state - clear all NSG associated with the LB, +delete the Web App Firewall associated with the LB if any, and delete the `default_ingress` BackendSet when the `IngressClass` is deleted with this annotation set to true. +Please note that users should first delete all `Ingress` resources associated with this `IngressClass` first, or orphaned resources like Listeners, BackendSets, etc. will +still be present on the LB after the `IngressClass` is deleted + +Example: +```yaml +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + annotations: + oci-native-ingress.oraclecloud.com/delete-protection-enabled: "true" +``` ### Dependency management Module [vendoring](https://go.dev/ref/mod#vendoring) is used to manage 3d-party modules in the project. diff --git a/pkg/controllers/backend/backend_test.go b/pkg/controllers/backend/backend_test.go index 4b948bd1..077251a7 100644 --- a/pkg/controllers/backend/backend_test.go +++ b/pkg/controllers/backend/backend_test.go @@ -295,6 +295,10 @@ func (m MockLoadBalancerClient) UpdateLoadBalancerShape(ctx context.Context, req return ociloadbalancer.UpdateLoadBalancerShapeResponse{}, nil } +func (m MockLoadBalancerClient) UpdateNetworkSecurityGroups(ctx context.Context, request ociloadbalancer.UpdateNetworkSecurityGroupsRequest) (ociloadbalancer.UpdateNetworkSecurityGroupsResponse, error) { + return ociloadbalancer.UpdateNetworkSecurityGroupsResponse{}, nil +} + func (m MockLoadBalancerClient) GetLoadBalancer(ctx context.Context, request ociloadbalancer.GetLoadBalancerRequest) (ociloadbalancer.GetLoadBalancerResponse, error) { res := util.SampleLoadBalancerResponse() return res, nil diff --git a/pkg/controllers/ingress/ingress.go b/pkg/controllers/ingress/ingress.go index 98521cf4..476bf0ba 100644 --- a/pkg/controllers/ingress/ingress.go +++ b/pkg/controllers/ingress/ingress.go @@ -522,7 +522,7 @@ func syncListener(ctx context.Context, namespace string, stateStore *state.State } if sslConfig != nil { - if !reflect.DeepEqual(listener.SslConfiguration.CertificateIds, sslConfig.CertificateIds) { + if listener.SslConfiguration == nil || !reflect.DeepEqual(listener.SslConfiguration.CertificateIds, sslConfig.CertificateIds) { klog.Infof("SSL config for listener update is %s", util.PrettyPrint(sslConfig)) needsUpdate = true } @@ -579,11 +579,10 @@ func syncBackendSet(ctx context.Context, ingress *networkingv1.Ingress, lbID str if err != nil { return err } - if sslConfig != nil { - if bs.SslConfiguration == nil || !reflect.DeepEqual(bs.SslConfiguration.TrustedCertificateAuthorityIds, sslConfig.TrustedCertificateAuthorityIds) { - klog.Infof("SSL config for backend set %s update is %s", *bs.Name, util.PrettyPrint(sslConfig)) - needsUpdate = true - } + + if backendSetSslConfigNeedsUpdate(sslConfig, &bs) { + klog.Infof("SSL config for backend set %s update is %s", *bs.Name, util.PrettyPrint(sslConfig)) + needsUpdate = true } healthChecker := stateStore.GetBackendSetHealthChecker(*bs.Name) @@ -601,7 +600,7 @@ func syncBackendSet(ctx context.Context, ingress *networkingv1.Ingress, lbID str } if needsUpdate { - err = wrapperClient.GetLbClient().UpdateBackendSet(context.TODO(), lb.Id, etag, bs, nil, sslConfig, healthChecker, &policy) + err = wrapperClient.GetLbClient().UpdateBackendSetDetails(context.TODO(), *lb.Id, etag, &bs, sslConfig, healthChecker, policy) if err != nil { return err } diff --git a/pkg/controllers/ingress/ingress_test.go b/pkg/controllers/ingress/ingress_test.go index 6ced764b..8e9f8a6a 100644 --- a/pkg/controllers/ingress/ingress_test.go +++ b/pkg/controllers/ingress/ingress_test.go @@ -225,6 +225,10 @@ func (m MockLoadBalancerClient) UpdateLoadBalancerShape(ctx context.Context, req return ociloadbalancer.UpdateLoadBalancerShapeResponse{}, nil } +func (m MockLoadBalancerClient) UpdateNetworkSecurityGroups(ctx context.Context, request ociloadbalancer.UpdateNetworkSecurityGroupsRequest) (ociloadbalancer.UpdateNetworkSecurityGroupsResponse, error) { + return ociloadbalancer.UpdateNetworkSecurityGroupsResponse{}, nil +} + func (m MockLoadBalancerClient) CreateLoadBalancer(ctx context.Context, request ociloadbalancer.CreateLoadBalancerRequest) (ociloadbalancer.CreateLoadBalancerResponse, error) { return ociloadbalancer.CreateLoadBalancerResponse{}, nil } diff --git a/pkg/controllers/ingress/util.go b/pkg/controllers/ingress/util.go index 019119dc..d4764fe2 100644 --- a/pkg/controllers/ingress/util.go +++ b/pkg/controllers/ingress/util.go @@ -429,3 +429,17 @@ func CreateOrGetCaBundleForBackendSet(namespace string, secretName string, compa func isTrustAuthorityCaBundle(id string) bool { return strings.Contains(id, "cabundle") } + +func backendSetSslConfigNeedsUpdate(calculatedConfig *ociloadbalancer.SslConfigurationDetails, + currentBackendSet *ociloadbalancer.BackendSet) bool { + if calculatedConfig == nil && currentBackendSet.SslConfiguration != nil { + return true + } + + if calculatedConfig != nil && (currentBackendSet.SslConfiguration == nil || + !reflect.DeepEqual(currentBackendSet.SslConfiguration.TrustedCertificateAuthorityIds, calculatedConfig.TrustedCertificateAuthorityIds)) { + return true + } + + return false +} diff --git a/pkg/controllers/ingress/util_test.go b/pkg/controllers/ingress/util_test.go index 9264be62..57f99b91 100644 --- a/pkg/controllers/ingress/util_test.go +++ b/pkg/controllers/ingress/util_test.go @@ -367,6 +367,34 @@ intermediatecert Expect(err).To(HaveOccurred()) } +func TestBackendSetSslConfigNeedsUpdate(t *testing.T) { + RegisterTestingT(t) + + caBundleId1 := []string{"caCert1"} + caBundleId2 := []string{"caCert2"} + + backendSetWithNilSslConfig := &ociloadbalancer.BackendSet{} + presentBackendSet1 := &ociloadbalancer.BackendSet{ + SslConfiguration: &ociloadbalancer.SslConfiguration{ + TrustedCertificateAuthorityIds: caBundleId1, + }, + } + presentBackendSet2 := &ociloadbalancer.BackendSet{ + SslConfiguration: &ociloadbalancer.SslConfiguration{ + TrustedCertificateAuthorityIds: caBundleId2, + }, + } + calculatedConfig1 := &ociloadbalancer.SslConfigurationDetails{ + TrustedCertificateAuthorityIds: caBundleId1, + } + + Expect(backendSetSslConfigNeedsUpdate(nil, backendSetWithNilSslConfig)).To(BeFalse()) + Expect(backendSetSslConfigNeedsUpdate(nil, presentBackendSet1)).To(BeTrue()) + Expect(backendSetSslConfigNeedsUpdate(calculatedConfig1, presentBackendSet1)).To(BeFalse()) + Expect(backendSetSslConfigNeedsUpdate(calculatedConfig1, presentBackendSet2)).To(BeTrue()) + Expect(backendSetSslConfigNeedsUpdate(calculatedConfig1, backendSetWithNilSslConfig)).To(BeTrue()) +} + func generateTestCertsAndKey() (string, string, string) { caCert := &x509.Certificate{ SerialNumber: big.NewInt(1), diff --git a/pkg/controllers/ingressclass/ingressclass.go b/pkg/controllers/ingressclass/ingressclass.go index 893570b9..31704310 100644 --- a/pkg/controllers/ingressclass/ingressclass.go +++ b/pkg/controllers/ingressclass/ingressclass.go @@ -207,33 +207,35 @@ func (c *Controller) sync(key string) error { return nil } -func (c *Controller) getLoadBalancer(ctx context.Context, ic *networkingv1.IngressClass) (*ociloadbalancer.LoadBalancer, error) { +func (c *Controller) getLoadBalancer(ctx context.Context, ic *networkingv1.IngressClass) (*ociloadbalancer.LoadBalancer, string, error) { lbID := util.GetIngressClassLoadBalancerId(ic) if lbID == "" { - klog.Errorf("LB id not set for ingressClass: %s", ic.Name) - return nil, nil // LoadBalancer ID not set, Trigger new LB creation + klog.Infof("LB id not set for ingressClass: %s", ic.Name) + return nil, "", nil // LoadBalancer ID not set, Trigger new LB creation } wrapperClient, ok := ctx.Value(util.WrapperClient).(*client.WrapperClient) if !ok { - return nil, fmt.Errorf(util.OciClientNotFoundInContextError) + return nil, "", fmt.Errorf(util.OciClientNotFoundInContextError) } - lb, _, err := wrapperClient.GetLbClient().GetLoadBalancer(context.TODO(), lbID) + + lb, etag, err := wrapperClient.GetLbClient().GetLoadBalancer(context.TODO(), lbID) if err != nil { klog.Errorf("Error while fetching LB %s for ingressClass: %s, err: %s", lbID, ic.Name, err.Error()) // Check if Service error 404, then ignore it since LB is not found. svcErr, ok := common.IsServiceError(err) if ok && svcErr.GetHTTPStatusCode() == 404 { - return nil, nil // Redirect new LB creation + return nil, "", nil // Redirect new LB creation } - return nil, err + return nil, "", err } - return lb, nil + + return lb, etag, nil } func (c *Controller) ensureLoadBalancer(ctx context.Context, ic *networkingv1.IngressClass) error { - lb, err := c.getLoadBalancer(ctx, ic) + lb, etag, err := c.getLoadBalancer(ctx, ic) if err != nil { return err } @@ -262,11 +264,12 @@ func (c *Controller) ensureLoadBalancer(ctx context.Context, ic *networkingv1.In klog.V(2).InfoS("Creating load balancer for ingress class", "ingressClass", ic.Name) createDetails := ociloadbalancer.CreateLoadBalancerDetails{ - CompartmentId: compartmentId, - DisplayName: common.String(util.GetIngressClassLoadBalancerName(ic, icp)), - ShapeName: common.String("flexible"), - SubnetIds: []string{util.GetIngressClassSubnetId(icp, c.defaultSubnetId)}, - IsPrivate: common.Bool(icp.Spec.IsPrivate), + CompartmentId: compartmentId, + DisplayName: common.String(util.GetIngressClassLoadBalancerName(ic, icp)), + ShapeName: common.String("flexible"), + SubnetIds: []string{util.GetIngressClassSubnetId(icp, c.defaultSubnetId)}, + IsPrivate: common.Bool(icp.Spec.IsPrivate), + NetworkSecurityGroupIds: util.GetIngressClassNetworkSecurityGroupIds(ic), BackendSets: map[string]ociloadbalancer.BackendSetDetails{ util.DefaultBackendSetName: { Policy: common.String("LEAST_CONNECTIONS"), @@ -300,7 +303,15 @@ func (c *Controller) ensureLoadBalancer(ctx context.Context, ic *networkingv1.In return err } } else { - c.checkForIngressClassParameterUpdates(ctx, lb, ic, icp) + err = c.checkForIngressClassParameterUpdates(ctx, lb, ic, icp, etag) + if err != nil { + return err + } + + err = c.checkForNetworkSecurityGroupsUpdate(ctx, ic) + if err != nil { + return err + } } if *lb.Id != util.GetIngressClassLoadBalancerId(ic) { @@ -342,7 +353,8 @@ func (c *Controller) setupWebApplicationFirewall(ctx context.Context, ic *networ return nil } -func (c *Controller) checkForIngressClassParameterUpdates(ctx context.Context, lb *ociloadbalancer.LoadBalancer, ic *networkingv1.IngressClass, icp *v1beta1.IngressClassParameters) error { +func (c *Controller) checkForIngressClassParameterUpdates(ctx context.Context, lb *ociloadbalancer.LoadBalancer, + ic *networkingv1.IngressClass, icp *v1beta1.IngressClassParameters, etag string) error { // check LoadBalancerName AND MinBandwidthMbps ,MaxBandwidthMbps displayName := util.GetIngressClassLoadBalancerName(ic, icp) wrapperClient, ok := ctx.Value(util.WrapperClient).(*client.WrapperClient) @@ -377,11 +389,11 @@ func (c *Controller) checkForIngressClassParameterUpdates(ctx context.Context, l req := ociloadbalancer.UpdateLoadBalancerShapeRequest{ LoadBalancerId: lb.Id, + IfMatch: common.String(etag), UpdateLoadBalancerShapeDetails: ociloadbalancer.UpdateLoadBalancerShapeDetails{ ShapeName: common.String("flexible"), ShapeDetails: shapeDetails, }, - OpcRetryToken: common.String(fmt.Sprintf("update-lb-shape-%s", ic.UID)), } klog.Infof("Update lb shape request: %s", util.PrettyPrint(req)) _, err := wrapperClient.GetLbClient().UpdateLoadBalancerShape(context.Background(), req) @@ -393,18 +405,90 @@ func (c *Controller) checkForIngressClassParameterUpdates(ctx context.Context, l return nil } +func (c *Controller) checkForNetworkSecurityGroupsUpdate(ctx context.Context, ic *networkingv1.IngressClass) error { + lb, _, err := c.getLoadBalancer(ctx, ic) + if err != nil { + return err + } + + wrapperClient, ok := ctx.Value(util.WrapperClient).(*client.WrapperClient) + if !ok { + return fmt.Errorf(util.OciClientNotFoundInContextError) + } + + nsgIdsFromSpec := util.GetIngressClassNetworkSecurityGroupIds(ic) + + /* + Only check if desired and actual slices have the same elements, ignoring order and duplicates + We don't check if lb.NetworkSecurityGroupIds is nil since util.StringSlicesHaveSameElements returns true if + one argument is nil and the other is empty. + */ + if util.StringSlicesHaveSameElements(nsgIdsFromSpec, lb.NetworkSecurityGroupIds) { + return nil + } + + _, err = wrapperClient.GetLbClient().UpdateNetworkSecurityGroups(context.Background(), *lb.Id, nsgIdsFromSpec) + return err +} + func (c *Controller) deleteIngressClass(ctx context.Context, ic *networkingv1.IngressClass) error { + if util.GetIngressClassDeleteProtectionEnabled(ic) { + err := c.clearLoadBalancer(ctx, ic) + if err != nil { + return err + } + } else { + err := c.deleteLoadBalancer(ctx, ic) + if err != nil { + return err + } + } - err := c.deleteLoadBalancer(ctx, ic) + err := c.deleteFinalizer(ctx, ic) if err != nil { return err } - err = c.deleteFinalizer(ctx, ic) + return nil +} + +// clearLoadBalancer clears the default_ingress backend, NSG attachment, and WAF firewall from the LB +func (c *Controller) clearLoadBalancer(ctx context.Context, ic *networkingv1.IngressClass) error { + lb, _, err := c.getLoadBalancer(ctx, ic) if err != nil { return err } + if lb == nil { + klog.Infof("Tried to clear LB for ic %s/%s, but it is deleted", ic.Namespace, ic.Name) + return nil + } + + wrapperClient, ok := ctx.Value(util.WrapperClient).(*client.WrapperClient) + if !ok { + return fmt.Errorf(util.OciClientNotFoundInContextError) + } + + fireWallId := util.GetIngressClassFireWallId(ic) + if fireWallId != "" { + wrapperClient.GetWafClient().DeleteWebAppFirewallWithId(fireWallId) + } + + nsgIds := util.GetIngressClassNetworkSecurityGroupIds(ic) + if len(nsgIds) > 0 { + _, err = wrapperClient.GetLbClient().UpdateNetworkSecurityGroups(context.Background(), *lb.Id, make([]string, 0)) + if err != nil { + klog.Errorf("While clearing LB %s, cannot clear NSG IDs due to %s, will proceed with IngressClass deletion for %s/%s", + *lb.Id, err.Error(), ic.Namespace, ic.Name) + } + } + + err = wrapperClient.GetLbClient().DeleteBackendSet(context.Background(), *lb.Id, util.DefaultBackendSetName) + if err != nil { + klog.Errorf("While clearing LB %s, cannot clear BackendSet %s due to %s, will proceed with IngressClass deletion for %s/%s", + *lb.Id, util.DefaultBackendSetName, err.Error(), ic.Namespace, ic.Name) + } + return nil } diff --git a/pkg/controllers/ingressclass/ingressclass_test.go b/pkg/controllers/ingressclass/ingressclass_test.go index b56121ff..29a681f8 100644 --- a/pkg/controllers/ingressclass/ingressclass_test.go +++ b/pkg/controllers/ingressclass/ingressclass_test.go @@ -117,6 +117,41 @@ func TestDeleteIngressClass(t *testing.T) { Expect(err).Should(BeNil()) } +func TestClearLoadBalancerWhenLBFound(t *testing.T) { + RegisterTestingT(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ingressClassList := util.GetIngressClassListWithLBSet("id") + ingressClassList.Items[0].Annotations[util.IngressClassFireWallIdAnnotation] = "firewallId" + ingressClassList.Items[0].Annotations[util.IngressClassNetworkSecurityGroupIdsAnnotation] = "nsgId" + c := inits(ctx, ingressClassList) + err := c.clearLoadBalancer(getContextWithClient(c, ctx), &ingressClassList.Items[0]) + Expect(err).Should(BeNil()) +} + +func TestClearLoadBalancerWhenLBNotFound(t *testing.T) { + RegisterTestingT(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ingressClassList := util.GetIngressClassListWithLBSet("notfound") + c := inits(ctx, ingressClassList) + err := c.clearLoadBalancer(getContextWithClient(c, ctx), &ingressClassList.Items[0]) + Expect(err).Should(BeNil()) +} + +func TestClearLoadBalancerWhenNetworkError(t *testing.T) { + RegisterTestingT(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ingressClassList := util.GetIngressClassListWithLBSet("networkerror") + c := inits(ctx, ingressClassList) + err := c.clearLoadBalancer(getContextWithClient(c, ctx), &ingressClassList.Items[0]) + Expect(err).ShouldNot(BeNil()) +} + func TestDeleteLoadBalancer(t *testing.T) { RegisterTestingT(t) ctx, cancel := context.WithCancel(context.Background()) @@ -182,10 +217,27 @@ func TestCheckForIngressClassParameterUpdates(t *testing.T) { MaxBandwidthMbps: 400, }, } - err = c.checkForIngressClassParameterUpdates(getContextWithClient(c, ctx), loadBalancer, &ingressClassList.Items[0], &icp) + err = c.checkForIngressClassParameterUpdates(getContextWithClient(c, ctx), loadBalancer, &ingressClassList.Items[0], &icp, "etag") Expect(err).Should(BeNil()) } +func TestCheckForNetworkSecurityGroupsUpdate(t *testing.T) { + RegisterTestingT(t) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ingressClassList := util.GetIngressClassResourceWithAnnotation("ingress-class-with-nsg", + map[string]string{ + util.IngressClassNetworkSecurityGroupIdsAnnotation: "id1,id2, id3", + util.IngressClassLoadBalancerIdAnnotation: "id", + }, "oci.oraclecloud.com/native-ingress-controller") + c := inits(ctx, ingressClassList) + + err := c.checkForNetworkSecurityGroupsUpdate(getContextWithClient(c, ctx), &ingressClassList.Items[0]) + Expect(err).To(BeNil()) +} + func TestDeleteFinalizer(t *testing.T) { RegisterTestingT(t) ctx, cancel := context.WithCancel(context.Background()) @@ -332,6 +384,15 @@ func (m MockLoadBalancerClient) UpdateLoadBalancerShape(ctx context.Context, req }, nil } +func (m MockLoadBalancerClient) UpdateNetworkSecurityGroups(ctx context.Context, + request ociloadbalancer.UpdateNetworkSecurityGroupsRequest) (ociloadbalancer.UpdateNetworkSecurityGroupsResponse, error) { + return ociloadbalancer.UpdateNetworkSecurityGroupsResponse{ + RawResponse: nil, + OpcWorkRequestId: common.String("id"), + OpcRequestId: common.String("id"), + }, nil +} + func (m MockLoadBalancerClient) CreateLoadBalancer(ctx context.Context, request ociloadbalancer.CreateLoadBalancerRequest) (ociloadbalancer.CreateLoadBalancerResponse, error) { id := "id" return ociloadbalancer.CreateLoadBalancerResponse{ diff --git a/pkg/controllers/nodeBackend/nodeBackend_test.go b/pkg/controllers/nodeBackend/nodeBackend_test.go index 7662f111..56665d5e 100644 --- a/pkg/controllers/nodeBackend/nodeBackend_test.go +++ b/pkg/controllers/nodeBackend/nodeBackend_test.go @@ -248,6 +248,10 @@ func (m MockLoadBalancerClient) UpdateLoadBalancerShape(ctx context.Context, req return ociloadbalancer.UpdateLoadBalancerShapeResponse{}, nil } +func (m MockLoadBalancerClient) UpdateNetworkSecurityGroups(ctx context.Context, request ociloadbalancer.UpdateNetworkSecurityGroupsRequest) (ociloadbalancer.UpdateNetworkSecurityGroupsResponse, error) { + return ociloadbalancer.UpdateNetworkSecurityGroupsResponse{}, nil +} + func (m MockLoadBalancerClient) GetLoadBalancer(ctx context.Context, request ociloadbalancer.GetLoadBalancerRequest) (ociloadbalancer.GetLoadBalancerResponse, error) { res := util.SampleLoadBalancerResponse() return res, nil diff --git a/pkg/controllers/routingpolicy/routingpolicy_test.go b/pkg/controllers/routingpolicy/routingpolicy_test.go index 9eb05180..a2ad19a4 100644 --- a/pkg/controllers/routingpolicy/routingpolicy_test.go +++ b/pkg/controllers/routingpolicy/routingpolicy_test.go @@ -220,6 +220,10 @@ func (m MockLoadBalancerClient) UpdateLoadBalancerShape(ctx context.Context, req return ociloadbalancer.UpdateLoadBalancerShapeResponse{}, nil } +func (m MockLoadBalancerClient) UpdateNetworkSecurityGroups(ctx context.Context, request ociloadbalancer.UpdateNetworkSecurityGroupsRequest) (ociloadbalancer.UpdateNetworkSecurityGroupsResponse, error) { + return ociloadbalancer.UpdateNetworkSecurityGroupsResponse{}, nil +} + func (m MockLoadBalancerClient) CreateLoadBalancer(ctx context.Context, request ociloadbalancer.CreateLoadBalancerRequest) (ociloadbalancer.CreateLoadBalancerResponse, error) { return ociloadbalancer.CreateLoadBalancerResponse{}, nil } diff --git a/pkg/loadbalancer/loadbalancer.go b/pkg/loadbalancer/loadbalancer.go index eb84bd99..c37b3de3 100644 --- a/pkg/loadbalancer/loadbalancer.go +++ b/pkg/loadbalancer/loadbalancer.go @@ -86,6 +86,33 @@ func (lbc *LoadBalancerClient) GetBackendSetHealth(ctx context.Context, lbID str return &resp.BackendSetHealth, nil } +func (lbc *LoadBalancerClient) UpdateNetworkSecurityGroups(ctx context.Context, lbId string, nsgIds []string) (loadbalancer.UpdateNetworkSecurityGroupsResponse, error) { + _, etag, err := lbc.GetLoadBalancer(ctx, lbId) + + req := loadbalancer.UpdateNetworkSecurityGroupsRequest{ + LoadBalancerId: common.String(lbId), + IfMatch: common.String(etag), + UpdateNetworkSecurityGroupsDetails: loadbalancer.UpdateNetworkSecurityGroupsDetails{ + NetworkSecurityGroupIds: nsgIds, + }, + } + + klog.Infof("Update LB NSG IDs request: %s", util.PrettyPrint(req)) + + resp, err := lbc.LbClient.UpdateNetworkSecurityGroups(ctx, req) + if err != nil { + return resp, err + } + + lbID, err := lbc.waitForWorkRequest(ctx, *resp.OpcWorkRequestId) + if err != nil { + return resp, err + } + + _, _, err = lbc.getLoadBalancerBustCache(ctx, lbID) + return resp, err +} + func (lbc *LoadBalancerClient) UpdateLoadBalancerShape(ctx context.Context, req loadbalancer.UpdateLoadBalancerShapeRequest) (response loadbalancer.UpdateLoadBalancerShapeResponse, err error) { resp, err := lbc.LbClient.UpdateLoadBalancerShape(ctx, req) if err != nil { @@ -451,6 +478,7 @@ func (lbc *LoadBalancerClient) updateRoutingPolicyRules(ctx context.Context, lbI return err } +// UpdateBackends updates Backends for backendSetName, while preserving sslConfig, policy, and healthChecker details func (lbc *LoadBalancerClient) UpdateBackends(ctx context.Context, lbID string, backendSetName string, backends []loadbalancer.BackendDetails) error { lb, etag, err := lbc.GetLoadBalancer(ctx, lbID) if err != nil { @@ -477,56 +505,60 @@ func (lbc *LoadBalancerClient) UpdateBackends(ctx context.Context, lbID string, return nil } - return lbc.UpdateBackendSet(ctx, lb.Id, etag, backendSet, backends, nil, nil, nil) -} - -func (lbc *LoadBalancerClient) UpdateBackendSet(ctx context.Context, lbID *string, etag string, backendSet loadbalancer.BackendSet, - backends []loadbalancer.BackendDetails, sslConfig *loadbalancer.SslConfigurationDetails, - healthCheckerDetails *loadbalancer.HealthCheckerDetails, policy *string) error { - - if backends == nil { - backends = make([]loadbalancer.BackendDetails, len(backendSet.Backends)) - for i := range backendSet.Backends { - backend := backendSet.Backends[i] - backends[i] = loadbalancer.BackendDetails{ - IpAddress: backend.IpAddress, - Port: backend.Port, - Weight: backend.Weight, - Drain: backend.Drain, - Backup: backend.Backup, - Offline: backend.Offline, - } + var sslConfig *loadbalancer.SslConfigurationDetails + if backendSet.SslConfiguration != nil { + sslConfig = &loadbalancer.SslConfigurationDetails{ + TrustedCertificateAuthorityIds: backendSet.SslConfiguration.TrustedCertificateAuthorityIds, } } - if sslConfig == nil && backendSet.SslConfiguration != nil { - sslConfig = &loadbalancer.SslConfigurationDetails{TrustedCertificateAuthorityIds: backendSet.SslConfiguration.TrustedCertificateAuthorityIds} - } - - if healthCheckerDetails == nil { - healthChecker := backendSet.HealthChecker - healthCheckerDetails = &loadbalancer.HealthCheckerDetails{ - Protocol: healthChecker.Protocol, - UrlPath: healthChecker.UrlPath, - Port: healthChecker.Port, - ReturnCode: healthChecker.ReturnCode, - Retries: healthChecker.Retries, - TimeoutInMillis: healthChecker.TimeoutInMillis, - IntervalInMillis: healthChecker.IntervalInMillis, - ResponseBodyRegex: healthChecker.ResponseBodyRegex, - IsForcePlainText: healthChecker.IsForcePlainText, - } + + healthCheckerDetails := &loadbalancer.HealthCheckerDetails{ + Protocol: backendSet.HealthChecker.Protocol, + UrlPath: backendSet.HealthChecker.UrlPath, + Port: backendSet.HealthChecker.Port, + ReturnCode: backendSet.HealthChecker.ReturnCode, + Retries: backendSet.HealthChecker.Retries, + TimeoutInMillis: backendSet.HealthChecker.TimeoutInMillis, + IntervalInMillis: backendSet.HealthChecker.IntervalInMillis, + ResponseBodyRegex: backendSet.HealthChecker.ResponseBodyRegex, + IsForcePlainText: backendSet.HealthChecker.IsForcePlainText, } - if policy == nil { - policy = backendSet.Policy + policy := *backendSet.Policy + + return lbc.UpdateBackendSet(ctx, lbID, etag, *backendSet.Name, policy, healthCheckerDetails, sslConfig, backends) +} + +// UpdateBackendSetDetails updates sslConfig, policy, and healthChecker details for backendSet, while preserving individual backends +func (lbc *LoadBalancerClient) UpdateBackendSetDetails(ctx context.Context, lbID string, etag string, + backendSet *loadbalancer.BackendSet, sslConfig *loadbalancer.SslConfigurationDetails, + healthCheckerDetails *loadbalancer.HealthCheckerDetails, policy string) error { + + backends := make([]loadbalancer.BackendDetails, len(backendSet.Backends)) + for i := range backendSet.Backends { + backend := backendSet.Backends[i] + backends[i] = loadbalancer.BackendDetails{ + IpAddress: backend.IpAddress, + Port: backend.Port, + Weight: backend.Weight, + Drain: backend.Drain, + Backup: backend.Backup, + Offline: backend.Offline, + } } + return lbc.UpdateBackendSet(ctx, lbID, etag, *backendSet.Name, policy, healthCheckerDetails, sslConfig, backends) +} + +func (lbc *LoadBalancerClient) UpdateBackendSet(ctx context.Context, lbID string, etag string, backendSetName string, + policy string, healthCheckerDetails *loadbalancer.HealthCheckerDetails, sslConfig *loadbalancer.SslConfigurationDetails, + backends []loadbalancer.BackendDetails) error { updateBackendSetRequest := loadbalancer.UpdateBackendSetRequest{ IfMatch: common.String(etag), - LoadBalancerId: lbID, - BackendSetName: common.String(*backendSet.Name), + LoadBalancerId: common.String(lbID), + BackendSetName: common.String(backendSetName), UpdateBackendSetDetails: loadbalancer.UpdateBackendSetDetails{ - Policy: policy, + Policy: common.String(policy), HealthChecker: healthCheckerDetails, SslConfiguration: sslConfig, Backends: backends, @@ -539,7 +571,8 @@ func (lbc *LoadBalancerClient) UpdateBackendSet(ctx context.Context, lbID *strin return err } - klog.Infof("Update backend set response: name: %s, work request id: %s, opc request id: %s.", *backendSet.Name, *resp.OpcWorkRequestId, *resp.OpcRequestId) + klog.Infof("Update backend set response: name: %s, work request id: %s, opc request id: %s.", + *updateBackendSetRequest.BackendSetName, *resp.OpcWorkRequestId, *resp.OpcRequestId) _, err = lbc.waitForWorkRequest(ctx, *resp.OpcWorkRequestId) return err } @@ -643,7 +676,7 @@ func (lbc *LoadBalancerClient) CreateListener(ctx context.Context, lbID string, if sslConfig == nil { return fmt.Errorf("no TLS configuration provided for a HTTP2 listener at port %d", listenerPort) } - + sslConfig.CipherSuiteName = common.String(util.ProtocolHTTP2DefaultCipherSuite) } diff --git a/pkg/loadbalancer/loadbalancer_test.go b/pkg/loadbalancer/loadbalancer_test.go index 9f96fadb..0a92accb 100644 --- a/pkg/loadbalancer/loadbalancer_test.go +++ b/pkg/loadbalancer/loadbalancer_test.go @@ -119,6 +119,26 @@ func TestLoadBalancerClient_UpdateBackends(t *testing.T) { } +func TestLoadBalancerClient_UpdateBackendSetDetails(t *testing.T) { + RegisterTestingT(t) + loadBalancerClient := setupLBClient() + + lbId := "id" + etag := "etag" + policy := "policy" + bsName := util.GenerateBackendSetName("default", "testecho1", 80) + + lb := util.SampleLoadBalancerResponse() + sslConfigDetails := ociloadbalancer.SslConfigurationDetails{TrustedCertificateAuthorityIds: []string{"trusted-cert"}} + healthCheckerDetails := ociloadbalancer.HealthCheckerDetails{} + + bs := lb.BackendSets[bsName] + + err := loadBalancerClient.UpdateBackendSetDetails(context.TODO(), lbId, etag, &bs, &sslConfigDetails, + &healthCheckerDetails, policy) + Expect(err).To(BeNil()) +} + func TestLoadBalancerClient_DeleteBackendSet(t *testing.T) { RegisterTestingT(t) loadBalancerClient := setupLBClient() @@ -185,6 +205,14 @@ func TestLoadBalancerClient_UpdateListener(t *testing.T) { Expect(err).To(BeNil()) } +func TestLoadBalancerClient_UpdateNetworkSecurityGroups(t *testing.T) { + RegisterTestingT(t) + loadBalancerClient := setupLBClient() + + _, err := loadBalancerClient.UpdateNetworkSecurityGroups(context.TODO(), "id", []string{"id1", "id2"}) + Expect(err).To(BeNil()) +} + func setupLBClient() *LoadBalancerClient { lbClient := GetLoadBalancerClient() @@ -211,6 +239,15 @@ func (m MockLoadBalancerClient) UpdateLoadBalancerShape(ctx context.Context, req return ociloadbalancer.UpdateLoadBalancerShapeResponse{}, nil } +func (m MockLoadBalancerClient) UpdateNetworkSecurityGroups(ctx context.Context, + request ociloadbalancer.UpdateNetworkSecurityGroupsRequest) (ociloadbalancer.UpdateNetworkSecurityGroupsResponse, error) { + return ociloadbalancer.UpdateNetworkSecurityGroupsResponse{ + RawResponse: nil, + OpcWorkRequestId: common.String("id"), + OpcRequestId: common.String("id"), + }, nil +} + func (m MockLoadBalancerClient) GetLoadBalancer(ctx context.Context, request ociloadbalancer.GetLoadBalancerRequest) (ociloadbalancer.GetLoadBalancerResponse, error) { res := util.SampleLoadBalancerResponse() return res, nil diff --git a/pkg/oci/client/loadbalancer.go b/pkg/oci/client/loadbalancer.go index ac3f118f..8d563603 100644 --- a/pkg/oci/client/loadbalancer.go +++ b/pkg/oci/client/loadbalancer.go @@ -11,6 +11,7 @@ type LoadBalancerInterface interface { CreateLoadBalancer(ctx context.Context, request loadbalancer.CreateLoadBalancerRequest) (loadbalancer.CreateLoadBalancerResponse, error) UpdateLoadBalancer(ctx context.Context, request loadbalancer.UpdateLoadBalancerRequest) (response loadbalancer.UpdateLoadBalancerResponse, err error) UpdateLoadBalancerShape(ctx context.Context, request loadbalancer.UpdateLoadBalancerShapeRequest) (response loadbalancer.UpdateLoadBalancerShapeResponse, err error) + UpdateNetworkSecurityGroups(ctx context.Context, request loadbalancer.UpdateNetworkSecurityGroupsRequest) (loadbalancer.UpdateNetworkSecurityGroupsResponse, error) DeleteLoadBalancer(ctx context.Context, request loadbalancer.DeleteLoadBalancerRequest) (loadbalancer.DeleteLoadBalancerResponse, error) GetWorkRequest(ctx context.Context, request loadbalancer.GetWorkRequestRequest) (loadbalancer.GetWorkRequestResponse, error) @@ -58,6 +59,10 @@ func (client LBClient) UpdateLoadBalancer(ctx context.Context, request loadbalan return client.lbClient.UpdateLoadBalancer(ctx, request) } +func (client LBClient) UpdateNetworkSecurityGroups(ctx context.Context, request loadbalancer.UpdateNetworkSecurityGroupsRequest) (loadbalancer.UpdateNetworkSecurityGroupsResponse, error) { + return client.lbClient.UpdateNetworkSecurityGroups(ctx, request) +} + func (client LBClient) DeleteLoadBalancer(ctx context.Context, request loadbalancer.DeleteLoadBalancerRequest) (loadbalancer.DeleteLoadBalancerResponse, error) { return client.lbClient.DeleteLoadBalancer(ctx, request) diff --git a/pkg/util/util.go b/pkg/util/util.go index e13867da..e8cd28f7 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -63,9 +63,11 @@ const ( // HTTP, HTTP2, TCP - accepted. IngressProtocolAnnotation = "oci-native-ingress.oraclecloud.com/protocol" - IngressPolicyAnnotation = "oci-native-ingress.oraclecloud.com/policy" - IngressClassWafPolicyAnnotation = "oci-native-ingress.oraclecloud.com/waf-policy-ocid" - IngressClassFireWallIdAnnotation = "oci-native-ingress.oraclecloud.com/firewall-id" + IngressPolicyAnnotation = "oci-native-ingress.oraclecloud.com/policy" + IngressClassWafPolicyAnnotation = "oci-native-ingress.oraclecloud.com/waf-policy-ocid" + IngressClassFireWallIdAnnotation = "oci-native-ingress.oraclecloud.com/firewall-id" + IngressClassNetworkSecurityGroupIdsAnnotation = "oci-native-ingress.oraclecloud.com/network-security-group-ids" + IngressClassDeleteProtectionEnabledAnnotation = "oci-native-ingress.oraclecloud.com/delete-protection-enabled" IngressHealthCheckProtocolAnnotation = "oci-native-ingress.oraclecloud.com/healthcheck-protocol" IngressHealthCheckPortAnnotation = "oci-native-ingress.oraclecloud.com/healthcheck-port" @@ -153,6 +155,40 @@ func GetIngressClassFireWallId(ic *networkingv1.IngressClass) string { return value } +func GetIngressClassNetworkSecurityGroupIds(ic *networkingv1.IngressClass) []string { + networkSecurityGroupIds := make([]string, 0) + value, ok := ic.Annotations[IngressClassNetworkSecurityGroupIdsAnnotation] + if !ok { + return networkSecurityGroupIds + } + + for _, id := range strings.Split(value, ",") { + trimmedId := strings.TrimSpace(id) + if trimmedId != "" { + networkSecurityGroupIds = append(networkSecurityGroupIds, trimmedId) + } + } + + return networkSecurityGroupIds +} + +func GetIngressClassDeleteProtectionEnabled(ic *networkingv1.IngressClass) bool { + annotation := IngressClassDeleteProtectionEnabledAnnotation + value, ok := ic.Annotations[annotation] + + if !ok || strings.TrimSpace(value) == "" { + return false + } + + result, err := strconv.ParseBool(value) + if err != nil { + klog.Errorf("Error parsing value %s for flag %s as boolean. Setting the default value as 'false'", value, annotation) + return false + } + + return result +} + func GetIngressProtocol(i *networkingv1.Ingress) string { protocol, ok := i.Annotations[IngressProtocolAnnotation] if !ok { @@ -719,3 +755,9 @@ func IsBackendServiceEqual(b1 *networkingv1.IngressBackend, b2 *networkingv1.Ing func IsIngressProtocolTCP(ingress *networkingv1.Ingress) bool { return GetIngressProtocol(ingress) == ProtocolTCP } + +// StringSlicesHaveSameElements checks if s1 and s2 have the same elements, ignoring order and duplicates. +// Returns true if one slice is nil and the other is empty. +func StringSlicesHaveSameElements(s1 []string, s2 []string) bool { + return sets.New(s1...).Equal(sets.New(s2...)) +} diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index aa65bd22..a3e19011 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -149,6 +149,63 @@ func TestGetIngressProtocol(t *testing.T) { Expect(result).Should(Equal(ProtocolHTTP)) } +func TestGetIngressClassNetworkSecurityGroupIds(t *testing.T) { + RegisterTestingT(t) + + getIngressClassWithNsgAnnotation := func(annotation string) *networkingv1.IngressClass { + return &networkingv1.IngressClass{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{IngressClassNetworkSecurityGroupIdsAnnotation: annotation}, + }, + } + } + + ingressClassWithNoAnnotation := &networkingv1.IngressClass{ + ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{}}, + } + ingressClassWithZeroNsg := getIngressClassWithNsgAnnotation("") + ingressClassWithOneNsg := getIngressClassWithNsgAnnotation(" id") + ingressClassWithMultipleNsg := getIngressClassWithNsgAnnotation(" id1, id2,id3,id4 ") + ingressClassWithRedundantCommas := getIngressClassWithNsgAnnotation(" id1, , id2,, id3,id4,, ") + + Expect(GetIngressClassNetworkSecurityGroupIds(ingressClassWithNoAnnotation)). + Should(Equal([]string{})) + Expect(GetIngressClassNetworkSecurityGroupIds(ingressClassWithZeroNsg)). + Should(Equal([]string{})) + Expect(GetIngressClassNetworkSecurityGroupIds(ingressClassWithOneNsg)). + Should(Equal([]string{"id"})) + Expect(GetIngressClassNetworkSecurityGroupIds(ingressClassWithMultipleNsg)). + Should(Equal([]string{"id1", "id2", "id3", "id4"})) + Expect(GetIngressClassNetworkSecurityGroupIds(ingressClassWithRedundantCommas)). + Should(Equal([]string{"id1", "id2", "id3", "id4"})) +} + +func TestGetIngressClassDeleteProtectionEnabled(t *testing.T) { + RegisterTestingT(t) + + getIngressClassWithDeleteProtectionEnabledAnnotation := func(annotation string) *networkingv1.IngressClass { + return &networkingv1.IngressClass{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{IngressClassDeleteProtectionEnabledAnnotation: annotation}, + }, + } + } + + ingressClassWithNoAnnotation := &networkingv1.IngressClass{ + ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{}}, + } + ingressClassWithEmptyAnnotation := getIngressClassWithDeleteProtectionEnabledAnnotation("") + ingressClassWithProtectionEnabled := getIngressClassWithDeleteProtectionEnabledAnnotation("true") + ingressClassWithProtectionDisabled := getIngressClassWithDeleteProtectionEnabledAnnotation("false") + ingressClassWithWrongAnnotation := getIngressClassWithDeleteProtectionEnabledAnnotation("n/a") + + Expect(GetIngressClassDeleteProtectionEnabled(ingressClassWithNoAnnotation)).Should(BeFalse()) + Expect(GetIngressClassDeleteProtectionEnabled(ingressClassWithEmptyAnnotation)).Should(BeFalse()) + Expect(GetIngressClassDeleteProtectionEnabled(ingressClassWithProtectionEnabled)).Should(BeTrue()) + Expect(GetIngressClassDeleteProtectionEnabled(ingressClassWithProtectionDisabled)).Should(BeFalse()) + Expect(GetIngressClassDeleteProtectionEnabled(ingressClassWithWrongAnnotation)).Should(BeFalse()) +} + func TestGetIngressClassLoadBalancerId(t *testing.T) { RegisterTestingT(t) lbId := "lbId" @@ -758,3 +815,30 @@ func TestIsBackendServiceEqual(t *testing.T) { Expect(IsBackendServiceEqual(b2, b4)).To(BeFalse()) Expect(IsBackendServiceEqual(b3, b4)).To(BeFalse()) } + +func TestStringSlicesHaveSameElements(t *testing.T) { + RegisterTestingT(t) + + var nilSlice []string = nil + emptySlice := make([]string, 0) + exampleSlice := []string{"a", "b", "d"} + exampleSliceGroup := [][]string{{"a", "b", "c"}, {"a", "c", "b"}, {"c", "b", "a"}, {"a", "b", "a", "c", "c"}} + + Expect(StringSlicesHaveSameElements(nilSlice, nilSlice)).Should(BeTrue()) + Expect(StringSlicesHaveSameElements(nilSlice, emptySlice)).Should(BeTrue()) + Expect(StringSlicesHaveSameElements(emptySlice, nilSlice)).Should(BeTrue()) + Expect(StringSlicesHaveSameElements(emptySlice, emptySlice)).Should(BeTrue()) + + Expect(StringSlicesHaveSameElements(nilSlice, exampleSlice)).Should(BeFalse()) + Expect(StringSlicesHaveSameElements(exampleSlice, emptySlice)).Should(BeFalse()) + + for _, val1 := range exampleSliceGroup { + for _, val2 := range exampleSliceGroup { + Expect(StringSlicesHaveSameElements(val1, val2)).Should(BeTrue()) + } + } + + for _, v := range exampleSliceGroup { + Expect(StringSlicesHaveSameElements(v, exampleSlice)).Should(BeFalse()) + } +}