Skip to content

Commit

Permalink
add tagging support for IngressClass
Browse files Browse the repository at this point in the history 
- Add support for defined and freeform tags via IngressClass annotations
- Swap opc-retry-token in UpdateLoadbalancer call in IngressClass controller logic for an ifMatch
  • Loading branch information
@piyush-tiwari
piyush-tiwari committed Oct 22, 2024
1 parent a48822c commit e74b161
Show file tree
Hide file tree
Showing 5 changed files with 248 additions and 73 deletions.
176 changes: 109 additions & 67 deletions pkg/controllers/ingressclass/ingressclass.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"fmt"
coreinformers "k8s.io/client-go/informers/core/v1"
corelisters "k8s.io/client-go/listers/core/v1"
"reflect"
"time"

"github.com/oracle/oci-native-ingress-controller/pkg/client"
Expand Down Expand Up @@ -234,8 +235,7 @@ func (c *Controller) getLoadBalancer(ctx context.Context, ic *networkingv1.Ingre
}

func (c *Controller) ensureLoadBalancer(ctx context.Context, ic *networkingv1.IngressClass) error {

lb, etag, err := c.getLoadBalancer(ctx, ic)
lb, _, err := c.getLoadBalancer(ctx, ic)
if err != nil {
return err
}
Expand All @@ -255,55 +255,13 @@ func (c *Controller) ensureLoadBalancer(ctx context.Context, ic *networkingv1.In
}
}

wrapperClient, ok := ctx.Value(util.WrapperClient).(*client.WrapperClient)
if !ok {
return fmt.Errorf(util.OciClientNotFoundInContextError)
}
compartmentId := common.String(util.GetIngressClassCompartmentId(icp, c.defaultCompartmentId))
if lb == nil {
klog.V(2).InfoS("Creating load balancer for ingress class", "ingressClass", ic.Name)

createDetails := ociloadbalancer.CreateLoadBalancerDetails{
CompartmentId: compartmentId,
DisplayName: common.String(util.GetIngressClassLoadBalancerName(ic, icp)),
ShapeName: common.String("flexible"),
SubnetIds: []string{util.GetIngressClassSubnetId(icp, c.defaultSubnetId)},
IsPrivate: common.Bool(icp.Spec.IsPrivate),
NetworkSecurityGroupIds: util.GetIngressClassNetworkSecurityGroupIds(ic),
BackendSets: map[string]ociloadbalancer.BackendSetDetails{
util.DefaultBackendSetName: {
Policy: common.String("LEAST_CONNECTIONS"),
HealthChecker: &ociloadbalancer.HealthCheckerDetails{
Protocol: common.String("TCP"),
},
},
},
FreeformTags: map[string]string{OnicResource: "loadbalancer"},
}

if icp.Spec.ReservedPublicAddressId != "" {
createDetails.ReservedIps = []ociloadbalancer.ReservedIp{{Id: common.String(icp.Spec.ReservedPublicAddressId)}}
}

createDetails.ShapeDetails = &ociloadbalancer.ShapeDetails{
MinimumBandwidthInMbps: common.Int(icp.Spec.MinBandwidthMbps),
MaximumBandwidthInMbps: common.Int(icp.Spec.MaxBandwidthMbps),
}

createLbRequest := ociloadbalancer.CreateLoadBalancerRequest{
// Use UID as retry token so multiple requests in 24 hours won't recreate the same LoadBalancer,
// but recreate of the IngressClass will trigger an LB within 24 hours.
// If you used ingress class name it would disallow creation of more LB's even in different clusters potentially.
OpcRetryToken: common.String(fmt.Sprintf("create-lb-%s", ic.UID)),
CreateLoadBalancerDetails: createDetails,
}
klog.Infof("Create lb request: %s", util.PrettyPrint(createLbRequest))
lb, err = wrapperClient.GetLbClient().CreateLoadBalancer(context.Background(), createLbRequest)
lb, err = c.createLoadBalancer(ctx, ic, icp)
if err != nil {
return err
return fmt.Errorf("unable to create LoadBalancer for IngressClass %s: %w", ic.Name, err)
}
} else {
err = c.checkForIngressClassParameterUpdates(ctx, lb, ic, icp, etag)
err = c.checkForIngressClassParameterUpdates(ctx, ic, icp)
if err != nil {
return err
}
Expand All @@ -314,6 +272,12 @@ func (c *Controller) ensureLoadBalancer(ctx context.Context, ic *networkingv1.In
}
}

wrapperClient, ok := ctx.Value(util.WrapperClient).(*client.WrapperClient)
if !ok {
return fmt.Errorf(util.OciClientNotFoundInContextError)
}
compartmentId := common.String(util.GetIngressClassCompartmentId(icp, c.defaultCompartmentId))

if *lb.Id != util.GetIngressClassLoadBalancerId(ic) {
klog.InfoS("Adding load balancer id to ingress class", "lbId", *lb.Id, "ingressClass", klog.KObj(ic))
patchError, done := util.PatchIngressClassWithAnnotation(wrapperClient.GetK8Client(), ic, util.IngressClassLoadBalancerIdAnnotation, *lb.Id)
Expand All @@ -334,6 +298,68 @@ func (c *Controller) ensureLoadBalancer(ctx context.Context, ic *networkingv1.In
return nil
}

func (c *Controller) createLoadBalancer(ctx context.Context, ic *networkingv1.IngressClass,
icp *v1beta1.IngressClassParameters) (*ociloadbalancer.LoadBalancer, error) {
klog.V(2).InfoS("Creating load balancer for ingress class", "ingressClass", ic.Name)
compartmentId := common.String(util.GetIngressClassCompartmentId(icp, c.defaultCompartmentId))
definedTags, err := util.GetIngressClassDefinedTags(ic)
if err != nil {
return nil, err
}
freeformTags, err := util.GetIngressClassFreeformTags(ic)
if err != nil {
return nil, err
}

createDetails := ociloadbalancer.CreateLoadBalancerDetails{
CompartmentId: compartmentId,
DisplayName: common.String(util.GetIngressClassLoadBalancerName(ic, icp)),
ShapeName: common.String("flexible"),
SubnetIds: []string{util.GetIngressClassSubnetId(icp, c.defaultSubnetId)},
IsPrivate: common.Bool(icp.Spec.IsPrivate),
NetworkSecurityGroupIds: util.GetIngressClassNetworkSecurityGroupIds(ic),
BackendSets: map[string]ociloadbalancer.BackendSetDetails{
util.DefaultBackendSetName: {
Policy: common.String("LEAST_CONNECTIONS"),
HealthChecker: &ociloadbalancer.HealthCheckerDetails{
Protocol: common.String("TCP"),
},
},
},
FreeformTags: freeformTags,
DefinedTags: definedTags,
}

if icp.Spec.ReservedPublicAddressId != "" {
createDetails.ReservedIps = []ociloadbalancer.ReservedIp{{Id: common.String(icp.Spec.ReservedPublicAddressId)}}
}

createDetails.ShapeDetails = &ociloadbalancer.ShapeDetails{
MinimumBandwidthInMbps: common.Int(icp.Spec.MinBandwidthMbps),
MaximumBandwidthInMbps: common.Int(icp.Spec.MaxBandwidthMbps),
}

createLbRequest := ociloadbalancer.CreateLoadBalancerRequest{
// Use UID as retry token so multiple requests in 24 hours won't recreate the same LoadBalancer,
// but recreate of the IngressClass will trigger an LB within 24 hours.
// If you used ingress class name it would disallow creation of more LB's even in different clusters potentially.
OpcRetryToken: common.String(fmt.Sprintf("create-lb-%s", ic.UID)),
CreateLoadBalancerDetails: createDetails,
}
klog.Infof("Create lb request: %s", util.PrettyPrint(createLbRequest))

wrapperClient, ok := ctx.Value(util.WrapperClient).(*client.WrapperClient)
if !ok {
return nil, fmt.Errorf(util.OciClientNotFoundInContextError)
}
lb, err := wrapperClient.GetLbClient().CreateLoadBalancer(context.Background(), createLbRequest)
if err != nil {
return nil, err
}

return lb, nil
}

func (c *Controller) setupWebApplicationFirewall(ctx context.Context, ic *networkingv1.IngressClass, compartmentId *string, lbId *string) error {
wrapperClient, ok := ctx.Value(util.WrapperClient).(*client.WrapperClient)
if !ok {
Expand All @@ -353,33 +379,43 @@ func (c *Controller) setupWebApplicationFirewall(ctx context.Context, ic *networ
return nil
}

func (c *Controller) checkForIngressClassParameterUpdates(ctx context.Context, lb *ociloadbalancer.LoadBalancer,
ic *networkingv1.IngressClass, icp *v1beta1.IngressClassParameters, etag string) error {
// check LoadBalancerName AND MinBandwidthMbps ,MaxBandwidthMbps
displayName := util.GetIngressClassLoadBalancerName(ic, icp)
func (c *Controller) checkForIngressClassParameterUpdates(ctx context.Context, ic *networkingv1.IngressClass, icp *v1beta1.IngressClassParameters) error {
lb, etag, err := c.getLoadBalancer(ctx, ic)
if err != nil {
return err
}

wrapperClient, ok := ctx.Value(util.WrapperClient).(*client.WrapperClient)
if !ok {
return fmt.Errorf(util.OciClientNotFoundInContextError)
}
if *lb.DisplayName != displayName {

detail := ociloadbalancer.UpdateLoadBalancerDetails{
DisplayName: &displayName,
}
req := ociloadbalancer.UpdateLoadBalancerRequest{
OpcRetryToken: common.String(fmt.Sprintf("update-lb-detail-%s", ic.UID)),
UpdateLoadBalancerDetails: detail,
LoadBalancerId: lb.Id,
}
// check LoadBalancerName, Defined and Freeform tags
displayName := util.GetIngressClassLoadBalancerName(ic, icp)
definedTags, err := util.GetIngressClassDefinedTags(ic)
if err != nil {
return err
}
freeformTags, err := util.GetIngressClassFreeformTags(ic)
if err != nil {
return err
}

klog.Infof("Update lb details request: %s", util.PrettyPrint(req))
_, err := wrapperClient.GetLbClient().UpdateLoadBalancer(context.Background(), req)
if *lb.DisplayName != displayName || !reflect.DeepEqual(lb.DefinedTags, definedTags) || !reflect.DeepEqual(lb.FreeformTags, freeformTags) {
_, err = wrapperClient.GetLbClient().UpdateLoadBalancer(context.Background(), *lb.Id, displayName, definedTags, freeformTags)
if err != nil {
return err
}

}

// refresh lb, etag information after last call
lb, etag, err = c.getLoadBalancer(ctx, ic)
if err != nil {
return err
}

// check LB Shape
if *lb.ShapeDetails.MaximumBandwidthInMbps != icp.Spec.MaxBandwidthMbps ||
*lb.ShapeDetails.MinimumBandwidthInMbps != icp.Spec.MinBandwidthMbps {
shapeDetails := &ociloadbalancer.ShapeDetails{
Expand Down Expand Up @@ -460,7 +496,7 @@ func (c *Controller) clearLoadBalancer(ctx context.Context, ic *networkingv1.Ing
}

if lb == nil {
klog.Infof("Tried to clear LB for ic %s/%s, but it is deleted", ic.Namespace, ic.Name)
klog.Infof("Tried to clear LB for ic %s, but it is deleted", ic.Name)
return nil
}

Expand All @@ -478,15 +514,21 @@ func (c *Controller) clearLoadBalancer(ctx context.Context, ic *networkingv1.Ing
if len(nsgIds) > 0 {
_, err = wrapperClient.GetLbClient().UpdateNetworkSecurityGroups(context.Background(), *lb.Id, make([]string, 0))
if err != nil {
klog.Errorf("While clearing LB %s, cannot clear NSG IDs due to %s, will proceed with IngressClass deletion for %s/%s",
*lb.Id, err.Error(), ic.Namespace, ic.Name)
klog.Errorf("While clearing LB %s, cannot clear NSG IDs due to %s, will proceed with IngressClass deletion for %s",
*lb.Id, err.Error(), ic.Name)
}
}

err = wrapperClient.GetLbClient().DeleteBackendSet(context.Background(), *lb.Id, util.DefaultBackendSetName)
if err != nil {
klog.Errorf("While clearing LB %s, cannot clear BackendSet %s due to %s, will proceed with IngressClass deletion for %s/%s",
*lb.Id, util.DefaultBackendSetName, err.Error(), ic.Namespace, ic.Name)
klog.Errorf("While clearing LB %s, cannot clear BackendSet %s due to %s, will proceed with IngressClass deletion for %s",
*lb.Id, util.DefaultBackendSetName, err.Error(), ic.Name)
}

_, err = wrapperClient.GetLbClient().UpdateLoadBalancer(context.Background(), *lb.Id, *lb.DisplayName, map[string]map[string]interface{}{}, map[string]string{})
if err != nil {
klog.Errorf("While clearing LB %s, cannot clear tags due to %s, will proceed with IngressClass deletion for %s",
*lb.Id, err.Error(), ic.Name)
}

return nil
Expand Down
7 changes: 2 additions & 5 deletions pkg/controllers/ingressclass/ingressclass_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,11 +202,8 @@ func TestCheckForIngressClassParameterUpdates(t *testing.T) {
RegisterTestingT(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ingressClassList := util.GetIngressClassList()
ingressClassList := util.GetIngressClassListWithLBSet("id")
c := inits(ctx, ingressClassList)
mockClient, err := c.client.GetClient(&MockConfigGetter{})
Expect(err).To(BeNil())
loadBalancer, _, _ := mockClient.GetLbClient().GetLoadBalancer(context.TODO(), "id")
icp := v1beta1.IngressClassParameters{
Spec: v1beta1.IngressClassParametersSpec{
CompartmentId: "",
Expand All @@ -217,7 +214,7 @@ func TestCheckForIngressClassParameterUpdates(t *testing.T) {
MaxBandwidthMbps: 400,
},
}
err = c.checkForIngressClassParameterUpdates(getContextWithClient(c, ctx), loadBalancer, &ingressClassList.Items[0], &icp, "etag")
err := c.checkForIngressClassParameterUpdates(getContextWithClient(c, ctx), &ingressClassList.Items[0], &icp)
Expect(err).Should(BeNil())
}

Expand Down
22 changes: 21 additions & 1 deletion pkg/loadbalancer/loadbalancer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,9 @@ func (lbc *LoadBalancerClient) GetBackendSetHealth(ctx context.Context, lbID str

func (lbc *LoadBalancerClient) UpdateNetworkSecurityGroups(ctx context.Context, lbId string, nsgIds []string) (loadbalancer.UpdateNetworkSecurityGroupsResponse, error) {
_, etag, err := lbc.GetLoadBalancer(ctx, lbId)
if err != nil {
return loadbalancer.UpdateNetworkSecurityGroupsResponse{}, err
}

req := loadbalancer.UpdateNetworkSecurityGroupsRequest{
LoadBalancerId: common.String(lbId),
Expand Down Expand Up @@ -128,7 +131,24 @@ func (lbc *LoadBalancerClient) UpdateLoadBalancerShape(ctx context.Context, req
return resp, err
}

func (lbc *LoadBalancerClient) UpdateLoadBalancer(ctx context.Context, req loadbalancer.UpdateLoadBalancerRequest) (*loadbalancer.LoadBalancer, error) {
func (lbc *LoadBalancerClient) UpdateLoadBalancer(ctx context.Context, lbId string, displayName string, definedTags map[string]map[string]interface{},
freeformTags map[string]string) (*loadbalancer.LoadBalancer, error) {
_, etag, err := lbc.GetLoadBalancer(ctx, lbId)
if err != nil {
return nil, err
}

req := loadbalancer.UpdateLoadBalancerRequest{
LoadBalancerId: common.String(lbId),
IfMatch: common.String(etag),
UpdateLoadBalancerDetails: loadbalancer.UpdateLoadBalancerDetails{
DisplayName: common.String(displayName),
DefinedTags: definedTags,
FreeformTags: freeformTags,
},
}

klog.Infof("Update lb details request: %s", util.PrettyPrint(req))
resp, err := lbc.LbClient.UpdateLoadBalancer(ctx, req)
if err != nil {
return nil, err
Expand Down
40 changes: 40 additions & 0 deletions pkg/util/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ const (
IngressClassFireWallIdAnnotation = "oci-native-ingress.oraclecloud.com/firewall-id"
IngressClassNetworkSecurityGroupIdsAnnotation = "oci-native-ingress.oraclecloud.com/network-security-group-ids"
IngressClassDeleteProtectionEnabledAnnotation = "oci-native-ingress.oraclecloud.com/delete-protection-enabled"
IngressClassDefinedTagsAnnotation = "oci-native-ingress.oraclecloud.com/defined-tags"
IngressClassFreeformTagsAnnotation = "oci-native-ingress.oraclecloud.com/freeform-tags"

IngressHealthCheckProtocolAnnotation = "oci-native-ingress.oraclecloud.com/healthcheck-protocol"
IngressHealthCheckPortAnnotation = "oci-native-ingress.oraclecloud.com/healthcheck-port"
Expand Down Expand Up @@ -189,6 +191,44 @@ func GetIngressClassDeleteProtectionEnabled(ic *networkingv1.IngressClass) bool
return result
}

func GetIngressClassDefinedTags(ic *networkingv1.IngressClass) (map[string]map[string]interface{}, error) {
annotation := IngressClassDefinedTagsAnnotation
value, ok := ic.Annotations[annotation]

// value of defined tags can only be strings for now, but we will allow anything that fits the type
// specified by LoadBalancer.DefinedTags
definedTags := map[string]map[string]interface{}{}

if !ok || strings.TrimSpace(value) == "" {
return definedTags, nil
}

err := json.Unmarshal([]byte(value), &definedTags)
if err != nil {
return nil, fmt.Errorf("error parsing value %s for annotation %s: %w", value, annotation, err)
}

return definedTags, nil
}

func GetIngressClassFreeformTags(ic *networkingv1.IngressClass) (map[string]string, error) {
annotation := IngressClassFreeformTagsAnnotation
value, ok := ic.Annotations[annotation]

freeformTags := map[string]string{}

if !ok || strings.TrimSpace(value) == "" {
return freeformTags, nil
}

err := json.Unmarshal([]byte(value), &freeformTags)
if err != nil {
return nil, fmt.Errorf("error parsing value %s for annotation %s: %w", value, annotation, err)
}

return freeformTags, nil
}

func GetIngressProtocol(i *networkingv1.Ingress) string {
protocol, ok := i.Annotations[IngressProtocolAnnotation]
if !ok {
Expand Down
Loading

0 comments on commit e74b161

Please sign in to comment.