Skip to content

Commit

Permalink
OKE-26958 Web application firewall integration (#14)
Browse files Browse the repository at this point in the history 
* Added Web Application Firewall Support
* OKE-27261-Fixed-multi-path-bug
  • Loading branch information
@Inbaraj-S
Inbaraj-S authored Aug 17, 2023
1 parent 8202620 commit ad04c22
Show file tree
Hide file tree
Showing 108 changed files with 11,454 additions and 173 deletions.
35 changes: 35 additions & 0 deletions pkg/client/client.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package client

import (
"github.com/oracle/oci-native-ingress-controller/pkg/certificate"
"github.com/oracle/oci-native-ingress-controller/pkg/loadbalancer"
"github.com/oracle/oci-native-ingress-controller/pkg/waf"
"k8s.io/client-go/kubernetes"
)

type ClientProvider struct {
kubernetesClient kubernetes.Interface
wafClient *waf.Client
lbClient *loadbalancer.LoadBalancerClient
certificatesClient *certificate.CertificatesClient
}

func NewWrapperClient(kubernetesClient kubernetes.Interface, wafClient *waf.Client, lbClient *loadbalancer.LoadBalancerClient, certificatesClient *certificate.CertificatesClient) *ClientProvider {
return &ClientProvider{kubernetesClient: kubernetesClient, wafClient: wafClient, lbClient: lbClient, certificatesClient: certificatesClient}
}

func (c ClientProvider) GetK8Client() kubernetes.Interface {
return c.kubernetesClient
}

func (c ClientProvider) GetWafClient() *waf.Client {
return c.wafClient
}

func (c ClientProvider) GetLbClient() *loadbalancer.LoadBalancerClient {
return c.lbClient
}

func (c ClientProvider) GetCertClient() *certificate.CertificatesClient {
return c.certificatesClient
}
25 changes: 9 additions & 16 deletions pkg/controllers/backend/backend.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ import (
"fmt"
"time"

"github.com/oracle/oci-native-ingress-controller/pkg/client"
"github.com/oracle/oci-native-ingress-controller/pkg/controllers/ingressclass"

"k8s.io/klog/v2"

"github.com/oracle/oci-go-sdk/v65/common"
Expand All @@ -33,13 +33,11 @@ import (
"k8s.io/apimachinery/pkg/util/strategicpatch"
"k8s.io/apimachinery/pkg/util/wait"
networkinginformers "k8s.io/client-go/informers/networking/v1"
"k8s.io/client-go/kubernetes"
corelisters "k8s.io/client-go/listers/core/v1"
networkinglisters "k8s.io/client-go/listers/networking/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/workqueue"

"github.com/oracle/oci-native-ingress-controller/pkg/loadbalancer"
"github.com/oracle/oci-native-ingress-controller/pkg/util"
)

Expand All @@ -54,11 +52,8 @@ type Controller struct {
podLister corelisters.PodLister
endpointLister corelisters.EndpointsLister

queue workqueue.RateLimitingInterface

clientset kubernetes.Interface

lbClient *loadbalancer.LoadBalancerClient
queue workqueue.RateLimitingInterface
client *client.ClientProvider
}

func NewController(
Expand All @@ -68,8 +63,7 @@ func NewController(
serviceLister corelisters.ServiceLister,
endpointLister corelisters.EndpointsLister,
podLister corelisters.PodLister,
client kubernetes.Interface,
lbClient *loadbalancer.LoadBalancerClient,
client *client.ClientProvider,
) *Controller {

c := &Controller{
Expand All @@ -79,8 +73,7 @@ func NewController(
serviceLister: serviceLister,
endpointLister: endpointLister,
podLister: podLister,
clientset: client,
lbClient: lbClient,
client: client,
queue: workqueue.NewRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(10*time.Second, 5*time.Minute)),
}

Expand Down Expand Up @@ -207,12 +200,12 @@ func (c *Controller) ensureBackends(ingressClass *networkingv1.IngressClass, lbI
}

backendSetName := util.GenerateBackendSetName(ingress.Namespace, svcName, svcPort)
err = c.lbClient.UpdateBackends(context.TODO(), lbID, backendSetName, backends)
err = c.client.GetLbClient().UpdateBackends(context.TODO(), lbID, backendSetName, backends)
if err != nil {
return fmt.Errorf("unable to update backends for %s/%s: %w", ingressClass.Name, backendSetName, err)
}

backendSetHealth, err := c.lbClient.GetBackendSetHealth(context.TODO(), lbID, backendSetName)
backendSetHealth, err := c.client.GetLbClient().GetBackendSetHealth(context.TODO(), lbID, backendSetName)
if err != nil {
return fmt.Errorf("unable to fetch backendset health: %w", err)
}
Expand Down Expand Up @@ -249,7 +242,7 @@ func (c *Controller) syncDefaultBackend(lbID string, ingresses []*networkingv1.I
return nil
}

err = c.lbClient.UpdateBackends(context.TODO(), lbID, ingressclass.DefaultIngress, backends)
err = c.client.GetLbClient().UpdateBackends(context.TODO(), lbID, ingressclass.DefaultIngress, backends)
if err != nil {
return err
}
Expand Down Expand Up @@ -378,7 +371,7 @@ func (c *Controller) ensurePodReadinessCondition(pod *corev1.Pod, readinessGate
return fmt.Errorf("unable to build pod condition for %s/%s: %w", pod.Namespace, pod.Name, err)
}

_, err = c.clientset.CoreV1().Pods(pod.Namespace).Patch(context.TODO(), pod.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status")
_, err = c.client.GetK8Client().CoreV1().Pods(pod.Namespace).Patch(context.TODO(), pod.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status")
if err != nil {
return fmt.Errorf("unable to remove readiness gate %s from pod %s/%s: %w", readinessGate, pod.Namespace, pod.Name, err)
}
Expand Down
10 changes: 6 additions & 4 deletions pkg/controllers/backend/backend_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,9 @@ import (
. "github.com/onsi/gomega"
"github.com/oracle/oci-go-sdk/v65/common"
ociloadbalancer "github.com/oracle/oci-go-sdk/v65/loadbalancer"
"github.com/oracle/oci-native-ingress-controller/pkg/client"
lb "github.com/oracle/oci-native-ingress-controller/pkg/loadbalancer"
"github.com/oracle/oci-native-ingress-controller/pkg/oci/client"
ociclient "github.com/oracle/oci-native-ingress-controller/pkg/oci/client"
"github.com/oracle/oci-native-ingress-controller/pkg/util"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
Expand Down Expand Up @@ -187,8 +188,9 @@ func inits(ctx context.Context, ingressClassList *networkingv1.IngressClassList,
Cache: map[string]*lb.LbCacheObj{},
}

ingressClassInformer, ingressInformer, serviceLister, endpointLister, podLister, client := setUp(ctx, ingressClassList, ingressList, testService, endpoints, pod)
c := NewController("oci.oraclecloud.com/native-ingress-controller", ingressClassInformer, ingressInformer, serviceLister, endpointLister, podLister, client, loadBalancerClient)
ingressClassInformer, ingressInformer, serviceLister, endpointLister, podLister, k8client := setUp(ctx, ingressClassList, ingressList, testService, endpoints, pod)
client := client.NewWrapperClient(k8client, nil, loadBalancerClient, nil)
c := NewController("oci.oraclecloud.com/native-ingress-controller", ingressClassInformer, ingressInformer, serviceLister, endpointLister, podLister, client)
return c
}

Expand Down Expand Up @@ -238,7 +240,7 @@ func TestBuildPodConditionPatch(t *testing.T) {
Expect(bytes.Equal(patch, []byte("{\"status\":{\"conditions\":[{\"lastProbeTime\":null,\"lastTransitionTime\":null,\"status\":\"True\",\"type\":\"ContainersReady\"}]}}"))).Should(Equal(true))
}

func getLoadBalancerClient() client.LoadBalancerInterface {
func getLoadBalancerClient() ociclient.LoadBalancerInterface {
return &MockLoadBalancerClient{}
}

Expand Down
53 changes: 24 additions & 29 deletions pkg/controllers/ingress/ingress.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"reflect"
"time"

"github.com/oracle/oci-native-ingress-controller/pkg/client"
"github.com/prometheus/client_golang/prometheus"

"github.com/pkg/errors"
Expand All @@ -35,7 +36,6 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
networkinginformers "k8s.io/client-go/informers/networking/v1"
"k8s.io/client-go/kubernetes"
corelisters "k8s.io/client-go/listers/core/v1"
networkinglisters "k8s.io/client-go/listers/networking/v1"
"k8s.io/client-go/tools/cache"
Expand All @@ -57,18 +57,15 @@ type Controller struct {
serviceLister corelisters.ServiceLister
queue workqueue.RateLimitingInterface
informer networkinginformers.IngressInformer
client kubernetes.Interface

lbClient *loadbalancer.LoadBalancerClient
certificatesClient *certificate.CertificatesClient
client *client.ClientProvider
metricsCollector *metric.IngressCollector
}

// NewController creates a new Controller.
func NewController(controllerClass string, defaultCompartmentId string,
ingressClassInformer networkinginformers.IngressClassInformer, ingressInformer networkinginformers.IngressInformer,
serviceLister corelisters.ServiceLister,
client kubernetes.Interface, lbClient *loadbalancer.LoadBalancerClient, certificatesClient *certificate.CertificatesClient,
client *client.ClientProvider,
reg *prometheus.Registry) *Controller {

c := &Controller{
Expand All @@ -79,8 +76,6 @@ func NewController(controllerClass string, defaultCompartmentId string,
informer: ingressInformer,
serviceLister: serviceLister,
client: client,
lbClient: lbClient,
certificatesClient: certificatesClient,
queue: workqueue.NewRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(10*time.Second, 5*time.Minute)),
metricsCollector: metric.NewIngressCollector(controllerClass, reg),
}
Expand Down Expand Up @@ -248,7 +243,7 @@ func (c *Controller) sync(key string) error {

func (c *Controller) ensureLoadBalancerIP(lbID string, ingress *networkingv1.Ingress) error {

lb, _, err := c.lbClient.GetLoadBalancer(context.TODO(), lbID)
lb, _, err := c.client.GetLbClient().GetLoadBalancer(context.TODO(), lbID)
if err != nil {
return errors.Wrapf(err, "unable to fetch ip from load balancer: %s", err.Error())
}
Expand Down Expand Up @@ -276,7 +271,7 @@ func (c *Controller) ensureLoadBalancerIP(lbID string, ingress *networkingv1.Ing
klog.V(2).InfoS("adding ip address to ingress", "ingress", klog.KObj(ingress), "ipAddress", ipAddress)

err = retry.RetryOnConflict(retry.DefaultBackoff, func() error {
latest, err := c.client.NetworkingV1().Ingresses(ingress.Namespace).Get(context.TODO(), ingress.Name, metav1.GetOptions{})
latest, err := c.client.GetK8Client().NetworkingV1().Ingresses(ingress.Namespace).Get(context.TODO(), ingress.Name, metav1.GetOptions{})
if err != nil {
return err
}
Expand All @@ -285,7 +280,7 @@ func (c *Controller) ensureLoadBalancerIP(lbID string, ingress *networkingv1.Ing
{IP: ipAddress},
}

_, err = c.client.NetworkingV1().Ingresses(ingress.Namespace).UpdateStatus(context.TODO(), latest, metav1.UpdateOptions{})
_, err = c.client.GetK8Client().NetworkingV1().Ingresses(ingress.Namespace).UpdateStatus(context.TODO(), latest, metav1.UpdateOptions{})
return err
})

Expand Down Expand Up @@ -316,7 +311,7 @@ func (c *Controller) ensureIngress(ingress *networkingv1.Ingress, ingressClass *

lbId := util.GetIngressClassLoadBalancerId(ingressClass)

lb, _, err := c.lbClient.GetLoadBalancer(context.TODO(), lbId)
lb, _, err := c.client.GetLbClient().GetLoadBalancer(context.TODO(), lbId)
if err != nil {
return err
}
Expand All @@ -337,14 +332,14 @@ func (c *Controller) ensureIngress(ingress *networkingv1.Ingress, ingressClass *
startBuildTime := util.GetCurrentTimeInUnixMillis()
klog.V(2).InfoS("creating backend set for ingress", "ingress", klog.KObj(ingress), "backendSetName", bsName)
artifact, artifactType := stateStore.GetTLSConfigForBackendSet(bsName)
backendSetSslConfig, err := certificate.GetSSLConfigForBackendSet(ingress.Namespace, artifactType, artifact, lb, bsName, c.defaultCompartmentId, c.certificatesClient, c.client)
backendSetSslConfig, err := certificate.GetSSLConfigForBackendSet(ingress.Namespace, artifactType, artifact, lb, bsName, c.defaultCompartmentId, c.client.GetCertClient(), c.client.GetK8Client())
if err != nil {
return err
}

healthChecker := stateStore.GetBackendSetHealthChecker(bsName)
policy := stateStore.GetBackendSetPolicy(bsName)
err = c.lbClient.CreateBackendSet(context.TODO(), lbId, bsName, policy, healthChecker, backendSetSslConfig)
err = c.client.GetLbClient().CreateBackendSet(context.TODO(), lbId, bsName, policy, healthChecker, backendSetSslConfig)
if err != nil {
return err
}
Expand Down Expand Up @@ -372,13 +367,13 @@ func (c *Controller) ensureIngress(ingress *networkingv1.Ingress, ingressClass *

var listenerSslConfig *ociloadbalancer.SslConfigurationDetails
artifact, artifactType := stateStore.GetTLSConfigForListener(port)
listenerSslConfig, err := certificate.GetSSLConfigForListener(ingress.Namespace, nil, artifactType, artifact, c.defaultCompartmentId, c.certificatesClient, c.client)
listenerSslConfig, err := certificate.GetSSLConfigForListener(ingress.Namespace, nil, artifactType, artifact, c.defaultCompartmentId, c.client.GetCertClient(), c.client.GetK8Client())
if err != nil {
return err
}

protocol := stateStore.GetListenerProtocol(port)
err = c.lbClient.CreateListener(context.TODO(), lbId, int(port), protocol, listenerSslConfig)
err = c.client.GetLbClient().CreateListener(context.TODO(), lbId, int(port), protocol, listenerSslConfig)
if err != nil {
return err
}
Expand All @@ -389,7 +384,7 @@ func (c *Controller) ensureIngress(ingress *networkingv1.Ingress, ingressClass *
return err
}

err = deleteBackendSets(actualBackendSets, desiredBackendSets, c.lbClient, lbId)
err = deleteBackendSets(actualBackendSets, desiredBackendSets, c.client.GetLbClient(), lbId)
if err != nil {
return err
}
Expand All @@ -399,7 +394,7 @@ func (c *Controller) ensureIngress(ingress *networkingv1.Ingress, ingressClass *
return err
}

return deleteListeners(actualListenerPorts, desiredListenerPorts, c.lbClient, lbId)
return deleteListeners(actualListenerPorts, desiredListenerPorts, c.client.GetLbClient(), lbId)
}

func handleIngressDelete(c *Controller, ingressClass *networkingv1.IngressClass) error {
Expand All @@ -412,7 +407,7 @@ func handleIngressDelete(c *Controller, ingressClass *networkingv1.IngressClass)

lbId := util.GetIngressClassLoadBalancerId(ingressClass)

lb, _, err := c.lbClient.GetLoadBalancer(context.TODO(), lbId)
lb, _, err := c.client.GetLbClient().GetLoadBalancer(context.TODO(), lbId)
if err != nil {
return err
}
Expand All @@ -422,7 +417,7 @@ func handleIngressDelete(c *Controller, ingressClass *networkingv1.IngressClass)
actualBackendSets.Insert(bsName)
}

err = deleteBackendSets(actualBackendSets, stateStore.GetAllBackendSetForIngressClass(), c.lbClient, lbId)
err = deleteBackendSets(actualBackendSets, stateStore.GetAllBackendSetForIngressClass(), c.client.GetLbClient(), lbId)
if err != nil {
return err
}
Expand All @@ -432,7 +427,7 @@ func handleIngressDelete(c *Controller, ingressClass *networkingv1.IngressClass)
actualListeners.Insert(int32(*listener.Port))
}

err = deleteListeners(actualListeners, stateStore.GetAllListenersForIngressClass(), c.lbClient, lbId)
err = deleteListeners(actualListeners, stateStore.GetAllListenersForIngressClass(), c.client.GetLbClient(), lbId)
if err != nil {
return err
}
Expand Down Expand Up @@ -474,7 +469,7 @@ func deleteListeners(actualListeners sets.Int32, desiredListeners sets.Int32, lb

func syncListener(namespace string, stateStore *state.StateStore, lbId *string, listenerName string, c *Controller) error {
startTime := util.GetCurrentTimeInUnixMillis()
lb, etag, err := c.lbClient.GetLoadBalancer(context.TODO(), *lbId)
lb, etag, err := c.client.GetLbClient().GetLoadBalancer(context.TODO(), *lbId)
if err != nil {
return err
}
Expand All @@ -488,7 +483,7 @@ func syncListener(namespace string, stateStore *state.StateStore, lbId *string,
artifact, artifactType := stateStore.GetTLSConfigForListener(int32(*listener.Port))
var sslConfig *ociloadbalancer.SslConfigurationDetails
if artifact != "" {
sslConfig, err = certificate.GetSSLConfigForListener(namespace, &listener, artifactType, artifact, c.defaultCompartmentId, c.certificatesClient, c.client)
sslConfig, err = certificate.GetSSLConfigForListener(namespace, &listener, artifactType, artifact, c.defaultCompartmentId, c.client.GetCertClient(), c.client.GetK8Client())
if err != nil {
return err
}
Expand All @@ -509,7 +504,7 @@ func syncListener(namespace string, stateStore *state.StateStore, lbId *string,
}

if needsUpdate {
err := c.lbClient.UpdateListener(context.TODO(), lbId, etag, listener, listener.RoutingPolicyName, sslConfig, &protocol)
err := c.client.GetLbClient().UpdateListener(context.TODO(), lbId, etag, listener, listener.RoutingPolicyName, sslConfig, &protocol)
if err != nil {
return err
}
Expand All @@ -524,7 +519,7 @@ func syncListener(namespace string, stateStore *state.StateStore, lbId *string,
func syncBackendSet(ingress *networkingv1.Ingress, lbID string, backendSetName string, stateStore *state.StateStore, c *Controller) error {

startTime := util.GetCurrentTimeInUnixMillis()
lb, etag, err := c.lbClient.GetLoadBalancer(context.TODO(), lbID)
lb, etag, err := c.client.GetLbClient().GetLoadBalancer(context.TODO(), lbID)
if err != nil {
return err
}
Expand All @@ -536,7 +531,7 @@ func syncBackendSet(ingress *networkingv1.Ingress, lbID string, backendSetName s

needsUpdate := false
artifact, artifactType := stateStore.GetTLSConfigForBackendSet(*bs.Name)
sslConfig, err := certificate.GetSSLConfigForBackendSet(ingress.Namespace, artifactType, artifact, lb, *bs.Name, c.defaultCompartmentId, c.certificatesClient, c.client)
sslConfig, err := certificate.GetSSLConfigForBackendSet(ingress.Namespace, artifactType, artifact, lb, *bs.Name, c.defaultCompartmentId, c.client.GetCertClient(), c.client.GetK8Client())
if err != nil {
return err
}
Expand All @@ -562,7 +557,7 @@ func syncBackendSet(ingress *networkingv1.Ingress, lbID string, backendSetName s
}

if needsUpdate {
err = c.lbClient.UpdateBackendSet(context.TODO(), lb.Id, etag, bs, nil, sslConfig, healthChecker, &policy)
err = c.client.GetLbClient().UpdateBackendSet(context.TODO(), lb.Id, etag, bs, nil, sslConfig, healthChecker, &policy)
if err != nil {
return err
}
Expand Down Expand Up @@ -641,7 +636,7 @@ func (c *Controller) ensureFinalizer(ingress *networkingv1.Ingress) error {
return err
}

_, err = c.client.NetworkingV1().Ingresses(ingress.Namespace).Patch(context.TODO(), ingress.Name, types.MergePatchType, patch, metav1.PatchOptions{})
_, err = c.client.GetK8Client().NetworkingV1().Ingresses(ingress.Namespace).Patch(context.TODO(), ingress.Name, types.MergePatchType, patch, metav1.PatchOptions{})
return err
})

Expand Down Expand Up @@ -671,7 +666,7 @@ func (c *Controller) deleteFinalizer(ingress *networkingv1.Ingress) error {
return err
}

_, err = c.client.NetworkingV1().Ingresses(ingress.Namespace).Patch(context.TODO(), ingress.Name, types.MergePatchType, patch, metav1.PatchOptions{})
_, err = c.client.GetK8Client().NetworkingV1().Ingresses(ingress.Namespace).Patch(context.TODO(), ingress.Name, types.MergePatchType, patch, metav1.PatchOptions{})
return err
})

Expand Down
Loading

0 comments on commit ad04c22

Please sign in to comment.