Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add L4 NetLB Dual-Stack Metrics #1937

Merged
merged 2 commits into from
Feb 16, 2023
Merged

Add L4 NetLB Dual-Stack Metrics #1937

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c02100a
Add L4 NetLB Dual-Stack Metrics
panslava Feb 7, 2023
d6dc142
Log metrics updates with V(2) for L4 Controllers
panslava Feb 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions pkg/l4lb/l4controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,21 +497,21 @@ func (l4c *L4Controller) publishMetrics(result *loadbalancers.L4ILBSyncResult, n
}
switch result.SyncType {
case loadbalancers.SyncTypeCreate, loadbalancers.SyncTypeUpdate:
klog.V(6).Infof("Internal L4 Loadbalancer for Service %s ensured, updating its state %v in metrics cache", namespacedName, result.MetricsState)
klog.V(2).Infof("Internal L4 Loadbalancer for Service %s ensured, updating its state %v in metrics cache", namespacedName, result.MetricsState)
l4c.ctx.ControllerMetrics.SetL4ILBService(namespacedName, result.MetricsState)
if l4c.enableDualStack {
klog.V(6).Infof("Internal L4 DualStack Loadbalancer for Service %s ensured, updating its state %v in metrics cache", namespacedName, result.MetricsState)
klog.V(2).Infof("Internal L4 DualStack Loadbalancer for Service %s ensured, updating its state %v in metrics cache", namespacedName, result.DualStackMetricsState)
l4c.ctx.ControllerMetrics.SetL4ILBDualStackService(namespacedName, result.DualStackMetricsState)
}
l4metrics.PublishILBSyncMetrics(result.Error == nil, result.SyncType, result.GCEResourceInError, utils.GetErrorType(result.Error), result.StartTime)

case loadbalancers.SyncTypeDelete:
// if service is successfully deleted, remove it from cache
if result.Error == nil {
klog.V(6).Infof("Internal L4 Loadbalancer for Service %s deleted, removing its state from metrics cache", namespacedName)
klog.V(2).Infof("Internal L4 Loadbalancer for Service %s deleted, removing its state from metrics cache", namespacedName)
l4c.ctx.ControllerMetrics.DeleteL4ILBService(namespacedName)
if l4c.enableDualStack {
klog.V(6).Infof("Internal L4 DualStack LoadBalancer for Service %s deleted, removing its state from metrics cache", namespacedName)
klog.V(2).Infof("Internal L4 DualStack LoadBalancer for Service %s deleted, removing its state from metrics cache", namespacedName)
l4c.ctx.ControllerMetrics.DeleteL4ILBDualStackService(namespacedName)
}
}
Expand Down
12 changes: 10 additions & 2 deletions pkg/l4lb/l4netlbcontroller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -633,14 +633,22 @@ func (lc *L4NetLBController) publishMetrics(result *loadbalancers.L4NetLBSyncRes
namespacedName := types.NamespacedName{Name: name, Namespace: namespace}.String()
switch result.SyncType {
case loadbalancers.SyncTypeCreate, loadbalancers.SyncTypeUpdate:
klog.V(4).Infof("External L4 Loadbalancer for Service %s ensured, updating its state %v in metrics cache", namespacedName, result.MetricsState)
klog.V(2).Infof("External L4 Loadbalancer for Service %s ensured, updating its state %v in metrics cache", namespacedName, result.MetricsState)
lc.ctx.ControllerMetrics.SetL4NetLBService(namespacedName, result.MetricsState)
if lc.enableDualStack {
klog.V(2).Infof("External L4 DualStack Loadbalancer for Service %s ensured, updating its state %v in metrics cache", namespacedName, result.DualStackMetricsState)
lc.ctx.ControllerMetrics.SetL4NetLBDualStackService(namespacedName, result.DualStackMetricsState)
}
lc.publishSyncMetrics(result)
case loadbalancers.SyncTypeDelete:
// if service is successfully deleted, remove it from cache
if result.Error == nil {
klog.V(4).Infof("External L4 Loadbalancer for Service %s deleted, removing its state from metrics cache", namespacedName)
klog.V(2).Infof("External L4 Loadbalancer for Service %s deleted, removing its state from metrics cache", namespacedName)
lc.ctx.ControllerMetrics.DeleteL4NetLBService(namespacedName)
if lc.enableDualStack {
klog.V(2).Infof("External L4 Loadbalancer for Service %s deleted, removing its state from metrics cache", namespacedName)
lc.ctx.ControllerMetrics.DeleteL4NetLBDualStackService(namespacedName)
}
}
lc.publishSyncMetrics(result)
default:
Expand Down
24 changes: 2 additions & 22 deletions pkg/loadbalancers/l4.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ type L4ILBSyncResult struct {
GCEResourceInError string
Status *corev1.LoadBalancerStatus
MetricsState metrics.L4ILBServiceState
DualStackMetricsState metrics.L4ILBDualStackServiceState
DualStackMetricsState metrics.L4DualStackServiceState
SyncType string
StartTime time.Time
}
Expand Down Expand Up @@ -287,7 +287,7 @@ func (l4 *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service
Annotations: make(map[string]string),
StartTime: time.Now(),
SyncType: SyncTypeCreate,
DualStackMetricsState: l4.getInitialDualStackMetricsState(),
DualStackMetricsState: metrics.InitServiceDualStackMetricsState(svc),
}

// If service already has an IP assigned, treat it as an update instead of a new Loadbalancer.
Expand Down Expand Up @@ -534,26 +534,6 @@ func (l4 *L4) hasAnnotation(annotationKey string) bool {
return false
}

func (l4 *L4) getInitialDualStackMetricsState() metrics.L4ILBDualStackServiceState {
// Always init stats with error, and update with Success when service was provisioned
state := metrics.L4ILBDualStackServiceState{
Status: metrics.StatusError,
}

var ipFamiliesStrings []string
for _, ipFamily := range l4.Service.Spec.IPFamilies {
ipFamiliesStrings = append(ipFamiliesStrings, string(ipFamily))
}
state.IPFamilies = strings.Join(ipFamiliesStrings, ",")

state.IPFamilyPolicy = ""
if l4.Service.Spec.IPFamilyPolicy != nil {
state.IPFamilyPolicy = string(*l4.Service.Spec.IPFamilyPolicy)
}

return state
}

func (l4 *L4) getOldForwardingRule() (*composite.ForwardingRule, error) {
bsName := l4.namer.L4Backend(l4.Service.Namespace, l4.Service.Name)
// Check if protocol has changed for this service. In this case, forwarding rule has different protocol and name
Expand Down
32 changes: 18 additions & 14 deletions pkg/loadbalancers/l4netlb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,29 +57,33 @@ type L4NetLB struct {
// L4NetLBSyncResult contains information about the outcome of an L4 NetLB sync. It stores the list of resource name annotations,
// sync error, the GCE resource that hit the error along with the error type, metrics and more fields.
type L4NetLBSyncResult struct {
Annotations map[string]string
Error error
GCEResourceInError string
Status *corev1.LoadBalancerStatus
MetricsState metrics.L4NetLBServiceState
SyncType string
StartTime time.Time
Annotations map[string]string
Error error
GCEResourceInError string
Status *corev1.LoadBalancerStatus
MetricsState metrics.L4NetLBServiceState
DualStackMetricsState metrics.L4DualStackServiceState
SyncType string
StartTime time.Time
}

func NewL4SyncResult(syncType string) *L4NetLBSyncResult {
func NewL4SyncResult(syncType string, svc *corev1.Service) *L4NetLBSyncResult {
startTime := time.Now()
result := &L4NetLBSyncResult{
Annotations: make(map[string]string),
StartTime: time.Now(),
SyncType: syncType,
Annotations: make(map[string]string),
StartTime: startTime,
SyncType: syncType,
MetricsState: metrics.InitL4NetLBServiceState(&startTime),
DualStackMetricsState: metrics.InitServiceDualStackMetricsState(svc),
}
result.MetricsState = metrics.InitL4NetLBServiceState(&result.StartTime)
return result
}

// SetMetricsForSuccessfulServiceSync should be call after successful sync.
func (r *L4NetLBSyncResult) SetMetricsForSuccessfulServiceSync() {
r.MetricsState.FirstSyncErrorTime = nil
r.MetricsState.InSuccess = true
r.DualStackMetricsState.Status = metrics.StatusSuccess
}

type L4NetLBParams struct {
Expand Down Expand Up @@ -117,7 +121,7 @@ func (l4netlb *L4NetLB) createKey(name string) (*meta.Key, error) {
// It returns a LoadBalancerStatus with the updated ForwardingRule IP address.
// This function does not link instances to Backend Service.
func (l4netlb *L4NetLB) EnsureFrontend(nodeNames []string, svc *corev1.Service) *L4NetLBSyncResult {
result := NewL4SyncResult(SyncTypeCreate)
result := NewL4SyncResult(SyncTypeCreate, svc)
// If service already has an IP assigned, treat it as an update instead of a new Loadbalancer.
if len(svc.Status.LoadBalancer.Ingress) > 0 {
result.SyncType = SyncTypeUpdate
Expand Down Expand Up @@ -281,7 +285,7 @@ func (l4netlb *L4NetLB) ensureIPv4NodesFirewall(nodeNames []string, ipAddress st
// EnsureLoadBalancerDeleted performs a cleanup of all GCE resources for the given loadbalancer service.
// It is health check, firewall rules and backend service
func (l4netlb *L4NetLB) EnsureLoadBalancerDeleted(svc *corev1.Service) *L4NetLBSyncResult {
result := NewL4SyncResult(SyncTypeDelete)
result := NewL4SyncResult(SyncTypeDelete, svc)
l4netlb.Service = svc

l4netlb.deleteIPv4ResourcesOnDelete(result)
Expand Down
152 changes: 99 additions & 53 deletions pkg/metrics/l4metrics_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,76 @@ func TestComputeL4NetLBMetrics(t *testing.T) {
}
}

func TestComputeL4NetLBDualStackMetrics(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
desc string
serviceStates []L4DualStackServiceState
expectL4NetLBDualStackCount map[L4DualStackServiceState]int
}{
{
desc: "empty input",
serviceStates: []L4DualStackServiceState{},
expectL4NetLBDualStackCount: map[L4DualStackServiceState]int{},
},
{
desc: "one l4 NetLB dual-stack service",
serviceStates: []L4DualStackServiceState{
newL4DualStackServiceState("IPv4", "SingleStack", StatusSuccess),
},
expectL4NetLBDualStackCount: map[L4DualStackServiceState]int{
L4DualStackServiceState{"IPv4", "SingleStack", StatusSuccess}: 1,
},
},
{
desc: "l4 NetLB dual-stack service in error state",
serviceStates: []L4DualStackServiceState{
newL4DualStackServiceState("IPv4", "SingleStack", StatusError),
},
expectL4NetLBDualStackCount: map[L4DualStackServiceState]int{
L4DualStackServiceState{"IPv4", "SingleStack", StatusError}: 1,
},
},
{
desc: "L4 NetLB dual-stack service with IPv4,IPv6 Families",
serviceStates: []L4DualStackServiceState{
newL4DualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess),
},
expectL4NetLBDualStackCount: map[L4DualStackServiceState]int{
L4DualStackServiceState{"IPv4,IPv6", "RequireDualStack", StatusSuccess}: 1,
},
},
{
desc: "many l4 NetLB dual-stack services",
serviceStates: []L4DualStackServiceState{
newL4DualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess),
newL4DualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess),
newL4DualStackServiceState("IPv4", "SingleStack", StatusError),
newL4DualStackServiceState("IPv6", "SingleStack", StatusSuccess),
newL4DualStackServiceState("IPv6", "SingleStack", StatusSuccess),
},
expectL4NetLBDualStackCount: map[L4DualStackServiceState]int{
L4DualStackServiceState{"IPv4,IPv6", "RequireDualStack", StatusSuccess}: 2,
L4DualStackServiceState{"IPv4", "SingleStack", StatusError}: 1,
L4DualStackServiceState{"IPv6", "SingleStack", StatusSuccess}: 2,
},
},
} {
tc := tc
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
newMetrics := FakeControllerMetrics()
for i, serviceState := range tc.serviceStates {
newMetrics.SetL4NetLBDualStackService(fmt.Sprint(i), serviceState)
}
got := newMetrics.computeL4NetLBDualStackMetrics()
if diff := cmp.Diff(tc.expectL4NetLBDualStackCount, got); diff != "" {
t.Fatalf("Got diff for L4 NetLB Dual-Stack service counts (-want +got):\n%s", diff)
}
})
}
}

func newL4NetLBServiceState(inSuccess, managed, premium, userError bool, errorTimestamp *time.Time) L4NetLBServiceState {
return L4NetLBServiceState{
IsPremiumTier: premium,
Expand Down Expand Up @@ -435,78 +505,54 @@ func TestComputeL4ILBDualStackMetrics(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
desc string
serviceStates []L4ILBDualStackServiceState
expectL4ILBDualStackCount map[L4ILBDualStackServiceState]int
serviceStates []L4DualStackServiceState
expectL4ILBDualStackCount map[L4DualStackServiceState]int
}{
{
desc: "empty input",
serviceStates: []L4ILBDualStackServiceState{},
expectL4ILBDualStackCount: map[L4ILBDualStackServiceState]int{},
serviceStates: []L4DualStackServiceState{},
expectL4ILBDualStackCount: map[L4DualStackServiceState]int{},
},
{
desc: "one l4 ilb dual-stack service",
serviceStates: []L4ILBDualStackServiceState{
newL4ILBDualStackServiceState("IPv4", "SingleStack", StatusSuccess),
serviceStates: []L4DualStackServiceState{
newL4DualStackServiceState("IPv4", "SingleStack", StatusSuccess),
},
expectL4ILBDualStackCount: map[L4ILBDualStackServiceState]int{
L4ILBDualStackServiceState{
"IPv4",
"SingleStack",
StatusSuccess,
}: 1,
expectL4ILBDualStackCount: map[L4DualStackServiceState]int{
L4DualStackServiceState{"IPv4", "SingleStack", StatusSuccess}: 1,
},
},
{
desc: "l4 ilb dual-stack service in error state",
serviceStates: []L4ILBDualStackServiceState{
newL4ILBDualStackServiceState("IPv4", "SingleStack", StatusError),
serviceStates: []L4DualStackServiceState{
newL4DualStackServiceState("IPv4", "SingleStack", StatusError),
},
expectL4ILBDualStackCount: map[L4ILBDualStackServiceState]int{
L4ILBDualStackServiceState{
"IPv4",
"SingleStack",
StatusError,
}: 1,
expectL4ILBDualStackCount: map[L4DualStackServiceState]int{
L4DualStackServiceState{"IPv4", "SingleStack", StatusError}: 1,
},
},
{
desc: "L4 ILB dual-stack service with IPv4,IPv6 Families",
serviceStates: []L4ILBDualStackServiceState{
newL4ILBDualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess),
serviceStates: []L4DualStackServiceState{
newL4DualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess),
},
expectL4ILBDualStackCount: map[L4ILBDualStackServiceState]int{
L4ILBDualStackServiceState{
"IPv4,IPv6",
"RequireDualStack",
StatusSuccess,
}: 1,
expectL4ILBDualStackCount: map[L4DualStackServiceState]int{
L4DualStackServiceState{"IPv4,IPv6", "RequireDualStack", StatusSuccess}: 1,
},
},
{
desc: "many l4 ilb dual-stack services",
serviceStates: []L4ILBDualStackServiceState{
newL4ILBDualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess),
newL4ILBDualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess),
newL4ILBDualStackServiceState("IPv4", "SingleStack", StatusError),
newL4ILBDualStackServiceState("IPv6", "SingleStack", StatusSuccess),
newL4ILBDualStackServiceState("IPv6", "SingleStack", StatusSuccess),
},
expectL4ILBDualStackCount: map[L4ILBDualStackServiceState]int{
L4ILBDualStackServiceState{
"IPv4,IPv6",
"RequireDualStack",
StatusSuccess,
}: 2,
L4ILBDualStackServiceState{
"IPv4",
"SingleStack",
StatusError,
}: 1,
L4ILBDualStackServiceState{
"IPv6",
"SingleStack",
StatusSuccess,
}: 2,
serviceStates: []L4DualStackServiceState{
newL4DualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess),
newL4DualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess),
newL4DualStackServiceState("IPv4", "SingleStack", StatusError),
newL4DualStackServiceState("IPv6", "SingleStack", StatusSuccess),
newL4DualStackServiceState("IPv6", "SingleStack", StatusSuccess),
},
expectL4ILBDualStackCount: map[L4DualStackServiceState]int{
L4DualStackServiceState{"IPv4,IPv6", "RequireDualStack", StatusSuccess}: 2,
L4DualStackServiceState{"IPv4", "SingleStack", StatusError}: 1,
L4DualStackServiceState{"IPv6", "SingleStack", StatusSuccess}: 2,
},
},
} {
Expand All @@ -525,8 +571,8 @@ func TestComputeL4ILBDualStackMetrics(t *testing.T) {
}
}

func newL4ILBDualStackServiceState(ipFamilies string, ipFamilyPolicy string, status L4ILBDualStackServiceStateStatus) L4ILBDualStackServiceState {
return L4ILBDualStackServiceState{
func newL4DualStackServiceState(ipFamilies string, ipFamilyPolicy string, status L4DualStackServiceStatus) L4DualStackServiceState {
return L4DualStackServiceState{
IPFamilies: ipFamilies,
IPFamilyPolicy: ipFamilyPolicy,
Status: status,
Expand Down
Loading