add counter prometheus metric from fleet guard rail rejection

pkg/utils/controller/metrics/metrics.go

@@ -49,6 +49,13 @@ var (
 		Name: "fleet_workload_active_workers",
 		Help: "Number of currently used workers per controller",
 	}, []string{"controller"})
+
+	// GuardRailRejectionCount is a prometheus counter metrics which holds the total
+	// number of rejections performed by the fleet guard rail.
+	GuardRailRejectionCount = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "fleet_guard_rail_rejection_count",
+		Help: "Total number of rejection performed by fleet guard rail",
+	}, []string{"controller"})
 )
 
 func init() {
@@ -58,5 +65,6 @@ func init() {
 		FleetReconcileTime,
 		FleetWorkerCount,
 		FleetActiveWorkers,
+		GuardRailRejectionCount,
 	)
 }

pkg/webhook/fleetresourcehandler/fleetresourcehandler_webhook.go

@@ -81,7 +81,7 @@ func (v *fleetResourceValidator) Handle(ctx context.Context, req admission.Reque
 			response = v.handleEvent(ctx, req)
 		case req.Namespace != "":
 			klog.V(2).InfoS("handling namespaced resource in fleet reserved namespaces", "GVK", req.RequestKind, "namespacedName", namespacedName, "operation", req.Operation, "subResource", req.SubResource)
-			response = validation.ValidateUserForResource(req, v.whiteListedUsers)
+			response = validation.ValidateUserForResource(req, v.whiteListedUsers, true)
 		default:
 			klog.V(3).InfoS("resource is not monitored by fleet resource validator webhook", "GVK", req.RequestKind, "namespacedName", namespacedName, "operation", req.Operation, "subResource", req.SubResource)
 			response = admission.Allowed(fmt.Sprintf("user: %s in groups: %v is allowed to modify resource with GVK: %s", req.UserInfo.Username, req.UserInfo.Groups, req.Kind.String()))
@@ -114,7 +114,7 @@ func (v *fleetResourceValidator) handleV1Alpha1MemberCluster(req admission.Reque
 		}
 		return validation.ValidateV1Alpha1MemberClusterUpdate(currentMC, oldMC, req, v.whiteListedUsers)
 	}
-	return validation.ValidateUserForResource(req, v.whiteListedUsers)
+	return validation.ValidateUserForResource(req, v.whiteListedUsers, true)
 }
 
 // handleMemberCluster allows/denies the request to modify member cluster object after validation.
@@ -136,7 +136,7 @@ func (v *fleetResourceValidator) handleMemberCluster(req admission.Request) admi
 	}
 	isFleetMC := utils.IsFleetAnnotationPresent(currentMC.Annotations)
 	if isFleetMC {
-		return validation.ValidateUserForResource(req, v.whiteListedUsers)
+		return validation.ValidateUserForResource(req, v.whiteListedUsers, true)
 	}
 	klog.V(3).InfoS("upstream member cluster resource is allowed to be created/deleted by any user",
 		"user", req.UserInfo.Username, "groups", req.UserInfo.Groups, "operation", req.Operation, "kind", req.RequestKind.Kind, "subResource", req.SubResource, "namespacedName", types.NamespacedName{Name: req.Name, Namespace: req.Namespace})
@@ -148,7 +148,7 @@ func (v *fleetResourceValidator) handleFleetReservedNamespacedResource(ctx conte
 	var response admission.Response
 	if strings.HasPrefix(req.Namespace, fleetMemberNamespacePrefix) {
 		// check to see if valid users other than member agent is making the request.
-		response = validation.ValidateUserForResource(req, v.whiteListedUsers)
+		response = validation.ValidateUserForResource(req, v.whiteListedUsers, false)
 		// check to see if member agent is making the request only on Update.
 		if !response.Allowed {
 			// if namespace name is just "fleet-member", mcName variable becomes empty and the request is allowed since that namespaces is not watched by member agents.
@@ -157,7 +157,7 @@ func (v *fleetResourceValidator) handleFleetReservedNamespacedResource(ctx conte
 		}
 		return response
 	} else if strings.HasPrefix(req.Namespace, fleetNamespacePrefix) || strings.HasPrefix(req.Namespace, kubeNamespacePrefix) {
-		return validation.ValidateUserForResource(req, v.whiteListedUsers)
+		return validation.ValidateUserForResource(req, v.whiteListedUsers, true)
 	}
 	klog.V(3).InfoS("namespace name doesn't begin with fleet/kube prefix so we allow all operations on these namespaces",
 		"user", req.UserInfo.Username, "groups", req.UserInfo.Groups, "operation", req.Operation, "kind", req.RequestKind.Kind, "subResource", req.SubResource, "namespacedName", types.NamespacedName{Name: req.Name, Namespace: req.Namespace})
@@ -175,7 +175,7 @@ func (v *fleetResourceValidator) handleNamespace(req admission.Request) admissio
 	fleetMatchResult := strings.HasPrefix(req.Name, fleetNamespacePrefix)
 	kubeMatchResult := strings.HasPrefix(req.Name, kubeNamespacePrefix)
 	if fleetMatchResult || kubeMatchResult {
-		return validation.ValidateUserForResource(req, v.whiteListedUsers)
+		return validation.ValidateUserForResource(req, v.whiteListedUsers, true)
 	}
 	// only handling reserved namespaces with prefix fleet/kube.
 	return admission.Allowed("namespace name doesn't begin with fleet/kube prefix so we allow all operations on these namespaces")

pkg/webhook/validation/uservalidation.go

@@ -20,6 +20,7 @@ import (
 	clusterv1beta1 "go.goms.io/fleet/apis/cluster/v1beta1"
 	fleetv1alpha1 "go.goms.io/fleet/apis/v1alpha1"
 	"go.goms.io/fleet/pkg/utils"
+	"go.goms.io/fleet/pkg/utils/controller/metrics"
 )
 
 const (
@@ -49,6 +50,7 @@ func ValidateUserForFleetCRD(req admission.Request, whiteListedUsers []string, g
 	namespacedName := types.NamespacedName{Name: req.Name, Namespace: req.Namespace}
 	userInfo := req.UserInfo
 	if checkCRDGroup(group) && !isMasterGroupUserOrWhiteListedUser(whiteListedUsers, userInfo) {
+		metrics.GuardRailRejectionCount.WithLabelValues("fleet-guard-rail").Inc()
 		klog.V(2).InfoS(deniedModifyResource, "user", userInfo.Username, "groups", userInfo.Groups, "operation", req.Operation, "GVK", req.RequestKind, "subResource", req.SubResource, "namespacedName", namespacedName)
 		return admission.Denied(fmt.Sprintf(ResourceDeniedFormat, userInfo.Username, utils.GenerateGroupString(userInfo.Groups), req.Operation, req.RequestKind, req.SubResource, namespacedName))
 	}
@@ -57,13 +59,16 @@ func ValidateUserForFleetCRD(req admission.Request, whiteListedUsers []string, g
 }
 
 // ValidateUserForResource checks to see if user is allowed to modify argued resource modified by request.
-func ValidateUserForResource(req admission.Request, whiteListedUsers []string) admission.Response {
+func ValidateUserForResource(req admission.Request, whiteListedUsers []string, isTerminalCall bool) admission.Response {
 	namespacedName := types.NamespacedName{Name: req.Name, Namespace: req.Namespace}
 	userInfo := req.UserInfo
 	if isMasterGroupUserOrWhiteListedUser(whiteListedUsers, userInfo) || isUserAuthenticatedServiceAccount(userInfo) || isUserKubeScheduler(userInfo) || isUserKubeControllerManager(userInfo) || isNodeGroupUser(userInfo) {
 		klog.V(3).InfoS(allowedModifyResource, "user", userInfo.Username, "groups", userInfo.Groups, "operation", req.Operation, "GVK", req.RequestKind, "subResource", req.SubResource, "namespacedName", namespacedName)
 		return admission.Allowed(fmt.Sprintf(ResourceAllowedFormat, userInfo.Username, utils.GenerateGroupString(userInfo.Groups), req.Operation, req.RequestKind, req.SubResource, namespacedName))
 	}
+	if isTerminalCall {
+		metrics.GuardRailRejectionCount.WithLabelValues("fleet-guard-rail").Inc()
+	}
 	klog.V(2).InfoS(deniedModifyResource, "user", userInfo.Username, "groups", userInfo.Groups, "operation", req.Operation, "GVK", req.RequestKind, "subResource", req.SubResource, "namespacedName", namespacedName)
 	return admission.Denied(fmt.Sprintf(ResourceDeniedFormat, userInfo.Username, utils.GenerateGroupString(userInfo.Groups), req.Operation, req.RequestKind, req.SubResource, namespacedName))
 }
@@ -77,6 +82,7 @@ func ValidateV1Alpha1MemberClusterUpdate(currentMC, oldMC fleetv1alpha1.MemberCl
 	isAnnotationUpdated := isMapFieldUpdated(currentMC.GetAnnotations(), oldMC.GetAnnotations())
 	isObjUpdated, err := isMemberClusterUpdated(&currentMC, &oldMC)
 	if err != nil {
+		metrics.GuardRailRejectionCount.WithLabelValues("fleet-guard-rail").Inc()
 		return admission.Denied(err.Error())
 	}
 	if (isLabelUpdated || isAnnotationUpdated) && !isObjUpdated {
@@ -85,7 +91,7 @@ func ValidateV1Alpha1MemberClusterUpdate(currentMC, oldMC fleetv1alpha1.MemberCl
 		response = admission.Allowed(fmt.Sprintf(ResourceAllowedFormat, userInfo.Username, utils.GenerateGroupString(userInfo.Groups), req.Operation, req.RequestKind, req.SubResource, namespacedName))
 	}
 	if isObjUpdated {
-		response = ValidateUserForResource(req, whiteListedUsers)
+		response = ValidateUserForResource(req, whiteListedUsers, true)
 	}
 	return response
 }
@@ -95,6 +101,7 @@ func ValidateFleetMemberClusterUpdate(currentMC, oldMC clusterv1beta1.MemberClus
 	namespacedName := types.NamespacedName{Name: currentMC.GetName()}
 	userInfo := req.UserInfo
 	if areAllFleetAnnotationsRemoved(currentMC.Annotations, oldMC.Annotations) {
+		metrics.GuardRailRejectionCount.WithLabelValues("fleet-guard-rail").Inc()
 		klog.V(2).InfoS(deniedRemoveFleetAnnotation, "user", userInfo.Username, "groups", userInfo.Groups, "operation", req.Operation, "GVK", req.RequestKind, "subResource", req.SubResource, "namespacedName", namespacedName)
 		return admission.Denied(deniedRemoveFleetAnnotation)
 	}
@@ -103,11 +110,12 @@ func ValidateFleetMemberClusterUpdate(currentMC, oldMC clusterv1beta1.MemberClus
 	oldMC.Spec.Taints = nil
 	isObjUpdated, err := isMemberClusterUpdated(currentMC.DeepCopy(), oldMC.DeepCopy())
 	if err != nil {
+		metrics.GuardRailRejectionCount.WithLabelValues("fleet-guard-rail").Inc()
 		return admission.Denied(err.Error())
 	}
 	isAnnotationUpdated := isFleetAnnotationUpdated(currentMC.Annotations, oldMC.Annotations)
 	if isObjUpdated || isAnnotationUpdated {
-		return ValidateUserForResource(req, whiteListedUsers)
+		return ValidateUserForResource(req, whiteListedUsers, true)
 	}
 	// any user is allowed to modify labels, annotations, taints on fleet MC except fleet pre-fixed annotations.
 	klog.V(3).InfoS(allowedModifyResource, "user", userInfo.Username, "groups", userInfo.Groups, "operation", req.Operation, "GVK", req.RequestKind, "subResource", req.SubResource, "namespacedName", namespacedName)
@@ -119,12 +127,13 @@ func ValidatedUpstreamMemberClusterUpdate(currentMC, oldMC clusterv1beta1.Member
 	namespacedName := types.NamespacedName{Name: currentMC.GetName()}
 	userInfo := req.UserInfo
 	if isFleetAnnotationAdded(currentMC.Annotations, oldMC.Annotations) {
+		metrics.GuardRailRejectionCount.WithLabelValues("fleet-guard-rail").Inc()
 		klog.V(2).InfoS(deniedAddFleetAnnotation, "user", userInfo.Username, "groups", userInfo.Groups, "operation", req.Operation, "GVK", req.RequestKind, "subResource", req.SubResource, "namespacedName", namespacedName)
 		return admission.Denied(deniedAddFleetAnnotation)
 	}
 	// any user is allowed to modify MC spec for upstream MC.
 	if !equality.Semantic.DeepEqual(currentMC.Status, oldMC.Status) {
-		return ValidateUserForResource(req, whiteListedUsers)
+		return ValidateUserForResource(req, whiteListedUsers, true)
 	}
 	klog.V(3).InfoS(allowedModifyResource, "user", userInfo.Username, "groups", userInfo.Groups, "operation", req.Operation, "GVK", req.RequestKind, "subResource", req.SubResource, "namespacedName", namespacedName)
 	return admission.Allowed(fmt.Sprintf(ResourceAllowedFormat, userInfo.Username, utils.GenerateGroupString(userInfo.Groups), req.Operation, req.RequestKind, req.SubResource, namespacedName))
@@ -268,6 +277,7 @@ func ValidateMCIdentity(ctx context.Context, client client.Client, req admission
 		klog.V(3).InfoS(allowedModifyResource, "user", userInfo.Username, "groups", userInfo.Groups, "operation", req.Operation, "GVK", req.RequestKind, "subResource", req.SubResource, "namespacedName", namespacedName)
 		return admission.Allowed(fmt.Sprintf(ResourceAllowedFormat, userInfo.Username, utils.GenerateGroupString(userInfo.Groups), req.Operation, req.RequestKind, req.SubResource, namespacedName))
 	}
+	metrics.GuardRailRejectionCount.WithLabelValues("fleet-guard-rail").Inc()
 	klog.V(2).InfoS(deniedModifyResource, "user", userInfo.Username, "groups", userInfo.Groups, "operation", req.Operation, "GVK", req.RequestKind, "subResource", req.SubResource, "namespacedName", namespacedName)
 	return admission.Denied(fmt.Sprintf(ResourceDeniedFormat, userInfo.Username, utils.GenerateGroupString(userInfo.Groups), req.Operation, req.RequestKind, req.SubResource, namespacedName))
 }

pkg/webhook/validation/uservalidation_test.go

@@ -148,7 +148,7 @@ func TestValidateUserForResource(t *testing.T) {
 
 	for testName, testCase := range testCases {
 		t.Run(testName, func(t *testing.T) {
-			gotResult := ValidateUserForResource(testCase.req, testCase.whiteListedUsers)
+			gotResult := ValidateUserForResource(testCase.req, testCase.whiteListedUsers, true)
 			assert.Equal(t, testCase.wantResponse, gotResult, utils.TestCaseMsg, testName)
 		})
 	}

pkg/webhook/webhook.go

@@ -47,6 +47,7 @@ import (
 	placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1"
 	fleetv1alpha1 "go.goms.io/fleet/apis/v1alpha1"
 	"go.goms.io/fleet/cmd/hubagent/options"
+	"go.goms.io/fleet/pkg/utils/controller/metrics"
 	"go.goms.io/fleet/pkg/webhook/clusterresourceoverride"
 	"go.goms.io/fleet/pkg/webhook/clusterresourceplacement"
 	"go.goms.io/fleet/pkg/webhook/fleetresourcehandler"
@@ -174,9 +175,14 @@ func (w *Config) Start(ctx context.Context) error {
 		klog.ErrorS(err, "unable to setup webhook configurations in apiserver")
 		return err
 	}
+	w.initMetrics()
 	return nil
 }
 
+func (w *Config) initMetrics() {
+	metrics.GuardRailRejectionCount.WithLabelValues("fleet-guard-rail").Add(0)
+}
+
 // createFleetWebhookConfiguration creates the ValidatingWebhookConfiguration object for the webhook.
 func (w *Config) createFleetWebhookConfiguration(ctx context.Context) error {
 	if err := w.createValidatingWebhookConfiguration(ctx, w.buildFleetValidatingWebhooks(), fleetValidatingWebhookCfgName); err != nil {