feat: add priority to resource customizations

Add a priority field to resource customizations, so that we can
untangle precedence of wildcard health checks. Update docs as well,
since they were previously incorrect (wildcards can only be used with
"old-style" health checks due to K8s). Fixes argoproj#16905.

Signed-off-by: Blake Pettersson <blake.pettersson@gmail.com>

docs/operator-manual/health.md

@@ -98,20 +98,42 @@ data:
     return hs
 ```
 
-In order to prevent duplication of the custom health check for potentially multiple resources, it is also possible to specify a wildcard in the resource kind, and anywhere in the resource group, like this:
+In order to prevent duplication of custom health checks for potentially multiple resources, it is also possible to 
+specify a wildcard in the resource kind, and anywhere in the resource group, like this:
 
 ```yaml
-  resource.customizations.health.ec2.aws.crossplane.io_*: |
-    ...
+  resource.customizations: |
+    ec2.aws.crossplane.io/*:
+      health.lua: |
+        ...
 ```
 
 ```yaml
-  resource.customizations.health.*.aws.crossplane.io_*: |
-    ...
+  # If a key _begins_ with a wildcard, please ensure that the GVK key is quoted.
+  resource.customizations: |
+    "*.aws.crossplane.io/*":
+      health.lua: |
+        ...
 ```
 
 !!!important
-    Please, note that there can be ambiguous resolution of wildcards, see [#16905](https://github.com/argoproj/argo-cd/issues/16905)
+    Please, note that wildcards are only supported when using the `resource.customizations` key, the `resource.customizations.health.<group>_<kind>`
+    style keys do not work since wildcards (`*`) are not supported in Kubernetes configmap keys.
+
+Note that there can be an ambiguous resolution of wildcard health checks if two or more of them potentially matches with 
+a given resource. To amend that, a priority can be specified on a resource customization - the higher the priority, the
+higher the precedence of the health check.
+
+```yaml
+  resource.customizations: |
+    ec2.aws.crossplane.io/*:
+      priority: 5
+      health.lua: |
+        ...
+```
+
+The priority is only relevant for health checks, and can only be defined when using the `resource.customizations` format
+since it only makes sense when used with wildcards. 
 
 The `obj` is a global variable which contains the resource. The script must return an object with status and optional message field.
 The custom health check might return one of the following health statuses:
@@ -121,7 +143,7 @@ The custom health check might return one of the following health statuses:
   * `Degraded` - the resource is degraded
   * `Suspended` - the resource is suspended and waiting for some external event to resume (e.g. suspended CronJob or paused Deployment)
 
-By default health typically returns `Progressing` status.
+By default, health typically returns a `Progressing` status.
 
 NOTE: As a security measure, access to the standard Lua libraries will be disabled by default. Admins can control access by
 setting `resource.customizations.useOpenLibs.<group>_<kind>`. In the following example, standard libraries are enabled for health check of `cert-manager.io/Certificate`.

pkg/apis/application/v1alpha1/types.go

@@ -2083,6 +2083,7 @@ type rawResourceOverride struct {
 	IgnoreDifferences     string           `json:"ignoreDifferences,omitempty"`
 	IgnoreResourceUpdates string           `json:"ignoreResourceUpdates,omitempty"`
 	KnownTypeFields       []KnownTypeField `json:"knownTypeFields,omitempty"`
+	Priority              int32            `json:"priority,omitempty"`
 }
 
 // ResourceOverride holds configuration to customize resource diffing and health assessment
@@ -2094,6 +2095,7 @@ type ResourceOverride struct {
 	IgnoreDifferences     OverrideIgnoreDiff `protobuf:"bytes,2,opt,name=ignoreDifferences"`
 	IgnoreResourceUpdates OverrideIgnoreDiff `protobuf:"bytes,6,opt,name=ignoreResourceUpdates"`
 	KnownTypeFields       []KnownTypeField   `protobuf:"bytes,4,opt,name=knownTypeFields"`
+	Priority              int32              `protobuf:"bytes,7,opt,name=priority"`
 }
 
 // TODO: describe this method
@@ -2106,6 +2108,7 @@ func (s *ResourceOverride) UnmarshalJSON(data []byte) error {
 	s.HealthLua = raw.HealthLua
 	s.UseOpenLibs = raw.UseOpenLibs
 	s.Actions = raw.Actions
+	s.Priority = raw.Priority
 	err := yaml.Unmarshal([]byte(raw.IgnoreDifferences), &s.IgnoreDifferences)
 	if err != nil {
 		return err
@@ -2127,7 +2130,7 @@ func (s ResourceOverride) MarshalJSON() ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	raw := &rawResourceOverride{s.HealthLua, s.UseOpenLibs, s.Actions, string(ignoreDifferencesData), string(ignoreResourceUpdatesData), s.KnownTypeFields}
+	raw := &rawResourceOverride{s.HealthLua, s.UseOpenLibs, s.Actions, string(ignoreDifferencesData), string(ignoreResourceUpdatesData), s.KnownTypeFields, s.Priority}
 	return json.Marshal(raw)
 }
 

util/lua/lua.go

@@ -429,12 +429,18 @@ func GetConfigMapKey(gvk schema.GroupVersionKind) string {
 func GetWildcardConfigMapKey(vm VM, gvk schema.GroupVersionKind) string {
 	gvkKeyToMatch := GetConfigMapKey(gvk)
 
-	for key := range vm.ResourceOverrides {
+	matchedKey := ""
+	var resourceOverride *appv1.ResourceOverride
+
+	for key, value := range vm.ResourceOverrides {
 		if glob.Match(key, gvkKeyToMatch) {
-			return key
+			if resourceOverride == nil || (value.Priority > resourceOverride.Priority) {
+				resourceOverride = &value
+				matchedKey = key
+			}
 		}
 	}
-	return ""
+	return matchedKey
 }
 
 func (vm VM) getPredefinedLuaScripts(objKey string, scriptFile string) (string, error) {

util/lua/lua_test.go

@@ -787,6 +787,11 @@ return hs`
 	const healthWildcardOverrideScript = `
  hs = {}
  hs.status = "Healthy"
+ return hs`
+
+	const healthWildcardOverrideScriptWithPriority = `
+ hs = {}
+ hs.status = "UnHealthy"
  return hs`
 
 	getHealthOverride := func(openLibs bool) ResourceHealthOverrides {
@@ -804,6 +809,16 @@ return hs`
 		},
 	}
 
+	getWildcardHealthOverrideWithPriority := ResourceHealthOverrides{
+		"*.aws.crossplane.io/*": appv1.ResourceOverride{
+			HealthLua: healthWildcardOverrideScript,
+		},
+		"*.aws*": appv1.ResourceOverride{
+			HealthLua: healthWildcardOverrideScriptWithPriority,
+			Priority:  10,
+		},
+	}
+
 	t.Run("Enable Lua standard lib", func(t *testing.T) {
 		testObj := StrToUnstructured(testSA)
 		overrides := getHealthOverride(true)
@@ -837,6 +852,15 @@ return hs`
 		assert.Equal(t, expectedStatus, status)
 	})
 
+	t.Run("Get resource health for wildcard override with priority", func(t *testing.T) {
+		testObj := StrToUnstructured(ec2AWSCrossplaneObjJson)
+		overrides := getWildcardHealthOverrideWithPriority
+		status, err := overrides.GetResourceHealth(testObj)
+		require.NoError(t, err)
+		expectedStatus := &health.HealthStatus{Status: "Unknown", Message: "Lua returned an invalid health status"}
+		assert.Equal(t, expectedStatus, status)
+	})
+
 	t.Run("Resource health for wildcard override not found", func(t *testing.T) {
 		testObj := StrToUnstructured(testSA)
 		overrides := getWildcardHealthOverride