Add ML detector custom rules

This commit adds custom rules to machine learning
detectors. The name of types used closely follows those
used in Elasticsearch source.

Relates: #3615

src/Nest/XPack/MachineLearning/Job/Detectors/DetectionRule.cs

@@ -0,0 +1,205 @@
+using System;
+using System.Collections.Generic;
+using System.Linq.Expressions;
+using System.Runtime.Serialization;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
+
+namespace Nest
+{
+	/// <summary>
+	/// A machine learning detection rule
+	/// </summary>
+	[JsonConverter(typeof(ReadAsTypeJsonConverter<DetectionRule>))]
+	public interface IDetectionRule
+	{
+		/// <summary>
+		/// The actions to take when rule is satisfied
+		/// </summary>
+		[JsonProperty("actions")]
+		IEnumerable<RuleAction> Actions { get; set; }
+
+		/// <summary>
+		/// The conditions of the rule
+		/// </summary>
+		[JsonProperty("conditions")]
+		IEnumerable<IRuleCondition> Conditions { get; set; }
+
+		/// <summary>
+		/// The scopes of the rule
+		/// </summary>
+		[JsonProperty("scope")]
+		IReadOnlyDictionary<Field, FilterRef> Scope { get; set; }
+	}
+
+	/// <inheritdoc />
+	public class DetectionRule : IDetectionRule
+	{
+		/// <inheritdoc />
+		public IEnumerable<RuleAction> Actions { get; set; }
+
+		/// <inheritdoc />
+		public IEnumerable<IRuleCondition> Conditions { get; set; }
+
+		/// <inheritdoc />
+		public IReadOnlyDictionary<Field, FilterRef> Scope { get; set; }
+	}
+
+	public class DetectionRulesDescriptor
+		: DescriptorPromiseBase<DetectionRulesDescriptor, List<IDetectionRule>>
+	{
+		public DetectionRulesDescriptor() : base(new List<IDetectionRule>()) { }
+
+		private DetectionRulesDescriptor Add(IDetectionRule m)
+		{
+			PromisedValue.Add(m);
+			return this;
+		}
+
+		public DetectionRulesDescriptor Rule(Func<DetectionRuleDescriptor, IDetectionRule> selector) =>
+			Add(selector.Invoke(new DetectionRuleDescriptor()));
+	}
+
+	public class DetectionRuleDescriptor : DescriptorBase<DetectionRuleDescriptor, IDetectionRule>, IDetectionRule
+	{
+		IEnumerable<RuleAction> IDetectionRule.Actions { get; set; }
+		IEnumerable<IRuleCondition> IDetectionRule.Conditions { get; set; }
+		IReadOnlyDictionary<Field, FilterRef> IDetectionRule.Scope { get; set; }
+
+		public DetectionRuleDescriptor Actions(IEnumerable<RuleAction> actions) => Assign(a => a.Actions = actions);
+
+		public DetectionRuleDescriptor Actions(params RuleAction[] actions) => Assign(a => a.Actions = actions);
+
+		public DetectionRuleDescriptor Scope<T>(Func<ScopeDescriptor<T>, IPromise<IReadOnlyDictionary<Field, FilterRef>>> selector) where T : class =>
+			Assign(a => a.Scope = selector.Invoke(new ScopeDescriptor<T>()).Value);
+
+		public DetectionRuleDescriptor Conditions(Func<RuleConditionsDescriptor, IPromise<List<IRuleCondition>>> selector) =>
+			Assign(a => a.Conditions = selector?.Invoke(new RuleConditionsDescriptor())?.Value);
+	}
+
+	public class RuleConditionsDescriptor : DescriptorPromiseBase<RuleConditionsDescriptor, List<IRuleCondition>>
+	{
+		public RuleConditionsDescriptor() : base(new List<IRuleCondition>()) { }
+
+		public RuleConditionsDescriptor Condition(Func<RuleConditionDescriptor, IRuleCondition> selector)
+		{
+			PromisedValue.AddIfNotNull(selector?.Invoke(new RuleConditionDescriptor()));
+			return this;
+		}
+	}
+
+	public class RuleConditionDescriptor : DescriptorBase<RuleConditionDescriptor, IRuleCondition>, IRuleCondition
+	{
+		AppliesTo IRuleCondition.AppliesTo { get; set; }
+		ConditionOperator IRuleCondition.Operator { get; set; }
+		double IRuleCondition.Value { get; set; }
+
+		public RuleConditionDescriptor AppliesTo(AppliesTo appliesTo) => Assign(a => a.AppliesTo = appliesTo);
+
+		public RuleConditionDescriptor Operator(ConditionOperator @operator) => Assign(a => a.Operator = @operator);
+
+		public RuleConditionDescriptor Value(double value) => Assign(a => a.Value = value);
+	}
+
+	public class ScopeDescriptor<T> : DescriptorPromiseBase<ScopeDescriptor<T>, IReadOnlyDictionary<Field, FilterRef>> where T : class
+	{
+		public ScopeDescriptor() : base(new Dictionary<Field, FilterRef>()) { }
+
+		private Dictionary<Field, FilterRef> Dictionary => (Dictionary<Field, FilterRef>)PromisedValue;
+
+		public ScopeDescriptor<T> Scope(Field field, FilterRef filterRef)
+		{
+			Dictionary[field] = filterRef;
+			return this;
+		}
+
+		public ScopeDescriptor<T> Scope(Expression<Func<T, object>> field, FilterRef filterRef)
+		{
+			Dictionary[field] = filterRef;
+			return this;
+		}
+	}
+
+	public class FilterRef
+	{
+		[JsonProperty("filter_id")]
+		public Id FilterId { get; set; }
+
+		[JsonProperty("filter_type")]
+		public RuleFilterType? FilterType { get; set; }
+	}
+
+	[JsonConverter(typeof(StringEnumConverter))]
+	public enum RuleFilterType
+	{
+		[EnumMember(Value = "include")]
+		Include,
+
+		[EnumMember(Value = "exclude")]
+		Exclude
+	}
+
+	[JsonConverter(typeof(ReadAsTypeJsonConverter<RuleCondition>))]
+	public interface IRuleCondition
+	{
+		[JsonProperty("applies_to")]
+		AppliesTo AppliesTo { get; set; }
+
+		[JsonProperty("operator")]
+		ConditionOperator Operator { get; set; }
+
+		[JsonProperty("value")]
+		double Value { get; set; }
+	}
+
+	public class RuleCondition : IRuleCondition
+	{
+		public AppliesTo AppliesTo { get; set; }
+
+		public ConditionOperator Operator { get; set; }
+
+		public double Value { get; set; }
+	}
+
+	[JsonConverter(typeof(StringEnumConverter))]
+	public enum ConditionOperator
+	{
+		[EnumMember(Value = "gt")]
+		GreaterThan,
+
+		[EnumMember(Value = "gte")]
+		GreaterThanOrEqual,
+
+		[EnumMember(Value = "lt")]
+		LessThan,
+
+		[EnumMember(Value = "lte")]
+		LessThanOrEqual,
+	}
+
+	[JsonConverter(typeof(StringEnumConverter))]
+	public enum RuleAction
+	{
+		[EnumMember(Value = "skip_result")]
+		SkipResult,
+
+		[EnumMember(Value = "skip_model_update")]
+		SkipModelUpdate
+	}
+
+	[JsonConverter(typeof(StringEnumConverter))]
+	public enum AppliesTo
+	{
+		[EnumMember(Value = "actual")]
+		Actual,
+
+		[EnumMember(Value = "typical")]
+		Typical,
+
+		[EnumMember(Value = "diff_from_typical")]
+		DiffFromTypical,
+
+		[EnumMember(Value = "time")]
+		Time
+	}
+}

src/Nest/XPack/MachineLearning/Job/Detectors/Detector.cs

@@ -1,27 +1,64 @@
 using System;
 using System.Collections.Generic;
+using System.Linq.Expressions;
+using System.Runtime.Serialization;
 using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
 using Newtonsoft.Json.Linq;
 
 namespace Nest
 {
+	/// <summary>
+	/// A machine learning detector
+	/// </summary>
 	[ContractJsonConverter(typeof(DetectorConverter))]
 	public interface IDetector
 	{
+		/// <summary>
+		/// A description of the detector. For example, "Low event rate".
+		/// </summary>
 		[JsonProperty("detector_description")]
 		string DetectorDescription { get; set; }
 
+		/// <summary>
+		/// A unique identifier for the detector. This identifier is based on the order of the
+		/// detectors in the analysis config, starting at zero. You can use this
+		/// identifier when you want to update a specific detector.
+		/// </summary>
 		[JsonProperty("detector_index")]
 		int? DetectorIndex { get; set; }
 
+		/// <summary>
+		/// If set, frequent entities are excluded from influencing the anomaly results.
+		/// Entities can be considered frequent over time or frequent in a population.
+		/// If you are working with both over and by fields, then you can set exclude_frequent
+		/// to all for both fields, or to by or over for those specific fields.
+		/// </summary>
 		[JsonProperty("exclude_frequent")]
 		ExcludeFrequent? ExcludeFrequent { get; set; }
 
+		/// <summary>
+		/// The analysis function used
+		/// </summary>
 		[JsonProperty("function")]
 		string Function { get; }
 
+		/// <summary>
+		/// Defines whether a new series is used as the null series when there
+		/// is no value for the by or partition fields. The default value is <c>false</c>.
+		/// </summary>
 		[JsonProperty("use_null")]
 		bool? UseNull { get; set; }
+
+		/// <summary>
+		/// Custom rules enable you to change the behaviour of anomaly detectors based on domain-specific knowledge.
+		/// Custom rules describe when a detector should take a certain action instead of following its default behaviour.
+		/// To specify the "when", a rule uses a scope and conditions. You can think of scope as the categorical
+		/// specification of a rule, while conditions are the numerical part. A rule can have a scope,
+		/// one or more conditions, or a combination of scope and conditions.
+		/// </summary>
+		[JsonProperty("custom_rules")]
+		IEnumerable<IDetectionRule> CustomRules { get; set; }
 	}
 
 	internal class DetectorConverter : JsonConverter
@@ -119,41 +156,78 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
 		public override bool CanConvert(Type objectType) => true;
 	}
 
+	/// <summary>
+	/// A machine learning detector with a field name
+	/// </summary>
 	public interface IFieldNameDetector : IDetector
 	{
+		/// <summary>
+		/// The field that the detector uses in the function. If you use an event
+		/// rate function such as count or rare, do not specify this field.
+		/// </summary>
 		[JsonProperty("field_name")]
 		Field FieldName { get; set; }
 	}
 
+	/// <summary>
+	/// A machine learning detector with a by field
+	/// </summary>
 	public interface IByFieldNameDetector : IDetector
 	{
+		/// <summary>
+		/// The field used to split the data. In particular, this property is used for analyzing the splits with
+		/// respect to their own history. It is used for finding unusual values in the context of the split.
+		/// </summary>
 		[JsonProperty("by_field_name")]
 		Field ByFieldName { get; set; }
 	}
 
+	/// <summary>
+	/// A machine learning detector with an over field
+	/// </summary>
 	public interface IOverFieldNameDetector : IDetector
 	{
+		/// <summary>
+		/// The field used to split the data. In particular, this property is used for analyzing the splits
+		/// with respect to the history of all splits. It is used for finding unusual values in the population of all splits.
+		/// </summary>
 		[JsonProperty("over_field_name")]
 		Field OverFieldName { get; set; }
 	}
 
+	/// <summary>
+	/// A machine learning detector with a partition field
+	/// </summary>
 	public interface IPartitionFieldNameDetector : IDetector
 	{
+		/// <summary>
+		/// The field used to segment the analysis. When you use this property, you have completely
+		/// independent baselines for each value of this field.
+		/// </summary>
 		[JsonProperty("partition_field_name")]
 		Field PartitionFieldName { get; set; }
 	}
 
+	/// <inheritdoc />
 	public abstract class DetectorBase : IDetector
 	{
 		protected DetectorBase(string function) => Function = function;
 
+		/// <inheritdoc />
 		public string DetectorDescription { get; set; }
+		/// <inheritdoc />
 		public int? DetectorIndex { get; set; }
+		/// <inheritdoc />
 		public ExcludeFrequent? ExcludeFrequent { get; set; }
+		/// <inheritdoc />
 		public string Function { get; }
+		/// <inheritdoc />
 		public bool? UseNull { get; set; }
+		/// <inheritdoc />
+		public IEnumerable<IDetectionRule> CustomRules { get; set; }
 	}
 
+	/// <inheritdoc cref="IDetector" />
 	public abstract class DetectorDescriptorBase<TDetectorDescriptor, TDetectorInterface>
 		: DescriptorBase<TDetectorDescriptor, TDetectorInterface>, IDetector
 		where TDetectorDescriptor : DetectorDescriptorBase<TDetectorDescriptor, TDetectorInterface>, TDetectorInterface
@@ -168,14 +242,23 @@ public abstract class DetectorDescriptorBase<TDetectorDescriptor, TDetectorInter
 		ExcludeFrequent? IDetector.ExcludeFrequent { get; set; }
 		string IDetector.Function => _function;
 		bool? IDetector.UseNull { get; set; }
+		IEnumerable<IDetectionRule> IDetector.CustomRules { get; set; }
 
+		/// <inheritdoc cref="IDetector.DetectorDescription" />
 		public TDetectorDescriptor DetectorDescription(string description) => Assign(a => a.DetectorDescription = description);
 
+		/// <inheritdoc cref="IDetector.ExcludeFrequent" />
 		public TDetectorDescriptor ExcludeFrequent(ExcludeFrequent? excludeFrequent) => Assign(a => a.ExcludeFrequent = excludeFrequent);
 
+		/// <inheritdoc cref="IDetector.UseNull" />
 		public TDetectorDescriptor UseNull(bool? useNull = true) => Assign(a => a.UseNull = useNull);
 
+		/// <inheritdoc cref="IDetector.DetectorIndex" />
 		public TDetectorDescriptor DetectorIndex(int? detectorIndex) => Assign(a => a.DetectorIndex = detectorIndex);
+
+		/// <inheritdoc cref="IDetector.CustomRules" />
+		public TDetectorDescriptor CustomRules(Func<DetectionRulesDescriptor, IPromise<List<IDetectionRule>>> selector) =>
+			Assign(a => a.CustomRules = selector.Invoke(new DetectionRulesDescriptor()).Value);
 	}
 
 	public class DetectorsDescriptor<T> : DescriptorPromiseBase<DetectorsDescriptor<T>, IList<IDetector>> where T : class

src/Tests/Tests/CodeStandards/Descriptors.doc.cs

@@ -212,6 +212,9 @@ from m in d.GetMethods()
 				where !(m.Name == nameof(SortDescriptor<object>.Descending) && dt == typeof(SortDescriptor<>))
 				where !(m.Name == nameof(ClrTypeMappingDescriptor<object>.DisableIdInference) && dt == typeof(ClrTypeMappingDescriptor<>))
 				where !(m.Name == nameof(ClrTypeMappingDescriptor.DisableIdInference) && dt == typeof(ClrTypeMappingDescriptor))
+				where !(m.Name == nameof(RuleConditionDescriptor.AppliesTo) && dt == typeof(RuleConditionDescriptor))
+				where !(m.Name == nameof(RuleConditionDescriptor.Operator) && dt == typeof(RuleConditionDescriptor))
+				where !(m.Name == nameof(RuleConditionDescriptor.Value) && dt == typeof(RuleConditionDescriptor))
 
 				select new {m, d, p};