[Security Solution] Add EQL query editable component with EQL options…

… fields (#199115)

**Partially addresses:** #171520

## Summary

This PR adds is built on top of #193828 and #196948 and adds an EQL Query editable component with EQL Options fields (`event_category_override`, `timestamp_field` and `tiebreaker_field`) for Three Way Diff tab's final edit side of the upgrade prebuilt rule workflow.

## Details

This PR make a set of changes to make existing EQL Query bar component easily reusable and type safe when used in forms. In particular the following was done

- EQL query bar was wrapped in `EqlQueryEdit` component with `UseField` inside. It helps to make it type safe avoiding issues like passing invalid types to `EqlQueryBar`. `UseField` types component properties as `Record<string, any>` so potentially any refactoring can break some functionality. For example code in Timeline passes `DataViewSpec` where `DataViewBase` is expected while these two types aren't fully compatible.
- Validation was added directly to `EqlQueryEdit`. Passing field configuration to `UseField` rewrites field configuration defined in from schema. It leads to cases when validation is defined in both form schema and as a field configuration for `UseFields`. Additionally we can reduce reusing complexity by incapsulating absolutely required validation in `EqlQueryEdit` component.
- Empty string `tiebreakerField` was removed in Timelines. `tiebreakerField` is part of EQL options used for EQL validation. EQL validation endpoint `/internal/search/eql` returns an error when an empty string provided for `tiebreakerField`. This problem didn't surface earlier since It looks like EQL options weren't provided correctly before this PR. Timeline EQL validation requests were sent without EQL options.

## How to test

The simplest way to test is via patching installed prebuilt rules via Rule Patch API. Please follow steps below

- Ensure the `prebuiltRulesCustomizationEnabled` feature flag is enabled
- Run Kibana locally
- Install an EQL prebuilt rule, e.g. `Potential Code Execution via Postgresql` with rule_id `2a692072-d78d-42f3-a48a-775677d79c4e`
- Patch the installed rule by running a query below

```bash
curl -X PATCH --user elastic:changeme  -H 'Content-Type: application/json' -H 'kbn-xsrf: 123' -H "elastic-api-version: 2023-10-31" -d '{"rule_id":"2a692072-d78d-42f3-a48a-775677d79c4e","version":1,"query":"process where process.name == \"cmd.exe\"","language":"eql","event_category_override": "test","timestamp_field": "@timestamp","tiebreaker_field": "tiebreaker"}' http://localhost:5601/kbn/api/detection_engine/rules
```

- Open `Detection Rules (SIEM)` Page -> `Rule Updates` -> click on `Potential Code Execution via Postgresql` rule -> expand `EQL Query` to see EQL Query -> press `Edit` button

## Screenshots

- EQL Query in Prebuilt Rules Update workflow
<img width="2560" alt="image" src="https://github.com/user-attachments/assets/59d157b2-6aca-4b21-95d0-f71a2e174df2">

- event_category_override + tiebreaker_field + timestamp_field (aka EQL options) in Prebuilt Rules Update workflow
<img width="2552" alt="image" src="https://github.com/user-attachments/assets/1886d3b4-98f9-40a7-954c-2a6d4b8e925a">

- Examples of invalid EQL
<img width="2560" alt="image" src="https://github.com/user-attachments/assets/d584deca-7903-45c5-9499-718552df441c">

<img width="2548" alt="image" src="https://github.com/user-attachments/assets/b734e22c-ab62-4624-85d0-e4e6dbb9d523">

packages/kbn-securitysolution-utils/index.ts

@@ -12,3 +12,4 @@ export * from './src/axios';
 export * from './src/transform_data_to_ndjson';
 export * from './src/path_validations';
 export * from './src/esql';
+export * from './src/debounce_async/debounce_async';

x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/eql_query_bar/validators.test.ts → packages/kbn-securitysolution-utils/src/debounce_async/debounce_async.test.ts

@@ -1,22 +1,20 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { debounceAsync } from './validators';
+import { debounceAsync } from './debounce_async';
 
 jest.useFakeTimers({ legacyFakeTimers: true });
 
 describe('debounceAsync', () => {
-  let fn: jest.Mock;
-
-  beforeEach(() => {
-    fn = jest.fn().mockResolvedValueOnce('first');
-  });
-
   it('resolves with the underlying invocation result', async () => {
+    const fn = jest.fn().mockResolvedValueOnce('first');
+
     const debounced = debounceAsync(fn, 0);
     const promise = debounced();
     jest.runOnlyPendingTimers();
@@ -25,6 +23,8 @@ describe('debounceAsync', () => {
   });
 
   it('resolves intermediate calls when the next invocation resolves', async () => {
+    const fn = jest.fn().mockResolvedValueOnce('first');
+
     const debounced = debounceAsync(fn, 200);
     fn.mockResolvedValueOnce('second');
 
@@ -39,6 +39,8 @@ describe('debounceAsync', () => {
   });
 
   it('debounces the function', async () => {
+    const fn = jest.fn().mockResolvedValueOnce('first');
+
     const debounced = debounceAsync(fn, 200);
 
     debounced();

packages/kbn-securitysolution-utils/src/debounce_async/debounce_async.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+/**
+ * Unlike lodash's debounce, which resolves intermediate calls with the most
+ * recent value, this implementation waits to resolve intermediate calls until
+ * the next invocation resolves.
+ *
+ * @param fn an async function
+ *
+ * @returns A debounced async function that resolves on the next invocation
+ */
+export function debounceAsync<Args extends unknown[], Result>(
+  fn: (...args: Args) => Result,
+  intervalMs: number
+): (...args: Args) => Promise<Awaited<Result>> {
+  let timeoutId: ReturnType<typeof setTimeout> | undefined;
+  let resolve: (value: Awaited<Result>) => void;
+  let promise = new Promise<Awaited<Result>>((_resolve) => {
+    resolve = _resolve;
+  });
+
+  return (...args) => {
+    if (timeoutId) {
+      clearTimeout(timeoutId);
+    }
+
+    timeoutId = setTimeout(async () => {
+      resolve(await fn(...args));
+      promise = new Promise((_resolve) => {
+        resolve = _resolve;
+      });
+    }, intervalMs);
+
+    return promise;
+  };
+}

x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_field_types.ts

@@ -9,14 +9,17 @@ import { z } from '@kbn/zod';
 import {
   BuildingBlockType,
   DataViewId,
+  EventCategoryOverride,
   IndexPatternArray,
   KqlQueryLanguage,
   RuleFilterArray,
   RuleNameOverride,
   RuleQuery,
   SavedQueryId,
+  TiebreakerField,
   TimelineTemplateId,
   TimelineTemplateTitle,
+  TimestampField,
   TimestampOverride,
   TimestampOverrideFallbackDisabled,
 } from '../../../../model/rule_schema';
@@ -78,6 +81,9 @@ export const RuleEqlQuery = z.object({
   query: RuleQuery,
   language: z.literal('eql'),
   filters: RuleFilterArray,
+  event_category_override: EventCategoryOverride.optional(),
+  timestamp_field: TimestampField.optional(),
+  tiebreaker_field: TiebreakerField.optional(),
 });
 
 export type RuleEsqlQuery = z.infer<typeof RuleEsqlQuery>;

x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/diffable_rule/diffable_rule.ts

@@ -9,7 +9,6 @@ import { z } from '@kbn/zod';
 import {
   AlertSuppression,
   AnomalyThreshold,
-  EventCategoryOverride,
   HistoryWindowStart,
   InvestigationFields,
   InvestigationGuide,
@@ -37,8 +36,6 @@ import {
   ThreatMapping,
   Threshold,
   ThresholdAlertSuppression,
-  TiebreakerField,
-  TimestampField,
 } from '../../../../model/rule_schema';
 
 import {
@@ -113,9 +110,6 @@ export const DiffableEqlFields = z.object({
   type: z.literal('eql'),
   eql_query: RuleEqlQuery, // NOTE: new field
   data_source: RuleDataSource.optional(), // NOTE: new field
-  event_category_override: EventCategoryOverride.optional(),
-  timestamp_field: TimestampField.optional(),
-  tiebreaker_field: TiebreakerField.optional(),
   alert_suppression: AlertSuppression.optional(),
 });
 

x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/convert_rule_to_diffable.ts

@@ -175,11 +175,15 @@ const extractDiffableEqlFieldsFromRuleObject = (
 ): RequiredOptional<DiffableEqlFields> => {
   return {
     type: rule.type,
-    eql_query: extractRuleEqlQuery(rule.query, rule.language, rule.filters),
+    eql_query: extractRuleEqlQuery({
+      query: rule.query,
+      language: rule.language,
+      filters: rule.filters,
+      eventCategoryOverride: rule.event_category_override,
+      timestampField: rule.timestamp_field,
+      tiebreakerField: rule.tiebreaker_field,
+    }),
     data_source: extractRuleDataSource(rule.index, rule.data_view_id),
-    event_category_override: rule.event_category_override,
-    timestamp_field: rule.timestamp_field,
-    tiebreaker_field: rule.tiebreaker_field,
     alert_suppression: rule.alert_suppression,
   };
 };

x-pack/plugins/security_solution/common/detection_engine/prebuilt_rules/diff/extract_rule_data_query.ts

@@ -8,9 +8,12 @@
 import type {
   EqlQueryLanguage,
   EsqlQueryLanguage,
+  EventCategoryOverride,
   KqlQueryLanguage,
   RuleFilterArray,
   RuleQuery,
+  TiebreakerField,
+  TimestampField,
 } from '../../../api/detection_engine/model/rule_schema';
 import type {
   InlineKqlQuery,
@@ -49,15 +52,23 @@ export const extractInlineKqlQuery = (
   };
 };
 
-export const extractRuleEqlQuery = (
-  query: RuleQuery,
-  language: EqlQueryLanguage,
-  filters: RuleFilterArray | undefined
-): RuleEqlQuery => {
+interface ExtractRuleEqlQueryParams {
+  query: RuleQuery;
+  language: EqlQueryLanguage;
+  filters: RuleFilterArray | undefined;
+  eventCategoryOverride: EventCategoryOverride | undefined;
+  timestampField: TimestampField | undefined;
+  tiebreakerField: TiebreakerField | undefined;
+}
+
+export const extractRuleEqlQuery = (params: ExtractRuleEqlQueryParams): RuleEqlQuery => {
   return {
-    query,
-    language,
-    filters: filters ?? [],
+    query: params.query,
+    language: params.language,
+    filters: params.filters ?? [],
+    event_category_override: params.eventCategoryOverride,
+    timestamp_field: params.timestampField,
+    tiebreaker_field: params.tiebreakerField,
   };
 };
 

x-pack/plugins/security_solution/common/search_strategy/timeline/events/eql/index.ts

@@ -7,7 +7,7 @@
 
 export type {
   TimelineEqlResponse,
-  EqlOptionsData,
-  EqlOptionsSelected,
+  EqlFieldsComboBoxOptions,
+  EqlOptions,
   FieldsEqlOptions,
 } from '@kbn/timelines-plugin/common';

x-pack/plugins/security_solution/public/common/hooks/eql/api.test.ts

@@ -34,7 +34,7 @@ const triggerValidateEql = () => {
     query: 'any where true',
     signal,
     runtimeMappings: undefined,
-    options: undefined,
+    eqlOptions: undefined,
   });
 };
 

x-pack/plugins/security_solution/public/common/hooks/eql/api.ts

@@ -11,7 +11,7 @@ import type { EqlSearchStrategyRequest, EqlSearchStrategyResponse } from '@kbn/d
 import { EQL_SEARCH_STRATEGY } from '@kbn/data-plugin/common';
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
-import type { EqlOptionsSelected } from '../../../../common/search_strategy';
+import type { EqlOptions } from '../../../../common/search_strategy';
 import {
   getValidationErrors,
   isErrorResponse,
@@ -31,9 +31,9 @@ interface Params {
   dataViewTitle: string;
   query: string;
   data: DataPublicPluginStart;
-  signal: AbortSignal;
   runtimeMappings: estypes.MappingRuntimeFields | undefined;
-  options: Omit<EqlOptionsSelected, 'query' | 'size'> | undefined;
+  eqlOptions: Omit<EqlOptions, 'query' | 'size'> | undefined;
+  signal?: AbortSignal;
 }
 
 export interface EqlResponseError {
@@ -51,9 +51,9 @@ export const validateEql = async ({
   data,
   dataViewTitle,
   query,
-  signal,
   runtimeMappings,
-  options,
+  eqlOptions,
+  signal,
 }: Params): Promise<ValidateEqlResponse> => {
   try {
     const { rawResponse: response } = await firstValueFrom(
@@ -62,9 +62,12 @@ export const validateEql = async ({
           params: {
             index: dataViewTitle,
             body: { query, runtime_mappings: runtimeMappings, size: 0 },
-            timestamp_field: options?.timestampField,
-            tiebreaker_field: options?.tiebreakerField || undefined,
-            event_category_field: options?.eventCategoryField,
+            // Prevent passing empty string values
+            timestamp_field: eqlOptions?.timestampField ? eqlOptions.timestampField : undefined,
+            tiebreaker_field: eqlOptions?.tiebreakerField ? eqlOptions.tiebreakerField : undefined,
+            event_category_field: eqlOptions?.eventCategoryField
+              ? eqlOptions.eventCategoryField
+              : undefined,
           },
           options: { ignore: [400] },
         },
@@ -79,26 +82,31 @@ export const validateEql = async ({
         valid: false,
         error: { code: EQL_ERROR_CODES.INVALID_SYNTAX, messages: getValidationErrors(response) },
       };
-    } else if (isVerificationErrorResponse(response) || isMappingErrorResponse(response)) {
+    }
+
+    if (isVerificationErrorResponse(response) || isMappingErrorResponse(response)) {
       return {
         valid: false,
         error: { code: EQL_ERROR_CODES.INVALID_EQL, messages: getValidationErrors(response) },
       };
-    } else if (isErrorResponse(response)) {
+    }
+
+    if (isErrorResponse(response)) {
       return {
         valid: false,
         error: { code: EQL_ERROR_CODES.FAILED_REQUEST, error: new Error(JSON.stringify(response)) },
       };
-    } else {
-      return { valid: true };
     }
+
+    return { valid: true };
   } catch (error) {
     if (error instanceof Error && error.message.startsWith('index_not_found_exception')) {
       return {
         valid: false,
         error: { code: EQL_ERROR_CODES.MISSING_DATA_SOURCE, messages: [error.message] },
       };
     }
+
     return {
       valid: false,
       error: {

x-pack/plugins/security_solution/public/common/mock/global_state.ts

@@ -349,7 +349,6 @@ export const mockGlobalState: State = {
         description: '',
         eqlOptions: {
           eventCategoryField: 'event.category',
-          tiebreakerField: '',
           timestampField: '@timestamp',
         },
         eventIdToNoteIds: { '1': ['1'] },

x-pack/plugins/security_solution/public/common/mock/timeline_results.ts

@@ -2042,7 +2042,6 @@ export const defaultTimelineProps: CreateTimelineProps = {
       eventCategoryField: 'event.category',
       query: '',
       size: 100,
-      tiebreakerField: '',
       timestampField: '@timestamp',
     },
     eventIdToNoteIds: {},