[Security Solution][Timeline] Rebuild nested fields structure from fi…

…elds response (elastic#96187)

* First pass at rebuilding nested object structure from fields response

* Always requests TIMELINE_CTI_FIELDS as part of request

This only works for one level of nesting; will be extending tests to
allow for multiple levels momentarily.

* Build objects from arbitrary levels of nesting

This is a recursive implementation, but recursion depth is limited to
the number of levels of nesting, with arguments reducing in size as we
go (i.e. logarithmic)

* Simplify parsing logic, perf improvements

* Order short-circuiting conditions by cost, ascending
* Simplify object building for non-nested objects from fields
  * The non-nested case is the same as the base recursive case, so
    always call our recursive function if building from .fields
* Simplify getNestedParentPath
  * We can do a few simple string comparison rather than building up
    multiple strings/arrays
* Don't call getNestedParentPath unnecessarily, only if we have a field

* Simplify if branching

By definition, nestedParentFieldName can never be equal to fieldName, which means
there are only two branches here.

* Declare/export a more accurate fields type

Each top-level field value can be either an array of leaf values
(unknown[]), or an array of nested fields.

* Remove unnecessary condition

If fieldName is null or undefined, there is no reason to search for it
in dataFields. Looking through the git history this looks to be dead
code as a result of refactoring, as opposed to a legitimate bugfix, so
I'm removing it.

* Fix failing tests

* one was a test failure due to my modifying mock data
* one may have been a legitimate bug where we don't handle a hit without
  a fields response; I need to follow up with Xavier to verify.

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/events/index.ts

@@ -26,10 +26,12 @@ export interface EventsActionGroupData {
   doc_count: number;
 }
 
+export type Fields = Record<string, unknown[] | Fields[]>;
+
 export interface EventHit extends SearchHit {
   sort: string[];
   _source: EventSource;
-  fields: Record<string, unknown[]>;
+  fields: Fields;
   aggregations: {
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     [agg: string]: any;

x-pack/plugins/security_solution/common/utils/mock_event_details.ts

@@ -40,7 +40,7 @@ export const eventHit = {
     'source.geo.location': [{ coordinates: [118.7778, 32.0617], type: 'Point' }],
     'threat.indicator': [
       {
-        'matched.field': ['matched_field'],
+        'matched.field': ['matched_field', 'other_matched_field'],
         first_seen: ['2021-02-22T17:29:25.195Z'],
         provider: ['yourself'],
         type: ['custom'],
@@ -259,8 +259,8 @@ export const eventDetailsFormattedFields = [
   {
     category: 'threat',
     field: 'threat.indicator.matched.field',
-    values: ['matched_field', 'matched_field_2'],
-    originalValue: ['matched_field', 'matched_field_2'],
+    values: ['matched_field', 'other_matched_field', 'matched_field_2'],
+    originalValue: ['matched_field', 'other_matched_field', 'matched_field_2'],
     isObjectArray: false,
   },
   {

x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/constants.ts

@@ -5,6 +5,15 @@
  * 2.0.
  */
 
+export const TIMELINE_CTI_FIELDS = [
+  'threat.indicator.event.dataset',
+  'threat.indicator.event.reference',
+  'threat.indicator.matched.atomic',
+  'threat.indicator.matched.field',
+  'threat.indicator.matched.type',
+  'threat.indicator.provider',
+];
+
 export const TIMELINE_EVENTS_FIELDS = [
   '@timestamp',
   'signal.status',
@@ -230,4 +239,5 @@ export const TIMELINE_EVENTS_FIELDS = [
   'zeek.ssl.established',
   'zeek.ssl.resumed',
   'zeek.ssl.version',
+  ...TIMELINE_CTI_FIELDS,
 ];

x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts

@@ -5,10 +5,10 @@
  * 2.0.
  */
 
+import { eventHit } from '../../../../../../common/utils/mock_event_details';
 import { EventHit } from '../../../../../../common/search_strategy';
 import { TIMELINE_EVENTS_FIELDS } from './constants';
-import { formatTimelineData } from './helpers';
-import { eventHit } from '../../../../../../common/utils/mock_event_details';
+import { buildObjectForFieldPath, formatTimelineData } from './helpers';
 
 describe('#formatTimelineData', () => {
   it('happy path', async () => {
@@ -42,12 +42,12 @@ describe('#formatTimelineData', () => {
             value: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'],
           },
           {
-            field: 'source.geo.location',
-            value: [`{"lon":118.7778,"lat":32.0617}`],
+            field: 'threat.indicator.matched.field',
+            value: ['matched_field', 'other_matched_field', 'matched_field_2'],
           },
           {
-            field: 'threat.indicator.matched.field',
-            value: ['matched_field', 'matched_field_2'],
+            field: 'source.geo.location',
+            value: [`{"lon":118.7778,"lat":32.0617}`],
           },
         ],
         ecs: {
@@ -94,6 +94,34 @@ describe('#formatTimelineData', () => {
           user: {
             name: ['jenkins'],
           },
+          threat: {
+            indicator: [
+              {
+                event: {
+                  dataset: [],
+                  reference: [],
+                },
+                matched: {
+                  atomic: ['matched_atomic'],
+                  field: ['matched_field', 'other_matched_field'],
+                  type: [],
+                },
+                provider: ['yourself'],
+              },
+              {
+                event: {
+                  dataset: [],
+                  reference: [],
+                },
+                matched: {
+                  atomic: ['matched_atomic_2'],
+                  field: ['matched_field_2'],
+                  type: [],
+                },
+                provider: ['other_you'],
+              },
+            ],
+          },
         },
       },
     });
@@ -371,4 +399,173 @@ describe('#formatTimelineData', () => {
       },
     });
   });
+
+  describe('buildObjectForFieldPath', () => {
+    it('builds an object from a single non-nested field', () => {
+      expect(buildObjectForFieldPath('@timestamp', eventHit)).toEqual({
+        '@timestamp': ['2020-11-17T14:48:08.922Z'],
+      });
+    });
+
+    it('builds an object with no fields response', () => {
+      const { fields, ...fieldLessHit } = eventHit;
+      // @ts-expect-error fieldLessHit is intentionally missing fields
+      expect(buildObjectForFieldPath('@timestamp', fieldLessHit)).toEqual({
+        '@timestamp': [],
+      });
+    });
+
+    it('does not misinterpret non-nested fields with a common prefix', () => {
+      // @ts-expect-error hit is minimal
+      const hit: EventHit = {
+        fields: {
+          'foo.bar': ['baz'],
+          'foo.barBaz': ['foo'],
+        },
+      };
+
+      expect(buildObjectForFieldPath('foo.barBaz', hit)).toEqual({
+        foo: { barBaz: ['foo'] },
+      });
+    });
+
+    it('builds an array of objects from a nested field', () => {
+      // @ts-expect-error hit is minimal
+      const hit: EventHit = {
+        fields: {
+          foo: [{ bar: ['baz'] }],
+        },
+      };
+      expect(buildObjectForFieldPath('foo.bar', hit)).toEqual({
+        foo: [{ bar: ['baz'] }],
+      });
+    });
+
+    it('builds intermediate objects for nested fields', () => {
+      // @ts-expect-error nestedHit is minimal
+      const nestedHit: EventHit = {
+        fields: {
+          'foo.bar': [
+            {
+              baz: ['host.name'],
+            },
+          ],
+        },
+      };
+      expect(buildObjectForFieldPath('foo.bar.baz', nestedHit)).toEqual({
+        foo: {
+          bar: [
+            {
+              baz: ['host.name'],
+            },
+          ],
+        },
+      });
+    });
+
+    it('builds intermediate objects at multiple levels', () => {
+      expect(buildObjectForFieldPath('threat.indicator.matched.atomic', eventHit)).toEqual({
+        threat: {
+          indicator: [
+            {
+              matched: {
+                atomic: ['matched_atomic'],
+              },
+            },
+            {
+              matched: {
+                atomic: ['matched_atomic_2'],
+              },
+            },
+          ],
+        },
+      });
+    });
+
+    it('preserves multiple values for a single leaf', () => {
+      expect(buildObjectForFieldPath('threat.indicator.matched.field', eventHit)).toEqual({
+        threat: {
+          indicator: [
+            {
+              matched: {
+                field: ['matched_field', 'other_matched_field'],
+              },
+            },
+            {
+              matched: {
+                field: ['matched_field_2'],
+              },
+            },
+          ],
+        },
+      });
+    });
+
+    describe('multiple levels of nested fields', () => {
+      let nestedHit: EventHit;
+
+      beforeEach(() => {
+        // @ts-expect-error nestedHit is minimal
+        nestedHit = {
+          fields: {
+            'nested_1.foo': [
+              {
+                'nested_2.bar': [
+                  { leaf: ['leaf_value'], leaf_2: ['leaf_2_value'] },
+                  { leaf_2: ['leaf_2_value_2', 'leaf_2_value_3'] },
+                ],
+              },
+              {
+                'nested_2.bar': [
+                  { leaf: ['leaf_value_2'], leaf_2: ['leaf_2_value_4'] },
+                  { leaf: ['leaf_value_3'], leaf_2: ['leaf_2_value_5'] },
+                ],
+              },
+            ],
+          },
+        };
+      });
+
+      it('includes objects without the field', () => {
+        expect(buildObjectForFieldPath('nested_1.foo.nested_2.bar.leaf', nestedHit)).toEqual({
+          nested_1: {
+            foo: [
+              {
+                nested_2: {
+                  bar: [{ leaf: ['leaf_value'] }, { leaf: [] }],
+                },
+              },
+              {
+                nested_2: {
+                  bar: [{ leaf: ['leaf_value_2'] }, { leaf: ['leaf_value_3'] }],
+                },
+              },
+            ],
+          },
+        });
+      });
+
+      it('groups multiple leaf values', () => {
+        expect(buildObjectForFieldPath('nested_1.foo.nested_2.bar.leaf_2', nestedHit)).toEqual({
+          nested_1: {
+            foo: [
+              {
+                nested_2: {
+                  bar: [
+                    { leaf_2: ['leaf_2_value'] },
+                    { leaf_2: ['leaf_2_value_2', 'leaf_2_value_3'] },
+                  ],
+                },
+              },
+              {
+                nested_2: {
+                  bar: [{ leaf_2: ['leaf_2_value_4'] }, { leaf_2: ['leaf_2_value_5'] }],
+                },
+              },
+            ],
+          },
+        });
+      });
+    });
+  });
 });