[Discover] New field stats in Discover sidebar popover

docs/management/advanced-options.asciidoc

@@ -320,6 +320,10 @@ the minimum and maximum values of a numeric field or a map of a geo field.
 [[discover:showMultiFields]]`discover:showMultiFields`::
 Controls the display of multi-fields in the expanded document view.
 
+[[discover:showLegacyFieldTopValues]]`discover:showLegacyFieldTopValues`::
+This setting will calculate Top Values for a field based only on the loaded records on Discover page. 
+To use the new and more accurate view, turn off this option.
+
 [[discover-sort-defaultorder]]`discover:sort:defaultOrder`::
 The default sort direction for time-based data views.
 

src/plugins/discover/common/index.ts

@@ -28,4 +28,5 @@ export const TRUNCATE_MAX_HEIGHT = 'truncate:maxHeight';
 export const ROW_HEIGHT_OPTION = 'discover:rowHeightOption';
 export const SEARCH_EMBEDDABLE_TYPE = 'search';
 export const HIDE_ANNOUNCEMENTS = 'hideAnnouncements';
+export const SHOW_LEGACY_FIELD_TOP_VALUES = 'discover:showLegacyFieldTopValues';
 export const ENABLE_SQL = 'discover:enableSql';

src/plugins/discover/kibana.json

@@ -17,7 +17,8 @@
     "savedObjectsManagement",
     "dataViewFieldEditor",
     "dataViewEditor",
-    "expressions"
+    "expressions",
+    "unifiedFieldList"
   ],
   "optionalPlugins": [
     "home",

src/plugins/discover/public/__mocks__/__storybook_mocks__/with_discover_services.tsx

@@ -70,6 +70,10 @@ const services = {
           getAbsoluteTime: () => {
             return { from: '2020-05-14T11:05:13.590', to: '2020-05-14T11:20:13.590' };
           },
+          getTime: () => ({
+            from: 'now-7d',
+            to: 'now',
+          }),
         },
       },
       savedQueries: { findSavedQueries: () => Promise.resolve({ queries: [] as SavedQuery[] }) },

src/plugins/discover/public/application/main/components/layout/__stories__/get_layout_props.ts

@@ -172,6 +172,7 @@ export function getDocumentsLayoutProps(dataView: DataView) {
         language: 'kuery',
         query: '',
       },
+      filters: [],
     },
   } as unknown as DiscoverLayoutProps;
 }
@@ -186,6 +187,7 @@ export const getPlainRecordLayoutProps = (dataView: DataView) => {
       query: {
         sql: 'SELECT * FROM "kibana_sample_data_ecommerce"',
       },
+      filters: [],
     },
   } as unknown as DiscoverLayoutProps;
 };

src/plugins/discover/public/application/main/components/sidebar/__stories__/discover_field_details.stories.tsx

@@ -88,7 +88,7 @@ storiesOf('components/sidebar/DiscoverFieldDetails', module)
     <DiscoverFieldDetails
       field={field}
       dataView={dataView}
-      details={{ buckets: [], error: 'An error occurred', exists: 1, total: 2, columns: [] }}
+      details={{ buckets: [], error: 'An error occurred', exists: 1, total: 2 }}
       onAddFilter={() => {}}
     />
   ));

src/plugins/discover/public/application/main/components/sidebar/discover_field.test.tsx

@@ -6,15 +6,43 @@
  * Side Public License, v 1.
  */
 
+import { act } from 'react-dom/test-utils';
+import { EuiPopover, EuiProgress, EuiButtonIcon } from '@elastic/eui';
+import { ReactWrapper } from 'enzyme';
 import React from 'react';
 import { findTestSubject } from '@elastic/eui/lib/test';
 import { mountWithIntl } from '@kbn/test-jest-helpers';
-
-import { DiscoverField } from './discover_field';
+import { chartPluginMock } from '@kbn/charts-plugin/public/mocks';
+import { dataViewPluginMocks } from '@kbn/data-views-plugin/public/mocks';
+import { fieldFormatsServiceMock } from '@kbn/field-formats-plugin/public/mocks';
+import { dataPluginMock } from '@kbn/data-plugin/public/mocks';
+import { DiscoverField, DiscoverFieldProps } from './discover_field';
 import { DataViewField } from '@kbn/data-views-plugin/public';
 import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public';
 import { stubDataView } from '@kbn/data-views-plugin/common/data_view.stub';
 
+jest.mock('@kbn/unified-field-list-plugin/public/services/field_stats', () => ({
+  loadFieldStats: jest.fn().mockResolvedValue({
+    totalDocuments: 1624,
+    sampledDocuments: 1624,
+    sampledValues: 3248,
+    topValues: {
+      buckets: [
+        {
+          count: 2042,
+          key: 'osx',
+        },
+        {
+          count: 1206,
+          key: 'winx',
+        },
+      ],
+    },
+  }),
+}));
+
+const dataServiceMock = dataPluginMock.createStartContract();
+
 jest.mock('../../../../kibana_services', () => ({
   getUiActions: jest.fn(() => {
     return {
@@ -23,16 +51,18 @@ jest.mock('../../../../kibana_services', () => ({
   }),
 }));
 
-function getComponent({
+async function getComponent({
   selected = false,
-  showDetails = false,
+  showFieldStats = false,
   field,
   onAddFilterExists = true,
+  showLegacyFieldTopValues = false,
 }: {
   selected?: boolean;
-  showDetails?: boolean;
+  showFieldStats?: boolean;
   field?: DataViewField;
   onAddFilterExists?: boolean;
+  showLegacyFieldTopValues?: boolean;
 }) {
   const finalField =
     field ??
@@ -47,15 +77,20 @@ function getComponent({
       readFromDocValues: true,
     });
 
-  const props = {
+  const props: DiscoverFieldProps = {
     dataView: stubDataView,
     field: finalField,
-    getDetails: jest.fn(() => ({ buckets: [], error: '', exists: 1, total: 2, columns: [] })),
+    getDetails: jest.fn(() => ({ buckets: [], error: '', exists: 1, total: 2 })),
     ...(onAddFilterExists && { onAddFilter: jest.fn() }),
     onAddField: jest.fn(),
     onRemoveField: jest.fn(),
-    showDetails,
+    showFieldStats,
     selected,
+    state: {
+      query: { query: '', language: 'lucene' },
+      filters: [],
+    },
+    contextualFields: [],
   };
   const services = {
     history: () => ({
@@ -73,10 +108,32 @@ function getComponent({
         if (key === 'fields:popularLimit') {
           return 5;
         }
+        if (key === 'discover:showLegacyFieldTopValues') {
+          return showLegacyFieldTopValues;
+        }
+      },
+    },
+    data: {
+      ...dataServiceMock,
+      query: {
+        ...dataServiceMock.query,
+        timefilter: {
+          ...dataServiceMock.query.timefilter,
+          timefilter: {
+            ...dataServiceMock.query.timefilter.timefilter,
+            getAbsoluteTime: () => ({
+              from: '2021-08-31T22:00:00.000Z',
+              to: '2022-09-01T09:16:29.553Z',
+            }),
+          },
+        },
       },
     },
+    dataViews: dataViewPluginMocks.createStartContract(),
+    fieldFormats: fieldFormatsServiceMock.createStartContract(),
+    charts: chartPluginMock.createSetupContract(),
   };
-  const comp = mountWithIntl(
+  const comp = await mountWithIntl(
     <KibanaContextProvider services={services}>
       <DiscoverField {...props} />
     </KibanaContextProvider>
@@ -85,22 +142,26 @@ function getComponent({
 }
 
 describe('discover sidebar field', function () {
-  it('should allow selecting fields', function () {
-    const { comp, props } = getComponent({});
+  it('should allow selecting fields', async function () {
+    const { comp, props } = await getComponent({});
     findTestSubject(comp, 'fieldToggle-bytes').simulate('click');
     expect(props.onAddField).toHaveBeenCalledWith('bytes');
   });
-  it('should allow deselecting fields', function () {
-    const { comp, props } = getComponent({ selected: true });
+  it('should allow deselecting fields', async function () {
+    const { comp, props } = await getComponent({ selected: true });
     findTestSubject(comp, 'fieldToggle-bytes').simulate('click');
     expect(props.onRemoveField).toHaveBeenCalledWith('bytes');
   });
-  it('should trigger getDetails', function () {
-    const { comp, props } = getComponent({ selected: true });
+  it('should trigger getDetails', async function () {
+    const { comp, props } = await getComponent({
+      selected: true,
+      showFieldStats: true,
+      showLegacyFieldTopValues: true,
+    });
     findTestSubject(comp, 'field-bytes-showDetails').simulate('click');
     expect(props.getDetails).toHaveBeenCalledWith(props.field);
   });
-  it('should not allow clicking on _source', function () {
+  it('should not allow clicking on _source', async function () {
     const field = new DataViewField({
       name: '_source',
       type: '_source',
@@ -109,14 +170,15 @@ describe('discover sidebar field', function () {
       aggregatable: true,
       readFromDocValues: true,
     });
-    const { comp, props } = getComponent({
+    const { comp, props } = await getComponent({
       selected: true,
       field,
+      showLegacyFieldTopValues: true,
     });
     findTestSubject(comp, 'field-_source-showDetails').simulate('click');
     expect(props.getDetails).not.toHaveBeenCalled();
   });
-  it('displays warning for conflicting fields', function () {
+  it('displays warning for conflicting fields', async function () {
     const field = new DataViewField({
       name: 'troubled_field',
       type: 'conflict',
@@ -125,23 +187,26 @@ describe('discover sidebar field', function () {
       aggregatable: true,
       readFromDocValues: false,
     });
-    const { comp } = getComponent({
+    const { comp } = await getComponent({
       selected: true,
       field,
     });
     const dscField = findTestSubject(comp, 'field-troubled_field-showDetails');
     expect(dscField.find('.kbnFieldButton__infoIcon').length).toEqual(1);
   });
-  it('should not execute getDetails when rendered, since it can be expensive', function () {
-    const { props } = getComponent({});
-    expect(props.getDetails.mock.calls.length).toEqual(0);
+  it('should not execute getDetails when rendered, since it can be expensive', async function () {
+    const { props } = await getComponent({});
+    expect(props.getDetails).toHaveBeenCalledTimes(0);
   });
-  it('should execute getDetails when show details is requested', function () {
-    const { props, comp } = getComponent({});
+  it('should execute getDetails when show details is requested', async function () {
+    const { props, comp } = await getComponent({
+      showFieldStats: true,
+      showLegacyFieldTopValues: true,
+    });
     findTestSubject(comp, 'field-bytes-showDetails').simulate('click');
-    expect(props.getDetails.mock.calls.length).toEqual(1);
+    expect(props.getDetails).toHaveBeenCalledTimes(1);
   });
-  it('should not return the popover if onAddFilter is not provided', function () {
+  it('should not return the popover if onAddFilter is not provided', async function () {
     const field = new DataViewField({
       name: '_source',
       type: '_source',
@@ -150,12 +215,45 @@ describe('discover sidebar field', function () {
       aggregatable: true,
       readFromDocValues: true,
     });
-    const { comp } = getComponent({
+    const { comp } = await getComponent({
       selected: true,
       field,
       onAddFilterExists: false,
     });
     const popover = findTestSubject(comp, 'discoverFieldListPanelPopover');
     expect(popover.length).toBe(0);
   });
+  it('should request field stats', async function () {
+    const field = new DataViewField({
+      name: 'machine.os.raw',
+      type: 'string',
+      esTypes: ['keyword'],
+      aggregatable: true,
+      searchable: true,
+    });
+    let comp: ReactWrapper;
+
+    await act(async () => {
+      const result = await getComponent({ showFieldStats: true, field, onAddFilterExists: true });
+      comp = result.comp;
+      await comp.update();
+    });
+
+    await act(async () => {
+      const fieldItem = findTestSubject(comp, 'field-machine.os.raw-showDetails');
+      await fieldItem.simulate('click');
+      await comp.update();
+    });
+
+    await comp!.update();
+
+    expect(comp!.find(EuiPopover).prop('isOpen')).toBe(true);
+    expect(findTestSubject(comp!, 'dscFieldStats-title').text()).toBe('Top values');
+    expect(findTestSubject(comp!, 'dscFieldStats-topValues-bucket')).toHaveLength(2);
+    expect(
+      findTestSubject(comp!, 'dscFieldStats-topValues-formattedFieldValue').first().text()
+    ).toBe('osx');
+    expect(comp!.find(EuiProgress)).toHaveLength(2);
+    expect(findTestSubject(comp!, 'dscFieldStats-topValues').find(EuiButtonIcon)).toHaveLength(4);
+  });
 });