[Lens] Other bucket defaults to false for top values greater than equal 1000

x-pack/plugins/lens/public/datasources/form_based/dimension_panel/dimension_editor.tsx

@@ -67,6 +67,7 @@ import {
   CalloutWarning,
   DimensionEditorGroupsOptions,
   isLayerChangingDueToDecimalsPercentile,
+  isLayerChangingDueToOtherBucketChange,
 } from './dimensions_editor_helpers';
 import type { TemporaryState } from './dimensions_editor_helpers';
 import { FieldInput } from './field_input';
@@ -133,6 +134,8 @@ export function DimensionEditor(props: DimensionEditorProps) {
 
   const [hasRankingToastFired, setRankingToastAsFired] = useState(false);
 
+  const [hasOtherBucketToastFired, setHasOtherBucketToastFired] = useState(false);
+
   const onHelpClick = () => setIsHelpOpen((prevIsHelpOpen) => !prevIsHelpOpen);
   const closeHelp = () => setIsHelpOpen(false);
 
@@ -145,6 +148,25 @@ export function DimensionEditor(props: DimensionEditorProps) {
     [layerId, setState]
   );
 
+  const fireOrResetOtherBucketToast = useCallback(
+    (newLayer: FormBasedLayer) => {
+      if (isLayerChangingDueToOtherBucketChange(state.layers[layerId], newLayer)) {
+        props.notifications.toasts.add({
+          title: i18n.translate('xpack.lens.uiInfo.otherBucketChangeTitle', {
+            defaultMessage: '“Group remaining values as Other” disabled',
+          }),
+          text: i18n.translate('xpack.lens.uiInfo.otherBucketDisabled', {
+            defaultMessage:
+              'Values >= 1000 may slow performance. Re-enable the setting in “Advanced” options.',
+          }),
+        });
+      }
+      // resets the flag
+      setHasOtherBucketToastFired(!hasOtherBucketToastFired);
+    },
+    [layerId, props.notifications.toasts, state.layers, hasOtherBucketToastFired]
+  );
+
   const fireOrResetRandomSamplingToast = useCallback(
     (newLayer: FormBasedLayer) => {
       // if prev and current sampling state is different, show a toast to the user
@@ -189,8 +211,9 @@ export function DimensionEditor(props: DimensionEditorProps) {
     (newLayer: FormBasedLayer) => {
       fireOrResetRandomSamplingToast(newLayer);
       fireOrResetRankingToast(newLayer);
+      fireOrResetOtherBucketToast(newLayer);
     },
-    [fireOrResetRandomSamplingToast, fireOrResetRankingToast]
+    [fireOrResetRandomSamplingToast, fireOrResetRankingToast, fireOrResetOtherBucketToast]
   );
 
   const setStateWrapper = useCallback(

x-pack/plugins/lens/public/datasources/form_based/dimension_panel/dimension_editor_helpers.test.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { TermsIndexPatternColumn } from '../operations';
+import type { FormBasedLayer } from '../types';
+import { isLayerChangingDueToOtherBucketChange } from './dimensions_editor_helpers';
+
+describe('isLayerChangingDueToOtherBucketChange', () => {
+  function getLayer(otherBucket: boolean, size: number) {
+    return {
+      indexPatternId: '1',
+      columnOrder: ['col1', 'col2'],
+      columns: {
+        col1: {
+          label: 'My Op',
+          dataType: 'string',
+          isBucketed: true,
+          operationType: 'terms',
+          sourceField: 'source',
+          params: {
+            size,
+            orderDirection: 'asc',
+            otherBucket,
+            orderBy: {
+              type: 'alphabetical',
+            },
+          },
+        } as TermsIndexPatternColumn,
+        col2: {
+          label: 'My Op',
+          dataType: 'number',
+          isBucketed: false,
+          operationType: 'average',
+          sourceField: 'memory',
+        },
+      },
+    } as FormBasedLayer;
+  }
+
+  it('should return true if it changes programatically from size smaller to 1000 to a greater one', () => {
+    const prevLayer = getLayer(true, 5);
+    const newLayer = getLayer(false, 1000);
+    expect(isLayerChangingDueToOtherBucketChange(prevLayer, newLayer)).toBeTruthy();
+  });
+
+  it('should return false if it changes from size smaller to 1000 to another smaller than 1000', () => {
+    const prevLayer = getLayer(true, 5);
+    const newLayer = getLayer(true, 999);
+    expect(isLayerChangingDueToOtherBucketChange(prevLayer, newLayer)).toBeFalsy();
+  });
+
+  it('should return false if it changes from size greater than 1000 to another smaller than 1000', () => {
+    const prevLayer = getLayer(false, 1001);
+    const newLayer = getLayer(true, 4);
+    expect(isLayerChangingDueToOtherBucketChange(prevLayer, newLayer)).toBeFalsy();
+  });
+});

x-pack/plugins/lens/public/datasources/form_based/dimension_panel/dimensions_editor_helpers.tsx

@@ -25,6 +25,7 @@ import {
 } from '../operations';
 import { isColumnOfType } from '../operations/definitions/helpers';
 import { FormBasedLayer } from '../types';
+import { MAX_TERMS_OTHER_ENABLED } from '../operations/definitions/terms/constants';
 
 export const formulaOperationName = 'formula';
 export const staticValueOperationName = 'static_value';
@@ -35,6 +36,37 @@ export const nonQuickFunctions = new Set([formulaOperationName, staticValueOpera
 
 export type TemporaryState = typeof quickFunctionsName | typeof staticValueOperationName | 'none';
 
+export function isLayerChangingDueToOtherBucketChange(
+  prevLayer: FormBasedLayer,
+  newLayer: FormBasedLayer
+) {
+  // Finds the other bucket in prevState and return its value
+  const prevStateTermsColumns = Object.entries(prevLayer.columns)
+    .map(([id, column]) => {
+      if (isColumnOfType<TermsIndexPatternColumn>('terms', column)) {
+        return { id, otherBucket: column.params.otherBucket, termsSize: column.params.size };
+      }
+    })
+    .filter(nonNullable);
+  // Checks if the terms columns have changed the otherBucket value programatically.
+  // This happens when the terms size is greater than equal MAX_TERMS_OTHER_ENABLED
+  // and the previous state terms size is lower than MAX_TERMS_OTHER_ENABLED
+  const hasChangedOtherBucket = prevStateTermsColumns.some(({ id, otherBucket, termsSize }) => {
+    const newStateTermsColumn = newLayer.columns[id];
+    if (!isColumnOfType<TermsIndexPatternColumn>('terms', newStateTermsColumn)) {
+      return false;
+    }
+
+    return (
+      newStateTermsColumn.params.otherBucket !== otherBucket &&
+      !newStateTermsColumn.params.otherBucket &&
+      newStateTermsColumn.params.size >= MAX_TERMS_OTHER_ENABLED &&
+      termsSize < MAX_TERMS_OTHER_ENABLED
+    );
+  });
+  return hasChangedOtherBucket;
+}
+
 export function isLayerChangingDueToDecimalsPercentile(
   prevLayer: FormBasedLayer,
   newLayer: FormBasedLayer

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/terms/constants.ts

@@ -12,3 +12,5 @@ export const DEFAULT_MAX_DOC_COUNT = 1;
 export const supportedTypes = new Set(['string', 'boolean', 'number', 'ip']);
 
 export const MULTI_KEY_VISUAL_SEPARATOR = '›';
+
+export const MAX_TERMS_OTHER_ENABLED = 1000;

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/terms/helpers.test.ts

@@ -16,6 +16,7 @@ import {
   getDisallowedTermsMessage,
   getMultiTermsScriptedFieldErrorMessage,
   isSortableByColumn,
+  getOtherBucketSwitchDefault,
 } from './helpers';
 import { ReferenceBasedIndexPatternColumn } from '../column_types';
 import type { PercentileRanksIndexPatternColumn } from '../percentile_ranks';
@@ -601,4 +602,89 @@ describe('isSortableByColumn()', () => {
       ).toBeTruthy();
     });
   });
+
+  describe('other bucket defaults', () => {
+    it('should default to true if size < 1000 and previous otherBucket is not set', () => {
+      const column = {
+        label: `Top value of test`,
+        dataType: 'string',
+        isBucketed: true,
+        operationType: 'terms',
+        params: {
+          orderBy: { type: 'alphabetical' },
+          size: 3,
+          orderDirection: 'asc',
+        },
+        sourceField: 'test',
+      } as TermsIndexPatternColumn;
+      expect(getOtherBucketSwitchDefault(column, 10)).toBeTruthy();
+    });
+
+    it('should default to false if size > 1000 and previous otherBucket is not set', () => {
+      const column = {
+        label: `Top value of test`,
+        dataType: 'string',
+        isBucketed: true,
+        operationType: 'terms',
+        params: {
+          orderBy: { type: 'alphabetical' },
+          size: 3,
+          orderDirection: 'asc',
+        },
+        sourceField: 'test',
+      } as TermsIndexPatternColumn;
+      expect(getOtherBucketSwitchDefault(column, 1000)).toBeFalsy();
+    });
+
+    it('should default to true if size < 1000 and previous otherBucket is set to true', () => {
+      const column = {
+        label: `Top value of test`,
+        dataType: 'string',
+        isBucketed: true,
+        operationType: 'terms',
+        params: {
+          orderBy: { type: 'alphabetical' },
+          size: 3,
+          orderDirection: 'asc',
+          otherBucket: true,
+        },
+        sourceField: 'test',
+      } as TermsIndexPatternColumn;
+      expect(getOtherBucketSwitchDefault(column, 10)).toBeTruthy();
+    });
+
+    it('should default to false if size > 1000 and previous otherBucket is set to true', () => {
+      const column = {
+        label: `Top value of test`,
+        dataType: 'string',
+        isBucketed: true,
+        operationType: 'terms',
+        params: {
+          orderBy: { type: 'alphabetical' },
+          size: 3,
+          orderDirection: 'asc',
+          otherBucket: true,
+        },
+        sourceField: 'test',
+      } as TermsIndexPatternColumn;
+      expect(getOtherBucketSwitchDefault(column, 1001)).toBeFalsy();
+    });
+
+    it('should default to false if size < 1000 and previous otherBucket is set to false', () => {
+      const column = {
+        label: `Top value of test`,
+        dataType: 'string',
+        isBucketed: true,
+        operationType: 'terms',
+        params: {
+          orderBy: { type: 'alphabetical' },
+          size: 1005,
+          orderDirection: 'asc',
+          otherBucket: false,
+        },
+        sourceField: 'test',
+      } as TermsIndexPatternColumn;
+      expect(getOtherBucketSwitchDefault(column, 6)).toBeFalsy();
+    });
+  });
 });

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/terms/helpers.ts

@@ -25,7 +25,7 @@ import type { PercentileRanksIndexPatternColumn } from '../percentile_ranks';
 import type { PercentileIndexPatternColumn } from '../percentile';
 
 import type { FormBasedLayer } from '../../../types';
-import { MULTI_KEY_VISUAL_SEPARATOR, supportedTypes } from './constants';
+import { MULTI_KEY_VISUAL_SEPARATOR, supportedTypes, MAX_TERMS_OTHER_ENABLED } from './constants';
 import { isColumnOfType } from '../helpers';
 
 const fullSeparatorString = ` ${MULTI_KEY_VISUAL_SEPARATOR} `;
@@ -327,3 +327,8 @@ export function getFieldsByValidationState(
     invalidFields,
   };
 }
+
+export function getOtherBucketSwitchDefault(column: TermsIndexPatternColumn, size: number) {
+  const otherBucketValue = column.params.otherBucket;
+  return (otherBucketValue || otherBucketValue === undefined) && size < MAX_TERMS_OTHER_ENABLED;
+}

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/terms/index.tsx

@@ -46,6 +46,7 @@ import {
   isSortableByColumn,
   isPercentileRankSortable,
   isPercentileSortable,
+  getOtherBucketSwitchDefault,
 } from './helpers';
 import {
   DEFAULT_MAX_DOC_COUNT,
@@ -733,6 +734,7 @@ The top values of a specified field ranked by the chosen metric.
                   params: {
                     ...currentColumn.params,
                     size: value,
+                    otherBucket: getOtherBucketSwitchDefault(currentColumn, value),
                   },
                 },
               } as Record<string, TermsIndexPatternColumn>,

x-pack/plugins/lens/public/datasources/form_based/operations/definitions/terms/terms.test.tsx

@@ -2680,6 +2680,7 @@ describe('terms', () => {
             params: {
               ...(layer.columns.col1 as TermsIndexPatternColumn).params,
               size: 7,
+              otherBucket: true,
             },
           },
         },