[8.x] [DataUsage][Serverless] data stream filter and date range filte…

…r fixes (#200731) (#200867)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[DataUsage][Serverless] data stream filter and date range filter
fixes (#200731)](#200731)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT
[{"author":{"name":"Ash","email":"1849116+ashokaditya@users.noreply.github.com"},"sourceCommit":{"committedDate":"2024-11-20T09:15:37Z","message":"[DataUsage][Serverless]
data stream filter and date range filter fixes (#200731)\n\n##
Summary\n\nAdds changes that:\n\n1. Shows more than 50 datastream
options if present and pre-selects at\nmost 50 on initial load.\n2.
Limits date range selection to 10 days from now.\n3. Allows date/time
selection to hours and minutes for ease of use.\n4. Pre-selects
start/end time range based on timezone offset from UTC on\nfirst
load\n\n### Checklist\n- [x] [Unit or
functional\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\nwere
updated or added to match the most common scenarios\n- [ ] [Flaky
Test\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\nused on any tests changed\n\n---------\n\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"e0129e0ea3bfcc4b053f29d04a5994876c4bae7b","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","ci:build-serverless-image","backport:version","v8.17.0"],"title":"[DataUsage][Serverless]
data stream filter and date range filter
fixes","number":200731,"url":"https://github.com/elastic/kibana/pull/200731","mergeCommit":{"message":"[DataUsage][Serverless]
data stream filter and date range filter fixes (#200731)\n\n##
Summary\n\nAdds changes that:\n\n1. Shows more than 50 datastream
options if present and pre-selects at\nmost 50 on initial load.\n2.
Limits date range selection to 10 days from now.\n3. Allows date/time
selection to hours and minutes for ease of use.\n4. Pre-selects
start/end time range based on timezone offset from UTC on\nfirst
load\n\n### Checklist\n- [x] [Unit or
functional\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\nwere
updated or added to match the most common scenarios\n- [ ] [Flaky
Test\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\nused on any tests changed\n\n---------\n\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"e0129e0ea3bfcc4b053f29d04a5994876c4bae7b"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/200731","number":200731,"mergeCommit":{"message":"[DataUsage][Serverless]
data stream filter and date range filter fixes (#200731)\n\n##
Summary\n\nAdds changes that:\n\n1. Shows more than 50 datastream
options if present and pre-selects at\nmost 50 on initial load.\n2.
Limits date range selection to 10 days from now.\n3. Allows date/time
selection to hours and minutes for ease of use.\n4. Pre-selects
start/end time range based on timezone offset from UTC on\nfirst
load\n\n### Checklist\n- [x] [Unit or
functional\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\nwere
updated or added to match the most common scenarios\n- [ ] [Flaky
Test\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\nused on any tests changed\n\n---------\n\nCo-authored-by:
kibanamachine
<42973632+kibanamachine@users.noreply.github.com>","sha":"e0129e0ea3bfcc4b053f29d04a5994876c4bae7b"}},{"branch":"8.x","label":"v8.17.0","branchLabelMappingKey":"^v8.17.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Ash <1849116+ashokaditya@users.noreply.github.com>

x-pack/plugins/data_usage/common/index.ts

@@ -12,6 +12,8 @@ export const PLUGIN_NAME = i18n.translate('xpack.dataUsage.name', {
   defaultMessage: 'Data Usage',
 });
 
+export const DEFAULT_SELECTED_OPTIONS = 50 as const;
+
 export const DATA_USAGE_API_ROUTE_PREFIX = '/api/data_usage/';
 export const DATA_USAGE_METRICS_API_ROUTE = `/internal${DATA_USAGE_API_ROUTE_PREFIX}metrics`;
 export const DATA_USAGE_DATA_STREAMS_API_ROUTE = `/internal${DATA_USAGE_API_ROUTE_PREFIX}data_streams`;

x-pack/plugins/data_usage/public/app/components/data_usage_metrics.test.tsx

@@ -4,9 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
 import React from 'react';
-
 import { render, waitFor } from '@testing-library/react';
 import userEvent, { type UserEvent } from '@testing-library/user-event';
 import { DataUsageMetrics } from './data_usage_metrics';
@@ -147,6 +145,13 @@ const getBaseMockedDataUsageMetrics = () => ({
   refetch: jest.fn(),
 });
 
+const generateDataStreams = (count: number) => {
+  return Array.from({ length: count }, (_, i) => ({
+    name: `.ds-${i}`,
+    storageSizeBytes: 1024 ** 2 * (22 / 7),
+  }));
+};
+
 describe('DataUsageMetrics', () => {
   let user: UserEvent;
   const testId = 'test';
@@ -174,8 +179,9 @@ describe('DataUsageMetrics', () => {
 
   it('should show date filter', () => {
     const { getByTestId } = render(<DataUsageMetrics data-test-subj={testId} />);
-    expect(getByTestId(`${testIdFilter}-date-range`)).toBeTruthy();
-    expect(getByTestId(`${testIdFilter}-date-range`).textContent).toContain('Last 24 hours');
+    const dateFilter = getByTestId(`${testIdFilter}-date-range`);
+    expect(dateFilter).toBeTruthy();
+    expect(dateFilter.textContent).toContain('to');
     expect(getByTestId(`${testIdFilter}-super-refresh-button`)).toBeTruthy();
   });
 
@@ -196,28 +202,7 @@ describe('DataUsageMetrics', () => {
   it('should show selected data streams on the filter', () => {
     mockUseGetDataUsageDataStreams.mockReturnValue({
       error: undefined,
-      data: [
-        {
-          name: '.ds-1',
-          storageSizeBytes: 10000,
-        },
-        {
-          name: '.ds-2',
-          storageSizeBytes: 20000,
-        },
-        {
-          name: '.ds-3',
-          storageSizeBytes: 10300,
-        },
-        {
-          name: '.ds-4',
-          storageSizeBytes: 23000,
-        },
-        {
-          name: '.ds-5',
-          storageSizeBytes: 23200,
-        },
-      ],
+      data: generateDataStreams(5),
       isFetching: false,
     });
     const { getByTestId } = render(<DataUsageMetrics data-test-subj={testId} />);
@@ -226,46 +211,35 @@ describe('DataUsageMetrics', () => {
     );
   });
 
+  it('should show at most 50 selected data streams on the filter', async () => {
+    mockUseGetDataUsageDataStreams.mockReturnValue({
+      error: undefined,
+      data: generateDataStreams(100),
+      isFetching: false,
+    });
+    const { getByTestId } = render(<DataUsageMetrics data-test-subj={testId} />);
+    const toggleFilterButton = getByTestId(`${testIdFilter}-dataStreams-popoverButton`);
+
+    expect(toggleFilterButton).toHaveTextContent('Data streams50');
+  });
+
   it('should allow de-selecting all but one data stream option', async () => {
     mockUseGetDataUsageDataStreams.mockReturnValue({
       error: undefined,
-      data: [
-        {
-          name: '.ds-1',
-          storageSizeBytes: 10000,
-        },
-        {
-          name: '.ds-2',
-          storageSizeBytes: 20000,
-        },
-        {
-          name: '.ds-3',
-          storageSizeBytes: 10300,
-        },
-        {
-          name: '.ds-4',
-          storageSizeBytes: 23000,
-        },
-        {
-          name: '.ds-5',
-          storageSizeBytes: 23200,
-        },
-      ],
+      data: generateDataStreams(5),
       isFetching: false,
     });
     const { getByTestId, getAllByTestId } = render(<DataUsageMetrics data-test-subj={testId} />);
-    expect(getByTestId(`${testIdFilter}-dataStreams-popoverButton`)).toHaveTextContent(
-      'Data streams5'
-    );
-    await user.click(getByTestId(`${testIdFilter}-dataStreams-popoverButton`));
+    const toggleFilterButton = getByTestId(`${testIdFilter}-dataStreams-popoverButton`);
+
+    expect(toggleFilterButton).toHaveTextContent('Data streams5');
+    await user.click(toggleFilterButton);
     const allFilterOptions = getAllByTestId('dataStreams-filter-option');
     for (let i = 0; i < allFilterOptions.length - 1; i++) {
       await user.click(allFilterOptions[i]);
     }
 
-    expect(getByTestId(`${testIdFilter}-dataStreams-popoverButton`)).toHaveTextContent(
-      'Data streams1'
-    );
+    expect(toggleFilterButton).toHaveTextContent('Data streams1');
   });
 
   it('should not call usage metrics API if no data streams', async () => {

x-pack/plugins/data_usage/public/app/components/data_usage_metrics.tsx

@@ -72,7 +72,9 @@ export const DataUsageMetrics = memo(
         setUrlMetricTypesFilter(metricsFilters.metricTypes.join(','));
       }
       if (!dataStreamsFromUrl && dataStreams) {
-        setUrlDataStreamsFilter(dataStreams.map((ds) => ds.name).join(','));
+        const hasMoreThan50 = dataStreams.length > 50;
+        const _dataStreams = hasMoreThan50 ? dataStreams.slice(0, 50) : dataStreams;
+        setUrlDataStreamsFilter(_dataStreams.map((ds) => ds.name).join(','));
       }
       if (!startDateFromUrl || !endDateFromUrl) {
         setUrlDateRangeFilter({ startDate: metricsFilters.from, endDate: metricsFilters.to });

x-pack/plugins/data_usage/public/app/components/filters/date_picker.tsx

@@ -15,6 +15,7 @@ import type {
   OnRefreshChangeProps,
 } from '@elastic/eui/src/components/date_picker/types';
 import { UI_SETTINGS } from '@kbn/data-plugin/common';
+import moment from 'moment';
 import { useTestIdGenerator } from '../../../hooks/use_test_id_generator';
 
 export interface DateRangePickerValues {
@@ -66,7 +67,7 @@ export const UsageMetricsDateRangePicker = memo<UsageMetricsDateRangePickerProps
       <EuiSuperDatePicker
         data-test-subj={getTestId('date-range')}
         isLoading={isDataLoading}
-        dateFormat={uiSettings.get('dateFormat')}
+        dateFormat={'MMM D, YYYY @ HH:mm'}
         commonlyUsedRanges={commonlyUsedRanges}
         end={dateRangePickerState.endDate}
         isPaused={!dateRangePickerState.autoRefreshOptions.enabled}
@@ -77,7 +78,11 @@ export const UsageMetricsDateRangePicker = memo<UsageMetricsDateRangePickerProps
         recentlyUsedRanges={dateRangePickerState.recentlyUsedDateRanges}
         start={dateRangePickerState.startDate}
         showUpdateButton={false}
+        timeFormat={'HH:mm'}
         updateButtonProps={{ iconOnly: false, fill: false }}
+        utcOffset={moment().utcOffset() / 60}
+        maxDate={moment()}
+        minDate={moment().subtract(9, 'days').startOf('day')}
         width="auto"
       />
     );

x-pack/plugins/data_usage/public/app/hooks/use_charts_filter.tsx

@@ -11,6 +11,7 @@ import {
   METRIC_TYPE_API_VALUES_TO_UI_OPTIONS_MAP,
   METRIC_TYPE_VALUES,
 } from '../../../common/rest_types';
+import { DEFAULT_SELECTED_OPTIONS } from '../../../common';
 import { FILTER_NAMES } from '../translations';
 import { useDataUsageMetricsUrlParams } from './use_charts_url_params';
 import { formatBytes } from '../../utils/format_bytes';
@@ -77,15 +78,17 @@ export const useChartsFilter = ({
           'data-test-subj': `${filterOptions.filterName}-filter-option`,
         }))
       : isDataStreamsFilter && !!filterOptions.options.length
-      ? filterOptions.options?.map((filterOption) => ({
+      ? filterOptions.options?.map((filterOption, i) => ({
           key: filterOption,
           label: filterOption,
           append: formatBytes(filterOptions.appendOptions?.[filterOption] ?? 0),
           checked: selectedDataStreamsFromUrl
             ? selectedDataStreamsFromUrl.includes(filterOption)
               ? 'on'
               : undefined
-            : 'on',
+            : i < DEFAULT_SELECTED_OPTIONS
+            ? 'on'
+            : undefined,
           'data-test-subj': `${filterOptions.filterName}-filter-option`,
         }))
       : []

x-pack/plugins/data_usage/public/app/hooks/use_charts_url_params.test.tsx

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import moment from 'moment';
 import { METRIC_TYPE_VALUES, MetricTypes } from '../../../common/rest_types';
 import { getDataUsageMetricsFiltersFromUrlParams } from './use_charts_url_params';
 
@@ -56,12 +57,12 @@ describe('#getDataUsageMetricsFiltersFromUrlParams', () => {
   it('should use given relative startDate and endDate values URL params', () => {
     expect(
       getDataUsageMetricsFiltersFromUrlParams({
-        startDate: 'now-24h/h',
-        endDate: 'now',
+        startDate: moment().subtract(24, 'hours').toISOString(),
+        endDate: moment().toISOString(),
       })
     ).toEqual({
-      endDate: 'now',
-      startDate: 'now-24h/h',
+      endDate: moment().toISOString(),
+      startDate: moment().subtract(24, 'hours').toISOString(),
     });
   });
 

x-pack/plugins/data_usage/public/app/hooks/use_date_picker.tsx

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import moment from 'moment';
 import { useCallback, useState } from 'react';
 import type {
   DurationRange,
@@ -18,8 +19,8 @@ export const DEFAULT_DATE_RANGE_OPTIONS = Object.freeze({
     enabled: false,
     duration: 10000,
   },
-  startDate: 'now-24h/h',
-  endDate: 'now',
+  startDate: moment().subtract(24, 'hours').startOf('day').toISOString(),
+  endDate: moment().toISOString(),
   recentlyUsedDateRanges: [],
 });
 

x-pack/plugins/data_usage/public/hooks/use_get_data_streams.ts

@@ -8,7 +8,7 @@
 import type { UseQueryOptions, UseQueryResult } from '@tanstack/react-query';
 import { useQuery } from '@tanstack/react-query';
 import type { IHttpFetchError } from '@kbn/core-http-browser';
-import { DATA_USAGE_DATA_STREAMS_API_ROUTE } from '../../common';
+import { DATA_USAGE_DATA_STREAMS_API_ROUTE, DEFAULT_SELECTED_OPTIONS } from '../../common';
 import { useKibanaContextForPlugin } from '../utils/use_kibana';
 
 type GetDataUsageDataStreamsResponse = Array<{
@@ -17,11 +17,6 @@ type GetDataUsageDataStreamsResponse = Array<{
   selected: boolean;
 }>;
 
-const PAGING_PARAMS = Object.freeze({
-  default: 50,
-  all: 10000,
-});
-
 export const useGetDataUsageDataStreams = ({
   selectedDataStreams,
   options = {
@@ -51,14 +46,14 @@ export const useGetDataUsageDataStreams = ({
         selected: GetDataUsageDataStreamsResponse;
         rest: GetDataUsageDataStreamsResponse;
       }>(
-        (acc, ds) => {
+        (acc, ds, i) => {
           const item = {
             name: ds.name,
             storageSizeBytes: ds.storageSizeBytes,
             selected: ds.selected,
           };
 
-          if (selectedDataStreams?.includes(ds.name)) {
+          if (selectedDataStreams?.includes(ds.name) && i < DEFAULT_SELECTED_OPTIONS) {
             acc.selected.push({ ...item, selected: true });
           } else {
             acc.rest.push({ ...item, selected: false });
@@ -69,20 +64,10 @@ export const useGetDataUsageDataStreams = ({
         { selected: [], rest: [] }
       );
 
-      let selectedDataStreamsCount = 0;
-      if (selectedDataStreams) {
-        selectedDataStreamsCount = selectedDataStreams.length;
-      }
-
       return [
         ...augmentedDataStreamsBasedOnSelectedItems.selected,
         ...augmentedDataStreamsBasedOnSelectedItems.rest,
-      ].slice(
-        0,
-        selectedDataStreamsCount >= PAGING_PARAMS.default
-          ? selectedDataStreamsCount + 10
-          : PAGING_PARAMS.default
-      );
+      ];
     },
   });
 };

x-pack/plugins/data_usage/public/hooks/use_get_usage_metrics.test.tsx

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import moment from 'moment';
 import React, { ReactNode } from 'react';
 import { QueryClient, QueryClientProvider, useQuery as _useQuery } from '@tanstack/react-query';
 import { renderHook } from '@testing-library/react-hooks';
@@ -41,8 +42,8 @@ jest.mock('../utils/use_kibana', () => {
 });
 
 const defaultUsageMetricsRequestBody = {
-  from: 'now-15m',
-  to: 'now',
+  from: moment().subtract(15, 'minutes').toISOString(),
+  to: moment().toISOString(),
   metricTypes: ['ingest_rate'],
   dataStreams: ['ds-1'],
 };

x-pack/plugins/data_usage/server/routes/internal/usage_metrics.test.ts

@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
+import moment from 'moment';
 import type { MockedKeys } from '@kbn/utility-types-jest';
 import type { CoreSetup } from '@kbn/core/server';
 import { registerUsageMetricsRoute } from './usage_metrics';
@@ -56,8 +56,8 @@ describe('registerUsageMetricsRoute', () => {
 
     const mockRequest = httpServerMock.createKibanaRequest({
       body: {
-        from: 'now-15m',
-        to: 'now',
+        from: moment().subtract(15, 'minutes').toISOString(),
+        to: moment().toISOString(),
         metricTypes: ['ingest_rate'],
         dataStreams: [],
       },
@@ -123,8 +123,8 @@ describe('registerUsageMetricsRoute', () => {
 
     const mockRequest = httpServerMock.createKibanaRequest({
       body: {
-        from: 'now-15m',
-        to: 'now',
+        from: moment().subtract(15, 'minutes').toISOString(),
+        to: moment().toISOString(),
         metricTypes: ['ingest_rate', 'storage_retained'],
         dataStreams: ['.ds-1', '.ds-2'],
       },
@@ -191,8 +191,8 @@ describe('registerUsageMetricsRoute', () => {
 
     const mockRequest = httpServerMock.createKibanaRequest({
       body: {
-        from: 'now-15m',
-        to: 'now',
+        from: moment().subtract(15, 'minutes').toISOString(),
+        to: moment().toISOString(),
         metricTypes: ['ingest_rate'],
         dataStreams: ['.ds-1', '.ds-2'],
       },