diff --git a/x-pack/plugins/ingest_manager/common/constants/routes.ts b/x-pack/plugins/ingest_manager/common/constants/routes.ts index 616195b32f266..98ca52651a2ae 100644 --- a/x-pack/plugins/ingest_manager/common/constants/routes.ts +++ b/x-pack/plugins/ingest_manager/common/constants/routes.ts @@ -5,9 +5,10 @@ */ // Base API paths export const API_ROOT = `/api/ingest_manager`; +export const EPM_API_ROOT = `${API_ROOT}/epm`; +export const DATA_STREAM_API_ROOT = `${API_ROOT}/data_streams`; export const DATASOURCE_API_ROOT = `${API_ROOT}/datasources`; export const AGENT_CONFIG_API_ROOT = `${API_ROOT}/agent_configs`; -export const EPM_API_ROOT = `${API_ROOT}/epm`; export const FLEET_API_ROOT = `${API_ROOT}/fleet`; // EPM API routes @@ -23,6 +24,11 @@ export const EPM_API_ROUTES = { CATEGORIES_PATTERN: `${EPM_API_ROOT}/categories`, }; +// Data stream API routes +export const DATA_STREAM_API_ROUTES = { + LIST_PATTERN: `${DATA_STREAM_API_ROOT}`, +}; + // Datasource API routes export const DATASOURCE_API_ROUTES = { LIST_PATTERN: `${DATASOURCE_API_ROOT}`, diff --git a/x-pack/plugins/ingest_manager/common/services/routes.ts b/x-pack/plugins/ingest_manager/common/services/routes.ts index f2343b1039151..46b76d886f3cd 100644 --- a/x-pack/plugins/ingest_manager/common/services/routes.ts +++ b/x-pack/plugins/ingest_manager/common/services/routes.ts @@ -8,6 +8,7 @@ import { EPM_API_ROUTES, DATASOURCE_API_ROUTES, AGENT_CONFIG_API_ROUTES, + DATA_STREAM_API_ROUTES, FLEET_SETUP_API_ROUTES, AGENT_API_ROUTES, ENROLLMENT_API_KEY_ROUTES, @@ -88,6 +89,12 @@ export const agentConfigRouteService = { }, }; +export const dataStreamRouteService = { + getListPath: () => { + return DATA_STREAM_API_ROUTES.LIST_PATTERN; + }, +}; + export const fleetSetupRouteService = { getFleetSetupPath: () => FLEET_SETUP_API_ROUTES.INFO_PATTERN, postFleetSetupPath: () => FLEET_SETUP_API_ROUTES.CREATE_PATTERN, diff --git a/x-pack/plugins/ingest_manager/common/types/models/data_stream.ts b/x-pack/plugins/ingest_manager/common/types/models/data_stream.ts new file mode 100644 index 0000000000000..7da9bbad1b170 --- /dev/null +++ b/x-pack/plugins/ingest_manager/common/types/models/data_stream.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface DataStream { + index: string; + dataset: string; + namespace: string; + type: string; + package: string; + last_activity: string; + size_in_bytes: number; +} diff --git a/x-pack/plugins/ingest_manager/common/types/models/index.ts b/x-pack/plugins/ingest_manager/common/types/models/index.ts index 579b510e52daa..f73ab7af636a9 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/index.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/index.ts @@ -7,6 +7,7 @@ export * from './agent'; export * from './agent_config'; export * from './datasource'; +export * from './data_stream'; export * from './output'; export * from './epm'; export * from './enrollment_api_key'; diff --git a/x-pack/plugins/ingest_manager/server/types/rest_spec/fleet_setup.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/data_stream.ts similarity index 59% rename from x-pack/plugins/ingest_manager/server/types/rest_spec/fleet_setup.ts rename to x-pack/plugins/ingest_manager/common/types/rest_spec/data_stream.ts index 2244bcd44043f..24f8110562bfc 100644 --- a/x-pack/plugins/ingest_manager/server/types/rest_spec/fleet_setup.ts +++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/data_stream.ts @@ -3,11 +3,8 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import { DataStream } from '../models'; -export const GetFleetSetupRequestSchema = {}; - -export const CreateFleetSetupRequestSchema = {}; - -export interface CreateFleetSetupResponse { - isInitialized: boolean; +export interface GetDataStreamsResponse { + data_streams: DataStream[]; } diff --git a/x-pack/plugins/ingest_manager/common/types/rest_spec/fleet_setup.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/fleet_setup.ts index dc1d748a8743a..c4ba8ee595acf 100644 --- a/x-pack/plugins/ingest_manager/common/types/rest_spec/fleet_setup.ts +++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/fleet_setup.ts @@ -4,16 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -// eslint-disable-next-line @typescript-eslint/no-empty-interface -export interface GetFleetSetupRequest {} - -export interface CreateFleetSetupRequest { - body: { - fleet_enroll_username: string; - fleet_enroll_password: string; - }; -} - export interface CreateFleetSetupResponse { isInitialized: boolean; } diff --git a/x-pack/plugins/ingest_manager/common/types/rest_spec/index.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/index.ts index abe1bc8e3eddb..c1805023f497a 100644 --- a/x-pack/plugins/ingest_manager/common/types/rest_spec/index.ts +++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/index.ts @@ -5,6 +5,7 @@ */ export * from './common'; export * from './datasource'; +export * from './data_stream'; export * from './agent'; export * from './agent_config'; export * from './fleet_setup'; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/constants/index.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/constants/index.ts index 282ea8dbee3a2..619d03651dd96 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/constants/index.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/constants/index.ts @@ -12,6 +12,7 @@ export const EPM_LIST_INSTALLED_PACKAGES_PATH = `${EPM_PATH}/installed`; export const EPM_DETAIL_VIEW_PATH = `${EPM_PATH}/detail/:pkgkey/:panel?`; export const AGENT_CONFIG_PATH = '/configs'; export const AGENT_CONFIG_DETAILS_PATH = `${AGENT_CONFIG_PATH}/`; +export const DATA_STREAM_PATH = '/data-streams'; export const FLEET_PATH = '/fleet'; export const FLEET_AGENTS_PATH = `${FLEET_PATH}/agents`; export const FLEET_AGENT_DETAIL_PATH = `${FLEET_AGENTS_PATH}/`; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/data_stream.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/data_stream.ts new file mode 100644 index 0000000000000..9acf4b1e17449 --- /dev/null +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/data_stream.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { useRequest } from './use_request'; +import { dataStreamRouteService } from '../../services'; +import { GetDataStreamsResponse } from '../../types'; + +export const useGetDataStreams = () => { + return useRequest({ + path: dataStreamRouteService.getListPath(), + method: 'get', + }); +}; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/index.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/index.ts index 5014049407e65..084aba9a34309 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/index.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/index.ts @@ -6,6 +6,7 @@ export { setHttpClient, sendRequest, useRequest } from './use_request'; export * from './agent_config'; export * from './datasource'; +export * from './data_stream'; export * from './agents'; export * from './enrollment_api_keys'; export * from './epm'; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/index.tsx index f7c2805c6ea7c..6485862830d8a 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/index.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/index.tsx @@ -16,10 +16,10 @@ import { IngestManagerConfigType, IngestManagerStartDeps, } from '../../plugin'; -import { EPM_PATH, FLEET_PATH, AGENT_CONFIG_PATH } from './constants'; +import { EPM_PATH, FLEET_PATH, AGENT_CONFIG_PATH, DATA_STREAM_PATH } from './constants'; import { DefaultLayout, WithoutHeaderLayout } from './layouts'; import { Loading, Error } from './components'; -import { IngestManagerOverview, EPMApp, AgentConfigApp, FleetApp } from './sections'; +import { IngestManagerOverview, EPMApp, AgentConfigApp, FleetApp, DataStreamApp } from './sections'; import { CoreContext, DepsContext, ConfigContext, setHttpClient, useConfig } from './hooks'; import { PackageInstallProvider } from './sections/epm/hooks'; import { sendSetup } from './hooks/use_request/setup'; @@ -98,6 +98,11 @@ const IngestManagerRoutes = ({ ...rest }) => { + + + + + diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx index 345fd535b8ecc..f1f9063de72f0 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx @@ -10,7 +10,7 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { Section } from '../sections'; import { AlphaMessaging } from '../components'; import { useLink, useConfig } from '../hooks'; -import { EPM_PATH, FLEET_PATH, AGENT_CONFIG_PATH } from '../constants'; +import { EPM_PATH, FLEET_PATH, AGENT_CONFIG_PATH, DATA_STREAM_PATH } from '../constants'; interface Props { section?: Section; @@ -76,6 +76,12 @@ export const DefaultLayout: React.FunctionComponent = ({ section, childre defaultMessage="Fleet" /> + + + diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/index.tsx index 0498e814440c7..1ea162252c741 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/index.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/index.tsx @@ -191,7 +191,6 @@ export const AgentConfigListPage: React.FunctionComponent<{}> = () => { defaultMessage: 'Name', }), width: '20%', - // FIXME: use version once available - see: https://github.com/elastic/kibana/issues/56750 render: (name: string, agentConfig: AgentConfig) => ( diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/index.tsx new file mode 100644 index 0000000000000..7b0641e66fd43 --- /dev/null +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/index.tsx @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import React from 'react'; +import { HashRouter as Router, Route, Switch } from 'react-router-dom'; +import { DataStreamListPage } from './list_page'; + +export const DataStreamApp: React.FunctionComponent = () => { + return ( + + + + + + + + ); +}; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx new file mode 100644 index 0000000000000..d7a3e933f3bb5 --- /dev/null +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx @@ -0,0 +1,283 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import React, { useMemo } from 'react'; +import { + EuiBadge, + EuiButton, + EuiText, + EuiFlexGroup, + EuiFlexItem, + EuiEmptyPrompt, + EuiInMemoryTable, + EuiTableActionsColumnType, + EuiTableFieldDataColumnType, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage, FormattedDate } from '@kbn/i18n/react'; +import { DataStream } from '../../../types'; +import { WithHeaderLayout } from '../../../layouts'; +import { useGetDataStreams, useStartDeps, usePagination } from '../../../hooks'; + +const DataStreamListPageLayout: React.FunctionComponent = ({ children }) => ( + + + +

+ +

+ + + + +

+ +

+ + + + } + > + {children} + +); + +export const DataStreamListPage: React.FunctionComponent<{}> = () => { + const { + data: { fieldFormats }, + } = useStartDeps(); + + const { pagination, pageSizeOptions } = usePagination(); + + // Fetch agent configs + const { isLoading, data: dataStreamsData, sendRequest } = useGetDataStreams(); + + // Some configs retrieved, set up table props + const columns = useMemo(() => { + const cols: Array< + EuiTableFieldDataColumnType | EuiTableActionsColumnType + > = [ + { + field: 'dataset', + sortable: true, + width: '25%', + truncateText: true, + name: i18n.translate('xpack.ingestManager.dataStreamList.datasetColumnTitle', { + defaultMessage: 'Dataset', + }), + }, + { + field: 'type', + sortable: true, + truncateText: true, + name: i18n.translate('xpack.ingestManager.dataStreamList.typeColumnTitle', { + defaultMessage: 'Type', + }), + }, + { + field: 'namespace', + sortable: true, + truncateText: true, + name: i18n.translate('xpack.ingestManager.dataStreamList.namespaceColumnTitle', { + defaultMessage: 'Namespace', + }), + render: (namespace: string) => { + return namespace ? {namespace} : ''; + }, + }, + { + field: 'package', + sortable: true, + truncateText: true, + name: i18n.translate('xpack.ingestManager.dataStreamList.integrationColumnTitle', { + defaultMessage: 'Integration', + }), + }, + { + field: 'last_activity', + sortable: true, + width: '25%', + dataType: 'date', + name: i18n.translate('xpack.ingestManager.dataStreamList.lastActivityColumnTitle', { + defaultMessage: 'Last activity', + }), + render: (date: DataStream['last_activity']) => { + try { + const formatter = fieldFormats.getInstance('date'); + return formatter.convert(date); + } catch (e) { + return ; + } + }, + }, + { + field: 'size_in_bytes', + sortable: true, + name: i18n.translate('xpack.ingestManager.dataStreamList.sizeColumnTitle', { + defaultMessage: 'Size', + }), + render: (size: DataStream['size_in_bytes']) => { + try { + const formatter = fieldFormats.getInstance('bytes'); + return formatter.convert(size); + } catch (e) { + return `${size}b`; + } + }, + }, + ]; + return cols; + }, [fieldFormats]); + + const emptyPrompt = useMemo( + () => ( + + + + } + /> + ), + [] + ); + + const filterOptions: { [key: string]: string[] } = { + dataset: [], + type: [], + namespace: [], + package: [], + }; + + if (dataStreamsData && dataStreamsData.data_streams.length) { + dataStreamsData.data_streams.forEach(stream => { + const { dataset, type, namespace, package: pkg } = stream; + if (!filterOptions.dataset.includes(dataset)) { + filterOptions.dataset.push(dataset); + } + if (!filterOptions.type.includes(type)) { + filterOptions.type.push(type); + } + if (!filterOptions.namespace.includes(namespace)) { + filterOptions.namespace.push(namespace); + } + if (!filterOptions.package.includes(pkg)) { + filterOptions.package.push(pkg); + } + }); + } + + return ( + + + ) : dataStreamsData && !dataStreamsData.data_streams.length ? ( + emptyPrompt + ) : ( + + ) + } + items={dataStreamsData ? dataStreamsData.data_streams : []} + itemId="index" + columns={columns} + pagination={{ + initialPageSize: pagination.pageSize, + pageSizeOptions, + }} + sorting={true} + search={{ + toolsRight: [ + sendRequest()}> + + , + ], + box: { + placeholder: i18n.translate( + 'xpack.ingestManager.dataStreamList.searchPlaceholderTitle', + { + defaultMessage: 'Filter data streams', + } + ), + incremental: true, + }, + filters: [ + { + type: 'field_value_selection', + field: 'dataset', + name: i18n.translate('xpack.ingestManager.dataStreamList.datasetColumnTitle', { + defaultMessage: 'Dataset', + }), + multiSelect: 'or', + options: filterOptions.dataset.map(option => ({ + value: option, + name: option, + })), + }, + { + type: 'field_value_selection', + field: 'type', + name: i18n.translate('xpack.ingestManager.dataStreamList.typeColumnTitle', { + defaultMessage: 'Type', + }), + multiSelect: 'or', + options: filterOptions.type.map(option => ({ + value: option, + name: option, + })), + }, + { + type: 'field_value_selection', + field: 'namespace', + name: i18n.translate('xpack.ingestManager.dataStreamList.namespaceColumnTitle', { + defaultMessage: 'Namespace', + }), + multiSelect: 'or', + options: filterOptions.namespace.map(option => ({ + value: option, + name: option, + })), + }, + { + type: 'field_value_selection', + field: 'package', + name: i18n.translate('xpack.ingestManager.dataStreamList.integrationColumnTitle', { + defaultMessage: 'Integration', + }), + multiSelect: 'or', + options: filterOptions.package.map(option => ({ + value: option, + name: option, + })), + }, + ], + }} + /> + + ); +}; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/index.tsx index c691bb609d435..1f46c4cc820cb 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/index.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/index.tsx @@ -6,6 +6,7 @@ export { IngestManagerOverview } from './overview'; export { EPMApp } from './epm'; export { AgentConfigApp } from './agent_config'; +export { DataStreamApp } from './data_stream'; export { FleetApp } from './fleet'; -export type Section = 'overview' | 'epm' | 'agent_config' | 'fleet'; +export type Section = 'overview' | 'epm' | 'agent_config' | 'fleet' | 'data_stream'; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts index 5ebd1300baf65..53dbe295718c5 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts @@ -9,6 +9,7 @@ export { getFlattenedObject } from '../../../../../../../src/core/utils'; export { agentConfigRouteService, datasourceRouteService, + dataStreamRouteService, fleetSetupRouteService, agentRouteService, enrollmentAPIKeyRouteService, diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts index 75194d3397f90..8ca1495a94071 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts @@ -17,6 +17,7 @@ export { DatasourceInput, DatasourceInputStream, DatasourceConfigRecordEntry, + DataStream, // API schemas - Agent Config GetAgentConfigsResponse, GetAgentConfigsResponseItem, @@ -30,6 +31,8 @@ export { // API schemas - Datasource CreateDatasourceRequest, CreateDatasourceResponse, + // API schemas - Data Streams + GetDataStreamsResponse, // API schemas - Agents GetAgentsResponse, GetAgentsRequest, diff --git a/x-pack/plugins/ingest_manager/server/constants/index.ts b/x-pack/plugins/ingest_manager/server/constants/index.ts index 6ac92ca5d2a91..b2e72fefe5997 100644 --- a/x-pack/plugins/ingest_manager/server/constants/index.ts +++ b/x-pack/plugins/ingest_manager/server/constants/index.ts @@ -12,6 +12,7 @@ export { // Routes PLUGIN_ID, EPM_API_ROUTES, + DATA_STREAM_API_ROUTES, DATASOURCE_API_ROUTES, AGENT_API_ROUTES, AGENT_CONFIG_API_ROUTES, diff --git a/x-pack/plugins/ingest_manager/server/plugin.ts b/x-pack/plugins/ingest_manager/server/plugin.ts index c07d2efba6756..55aea4b1a4cdd 100644 --- a/x-pack/plugins/ingest_manager/server/plugin.ts +++ b/x-pack/plugins/ingest_manager/server/plugin.ts @@ -33,6 +33,7 @@ import { registerEncryptedSavedObjects } from './saved_objects'; import { registerEPMRoutes, registerDatasourceRoutes, + registerDataStreamRoutes, registerAgentConfigRoutes, registerSetupRoutes, registerAgentRoutes, @@ -141,6 +142,7 @@ export class IngestManagerPlugin // Register routes registerAgentConfigRoutes(router); registerDatasourceRoutes(router); + registerDataStreamRoutes(router); // Conditional routes if (config.epm.enabled) { diff --git a/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts new file mode 100644 index 0000000000000..a24518d644c4c --- /dev/null +++ b/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { RequestHandler } from 'src/core/server'; +import { DataStream } from '../../types'; +import { GetDataStreamsResponse } from '../../../common'; + +const DATA_STREAM_INDEX_PATTERN = 'logs-*-*,metrics-*-*'; + +export const getListHandler: RequestHandler = async (context, request, response) => { + const callCluster = context.core.elasticsearch.dataClient.callAsCurrentUser; + + try { + // Get stats (size on disk) of all potentially matching indices + const { indices: indexStats } = await callCluster('indices.stats', { + index: DATA_STREAM_INDEX_PATTERN, + metric: ['store'], + }); + + // Get all matching indices and info about each + // This returns the top 100,000 indices (as buckets) by last activity + const { + aggregations: { + index: { buckets: indexResults }, + }, + } = await callCluster('search', { + index: DATA_STREAM_INDEX_PATTERN, + body: { + size: 0, + query: { + bool: { + must: [ + { + exists: { + field: 'stream.namespace', + }, + }, + { + exists: { + field: 'stream.dataset', + }, + }, + ], + }, + }, + aggs: { + index: { + terms: { + field: '_index', + size: 100000, + order: { + last_activity: 'desc', + }, + }, + aggs: { + dataset: { + terms: { + field: 'stream.dataset', + size: 1, + }, + }, + namespace: { + terms: { + field: 'stream.namespace', + size: 1, + }, + }, + type: { + terms: { + field: 'stream.type', + size: 1, + }, + }, + package: { + terms: { + field: 'event.module', + size: 1, + }, + }, + last_activity: { + max: { + field: '@timestamp', + }, + }, + }, + }, + }, + }, + }); + + const dataStreams: DataStream[] = (indexResults as any[]).map(result => { + const { + key: indexName, + dataset: { buckets: datasetBuckets }, + namespace: { buckets: namespaceBuckets }, + type: { buckets: typeBuckets }, + package: { buckets: packageBuckets }, + last_activity: { value_as_string: lastActivity }, + } = result; + return { + index: indexName, + dataset: datasetBuckets.length ? datasetBuckets[0].key : '', + namespace: namespaceBuckets.length ? namespaceBuckets[0].key : '', + type: typeBuckets.length ? typeBuckets[0].key : '', + package: packageBuckets.length ? packageBuckets[0].key : '', + last_activity: lastActivity, + size_in_bytes: indexStats[indexName] ? indexStats[indexName].total.store.size_in_bytes : 0, + }; + }); + + const body: GetDataStreamsResponse = { + data_streams: dataStreams, + }; + return response.ok({ + body, + }); + } catch (e) { + return response.customError({ + statusCode: 500, + body: { message: e.message }, + }); + } +}; diff --git a/x-pack/plugins/ingest_manager/server/routes/data_streams/index.ts b/x-pack/plugins/ingest_manager/server/routes/data_streams/index.ts new file mode 100644 index 0000000000000..39502eba89a6a --- /dev/null +++ b/x-pack/plugins/ingest_manager/server/routes/data_streams/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { IRouter } from 'src/core/server'; +import { PLUGIN_ID, DATA_STREAM_API_ROUTES } from '../../constants'; +import { getListHandler } from './handlers'; + +export const registerRoutes = (router: IRouter) => { + // List of data streams + router.get( + { + path: DATA_STREAM_API_ROUTES.LIST_PATTERN, + validate: false, + options: { tags: [`access:${PLUGIN_ID}-read`] }, + }, + getListHandler + ); +}; diff --git a/x-pack/plugins/ingest_manager/server/routes/index.ts b/x-pack/plugins/ingest_manager/server/routes/index.ts index 33d75f3ab82cd..8a186c5485024 100644 --- a/x-pack/plugins/ingest_manager/server/routes/index.ts +++ b/x-pack/plugins/ingest_manager/server/routes/index.ts @@ -5,6 +5,7 @@ */ export { registerRoutes as registerAgentConfigRoutes } from './agent_config'; export { registerRoutes as registerDatasourceRoutes } from './datasource'; +export { registerRoutes as registerDataStreamRoutes } from './data_streams'; export { registerRoutes as registerEPMRoutes } from './epm'; export { registerRoutes as registerSetupRoutes } from './setup'; export { registerRoutes as registerAgentRoutes } from './agent'; diff --git a/x-pack/plugins/ingest_manager/server/routes/setup/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/setup/handlers.ts index 5c66f9008e2a3..837e73b966feb 100644 --- a/x-pack/plugins/ingest_manager/server/routes/setup/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/setup/handlers.ts @@ -5,7 +5,7 @@ */ import { RequestHandler } from 'src/core/server'; import { outputService } from '../../services'; -import { CreateFleetSetupResponse } from '../../types'; +import { CreateFleetSetupResponse } from '../../../common'; import { setupIngestManager, setupFleet } from '../../services/setup'; export const getFleetSetupHandler: RequestHandler = async (context, request, response) => { diff --git a/x-pack/plugins/ingest_manager/server/routes/setup/index.ts b/x-pack/plugins/ingest_manager/server/routes/setup/index.ts index a2c641503e825..edc9a0a268161 100644 --- a/x-pack/plugins/ingest_manager/server/routes/setup/index.ts +++ b/x-pack/plugins/ingest_manager/server/routes/setup/index.ts @@ -5,7 +5,6 @@ */ import { IRouter } from 'src/core/server'; import { PLUGIN_ID, FLEET_SETUP_API_ROUTES, SETUP_API_ROUTE } from '../../constants'; -import { GetFleetSetupRequestSchema, CreateFleetSetupRequestSchema } from '../../types'; import { getFleetSetupHandler, createFleetSetupHandler, @@ -28,7 +27,7 @@ export const registerRoutes = (router: IRouter) => { router.get( { path: FLEET_SETUP_API_ROUTES.INFO_PATTERN, - validate: GetFleetSetupRequestSchema, + validate: false, options: { tags: [`access:${PLUGIN_ID}-read`] }, }, getFleetSetupHandler @@ -38,7 +37,7 @@ export const registerRoutes = (router: IRouter) => { router.post( { path: FLEET_SETUP_API_ROUTES.CREATE_PATTERN, - validate: CreateFleetSetupRequestSchema, + validate: false, options: { tags: [`access:${PLUGIN_ID}-all`] }, }, createFleetSetupHandler diff --git a/x-pack/plugins/ingest_manager/server/types/index.tsx b/x-pack/plugins/ingest_manager/server/types/index.tsx index 105f9039f1e98..aa5496cc836b7 100644 --- a/x-pack/plugins/ingest_manager/server/types/index.tsx +++ b/x-pack/plugins/ingest_manager/server/types/index.tsx @@ -22,6 +22,7 @@ export { AgentConfig, NewAgentConfig, AgentConfigStatus, + DataStream, Output, NewOutput, OutputType, diff --git a/x-pack/plugins/ingest_manager/server/types/rest_spec/index.ts b/x-pack/plugins/ingest_manager/server/types/rest_spec/index.ts index c143cd3b35f91..42b607fa1c715 100644 --- a/x-pack/plugins/ingest_manager/server/types/rest_spec/index.ts +++ b/x-pack/plugins/ingest_manager/server/types/rest_spec/index.ts @@ -9,5 +9,4 @@ export * from './agent'; export * from './datasource'; export * from './epm'; export * from './enrollment_api_key'; -export * from './fleet_setup'; export * from './install_script';