From e0e62b1188c07067a5976f76ebe16bd29a1af9cd Mon Sep 17 00:00:00 2001
From: Jen Huang <its.jenetic@gmail.com>
Date: Tue, 21 Apr 2020 11:12:12 -0700
Subject: [PATCH 1/6] Clean up fleet setup request/response typings

---
 .../common/types/rest_spec/fleet_setup.ts           | 10 ----------
 .../ingest_manager/server/routes/setup/handlers.ts  |  2 +-
 .../ingest_manager/server/routes/setup/index.ts     |  5 ++---
 .../server/types/rest_spec/fleet_setup.ts           | 13 -------------
 .../ingest_manager/server/types/rest_spec/index.ts  |  1 -
 5 files changed, 3 insertions(+), 28 deletions(-)
 delete mode 100644 x-pack/plugins/ingest_manager/server/types/rest_spec/fleet_setup.ts

diff --git a/x-pack/plugins/ingest_manager/common/types/rest_spec/fleet_setup.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/fleet_setup.ts
index dc1d748a8743a..c4ba8ee595acf 100644
--- a/x-pack/plugins/ingest_manager/common/types/rest_spec/fleet_setup.ts
+++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/fleet_setup.ts
@@ -4,16 +4,6 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-// eslint-disable-next-line @typescript-eslint/no-empty-interface
-export interface GetFleetSetupRequest {}
-
-export interface CreateFleetSetupRequest {
-  body: {
-    fleet_enroll_username: string;
-    fleet_enroll_password: string;
-  };
-}
-
 export interface CreateFleetSetupResponse {
   isInitialized: boolean;
 }
diff --git a/x-pack/plugins/ingest_manager/server/routes/setup/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/setup/handlers.ts
index 5c66f9008e2a3..837e73b966feb 100644
--- a/x-pack/plugins/ingest_manager/server/routes/setup/handlers.ts
+++ b/x-pack/plugins/ingest_manager/server/routes/setup/handlers.ts
@@ -5,7 +5,7 @@
  */
 import { RequestHandler } from 'src/core/server';
 import { outputService } from '../../services';
-import { CreateFleetSetupResponse } from '../../types';
+import { CreateFleetSetupResponse } from '../../../common';
 import { setupIngestManager, setupFleet } from '../../services/setup';
 
 export const getFleetSetupHandler: RequestHandler = async (context, request, response) => {
diff --git a/x-pack/plugins/ingest_manager/server/routes/setup/index.ts b/x-pack/plugins/ingest_manager/server/routes/setup/index.ts
index a2c641503e825..edc9a0a268161 100644
--- a/x-pack/plugins/ingest_manager/server/routes/setup/index.ts
+++ b/x-pack/plugins/ingest_manager/server/routes/setup/index.ts
@@ -5,7 +5,6 @@
  */
 import { IRouter } from 'src/core/server';
 import { PLUGIN_ID, FLEET_SETUP_API_ROUTES, SETUP_API_ROUTE } from '../../constants';
-import { GetFleetSetupRequestSchema, CreateFleetSetupRequestSchema } from '../../types';
 import {
   getFleetSetupHandler,
   createFleetSetupHandler,
@@ -28,7 +27,7 @@ export const registerRoutes = (router: IRouter) => {
   router.get(
     {
       path: FLEET_SETUP_API_ROUTES.INFO_PATTERN,
-      validate: GetFleetSetupRequestSchema,
+      validate: false,
       options: { tags: [`access:${PLUGIN_ID}-read`] },
     },
     getFleetSetupHandler
@@ -38,7 +37,7 @@ export const registerRoutes = (router: IRouter) => {
   router.post(
     {
       path: FLEET_SETUP_API_ROUTES.CREATE_PATTERN,
-      validate: CreateFleetSetupRequestSchema,
+      validate: false,
       options: { tags: [`access:${PLUGIN_ID}-all`] },
     },
     createFleetSetupHandler
diff --git a/x-pack/plugins/ingest_manager/server/types/rest_spec/fleet_setup.ts b/x-pack/plugins/ingest_manager/server/types/rest_spec/fleet_setup.ts
deleted file mode 100644
index 2244bcd44043f..0000000000000
--- a/x-pack/plugins/ingest_manager/server/types/rest_spec/fleet_setup.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-export const GetFleetSetupRequestSchema = {};
-
-export const CreateFleetSetupRequestSchema = {};
-
-export interface CreateFleetSetupResponse {
-  isInitialized: boolean;
-}
diff --git a/x-pack/plugins/ingest_manager/server/types/rest_spec/index.ts b/x-pack/plugins/ingest_manager/server/types/rest_spec/index.ts
index c143cd3b35f91..42b607fa1c715 100644
--- a/x-pack/plugins/ingest_manager/server/types/rest_spec/index.ts
+++ b/x-pack/plugins/ingest_manager/server/types/rest_spec/index.ts
@@ -9,5 +9,4 @@ export * from './agent';
 export * from './datasource';
 export * from './epm';
 export * from './enrollment_api_key';
-export * from './fleet_setup';
 export * from './install_script';

From 1fc656ab853dff36990b54a9f4b00b9950aa9373 Mon Sep 17 00:00:00 2001
From: Jen Huang <its.jenetic@gmail.com>
Date: Tue, 21 Apr 2020 13:31:06 -0700
Subject: [PATCH 2/6] Add data stream model and list route, handler, and
 request/response types

---
 .../ingest_manager/common/constants/routes.ts |   8 +-
 .../common/types/models/data_stream.ts        |  15 +++
 .../common/types/models/index.ts              |   1 +
 .../common/types/rest_spec/data_stream.ts     |  10 ++
 .../common/types/rest_spec/index.ts           |   1 +
 .../ingest_manager/server/constants/index.ts  |   1 +
 .../plugins/ingest_manager/server/plugin.ts   |   2 +
 .../server/routes/data_streams/handlers.ts    | 127 ++++++++++++++++++
 .../server/routes/data_streams/index.ts       |  20 +++
 .../ingest_manager/server/routes/index.ts     |   1 +
 .../ingest_manager/server/types/index.tsx     |   1 +
 11 files changed, 186 insertions(+), 1 deletion(-)
 create mode 100644 x-pack/plugins/ingest_manager/common/types/models/data_stream.ts
 create mode 100644 x-pack/plugins/ingest_manager/common/types/rest_spec/data_stream.ts
 create mode 100644 x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts
 create mode 100644 x-pack/plugins/ingest_manager/server/routes/data_streams/index.ts

diff --git a/x-pack/plugins/ingest_manager/common/constants/routes.ts b/x-pack/plugins/ingest_manager/common/constants/routes.ts
index a31d38a723c2c..df304fe91b78f 100644
--- a/x-pack/plugins/ingest_manager/common/constants/routes.ts
+++ b/x-pack/plugins/ingest_manager/common/constants/routes.ts
@@ -5,9 +5,10 @@
  */
 // Base API paths
 export const API_ROOT = `/api/ingest_manager`;
+export const EPM_API_ROOT = `${API_ROOT}/epm`;
+export const DATA_STREAM_API_ROOT = `${API_ROOT}/data_streams`;
 export const DATASOURCE_API_ROOT = `${API_ROOT}/datasources`;
 export const AGENT_CONFIG_API_ROOT = `${API_ROOT}/agent_configs`;
-export const EPM_API_ROOT = `${API_ROOT}/epm`;
 export const FLEET_API_ROOT = `${API_ROOT}/fleet`;
 
 // EPM API routes
@@ -23,6 +24,11 @@ export const EPM_API_ROUTES = {
   CATEGORIES_PATTERN: `${EPM_API_ROOT}/categories`,
 };
 
+// Data stream API routes
+export const DATA_STREAM_API_ROUTES = {
+  LIST_PATTERN: `${DATA_STREAM_API_ROOT}`,
+};
+
 // Datasource API routes
 export const DATASOURCE_API_ROUTES = {
   LIST_PATTERN: `${DATASOURCE_API_ROOT}`,
diff --git a/x-pack/plugins/ingest_manager/common/types/models/data_stream.ts b/x-pack/plugins/ingest_manager/common/types/models/data_stream.ts
new file mode 100644
index 0000000000000..7da9bbad1b170
--- /dev/null
+++ b/x-pack/plugins/ingest_manager/common/types/models/data_stream.ts
@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export interface DataStream {
+  index: string;
+  dataset: string;
+  namespace: string;
+  type: string;
+  package: string;
+  last_activity: string;
+  size_in_bytes: number;
+}
diff --git a/x-pack/plugins/ingest_manager/common/types/models/index.ts b/x-pack/plugins/ingest_manager/common/types/models/index.ts
index 579b510e52daa..f73ab7af636a9 100644
--- a/x-pack/plugins/ingest_manager/common/types/models/index.ts
+++ b/x-pack/plugins/ingest_manager/common/types/models/index.ts
@@ -7,6 +7,7 @@
 export * from './agent';
 export * from './agent_config';
 export * from './datasource';
+export * from './data_stream';
 export * from './output';
 export * from './epm';
 export * from './enrollment_api_key';
diff --git a/x-pack/plugins/ingest_manager/common/types/rest_spec/data_stream.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/data_stream.ts
new file mode 100644
index 0000000000000..24f8110562bfc
--- /dev/null
+++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/data_stream.ts
@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { DataStream } from '../models';
+
+export interface GetDataStreamsResponse {
+  data_streams: DataStream[];
+}
diff --git a/x-pack/plugins/ingest_manager/common/types/rest_spec/index.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/index.ts
index abe1bc8e3eddb..c1805023f497a 100644
--- a/x-pack/plugins/ingest_manager/common/types/rest_spec/index.ts
+++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/index.ts
@@ -5,6 +5,7 @@
  */
 export * from './common';
 export * from './datasource';
+export * from './data_stream';
 export * from './agent';
 export * from './agent_config';
 export * from './fleet_setup';
diff --git a/x-pack/plugins/ingest_manager/server/constants/index.ts b/x-pack/plugins/ingest_manager/server/constants/index.ts
index 6ac92ca5d2a91..b2e72fefe5997 100644
--- a/x-pack/plugins/ingest_manager/server/constants/index.ts
+++ b/x-pack/plugins/ingest_manager/server/constants/index.ts
@@ -12,6 +12,7 @@ export {
   // Routes
   PLUGIN_ID,
   EPM_API_ROUTES,
+  DATA_STREAM_API_ROUTES,
   DATASOURCE_API_ROUTES,
   AGENT_API_ROUTES,
   AGENT_CONFIG_API_ROUTES,
diff --git a/x-pack/plugins/ingest_manager/server/plugin.ts b/x-pack/plugins/ingest_manager/server/plugin.ts
index 4dd070a7414f0..9b8e1702f4f2f 100644
--- a/x-pack/plugins/ingest_manager/server/plugin.ts
+++ b/x-pack/plugins/ingest_manager/server/plugin.ts
@@ -32,6 +32,7 @@ import {
 import {
   registerEPMRoutes,
   registerDatasourceRoutes,
+  registerDataStreamRoutes,
   registerAgentConfigRoutes,
   registerSetupRoutes,
   registerAgentRoutes,
@@ -130,6 +131,7 @@ export class IngestManagerPlugin implements Plugin<IngestManagerSetupContract> {
     // Register routes
     registerAgentConfigRoutes(router);
     registerDatasourceRoutes(router);
+    registerDataStreamRoutes(router);
 
     // Conditional routes
     if (config.epm.enabled) {
diff --git a/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts
new file mode 100644
index 0000000000000..0a19997111310
--- /dev/null
+++ b/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts
@@ -0,0 +1,127 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { RequestHandler } from 'src/core/server';
+import { DataStream } from '../../types';
+import { GetDataStreamsResponse } from '../../../common';
+
+const DATA_STREAM_INDEX_PATTERN = 'logs-*-*,metrics-*-*';
+
+export const getListHandler: RequestHandler = async (context, request, response) => {
+  const callCluster = context.core.elasticsearch.dataClient.callAsCurrentUser;
+
+  try {
+    // Get stats (size on disk) of all potentially matching indices
+    const { indices: indexStats } = await callCluster('indices.stats', {
+      index: DATA_STREAM_INDEX_PATTERN,
+      metric: ['store'],
+    });
+
+    // Get all matching indices and info about each
+    // This returns the top 100,000 indices (as buckets) by last activity
+    const {
+      aggregations: {
+        index: { buckets: indexResults },
+      },
+    } = await callCluster('search', {
+      index: DATA_STREAM_INDEX_PATTERN,
+      body: {
+        size: 0,
+        query: {
+          bool: {
+            must: [
+              {
+                exists: {
+                  field: 'fields.stream.namespace',
+                },
+              },
+              {
+                exists: {
+                  field: 'fields.stream.dataset',
+                },
+              },
+            ],
+          },
+        },
+        aggs: {
+          index: {
+            terms: {
+              field: '_index',
+              size: 100000,
+              order: {
+                last_activity: 'desc',
+              },
+            },
+            aggs: {
+              dataset: {
+                terms: {
+                  field: 'fields.stream.dataset',
+                  size: 1,
+                },
+              },
+              namespace: {
+                terms: {
+                  field: 'fields.stream.namespace',
+                  size: 1,
+                },
+              },
+              type: {
+                terms: {
+                  field: 'fields.stream.type',
+                  size: 1,
+                },
+              },
+              package: {
+                terms: {
+                  field: 'event.module',
+                  size: 1,
+                },
+              },
+              last_activity: {
+                max: {
+                  field: '@timestamp',
+                },
+              },
+            },
+          },
+        },
+      },
+    });
+
+    const dataStreams: DataStream[] = [];
+
+    (indexResults as any[]).forEach(result => {
+      const {
+        key: indexName,
+        dataset: { buckets: datasetBuckets },
+        namespace: { buckets: namespaceBuckets },
+        type: { buckets: typeBuckets },
+        package: { buckets: packageBuckets },
+        last_activity: { value_as_string: lastActivity },
+      } = result;
+      dataStreams.push({
+        index: indexName,
+        dataset: datasetBuckets.length ? datasetBuckets[0].key : '',
+        namespace: namespaceBuckets.length ? namespaceBuckets[0].key : '',
+        type: typeBuckets.length ? typeBuckets[0].key : '',
+        package: packageBuckets.length ? packageBuckets[0].key : '',
+        last_activity: lastActivity,
+        size_in_bytes: indexStats[indexName] ? indexStats[indexName].total.store.size_in_bytes : 0,
+      });
+    });
+
+    const body: GetDataStreamsResponse = {
+      data_streams: dataStreams,
+    };
+    return response.ok({
+      body,
+    });
+  } catch (e) {
+    return response.customError({
+      statusCode: 500,
+      body: { message: e.message },
+    });
+  }
+};
diff --git a/x-pack/plugins/ingest_manager/server/routes/data_streams/index.ts b/x-pack/plugins/ingest_manager/server/routes/data_streams/index.ts
new file mode 100644
index 0000000000000..39502eba89a6a
--- /dev/null
+++ b/x-pack/plugins/ingest_manager/server/routes/data_streams/index.ts
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { IRouter } from 'src/core/server';
+import { PLUGIN_ID, DATA_STREAM_API_ROUTES } from '../../constants';
+import { getListHandler } from './handlers';
+
+export const registerRoutes = (router: IRouter) => {
+  // List of data streams
+  router.get(
+    {
+      path: DATA_STREAM_API_ROUTES.LIST_PATTERN,
+      validate: false,
+      options: { tags: [`access:${PLUGIN_ID}-read`] },
+    },
+    getListHandler
+  );
+};
diff --git a/x-pack/plugins/ingest_manager/server/routes/index.ts b/x-pack/plugins/ingest_manager/server/routes/index.ts
index 33d75f3ab82cd..8a186c5485024 100644
--- a/x-pack/plugins/ingest_manager/server/routes/index.ts
+++ b/x-pack/plugins/ingest_manager/server/routes/index.ts
@@ -5,6 +5,7 @@
  */
 export { registerRoutes as registerAgentConfigRoutes } from './agent_config';
 export { registerRoutes as registerDatasourceRoutes } from './datasource';
+export { registerRoutes as registerDataStreamRoutes } from './data_streams';
 export { registerRoutes as registerEPMRoutes } from './epm';
 export { registerRoutes as registerSetupRoutes } from './setup';
 export { registerRoutes as registerAgentRoutes } from './agent';
diff --git a/x-pack/plugins/ingest_manager/server/types/index.tsx b/x-pack/plugins/ingest_manager/server/types/index.tsx
index 1cd5622c0c7b0..144929a385b95 100644
--- a/x-pack/plugins/ingest_manager/server/types/index.tsx
+++ b/x-pack/plugins/ingest_manager/server/types/index.tsx
@@ -47,6 +47,7 @@ export {
   RegistrySearchResults,
   RegistrySearchResult,
   DefaultPackages,
+  DataStream,
 } from '../../common';
 
 export type CallESAsCurrentUser = ScopedClusterClient['callAsCurrentUser'];

From 5699a1a42dd58f24b43d13096d5b7f2b77fc5226 Mon Sep 17 00:00:00 2001
From: Jen Huang <its.jenetic@gmail.com>
Date: Tue, 21 Apr 2020 15:58:02 -0700
Subject: [PATCH 3/6] Initial pass at data streams list

---
 .../ingest_manager/common/services/routes.ts  |   7 +
 .../ingest_manager/constants/index.ts         |   1 +
 .../hooks/use_request/data_stream.ts          |  15 +
 .../ingest_manager/hooks/use_request/index.ts |   1 +
 .../applications/ingest_manager/index.tsx     |   9 +-
 .../ingest_manager/layouts/default.tsx        |   8 +-
 .../sections/agent_config/list_page/index.tsx |   1 -
 .../sections/data_stream/index.tsx            |  20 ++
 .../sections/data_stream/list_page/index.tsx  | 262 ++++++++++++++++++
 .../ingest_manager/sections/index.tsx         |   3 +-
 .../ingest_manager/services/index.ts          |   1 +
 .../ingest_manager/types/index.ts             |   3 +
 12 files changed, 326 insertions(+), 5 deletions(-)
 create mode 100644 x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/data_stream.ts
 create mode 100644 x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/index.tsx
 create mode 100644 x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx

diff --git a/x-pack/plugins/ingest_manager/common/services/routes.ts b/x-pack/plugins/ingest_manager/common/services/routes.ts
index 7cc6fc3c66afb..c8ea6fb3a04cf 100644
--- a/x-pack/plugins/ingest_manager/common/services/routes.ts
+++ b/x-pack/plugins/ingest_manager/common/services/routes.ts
@@ -8,6 +8,7 @@ import {
   EPM_API_ROUTES,
   DATASOURCE_API_ROUTES,
   AGENT_CONFIG_API_ROUTES,
+  DATA_STREAM_API_ROUTES,
   FLEET_SETUP_API_ROUTES,
   AGENT_API_ROUTES,
   ENROLLMENT_API_KEY_ROUTES,
@@ -88,6 +89,12 @@ export const agentConfigRouteService = {
   },
 };
 
+export const dataStreamRouteService = {
+  getListPath: () => {
+    return DATA_STREAM_API_ROUTES.LIST_PATTERN;
+  },
+};
+
 export const fleetSetupRouteService = {
   getFleetSetupPath: () => FLEET_SETUP_API_ROUTES.INFO_PATTERN,
   postFleetSetupPath: () => FLEET_SETUP_API_ROUTES.CREATE_PATTERN,
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/constants/index.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/constants/index.ts
index 282ea8dbee3a2..619d03651dd96 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/constants/index.ts
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/constants/index.ts
@@ -12,6 +12,7 @@ export const EPM_LIST_INSTALLED_PACKAGES_PATH = `${EPM_PATH}/installed`;
 export const EPM_DETAIL_VIEW_PATH = `${EPM_PATH}/detail/:pkgkey/:panel?`;
 export const AGENT_CONFIG_PATH = '/configs';
 export const AGENT_CONFIG_DETAILS_PATH = `${AGENT_CONFIG_PATH}/`;
+export const DATA_STREAM_PATH = '/data-streams';
 export const FLEET_PATH = '/fleet';
 export const FLEET_AGENTS_PATH = `${FLEET_PATH}/agents`;
 export const FLEET_AGENT_DETAIL_PATH = `${FLEET_AGENTS_PATH}/`;
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/data_stream.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/data_stream.ts
new file mode 100644
index 0000000000000..507fc2aa9e9ba
--- /dev/null
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/data_stream.ts
@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { useRequest, sendRequest } from './use_request';
+import { dataStreamRouteService } from '../../services';
+import { GetDataStreamsResponse } from '../../types';
+
+export const useGetDataStreams = () => {
+  return useRequest<GetDataStreamsResponse>({
+    path: dataStreamRouteService.getListPath(),
+    method: 'get',
+  });
+};
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/index.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/index.ts
index 5014049407e65..084aba9a34309 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/index.ts
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/index.ts
@@ -6,6 +6,7 @@
 export { setHttpClient, sendRequest, useRequest } from './use_request';
 export * from './agent_config';
 export * from './datasource';
+export * from './data_stream';
 export * from './agents';
 export * from './enrollment_api_keys';
 export * from './epm';
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/index.tsx
index f7c2805c6ea7c..6485862830d8a 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/index.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/index.tsx
@@ -16,10 +16,10 @@ import {
   IngestManagerConfigType,
   IngestManagerStartDeps,
 } from '../../plugin';
-import { EPM_PATH, FLEET_PATH, AGENT_CONFIG_PATH } from './constants';
+import { EPM_PATH, FLEET_PATH, AGENT_CONFIG_PATH, DATA_STREAM_PATH } from './constants';
 import { DefaultLayout, WithoutHeaderLayout } from './layouts';
 import { Loading, Error } from './components';
-import { IngestManagerOverview, EPMApp, AgentConfigApp, FleetApp } from './sections';
+import { IngestManagerOverview, EPMApp, AgentConfigApp, FleetApp, DataStreamApp } from './sections';
 import { CoreContext, DepsContext, ConfigContext, setHttpClient, useConfig } from './hooks';
 import { PackageInstallProvider } from './sections/epm/hooks';
 import { sendSetup } from './hooks/use_request/setup';
@@ -98,6 +98,11 @@ const IngestManagerRoutes = ({ ...rest }) => {
               <AgentConfigApp />
             </DefaultLayout>
           </Route>
+          <Route path={DATA_STREAM_PATH}>
+            <DefaultLayout section="data_stream">
+              <DataStreamApp />
+            </DefaultLayout>
+          </Route>
           <ProtectedRoute path={FLEET_PATH} isAllowed={fleet.enabled}>
             <DefaultLayout section="fleet">
               <FleetApp />
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx
index 345fd535b8ecc..f1f9063de72f0 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx
@@ -10,7 +10,7 @@ import { FormattedMessage } from '@kbn/i18n/react';
 import { Section } from '../sections';
 import { AlphaMessaging } from '../components';
 import { useLink, useConfig } from '../hooks';
-import { EPM_PATH, FLEET_PATH, AGENT_CONFIG_PATH } from '../constants';
+import { EPM_PATH, FLEET_PATH, AGENT_CONFIG_PATH, DATA_STREAM_PATH } from '../constants';
 
 interface Props {
   section?: Section;
@@ -76,6 +76,12 @@ export const DefaultLayout: React.FunctionComponent<Props> = ({ section, childre
                   defaultMessage="Fleet"
                 />
               </EuiTab>
+              <EuiTab isSelected={section === 'data_stream'} href={useLink(DATA_STREAM_PATH)}>
+                <FormattedMessage
+                  id="xpack.ingestManager.appNavigation.dataStreamsLinkText"
+                  defaultMessage="Data streams"
+                />
+              </EuiTab>
             </EuiTabs>
           </EuiFlexItem>
         </EuiFlexGroup>
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/index.tsx
index 0498e814440c7..1ea162252c741 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/index.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/index.tsx
@@ -191,7 +191,6 @@ export const AgentConfigListPage: React.FunctionComponent<{}> = () => {
           defaultMessage: 'Name',
         }),
         width: '20%',
-        // FIXME: use version once available - see: https://github.com/elastic/kibana/issues/56750
         render: (name: string, agentConfig: AgentConfig) => (
           <EuiFlexGroup gutterSize="s" alignItems="baseline" style={{ minWidth: 0 }}>
             <EuiFlexItem grow={false} style={NO_WRAP_TRUNCATE_STYLE}>
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/index.tsx
new file mode 100644
index 0000000000000..7b0641e66fd43
--- /dev/null
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/index.tsx
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import React from 'react';
+import { HashRouter as Router, Route, Switch } from 'react-router-dom';
+import { DataStreamListPage } from './list_page';
+
+export const DataStreamApp: React.FunctionComponent = () => {
+  return (
+    <Router>
+      <Switch>
+        <Route path="/data-streams">
+          <DataStreamListPage />
+        </Route>
+      </Switch>
+    </Router>
+  );
+};
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx
new file mode 100644
index 0000000000000..7c8202162abae
--- /dev/null
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx
@@ -0,0 +1,262 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import React, { useMemo } from 'react';
+import {
+  EuiBadge,
+  EuiText,
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiEmptyPrompt,
+  EuiInMemoryTable,
+  EuiTableActionsColumnType,
+  EuiTableFieldDataColumnType,
+} from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage, FormattedDate } from '@kbn/i18n/react';
+import { DataStream } from '../../../types';
+import { WithHeaderLayout } from '../../../layouts';
+import { useGetDataStreams, useStartDeps, usePagination } from '../../../hooks';
+
+const DataStreamListPageLayout: React.FunctionComponent = ({ children }) => (
+  <WithHeaderLayout
+    leftColumn={
+      <EuiFlexGroup direction="column" gutterSize="m">
+        <EuiFlexItem>
+          <EuiText>
+            <h1>
+              <FormattedMessage
+                id="xpack.ingestManager.dataStreamList.pageTitle"
+                defaultMessage="Data streams"
+              />
+            </h1>
+          </EuiText>
+        </EuiFlexItem>
+        <EuiFlexItem>
+          <EuiText color="subdued">
+            <p>
+              <FormattedMessage
+                id="xpack.ingestManager.dataStreamList.pageSubtitle"
+                defaultMessage="Manage the data created by your agents."
+              />
+            </p>
+          </EuiText>
+        </EuiFlexItem>
+      </EuiFlexGroup>
+    }
+  >
+    {children}
+  </WithHeaderLayout>
+);
+
+export const DataStreamListPage: React.FunctionComponent<{}> = () => {
+  const {
+    data: { fieldFormats },
+  } = useStartDeps();
+
+  const { pagination, pageSizeOptions } = usePagination();
+
+  // Fetch agent configs
+  const { isLoading, data: dataStreamsData } = useGetDataStreams();
+
+  // Some configs retrieved, set up table props
+  const columns = useMemo(() => {
+    const cols: Array<
+      EuiTableFieldDataColumnType<DataStream> | EuiTableActionsColumnType<DataStream>
+    > = [
+      {
+        field: 'index',
+        name: i18n.translate('xpack.ingestManager.dataStreamList.indexColumnTitle', {
+          defaultMessage: 'Index',
+        }),
+      },
+      {
+        field: 'dataset',
+        name: i18n.translate('xpack.ingestManager.dataStreamList.datasetColumnTitle', {
+          defaultMessage: 'Dataset',
+        }),
+      },
+      {
+        field: 'type',
+        name: i18n.translate('xpack.ingestManager.dataStreamList.typeColumnTitle', {
+          defaultMessage: 'Type',
+        }),
+      },
+      {
+        field: 'namespace',
+        name: i18n.translate('xpack.ingestManager.dataStreamList.namespaceColumnTitle', {
+          defaultMessage: 'Namespace',
+        }),
+        render: (namespace: DataStream['namespace']) => {
+          return <EuiBadge color="hollow">{namespace}</EuiBadge>;
+        },
+      },
+      {
+        field: 'package',
+        name: i18n.translate('xpack.ingestManager.dataStreamList.integrationColumnTitle', {
+          defaultMessage: 'Integration',
+        }),
+      },
+      {
+        field: 'last_activity',
+        dataType: 'date',
+        name: i18n.translate('xpack.ingestManager.dataStreamList.lastActivityColumnTitle', {
+          defaultMessage: 'Last activity',
+        }),
+        render: (date: DataStream['last_activity']) => {
+          try {
+            const formatter = fieldFormats.getInstance('date');
+            return formatter.convert(date);
+          } catch (e) {
+            return <FormattedDate value={date} year="numeric" month="short" day="2-digit" />;
+          }
+        },
+      },
+      {
+        field: 'size_in_bytes',
+        name: i18n.translate('xpack.ingestManager.dataStreamList.sizeColumnTitle', {
+          defaultMessage: 'Size',
+        }),
+        render: (size: DataStream['size_in_bytes']) => {
+          try {
+            const formatter = fieldFormats.getInstance('bytes');
+            return formatter.convert(size);
+          } catch (e) {
+            return `${size}b`;
+          }
+        },
+      },
+    ];
+    return cols;
+  }, [fieldFormats]);
+
+  const emptyPrompt = useMemo(
+    () => (
+      <EuiEmptyPrompt
+        title={
+          <h2>
+            <FormattedMessage
+              id="xpack.ingestManager.dataStreamList.noDataStreamsPrompt"
+              defaultMessage="No data streams"
+            />
+          </h2>
+        }
+      />
+    ),
+    []
+  );
+
+  const filterOptions: { [key: string]: string[] } = {
+    dataset: [],
+    type: [],
+    namespace: [],
+    package: [],
+  };
+
+  if (dataStreamsData && dataStreamsData.data_streams.length) {
+    dataStreamsData.data_streams.forEach(stream => {
+      const { dataset, type, namespace, package: pkg } = stream;
+      if (!filterOptions.dataset.includes(dataset)) {
+        filterOptions.dataset.push(dataset);
+      }
+      if (!filterOptions.type.includes(type)) {
+        filterOptions.type.push(type);
+      }
+      if (!filterOptions.namespace.includes(namespace)) {
+        filterOptions.namespace.push(namespace);
+      }
+      if (!filterOptions.package.includes(pkg)) {
+        filterOptions.package.push(pkg);
+      }
+    });
+  }
+
+  return (
+    <DataStreamListPageLayout>
+      <EuiInMemoryTable
+        loading={isLoading}
+        hasActions={true}
+        message={
+          isLoading ? (
+            <FormattedMessage
+              id="xpack.ingestManager.dataStreamList.loadingDataStreamsMessage"
+              defaultMessage="Loading data streams…"
+            />
+          ) : dataStreamsData && !dataStreamsData.data_streams.length ? (
+            emptyPrompt
+          ) : null
+        }
+        items={dataStreamsData ? dataStreamsData.data_streams : []}
+        itemId="index"
+        columns={columns}
+        pagination={{
+          initialPageSize: pagination.pageSize,
+          pageSizeOptions,
+        }}
+        search={{
+          box: {
+            placeholder: i18n.translate(
+              'xpack.ingestManager.dataStreamList.searchPlaceholderTitle',
+              {
+                defaultMessage: 'Filter data streams',
+              }
+            ),
+            incremental: true,
+          },
+          filters: [
+            {
+              type: 'field_value_selection',
+              field: 'dataset',
+              name: i18n.translate('xpack.ingestManager.dataStreamList.datasetColumnTitle', {
+                defaultMessage: 'Dataset',
+              }),
+              multiSelect: true,
+              options: filterOptions.dataset.map(option => ({
+                value: option,
+                name: option,
+              })),
+            },
+            {
+              type: 'field_value_selection',
+              field: 'type',
+              name: i18n.translate('xpack.ingestManager.dataStreamList.typeColumnTitle', {
+                defaultMessage: 'Type',
+              }),
+              multiSelect: true,
+              options: filterOptions.type.map(option => ({
+                value: option,
+                name: option,
+              })),
+            },
+            {
+              type: 'field_value_selection',
+              field: 'namespace',
+              name: i18n.translate('xpack.ingestManager.dataStreamList.namespaceColumnTitle', {
+                defaultMessage: 'Namespace',
+              }),
+              multiSelect: true,
+              options: filterOptions.namespace.map(option => ({
+                value: option,
+                name: option,
+              })),
+            },
+            {
+              type: 'field_value_selection',
+              field: 'package',
+              name: i18n.translate('xpack.ingestManager.dataStreamList.integrationColumnTitle', {
+                defaultMessage: 'Integration',
+              }),
+              multiSelect: true,
+              options: filterOptions.package.map(option => ({
+                value: option,
+                name: option,
+              })),
+            },
+          ],
+        }}
+      />
+    </DataStreamListPageLayout>
+  );
+};
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/index.tsx
index c691bb609d435..1f46c4cc820cb 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/index.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/index.tsx
@@ -6,6 +6,7 @@
 export { IngestManagerOverview } from './overview';
 export { EPMApp } from './epm';
 export { AgentConfigApp } from './agent_config';
+export { DataStreamApp } from './data_stream';
 export { FleetApp } from './fleet';
 
-export type Section = 'overview' | 'epm' | 'agent_config' | 'fleet';
+export type Section = 'overview' | 'epm' | 'agent_config' | 'fleet' | 'data_stream';
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts
index 5ebd1300baf65..53dbe295718c5 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts
@@ -9,6 +9,7 @@ export { getFlattenedObject } from '../../../../../../../src/core/utils';
 export {
   agentConfigRouteService,
   datasourceRouteService,
+  dataStreamRouteService,
   fleetSetupRouteService,
   agentRouteService,
   enrollmentAPIKeyRouteService,
diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts
index 32615278b67d7..608c984cfba39 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/types/index.ts
@@ -17,6 +17,7 @@ export {
   DatasourceInput,
   DatasourceInputStream,
   DatasourceConfigRecordEntry,
+  DataStream,
   // API schemas - Agent Config
   GetAgentConfigsResponse,
   GetAgentConfigsResponseItem,
@@ -30,6 +31,8 @@ export {
   // API schemas - Datasource
   CreateDatasourceRequest,
   CreateDatasourceResponse,
+  // API schemas - Data Streams
+  GetDataStreamsResponse,
   // API schemas - Agents
   GetAgentsResponse,
   GetAgentsRequest,

From f0b0b39d54435eb74694dfeea03f890eaf933c39 Mon Sep 17 00:00:00 2001
From: Jen Huang <its.jenetic@gmail.com>
Date: Tue, 21 Apr 2020 16:29:37 -0700
Subject: [PATCH 4/6] Table styling fixes

---
 .../sections/data_stream/list_page/index.tsx  | 49 +++++++++++++------
 1 file changed, 35 insertions(+), 14 deletions(-)

diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx
index 7c8202162abae..d7a3e933f3bb5 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx
@@ -6,6 +6,7 @@
 import React, { useMemo } from 'react';
 import {
   EuiBadge,
+  EuiButton,
   EuiText,
   EuiFlexGroup,
   EuiFlexItem,
@@ -59,48 +60,53 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => {
   const { pagination, pageSizeOptions } = usePagination();
 
   // Fetch agent configs
-  const { isLoading, data: dataStreamsData } = useGetDataStreams();
+  const { isLoading, data: dataStreamsData, sendRequest } = useGetDataStreams();
 
   // Some configs retrieved, set up table props
   const columns = useMemo(() => {
     const cols: Array<
       EuiTableFieldDataColumnType<DataStream> | EuiTableActionsColumnType<DataStream>
     > = [
-      {
-        field: 'index',
-        name: i18n.translate('xpack.ingestManager.dataStreamList.indexColumnTitle', {
-          defaultMessage: 'Index',
-        }),
-      },
       {
         field: 'dataset',
+        sortable: true,
+        width: '25%',
+        truncateText: true,
         name: i18n.translate('xpack.ingestManager.dataStreamList.datasetColumnTitle', {
           defaultMessage: 'Dataset',
         }),
       },
       {
         field: 'type',
+        sortable: true,
+        truncateText: true,
         name: i18n.translate('xpack.ingestManager.dataStreamList.typeColumnTitle', {
           defaultMessage: 'Type',
         }),
       },
       {
         field: 'namespace',
+        sortable: true,
+        truncateText: true,
         name: i18n.translate('xpack.ingestManager.dataStreamList.namespaceColumnTitle', {
           defaultMessage: 'Namespace',
         }),
-        render: (namespace: DataStream['namespace']) => {
-          return <EuiBadge color="hollow">{namespace}</EuiBadge>;
+        render: (namespace: string) => {
+          return namespace ? <EuiBadge color="hollow">{namespace}</EuiBadge> : '';
         },
       },
       {
         field: 'package',
+        sortable: true,
+        truncateText: true,
         name: i18n.translate('xpack.ingestManager.dataStreamList.integrationColumnTitle', {
           defaultMessage: 'Integration',
         }),
       },
       {
         field: 'last_activity',
+        sortable: true,
+        width: '25%',
         dataType: 'date',
         name: i18n.translate('xpack.ingestManager.dataStreamList.lastActivityColumnTitle', {
           defaultMessage: 'Last activity',
@@ -116,6 +122,7 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => {
       },
       {
         field: 'size_in_bytes',
+        sortable: true,
         name: i18n.translate('xpack.ingestManager.dataStreamList.sizeColumnTitle', {
           defaultMessage: 'Size',
         }),
@@ -186,7 +193,12 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => {
             />
           ) : dataStreamsData && !dataStreamsData.data_streams.length ? (
             emptyPrompt
-          ) : null
+          ) : (
+            <FormattedMessage
+              id="xpack.ingestManager.dataStreamList.noFilteredDataStreamsMessage"
+              defaultMessage="No matching data streams found"
+            />
+          )
         }
         items={dataStreamsData ? dataStreamsData.data_streams : []}
         itemId="index"
@@ -195,7 +207,16 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => {
           initialPageSize: pagination.pageSize,
           pageSizeOptions,
         }}
+        sorting={true}
         search={{
+          toolsRight: [
+            <EuiButton color="primary" iconType="refresh" onClick={() => sendRequest()}>
+              <FormattedMessage
+                id="xpack.ingestManager.dataStreamList.reloadDataStreamsButtonText"
+                defaultMessage="Reload"
+              />
+            </EuiButton>,
+          ],
           box: {
             placeholder: i18n.translate(
               'xpack.ingestManager.dataStreamList.searchPlaceholderTitle',
@@ -212,7 +233,7 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => {
               name: i18n.translate('xpack.ingestManager.dataStreamList.datasetColumnTitle', {
                 defaultMessage: 'Dataset',
               }),
-              multiSelect: true,
+              multiSelect: 'or',
               options: filterOptions.dataset.map(option => ({
                 value: option,
                 name: option,
@@ -224,7 +245,7 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => {
               name: i18n.translate('xpack.ingestManager.dataStreamList.typeColumnTitle', {
                 defaultMessage: 'Type',
               }),
-              multiSelect: true,
+              multiSelect: 'or',
               options: filterOptions.type.map(option => ({
                 value: option,
                 name: option,
@@ -236,7 +257,7 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => {
               name: i18n.translate('xpack.ingestManager.dataStreamList.namespaceColumnTitle', {
                 defaultMessage: 'Namespace',
               }),
-              multiSelect: true,
+              multiSelect: 'or',
               options: filterOptions.namespace.map(option => ({
                 value: option,
                 name: option,
@@ -248,7 +269,7 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => {
               name: i18n.translate('xpack.ingestManager.dataStreamList.integrationColumnTitle', {
                 defaultMessage: 'Integration',
               }),
-              multiSelect: true,
+              multiSelect: 'or',
               options: filterOptions.package.map(option => ({
                 value: option,
                 name: option,

From d8f9bc26e3371bcd44620b458fda63cc0df7aad7 Mon Sep 17 00:00:00 2001
From: Jen Huang <its.jenetic@gmail.com>
Date: Wed, 22 Apr 2020 09:30:18 -0700
Subject: [PATCH 5/6] Fix types, fix field names

---
 .../ingest_manager/hooks/use_request/data_stream.ts    |  2 +-
 .../server/routes/data_streams/handlers.ts             | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/data_stream.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/data_stream.ts
index 507fc2aa9e9ba..9acf4b1e17449 100644
--- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/data_stream.ts
+++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_request/data_stream.ts
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { useRequest, sendRequest } from './use_request';
+import { useRequest } from './use_request';
 import { dataStreamRouteService } from '../../services';
 import { GetDataStreamsResponse } from '../../types';
 
diff --git a/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts
index 0a19997111310..1df69019a826a 100644
--- a/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts
+++ b/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts
@@ -34,12 +34,12 @@ export const getListHandler: RequestHandler = async (context, request, response)
             must: [
               {
                 exists: {
-                  field: 'fields.stream.namespace',
+                  field: 'stream.namespace',
                 },
               },
               {
                 exists: {
-                  field: 'fields.stream.dataset',
+                  field: 'stream.dataset',
                 },
               },
             ],
@@ -57,19 +57,19 @@ export const getListHandler: RequestHandler = async (context, request, response)
             aggs: {
               dataset: {
                 terms: {
-                  field: 'fields.stream.dataset',
+                  field: 'stream.dataset',
                   size: 1,
                 },
               },
               namespace: {
                 terms: {
-                  field: 'fields.stream.namespace',
+                  field: 'stream.namespace',
                   size: 1,
                 },
               },
               type: {
                 terms: {
-                  field: 'fields.stream.type',
+                  field: 'stream.type',
                   size: 1,
                 },
               },

From 0bbc12fec4977889193b87ab434fa90a6c6640b9 Mon Sep 17 00:00:00 2001
From: Jen Huang <its.jenetic@gmail.com>
Date: Thu, 23 Apr 2020 09:53:59 -0700
Subject: [PATCH 6/6] Change forEach to map

---
 .../ingest_manager/server/routes/data_streams/handlers.ts | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts
index 1df69019a826a..a24518d644c4c 100644
--- a/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts
+++ b/x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts
@@ -90,9 +90,7 @@ export const getListHandler: RequestHandler = async (context, request, response)
       },
     });
 
-    const dataStreams: DataStream[] = [];
-
-    (indexResults as any[]).forEach(result => {
+    const dataStreams: DataStream[] = (indexResults as any[]).map(result => {
       const {
         key: indexName,
         dataset: { buckets: datasetBuckets },
@@ -101,7 +99,7 @@ export const getListHandler: RequestHandler = async (context, request, response)
         package: { buckets: packageBuckets },
         last_activity: { value_as_string: lastActivity },
       } = result;
-      dataStreams.push({
+      return {
         index: indexName,
         dataset: datasetBuckets.length ? datasetBuckets[0].key : '',
         namespace: namespaceBuckets.length ? namespaceBuckets[0].key : '',
@@ -109,7 +107,7 @@ export const getListHandler: RequestHandler = async (context, request, response)
         package: packageBuckets.length ? packageBuckets[0].key : '',
         last_activity: lastActivity,
         size_in_bytes: indexStats[indexName] ? indexStats[indexName].total.store.size_in_bytes : 0,
-      });
+      };
     });
 
     const body: GetDataStreamsResponse = {