[Ingest Manager] Adjust dataset aggs to use datastream fields instead (…

…elastic#74342)

* [Ingest Manager] Adjust dataset aggs to use datastream fields instead

Elastic Agent and Elasticsearch are switching over from using dataset.* to datastream.*. This adjust the aggregation on the dataset page to get the datastreams.

For this to work properly, the most recent version of Elasticsearch 7.9 must be used and is pending updates on all the packages to ship also the datastream fields, see elastic/integrations#213

* Update datastream to data_stream

* Update data stream name generation

* Fix typo

* Temporarily use datastream instead of data_stream

* updating to use `data_stream` instead of `datastream`

Co-authored-by: ruflin <spam@ruflin.com>
Co-authored-by: Jen Huang <its.jenetic@gmail.com>

x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts

@@ -31,12 +31,12 @@ export const getListHandler: RequestHandler = async (context, request, response)
             must: [
               {
                 exists: {
-                  field: 'dataset.namespace',
+                  field: 'data_stream.namespace',
                 },
               },
               {
                 exists: {
-                  field: 'dataset.name',
+                  field: 'data_stream.dataset',
                 },
               },
             ],
@@ -54,19 +54,19 @@ export const getListHandler: RequestHandler = async (context, request, response)
             aggs: {
               dataset: {
                 terms: {
-                  field: 'dataset.name',
+                  field: 'data_stream.dataset',
                   size: 1,
                 },
               },
               namespace: {
                 terms: {
-                  field: 'dataset.namespace',
+                  field: 'data_stream.namespace',
                   size: 1,
                 },
               },
               type: {
                 terms: {
-                  field: 'dataset.type',
+                  field: 'data_stream.type',
                   size: 1,
                 },
               },

x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts

@@ -393,14 +393,14 @@ const updateExistingIndex = async ({
   // are added in https://github.com/elastic/kibana/issues/66551.  namespace value we will continue
   // to skip updating and assume the value in the index mapping is correct
   delete mappings.properties.stream;
-  delete mappings.properties.dataset;
+  delete mappings.properties.data_stream;
 
-  // get the dataset values from the index template to compose data stream name
+  // get the data_stream values from the index template to compose data stream name
   const indexMappings = await getIndexMappings(indexName, callCluster);
-  const dataset = indexMappings[indexName].mappings.properties.dataset.properties;
-  if (!dataset.type.value || !dataset.name.value || !dataset.namespace.value)
-    throw new Error(`dataset values are missing from the index template ${indexName}`);
-  const dataStreamName = `${dataset.type.value}-${dataset.name.value}-${dataset.namespace.value}`;
+  const dataStream = indexMappings[indexName].mappings.properties.data_stream.properties;
+  if (!dataStream.type.value || !dataStream.dataset.value || !dataStream.namespace.value)
+    throw new Error(`data_stream values are missing from the index template ${indexName}`);
+  const dataStreamName = `${dataStream.type.value}-${dataStream.dataset.value}-${dataStream.namespace.value}`;
 
   // try to update the mappings first
   try {