Merge branch 'main' into feat/osquery-manager-prebuilt-packs

docs/maps/maps-aggregations.asciidoc

@@ -42,22 +42,24 @@ image::maps/images/grid_to_docs.gif[]
 
 [role="xpack"]
 [[maps-grid-aggregation]]
-=== Grid aggregation
+=== Clusters
 
-Grid aggregation layers use {ref}/search-aggregations-bucket-geotilegrid-aggregation.html[GeoTile grid aggregation] to group your documents into grids. You can calculate metrics for each gridded cell.
+Clusters use {ref}/search-aggregations-bucket-geotilegrid-aggregation.html[Geotile grid aggregation] or {ref}/search-aggregations-bucket-geohexgrid-aggregation.html[Geohex grid aggregation] to group your documents into grids. You can calculate metrics for each gridded cell.
 
-Symbolize grid aggregation metrics as:
+Symbolize cluster metrics as:
 
-*Clusters*:: Creates a <<vector-layer, vector layer>> with a cluster symbol for each gridded cell.
+*Clusters*:: Uses {ref}/search-aggregations-bucket-geotilegrid-aggregation.html[Geotile grid aggregation] to group your documents into grids. Creates a <<vector-layer, vector layer>> with a cluster symbol for each gridded cell.
 The cluster location is the weighted centroid for all documents in the gridded cell.
 
-*Grid rectangles*:: Creates a <<vector-layer, vector layer>> with a bounding box polygon for each gridded cell.
+*Grids*:: Uses {ref}/search-aggregations-bucket-geotilegrid-aggregation.html[Geotile grid aggregation] to group your documents into grids. Creates a <<vector-layer, vector layer>> with a bounding box polygon for each gridded cell.
 
-*Heat map*:: Creates a <<heatmap-layer, heat map layer>> that clusters the weighted centroids for each gridded cell.
+*Heat map*:: Uses {ref}/search-aggregations-bucket-geotilegrid-aggregation.html[Geotile grid aggregation] to group your documents into grids. Creates a <<heatmap-layer, heat map layer>> that clusters the weighted centroids for each gridded cell.
 
-To enable a grid aggregation layer:
+*Hexbins*:: Uses {ref}/search-aggregations-bucket-geohexgrid-aggregation.html[Geohex grid aggregation] to group your documents into H3 hexagon grids. Creates a <<vector-layer, vector layer>> with a hexagon polygon for each gridded cell.
 
-. Click *Add layer*, then select the *Clusters and grids* or *Heat map* layer.
+To enable a clusters layer:
+
+. Click *Add layer*, then select the *Clusters* or *Heat map* layer.
 
 To enable a blended layer that dynamically shows clusters or documents:
 

docs/maps/maps-getting-started.asciidoc

@@ -128,7 +128,7 @@ traffic. Larger circles will symbolize grids with
 more total bytes transferred, and smaller circles will symbolize
 grids with less bytes transferred.
 
-. Click **Add layer**, and select **Clusters and grids**.
+. Click **Add layer**, and select **Clusters**.
 . Set **Data view** to **kibana_sample_data_logs**.
 . Click **Add layer**.
 . In **Layer settings**, set:

docs/maps/vector-layer.asciidoc

@@ -11,7 +11,7 @@ To add a vector layer to your map, click *Add layer*, then select one of the fol
 
 *Choropleth*:: Shaded areas to compare statistics across boundaries.
 
-*Clusters and grids*:: Geospatial data grouped in grids with metrics for each gridded cell.
+*Clusters*:: Geospatial data grouped in grids with metrics for each gridded cell.
 The index must contain at least one field mapped as {ref}/geo-point.html[geo_point] or {ref}/geo-shape.html[geo_shape].
 
 *Create index*:: Draw shapes on the map and index in Elasticsearch.

docs/settings/task-manager-settings.asciidoc

@@ -40,6 +40,11 @@ These non-persisted action tasks have a risk that they won't be run at all if th
 `xpack.task_manager.ephemeral_tasks.request_capacity`::
 Sets the size of the ephemeral queue defined above. Defaults to 10.
 
+`xpack.task_manager.event_loop_delay.monitor`::
+Enables event loop delay monitoring, which will log a warning when a task causes an event loop delay which exceeds the `warn_threshold` setting.  Defaults to true.
+
+`xpack.task_manager.event_loop_delay.warn_threshold`::
+Sets the amount of event loop delay during a task execution which will cause a warning to be logged. Defaults to 5000 milliseconds (5 seconds).
 
 [float]
 [[task-manager-health-settings]]

package.json

@@ -85,7 +85,7 @@
     "**/istanbul-lib-coverage": "^3.2.0",
     "**/json-schema": "^0.4.0",
     "**/minimatch": "^3.1.2",
-    "**/minimist": "^1.2.5",
+    "**/minimist": "^1.2.6",
     "**/node-forge": "^1.3.0",
     "**/pdfkit/crypto-js": "4.0.0",
     "**/react-syntax-highlighter": "^15.3.1",
@@ -648,7 +648,7 @@
     "@types/mime": "^2.0.1",
     "@types/mime-types": "^2.1.0",
     "@types/minimatch": "^2.0.29",
-    "@types/minimist": "^1.2.1",
+    "@types/minimist": "^1.2.2",
     "@types/mocha": "^9.1.0",
     "@types/mock-fs": "^4.13.1",
     "@types/moment-timezone": "^0.5.12",
@@ -841,7 +841,7 @@
     "lmdb-store": "^1.6.11",
     "marge": "^1.0.1",
     "micromatch": "3.1.10",
-    "minimist": "^1.2.5",
+    "minimist": "^1.2.6",
     "mkdirp": "0.5.1",
     "mocha": "^9.1.0",
     "mocha-junit-reporter": "^2.0.2",

packages/kbn-pm/dist/index.js

@@ -16447,7 +16447,7 @@ module.exports = function (args, opts) {
         var o = obj;
         for (var i = 0; i < keys.length-1; i++) {
             var key = keys[i];
-            if (key === '__proto__') return;
+            if (isConstructorOrProto(o, key)) return;
             if (o[key] === undefined) o[key] = {};
             if (o[key] === Object.prototype || o[key] === Number.prototype
                 || o[key] === String.prototype) o[key] = {};
@@ -16456,7 +16456,7 @@ module.exports = function (args, opts) {
         }
 
         var key = keys[keys.length - 1];
-        if (key === '__proto__') return;
+        if (isConstructorOrProto(o, key)) return;
         if (o === Object.prototype || o === Number.prototype
             || o === String.prototype) o = {};
         if (o === Array.prototype) o = [];
@@ -16621,6 +16621,10 @@ function isNumber (x) {
 }
 
 
+function isConstructorOrProto (obj, key) {
+    return key === 'constructor' && typeof obj[key] === 'function' || key === '__proto__';
+}
+
 
 /***/ }),
 /* 229 */

src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker

@@ -379,6 +379,8 @@ kibana_vars=(
     xpack.task_manager.poll_interval
     xpack.task_manager.request_capacity
     xpack.task_manager.version_conflict_threshold
+    xpack.task_manager.event_loop_delay.monitor
+    xpack.task_manager.event_loop_delay.warn_threshold
     xpack.uptime.index
 )
 

src/plugins/data/common/datatable_utilities/datatable_utilities_service.test.ts

@@ -8,7 +8,7 @@
 
 import { createStubDataView } from 'src/plugins/data_views/common/mocks';
 import type { DataViewsContract } from 'src/plugins/data_views/common';
-import type { DatatableColumn } from 'src/plugins/expressions/common';
+import type { Datatable, DatatableColumn } from 'src/plugins/expressions/common';
 import { FieldFormat } from 'src/plugins/field_formats/common';
 import { fieldFormatsMock } from 'src/plugins/field_formats/common/mocks';
 import type { AggsCommonStart } from '../search';
@@ -106,6 +106,16 @@ describe('DatatableUtilitiesService', () => {
     });
   });
 
+  describe('getTotalCount', () => {
+    it('should return a total hits count', () => {
+      const table = {
+        meta: { statistics: { totalCount: 100 } },
+      } as unknown as Datatable;
+
+      expect(datatableUtilitiesService.getTotalCount(table)).toBe(100);
+    });
+  });
+
   describe('setFieldFormat', () => {
     it('should set new field format', () => {
       const column = { meta: {} } as DatatableColumn;

src/plugins/data/common/datatable_utilities/datatable_utilities_service.ts

@@ -7,7 +7,7 @@
  */
 
 import type { DataView, DataViewsContract, DataViewField } from 'src/plugins/data_views/common';
-import type { DatatableColumn } from 'src/plugins/expressions/common';
+import type { Datatable, DatatableColumn } from 'src/plugins/expressions/common';
 import type { FieldFormatsStartCommon, FieldFormat } from 'src/plugins/field_formats/common';
 import type { AggsCommonStart, AggConfig, CreateAggConfigParams, IAggType } from '../search';
 
@@ -77,6 +77,10 @@ export class DatatableUtilitiesService {
     return params?.interval;
   }
 
+  getTotalCount(table: Datatable): number | undefined {
+    return table.meta?.statistics?.totalCount;
+  }
+
   isFilterable(column: DatatableColumn): boolean {
     if (column.meta.source !== 'esaggs') {
       return false;

src/plugins/data/common/search/expressions/eql_raw_response.test.ts

@@ -43,6 +43,22 @@ describe('eqlRawResponse', () => {
       const result = eqlRawResponse.to!.datatable(response, {});
       expect(result).toMatchSnapshot();
     });
+
+    test('extracts total hits number', () => {
+      const response: EqlRawResponse = {
+        type: 'eql_raw_response',
+        body: {
+          hits: {
+            events: [],
+            total: {
+              value: 2,
+            },
+          },
+        },
+      };
+      const result = eqlRawResponse.to!.datatable(response, {});
+      expect(result).toHaveProperty('meta.statistics.totalCount', 2);
+    });
   });
 
   describe('converts sequences to table', () => {

src/plugins/data/common/search/expressions/eql_raw_response.ts

@@ -125,6 +125,9 @@ export const eqlRawResponse: EqlRawResponseExpressionTypeDefinition = {
         meta: {
           type: 'eql',
           source: '*',
+          statistics: {
+            totalCount: (context.body as EqlSearchResponse<unknown>).hits.total?.value,
+          },
         },
         columns,
         rows,

src/plugins/data/common/search/expressions/es_raw_response.ts

@@ -82,6 +82,12 @@ export const esRawResponse: EsRawResponseExpressionTypeDefinition = {
         meta: {
           type: 'esdsl',
           source: '*',
+          statistics: {
+            totalCount:
+              typeof context.body.hits.total === 'number'
+                ? context.body.hits.total
+                : context.body.hits.total?.value,
+          },
         },
         columns,
         rows,

src/plugins/data/common/search/tabify/tabify.test.ts

@@ -52,6 +52,10 @@ describe('tabifyAggResponse Integration', () => {
 
     expect(resp.rows[0]).toEqual({ 'col-0-1': 1000 });
     expect(resp.columns[0]).toHaveProperty('name', aggConfigs.aggs[0].makeLabel());
+
+    expect(resp).toHaveProperty('meta.type', 'esaggs');
+    expect(resp).toHaveProperty('meta.source', '1234');
+    expect(resp).toHaveProperty('meta.statistics.totalCount', 1000);
   });
 
   describe('scaleMetricValues performance check', () => {

src/plugins/data/common/search/tabify/tabify.ts

@@ -7,6 +7,7 @@
  */
 
 import { get } from 'lodash';
+import type { Datatable } from 'src/plugins/expressions';
 import { TabbedAggResponseWriter } from './response_writer';
 import { TabifyBuckets } from './buckets';
 import type { TabbedResponseWriterOptions } from './types';
@@ -20,7 +21,7 @@ export function tabifyAggResponse(
   aggConfigs: IAggConfigs,
   esResponse: Record<string, any>,
   respOpts?: Partial<TabbedResponseWriterOptions>
-) {
+): Datatable {
   /**
    * read an aggregation from a bucket, which *might* be found at key (if
    * the response came in object form), and will recurse down the aggregation
@@ -152,5 +153,14 @@ export function tabifyAggResponse(
 
   collectBucket(aggConfigs, write, topLevelBucket, '', 1);
 
-  return write.response();
+  return {
+    ...write.response(),
+    meta: {
+      type: 'esaggs',
+      source: aggConfigs.indexPattern.id,
+      statistics: {
+        totalCount: esResponse.hits?.total,
+      },
+    },
+  };
 }

src/plugins/expressions/common/expression_functions/specs/map_column.ts

@@ -95,7 +95,7 @@ export const mapColumn: ExpressionFunctionDefinition<
         input.rows.map((row) =>
           args
             .expression({
-              type: 'datatable',
+              ...input,
               columns: [...input.columns],
               rows: [row],
             })
@@ -129,9 +129,9 @@ export const mapColumn: ExpressionFunctionDefinition<
         };
 
         return {
+          ...input,
           columns,
           rows,
-          type: 'datatable',
         };
       })
     );

src/plugins/expressions/common/expression_functions/specs/math_column.ts

@@ -79,7 +79,7 @@ export const mathColumn: ExpressionFunctionDefinition<
       input.rows.map(async (row) => {
         const result = await math.fn(
           {
-            type: 'datatable',
+            ...input,
             columns: input.columns,
             rows: [row],
           },
@@ -128,7 +128,7 @@ export const mathColumn: ExpressionFunctionDefinition<
     columns.push(newColumn);
 
     return {
-      type: 'datatable',
+      ...input,
       columns,
       rows: newRows,
     } as Datatable;

src/plugins/expressions/common/expression_types/specs/datatable.ts

@@ -91,12 +91,45 @@ export interface DatatableColumn {
   meta: DatatableColumnMeta;
 }
 
+/**
+ * Metadata with statistics about the `Datatable` source.
+ */
+export interface DatatableMetaStatistics {
+  /**
+   * Total hits number returned for the request generated the `Datatable`.
+   */
+  totalCount?: number;
+}
+
+/**
+ * The `Datatable` meta information.
+ */
+export interface DatatableMeta {
+  /**
+   * Statistics about the `Datatable` source.
+   */
+  statistics?: DatatableMetaStatistics;
+
+  /**
+   * The `Datatable` type (e.g. `essql`, `eql`, `esdsl`, etc.).
+   */
+  type?: string;
+
+  /**
+   * The `Datatable` data source.
+   */
+  source?: string;
+
+  [key: string]: unknown;
+}
+
 /**
  * A `Datatable` in Canvas is a unique structure that represents tabulated data.
  */
 export interface Datatable {
   type: typeof name;
   columns: DatatableColumn[];
+  meta?: DatatableMeta;
   rows: DatatableRow[];
 }