Merge remote-tracking branch 'upstream/main' into embeddableRebuild/a…

…ddNewRegistry

.github/CODEOWNERS

@@ -383,6 +383,7 @@ packages/kbn-eslint-plugin-imports @elastic/kibana-operations
 packages/kbn-eslint-plugin-telemetry @elastic/obs-knowledge-team
 examples/eso_model_version_example @elastic/kibana-security
 x-pack/test/encrypted_saved_objects_api_integration/plugins/api_consumer_plugin @elastic/kibana-security
+packages/kbn-esql-utils @elastic/kibana-visualizations
 packages/kbn-event-annotation-common @elastic/kibana-visualizations
 packages/kbn-event-annotation-components @elastic/kibana-visualizations
 src/plugins/event_annotation_listing @elastic/kibana-visualizations
@@ -529,6 +530,7 @@ x-pack/packages/maps/vector_tile_utils @elastic/kibana-gis
 x-pack/plugins/metrics_data_access @elastic/obs-knowledge-team
 x-pack/packages/ml/agg_utils @elastic/ml-ui
 x-pack/packages/ml/anomaly_utils @elastic/ml-ui
+x-pack/packages/ml/cancellable_search @elastic/ml-ui
 x-pack/packages/ml/category_validator @elastic/ml-ui
 x-pack/packages/ml/chi2test @elastic/ml-ui
 x-pack/packages/ml/creation_wizard_utils @elastic/ml-ui

package.json

@@ -422,6 +422,7 @@
     "@kbn/es-ui-shared-plugin": "link:src/plugins/es_ui_shared",
     "@kbn/eso-model-version-example": "link:examples/eso_model_version_example",
     "@kbn/eso-plugin": "link:x-pack/test/encrypted_saved_objects_api_integration/plugins/api_consumer_plugin",
+    "@kbn/esql-utils": "link:packages/kbn-esql-utils",
     "@kbn/event-annotation-common": "link:packages/kbn-event-annotation-common",
     "@kbn/event-annotation-components": "link:packages/kbn-event-annotation-components",
     "@kbn/event-annotation-listing-plugin": "link:src/plugins/event_annotation_listing",
@@ -549,6 +550,7 @@
     "@kbn/metrics-data-access-plugin": "link:x-pack/plugins/metrics_data_access",
     "@kbn/ml-agg-utils": "link:x-pack/packages/ml/agg_utils",
     "@kbn/ml-anomaly-utils": "link:x-pack/packages/ml/anomaly_utils",
+    "@kbn/ml-cancellable-search": "link:x-pack/packages/ml/cancellable_search",
     "@kbn/ml-category-validator": "link:x-pack/packages/ml/category_validator",
     "@kbn/ml-chi2test": "link:x-pack/packages/ml/chi2test",
     "@kbn/ml-creation-wizard-utils": "link:x-pack/packages/ml/creation_wizard_utils",

packages/deeplinks/ml/deep_links.ts

@@ -27,6 +27,7 @@ export type LinkId =
   | 'nodesOverview'
   | 'nodes'
   | 'memoryUsage'
+  | 'esqlDataVisualizer'
   | 'dataVisualizer'
   | 'fileUpload'
   | 'indexDataVisualizer'

packages/default-nav/ml/default_navigation.ts

@@ -121,6 +121,15 @@ export const defaultNavigation: MlNodeDefinition = {
             );
           },
         },
+        {
+          title: i18n.translate('defaultNavigation.ml.esqlDataVisualizer', {
+            defaultMessage: 'ES|QL',
+          }),
+          link: 'ml:esqlDataVisualizer',
+          getIsActive: ({ pathNameSerialized, prepend }) => {
+            return pathNameSerialized.includes(prepend('/app/ml/datavisualizer/esql'));
+          },
+        },
         {
           title: i18n.translate('defaultNavigation.ml.dataComparison', {
             defaultMessage: 'Data drift',

packages/kbn-alerts-as-data-utils/src/schemas/generated/security_schema.ts

@@ -122,6 +122,7 @@ const SecurityAlertOptional = rt.partial({
   'ecs.version': schemaString,
   'event.action': schemaString,
   'event.kind': schemaString,
+  'host.asset.criticality': schemaString,
   'kibana.alert.action_group': schemaString,
   'kibana.alert.ancestors.rule': schemaString,
   'kibana.alert.building_block_type': schemaString,
@@ -204,6 +205,7 @@ const SecurityAlertOptional = rt.partial({
   'kibana.alert.workflow_user': schemaString,
   'kibana.version': schemaString,
   tags: schemaStringArray,
+  'user.asset.criticality': schemaString,
 });
 
 // prettier-ignore

packages/kbn-es-query/tsconfig.json

@@ -15,7 +15,7 @@
   "kbn_references": [
     "@kbn/utility-types",
     "@kbn/i18n",
-    "@kbn/safer-lodash-set",
+    "@kbn/safer-lodash-set"
   ],
   "exclude": [
     "target/**/*",

packages/kbn-esql-utils/README.md

@@ -0,0 +1,4 @@
+# @kbn/esql-utils
+
+This package contains utilities for ES|QL.
+

packages/kbn-esql-utils/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { getESQLAdHocDataview } from './src';

packages/kbn-esql-utils/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-esql-utils'],
+};

packages/kbn-esql-utils/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/esql-utils",
+  "owner": "@elastic/kibana-visualizations"
+}

packages/kbn-esql-utils/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "@kbn/esql-utils",
+  "private": true,
+  "version": "1.0.0",
+  "license": "SSPL-1.0 OR Elastic License 2.0",
+  "sideEffects": false
+}

packages/kbn-esql-utils/src/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export * from './utils';

packages/kbn-esql-utils/src/utils/index.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public';
+
+// uses browser sha256 method with fallback if unavailable
+async function sha256(str: string) {
+  if (crypto.subtle) {
+    const enc = new TextEncoder();
+    const hash = await crypto.subtle.digest('SHA-256', enc.encode(str));
+    return Array.from(new Uint8Array(hash))
+      .map((v) => v.toString(16).padStart(2, '0'))
+      .join('');
+  } else {
+    const { sha256: sha256fn } = await import('./sha256');
+    return sha256fn(str);
+  }
+}
+
+// Some applications need to have a dataview to work properly with ES|QL queries
+// This is a helper to create one. The id is constructed from the indexpattern.
+// As there are no runtime fields or field formatters or default time fields
+// the same adhoc dataview can be constructed/used. This comes with great advantages such
+// as solving the problem descibed here https://github.com/elastic/kibana/issues/168131
+export async function getESQLAdHocDataview(
+  indexPattern: string,
+  dataViewsService: DataViewsPublicPluginStart
+) {
+  return await dataViewsService.create({
+    title: indexPattern,
+    id: await sha256(`esql-${indexPattern}`),
+  });
+}

packages/kbn-esql-utils/src/utils/sha256.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Sha256 } from '@kbn/crypto-browser';
+
+export const sha256 = async (str: string) => new Sha256().update(str).digest('hex');

packages/kbn-esql-utils/tsconfig.json

@@ -0,0 +1,22 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node",
+      "react"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+    "**/*.tsx",
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/data-views-plugin",
+    "@kbn/crypto-browser",
+  ]
+}

packages/kbn-monaco/BUILD.bazel

@@ -29,6 +29,7 @@ SHARED_DEPS = [
   "@npm//antlr4ts",
   "@npm//monaco-editor",
   "@npm//monaco-yaml",
+  "@npm//js-levenshtein",
 ]
 
 webpack_cli(

packages/kbn-monaco/src/esql/language.ts

@@ -111,4 +111,25 @@ export const ESQLLang: CustomLangModuleType<ESQLCallbacks> = {
       },
     };
   },
+
+  getCodeActionProvider: (callbacks?: ESQLCallbacks): monaco.languages.CodeActionProvider => {
+    return {
+      async provideCodeActions(
+        model /** ITextModel*/,
+        range /** Range*/,
+        context /** CodeActionContext*/,
+        token /** CancellationToken*/
+      ) {
+        const astAdapter = new ESQLAstAdapter(
+          (...uris) => workerProxyService.getWorker(uris),
+          callbacks
+        );
+        const actions = await astAdapter.codeAction(model, range, context);
+        return {
+          actions,
+          dispose: () => {},
+        };
+      },
+    };
+  },
 };

packages/kbn-monaco/src/esql/lib/ast/autocomplete/autocomplete.test.ts

@@ -121,8 +121,8 @@ function getFunctionSignaturesByReturnType(
       }
       return true;
     })
-    .map(({ builtin: isBuiltinFn, name, signatures, ...defRest }) =>
-      isBuiltinFn ? `${name} $0` : `${name}($0)`
+    .map(({ type, name, signatures, ...defRest }) =>
+      type === 'builtin' ? `${name} $0` : `${name}($0)`
     );
 }
 

packages/kbn-monaco/src/esql/lib/ast/autocomplete/autocomplete.ts

@@ -37,7 +37,6 @@ import {
 import { collectVariables, excludeVariablesFromCurrentCommand } from '../shared/variables';
 import type {
   AstProviderFn,
-  ESQLAst,
   ESQLAstItem,
   ESQLCommand,
   ESQLCommandMode,
@@ -72,6 +71,7 @@ import {
 import { EDITOR_MARKER } from '../shared/constants';
 import { getAstContext, removeMarkerArgFromArgsList } from '../shared/context';
 import {
+  buildQueryUntilPreviousCommand,
   getFieldsByTypeHelper,
   getPolicyHelper,
   getSourcesHelper,
@@ -187,7 +187,7 @@ export async function suggest(
 
   const astContext = getAstContext(innerText, ast, offset);
   // build the correct query to fetch the list of fields
-  const queryForFields = buildQueryForFields(ast, finalText);
+  const queryForFields = buildQueryUntilPreviousCommand(ast, finalText);
   const { getFieldsByType, getFieldsMap } = getFieldsByTypeRetriever(
     queryForFields,
     resourceRetriever
@@ -260,11 +260,6 @@ export async function suggest(
   return [];
 }
 
-export function buildQueryForFields(ast: ESQLAst, queryString: string) {
-  const prevCommand = ast[Math.max(ast.length - 2, 0)];
-  return prevCommand ? queryString.substring(0, prevCommand.location.max + 1) : queryString;
-}
-
 function getFieldsByTypeRetriever(queryString: string, resourceRetriever?: ESQLCallbacks) {
   const helpers = getFieldsByTypeHelper(queryString, resourceRetriever);
   return {
@@ -812,7 +807,7 @@ async function getBuiltinFunctionNextArgument(
           // technically another boolean value should be suggested, but it is a better experience
           // to actually suggest a wider set of fields/functions
           [
-            finalType === 'boolean' && getFunctionDefinition(nodeArg.name)?.builtin
+            finalType === 'boolean' && getFunctionDefinition(nodeArg.name)?.type === 'builtin'
               ? 'any'
               : finalType,
           ],
@@ -1013,7 +1008,7 @@ async function getFunctionArgsSuggestions(
         ? {
             ...suggestion,
             insertText:
-              hasMoreMandatoryArgs && !fnDefinition.builtin
+              hasMoreMandatoryArgs && fnDefinition.type !== 'builtin'
                 ? `${suggestion.insertText},`
                 : suggestion.insertText,
           }
@@ -1023,7 +1018,8 @@ async function getFunctionArgsSuggestions(
 
   return suggestions.map(({ insertText, ...rest }) => ({
     ...rest,
-    insertText: hasMoreMandatoryArgs && !fnDefinition.builtin ? `${insertText},` : insertText,
+    insertText:
+      hasMoreMandatoryArgs && fnDefinition.type !== 'builtin' ? `${insertText},` : insertText,
   }));
 }
 

packages/kbn-monaco/src/esql/lib/ast/autocomplete/factories.ts

@@ -18,7 +18,7 @@ import {
   CommandOptionsDefinition,
   CommandModeDefinition,
 } from '../definitions/types';
-import { getCommandDefinition } from '../shared/helpers';
+import { getCommandDefinition, shouldBeQuotedText } from '../shared/helpers';
 import { buildDocumentation, buildFunctionDocumentation } from './documentation_util';
 
 const allFunctions = statsAggregationFunctionDefinitions.concat(evalFunctionsDefinitions);
@@ -28,11 +28,8 @@ export const TRIGGER_SUGGESTION_COMMAND = {
   id: 'editor.action.triggerSuggest',
 };
 
-function getSafeInsertText(text: string, { dashSupported }: { dashSupported?: boolean } = {}) {
-  if (dashSupported) {
-    return /[^a-zA-Z\d_\.@-]/.test(text) ? `\`${text}\`` : text;
-  }
-  return /[^a-zA-Z\d_\.@]/.test(text) ? `\`${text}\`` : text;
+function getSafeInsertText(text: string, options: { dashSupported?: boolean } = {}) {
+  return shouldBeQuotedText(text, options) ? `\`${text}\`` : text;
 }
 
 export function getAutocompleteFunctionDefinition(fn: FunctionDefinition) {