[maps] ES|QL source (#173481)

Closes #167648

PR adds "ES|QL" card to "Add Layer" interface. Creates a layer renders
an ES|QL statement on the map

<img width="250" alt="Screenshot 2023-12-16 at 2 03 04 PM"
src="https://github.com/elastic/kibana/assets/373691/4d1e24f6-405b-4016-8e6f-4736742c6166">

<img width="800" alt="Screenshot 2023-12-16 at 1 54 24 PM"
src="https://github.com/elastic/kibana/assets/373691/8387551f-c3b5-4b15-84eb-aef18254d371">

### Known limitations
This PR is intended to be a first start and does not cover all
functionality. The following list identifies known limitations that will
have to be resolved in future work.
1. tooltips - Existing documents source supports lazy loading tooltips
to avoid pulling unused data on map render. How would this look for
ES|QL? Should tooltips only support local data?
2. ES|QL layer does not surface data view to unified search bar so
search type-ahead and filter bar will not show index-pattern fields from
ES|QL layer.
3. ES|QL layer does not surface geoField. This affects control for
drawing filters on map.
4. ES|QL layer does not support pulling field meta from Elasticsearch.
Instead, data-driven styling uses ranges from local data set. This will
be tricky as we can't just pull field ranges from index-pattern. Also
need to account for WHERE clause and other edge cases.
5. fit to bounds

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Nick Peihl <nickpeihl@gmail.com>

package.json

@@ -1111,6 +1111,7 @@
     "vega-spec-injector": "^0.0.2",
     "vega-tooltip": "^0.28.0",
     "vinyl": "^2.2.0",
+    "wellknown": "^0.5.0",
     "whatwg-fetch": "^3.0.0",
     "xml2js": "^0.5.0",
     "xstate": "^4.38.2",

packages/kbn-es-query/index.ts

@@ -55,6 +55,7 @@ export {
   getAggregateQueryMode,
   getIndexPatternFromSQLQuery,
   getIndexPatternFromESQLQuery,
+  getLimitFromESQLQuery,
   getLanguageDisplayName,
   cleanupESQLQueryForLensSuggestions,
 } from './src/es_query';

packages/kbn-es-query/src/es_query/es_aggregate_query.test.ts

@@ -12,6 +12,7 @@ import {
   getAggregateQueryMode,
   getIndexPatternFromSQLQuery,
   getIndexPatternFromESQLQuery,
+  getLimitFromESQLQuery,
   cleanupESQLQueryForLensSuggestions,
 } from './es_aggregate_query';
 
@@ -117,6 +118,33 @@ describe('sql query helpers', () => {
     });
   });
 
+  describe('getLimitFromESQLQuery', () => {
+    it('should return default limit when ES|QL query is empty', () => {
+      const limit = getLimitFromESQLQuery('');
+      expect(limit).toBe(500);
+    });
+
+    it('should return default limit when ES|QL query does not contain LIMIT command', () => {
+      const limit = getLimitFromESQLQuery('FROM foo');
+      expect(limit).toBe(500);
+    });
+
+    it('should return default limit when ES|QL query contains invalid LIMIT command', () => {
+      const limit = getLimitFromESQLQuery('FROM foo | LIMIT iAmNotANumber');
+      expect(limit).toBe(500);
+    });
+
+    it('should return limit when ES|QL query contains LIMIT command', () => {
+      const limit = getLimitFromESQLQuery('FROM foo | LIMIT 10000 | KEEP myField');
+      expect(limit).toBe(10000);
+    });
+
+    it('should return last limit when ES|QL query contains multiple LIMIT command', () => {
+      const limit = getLimitFromESQLQuery('FROM foo | LIMIT 200 | LIMIT 0');
+      expect(limit).toBe(0);
+    });
+  });
+
   describe('cleanupESQLQueryForLensSuggestions', () => {
     it('should not remove anything if a drop command is not present', () => {
       expect(cleanupESQLQueryForLensSuggestions('from a | eval b = 1')).toBe('from a | eval b = 1');

packages/kbn-es-query/src/es_query/es_aggregate_query.ts

@@ -5,10 +5,13 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
+
 import type { Query, AggregateQuery } from '../filters';
 
 type Language = keyof AggregateQuery;
 
+const DEFAULT_ESQL_LIMIT = 500;
+
 // Checks if the query is of type Query
 export function isOfQueryType(arg?: Query | AggregateQuery): arg is Query {
   return Boolean(arg && 'query' in arg);
@@ -67,6 +70,17 @@ export function getIndexPatternFromESQLQuery(esql?: string): string {
   return '';
 }
 
+export function getLimitFromESQLQuery(esql: string): number {
+  const limitCommands = esql.match(new RegExp(/LIMIT\s[0-9]+/, 'ig'));
+  if (!limitCommands) {
+    return DEFAULT_ESQL_LIMIT;
+  }
+
+  const lastIndex = limitCommands.length - 1;
+  const split = limitCommands[lastIndex].split(' ');
+  return parseInt(split[1], 10);
+}
+
 export function cleanupESQLQueryForLensSuggestions(esql?: string): string {
   const pipes = (esql || '').split('|');
   return pipes.filter((statement) => !/DROP\s/i.test(statement)).join('|');

packages/kbn-es-query/src/es_query/index.ts

@@ -20,6 +20,7 @@ export {
   getIndexPatternFromSQLQuery,
   getLanguageDisplayName,
   getIndexPatternFromESQLQuery,
+  getLimitFromESQLQuery,
   cleanupESQLQueryForLensSuggestions,
 } from './es_aggregate_query';
 export { fromCombinedFilter } from './from_combined_filter';

packages/kbn-es-types/index.ts

@@ -19,4 +19,7 @@ export type {
   ESFilter,
   MaybeReadonlyArray,
   ClusterDetails,
+  ESQLColumn,
+  ESQLRow,
+  ESQLSearchReponse,
 } from './src';

packages/kbn-es-types/src/index.ts

@@ -12,6 +12,9 @@ import {
   AggregateOfMap as AggregationResultOfMap,
   SearchHit,
   ClusterDetails,
+  ESQLColumn,
+  ESQLRow,
+  ESQLSearchReponse,
 } from './search';
 
 export type ESFilter = estypes.QueryDslQueryContainer;
@@ -41,4 +44,7 @@ export type {
   AggregationResultOfMap,
   SearchHit,
   ClusterDetails,
+  ESQLColumn,
+  ESQLRow,
+  ESQLSearchReponse,
 };

packages/kbn-es-types/src/search.ts

@@ -653,3 +653,15 @@ export interface ClusterDetails {
   _shards?: estypes.ShardStatistics;
   failures?: estypes.ShardFailure[];
 }
+
+export interface ESQLColumn {
+  name: string;
+  type: string;
+}
+
+export type ESQLRow = unknown[];
+
+export interface ESQLSearchReponse {
+  columns: ESQLColumn[];
+  values: ESQLRow[];
+}

packages/kbn-text-based-editor/src/editor_footer.tsx

@@ -200,6 +200,7 @@ interface EditorFooterProps {
   disableSubmitAction?: boolean;
   editorIsInline?: boolean;
   isSpaceReduced?: boolean;
+  isLoading?: boolean;
 }
 
 export const EditorFooter = memo(function EditorFooter({
@@ -214,6 +215,7 @@ export const EditorFooter = memo(function EditorFooter({
   disableSubmitAction,
   editorIsInline,
   isSpaceReduced,
+  isLoading,
 }: EditorFooterProps) {
   const { euiTheme } = useEuiTheme();
   const [isErrorPopoverOpen, setIsErrorPopoverOpen] = useState(false);
@@ -331,6 +333,7 @@ export const EditorFooter = memo(function EditorFooter({
                   size="s"
                   fill
                   onClick={runQuery}
+                  isLoading={isLoading}
                   isDisabled={Boolean(disableSubmitAction)}
                   data-test-subj="TextBasedLangEditor-run-query-button"
                   minWidth={isSpaceReduced ? false : undefined}

packages/kbn-text-based-editor/src/text_based_languages_editor.tsx

@@ -86,6 +86,8 @@ export interface TextBasedLanguagesEditorProps {
   errors?: Error[];
   /** Warning string as it comes from ES */
   warning?: string;
+  /** Disables the editor and displays loading icon in run button */
+  isLoading?: boolean;
   /** Disables the editor */
   isDisabled?: boolean;
   /** Indicator if the editor is on dark mode */
@@ -149,6 +151,7 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({
   detectTimestamp = false,
   errors: serverErrors,
   warning: serverWarning,
+  isLoading,
   isDisabled,
   isDarkMode,
   hideMinimizeButton,
@@ -540,7 +543,9 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({
     },
     overviewRulerBorder: false,
     readOnly:
-      isDisabled || Boolean(!isCompactFocused && codeOneLiner && codeOneLiner.includes('...')),
+      isLoading ||
+      isDisabled ||
+      Boolean(!isCompactFocused && codeOneLiner && codeOneLiner.includes('...')),
   };
 
   if (isCompactFocused) {
@@ -836,6 +841,7 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({
                         disableSubmitAction={disableSubmitAction}
                         hideRunQueryText={hideRunQueryText}
                         isSpaceReduced={isSpaceReduced}
+                        isLoading={isLoading}
                       />
                     )}
                   </div>
@@ -925,6 +931,7 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({
           editorIsInline={editorIsInline}
           disableSubmitAction={disableSubmitAction}
           isSpaceReduced={isSpaceReduced}
+          isLoading={isLoading}
           {...editorMessages}
         />
       )}

src/plugins/data/common/search/expressions/esql.ts

@@ -20,6 +20,7 @@ import { zipObject } from 'lodash';
 import { Observable, defer, throwError } from 'rxjs';
 import { catchError, map, switchMap, tap } from 'rxjs/operators';
 import { buildEsQuery } from '@kbn/es-query';
+import type { ESQLSearchReponse } from '@kbn/es-types';
 import { getEsQueryConfig } from '../../es_query';
 import { getTime } from '../../query';
 import { ESQL_SEARCH_STRATEGY, IKibanaSearchRequest, ISearchGeneric, KibanaContext } from '..';
@@ -90,14 +91,6 @@ interface ESQLSearchParams {
   locale?: string;
 }
 
-interface ESQLSearchReponse {
-  columns?: Array<{
-    name: string;
-    type: string;
-  }>;
-  values: unknown[][];
-}
-
 export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
   const essql: EsqlExpressionFunctionDefinition = {
     name: 'esql',

src/plugins/data/server/search/strategies/es_search/elasticsearch.ts

@@ -15,9 +15,3 @@ export type IndexAsString<Map> = {
 } & Map;
 
 export type Omit<T, K extends keyof T> = Pick<T, Exclude<keyof T, K>>;
-
-export interface BoolQuery {
-  must_not: Array<Record<string, any>>;
-  should: Array<Record<string, any>>;
-  filter: Array<Record<string, any>>;
-}

src/plugins/data/tsconfig.json

@@ -50,7 +50,8 @@
     "@kbn/search-errors",
     "@kbn/search-response-warnings",
     "@kbn/shared-ux-link-redirect-app",
-    "@kbn/bfetch-error"
+    "@kbn/bfetch-error",
+    "@kbn/es-types"
   ],
   "exclude": [
     "target/**/*",

x-pack/plugins/maps/common/constants.ts

@@ -69,6 +69,7 @@ export enum SOURCE_TYPES {
   ES_SEARCH = 'ES_SEARCH',
   ES_PEW_PEW = 'ES_PEW_PEW',
   ES_ML_ANOMALIES = 'ML_ANOMALIES',
+  ESQL = 'ESQL',
   EMS_XYZ = 'EMS_XYZ', // identifies a custom TMS source. EMS-prefix in the name is a little unfortunate :(
   WMS = 'WMS',
   KIBANA_TILEMAP = 'KIBANA_TILEMAP',
@@ -327,6 +328,7 @@ export enum WIZARD_ID {
   POINT_2_POINT = 'point2Point',
   ES_DOCUMENT = 'esDocument',
   ES_TOP_HITS = 'esTopHits',
+  ESQL = 'ESQL',
   KIBANA_BASEMAP = 'kibanaBasemap',
   MVT_VECTOR = 'mvtVector',
   WMS_LAYER = 'wmsLayer',

x-pack/plugins/maps/common/descriptor_types/source_descriptor_types.ts

@@ -9,6 +9,7 @@
 
 import { FeatureCollection } from 'geojson';
 import type { Query } from '@kbn/es-query';
+import type { ESQLColumn } from '@kbn/es-types';
 import { SortDirection } from '@kbn/data-plugin/common/search';
 import {
   AGG_TYPE,
@@ -37,6 +38,20 @@ export type EMSFileSourceDescriptor = AbstractSourceDescriptor & {
   tooltipProperties: string[];
 };
 
+export type ESQLSourceDescriptor = AbstractSourceDescriptor & {
+  // id: UUID
+  id: string;
+  esql: string;
+  columns: ESQLColumn[];
+  /*
+   * Date field used to narrow ES|QL requests by global time range
+   */
+  dateField?: string;
+  narrowByGlobalSearch: boolean;
+  narrowByMapBounds: boolean;
+  applyForceRefresh: boolean;
+};
+
 export type AbstractESSourceDescriptor = AbstractSourceDescriptor & {
   // id: UUID
   id: string;

x-pack/plugins/maps/common/telemetry/layer_stats_collector.ts

@@ -214,6 +214,10 @@ function getLayerKey(layerDescriptor: LayerDescriptor): LAYER_KEYS | null {
     return LAYER_KEYS.ES_ML_ANOMALIES;
   }
 
+  if (layerDescriptor.sourceDescriptor.type === SOURCE_TYPES.ESQL) {
+    return LAYER_KEYS.ESQL;
+  }
+
   if (layerDescriptor.sourceDescriptor.type === SOURCE_TYPES.ES_SEARCH) {
     const sourceDescriptor = layerDescriptor.sourceDescriptor as ESSearchSourceDescriptor;
 

x-pack/plugins/maps/common/telemetry/types.ts

@@ -27,6 +27,7 @@ export enum LAYER_KEYS {
   ES_AGG_HEXAGONS = 'es_agg_hexagons',
   ES_AGG_HEATMAP = 'es_agg_heatmap',
   ES_ML_ANOMALIES = 'es_ml_anomalies',
+  ESQL = 'esql',
   EMS_REGION = 'ems_region',
   EMS_BASEMAP = 'ems_basemap',
   KBN_TMS_RASTER = 'kbn_tms_raster',

x-pack/plugins/maps/kibana.jsonc

@@ -49,7 +49,8 @@
       "kibanaUtils",
       "usageCollection",
       "unifiedSearch",
-      "fieldFormats"
+      "fieldFormats",
+      "textBasedLanguages"
     ],
     "extraPublicDirs": [
       "common"

x-pack/plugins/maps/public/actions/layer_actions.ts

@@ -74,7 +74,7 @@ import { IVectorStyle } from '../classes/styles/vector/vector_style';
 import { notifyLicensedFeatureUsage } from '../licensed_features';
 import { IESAggField } from '../classes/fields/agg';
 import { IField } from '../classes/fields/field';
-import type { IESSource } from '../classes/sources/es_source';
+import type { IVectorSource } from '../classes/sources/vector_source';
 import { getDrawMode, getOpenTOCDetails } from '../selectors/ui_selectors';
 import { isLayerGroup, LayerGroup } from '../classes/layers/layer_group';
 import { isSpatialJoin } from '../classes/joins/is_spatial_join';
@@ -849,18 +849,23 @@ export function setTileState(
 }
 
 function clearInspectorAdapters(layer: ILayer, adapters: Adapters) {
-  if (isLayerGroup(layer) || !layer.getSource().isESSource()) {
+  if (isLayerGroup(layer)) {
     return;
   }
 
   if (adapters.vectorTiles) {
     adapters.vectorTiles.removeLayer(layer.getId());
   }
 
+  const source = layer.getSource();
+  if ('getInspectorRequestIds' in source) {
+    (source as IVectorSource).getInspectorRequestIds().forEach((id) => {
+      adapters.requests!.resetRequest(id);
+    });
+  }
+
   if (adapters.requests && 'getValidJoins' in layer) {
-    const vectorLayer = layer as IVectorLayer;
-    adapters.requests!.resetRequest((layer.getSource() as IESSource).getId());
-    vectorLayer.getValidJoins().forEach((join) => {
+    (layer as IVectorLayer).getValidJoins().forEach((join) => {
       adapters.requests!.resetRequest(join.getRightJoinSource().getId());
     });
   }

x-pack/plugins/maps/public/classes/layers/build_vector_request_meta.ts

@@ -26,7 +26,7 @@ export function buildVectorRequestMeta(
     applyGlobalQuery: source.getApplyGlobalQuery(),
     applyGlobalTime: source.getApplyGlobalTime(),
     sourceMeta: source.getSyncMeta(dataFilters),
-    applyForceRefresh: source.isESSource() ? source.getApplyForceRefresh() : false,
+    applyForceRefresh: source.getApplyForceRefresh(),
     isForceRefresh,
     isFeatureEditorOpenForLayer,
   };

x-pack/plugins/maps/public/classes/layers/raster_tile_layer/raster_tile_layer.ts

@@ -82,7 +82,7 @@ export class RasterTileLayer extends AbstractLayer {
       ...dataFilters,
       applyGlobalQuery: source.getApplyGlobalQuery(),
       applyGlobalTime: source.getApplyGlobalTime(),
-      applyForceRefresh: source.isESSource() ? source.getApplyForceRefresh() : false,
+      applyForceRefresh: source.getApplyForceRefresh(),
       sourceQuery: this.getQuery() || undefined,
       isForceRefresh,
     };

x-pack/plugins/maps/public/classes/layers/wizards/layer_wizard_registry.ts

@@ -48,6 +48,7 @@ export type LayerWizard = {
 export type RenderWizardArguments = {
   previewLayers: (layerDescriptors: LayerDescriptor[]) => void;
   mapColors: string[];
+  mostCommonDataViewId?: string;
   // multi-step arguments for wizards that supply 'prerequisiteSteps'
   currentStepId: string | null;
   isOnFinalStep: boolean;