From 1743e9e96b61398262b3a43a98cf7701e9121cb5 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 7 Jul 2020 17:26:23 +0200 Subject: [PATCH 01/15] [ML] Introduce ML API server side histograms endpoint. --- .../services/ml_api_service/index.ts | 29 ++- .../models/data_visualizer/data_visualizer.ts | 233 ++++++++++++++++++ .../ml/server/routes/data_visualizer.ts | 59 +++++ .../routes/schemas/data_visualizer_schema.ts | 8 + 4 files changed, 328 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts index d1b6f95f32bed..a073656e16978 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts @@ -27,7 +27,10 @@ import { ModelSnapshot, } from '../../../../common/types/anomaly_detection_jobs'; import { ES_AGGREGATION } from '../../../../common/constants/aggregation_types'; -import { FieldRequestConfig } from '../../datavisualizer/index_based/common'; +import { + FieldHistogramRequestConfig, + FieldRequestConfig, +} from '../../datavisualizer/index_based/common'; import { DataRecognizerConfigResponse, Module } from '../../../../common/types/modules'; import { getHttp } from '../../util/dependency_cache'; @@ -494,6 +497,30 @@ export function mlApiServicesProvider(httpService: HttpService) { }); }, + getVisualizerFieldHistograms({ + indexPatternTitle, + query, + samplerShardSize, + fields, + }: { + indexPatternTitle: string; + query: any; + samplerShardSize?: number; + fields?: FieldHistogramRequestConfig[]; + }) { + const body = JSON.stringify({ + query, + samplerShardSize, + fields, + }); + + return httpService.http({ + path: `${basePath()}/data_visualizer/get_field_histograms/${indexPatternTitle}`, + method: 'POST', + body, + }); + }, + getVisualizerOverallStats({ indexPatternTitle, query, diff --git a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts index d58c797b446db..737c2f9b85605 100644 --- a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts +++ b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts @@ -8,6 +8,7 @@ import { LegacyCallAPIOptions, LegacyAPICaller } from 'kibana/server'; import _ from 'lodash'; import { ML_JOB_FIELD_TYPES } from '../../../common/constants/field_types'; import { getSafeAggregationName } from '../../../common/util/job_utils'; +import { stringHash } from '../../../common/util/string_utils'; import { buildBaseFilterCriteria, buildSamplerAggregation, @@ -19,6 +20,8 @@ const SAMPLER_TOP_TERMS_SHARD_SIZE = 5000; const AGGREGATABLE_EXISTS_REQUEST_BATCH_SIZE = 200; const FIELDS_REQUEST_BATCH_SIZE = 10; +const MAX_CHART_COLUMNS = 20; + interface FieldData { fieldName: string; existsInDocs: boolean; @@ -35,6 +38,11 @@ export interface Field { cardinality: number; } +interface HistogramField { + fieldName: string; + type: string; +} + interface Distribution { percentiles: any[]; minPercentile: number; @@ -98,6 +106,70 @@ interface FieldExamples { examples: any[]; } +interface NumericColumnStats { + interval: number; + min: number; + max: number; +} +type NumericColumnStatsMap = Record; + +interface AggHistogram { + histogram: { + field: string; + interval: number; + }; +} + +interface AggCardinality { + cardinality: { + field: string; + }; +} + +interface AggTerms { + terms: { + field: string; + size: number; + }; +} + +interface NumericDataItem { + key: number; + key_as_string?: string; + doc_count: number; +} + +interface NumericChartData { + data: NumericDataItem[]; + id: string; + interval: number; + stats: [number, number]; + type: 'numeric'; +} + +interface OrdinalDataItem { + key: string; + key_as_string?: string; + doc_count: number; +} + +interface OrdinalChartData { + type: 'ordinal' | 'boolean'; + cardinality: number; + data: OrdinalDataItem[]; + id: string; +} + +interface UnsupportedChartData { + id: string; + type: 'unsupported'; +} + +type ChartRequestAgg = AggHistogram | AggCardinality | AggTerms; + +// type ChartDataItem = NumericDataItem | OrdinalDataItem; +type ChartData = NumericChartData | OrdinalChartData | UnsupportedChartData; + type BatchStats = | NumericFieldStats | StringFieldStats @@ -200,6 +272,167 @@ export class DataVisualizer { return stats; } + async getAggIntervals( + indexPatternTitle: string, + query: any, + fields: HistogramField[], + samplerShardSize: number + ): Promise { + const numericColumns = fields.filter((field) => { + return field.type === 'number' || field.type === 'number'; + }); + + if (numericColumns.length === 0) { + return {}; + } + + const minMaxAggs = numericColumns.reduce((aggs, c) => { + const id = stringHash(c.fieldName); + aggs[id] = { + stats: { + field: c.fieldName, + }, + }; + return aggs; + }, {} as Record); + + const respStats = await this.callAsCurrentUser('search', { + index: indexPatternTitle, + size: 0, + body: { + query, + aggs: minMaxAggs, + size: 0, + }, + }); + + return Object.keys(respStats.aggregations).reduce((p, aggName) => { + const stats = [respStats.aggregations[aggName].min, respStats.aggregations[aggName].max]; + if (!stats.includes(null)) { + const delta = respStats.aggregations[aggName].max - respStats.aggregations[aggName].min; + + let aggInterval = 1; + + if (delta > MAX_CHART_COLUMNS) { + aggInterval = Math.round(delta / MAX_CHART_COLUMNS); + } + + if (delta <= 1) { + aggInterval = delta / MAX_CHART_COLUMNS; + } + + p[aggName] = { interval: aggInterval, min: stats[0], max: stats[1] }; + } + + return p; + }, {} as NumericColumnStatsMap); + } + + // Obtains binned histograms for supplied list of fields. The statistics for each field in the + // returned array depend on the type of the field (keyword, number, date etc). + // Sampling will be used if supplied samplerShardSize > 0. + async getHistogramsForFields( + indexPatternTitle: string, + query: any, + fields: HistogramField[], + samplerShardSize: number + ): Promise { + const aggIntervals = await this.getAggIntervals( + indexPatternTitle, + query, + fields, + samplerShardSize + ); + + const chartDataAggs = fields.reduce((aggs, field) => { + const fieldName = field.fieldName; + const fieldType = field.type; + const id = stringHash(fieldName); + if (fieldType === 'number' || fieldType === 'date') { + if (aggIntervals[id] !== undefined) { + aggs[`${id}_histogram`] = { + histogram: { + field: fieldName, + interval: aggIntervals[id].interval !== 0 ? aggIntervals[id].interval : 1, + }, + }; + } + } else if (fieldType === 'string' || fieldType === 'boolean') { + if (fieldType === 'string') { + aggs[`${id}_cardinality`] = { + cardinality: { + field: fieldName, + }, + }; + } + aggs[`${id}_terms`] = { + terms: { + field: fieldName, + size: MAX_CHART_COLUMNS, + }, + }; + } + return aggs; + }, {} as Record); + + if (Object.keys(chartDataAggs).length === 0) { + return []; + } + + const respChartsData = await this.callAsCurrentUser('search', { + index: indexPatternTitle, + size: 0, + body: { + query, + aggs: chartDataAggs, + size: 0, + }, + }); + + const chartsData: ChartData[] = fields.map( + (field): ChartData => { + const fieldName = field.fieldName; + const fieldType = field.type; + const id = stringHash(field.fieldName); + + if (fieldType === 'number' || fieldType === 'date') { + if (aggIntervals[id] === undefined) { + return { + type: 'numeric', + data: [], + interval: 0, + stats: [0, 0], + id: fieldName, + }; + } + + return { + data: respChartsData.aggregations[`${id}_histogram`].buckets, + interval: aggIntervals[id].interval, + stats: [aggIntervals[id].min, aggIntervals[id].max], + type: 'numeric', + id: fieldName, + }; + } else if (fieldType === 'string' || fieldType === 'boolean') { + return { + type: fieldType === 'string' ? 'ordinal' : 'boolean', + cardinality: + fieldType === 'string' ? respChartsData.aggregations[`${id}_cardinality`].value : 2, + data: respChartsData.aggregations[`${id}_terms`].buckets, + id: fieldName, + }; + } + + return { + type: 'unsupported', + id: fieldName, + }; + } + ); + + return chartsData; + } + // Obtains statistics for supplied list of fields. The statistics for each field in the // returned array depend on the type of the field (keyword, number, date etc). // Sampling will be used if supplied samplerShardSize > 0. diff --git a/x-pack/plugins/ml/server/routes/data_visualizer.ts b/x-pack/plugins/ml/server/routes/data_visualizer.ts index 04008a896a1a2..c3c555253c995 100644 --- a/x-pack/plugins/ml/server/routes/data_visualizer.ts +++ b/x-pack/plugins/ml/server/routes/data_visualizer.ts @@ -9,6 +9,7 @@ import { wrapError } from '../client/error_wrapper'; import { DataVisualizer } from '../models/data_visualizer'; import { Field } from '../models/data_visualizer/data_visualizer'; import { + dataVisualizerFieldHistogramsSchema, dataVisualizerFieldStatsSchema, dataVisualizerOverallStatsSchema, indexPatternTitleSchema, @@ -65,10 +66,68 @@ function getStatsForFields( ); } +function getHistogramsForFields( + context: RequestHandlerContext, + indexPatternTitle: string, + query: any, + fields: Field[], + samplerShardSize: number +) { + const dv = new DataVisualizer(context.ml!.mlClient.callAsCurrentUser); + return dv.getHistogramsForFields(indexPatternTitle, query, fields, samplerShardSize); +} + /** * Routes for the index data visualizer. */ export function dataVisualizerRoutes({ router, mlLicense }: RouteInitialization) { + /** + * @apiGroup DataVisualizer + * + * @api {post} /api/ml/data_visualizer/get_field_stats/:indexPatternTitle Get stats for fields + * @apiName GetStatsForFields + * @apiDescription Returns the stats on individual fields in the specified index pattern. + * + * @apiSchema (params) indexPatternTitleSchema + * @apiSchema (body) dataVisualizerFieldHistogramsSchema + * + * @apiSuccess {Object} fieldName stats by field, keyed on the name of the field. + */ + router.post( + { + path: '/api/ml/data_visualizer/get_field_histograms/{indexPatternTitle}', + validate: { + params: indexPatternTitleSchema, + body: dataVisualizerFieldHistogramsSchema, + }, + options: { + tags: ['access:ml:canAccessML'], + }, + }, + mlLicense.basicLicenseAPIGuard(async (context, request, response) => { + try { + const { + params: { indexPatternTitle }, + body: { query, fields, samplerShardSize }, + } = request; + + const results = await getHistogramsForFields( + context, + indexPatternTitle, + query, + fields, + samplerShardSize + ); + + return response.ok({ + body: results, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); + /** * @apiGroup DataVisualizer * diff --git a/x-pack/plugins/ml/server/routes/schemas/data_visualizer_schema.ts b/x-pack/plugins/ml/server/routes/schemas/data_visualizer_schema.ts index b2d665954bd4d..14bc711886315 100644 --- a/x-pack/plugins/ml/server/routes/schemas/data_visualizer_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/data_visualizer_schema.ts @@ -11,6 +11,14 @@ export const indexPatternTitleSchema = schema.object({ indexPatternTitle: schema.string(), }); +export const dataVisualizerFieldHistogramsSchema = schema.object({ + /** Query to match documents in the index. */ + query: schema.any(), + fields: schema.arrayOf(schema.any()), + /** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */ + samplerShardSize: schema.number(), +}); + export const dataVisualizerFieldStatsSchema = schema.object({ /** Query to match documents in the index. */ query: schema.any(), From b66de7006d12ee84bd184019f91972eaaebaacf5 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 7 Jul 2020 17:35:35 +0200 Subject: [PATCH 02/15] [ML] Use histogram endpoint for analytics results pages. --- .../application/components/data_grid/index.ts | 2 +- .../components/data_grid/use_column_chart.tsx | 2 +- .../use_exploration_results.ts | 24 ++++++++++++------- .../outlier_exploration/use_outlier_data.ts | 24 ++++++++++++------- .../index_based/common/index.ts | 2 +- .../index_based/common/request.ts | 7 ++++++ .../index_based/data_loader/data_loader.ts | 17 ++++++++++++- 7 files changed, 58 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/ml/public/application/components/data_grid/index.ts b/x-pack/plugins/ml/public/application/components/data_grid/index.ts index 80bc6b861f742..8503fd7c5c1cd 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/index.ts +++ b/x-pack/plugins/ml/public/application/components/data_grid/index.ts @@ -12,7 +12,7 @@ export { showDataGridColumnChartErrorMessageToast, useRenderCellValue, } from './common'; -export { fetchChartsData, ChartData } from './use_column_chart'; +export { fetchChartsData, getFieldType, ChartData } from './use_column_chart'; export { useDataGrid } from './use_data_grid'; export { DataGrid } from './data_grid'; export { diff --git a/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.tsx index 6b207a999eb52..98d0501c87c69 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.tsx +++ b/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.tsx @@ -40,7 +40,7 @@ const getXScaleType = (kbnFieldType: KBN_FIELD_TYPES | undefined): XScaleType => } }; -const getFieldType = (schema: EuiDataGridColumn['schema']): KBN_FIELD_TYPES | undefined => { +export const getFieldType = (schema: EuiDataGridColumn['schema']): KBN_FIELD_TYPES | undefined => { if (schema === NON_AGGREGATABLE) { return undefined; } diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts index 796670f6a864d..41b7565f8cd64 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts @@ -12,16 +12,17 @@ import { CoreSetup } from 'src/core/public'; import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; +import { DataLoader } from '../../../../../datavisualizer/index_based/data_loader'; + import { - fetchChartsData, getDataGridSchemasFromFieldTypes, + getFieldType, showDataGridColumnChartErrorMessageToast, useDataGrid, useRenderCellValue, UseIndexDataReturnType, } from '../../../../../components/data_grid'; -import { SavedSearchQuery } from '../../../../../contexts/ml'; -import { ml } from '../../../../../services/ml_api_service'; +import { useMlContext, SavedSearchQuery } from '../../../../../contexts/ml'; import { getIndexData, getIndexFields, DataFrameAnalyticsConfig } from '../../../../common'; import { @@ -40,6 +41,8 @@ export const useExplorationResults = ( const needsDestIndexFields = indexPattern !== undefined && indexPattern.title === jobConfig?.source.index[0]; + const { kibanaConfig } = useMlContext(); + const columns: EuiDataGridColumn[] = []; if (jobConfig !== undefined) { @@ -74,12 +77,17 @@ export const useExplorationResults = ( const fetchColumnChartsData = async function () { try { - if (jobConfig !== undefined) { - const columnChartsData = await fetchChartsData( - jobConfig.dest.index, - ml.esSearch, + if (jobConfig !== undefined && indexPattern !== undefined) { + const dataLoader = new DataLoader(indexPattern, kibanaConfig); + const columnChartsData = await dataLoader.loadFieldHistograms( searchQuery, - columns.filter((cT) => dataGrid.visibleColumns.includes(cT.id)) + 5000, // samplerShardSize, + columns + .filter((cT) => dataGrid.visibleColumns.includes(cT.id)) + .map((cT) => ({ + fieldName: cT.id, + type: getFieldType(cT.schema), + })) ); dataGrid.setColumnCharts(columnChartsData); } diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts index beb6836bf801f..4bdba6df8875a 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts @@ -10,21 +10,22 @@ import { EuiDataGridColumn } from '@elastic/eui'; import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; +import { DataLoader } from '../../../../../datavisualizer/index_based/data_loader'; + import { useColorRange, COLOR_RANGE, COLOR_RANGE_SCALE, } from '../../../../../components/color_range_legend'; import { - fetchChartsData, + getFieldType, getDataGridSchemasFromFieldTypes, showDataGridColumnChartErrorMessageToast, useDataGrid, useRenderCellValue, UseIndexDataReturnType, } from '../../../../../components/data_grid'; -import { SavedSearchQuery } from '../../../../../contexts/ml'; -import { ml } from '../../../../../services/ml_api_service'; +import { useMlContext, SavedSearchQuery } from '../../../../../contexts/ml'; import { getToastNotifications } from '../../../../../util/dependency_cache'; import { getIndexData, getIndexFields, DataFrameAnalyticsConfig } from '../../../../common'; @@ -41,6 +42,8 @@ export const useOutlierData = ( const needsDestIndexFields = indexPattern !== undefined && indexPattern.title === jobConfig?.source.index[0]; + const { kibanaConfig } = useMlContext(); + const columns: EuiDataGridColumn[] = []; if (jobConfig !== undefined && indexPattern !== undefined) { @@ -81,12 +84,17 @@ export const useOutlierData = ( const fetchColumnChartsData = async function () { try { - if (jobConfig !== undefined) { - const columnChartsData = await fetchChartsData( - jobConfig.dest.index, - ml.esSearch, + if (jobConfig !== undefined && indexPattern !== undefined) { + const dataLoader = new DataLoader(indexPattern, kibanaConfig); + const columnChartsData = await dataLoader.loadFieldHistograms( searchQuery, - columns.filter((cT) => dataGrid.visibleColumns.includes(cT.id)) + 5000, // samplerShardSize, + columns + .filter((cT) => dataGrid.visibleColumns.includes(cT.id)) + .map((cT) => ({ + fieldName: cT.id, + type: getFieldType(cT.schema), + })) ); dataGrid.setColumnCharts(columnChartsData); } diff --git a/x-pack/plugins/ml/public/application/datavisualizer/index_based/common/index.ts b/x-pack/plugins/ml/public/application/datavisualizer/index_based/common/index.ts index 5618f701e4c5f..50278c300d103 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/index_based/common/index.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/index_based/common/index.ts @@ -5,4 +5,4 @@ */ export { FieldVisConfig } from './field_vis_config'; -export { FieldRequestConfig } from './request'; +export { FieldHistogramRequestConfig, FieldRequestConfig } from './request'; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/index_based/common/request.ts b/x-pack/plugins/ml/public/application/datavisualizer/index_based/common/request.ts index 9a886cbc899c2..fd4888b8729c1 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/index_based/common/request.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/index_based/common/request.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import { KBN_FIELD_TYPES } from '../../../../../../../../src/plugins/data/public'; + import { ML_JOB_FIELD_TYPES } from '../../../../../common/constants/field_types'; export interface FieldRequestConfig { @@ -11,3 +13,8 @@ export interface FieldRequestConfig { type: ML_JOB_FIELD_TYPES; cardinality: number; } + +export interface FieldHistogramRequestConfig { + fieldName: string; + type?: KBN_FIELD_TYPES; +} diff --git a/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts b/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts index 7d1f456d2334f..8d981ec710c2d 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts @@ -13,7 +13,7 @@ import { SavedSearchQuery } from '../../../contexts/ml'; import { IndexPatternTitle } from '../../../../../common/types/kibana'; import { ml } from '../../../services/ml_api_service'; -import { FieldRequestConfig } from '../common'; +import { FieldHistogramRequestConfig, FieldRequestConfig } from '../common'; // List of system fields we don't want to display. const OMIT_FIELDS: string[] = ['_source', '_type', '_index', '_id', '_version', '_score']; @@ -91,6 +91,21 @@ export class DataLoader { return stats; } + async loadFieldHistograms( + query: string | SavedSearchQuery, + samplerShardSize: number, + fields: FieldHistogramRequestConfig[] + ): Promise { + const stats = await ml.getVisualizerFieldHistograms({ + indexPatternTitle: this._indexPatternTitle, + query, + samplerShardSize, + fields, + }); + + return stats; + } + displayError(err: any) { const toastNotifications = getToastNotifications(); if (err.statusCode === 500) { From e079df8d133ebb9e8a106ae8665e6de08887c8b0 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 8 Jul 2020 09:20:52 +0200 Subject: [PATCH 03/15] [ML] Use histogram endpoint for analytics wizard. --- .../hooks/use_index_data.ts | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts index ee0e5c1955ead..fd7b302051657 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts @@ -11,8 +11,11 @@ import { EuiDataGridColumn } from '@elastic/eui'; import { CoreSetup } from 'src/core/public'; import { IndexPattern } from '../../../../../../../../../src/plugins/data/public'; + +import { DataLoader } from '../../../../datavisualizer/index_based/data_loader'; + import { - fetchChartsData, + getFieldType, getDataGridSchemaFromKibanaFieldType, getFieldsFromKibanaIndexPattern, showDataGridColumnChartErrorMessageToast, @@ -22,6 +25,7 @@ import { SearchResponse7, UseIndexDataReturnType, } from '../../../../components/data_grid'; +import { useMlContext } from '../../../../contexts/ml'; import { getErrorMessage } from '../../../../../../common/util/errors'; import { INDEX_STATUS } from '../../../common/analytics'; import { ml } from '../../../../services/ml_api_service'; @@ -35,6 +39,8 @@ export const useIndexData = ( ): UseIndexDataReturnType => { const indexPatternFields = getFieldsFromKibanaIndexPattern(indexPattern); + const { kibanaConfig } = useMlContext(); + // EuiDataGrid State const columns: EuiDataGridColumn[] = [ ...indexPatternFields.map((id) => { @@ -105,11 +111,16 @@ export const useIndexData = ( const fetchColumnChartsData = async function () { try { - const columnChartsData = await fetchChartsData( - indexPattern.title, - ml.esSearch, + const dataLoader = new DataLoader(indexPattern, kibanaConfig); + const columnChartsData = await dataLoader.loadFieldHistograms( query, - columns.filter((cT) => dataGrid.visibleColumns.includes(cT.id)) + 5000, // samplerShardSize, + columns + .filter((cT) => dataGrid.visibleColumns.includes(cT.id)) + .map((cT) => ({ + fieldName: cT.id, + type: getFieldType(cT.schema), + })) ); dataGrid.setColumnCharts(columnChartsData); } catch (e) { From 8296524758eb54336021cc9a0c3dbb09a6a9398a Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 8 Jul 2020 09:37:40 +0200 Subject: [PATCH 04/15] [ML] Fix DataLoader dependencies and constructor. Use API endpoint for all data fetching. --- .../application/components/data_grid/index.ts | 2 +- .../components/data_grid/use_column_chart.tsx | 184 ------------------ .../hooks/use_index_data.ts | 5 +- .../use_exploration_results.ts | 6 +- .../outlier_exploration/use_outlier_data.ts | 6 +- .../index_based/data_loader/data_loader.ts | 15 +- .../datavisualizer/index_based/page.tsx | 3 +- x-pack/plugins/ml/public/shared.ts | 2 + .../public/app/hooks/use_index_data.ts | 17 +- .../transform/public/shared_imports.ts | 3 +- 10 files changed, 33 insertions(+), 210 deletions(-) diff --git a/x-pack/plugins/ml/public/application/components/data_grid/index.ts b/x-pack/plugins/ml/public/application/components/data_grid/index.ts index 8503fd7c5c1cd..4bbd3595e5a7e 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/index.ts +++ b/x-pack/plugins/ml/public/application/components/data_grid/index.ts @@ -12,7 +12,7 @@ export { showDataGridColumnChartErrorMessageToast, useRenderCellValue, } from './common'; -export { fetchChartsData, getFieldType, ChartData } from './use_column_chart'; +export { getFieldType, ChartData } from './use_column_chart'; export { useDataGrid } from './use_data_grid'; export { DataGrid } from './data_grid'; export { diff --git a/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.tsx index 98d0501c87c69..a762c44e243bf 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.tsx +++ b/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.tsx @@ -16,8 +16,6 @@ import { i18n } from '@kbn/i18n'; import { KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public'; -import { stringHash } from '../../../../common/util/string_utils'; - import { NON_AGGREGATABLE } from './common'; export const hoveredRow$ = new BehaviorSubject(null); @@ -67,188 +65,6 @@ export const getFieldType = (schema: EuiDataGridColumn['schema']): KBN_FIELD_TYP return fieldType; }; -interface NumericColumnStats { - interval: number; - min: number; - max: number; -} -type NumericColumnStatsMap = Record; -const getAggIntervals = async ( - indexPatternTitle: string, - esSearch: (payload: any) => Promise, - query: any, - columnTypes: EuiDataGridColumn[] -): Promise => { - const numericColumns = columnTypes.filter((cT) => { - const fieldType = getFieldType(cT.schema); - return fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE; - }); - - if (numericColumns.length === 0) { - return {}; - } - - const minMaxAggs = numericColumns.reduce((aggs, c) => { - const id = stringHash(c.id); - aggs[id] = { - stats: { - field: c.id, - }, - }; - return aggs; - }, {} as Record); - - const respStats = await esSearch({ - index: indexPatternTitle, - size: 0, - body: { - query, - aggs: minMaxAggs, - size: 0, - }, - }); - - return Object.keys(respStats.aggregations).reduce((p, aggName) => { - const stats = [respStats.aggregations[aggName].min, respStats.aggregations[aggName].max]; - if (!stats.includes(null)) { - const delta = respStats.aggregations[aggName].max - respStats.aggregations[aggName].min; - - let aggInterval = 1; - - if (delta > MAX_CHART_COLUMNS) { - aggInterval = Math.round(delta / MAX_CHART_COLUMNS); - } - - if (delta <= 1) { - aggInterval = delta / MAX_CHART_COLUMNS; - } - - p[aggName] = { interval: aggInterval, min: stats[0], max: stats[1] }; - } - - return p; - }, {} as NumericColumnStatsMap); -}; - -interface AggHistogram { - histogram: { - field: string; - interval: number; - }; -} - -interface AggCardinality { - cardinality: { - field: string; - }; -} - -interface AggTerms { - terms: { - field: string; - size: number; - }; -} - -type ChartRequestAgg = AggHistogram | AggCardinality | AggTerms; - -export const fetchChartsData = async ( - indexPatternTitle: string, - esSearch: (payload: any) => Promise, - query: any, - columnTypes: EuiDataGridColumn[] -): Promise => { - const aggIntervals = await getAggIntervals(indexPatternTitle, esSearch, query, columnTypes); - - const chartDataAggs = columnTypes.reduce((aggs, c) => { - const fieldType = getFieldType(c.schema); - const id = stringHash(c.id); - if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) { - if (aggIntervals[id] !== undefined) { - aggs[`${id}_histogram`] = { - histogram: { - field: c.id, - interval: aggIntervals[id].interval !== 0 ? aggIntervals[id].interval : 1, - }, - }; - } - } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) { - if (fieldType === KBN_FIELD_TYPES.STRING) { - aggs[`${id}_cardinality`] = { - cardinality: { - field: c.id, - }, - }; - } - aggs[`${id}_terms`] = { - terms: { - field: c.id, - size: MAX_CHART_COLUMNS, - }, - }; - } - return aggs; - }, {} as Record); - - if (Object.keys(chartDataAggs).length === 0) { - return []; - } - - const respChartsData = await esSearch({ - index: indexPatternTitle, - size: 0, - body: { - query, - aggs: chartDataAggs, - size: 0, - }, - }); - - const chartsData: ChartData[] = columnTypes.map( - (c): ChartData => { - const fieldType = getFieldType(c.schema); - const id = stringHash(c.id); - - if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) { - if (aggIntervals[id] === undefined) { - return { - type: 'numeric', - data: [], - interval: 0, - stats: [0, 0], - id: c.id, - }; - } - - return { - data: respChartsData.aggregations[`${id}_histogram`].buckets, - interval: aggIntervals[id].interval, - stats: [aggIntervals[id].min, aggIntervals[id].max], - type: 'numeric', - id: c.id, - }; - } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) { - return { - type: fieldType === KBN_FIELD_TYPES.STRING ? 'ordinal' : 'boolean', - cardinality: - fieldType === KBN_FIELD_TYPES.STRING - ? respChartsData.aggregations[`${id}_cardinality`].value - : 2, - data: respChartsData.aggregations[`${id}_terms`].buckets, - id: c.id, - }; - } - - return { - type: 'unsupported', - id: c.id, - }; - } - ); - - return chartsData; -}; - interface NumericDataItem { key: number; key_as_string?: string; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts index fd7b302051657..7ea44a2a7822e 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts @@ -25,7 +25,6 @@ import { SearchResponse7, UseIndexDataReturnType, } from '../../../../components/data_grid'; -import { useMlContext } from '../../../../contexts/ml'; import { getErrorMessage } from '../../../../../../common/util/errors'; import { INDEX_STATUS } from '../../../common/analytics'; import { ml } from '../../../../services/ml_api_service'; @@ -39,8 +38,6 @@ export const useIndexData = ( ): UseIndexDataReturnType => { const indexPatternFields = getFieldsFromKibanaIndexPattern(indexPattern); - const { kibanaConfig } = useMlContext(); - // EuiDataGrid State const columns: EuiDataGridColumn[] = [ ...indexPatternFields.map((id) => { @@ -111,7 +108,7 @@ export const useIndexData = ( const fetchColumnChartsData = async function () { try { - const dataLoader = new DataLoader(indexPattern, kibanaConfig); + const dataLoader = new DataLoader(indexPattern, toastNotifications); const columnChartsData = await dataLoader.loadFieldHistograms( query, 5000, // samplerShardSize, diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts index 41b7565f8cd64..57d448ca17d20 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts @@ -22,7 +22,7 @@ import { useRenderCellValue, UseIndexDataReturnType, } from '../../../../../components/data_grid'; -import { useMlContext, SavedSearchQuery } from '../../../../../contexts/ml'; +import { SavedSearchQuery } from '../../../../../contexts/ml'; import { getIndexData, getIndexFields, DataFrameAnalyticsConfig } from '../../../../common'; import { @@ -41,8 +41,6 @@ export const useExplorationResults = ( const needsDestIndexFields = indexPattern !== undefined && indexPattern.title === jobConfig?.source.index[0]; - const { kibanaConfig } = useMlContext(); - const columns: EuiDataGridColumn[] = []; if (jobConfig !== undefined) { @@ -78,7 +76,7 @@ export const useExplorationResults = ( const fetchColumnChartsData = async function () { try { if (jobConfig !== undefined && indexPattern !== undefined) { - const dataLoader = new DataLoader(indexPattern, kibanaConfig); + const dataLoader = new DataLoader(indexPattern, toastNotifications); const columnChartsData = await dataLoader.loadFieldHistograms( searchQuery, 5000, // samplerShardSize, diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts index 4bdba6df8875a..b852e41ea1810 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts @@ -25,7 +25,7 @@ import { useRenderCellValue, UseIndexDataReturnType, } from '../../../../../components/data_grid'; -import { useMlContext, SavedSearchQuery } from '../../../../../contexts/ml'; +import { SavedSearchQuery } from '../../../../../contexts/ml'; import { getToastNotifications } from '../../../../../util/dependency_cache'; import { getIndexData, getIndexFields, DataFrameAnalyticsConfig } from '../../../../common'; @@ -42,8 +42,6 @@ export const useOutlierData = ( const needsDestIndexFields = indexPattern !== undefined && indexPattern.title === jobConfig?.source.index[0]; - const { kibanaConfig } = useMlContext(); - const columns: EuiDataGridColumn[] = []; if (jobConfig !== undefined && indexPattern !== undefined) { @@ -85,7 +83,7 @@ export const useOutlierData = ( const fetchColumnChartsData = async function () { try { if (jobConfig !== undefined && indexPattern !== undefined) { - const dataLoader = new DataLoader(indexPattern, kibanaConfig); + const dataLoader = new DataLoader(indexPattern, getToastNotifications()); const columnChartsData = await dataLoader.loadFieldHistograms( searchQuery, 5000, // samplerShardSize, diff --git a/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts b/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts index 8d981ec710c2d..4f5ff656137b9 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts @@ -6,7 +6,8 @@ import { i18n } from '@kbn/i18n'; -import { getToastNotifications } from '../../../util/dependency_cache'; +import { CoreSetup } from 'src/core/public'; + import { IndexPattern } from '../../../../../../../../src/plugins/data/public'; import { SavedSearchQuery } from '../../../contexts/ml'; @@ -24,10 +25,15 @@ export class DataLoader { private _indexPattern: IndexPattern; private _indexPatternTitle: IndexPatternTitle = ''; private _maxExamples: number = MAX_EXAMPLES_DEFAULT; + private _toastNotifications: CoreSetup['notifications']['toasts']; - constructor(indexPattern: IndexPattern, kibanaConfig: any) { + constructor( + indexPattern: IndexPattern, + toastNotifications: CoreSetup['notifications']['toasts'] + ) { this._indexPattern = indexPattern; this._indexPatternTitle = indexPattern.title; + this._toastNotifications = toastNotifications; } async loadOverallData( @@ -107,9 +113,8 @@ export class DataLoader { } displayError(err: any) { - const toastNotifications = getToastNotifications(); if (err.statusCode === 500) { - toastNotifications.addDanger( + this._toastNotifications.addDanger( i18n.translate('xpack.ml.datavisualizer.dataLoader.internalServerErrorMessage', { defaultMessage: 'Error loading data in index {index}. {message}. ' + @@ -121,7 +126,7 @@ export class DataLoader { }) ); } else { - toastNotifications.addDanger( + this._toastNotifications.addDanger( i18n.translate('xpack.ml.datavisualizer.page.errorLoadingDataMessage', { defaultMessage: 'Error loading data in index {index}. {message}', values: { diff --git a/x-pack/plugins/ml/public/application/datavisualizer/index_based/page.tsx b/x-pack/plugins/ml/public/application/datavisualizer/index_based/page.tsx index 97b4043c9fd64..301cdec2f788b 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/index_based/page.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/index_based/page.tsx @@ -43,6 +43,7 @@ import { kbnTypeToMLJobType } from '../../util/field_types_utils'; import { useTimefilter } from '../../contexts/kibana'; import { timeBasedIndexCheck, getQueryFromSavedSearch } from '../../util/index_utils'; import { getTimeBucketsFromCache } from '../../util/time_buckets'; +import { getToastNotifications } from '../../util/dependency_cache'; import { useUrlState } from '../../util/url_state'; import { FieldRequestConfig, FieldVisConfig } from './common'; import { ActionsPanel } from './components/actions_panel'; @@ -107,7 +108,7 @@ export const Page: FC = () => { autoRefreshSelector: true, }); - const dataLoader = new DataLoader(currentIndexPattern, kibanaConfig); + const dataLoader = new DataLoader(currentIndexPattern, getToastNotifications()); const [globalState, setGlobalState] = useUrlState('_g'); useEffect(() => { if (globalState?.time !== undefined) { diff --git a/x-pack/plugins/ml/public/shared.ts b/x-pack/plugins/ml/public/shared.ts index 4b1d7ee733dcf..d559c1f6f027e 100644 --- a/x-pack/plugins/ml/public/shared.ts +++ b/x-pack/plugins/ml/public/shared.ts @@ -20,3 +20,5 @@ export * from './application/formatters/metric_change_description'; export * from './application/components/data_grid'; export * from './application/data_frame_analytics/common'; export * from './application/util/date_utils'; + +export { DataLoader } from './application/datavisualizer/index_based/data_loader'; diff --git a/x-pack/plugins/transform/public/app/hooks/use_index_data.ts b/x-pack/plugins/transform/public/app/hooks/use_index_data.ts index c821c183ad370..2b725aac33909 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_index_data.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_index_data.ts @@ -9,13 +9,14 @@ import { useEffect } from 'react'; import { EuiDataGridColumn } from '@elastic/eui'; import { - fetchChartsData, + getFieldType, getDataGridSchemaFromKibanaFieldType, getFieldsFromKibanaIndexPattern, getErrorMessage, showDataGridColumnChartErrorMessageToast, useDataGrid, useRenderCellValue, + DataLoader, EsSorting, SearchResponse7, UseIndexDataReturnType, @@ -107,13 +108,17 @@ export const useIndexData = ( const fetchColumnChartsData = async function () { try { - const columnChartsData = await fetchChartsData( - indexPattern.title, - api.esSearch, + const dataLoader = new DataLoader(indexPattern, toastNotifications); + const columnChartsData = await dataLoader.loadFieldHistograms( isDefaultQuery(query) ? matchAllQuery : query, - columns.filter((cT) => dataGrid.visibleColumns.includes(cT.id)) + 5000, // samplerShardSize, + columns + .filter((cT) => dataGrid.visibleColumns.includes(cT.id)) + .map((cT) => ({ + fieldName: cT.id, + type: getFieldType(cT.schema), + })) ); - setColumnCharts(columnChartsData); } catch (e) { showDataGridColumnChartErrorMessageToast(e, toastNotifications); diff --git a/x-pack/plugins/transform/public/shared_imports.ts b/x-pack/plugins/transform/public/shared_imports.ts index e0bbcd0b5d9db..a5cd71cfebf81 100644 --- a/x-pack/plugins/transform/public/shared_imports.ts +++ b/x-pack/plugins/transform/public/shared_imports.ts @@ -14,7 +14,7 @@ export { } from '../../../../src/plugins/es_ui_shared/public'; export { - fetchChartsData, + getFieldType, getErrorMessage, extractErrorMessage, formatHumanReadableDateTimeSeconds, @@ -26,6 +26,7 @@ export { useRenderCellValue, ChartData, DataGrid, + DataLoader, EsSorting, RenderCellValue, SearchResponse7, From b939a879527a2a1d6bc0b9f17d317c6ff64efb06 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 8 Jul 2020 10:30:28 +0200 Subject: [PATCH 05/15] [ML] Sampler support. Reintroduce KBN_FIELD_TYPES. --- .../models/data_visualizer/data_visualizer.ts | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts index 737c2f9b85605..8c9223b008ea2 100644 --- a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts +++ b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts @@ -6,6 +6,7 @@ import { LegacyCallAPIOptions, LegacyAPICaller } from 'kibana/server'; import _ from 'lodash'; +import { KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/server'; import { ML_JOB_FIELD_TYPES } from '../../../common/constants/field_types'; import { getSafeAggregationName } from '../../../common/util/job_utils'; import { stringHash } from '../../../common/util/string_utils'; @@ -279,7 +280,7 @@ export class DataVisualizer { samplerShardSize: number ): Promise { const numericColumns = fields.filter((field) => { - return field.type === 'number' || field.type === 'number'; + return field.type === KBN_FIELD_TYPES.NUMBER || field.type === KBN_FIELD_TYPES.DATE; }); if (numericColumns.length === 0) { @@ -301,15 +302,18 @@ export class DataVisualizer { size: 0, body: { query, - aggs: minMaxAggs, + aggs: buildSamplerAggregation(minMaxAggs, samplerShardSize), size: 0, }, }); - return Object.keys(respStats.aggregations).reduce((p, aggName) => { - const stats = [respStats.aggregations[aggName].min, respStats.aggregations[aggName].max]; + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + const aggregations = _.get(respStats.aggregations, aggsPath); + + return Object.keys(aggregations).reduce((p, aggName) => { + const stats = [aggregations[aggName].min, aggregations[aggName].max]; if (!stats.includes(null)) { - const delta = respStats.aggregations[aggName].max - respStats.aggregations[aggName].min; + const delta = aggregations[aggName].max - aggregations[aggName].min; let aggInterval = 1; @@ -348,7 +352,7 @@ export class DataVisualizer { const fieldName = field.fieldName; const fieldType = field.type; const id = stringHash(fieldName); - if (fieldType === 'number' || fieldType === 'date') { + if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) { if (aggIntervals[id] !== undefined) { aggs[`${id}_histogram`] = { histogram: { @@ -357,8 +361,8 @@ export class DataVisualizer { }, }; } - } else if (fieldType === 'string' || fieldType === 'boolean') { - if (fieldType === 'string') { + } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) { + if (fieldType === KBN_FIELD_TYPES.STRING) { aggs[`${id}_cardinality`] = { cardinality: { field: fieldName, @@ -384,18 +388,21 @@ export class DataVisualizer { size: 0, body: { query, - aggs: chartDataAggs, + aggs: buildSamplerAggregation(chartDataAggs, samplerShardSize), size: 0, }, }); + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + const aggregations = _.get(respChartsData.aggregations, aggsPath); + const chartsData: ChartData[] = fields.map( (field): ChartData => { const fieldName = field.fieldName; const fieldType = field.type; const id = stringHash(field.fieldName); - if (fieldType === 'number' || fieldType === 'date') { + if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) { if (aggIntervals[id] === undefined) { return { type: 'numeric', @@ -407,18 +414,18 @@ export class DataVisualizer { } return { - data: respChartsData.aggregations[`${id}_histogram`].buckets, + data: aggregations[`${id}_histogram`].buckets, interval: aggIntervals[id].interval, stats: [aggIntervals[id].min, aggIntervals[id].max], type: 'numeric', id: fieldName, }; - } else if (fieldType === 'string' || fieldType === 'boolean') { + } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) { return { - type: fieldType === 'string' ? 'ordinal' : 'boolean', + type: fieldType === KBN_FIELD_TYPES.STRING ? 'ordinal' : 'boolean', cardinality: - fieldType === 'string' ? respChartsData.aggregations[`${id}_cardinality`].value : 2, - data: respChartsData.aggregations[`${id}_terms`].buckets, + fieldType === KBN_FIELD_TYPES.STRING ? aggregations[`${id}_cardinality`].value : 2, + data: aggregations[`${id}_terms`].buckets, id: fieldName, }; } From 5daa85443de1415188c69dc81d9635550ac2c812 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 8 Jul 2020 11:49:57 +0200 Subject: [PATCH 06/15] [ML] Sample info tooltip. --- .../ml/common/constants/field_histograms.ts | 8 ++++ .../components/data_grid/data_grid.tsx | 41 ++++++++++++------- .../hooks/use_index_data.ts | 5 +-- .../use_exploration_results.ts | 5 +-- .../outlier_exploration/use_outlier_data.ts | 5 +-- .../index_based/data_loader/data_loader.ts | 7 ++-- .../services/ml_api_service/index.ts | 6 +-- .../public/app/hooks/use_index_data.ts | 5 +-- 8 files changed, 50 insertions(+), 32 deletions(-) create mode 100644 x-pack/plugins/ml/common/constants/field_histograms.ts diff --git a/x-pack/plugins/ml/common/constants/field_histograms.ts b/x-pack/plugins/ml/common/constants/field_histograms.ts new file mode 100644 index 0000000000000..5c86c00ac666f --- /dev/null +++ b/x-pack/plugins/ml/common/constants/field_histograms.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// Default sampler shard size used for field histograms +export const DEFAULT_SAMPLER_SHARD_SIZE = 5000; diff --git a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx index 9af7a869e0e56..d4be2eab13d26 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx +++ b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx @@ -20,10 +20,13 @@ import { EuiFlexItem, EuiSpacer, EuiTitle, + EuiToolTip, } from '@elastic/eui'; import { CoreSetup } from 'src/core/public'; +import { DEFAULT_SAMPLER_SHARD_SIZE } from '../../../../common/constants/field_histograms'; + import { INDEX_STATUS } from '../../data_frame_analytics/common'; import { euiDataGridStyle, euiDataGridToolbarSettings } from './common'; @@ -193,21 +196,31 @@ export const DataGrid: FC = memo( ...(chartsButtonVisible ? { additionalControls: ( - - {i18n.translate('xpack.ml.dataGrid.histogramButtonText', { - defaultMessage: 'Histogram charts', + + > + + {i18n.translate('xpack.ml.dataGrid.histogramButtonText', { + defaultMessage: 'Histogram charts', + })} + + ), } : {}), diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts index 7ea44a2a7822e..7ae462d5c9c20 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts @@ -110,14 +110,13 @@ export const useIndexData = ( try { const dataLoader = new DataLoader(indexPattern, toastNotifications); const columnChartsData = await dataLoader.loadFieldHistograms( - query, - 5000, // samplerShardSize, columns .filter((cT) => dataGrid.visibleColumns.includes(cT.id)) .map((cT) => ({ fieldName: cT.id, type: getFieldType(cT.schema), - })) + })), + query ); dataGrid.setColumnCharts(columnChartsData); } catch (e) { diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts index 57d448ca17d20..c1d73e4d19305 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts @@ -78,14 +78,13 @@ export const useExplorationResults = ( if (jobConfig !== undefined && indexPattern !== undefined) { const dataLoader = new DataLoader(indexPattern, toastNotifications); const columnChartsData = await dataLoader.loadFieldHistograms( - searchQuery, - 5000, // samplerShardSize, columns .filter((cT) => dataGrid.visibleColumns.includes(cT.id)) .map((cT) => ({ fieldName: cT.id, type: getFieldType(cT.schema), - })) + })), + searchQuery ); dataGrid.setColumnCharts(columnChartsData); } diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts index b852e41ea1810..68acf14eea957 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts @@ -85,14 +85,13 @@ export const useOutlierData = ( if (jobConfig !== undefined && indexPattern !== undefined) { const dataLoader = new DataLoader(indexPattern, getToastNotifications()); const columnChartsData = await dataLoader.loadFieldHistograms( - searchQuery, - 5000, // samplerShardSize, columns .filter((cT) => dataGrid.visibleColumns.includes(cT.id)) .map((cT) => ({ fieldName: cT.id, type: getFieldType(cT.schema), - })) + })), + searchQuery ); dataGrid.setColumnCharts(columnChartsData); } diff --git a/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts b/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts index 4f5ff656137b9..f617bb743e2a9 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts @@ -12,6 +12,7 @@ import { IndexPattern } from '../../../../../../../../src/plugins/data/public'; import { SavedSearchQuery } from '../../../contexts/ml'; import { IndexPatternTitle } from '../../../../../common/types/kibana'; +import { DEFAULT_SAMPLER_SHARD_SIZE } from '../../../../../common/constants/field_histograms'; import { ml } from '../../../services/ml_api_service'; import { FieldHistogramRequestConfig, FieldRequestConfig } from '../common'; @@ -98,15 +99,15 @@ export class DataLoader { } async loadFieldHistograms( + fields: FieldHistogramRequestConfig[], query: string | SavedSearchQuery, - samplerShardSize: number, - fields: FieldHistogramRequestConfig[] + samplerShardSize = DEFAULT_SAMPLER_SHARD_SIZE ): Promise { const stats = await ml.getVisualizerFieldHistograms({ indexPatternTitle: this._indexPatternTitle, query, - samplerShardSize, fields, + samplerShardSize, }); return stats; diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts index a073656e16978..599e4d4bb8a10 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts @@ -500,18 +500,18 @@ export function mlApiServicesProvider(httpService: HttpService) { getVisualizerFieldHistograms({ indexPatternTitle, query, - samplerShardSize, fields, + samplerShardSize, }: { indexPatternTitle: string; query: any; + fields: FieldHistogramRequestConfig[]; samplerShardSize?: number; - fields?: FieldHistogramRequestConfig[]; }) { const body = JSON.stringify({ query, - samplerShardSize, fields, + samplerShardSize, }); return httpService.http({ diff --git a/x-pack/plugins/transform/public/app/hooks/use_index_data.ts b/x-pack/plugins/transform/public/app/hooks/use_index_data.ts index 2b725aac33909..1642b49d6815d 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_index_data.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_index_data.ts @@ -110,14 +110,13 @@ export const useIndexData = ( try { const dataLoader = new DataLoader(indexPattern, toastNotifications); const columnChartsData = await dataLoader.loadFieldHistograms( - isDefaultQuery(query) ? matchAllQuery : query, - 5000, // samplerShardSize, columns .filter((cT) => dataGrid.visibleColumns.includes(cT.id)) .map((cT) => ({ fieldName: cT.id, type: getFieldType(cT.schema), - })) + })), + isDefaultQuery(query) ? matchAllQuery : query ); setColumnCharts(columnChartsData); } catch (e) { From 3f452145e46864074e5a2a501e798502cc43cd17 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 9 Jul 2020 12:45:48 +0200 Subject: [PATCH 07/15] [ML] Added jest and API integration tests. --- .../data_grid/use_column_chart.test.tsx | 20 +++ .../models/data_visualizer/data_visualizer.ts | 18 +-- .../ml/server/routes/data_visualizer.ts | 12 +- .../data_visualizer/get_field_histograms.ts | 122 ++++++++++++++++++ 4 files changed, 157 insertions(+), 15 deletions(-) create mode 100644 x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.test.tsx create mode 100644 x-pack/test/api_integration/apis/ml/data_visualizer/get_field_histograms.ts diff --git a/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.test.tsx b/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.test.tsx new file mode 100644 index 0000000000000..77628e394bfeb --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.test.tsx @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; + +import { getFieldType } from './use_column_chart'; + +describe('getFieldType', () => { + it('should return the Kibana field type for a given EUI data grid schema', () => { + expect(getFieldType('text')).toBe('string'); + expect(getFieldType('datetime')).toBe('date'); + expect(getFieldType('numeric')).toBe('number'); + expect(getFieldType('boolean')).toBe('boolean'); + expect(getFieldType('json')).toBe('object'); + expect(getFieldType('non-aggregatable')).toBe(undefined); + }); +}); diff --git a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts index 8c9223b008ea2..9512eda0303a0 100644 --- a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts +++ b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts @@ -39,7 +39,7 @@ export interface Field { cardinality: number; } -interface HistogramField { +export interface HistogramField { fieldName: string; type: string; } @@ -308,7 +308,8 @@ export class DataVisualizer { }); const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const aggregations = _.get(respStats.aggregations, aggsPath); + const aggregations = + aggsPath.length > 0 ? _.get(respStats.aggregations, aggsPath) : respStats.aggregations; return Object.keys(aggregations).reduce((p, aggName) => { const stats = [aggregations[aggName].min, aggregations[aggName].max]; @@ -317,12 +318,8 @@ export class DataVisualizer { let aggInterval = 1; - if (delta > MAX_CHART_COLUMNS) { - aggInterval = Math.round(delta / MAX_CHART_COLUMNS); - } - - if (delta <= 1) { - aggInterval = delta / MAX_CHART_COLUMNS; + if (delta > MAX_CHART_COLUMNS || delta <= 1) { + aggInterval = delta / (MAX_CHART_COLUMNS - 1); } p[aggName] = { interval: aggInterval, min: stats[0], max: stats[1] }; @@ -394,7 +391,10 @@ export class DataVisualizer { }); const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const aggregations = _.get(respChartsData.aggregations, aggsPath); + const aggregations = + aggsPath.length > 0 + ? _.get(respChartsData.aggregations, aggsPath) + : respChartsData.aggregations; const chartsData: ChartData[] = fields.map( (field): ChartData => { diff --git a/x-pack/plugins/ml/server/routes/data_visualizer.ts b/x-pack/plugins/ml/server/routes/data_visualizer.ts index c3c555253c995..9dd010e105b6e 100644 --- a/x-pack/plugins/ml/server/routes/data_visualizer.ts +++ b/x-pack/plugins/ml/server/routes/data_visualizer.ts @@ -7,7 +7,7 @@ import { RequestHandlerContext } from 'kibana/server'; import { wrapError } from '../client/error_wrapper'; import { DataVisualizer } from '../models/data_visualizer'; -import { Field } from '../models/data_visualizer/data_visualizer'; +import { Field, HistogramField } from '../models/data_visualizer/data_visualizer'; import { dataVisualizerFieldHistogramsSchema, dataVisualizerFieldStatsSchema, @@ -70,7 +70,7 @@ function getHistogramsForFields( context: RequestHandlerContext, indexPatternTitle: string, query: any, - fields: Field[], + fields: HistogramField[], samplerShardSize: number ) { const dv = new DataVisualizer(context.ml!.mlClient.callAsCurrentUser); @@ -84,14 +84,14 @@ export function dataVisualizerRoutes({ router, mlLicense }: RouteInitialization) /** * @apiGroup DataVisualizer * - * @api {post} /api/ml/data_visualizer/get_field_stats/:indexPatternTitle Get stats for fields - * @apiName GetStatsForFields - * @apiDescription Returns the stats on individual fields in the specified index pattern. + * @api {post} /api/ml/data_visualizer/get_field_stats/:indexPatternTitle Get histograms for fields + * @apiName GetHistogramsForFields + * @apiDescription Returns the histograms on a list fields in the specified index pattern. * * @apiSchema (params) indexPatternTitleSchema * @apiSchema (body) dataVisualizerFieldHistogramsSchema * - * @apiSuccess {Object} fieldName stats by field, keyed on the name of the field. + * @apiSuccess {Object} fieldName histograms by field, keyed on the name of the field. */ router.post( { diff --git a/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_histograms.ts b/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_histograms.ts new file mode 100644 index 0000000000000..8b21c367d29f6 --- /dev/null +++ b/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_histograms.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { USER } from '../../../../functional/services/ml/security_common'; +import { COMMON_REQUEST_HEADERS } from '../../../../functional/services/ml/common'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertestWithoutAuth'); + const ml = getService('ml'); + + const fieldHistogramsTestData = { + testTitle: 'returns histogram data for fields', + index: 'ft_farequote', + user: USER.ML_POWERUSER, + requestBody: { + query: { bool: { should: [{ match_phrase: { airline: 'JZA' } }], minimum_should_match: 1 } }, + fields: [ + { fieldName: '@timestamp', type: 'date' }, + { fieldName: 'airline', type: 'string' }, + { fieldName: 'responsetime', type: 'number' }, + ], + samplerShardSize: -1, // No sampling, as otherwise counts could vary on each run. + }, + expected: { + responseCode: 200, + responseBody: [ + { + dataLength: 20, + type: 'numeric', + id: '@timestamp', + }, + { type: 'ordinal', dataLength: 1, id: 'airline' }, + { + dataLength: 20, + type: 'numeric', + id: 'responsetime', + }, + ], + }, + }; + + const errorTestData = { + testTitle: 'returns error for index which does not exist', + index: 'ft_farequote_not_exists', + user: USER.ML_POWERUSER, + requestBody: { + query: { bool: { must: [{ match_all: {} }] } }, + fields: [{ fieldName: 'responsetime', type: 'number' }], + samplerShardSize: -1, + }, + expected: { + responseCode: 404, + responseBody: { + statusCode: 404, + error: 'Not Found', + message: + '[index_not_found_exception] no such index [ft_farequote_not_exists], with { resource.type="index_or_alias" & resource.id="ft_farequote_not_exists" & index_uuid="_na_" & index="ft_farequote_not_exists" }', + }, + }, + }; + + async function runGetFieldHistogramsRequest( + index: string, + user: USER, + requestBody: object, + expectedResponsecode: number + ): Promise { + const { body } = await supertest + .post(`/api/ml/data_visualizer/get_field_histograms/${index}`) + .auth(user, ml.securityCommon.getPasswordForUser(user)) + .set(COMMON_REQUEST_HEADERS) + .send(requestBody) + .expect(expectedResponsecode); + + return body; + } + + describe('get_field_histograms', function () { + before(async () => { + await esArchiver.loadIfNeeded('ml/farequote'); + await ml.testResources.setKibanaTimeZoneToUTC(); + }); + + it(`${fieldHistogramsTestData.testTitle}`, async () => { + const body = await runGetFieldHistogramsRequest( + fieldHistogramsTestData.index, + fieldHistogramsTestData.user, + fieldHistogramsTestData.requestBody, + fieldHistogramsTestData.expected.responseCode + ); + + const expected = fieldHistogramsTestData.expected; + + const actual = body.map((b: any) => ({ + dataLength: b.data.length, + type: b.type, + id: b.id, + })); + expect(actual).to.eql(expected.responseBody); + }); + + it(`${errorTestData.testTitle}`, async () => { + const body = await runGetFieldHistogramsRequest( + errorTestData.index, + errorTestData.user, + errorTestData.requestBody, + errorTestData.expected.responseCode + ); + + expect(body.error).to.eql(errorTestData.expected.responseBody.error); + expect(body.message).to.eql(errorTestData.expected.responseBody.message); + }); + }); +}; From ea9190062330274fdff22b1a3699cef5b82cdb1c Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 9 Jul 2020 14:20:26 +0200 Subject: [PATCH 08/15] [ML] Fix TS. --- .../{use_column_chart.test.tsx => use_column_chart.test.ts} | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) rename x-pack/plugins/ml/public/application/components/data_grid/{use_column_chart.test.tsx => use_column_chart.test.ts} (92%) diff --git a/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.test.tsx b/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.test.ts similarity index 92% rename from x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.test.tsx rename to x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.test.ts index 77628e394bfeb..1b35ef238d09e 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.test.tsx +++ b/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.test.ts @@ -4,11 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import React from 'react'; - import { getFieldType } from './use_column_chart'; -describe('getFieldType', () => { +describe('getFieldType()', () => { it('should return the Kibana field type for a given EUI data grid schema', () => { expect(getFieldType('text')).toBe('string'); expect(getFieldType('datetime')).toBe('date'); From 84203df4ebbf352f31a52f015778d7dc2c8f81e8 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 9 Jul 2020 16:41:23 +0200 Subject: [PATCH 09/15] [ML] Use setDependencyCache to initialize http when using DataLoader via transforms. --- x-pack/plugins/ml/public/shared.ts | 1 + x-pack/plugins/transform/public/app/app.tsx | 8 ++++++++ x-pack/plugins/transform/public/shared_imports.ts | 2 ++ 3 files changed, 11 insertions(+) diff --git a/x-pack/plugins/ml/public/shared.ts b/x-pack/plugins/ml/public/shared.ts index d559c1f6f027e..87daa2d2c1790 100644 --- a/x-pack/plugins/ml/public/shared.ts +++ b/x-pack/plugins/ml/public/shared.ts @@ -22,3 +22,4 @@ export * from './application/data_frame_analytics/common'; export * from './application/util/date_utils'; export { DataLoader } from './application/datavisualizer/index_based/data_loader'; +export { clearCache, setDependencyCache } from './application/util/dependency_cache'; diff --git a/x-pack/plugins/transform/public/app/app.tsx b/x-pack/plugins/transform/public/app/app.tsx index ccfdc8b0942fa..953417568aba3 100644 --- a/x-pack/plugins/transform/public/app/app.tsx +++ b/x-pack/plugins/transform/public/app/app.tsx @@ -15,6 +15,8 @@ import { KibanaContextProvider } from '../../../../../src/plugins/kibana_react/p import { API_BASE_PATH } from '../../common/constants'; +import { clearCache, setDependencyCache } from '../shared_imports'; + import { SectionError } from './components'; import { SECTION_SLUG } from './constants'; import { AuthorizationContext, AuthorizationProvider } from './lib/authorization'; @@ -60,6 +62,11 @@ export const App: FC<{ history: ScopedHistory }> = ({ history }) => { }; export const renderApp = (element: HTMLElement, appDependencies: AppDependencies) => { + // TODO Temporary fix to make the Data Grid Histograms in the Transforms Wizard work + setDependencyCache({ + http: appDependencies.http, + }); + const I18nContext = appDependencies.i18n.Context; render( @@ -75,5 +82,6 @@ export const renderApp = (element: HTMLElement, appDependencies: AppDependencies return () => { unmountComponentAtNode(element); + clearCache(); }; }; diff --git a/x-pack/plugins/transform/public/shared_imports.ts b/x-pack/plugins/transform/public/shared_imports.ts index a5cd71cfebf81..b5f6930cebb33 100644 --- a/x-pack/plugins/transform/public/shared_imports.ts +++ b/x-pack/plugins/transform/public/shared_imports.ts @@ -14,6 +14,7 @@ export { } from '../../../../src/plugins/es_ui_shared/public'; export { + clearCache, getFieldType, getErrorMessage, extractErrorMessage, @@ -21,6 +22,7 @@ export { getDataGridSchemaFromKibanaFieldType, getFieldsFromKibanaIndexPattern, multiColumnSortFactory, + setDependencyCache, showDataGridColumnChartErrorMessageToast, useDataGrid, useRenderCellValue, From a117e4374caa905335ba27b0fcd77916c86b5df6 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 13 Jul 2020 08:05:12 +0200 Subject: [PATCH 10/15] [ML] Histogram API endpoint for transforms. --- .../models/data_visualizer/data_visualizer.ts | 329 +++++++++--------- .../ml/server/models/data_visualizer/index.ts | 2 +- x-pack/plugins/ml/server/shared.ts | 1 + .../server/routes/api/field_histograms.ts | 50 +++ .../transform/server/routes/api/schema.ts | 17 + .../transform/server/shared_imports.ts | 7 + 6 files changed, 248 insertions(+), 158 deletions(-) create mode 100644 x-pack/plugins/transform/server/routes/api/field_histograms.ts create mode 100644 x-pack/plugins/transform/server/shared_imports.ts diff --git a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts index 9512eda0303a0..d1a4a0b585fbb 100644 --- a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts +++ b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { LegacyCallAPIOptions, LegacyAPICaller } from 'kibana/server'; +import { LegacyAPICaller } from 'kibana/server'; import _ from 'lodash'; import { KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/server'; import { ML_JOB_FIELD_TYPES } from '../../../common/constants/field_types'; @@ -179,12 +179,176 @@ type BatchStats = | DocumentCountStats | FieldExamples; +const getAggIntervals = async ( + callAsCurrentUser: LegacyAPICaller, + indexPatternTitle: string, + query: any, + fields: HistogramField[], + samplerShardSize: number +): Promise => { + const numericColumns = fields.filter((field) => { + return field.type === KBN_FIELD_TYPES.NUMBER || field.type === KBN_FIELD_TYPES.DATE; + }); + + if (numericColumns.length === 0) { + return {}; + } + + const minMaxAggs = numericColumns.reduce((aggs, c) => { + const id = stringHash(c.fieldName); + aggs[id] = { + stats: { + field: c.fieldName, + }, + }; + return aggs; + }, {} as Record); + + const respStats = await callAsCurrentUser('search', { + index: indexPatternTitle, + size: 0, + body: { + query, + aggs: buildSamplerAggregation(minMaxAggs, samplerShardSize), + size: 0, + }, + }); + + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + const aggregations = + aggsPath.length > 0 ? _.get(respStats.aggregations, aggsPath) : respStats.aggregations; + + return Object.keys(aggregations).reduce((p, aggName) => { + const stats = [aggregations[aggName].min, aggregations[aggName].max]; + if (!stats.includes(null)) { + const delta = aggregations[aggName].max - aggregations[aggName].min; + + let aggInterval = 1; + + if (delta > MAX_CHART_COLUMNS || delta <= 1) { + aggInterval = delta / (MAX_CHART_COLUMNS - 1); + } + + p[aggName] = { interval: aggInterval, min: stats[0], max: stats[1] }; + } + + return p; + }, {} as NumericColumnStatsMap); +}; + +// export for re-use by transforms plugin +export const getHistogramsForFields = async ( + callAsCurrentUser: LegacyAPICaller, + indexPatternTitle: string, + query: any, + fields: HistogramField[], + samplerShardSize: number +) => { + const aggIntervals = await getAggIntervals( + callAsCurrentUser, + indexPatternTitle, + query, + fields, + samplerShardSize + ); + + const chartDataAggs = fields.reduce((aggs, field) => { + const fieldName = field.fieldName; + const fieldType = field.type; + const id = stringHash(fieldName); + if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) { + if (aggIntervals[id] !== undefined) { + aggs[`${id}_histogram`] = { + histogram: { + field: fieldName, + interval: aggIntervals[id].interval !== 0 ? aggIntervals[id].interval : 1, + }, + }; + } + } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) { + if (fieldType === KBN_FIELD_TYPES.STRING) { + aggs[`${id}_cardinality`] = { + cardinality: { + field: fieldName, + }, + }; + } + aggs[`${id}_terms`] = { + terms: { + field: fieldName, + size: MAX_CHART_COLUMNS, + }, + }; + } + return aggs; + }, {} as Record); + + if (Object.keys(chartDataAggs).length === 0) { + return []; + } + + const respChartsData = await callAsCurrentUser('search', { + index: indexPatternTitle, + size: 0, + body: { + query, + aggs: buildSamplerAggregation(chartDataAggs, samplerShardSize), + size: 0, + }, + }); + + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + const aggregations = + aggsPath.length > 0 + ? _.get(respChartsData.aggregations, aggsPath) + : respChartsData.aggregations; + + const chartsData: ChartData[] = fields.map( + (field): ChartData => { + const fieldName = field.fieldName; + const fieldType = field.type; + const id = stringHash(field.fieldName); + + if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) { + if (aggIntervals[id] === undefined) { + return { + type: 'numeric', + data: [], + interval: 0, + stats: [0, 0], + id: fieldName, + }; + } + + return { + data: aggregations[`${id}_histogram`].buckets, + interval: aggIntervals[id].interval, + stats: [aggIntervals[id].min, aggIntervals[id].max], + type: 'numeric', + id: fieldName, + }; + } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) { + return { + type: fieldType === KBN_FIELD_TYPES.STRING ? 'ordinal' : 'boolean', + cardinality: + fieldType === KBN_FIELD_TYPES.STRING ? aggregations[`${id}_cardinality`].value : 2, + data: aggregations[`${id}_terms`].buckets, + id: fieldName, + }; + } + + return { + type: 'unsupported', + id: fieldName, + }; + } + ); + + return chartsData; +}; + export class DataVisualizer { - callAsCurrentUser: ( - endpoint: string, - clientParams: Record, - options?: LegacyCallAPIOptions - ) => Promise; + callAsCurrentUser: LegacyAPICaller; constructor(callAsCurrentUser: LegacyAPICaller) { this.callAsCurrentUser = callAsCurrentUser; @@ -273,62 +437,6 @@ export class DataVisualizer { return stats; } - async getAggIntervals( - indexPatternTitle: string, - query: any, - fields: HistogramField[], - samplerShardSize: number - ): Promise { - const numericColumns = fields.filter((field) => { - return field.type === KBN_FIELD_TYPES.NUMBER || field.type === KBN_FIELD_TYPES.DATE; - }); - - if (numericColumns.length === 0) { - return {}; - } - - const minMaxAggs = numericColumns.reduce((aggs, c) => { - const id = stringHash(c.fieldName); - aggs[id] = { - stats: { - field: c.fieldName, - }, - }; - return aggs; - }, {} as Record); - - const respStats = await this.callAsCurrentUser('search', { - index: indexPatternTitle, - size: 0, - body: { - query, - aggs: buildSamplerAggregation(minMaxAggs, samplerShardSize), - size: 0, - }, - }); - - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const aggregations = - aggsPath.length > 0 ? _.get(respStats.aggregations, aggsPath) : respStats.aggregations; - - return Object.keys(aggregations).reduce((p, aggName) => { - const stats = [aggregations[aggName].min, aggregations[aggName].max]; - if (!stats.includes(null)) { - const delta = aggregations[aggName].max - aggregations[aggName].min; - - let aggInterval = 1; - - if (delta > MAX_CHART_COLUMNS || delta <= 1) { - aggInterval = delta / (MAX_CHART_COLUMNS - 1); - } - - p[aggName] = { interval: aggInterval, min: stats[0], max: stats[1] }; - } - - return p; - }, {} as NumericColumnStatsMap); - } - // Obtains binned histograms for supplied list of fields. The statistics for each field in the // returned array depend on the type of the field (keyword, number, date etc). // Sampling will be used if supplied samplerShardSize > 0. @@ -338,106 +446,13 @@ export class DataVisualizer { fields: HistogramField[], samplerShardSize: number ): Promise { - const aggIntervals = await this.getAggIntervals( + return await getHistogramsForFields( + this.callAsCurrentUser, indexPatternTitle, query, fields, samplerShardSize ); - - const chartDataAggs = fields.reduce((aggs, field) => { - const fieldName = field.fieldName; - const fieldType = field.type; - const id = stringHash(fieldName); - if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) { - if (aggIntervals[id] !== undefined) { - aggs[`${id}_histogram`] = { - histogram: { - field: fieldName, - interval: aggIntervals[id].interval !== 0 ? aggIntervals[id].interval : 1, - }, - }; - } - } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) { - if (fieldType === KBN_FIELD_TYPES.STRING) { - aggs[`${id}_cardinality`] = { - cardinality: { - field: fieldName, - }, - }; - } - aggs[`${id}_terms`] = { - terms: { - field: fieldName, - size: MAX_CHART_COLUMNS, - }, - }; - } - return aggs; - }, {} as Record); - - if (Object.keys(chartDataAggs).length === 0) { - return []; - } - - const respChartsData = await this.callAsCurrentUser('search', { - index: indexPatternTitle, - size: 0, - body: { - query, - aggs: buildSamplerAggregation(chartDataAggs, samplerShardSize), - size: 0, - }, - }); - - const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); - const aggregations = - aggsPath.length > 0 - ? _.get(respChartsData.aggregations, aggsPath) - : respChartsData.aggregations; - - const chartsData: ChartData[] = fields.map( - (field): ChartData => { - const fieldName = field.fieldName; - const fieldType = field.type; - const id = stringHash(field.fieldName); - - if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) { - if (aggIntervals[id] === undefined) { - return { - type: 'numeric', - data: [], - interval: 0, - stats: [0, 0], - id: fieldName, - }; - } - - return { - data: aggregations[`${id}_histogram`].buckets, - interval: aggIntervals[id].interval, - stats: [aggIntervals[id].min, aggIntervals[id].max], - type: 'numeric', - id: fieldName, - }; - } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) { - return { - type: fieldType === KBN_FIELD_TYPES.STRING ? 'ordinal' : 'boolean', - cardinality: - fieldType === KBN_FIELD_TYPES.STRING ? aggregations[`${id}_cardinality`].value : 2, - data: aggregations[`${id}_terms`].buckets, - id: fieldName, - }; - } - - return { - type: 'unsupported', - id: fieldName, - }; - } - ); - - return chartsData; } // Obtains statistics for supplied list of fields. The statistics for each field in the diff --git a/x-pack/plugins/ml/server/models/data_visualizer/index.ts b/x-pack/plugins/ml/server/models/data_visualizer/index.ts index ed44e9b12e1d1..ca1df0fe8300c 100644 --- a/x-pack/plugins/ml/server/models/data_visualizer/index.ts +++ b/x-pack/plugins/ml/server/models/data_visualizer/index.ts @@ -4,4 +4,4 @@ * you may not use this file except in compliance with the Elastic License. */ -export { DataVisualizer } from './data_visualizer'; +export { getHistogramsForFields, DataVisualizer } from './data_visualizer'; diff --git a/x-pack/plugins/ml/server/shared.ts b/x-pack/plugins/ml/server/shared.ts index 3fca8ea1ba047..100433b23f7d1 100644 --- a/x-pack/plugins/ml/server/shared.ts +++ b/x-pack/plugins/ml/server/shared.ts @@ -8,3 +8,4 @@ export * from '../common/types/anomalies'; export * from '../common/types/anomaly_detection_jobs'; export * from './lib/capabilities/errors'; export { ModuleSetupPayload } from './shared_services/providers/modules'; +export { getHistogramsForFields } from './models/data_visualizer/'; diff --git a/x-pack/plugins/transform/server/routes/api/field_histograms.ts b/x-pack/plugins/transform/server/routes/api/field_histograms.ts new file mode 100644 index 0000000000000..732e5e072fe84 --- /dev/null +++ b/x-pack/plugins/transform/server/routes/api/field_histograms.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { wrapEsError } from '../../../../../legacy/server/lib/create_router/error_wrappers'; + +import { getHistogramsForFields } from '../../shared_imports'; +import { RouteDependencies } from '../../types'; + +import { addBasePath } from '../index'; + +import { wrapError } from './error_utils'; +import { fieldHistogramsSchema, indexPatternTitleSchema, IndexPatternTitleSchema } from './schema'; + +export function registerFieldHistogramsRoutes({ router, license }: RouteDependencies) { + router.get( + { + path: addBasePath('transforms/field_histograms/{indexPatternTitle}'), + validate: { + params: indexPatternTitleSchema, + body: fieldHistogramsSchema, + }, + }, + license.guardApiRoute(async (ctx, req, res) => { + const { indexPatternTitle } = req.params as IndexPatternTitleSchema; + const { query, fields, samplerShardSize } = req.body; + + try { + const resp = await getHistogramsForFields( + ctx.transform!.dataClient.callAsCurrentUser, + indexPatternTitle, + query, + fields, + samplerShardSize + ); + + return res.ok({ body: resp }); + } catch (e) { + return res.customError(wrapError(wrapEsError(e))); + } + }) + ); +} diff --git a/x-pack/plugins/transform/server/routes/api/schema.ts b/x-pack/plugins/transform/server/routes/api/schema.ts index 7da3f1ccfe55e..152ccd9128621 100644 --- a/x-pack/plugins/transform/server/routes/api/schema.ts +++ b/x-pack/plugins/transform/server/routes/api/schema.ts @@ -5,6 +5,23 @@ */ import { schema } from '@kbn/config-schema'; +export const fieldHistogramsSchema = schema.object({ + /** Query to match documents in the index. */ + query: schema.any(), + fields: schema.arrayOf(schema.any()), + /** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */ + samplerShardSize: schema.number(), +}); + +export const indexPatternTitleSchema = schema.object({ + /** Title of the index pattern for which to return stats. */ + indexPatternTitle: schema.string(), +}); + +export interface IndexPatternTitleSchema { + indexPatternTitle: string; +} + export const schemaTransformId = { params: schema.object({ transformId: schema.string(), diff --git a/x-pack/plugins/transform/server/shared_imports.ts b/x-pack/plugins/transform/server/shared_imports.ts new file mode 100644 index 0000000000000..d1f86ac375721 --- /dev/null +++ b/x-pack/plugins/transform/server/shared_imports.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { getHistogramsForFields } from '../../ml/server'; From 3306ff8938eb1f23a900608ffeabc22cbfa3cb47 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 14 Jul 2020 09:58:02 +0200 Subject: [PATCH 11/15] [ML] Fix API endpoint for transform. --- x-pack/plugins/ml/public/shared.ts | 3 -- x-pack/plugins/transform/public/app/app.tsx | 7 ----- .../transform/public/app/hooks/use_api.ts | 30 +++++++++++++++++++ .../public/app/hooks/use_index_data.ts | 5 ++-- .../transform/public/shared_imports.ts | 3 -- .../server/routes/api/field_histograms.ts | 4 +-- .../plugins/transform/server/routes/index.ts | 2 ++ 7 files changed, 36 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/ml/public/shared.ts b/x-pack/plugins/ml/public/shared.ts index 87daa2d2c1790..4b1d7ee733dcf 100644 --- a/x-pack/plugins/ml/public/shared.ts +++ b/x-pack/plugins/ml/public/shared.ts @@ -20,6 +20,3 @@ export * from './application/formatters/metric_change_description'; export * from './application/components/data_grid'; export * from './application/data_frame_analytics/common'; export * from './application/util/date_utils'; - -export { DataLoader } from './application/datavisualizer/index_based/data_loader'; -export { clearCache, setDependencyCache } from './application/util/dependency_cache'; diff --git a/x-pack/plugins/transform/public/app/app.tsx b/x-pack/plugins/transform/public/app/app.tsx index 953417568aba3..f2fa4adf841d2 100644 --- a/x-pack/plugins/transform/public/app/app.tsx +++ b/x-pack/plugins/transform/public/app/app.tsx @@ -15,8 +15,6 @@ import { KibanaContextProvider } from '../../../../../src/plugins/kibana_react/p import { API_BASE_PATH } from '../../common/constants'; -import { clearCache, setDependencyCache } from '../shared_imports'; - import { SectionError } from './components'; import { SECTION_SLUG } from './constants'; import { AuthorizationContext, AuthorizationProvider } from './lib/authorization'; @@ -62,11 +60,6 @@ export const App: FC<{ history: ScopedHistory }> = ({ history }) => { }; export const renderApp = (element: HTMLElement, appDependencies: AppDependencies) => { - // TODO Temporary fix to make the Data Grid Histograms in the Transforms Wizard work - setDependencyCache({ - http: appDependencies.http, - }); - const I18nContext = appDependencies.i18n.Context; render( diff --git a/x-pack/plugins/transform/public/app/hooks/use_api.ts b/x-pack/plugins/transform/public/app/hooks/use_api.ts index 56528370a3ab9..576000d31fc8b 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_api.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_api.ts @@ -5,6 +5,9 @@ */ import { useMemo } from 'react'; + +import { KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/public'; + import { TransformId, TransformEndpointRequest, @@ -17,6 +20,15 @@ import { useAppDependencies } from '../app_dependencies'; import { GetTransformsResponse, PreviewRequestBody } from '../common'; import { EsIndex } from './use_api_types'; +import { SavedSearchQuery } from './use_search_items'; + +// Default sampler shard size used for field histograms +export const DEFAULT_SAMPLER_SHARD_SIZE = 5000; + +export interface FieldHistogramRequestConfig { + fieldName: string; + type?: KBN_FIELD_TYPES; +} export const useApi = () => { const { http } = useAppDependencies(); @@ -85,6 +97,24 @@ export const useApi = () => { getIndices(): Promise { return http.get(`/api/index_management/indices`); }, + getHistogramsForFields( + indexPatternTitle: string, + fields: FieldHistogramRequestConfig[], + query: string | SavedSearchQuery, + samplerShardSize = DEFAULT_SAMPLER_SHARD_SIZE + ) { + const body = JSON.stringify({ + query, + fields, + samplerShardSize, + }); + + return http.post({ + path: `${API_BASE_PATH}field_histograms/${indexPatternTitle}`, + method: 'POST', + body, + }); + }, }), [http] ); diff --git a/x-pack/plugins/transform/public/app/hooks/use_index_data.ts b/x-pack/plugins/transform/public/app/hooks/use_index_data.ts index 1642b49d6815d..ad5850f26be2e 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_index_data.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_index_data.ts @@ -16,7 +16,6 @@ import { showDataGridColumnChartErrorMessageToast, useDataGrid, useRenderCellValue, - DataLoader, EsSorting, SearchResponse7, UseIndexDataReturnType, @@ -108,8 +107,8 @@ export const useIndexData = ( const fetchColumnChartsData = async function () { try { - const dataLoader = new DataLoader(indexPattern, toastNotifications); - const columnChartsData = await dataLoader.loadFieldHistograms( + const columnChartsData = await api.getHistogramsForFields( + indexPattern.title, columns .filter((cT) => dataGrid.visibleColumns.includes(cT.id)) .map((cT) => ({ diff --git a/x-pack/plugins/transform/public/shared_imports.ts b/x-pack/plugins/transform/public/shared_imports.ts index b5f6930cebb33..abbc39dd6c728 100644 --- a/x-pack/plugins/transform/public/shared_imports.ts +++ b/x-pack/plugins/transform/public/shared_imports.ts @@ -14,7 +14,6 @@ export { } from '../../../../src/plugins/es_ui_shared/public'; export { - clearCache, getFieldType, getErrorMessage, extractErrorMessage, @@ -22,13 +21,11 @@ export { getDataGridSchemaFromKibanaFieldType, getFieldsFromKibanaIndexPattern, multiColumnSortFactory, - setDependencyCache, showDataGridColumnChartErrorMessageToast, useDataGrid, useRenderCellValue, ChartData, DataGrid, - DataLoader, EsSorting, RenderCellValue, SearchResponse7, diff --git a/x-pack/plugins/transform/server/routes/api/field_histograms.ts b/x-pack/plugins/transform/server/routes/api/field_histograms.ts index 732e5e072fe84..d602e49338846 100644 --- a/x-pack/plugins/transform/server/routes/api/field_histograms.ts +++ b/x-pack/plugins/transform/server/routes/api/field_histograms.ts @@ -20,9 +20,9 @@ import { wrapError } from './error_utils'; import { fieldHistogramsSchema, indexPatternTitleSchema, IndexPatternTitleSchema } from './schema'; export function registerFieldHistogramsRoutes({ router, license }: RouteDependencies) { - router.get( + router.post( { - path: addBasePath('transforms/field_histograms/{indexPatternTitle}'), + path: addBasePath('field_histograms/{indexPatternTitle}'), validate: { params: indexPatternTitleSchema, body: fieldHistogramsSchema, diff --git a/x-pack/plugins/transform/server/routes/index.ts b/x-pack/plugins/transform/server/routes/index.ts index 07c21e58e64e4..4f35b094017a4 100644 --- a/x-pack/plugins/transform/server/routes/index.ts +++ b/x-pack/plugins/transform/server/routes/index.ts @@ -6,6 +6,7 @@ import { RouteDependencies } from '../types'; +import { registerFieldHistogramsRoutes } from './api/field_histograms'; import { registerPrivilegesRoute } from './api/privileges'; import { registerTransformsRoutes } from './api/transforms'; @@ -15,6 +16,7 @@ export const addBasePath = (uri: string): string => `${API_BASE_PATH}${uri}`; export class ApiRoutes { setup(dependencies: RouteDependencies) { + registerFieldHistogramsRoutes(dependencies); registerPrivilegesRoute(dependencies); registerTransformsRoutes(dependencies); } From 8eedef2bd990acabaf559eb578d88602156c1936 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 14 Jul 2020 10:23:32 +0200 Subject: [PATCH 12/15] [ML] e2e tests for outlier detection histogram charts. --- x-pack/plugins/transform/public/app/app.tsx | 1 - .../outlier_detection_creation.ts | 22 ++++++++ .../ml/data_frame_analytics_creation.ts | 52 +++++++++++++++++++ 3 files changed, 74 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/transform/public/app/app.tsx b/x-pack/plugins/transform/public/app/app.tsx index f2fa4adf841d2..ccfdc8b0942fa 100644 --- a/x-pack/plugins/transform/public/app/app.tsx +++ b/x-pack/plugins/transform/public/app/app.tsx @@ -75,6 +75,5 @@ export const renderApp = (element: HTMLElement, appDependencies: AppDependencies return () => { unmountComponentAtNode(element); - clearCache(); }; }; diff --git a/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts b/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts index 6cdb9caa1e2db..4ae93296f9be0 100644 --- a/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts +++ b/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts @@ -37,6 +37,18 @@ export default function ({ getService }: FtrProviderContext) { modelMemory: '5mb', createIndexPattern: true, expected: { + histogramCharts: [ + { chartAvailable: true, id: '1stFlrSF', legend: '334 - 4692' }, + { chartAvailable: true, id: 'BsmtFinSF1', legend: '0 - 5644' }, + { chartAvailable: true, id: 'BsmtQual', legend: '0 - 5' }, + { chartAvailable: true, id: 'CentralAir', legend: '2 categories' }, + { chartAvailable: true, id: 'Condition2', legend: '2 categories' }, + { chartAvailable: true, id: 'Electrical', legend: '2 categories' }, + { chartAvailable: true, id: 'ExterQual', legend: '1 - 4' }, + { chartAvailable: true, id: 'Exterior1st', legend: '2 categories' }, + { chartAvailable: true, id: 'Exterior2nd', legend: '3 categories' }, + { chartAvailable: true, id: 'Fireplaces', legend: '0 - 3' }, + ], row: { type: 'outlier_detection', status: 'stopped', @@ -84,6 +96,16 @@ export default function ({ getService }: FtrProviderContext) { await ml.dataFrameAnalyticsCreation.assertSourceDataPreviewExists(); }); + it('enables the source data preview histogram charts', async () => { + await ml.dataFrameAnalyticsCreation.enableSourceDataPreviewHistogramCharts(); + }); + + it('displays the source data preview histogram charts', async () => { + await ml.dataFrameAnalyticsCreation.assertSourceDataPreviewHistogramCharts( + testData.expected.histogramCharts + ); + }); + it('displays the include fields selection', async () => { await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists(); }); diff --git a/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts b/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts index 1b756bbaca5d8..fc4aaa4fbf5fd 100644 --- a/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts +++ b/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts @@ -128,6 +128,58 @@ export function MachineLearningDataFrameAnalyticsCreationProvider( await testSubjects.existOrFail('mlAnalyticsCreationDataGrid loaded', { timeout: 5000 }); }, + async assertIndexPreviewHistogramChartButtonExists() { + await testSubjects.existOrFail('mlAnalyticsCreationDataGridHistogramButton'); + }, + + async enableSourceDataPreviewHistogramCharts() { + await this.assertSourceDataPreviewHistogramChartButtonCheckState(false); + await testSubjects.click('mlAnalyticsCreationDataGridHistogramButton'); + await this.assertSourceDataPreviewHistogramChartButtonCheckState(true); + }, + + async assertSourceDataPreviewHistogramChartButtonCheckState(expectedCheckState: boolean) { + const actualCheckState = + (await testSubjects.getAttribute( + 'mlAnalyticsCreationDataGridHistogramButton', + 'aria-checked' + )) === 'true'; + expect(actualCheckState).to.eql( + expectedCheckState, + `Chart histogram button check state should be '${expectedCheckState}' (got '${actualCheckState}')` + ); + }, + + async assertSourceDataPreviewHistogramCharts( + expectedHistogramCharts: Array<{ chartAvailable: boolean; id: string; legend: string }> + ) { + // For each chart, get the content of each header cell and assert + // the legend text and column id and if the chart should be present or not. + await retry.tryForTime(5000, async () => { + for (const [index, expected] of expectedHistogramCharts.entries()) { + await testSubjects.existOrFail(`mlDataGridChart-${index}`); + + if (expected.chartAvailable) { + await testSubjects.existOrFail(`mlDataGridChart-${index}-histogram`); + } else { + await testSubjects.missingOrFail(`mlDataGridChart-${index}-histogram`); + } + + const actualLegend = await testSubjects.getVisibleText(`mlDataGridChart-${index}-legend`); + expect(actualLegend).to.eql( + expected.legend, + `Legend text for column '${index}' should be '${expected.legend}' (got '${actualLegend}')` + ); + + const actualId = await testSubjects.getVisibleText(`mlDataGridChart-${index}-id`); + expect(actualId).to.eql( + expected.id, + `Id text for column '${index}' should be '${expected.id}' (got '${actualId}')` + ); + } + }); + }, + async assertIncludeFieldsSelectionExists() { await testSubjects.existOrFail('mlAnalyticsCreateJobWizardIncludesSelect', { timeout: 5000 }); }, From b794e97124ba622040eed3522e4ef01c83e889c6 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 14 Jul 2020 10:26:08 +0200 Subject: [PATCH 13/15] [ML] schema comments. --- .../plugins/ml/server/routes/schemas/data_visualizer_schema.ts | 1 + x-pack/plugins/transform/server/routes/api/schema.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/x-pack/plugins/ml/server/routes/schemas/data_visualizer_schema.ts b/x-pack/plugins/ml/server/routes/schemas/data_visualizer_schema.ts index 14bc711886315..24e45514e1efc 100644 --- a/x-pack/plugins/ml/server/routes/schemas/data_visualizer_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/data_visualizer_schema.ts @@ -14,6 +14,7 @@ export const indexPatternTitleSchema = schema.object({ export const dataVisualizerFieldHistogramsSchema = schema.object({ /** Query to match documents in the index. */ query: schema.any(), + /** The fields to return histogram data. */ fields: schema.arrayOf(schema.any()), /** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */ samplerShardSize: schema.number(), diff --git a/x-pack/plugins/transform/server/routes/api/schema.ts b/x-pack/plugins/transform/server/routes/api/schema.ts index 152ccd9128621..8aadef81b221b 100644 --- a/x-pack/plugins/transform/server/routes/api/schema.ts +++ b/x-pack/plugins/transform/server/routes/api/schema.ts @@ -8,6 +8,7 @@ import { schema } from '@kbn/config-schema'; export const fieldHistogramsSchema = schema.object({ /** Query to match documents in the index. */ query: schema.any(), + /** The fields to return histogram data. */ fields: schema.arrayOf(schema.any()), /** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */ samplerShardSize: schema.number(), From 3883477b2535cca78ce989c40dea774d5d00ee53 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 14 Jul 2020 10:35:36 +0200 Subject: [PATCH 14/15] [ML] Use useMemo() for DataLoader. --- .../analytics_creation/hooks/use_index_data.ts | 7 +++++-- .../use_exploration_results.ts | 11 ++++++++--- .../outlier_exploration/use_outlier_data.ts | 13 ++++++++++--- .../application/datavisualizer/index_based/page.tsx | 7 +++++-- 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts index 7ae462d5c9c20..2cecffc993257 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useEffect } from 'react'; +import { useEffect, useMemo } from 'react'; import { EuiDataGridColumn } from '@elastic/eui'; @@ -106,9 +106,12 @@ export const useIndexData = ( // eslint-disable-next-line react-hooks/exhaustive-deps }, [indexPattern.title, JSON.stringify([query, pagination, sortingColumns])]); + const dataLoader = useMemo(() => new DataLoader(indexPattern, toastNotifications), [ + indexPattern, + ]); + const fetchColumnChartsData = async function () { try { - const dataLoader = new DataLoader(indexPattern, toastNotifications); const columnChartsData = await dataLoader.loadFieldHistograms( columns .filter((cT) => dataGrid.visibleColumns.includes(cT.id)) diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts index c1d73e4d19305..98dd40986e32b 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useEffect } from 'react'; +import { useEffect, useMemo } from 'react'; import { EuiDataGridColumn } from '@elastic/eui'; @@ -73,10 +73,15 @@ export const useExplorationResults = ( // eslint-disable-next-line react-hooks/exhaustive-deps }, [jobConfig && jobConfig.id, dataGrid.pagination, searchQuery, dataGrid.sortingColumns]); + const dataLoader = useMemo( + () => + indexPattern !== undefined ? new DataLoader(indexPattern, toastNotifications) : undefined, + [indexPattern] + ); + const fetchColumnChartsData = async function () { try { - if (jobConfig !== undefined && indexPattern !== undefined) { - const dataLoader = new DataLoader(indexPattern, toastNotifications); + if (jobConfig !== undefined && dataLoader !== undefined) { const columnChartsData = await dataLoader.loadFieldHistograms( columns .filter((cT) => dataGrid.visibleColumns.includes(cT.id)) diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts index 68acf14eea957..90294a09c0adc 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useEffect } from 'react'; +import { useEffect, useMemo } from 'react'; import { EuiDataGridColumn } from '@elastic/eui'; @@ -80,10 +80,17 @@ export const useOutlierData = ( // eslint-disable-next-line react-hooks/exhaustive-deps }, [jobConfig && jobConfig.id, dataGrid.pagination, searchQuery, dataGrid.sortingColumns]); + const dataLoader = useMemo( + () => + indexPattern !== undefined + ? new DataLoader(indexPattern, getToastNotifications()) + : undefined, + [indexPattern] + ); + const fetchColumnChartsData = async function () { try { - if (jobConfig !== undefined && indexPattern !== undefined) { - const dataLoader = new DataLoader(indexPattern, getToastNotifications()); + if (jobConfig !== undefined && dataLoader !== undefined) { const columnChartsData = await dataLoader.loadFieldHistograms( columns .filter((cT) => dataGrid.visibleColumns.includes(cT.id)) diff --git a/x-pack/plugins/ml/public/application/datavisualizer/index_based/page.tsx b/x-pack/plugins/ml/public/application/datavisualizer/index_based/page.tsx index 301cdec2f788b..3c332d305d7e9 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/index_based/page.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/index_based/page.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { FC, Fragment, useEffect, useState } from 'react'; +import React, { FC, Fragment, useEffect, useMemo, useState } from 'react'; import { merge } from 'rxjs'; import { i18n } from '@kbn/i18n'; @@ -108,7 +108,10 @@ export const Page: FC = () => { autoRefreshSelector: true, }); - const dataLoader = new DataLoader(currentIndexPattern, getToastNotifications()); + const dataLoader = useMemo(() => new DataLoader(currentIndexPattern, getToastNotifications()), [ + currentIndexPattern, + ]); + const [globalState, setGlobalState] = useUrlState('_g'); useEffect(() => { if (globalState?.time !== undefined) { From 5b2ecdc1a491e2ce37788259f80eb4e99637b3f9 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 14 Jul 2020 11:06:45 +0200 Subject: [PATCH 15/15] [ML] Fix API code structure. --- .../transform/public/app/hooks/use_api.ts | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/transform/public/app/hooks/use_api.ts b/x-pack/plugins/transform/public/app/hooks/use_api.ts index 576000d31fc8b..1d2752b9e939d 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_api.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_api.ts @@ -103,16 +103,12 @@ export const useApi = () => { query: string | SavedSearchQuery, samplerShardSize = DEFAULT_SAMPLER_SHARD_SIZE ) { - const body = JSON.stringify({ - query, - fields, - samplerShardSize, - }); - - return http.post({ - path: `${API_BASE_PATH}field_histograms/${indexPatternTitle}`, - method: 'POST', - body, + return http.post(`${API_BASE_PATH}field_histograms/${indexPatternTitle}`, { + body: JSON.stringify({ + query, + fields, + samplerShardSize, + }), }); }, }),