From 64f6670128be9cb0a11eee02f4d4ffcb5eb06467 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Mon, 20 Jan 2020 14:46:24 +0000 Subject: [PATCH] [ML] Improving job wizards with datafeed aggregations (#55180) (#55313) * [ML] Improving job wizards with datafeed aggregations * picking all agg keys for fields * function move and rename --- .../ml/common/constants/field_types.ts | 1 + .../job_creator/advanced_job_creator.ts | 2 +- .../job_creator/categorization_job_creator.ts | 2 +- .../new_job/common/job_creator/job_creator.ts | 19 ++++- .../job_creator/multi_metric_job_creator.ts | 2 +- .../job_creator/population_job_creator.ts | 2 +- .../job_creator/single_metric_job_creator.ts | 2 +- .../common/job_creator/util/general.ts | 72 ++++++++++++++----- .../time_field/time_field_select.tsx | 6 +- .../advanced_detector_modal.tsx | 31 ++++---- .../categorization_field_select.tsx | 8 +-- .../influencers/influencers_select.tsx | 4 +- .../summary_count_field_select.tsx | 6 +- 13 files changed, 104 insertions(+), 53 deletions(-) diff --git a/x-pack/legacy/plugins/ml/common/constants/field_types.ts b/x-pack/legacy/plugins/ml/common/constants/field_types.ts index 6c9af703c148a..9402e4c20e46f 100644 --- a/x-pack/legacy/plugins/ml/common/constants/field_types.ts +++ b/x-pack/legacy/plugins/ml/common/constants/field_types.ts @@ -16,3 +16,4 @@ export enum ML_JOB_FIELD_TYPES { } export const MLCATEGORY = 'mlcategory'; +export const DOC_COUNT = 'doc_count'; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/advanced_job_creator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/advanced_job_creator.ts index b0eb1b98cd02b..4530c00c10535 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/advanced_job_creator.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/advanced_job_creator.ts @@ -183,7 +183,7 @@ export class AdvancedJobCreator extends JobCreator { public cloneFromExistingJob(job: Job, datafeed: Datafeed) { this._overrideConfigs(job, datafeed); - const detectors = getRichDetectors(job, datafeed, this.scriptFields, true); + const detectors = getRichDetectors(job, datafeed, this.additionalFields, true); // keep track of the custom rules for each detector const customRules = this._detectors.map(d => d.custom_rules); diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts index 7c070ccc6bc53..71619311c4361 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts @@ -140,7 +140,7 @@ export class CategorizationJobCreator extends JobCreator { public cloneFromExistingJob(job: Job, datafeed: Datafeed) { this._overrideConfigs(job, datafeed); this.createdBy = CREATED_BY_LABEL.CATEGORIZATION; - const detectors = getRichDetectors(job, datafeed, this.scriptFields, false); + const detectors = getRichDetectors(job, datafeed, this.additionalFields, false); const dtr = detectors[0]; if (detectors.length && dtr.agg !== null && dtr.field !== null) { diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts index 513c8239db01e..90c189c9d6197 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts @@ -19,7 +19,7 @@ import { CREATED_BY_LABEL, SHARED_RESULTS_INDEX_NAME, } from '../../../../../../common/constants/new_job'; -import { isSparseDataJob } from './util/general'; +import { isSparseDataJob, collectAggs } from './util/general'; import { parseInterval } from '../../../../../../common/util/parse_interval'; import { Calendar } from '../../../../../../common/types/calendars'; import { mlCalendarService } from '../../../../services/calendar_service'; @@ -43,6 +43,7 @@ export class JobCreator { protected _aggs: Aggregation[] = []; protected _fields: Field[] = []; protected _scriptFields: Field[] = []; + protected _aggregationFields: Field[] = []; protected _sparseData: boolean = false; private _stopAllRefreshPolls: { stop: boolean; @@ -450,6 +451,14 @@ export class JobCreator { return this._scriptFields; } + public get aggregationFields(): Field[] { + return this._aggregationFields; + } + + public get additionalFields(): Field[] { + return [...this._scriptFields, ...this._aggregationFields]; + } + public get subscribers(): ProgressSubscriber[] { return this._subscribers; } @@ -603,6 +612,7 @@ export class JobCreator { } this._sparseData = isSparseDataJob(job, datafeed); + this._scriptFields = []; if (this._datafeed_config.script_fields !== undefined) { this._scriptFields = Object.keys(this._datafeed_config.script_fields).map(f => ({ id: f, @@ -610,8 +620,11 @@ export class JobCreator { type: ES_FIELD_TYPES.KEYWORD, aggregatable: true, })); - } else { - this._scriptFields = []; + } + + this._aggregationFields = []; + if (this._datafeed_config.aggregations?.buckets !== undefined) { + collectAggs(this._datafeed_config.aggregations.buckets, this._aggregationFields); } } } diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/multi_metric_job_creator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/multi_metric_job_creator.ts index 8a4411bf9025f..7c5fba028d9e8 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/multi_metric_job_creator.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/multi_metric_job_creator.ts @@ -153,7 +153,7 @@ export class MultiMetricJobCreator extends JobCreator { public cloneFromExistingJob(job: Job, datafeed: Datafeed) { this._overrideConfigs(job, datafeed); this.createdBy = CREATED_BY_LABEL.MULTI_METRIC; - const detectors = getRichDetectors(job, datafeed, this.scriptFields, false); + const detectors = getRichDetectors(job, datafeed, this.additionalFields, false); if (datafeed.aggregations !== undefined) { // if we've converting from a single metric job, diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/population_job_creator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/population_job_creator.ts index 9300e53c578e1..3009d68ca67ca 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/population_job_creator.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/population_job_creator.ts @@ -135,7 +135,7 @@ export class PopulationJobCreator extends JobCreator { public cloneFromExistingJob(job: Job, datafeed: Datafeed) { this._overrideConfigs(job, datafeed); this.createdBy = CREATED_BY_LABEL.POPULATION; - const detectors = getRichDetectors(job, datafeed, this.scriptFields, false); + const detectors = getRichDetectors(job, datafeed, this.additionalFields, false); this.removeAllDetectors(); diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/single_metric_job_creator.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/single_metric_job_creator.ts index f98fd4dbe970a..9f3500185c2bf 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/single_metric_job_creator.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/single_metric_job_creator.ts @@ -190,7 +190,7 @@ export class SingleMetricJobCreator extends JobCreator { public cloneFromExistingJob(job: Job, datafeed: Datafeed) { this._overrideConfigs(job, datafeed); this.createdBy = CREATED_BY_LABEL.SINGLE_METRIC; - const detectors = getRichDetectors(job, datafeed, this.scriptFields, false); + const detectors = getRichDetectors(job, datafeed, this.additionalFields, false); this.removeAllDetectors(); diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts index 6443539a9877d..e5b6212a4326e 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts @@ -11,7 +11,8 @@ import { ML_JOB_AGGREGATION, SPARSE_DATA_AGGREGATIONS, } from '../../../../../../../common/constants/aggregation_types'; -import { MLCATEGORY } from '../../../../../../../common/constants/field_types'; +import { MLCATEGORY, DOC_COUNT } from '../../../../../../../common/constants/field_types'; +import { ES_FIELD_TYPES } from '../../../../../../../../../../../src/plugins/data/public'; import { EVENT_RATE_FIELD_ID, Field, @@ -27,14 +28,14 @@ import { } from '../index'; import { CREATED_BY_LABEL, JOB_TYPE } from '../../../../../../../common/constants/new_job'; -const getFieldByIdFactory = (scriptFields: Field[]) => (id: string) => { +const getFieldByIdFactory = (additionalFields: Field[]) => (id: string) => { let field = newJobCapsService.getFieldById(id); // if no field could be found it may be a pretend field, like mlcategory or a script field if (field === null) { if (id === MLCATEGORY) { field = mlCategory; - } else if (scriptFields.length) { - field = scriptFields.find(f => f.id === id) || null; + } else if (additionalFields.length) { + field = additionalFields.find(f => f.id === id) || null; } } return field; @@ -44,12 +45,12 @@ const getFieldByIdFactory = (scriptFields: Field[]) => (id: string) => { export function getRichDetectors( job: Job, datafeed: Datafeed, - scriptFields: Field[], + additionalFields: Field[], advanced: boolean = false ) { const detectors = advanced ? getDetectorsAdvanced(job, datafeed) : getDetectors(job, datafeed); - const getFieldById = getFieldByIdFactory(scriptFields); + const getFieldById = getFieldByIdFactory(additionalFields); return detectors.map(d => { let field = null; @@ -82,19 +83,19 @@ export function getRichDetectors( }); } -export function createFieldOptions(fields: Field[]) { - return fields - .filter(f => f.id !== EVENT_RATE_FIELD_ID) - .map(f => ({ - label: f.name, - })) - .sort((a, b) => a.label.localeCompare(b.label)); -} - -export function createScriptFieldOptions(scriptFields: Field[]) { - return scriptFields.map(f => ({ - label: f.id, - })); +export function createFieldOptions(fields: Field[], additionalFields: Field[]) { + return [ + ...fields + .filter(f => f.id !== EVENT_RATE_FIELD_ID) + .map(f => ({ + label: f.name, + })), + ...additionalFields + .filter(f => fields.some(f2 => f2.id === f.id) === false) + .map(f => ({ + label: f.id, + })), + ].sort((a, b) => a.label.localeCompare(b.label)); } export function createMlcategoryFieldOption(categorizationFieldName: string | null) { @@ -108,6 +109,16 @@ export function createMlcategoryFieldOption(categorizationFieldName: string | nu ]; } +export function createDocCountFieldOption(usingAggregations: boolean) { + return usingAggregations + ? [ + { + label: DOC_COUNT, + }, + ] + : []; +} + function getDetectorsAdvanced(job: Job, datafeed: Datafeed) { return processFieldlessAggs(job.analysis_config.detectors); } @@ -305,3 +316,26 @@ export function getJobCreatorTitle(jobCreator: JobCreatorType) { return ''; } } + +// recurse through a datafeed aggregation object, +// adding top level keys from each nested agg to an array +// of fields +export function collectAggs(o: any, aggFields: Field[]) { + for (const i in o) { + if (o[i] !== null && typeof o[i] === 'object') { + if (i === 'aggregations' || i === 'aggs') { + Object.keys(o[i]).forEach(k => { + if (k !== 'aggregations' && i !== 'aggs') { + aggFields.push({ + id: k, + name: k, + type: ES_FIELD_TYPES.KEYWORD, + aggregatable: true, + }); + } + }); + } + collectAggs(o[i], aggFields); + } + } +} diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/time_field/time_field_select.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/time_field/time_field_select.tsx index f2e2516866835..9af1226d1fe6c 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/time_field/time_field_select.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/time_field/time_field_select.tsx @@ -4,9 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { FC } from 'react'; +import React, { FC, useContext } from 'react'; import { EuiComboBox, EuiComboBoxOptionProps } from '@elastic/eui'; +import { JobCreatorContext } from '../../../job_creator_context'; import { Field } from '../../../../../../../../../common/types/fields'; import { createFieldOptions } from '../../../../../common/job_creator/util/general'; @@ -17,7 +18,8 @@ interface Props { } export const TimeFieldSelect: FC = ({ fields, changeHandler, selectedField }) => { - const options: EuiComboBoxOptionProps[] = createFieldOptions(fields); + const { jobCreator } = useContext(JobCreatorContext); + const options: EuiComboBoxOptionProps[] = createFieldOptions(fields, jobCreator.additionalFields); const selection: EuiComboBoxOptionProps[] = []; if (selectedField !== null) { diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_detector_modal/advanced_detector_modal.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_detector_modal/advanced_detector_modal.tsx index 06c8068a9c005..753cea7adcb35 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_detector_modal/advanced_detector_modal.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_detector_modal/advanced_detector_modal.tsx @@ -18,7 +18,6 @@ import { JobCreatorContext } from '../../../job_creator_context'; import { AdvancedJobCreator } from '../../../../../common/job_creator'; import { createFieldOptions, - createScriptFieldOptions, createMlcategoryFieldOption, } from '../../../../../common/job_creator/util/general'; import { @@ -88,7 +87,7 @@ export const AdvancedDetectorModal: FC = ({ const [fieldOptionEnabled, setFieldOptionEnabled] = useState(true); const { descriptionPlaceholder, setDescriptionPlaceholder } = useDetectorPlaceholder(detector); - const usingScriptFields = jobCreator.scriptFields.length > 0; + const usingScriptFields = jobCreator.additionalFields.length > 0; // list of aggregation combobox options. const aggOptions: EuiComboBoxOptionProps[] = aggs @@ -98,12 +97,12 @@ export const AdvancedDetectorModal: FC = ({ // fields available for the selected agg const { currentFieldOptions, setCurrentFieldOptions } = useCurrentFieldOptions( detector.agg, - jobCreator.scriptFields + jobCreator.additionalFields, + fields ); const allFieldOptions: EuiComboBoxOptionProps[] = [ - ...createFieldOptions(fields), - ...createScriptFieldOptions(jobCreator.scriptFields), + ...createFieldOptions(fields, jobCreator.additionalFields), ].sort(comboBoxOptionsSort); const splitFieldOptions: EuiComboBoxOptionProps[] = [ @@ -127,7 +126,9 @@ export const AdvancedDetectorModal: FC = ({ return mlCategory; } return ( - fields.find(f => f.id === title) || jobCreator.scriptFields.find(f => f.id === title) || null + fields.find(f => f.id === title) || + jobCreator.additionalFields.find(f => f.id === title) || + null ); } @@ -365,21 +366,27 @@ function useDetectorPlaceholder(detector: RichDetector) { } // creates list of combobox options based on an aggregation's field list -function createFieldOptionsFromAgg(agg: Aggregation | null) { - return createFieldOptions(agg !== null && agg.fields !== undefined ? agg.fields : []); +function createFieldOptionsFromAgg(agg: Aggregation | null, additionalFields: Field[]) { + return createFieldOptions( + agg !== null && agg.fields !== undefined ? agg.fields : [], + additionalFields + ); } // custom hook for storing combobox options based on an aggregation field list -function useCurrentFieldOptions(aggregation: Aggregation | null, scriptFields: Field[]) { +function useCurrentFieldOptions( + aggregation: Aggregation | null, + additionalFields: Field[], + fields: Field[] +) { const [currentFieldOptions, setCurrentFieldOptions] = useState( - createFieldOptionsFromAgg(aggregation) + createFieldOptionsFromAgg(aggregation, additionalFields) ); - const scriptFieldOptions = createScriptFieldOptions(scriptFields); return { currentFieldOptions, setCurrentFieldOptions: (agg: Aggregation | null) => - setCurrentFieldOptions([...createFieldOptionsFromAgg(agg), ...scriptFieldOptions]), + setCurrentFieldOptions(createFieldOptionsFromAgg(agg, additionalFields)), }; } diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field_select.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field_select.tsx index d995d40284aba..6451c2785eae0 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field_select.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field_select.tsx @@ -9,10 +9,7 @@ import { EuiComboBox, EuiComboBoxOptionProps } from '@elastic/eui'; import { JobCreatorContext } from '../../../job_creator_context'; import { Field } from '../../../../../../../../../common/types/fields'; -import { - createFieldOptions, - createScriptFieldOptions, -} from '../../../../../common/job_creator/util/general'; +import { createFieldOptions } from '../../../../../common/job_creator/util/general'; interface Props { fields: Field[]; @@ -23,8 +20,7 @@ interface Props { export const CategorizationFieldSelect: FC = ({ fields, changeHandler, selectedField }) => { const { jobCreator } = useContext(JobCreatorContext); const options: EuiComboBoxOptionProps[] = [ - ...createFieldOptions(fields), - ...createScriptFieldOptions(jobCreator.scriptFields), + ...createFieldOptions(fields, jobCreator.additionalFields), ]; const selection: EuiComboBoxOptionProps[] = []; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/influencers/influencers_select.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/influencers/influencers_select.tsx index 639bdb9ec76bf..d4ac470f4ea4f 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/influencers/influencers_select.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/influencers/influencers_select.tsx @@ -11,7 +11,6 @@ import { JobCreatorContext } from '../../../job_creator_context'; import { Field } from '../../../../../../../../../common/types/fields'; import { createFieldOptions, - createScriptFieldOptions, createMlcategoryFieldOption, } from '../../../../../common/job_creator/util/general'; @@ -24,8 +23,7 @@ interface Props { export const InfluencersSelect: FC = ({ fields, changeHandler, selectedInfluencers }) => { const { jobCreator } = useContext(JobCreatorContext); const options: EuiComboBoxOptionProps[] = [ - ...createFieldOptions(fields), - ...createScriptFieldOptions(jobCreator.scriptFields), + ...createFieldOptions(fields, jobCreator.additionalFields), ...createMlcategoryFieldOption(jobCreator.categorizationFieldName), ]; diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/summary_count_field_select.tsx b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/summary_count_field_select.tsx index efe32e3173cad..6fe3aaf0a8652 100644 --- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/summary_count_field_select.tsx +++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/summary_count_field_select.tsx @@ -11,7 +11,7 @@ import { JobCreatorContext } from '../../../job_creator_context'; import { Field } from '../../../../../../../../../common/types/fields'; import { createFieldOptions, - createScriptFieldOptions, + createDocCountFieldOption, } from '../../../../../common/job_creator/util/general'; interface Props { @@ -23,8 +23,8 @@ interface Props { export const SummaryCountFieldSelect: FC = ({ fields, changeHandler, selectedField }) => { const { jobCreator } = useContext(JobCreatorContext); const options: EuiComboBoxOptionProps[] = [ - ...createFieldOptions(fields), - ...createScriptFieldOptions(jobCreator.scriptFields), + ...createFieldOptions(fields, jobCreator.additionalFields), + ...createDocCountFieldOption(jobCreator.aggregationFields.length > 0), ]; const selection: EuiComboBoxOptionProps[] = [];