Skip to content

Commit

Permalink
[ML] Improving job wizards with datafeed aggregations (#55180) (#55313)
Browse files Browse the repository at this point in the history
* [ML] Improving job wizards with datafeed aggregations

* picking all agg keys for fields

* function move and rename
  • Loading branch information
jgowdyelastic authored Jan 20, 2020
1 parent 0d10f0e commit 64f6670
Show file tree
Hide file tree
Showing 13 changed files with 104 additions and 53 deletions.
1 change: 1 addition & 0 deletions x-pack/legacy/plugins/ml/common/constants/field_types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,4 @@ export enum ML_JOB_FIELD_TYPES {
}

export const MLCATEGORY = 'mlcategory';
export const DOC_COUNT = 'doc_count';
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ export class AdvancedJobCreator extends JobCreator {

public cloneFromExistingJob(job: Job, datafeed: Datafeed) {
this._overrideConfigs(job, datafeed);
const detectors = getRichDetectors(job, datafeed, this.scriptFields, true);
const detectors = getRichDetectors(job, datafeed, this.additionalFields, true);

// keep track of the custom rules for each detector
const customRules = this._detectors.map(d => d.custom_rules);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ export class CategorizationJobCreator extends JobCreator {
public cloneFromExistingJob(job: Job, datafeed: Datafeed) {
this._overrideConfigs(job, datafeed);
this.createdBy = CREATED_BY_LABEL.CATEGORIZATION;
const detectors = getRichDetectors(job, datafeed, this.scriptFields, false);
const detectors = getRichDetectors(job, datafeed, this.additionalFields, false);

const dtr = detectors[0];
if (detectors.length && dtr.agg !== null && dtr.field !== null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import {
CREATED_BY_LABEL,
SHARED_RESULTS_INDEX_NAME,
} from '../../../../../../common/constants/new_job';
import { isSparseDataJob } from './util/general';
import { isSparseDataJob, collectAggs } from './util/general';
import { parseInterval } from '../../../../../../common/util/parse_interval';
import { Calendar } from '../../../../../../common/types/calendars';
import { mlCalendarService } from '../../../../services/calendar_service';
Expand All @@ -43,6 +43,7 @@ export class JobCreator {
protected _aggs: Aggregation[] = [];
protected _fields: Field[] = [];
protected _scriptFields: Field[] = [];
protected _aggregationFields: Field[] = [];
protected _sparseData: boolean = false;
private _stopAllRefreshPolls: {
stop: boolean;
Expand Down Expand Up @@ -450,6 +451,14 @@ export class JobCreator {
return this._scriptFields;
}

public get aggregationFields(): Field[] {
return this._aggregationFields;
}

public get additionalFields(): Field[] {
return [...this._scriptFields, ...this._aggregationFields];
}

public get subscribers(): ProgressSubscriber[] {
return this._subscribers;
}
Expand Down Expand Up @@ -603,15 +612,19 @@ export class JobCreator {
}
this._sparseData = isSparseDataJob(job, datafeed);

this._scriptFields = [];
if (this._datafeed_config.script_fields !== undefined) {
this._scriptFields = Object.keys(this._datafeed_config.script_fields).map(f => ({
id: f,
name: f,
type: ES_FIELD_TYPES.KEYWORD,
aggregatable: true,
}));
} else {
this._scriptFields = [];
}

this._aggregationFields = [];
if (this._datafeed_config.aggregations?.buckets !== undefined) {
collectAggs(this._datafeed_config.aggregations.buckets, this._aggregationFields);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ export class MultiMetricJobCreator extends JobCreator {
public cloneFromExistingJob(job: Job, datafeed: Datafeed) {
this._overrideConfigs(job, datafeed);
this.createdBy = CREATED_BY_LABEL.MULTI_METRIC;
const detectors = getRichDetectors(job, datafeed, this.scriptFields, false);
const detectors = getRichDetectors(job, datafeed, this.additionalFields, false);

if (datafeed.aggregations !== undefined) {
// if we've converting from a single metric job,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ export class PopulationJobCreator extends JobCreator {
public cloneFromExistingJob(job: Job, datafeed: Datafeed) {
this._overrideConfigs(job, datafeed);
this.createdBy = CREATED_BY_LABEL.POPULATION;
const detectors = getRichDetectors(job, datafeed, this.scriptFields, false);
const detectors = getRichDetectors(job, datafeed, this.additionalFields, false);

this.removeAllDetectors();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ export class SingleMetricJobCreator extends JobCreator {
public cloneFromExistingJob(job: Job, datafeed: Datafeed) {
this._overrideConfigs(job, datafeed);
this.createdBy = CREATED_BY_LABEL.SINGLE_METRIC;
const detectors = getRichDetectors(job, datafeed, this.scriptFields, false);
const detectors = getRichDetectors(job, datafeed, this.additionalFields, false);

this.removeAllDetectors();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ import {
ML_JOB_AGGREGATION,
SPARSE_DATA_AGGREGATIONS,
} from '../../../../../../../common/constants/aggregation_types';
import { MLCATEGORY } from '../../../../../../../common/constants/field_types';
import { MLCATEGORY, DOC_COUNT } from '../../../../../../../common/constants/field_types';
import { ES_FIELD_TYPES } from '../../../../../../../../../../../src/plugins/data/public';
import {
EVENT_RATE_FIELD_ID,
Field,
Expand All @@ -27,14 +28,14 @@ import {
} from '../index';
import { CREATED_BY_LABEL, JOB_TYPE } from '../../../../../../../common/constants/new_job';

const getFieldByIdFactory = (scriptFields: Field[]) => (id: string) => {
const getFieldByIdFactory = (additionalFields: Field[]) => (id: string) => {
let field = newJobCapsService.getFieldById(id);
// if no field could be found it may be a pretend field, like mlcategory or a script field
if (field === null) {
if (id === MLCATEGORY) {
field = mlCategory;
} else if (scriptFields.length) {
field = scriptFields.find(f => f.id === id) || null;
} else if (additionalFields.length) {
field = additionalFields.find(f => f.id === id) || null;
}
}
return field;
Expand All @@ -44,12 +45,12 @@ const getFieldByIdFactory = (scriptFields: Field[]) => (id: string) => {
export function getRichDetectors(
job: Job,
datafeed: Datafeed,
scriptFields: Field[],
additionalFields: Field[],
advanced: boolean = false
) {
const detectors = advanced ? getDetectorsAdvanced(job, datafeed) : getDetectors(job, datafeed);

const getFieldById = getFieldByIdFactory(scriptFields);
const getFieldById = getFieldByIdFactory(additionalFields);

return detectors.map(d => {
let field = null;
Expand Down Expand Up @@ -82,19 +83,19 @@ export function getRichDetectors(
});
}

export function createFieldOptions(fields: Field[]) {
return fields
.filter(f => f.id !== EVENT_RATE_FIELD_ID)
.map(f => ({
label: f.name,
}))
.sort((a, b) => a.label.localeCompare(b.label));
}

export function createScriptFieldOptions(scriptFields: Field[]) {
return scriptFields.map(f => ({
label: f.id,
}));
export function createFieldOptions(fields: Field[], additionalFields: Field[]) {
return [
...fields
.filter(f => f.id !== EVENT_RATE_FIELD_ID)
.map(f => ({
label: f.name,
})),
...additionalFields
.filter(f => fields.some(f2 => f2.id === f.id) === false)
.map(f => ({
label: f.id,
})),
].sort((a, b) => a.label.localeCompare(b.label));
}

export function createMlcategoryFieldOption(categorizationFieldName: string | null) {
Expand All @@ -108,6 +109,16 @@ export function createMlcategoryFieldOption(categorizationFieldName: string | nu
];
}

export function createDocCountFieldOption(usingAggregations: boolean) {
return usingAggregations
? [
{
label: DOC_COUNT,
},
]
: [];
}

function getDetectorsAdvanced(job: Job, datafeed: Datafeed) {
return processFieldlessAggs(job.analysis_config.detectors);
}
Expand Down Expand Up @@ -305,3 +316,26 @@ export function getJobCreatorTitle(jobCreator: JobCreatorType) {
return '';
}
}

// recurse through a datafeed aggregation object,
// adding top level keys from each nested agg to an array
// of fields
export function collectAggs(o: any, aggFields: Field[]) {
for (const i in o) {
if (o[i] !== null && typeof o[i] === 'object') {
if (i === 'aggregations' || i === 'aggs') {
Object.keys(o[i]).forEach(k => {
if (k !== 'aggregations' && i !== 'aggs') {
aggFields.push({
id: k,
name: k,
type: ES_FIELD_TYPES.KEYWORD,
aggregatable: true,
});
}
});
}
collectAggs(o[i], aggFields);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@
* you may not use this file except in compliance with the Elastic License.
*/

import React, { FC } from 'react';
import React, { FC, useContext } from 'react';
import { EuiComboBox, EuiComboBoxOptionProps } from '@elastic/eui';

import { JobCreatorContext } from '../../../job_creator_context';
import { Field } from '../../../../../../../../../common/types/fields';
import { createFieldOptions } from '../../../../../common/job_creator/util/general';

Expand All @@ -17,7 +18,8 @@ interface Props {
}

export const TimeFieldSelect: FC<Props> = ({ fields, changeHandler, selectedField }) => {
const options: EuiComboBoxOptionProps[] = createFieldOptions(fields);
const { jobCreator } = useContext(JobCreatorContext);
const options: EuiComboBoxOptionProps[] = createFieldOptions(fields, jobCreator.additionalFields);

const selection: EuiComboBoxOptionProps[] = [];
if (selectedField !== null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import { JobCreatorContext } from '../../../job_creator_context';
import { AdvancedJobCreator } from '../../../../../common/job_creator';
import {
createFieldOptions,
createScriptFieldOptions,
createMlcategoryFieldOption,
} from '../../../../../common/job_creator/util/general';
import {
Expand Down Expand Up @@ -88,7 +87,7 @@ export const AdvancedDetectorModal: FC<Props> = ({
const [fieldOptionEnabled, setFieldOptionEnabled] = useState(true);
const { descriptionPlaceholder, setDescriptionPlaceholder } = useDetectorPlaceholder(detector);

const usingScriptFields = jobCreator.scriptFields.length > 0;
const usingScriptFields = jobCreator.additionalFields.length > 0;
// list of aggregation combobox options.

const aggOptions: EuiComboBoxOptionProps[] = aggs
Expand All @@ -98,12 +97,12 @@ export const AdvancedDetectorModal: FC<Props> = ({
// fields available for the selected agg
const { currentFieldOptions, setCurrentFieldOptions } = useCurrentFieldOptions(
detector.agg,
jobCreator.scriptFields
jobCreator.additionalFields,
fields
);

const allFieldOptions: EuiComboBoxOptionProps[] = [
...createFieldOptions(fields),
...createScriptFieldOptions(jobCreator.scriptFields),
...createFieldOptions(fields, jobCreator.additionalFields),
].sort(comboBoxOptionsSort);

const splitFieldOptions: EuiComboBoxOptionProps[] = [
Expand All @@ -127,7 +126,9 @@ export const AdvancedDetectorModal: FC<Props> = ({
return mlCategory;
}
return (
fields.find(f => f.id === title) || jobCreator.scriptFields.find(f => f.id === title) || null
fields.find(f => f.id === title) ||
jobCreator.additionalFields.find(f => f.id === title) ||
null
);
}

Expand Down Expand Up @@ -365,21 +366,27 @@ function useDetectorPlaceholder(detector: RichDetector) {
}

// creates list of combobox options based on an aggregation's field list
function createFieldOptionsFromAgg(agg: Aggregation | null) {
return createFieldOptions(agg !== null && agg.fields !== undefined ? agg.fields : []);
function createFieldOptionsFromAgg(agg: Aggregation | null, additionalFields: Field[]) {
return createFieldOptions(
agg !== null && agg.fields !== undefined ? agg.fields : [],
additionalFields
);
}

// custom hook for storing combobox options based on an aggregation field list
function useCurrentFieldOptions(aggregation: Aggregation | null, scriptFields: Field[]) {
function useCurrentFieldOptions(
aggregation: Aggregation | null,
additionalFields: Field[],
fields: Field[]
) {
const [currentFieldOptions, setCurrentFieldOptions] = useState(
createFieldOptionsFromAgg(aggregation)
createFieldOptionsFromAgg(aggregation, additionalFields)
);
const scriptFieldOptions = createScriptFieldOptions(scriptFields);

return {
currentFieldOptions,
setCurrentFieldOptions: (agg: Aggregation | null) =>
setCurrentFieldOptions([...createFieldOptionsFromAgg(agg), ...scriptFieldOptions]),
setCurrentFieldOptions(createFieldOptionsFromAgg(agg, additionalFields)),
};
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,7 @@ import { EuiComboBox, EuiComboBoxOptionProps } from '@elastic/eui';

import { JobCreatorContext } from '../../../job_creator_context';
import { Field } from '../../../../../../../../../common/types/fields';
import {
createFieldOptions,
createScriptFieldOptions,
} from '../../../../../common/job_creator/util/general';
import { createFieldOptions } from '../../../../../common/job_creator/util/general';

interface Props {
fields: Field[];
Expand All @@ -23,8 +20,7 @@ interface Props {
export const CategorizationFieldSelect: FC<Props> = ({ fields, changeHandler, selectedField }) => {
const { jobCreator } = useContext(JobCreatorContext);
const options: EuiComboBoxOptionProps[] = [
...createFieldOptions(fields),
...createScriptFieldOptions(jobCreator.scriptFields),
...createFieldOptions(fields, jobCreator.additionalFields),
];

const selection: EuiComboBoxOptionProps[] = [];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import { JobCreatorContext } from '../../../job_creator_context';
import { Field } from '../../../../../../../../../common/types/fields';
import {
createFieldOptions,
createScriptFieldOptions,
createMlcategoryFieldOption,
} from '../../../../../common/job_creator/util/general';

Expand All @@ -24,8 +23,7 @@ interface Props {
export const InfluencersSelect: FC<Props> = ({ fields, changeHandler, selectedInfluencers }) => {
const { jobCreator } = useContext(JobCreatorContext);
const options: EuiComboBoxOptionProps[] = [
...createFieldOptions(fields),
...createScriptFieldOptions(jobCreator.scriptFields),
...createFieldOptions(fields, jobCreator.additionalFields),
...createMlcategoryFieldOption(jobCreator.categorizationFieldName),
];

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { JobCreatorContext } from '../../../job_creator_context';
import { Field } from '../../../../../../../../../common/types/fields';
import {
createFieldOptions,
createScriptFieldOptions,
createDocCountFieldOption,
} from '../../../../../common/job_creator/util/general';

interface Props {
Expand All @@ -23,8 +23,8 @@ interface Props {
export const SummaryCountFieldSelect: FC<Props> = ({ fields, changeHandler, selectedField }) => {
const { jobCreator } = useContext(JobCreatorContext);
const options: EuiComboBoxOptionProps[] = [
...createFieldOptions(fields),
...createScriptFieldOptions(jobCreator.scriptFields),
...createFieldOptions(fields, jobCreator.additionalFields),
...createDocCountFieldOption(jobCreator.aggregationFields.length > 0),
];

const selection: EuiComboBoxOptionProps[] = [];
Expand Down

0 comments on commit 64f6670

Please sign in to comment.