Skip to content

Commit

Permalink
Merge branch 'master' into lowercase-normalizer
Browse files Browse the repository at this point in the history
  • Loading branch information
elasticmachine authored Oct 1, 2020
2 parents 7921a1f + bad6eab commit 6a70761
Show file tree
Hide file tree
Showing 115 changed files with 1,267 additions and 656 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,11 @@ export interface ModuleDescriptor<JobType extends string> {
partitionField?: string
) => Promise<SetupMlModuleResponsePayload>;
cleanUpModule: (spaceId: string, sourceId: string) => Promise<DeleteJobsResponsePayload>;
validateSetupIndices?: (
validateSetupIndices: (
indices: string[],
timestampField: string
) => Promise<ValidationIndicesResponsePayload>;
validateSetupDatasets?: (
validateSetupDatasets: (
indices: string[],
timestampField: string,
startTime: number,
Expand Down
289 changes: 289 additions & 0 deletions x-pack/plugins/infra/public/containers/ml/infra_ml_setup_state.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,289 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import { isEqual } from 'lodash';
import { useCallback, useEffect, useMemo, useState } from 'react';
import { usePrevious } from 'react-use';
import {
combineDatasetFilters,
DatasetFilter,
filterDatasetFilter,
isExampleDataIndex,
} from '../../../common/infra_ml';
import {
AvailableIndex,
ValidationIndicesError,
ValidationUIError,
} from '../../components/logging/log_analysis_setup/initial_configuration_step';
import { useTrackedPromise } from '../../utils/use_tracked_promise';
import { ModuleDescriptor, ModuleSourceConfiguration } from './infra_ml_module_types';

type SetupHandler = (
indices: string[],
startTime: number | undefined,
endTime: number | undefined,
datasetFilter: DatasetFilter
) => void;

interface AnalysisSetupStateArguments<JobType extends string> {
cleanUpAndSetUpModule: SetupHandler;
moduleDescriptor: ModuleDescriptor<JobType>;
setUpModule: SetupHandler;
sourceConfiguration: ModuleSourceConfiguration;
}

const fourWeeksInMs = 86400000 * 7 * 4;

export const useAnalysisSetupState = <JobType extends string>({
cleanUpAndSetUpModule,
moduleDescriptor: { validateSetupDatasets, validateSetupIndices },
setUpModule,
sourceConfiguration,
}: AnalysisSetupStateArguments<JobType>) => {
const [startTime, setStartTime] = useState<number | undefined>(Date.now() - fourWeeksInMs);
const [endTime, setEndTime] = useState<number | undefined>(undefined);

const isTimeRangeValid = useMemo(
() => (startTime != null && endTime != null ? startTime < endTime : true),
[endTime, startTime]
);

const [validatedIndices, setValidatedIndices] = useState<AvailableIndex[]>(
sourceConfiguration.indices.map((indexName) => ({
name: indexName,
validity: 'unknown' as const,
}))
);

const updateIndicesWithValidationErrors = useCallback(
(validationErrors: ValidationIndicesError[]) =>
setValidatedIndices((availableIndices) =>
availableIndices.map((previousAvailableIndex) => {
const indexValiationErrors = validationErrors.filter(
({ index }) => index === previousAvailableIndex.name
);

if (indexValiationErrors.length > 0) {
return {
validity: 'invalid',
name: previousAvailableIndex.name,
errors: indexValiationErrors,
};
} else if (previousAvailableIndex.validity === 'valid') {
return {
...previousAvailableIndex,
validity: 'valid',
errors: [],
};
} else {
return {
validity: 'valid',
name: previousAvailableIndex.name,
isSelected: !isExampleDataIndex(previousAvailableIndex.name),
availableDatasets: [],
datasetFilter: {
type: 'includeAll' as const,
},
};
}
})
),
[]
);

const updateIndicesWithAvailableDatasets = useCallback(
(availableDatasets: Array<{ indexName: string; datasets: string[] }>) =>
setValidatedIndices((availableIndices) =>
availableIndices.map((previousAvailableIndex) => {
if (previousAvailableIndex.validity !== 'valid') {
return previousAvailableIndex;
}

const availableDatasetsForIndex = availableDatasets.filter(
({ indexName }) => indexName === previousAvailableIndex.name
);
const newAvailableDatasets = availableDatasetsForIndex.flatMap(
({ datasets }) => datasets
);

// filter out datasets that have disappeared if this index' datasets were updated
const newDatasetFilter: DatasetFilter =
availableDatasetsForIndex.length > 0
? filterDatasetFilter(previousAvailableIndex.datasetFilter, (dataset) =>
newAvailableDatasets.includes(dataset)
)
: previousAvailableIndex.datasetFilter;

return {
...previousAvailableIndex,
availableDatasets: newAvailableDatasets,
datasetFilter: newDatasetFilter,
};
})
),
[]
);

const validIndexNames = useMemo(
() => validatedIndices.filter((index) => index.validity === 'valid').map((index) => index.name),
[validatedIndices]
);

const selectedIndexNames = useMemo(
() =>
validatedIndices
.filter((index) => index.validity === 'valid' && index.isSelected)
.map((i) => i.name),
[validatedIndices]
);

const datasetFilter = useMemo(
() =>
validatedIndices
.flatMap((validatedIndex) =>
validatedIndex.validity === 'valid'
? validatedIndex.datasetFilter
: { type: 'includeAll' as const }
)
.reduce(combineDatasetFilters, { type: 'includeAll' as const }),
[validatedIndices]
);

const [validateIndicesRequest, validateIndices] = useTrackedPromise(
{
cancelPreviousOn: 'resolution',
createPromise: async () => {
return await validateSetupIndices(
sourceConfiguration.indices,
sourceConfiguration.timestampField
);
},
onResolve: ({ data: { errors } }) => {
updateIndicesWithValidationErrors(errors);
},
onReject: () => {
setValidatedIndices([]);
},
},
[sourceConfiguration.indices, sourceConfiguration.timestampField]
);

const [validateDatasetsRequest, validateDatasets] = useTrackedPromise(
{
cancelPreviousOn: 'resolution',
createPromise: async () => {
if (validIndexNames.length === 0) {
return { data: { datasets: [] } };
}

return await validateSetupDatasets(
validIndexNames,
sourceConfiguration.timestampField,
startTime ?? 0,
endTime ?? Date.now()
);
},
onResolve: ({ data: { datasets } }) => {
updateIndicesWithAvailableDatasets(datasets);
},
},
[validIndexNames, sourceConfiguration.timestampField, startTime, endTime]
);

const setUp = useCallback(() => {
return setUpModule(selectedIndexNames, startTime, endTime, datasetFilter);
}, [setUpModule, selectedIndexNames, startTime, endTime, datasetFilter]);

const cleanUpAndSetUp = useCallback(() => {
return cleanUpAndSetUpModule(selectedIndexNames, startTime, endTime, datasetFilter);
}, [cleanUpAndSetUpModule, selectedIndexNames, startTime, endTime, datasetFilter]);

const isValidating = useMemo(
() => validateIndicesRequest.state === 'pending' || validateDatasetsRequest.state === 'pending',
[validateDatasetsRequest.state, validateIndicesRequest.state]
);

const validationErrors = useMemo<ValidationUIError[]>(() => {
if (isValidating) {
return [];
}

return [
// validate request status
...(validateIndicesRequest.state === 'rejected' ||
validateDatasetsRequest.state === 'rejected'
? [{ error: 'NETWORK_ERROR' as const }]
: []),
// validation request results
...validatedIndices.reduce<ValidationUIError[]>((errors, index) => {
return index.validity === 'invalid' && selectedIndexNames.includes(index.name)
? [...errors, ...index.errors]
: errors;
}, []),
// index count
...(selectedIndexNames.length === 0 ? [{ error: 'TOO_FEW_SELECTED_INDICES' as const }] : []),
// time range
...(!isTimeRangeValid ? [{ error: 'INVALID_TIME_RANGE' as const }] : []),
];
}, [
isValidating,
validateIndicesRequest.state,
validateDatasetsRequest.state,
validatedIndices,
selectedIndexNames,
isTimeRangeValid,
]);

const prevStartTime = usePrevious(startTime);
const prevEndTime = usePrevious(endTime);
const prevValidIndexNames = usePrevious(validIndexNames);

useEffect(() => {
if (!isTimeRangeValid) {
return;
}

validateIndices();
}, [isTimeRangeValid, validateIndices]);

useEffect(() => {
if (!isTimeRangeValid) {
return;
}

if (
startTime !== prevStartTime ||
endTime !== prevEndTime ||
!isEqual(validIndexNames, prevValidIndexNames)
) {
validateDatasets();
}
}, [
endTime,
isTimeRangeValid,
prevEndTime,
prevStartTime,
prevValidIndexNames,
startTime,
validIndexNames,
validateDatasets,
]);

return {
cleanUpAndSetUp,
datasetFilter,
endTime,
isValidating,
selectedIndexNames,
setEndTime,
setStartTime,
setUp,
startTime,
validatedIndices,
setValidatedIndices,
validationErrors,
};
};
Loading

0 comments on commit 6a70761

Please sign in to comment.