From c0c62723075bd0660db511d723035ff636787695 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Sat, 20 Mar 2021 17:14:17 -0600 Subject: [PATCH] [ml] migrate file_data_visualizer/analyze_file to file_upload plugin (#94259) (#95022) * [ml] migrate file_data_visualizer/analyze_file to file_upload plugin * tslint * give analyze route access to ml user Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- x-pack/plugins/file_upload/common/types.ts | 67 +++++++++++++++++ .../server/analyze_file.tsx} | 45 ++++++------ .../plugins/file_upload/server/import_data.ts | 3 +- x-pack/plugins/file_upload/server/routes.ts | 52 +++++++++++++- x-pack/plugins/file_upload/server/schemas.ts | 17 +++++ .../plugins/ml/common/types/capabilities.ts | 8 ++- .../ml/common/types/file_datavisualizer.ts | 71 ------------------- .../analysis_summary/analysis_summary.tsx | 2 +- .../combined_fields/combined_fields_form.tsx | 2 +- .../components/combined_fields/geo_point.tsx | 2 +- .../components/combined_fields/utils.ts | 7 +- .../explanation_flyout/explanation_flyout.tsx | 2 +- .../fields_stats_grid/create_fields.ts | 2 +- .../fields_stats_grid/fields_stats_grid.tsx | 2 +- .../fields_stats_grid/get_field_names.ts | 2 +- .../filebeat_config_flyout/filebeat_config.ts | 2 +- .../filebeat_config_flyout.tsx | 2 +- .../components/import_settings/advanced.tsx | 2 +- .../import_settings/import_settings.tsx | 2 +- .../results_links/results_links.tsx | 2 +- .../components/results_view/results_view.tsx | 2 +- .../file_based/components/utils/utils.ts | 2 +- .../services/ml_api_service/datavisualizer.ts | 4 +- .../models/file_data_visualizer/index.ts | 8 --- x-pack/plugins/ml/server/plugin.ts | 2 - .../ml/server/routes/file_data_visualizer.ts | 60 ---------------- .../schemas/file_data_visualizer_schema.ts | 27 ------- 27 files changed, 181 insertions(+), 218 deletions(-) rename x-pack/plugins/{ml/server/models/file_data_visualizer/file_data_visualizer.ts => file_upload/server/analyze_file.tsx} (58%) delete mode 100644 x-pack/plugins/ml/common/types/file_datavisualizer.ts delete mode 100644 x-pack/plugins/ml/server/models/file_data_visualizer/index.ts delete mode 100644 x-pack/plugins/ml/server/routes/file_data_visualizer.ts delete mode 100644 x-pack/plugins/ml/server/routes/schemas/file_data_visualizer_schema.ts diff --git a/x-pack/plugins/file_upload/common/types.ts b/x-pack/plugins/file_upload/common/types.ts index c01e514f0f720e..3c385bdf8f3df1 100644 --- a/x-pack/plugins/file_upload/common/types.ts +++ b/x-pack/plugins/file_upload/common/types.ts @@ -5,6 +5,73 @@ * 2.0. */ +import { ES_FIELD_TYPES } from '../../../../src/plugins/data/common'; + +export interface InputOverrides { + [key: string]: string | undefined; +} + +export type FormattedOverrides = InputOverrides & { + column_names: string[]; + has_header_row: boolean; + should_trim_fields: boolean; +}; + +export interface AnalysisResult { + results: FindFileStructureResponse; + overrides?: FormattedOverrides; +} + +export interface FindFileStructureResponse { + charset: string; + has_header_row: boolean; + has_byte_order_marker: boolean; + format: string; + field_stats: { + [fieldName: string]: { + count: number; + cardinality: number; + top_hits: Array<{ count: number; value: any }>; + mean_value?: number; + median_value?: number; + max_value?: number; + min_value?: number; + earliest?: string; + latest?: string; + }; + }; + sample_start: string; + num_messages_analyzed: number; + mappings: { + properties: { + [fieldName: string]: { + // including all possible Elasticsearch types + // since find_file_structure API can be enhanced to include new fields in the future + type: Exclude< + ES_FIELD_TYPES, + ES_FIELD_TYPES._ID | ES_FIELD_TYPES._INDEX | ES_FIELD_TYPES._SOURCE | ES_FIELD_TYPES._TYPE + >; + format?: string; + }; + }; + }; + quote: string; + delimiter: string; + need_client_timezone: boolean; + num_lines_analyzed: number; + column_names?: string[]; + explanation?: string[]; + grok_pattern?: string; + multiline_start_pattern?: string; + exclude_lines_pattern?: string; + java_timestamp_formats?: string[]; + joda_timestamp_formats?: string[]; + timestamp_field?: string; + should_trim_fields?: boolean; +} + +export type InputData = any[]; + export interface ImportResponse { success: boolean; id: string; diff --git a/x-pack/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts b/x-pack/plugins/file_upload/server/analyze_file.tsx similarity index 58% rename from x-pack/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts rename to x-pack/plugins/file_upload/server/analyze_file.tsx index 6e57e997e5f008..394573eb0cca5a 100644 --- a/x-pack/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts +++ b/x-pack/plugins/file_upload/server/analyze_file.tsx @@ -9,32 +9,29 @@ import { IScopedClusterClient } from 'kibana/server'; import { AnalysisResult, FormattedOverrides, + InputData, InputOverrides, FindFileStructureResponse, -} from '../../../common/types/file_datavisualizer'; - -export type InputData = any[]; - -export function fileDataVisualizerProvider(client: IScopedClusterClient) { - async function analyzeFile(data: InputData, overrides: InputOverrides): Promise { - overrides.explain = overrides.explain === undefined ? 'true' : overrides.explain; - const { - body, - } = await client.asInternalUser.textStructure.findStructure({ - body: data, - ...overrides, - }); - - const { hasOverrides, reducedOverrides } = formatOverrides(overrides); - - return { - ...(hasOverrides && { overrides: reducedOverrides }), - results: body, - }; - } +} from '../common'; + +export async function analyzeFile( + client: IScopedClusterClient, + data: InputData, + overrides: InputOverrides +): Promise { + overrides.explain = overrides.explain === undefined ? 'true' : overrides.explain; + const { + body, + } = await client.asInternalUser.textStructure.findStructure({ + body: data, + ...overrides, + }); + + const { hasOverrides, reducedOverrides } = formatOverrides(overrides); return { - analyzeFile, + ...(hasOverrides && { overrides: reducedOverrides }), + results: body, }; } @@ -42,8 +39,8 @@ function formatOverrides(overrides: InputOverrides) { let hasOverrides = false; const reducedOverrides: FormattedOverrides = Object.keys(overrides).reduce((acc, overrideKey) => { - const overrideValue: string = overrides[overrideKey]; - if (overrideValue !== '') { + const overrideValue: string | undefined = overrides[overrideKey]; + if (overrideValue !== undefined && overrideValue !== '') { if (overrideKey === 'column_names') { acc.column_names = overrideValue.split(','); } else if (overrideKey === 'has_header_row') { diff --git a/x-pack/plugins/file_upload/server/import_data.ts b/x-pack/plugins/file_upload/server/import_data.ts index 510302beaf6dda..f93d73647ed0e1 100644 --- a/x-pack/plugins/file_upload/server/import_data.ts +++ b/x-pack/plugins/file_upload/server/import_data.ts @@ -10,13 +10,12 @@ import { INDEX_META_DATA_CREATED_BY } from '../common/constants'; import { ImportResponse, ImportFailure, + InputData, Settings, Mappings, IngestPipelineWrapper, } from '../common'; -export type InputData = any[]; - export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { async function importData( id: string | undefined, diff --git a/x-pack/plugins/file_upload/server/routes.ts b/x-pack/plugins/file_upload/server/routes.ts index d7b7b8f99edd99..4f4adb29f6b0bb 100644 --- a/x-pack/plugins/file_upload/server/routes.ts +++ b/x-pack/plugins/file_upload/server/routes.ts @@ -5,13 +5,21 @@ * 2.0. */ +import { schema } from '@kbn/config-schema'; import { IRouter, IScopedClusterClient } from 'kibana/server'; -import { MAX_FILE_SIZE_BYTES, IngestPipelineWrapper, Mappings, Settings } from '../common'; +import { + MAX_FILE_SIZE_BYTES, + IngestPipelineWrapper, + InputData, + Mappings, + Settings, +} from '../common'; import { wrapError } from './error_wrapper'; -import { InputData, importDataProvider } from './import_data'; +import { analyzeFile } from './analyze_file'; +import { importDataProvider } from './import_data'; import { updateTelemetry } from './telemetry'; -import { importFileBodySchema, importFileQuerySchema } from './schemas'; +import { analyzeFileQuerySchema, importFileBodySchema, importFileQuerySchema } from './schemas'; function importData( client: IScopedClusterClient, @@ -30,6 +38,44 @@ function importData( * Routes for the file upload. */ export function fileUploadRoutes(router: IRouter) { + /** + * @apiGroup FileDataVisualizer + * + * @api {post} /api/file_upload/analyze_file Analyze file data + * @apiName AnalyzeFile + * @apiDescription Performs analysis of the file data. + * + * @apiSchema (query) analyzeFileQuerySchema + */ + router.post( + { + path: '/api/file_upload/analyze_file', + validate: { + body: schema.any(), + query: analyzeFileQuerySchema, + }, + options: { + body: { + accepts: ['text/*', 'application/json'], + maxBytes: MAX_FILE_SIZE_BYTES, + }, + tags: ['access:fileUpload:analyzeFile'], + }, + }, + async (context, request, response) => { + try { + const result = await analyzeFile( + context.core.elasticsearch.client, + request.body, + request.query + ); + return response.ok({ body: result }); + } catch (e) { + return response.customError(wrapError(e)); + } + } + ); + /** * @apiGroup FileDataVisualizer * diff --git a/x-pack/plugins/file_upload/server/schemas.ts b/x-pack/plugins/file_upload/server/schemas.ts index bfef9885b3216a..a0d54cf9ec5536 100644 --- a/x-pack/plugins/file_upload/server/schemas.ts +++ b/x-pack/plugins/file_upload/server/schemas.ts @@ -7,6 +7,23 @@ import { schema } from '@kbn/config-schema'; +export const analyzeFileQuerySchema = schema.object({ + charset: schema.maybe(schema.string()), + column_names: schema.maybe(schema.string()), + delimiter: schema.maybe(schema.string()), + explain: schema.maybe(schema.string()), + format: schema.maybe(schema.string()), + grok_pattern: schema.maybe(schema.string()), + has_header_row: schema.maybe(schema.string()), + line_merge_size_limit: schema.maybe(schema.string()), + lines_to_sample: schema.maybe(schema.string()), + quote: schema.maybe(schema.string()), + should_trim_fields: schema.maybe(schema.string()), + timeout: schema.maybe(schema.string()), + timestamp_field: schema.maybe(schema.string()), + timestamp_format: schema.maybe(schema.string()), +}); + export const importFileQuerySchema = schema.object({ id: schema.maybe(schema.string()), }); diff --git a/x-pack/plugins/ml/common/types/capabilities.ts b/x-pack/plugins/ml/common/types/capabilities.ts index 61a5013642cd7a..129b496c00149f 100644 --- a/x-pack/plugins/ml/common/types/capabilities.ts +++ b/x-pack/plugins/ml/common/types/capabilities.ts @@ -102,7 +102,11 @@ export function getPluginPrivileges() { return { admin: { ...privilege, - api: ['fileUpload:import', ...allMlCapabilitiesKeys.map((k) => `ml:${k}`)], + api: [ + 'fileUpload:import', + 'fileUpload:analyzeFile', + ...allMlCapabilitiesKeys.map((k) => `ml:${k}`), + ], catalogue: [PLUGIN_ID, `${PLUGIN_ID}_file_data_visualizer`], ui: allMlCapabilitiesKeys, savedObject: { @@ -116,7 +120,7 @@ export function getPluginPrivileges() { }, user: { ...privilege, - api: userMlCapabilitiesKeys.map((k) => `ml:${k}`), + api: ['fileUpload:analyzeFile', ...userMlCapabilitiesKeys.map((k) => `ml:${k}`)], catalogue: [PLUGIN_ID], management: { insightsAndAlerting: [] }, ui: userMlCapabilitiesKeys, diff --git a/x-pack/plugins/ml/common/types/file_datavisualizer.ts b/x-pack/plugins/ml/common/types/file_datavisualizer.ts deleted file mode 100644 index 500ee988234853..00000000000000 --- a/x-pack/plugins/ml/common/types/file_datavisualizer.ts +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ES_FIELD_TYPES } from '../../../../../src/plugins/data/common'; - -export interface InputOverrides { - [key: string]: string; -} - -export type FormattedOverrides = InputOverrides & { - column_names: string[]; - has_header_row: boolean; - should_trim_fields: boolean; -}; - -export interface AnalysisResult { - results: FindFileStructureResponse; - overrides?: FormattedOverrides; -} - -export interface FindFileStructureResponse { - charset: string; - has_header_row: boolean; - has_byte_order_marker: boolean; - format: string; - field_stats: { - [fieldName: string]: { - count: number; - cardinality: number; - top_hits: Array<{ count: number; value: any }>; - mean_value?: number; - median_value?: number; - max_value?: number; - min_value?: number; - earliest?: string; - latest?: string; - }; - }; - sample_start: string; - num_messages_analyzed: number; - mappings: { - properties: { - [fieldName: string]: { - // including all possible Elasticsearch types - // since find_file_structure API can be enhanced to include new fields in the future - type: Exclude< - ES_FIELD_TYPES, - ES_FIELD_TYPES._ID | ES_FIELD_TYPES._INDEX | ES_FIELD_TYPES._SOURCE | ES_FIELD_TYPES._TYPE - >; - format?: string; - }; - }; - }; - quote: string; - delimiter: string; - need_client_timezone: boolean; - num_lines_analyzed: number; - column_names?: string[]; - explanation?: string[]; - grok_pattern?: string; - multiline_start_pattern?: string; - exclude_lines_pattern?: string; - java_timestamp_formats?: string[]; - joda_timestamp_formats?: string[]; - timestamp_field?: string; - should_trim_fields?: boolean; -} diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/analysis_summary/analysis_summary.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/analysis_summary/analysis_summary.tsx index 0691ed6fdd4417..d787ee5cb844e2 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/analysis_summary/analysis_summary.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/analysis_summary/analysis_summary.tsx @@ -9,7 +9,7 @@ import { FormattedMessage } from '@kbn/i18n/react'; import React, { FC } from 'react'; import { EuiTitle, EuiSpacer, EuiDescriptionList } from '@elastic/eui'; -import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; +import { FindFileStructureResponse } from '../../../../../../../file_upload/common'; export const AnalysisSummary: FC<{ results: FindFileStructureResponse }> = ({ results }) => { const items = createDisplayItems(results); diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_fields_form.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_fields_form.tsx index 0b67b24b65c5e7..02ead5c26f9592 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_fields_form.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/combined_fields_form.tsx @@ -29,7 +29,7 @@ import { removeCombinedFieldsFromMappings, removeCombinedFieldsFromPipeline, } from './utils'; -import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; +import { FindFileStructureResponse } from '../../../../../../../file_upload/common'; interface Props { mappingsString: string; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/geo_point.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/geo_point.tsx index d12de6c5c91a51..5ae2e5de681c3e 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/geo_point.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/geo_point.tsx @@ -29,7 +29,7 @@ import { getFieldNames, getNameCollisionMsg, } from './utils'; -import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; +import { FindFileStructureResponse } from '../../../../../../../file_upload/common'; interface Props { addCombinedField: (combinedField: CombinedField) => void; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts index da1efe034bd54c..ab08398fcda025 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts @@ -9,8 +9,11 @@ import { i18n } from '@kbn/i18n'; import { cloneDeep } from 'lodash'; import uuid from 'uuid/v4'; import { CombinedField } from './types'; -import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; -import { IngestPipeline, Mappings } from '../../../../../../../file_upload/common'; +import { + FindFileStructureResponse, + IngestPipeline, + Mappings, +} from '../../../../../../../file_upload/common'; const COMMON_LAT_NAMES = ['latitude', 'lat']; const COMMON_LON_NAMES = ['longitude', 'long', 'lon']; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/explanation_flyout.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/explanation_flyout.tsx index 199b03b728a967..579f2e3340954d 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/explanation_flyout.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/explanation_flyout.tsx @@ -20,7 +20,7 @@ import { EuiText, EuiSubSteps, } from '@elastic/eui'; -import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; +import { FindFileStructureResponse } from '../../../../../../../file_upload/common'; interface Props { results: FindFileStructureResponse; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/fields_stats_grid/create_fields.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/fields_stats_grid/create_fields.ts index 86ca1fbbe6f824..fdbb35d27c531e 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/fields_stats_grid/create_fields.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/fields_stats_grid/create_fields.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; +import { FindFileStructureResponse } from '../../../../../../../file_upload/common'; import { getFieldNames, getSupportedFieldType } from './get_field_names'; import { FileBasedFieldVisConfig } from '../../../stats_table/types'; import { ML_JOB_FIELD_TYPES } from '../../../../../../common/constants/field_types'; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/fields_stats_grid/fields_stats_grid.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/fields_stats_grid/fields_stats_grid.tsx index 364242e9bf325a..1029d58b4c6395 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/fields_stats_grid/fields_stats_grid.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/fields_stats_grid/fields_stats_grid.tsx @@ -7,7 +7,7 @@ import React, { useMemo, FC } from 'react'; import { EuiFlexGroup, EuiSpacer } from '@elastic/eui'; -import type { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; +import type { FindFileStructureResponse } from '../../../../../../../file_upload/common'; import { DataVisualizerTable, ItemIdToExpandedRowMap } from '../../../stats_table'; import type { FileBasedFieldVisConfig } from '../../../stats_table/types/field_vis_config'; import { FileBasedDataVisualizerExpandedRow } from '../expanded_row'; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/fields_stats_grid/get_field_names.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/fields_stats_grid/get_field_names.ts index 2e0c3feeee4f10..d1cb361a84a729 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/fields_stats_grid/get_field_names.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/fields_stats_grid/get_field_names.ts @@ -6,7 +6,7 @@ */ import { difference } from 'lodash'; -import type { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; +import type { FindFileStructureResponse } from '../../../../../../../file_upload/common'; import { MlJobFieldType } from '../../../../../../common/types/field_types'; import { ML_JOB_FIELD_TYPES } from '../../../../../../common/constants/field_types'; import { ES_FIELD_TYPES } from '../../../../../../../../../src/plugins/data/common'; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts index da22b807f4332b..2254110432bdba 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts @@ -6,7 +6,7 @@ */ import { i18n } from '@kbn/i18n'; -import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; +import { FindFileStructureResponse } from '../../../../../../../file_upload/common'; export function createFilebeatConfig( index: string, diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx index a288579f91b585..c3b53d44300873 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx @@ -23,7 +23,7 @@ import { } from '@elastic/eui'; import { createFilebeatConfig } from './filebeat_config'; import { useMlKibana } from '../../../../contexts/kibana'; -import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; +import { FindFileStructureResponse } from '../../../../../../../file_upload/common'; export enum EDITOR_MODE { HIDDEN, diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.tsx index 94f4aaa1c4bab3..eb0e09973f0e32 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.tsx @@ -20,7 +20,7 @@ import { import { CombinedField, CombinedFieldsForm } from '../combined_fields'; import { MLJobEditor, ML_EDITOR_MODE } from '../../../../jobs/jobs_list/components/ml_job_editor'; -import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; +import { FindFileStructureResponse } from '../../../../../../../file_upload/common'; const EDITOR_HEIGHT = '300px'; interface Props { diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/import_settings.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/import_settings.tsx index c75d2074e60a39..5a9597723a0b59 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/import_settings.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/import_settings.tsx @@ -13,7 +13,7 @@ import { EuiTabbedContent, EuiSpacer } from '@elastic/eui'; import { SimpleSettings } from './simple'; import { AdvancedSettings } from './advanced'; import { CombinedField } from '../combined_fields'; -import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; +import { FindFileStructureResponse } from '../../../../../../../file_upload/common'; interface Props { index: string; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.tsx index 6ce75a4be840e9..90b8fb4ac0cbbd 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.tsx @@ -20,7 +20,7 @@ import { DISCOVER_APP_URL_GENERATOR, DiscoverUrlGeneratorState, } from '../../../../../../../../../src/plugins/discover/public'; -import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; +import { FindFileStructureResponse } from '../../../../../../../file_upload/common'; const RECHECK_DELAY_MS = 3000; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.tsx index cfcdd96dc3c8a0..7431bfd4295e4e 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.tsx @@ -20,7 +20,7 @@ import { EuiFlexGroup, EuiFlexItem, } from '@elastic/eui'; -import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; +import { FindFileStructureResponse } from '../../../../../../../file_upload/common'; import { FileContents } from '../file_contents'; import { AnalysisSummary } from '../analysis_summary'; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts index ebde771603fcfb..85ca27cdf90fff 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts @@ -7,7 +7,7 @@ import { isEqual } from 'lodash'; import { ml } from '../../../../services/ml_api_service'; -import { AnalysisResult, InputOverrides } from '../../../../../../common/types/file_datavisualizer'; +import { AnalysisResult, InputOverrides } from '../../../../../../../file_upload/common'; import { MB } from '../../../../../../../file_upload/public'; export const DEFAULT_LINES_TO_SAMPLE = 1000; diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts index 98a0d7b9b0a94f..ccea697de4b9c5 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts @@ -7,13 +7,11 @@ import { http } from '../http_service'; -import { basePath } from './index'; - export const fileDatavisualizer = { analyzeFile(file: string, params: Record = {}) { const body = JSON.stringify(file); return http({ - path: `${basePath()}/file_data_visualizer/analyze_file`, + path: '/api/file_upload/analyze_file', method: 'POST', body, query: params, diff --git a/x-pack/plugins/ml/server/models/file_data_visualizer/index.ts b/x-pack/plugins/ml/server/models/file_data_visualizer/index.ts deleted file mode 100644 index 8e4c9935f55379..00000000000000 --- a/x-pack/plugins/ml/server/models/file_data_visualizer/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export { fileDataVisualizerProvider, InputData } from './file_data_visualizer'; diff --git a/x-pack/plugins/ml/server/plugin.ts b/x-pack/plugins/ml/server/plugin.ts index c4ee1fd76530e8..173b30716c6b66 100644 --- a/x-pack/plugins/ml/server/plugin.ts +++ b/x-pack/plugins/ml/server/plugin.ts @@ -34,7 +34,6 @@ import { dataFrameAnalyticsRoutes } from './routes/data_frame_analytics'; import { dataRecognizer } from './routes/modules'; import { dataVisualizerRoutes } from './routes/data_visualizer'; import { fieldsService } from './routes/fields_service'; -import { fileDataVisualizerRoutes } from './routes/file_data_visualizer'; import { filtersRoutes } from './routes/filters'; import { indicesRoutes } from './routes/indices'; import { jobAuditMessagesRoutes } from './routes/job_audit_messages'; @@ -172,7 +171,6 @@ export class MlServerPlugin dataRecognizer(routeInit); dataVisualizerRoutes(routeInit); fieldsService(routeInit); - fileDataVisualizerRoutes(routeInit); filtersRoutes(routeInit); indicesRoutes(routeInit); jobAuditMessagesRoutes(routeInit); diff --git a/x-pack/plugins/ml/server/routes/file_data_visualizer.ts b/x-pack/plugins/ml/server/routes/file_data_visualizer.ts deleted file mode 100644 index 6b200c59f57d5a..00000000000000 --- a/x-pack/plugins/ml/server/routes/file_data_visualizer.ts +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IScopedClusterClient } from 'kibana/server'; -import { schema } from '@kbn/config-schema'; -import { MAX_FILE_SIZE_BYTES } from '../../../file_upload/common'; -import { InputOverrides } from '../../common/types/file_datavisualizer'; -import { wrapError } from '../client/error_wrapper'; -import { InputData, fileDataVisualizerProvider } from '../models/file_data_visualizer'; - -import { RouteInitialization } from '../types'; -import { analyzeFileQuerySchema } from './schemas/file_data_visualizer_schema'; - -function analyzeFiles(client: IScopedClusterClient, data: InputData, overrides: InputOverrides) { - const { analyzeFile } = fileDataVisualizerProvider(client); - return analyzeFile(data, overrides); -} - -/** - * Routes for the file data visualizer. - */ -export function fileDataVisualizerRoutes({ router, routeGuard }: RouteInitialization) { - /** - * @apiGroup FileDataVisualizer - * - * @api {post} /api/ml/file_data_visualizer/analyze_file Analyze file data - * @apiName AnalyzeFile - * @apiDescription Performs analysis of the file data. - * - * @apiSchema (query) analyzeFileQuerySchema - */ - router.post( - { - path: '/api/ml/file_data_visualizer/analyze_file', - validate: { - body: schema.any(), - query: analyzeFileQuerySchema, - }, - options: { - body: { - accepts: ['text/*', 'application/json'], - maxBytes: MAX_FILE_SIZE_BYTES, - }, - tags: ['access:ml:canFindFileStructure'], - }, - }, - routeGuard.basicLicenseAPIGuard(async ({ client, request, response }) => { - try { - const result = await analyzeFiles(client, request.body, request.query); - return response.ok({ body: result }); - } catch (e) { - return response.customError(wrapError(e)); - } - }) - ); -} diff --git a/x-pack/plugins/ml/server/routes/schemas/file_data_visualizer_schema.ts b/x-pack/plugins/ml/server/routes/schemas/file_data_visualizer_schema.ts deleted file mode 100644 index 7d368ed38b2e3a..00000000000000 --- a/x-pack/plugins/ml/server/routes/schemas/file_data_visualizer_schema.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { schema } from '@kbn/config-schema'; - -export const analyzeFileQuerySchema = schema.maybe( - schema.object({ - charset: schema.maybe(schema.string()), - column_names: schema.maybe(schema.string()), - delimiter: schema.maybe(schema.string()), - explain: schema.maybe(schema.string()), - format: schema.maybe(schema.string()), - grok_pattern: schema.maybe(schema.string()), - has_header_row: schema.maybe(schema.string()), - line_merge_size_limit: schema.maybe(schema.string()), - lines_to_sample: schema.maybe(schema.string()), - quote: schema.maybe(schema.string()), - should_trim_fields: schema.maybe(schema.string()), - timeout: schema.maybe(schema.string()), - timestamp_field: schema.maybe(schema.string()), - timestamp_format: schema.maybe(schema.string()), - }) -);