[ml] migrate file_data_visualizer/analyze_file to file_upload plugin (#…

…94259) * [ml] migrate file_data_visualizer/analyze_file to file_upload plugin * tslint * give analyze route access to ml user Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
elastic · Mar 20, 2021 · a6c0ff6 · a6c0ff6
1 parent dd7ea1d
commit a6c0ff6
Show file tree

Hide file tree

Showing 27 changed files with 181 additions and 218 deletions.
diff --git a/x-pack/plugins/file_upload/common/types.ts b/x-pack/plugins/file_upload/common/types.ts
@@ -5,6 +5,73 @@
  * 2.0.
  */
 
+import { ES_FIELD_TYPES } from '../../../../src/plugins/data/common';
+
+export interface InputOverrides {
+  [key: string]: string | undefined;
+}
+
+export type FormattedOverrides = InputOverrides & {
+  column_names: string[];
+  has_header_row: boolean;
+  should_trim_fields: boolean;
+};
+
+export interface AnalysisResult {
+  results: FindFileStructureResponse;
+  overrides?: FormattedOverrides;
+}
+
+export interface FindFileStructureResponse {
+  charset: string;
+  has_header_row: boolean;
+  has_byte_order_marker: boolean;
+  format: string;
+  field_stats: {
+    [fieldName: string]: {
+      count: number;
+      cardinality: number;
+      top_hits: Array<{ count: number; value: any }>;
+      mean_value?: number;
+      median_value?: number;
+      max_value?: number;
+      min_value?: number;
+      earliest?: string;
+      latest?: string;
+    };
+  };
+  sample_start: string;
+  num_messages_analyzed: number;
+  mappings: {
+    properties: {
+      [fieldName: string]: {
+        // including all possible Elasticsearch types
+        // since find_file_structure API can be enhanced to include new fields in the future
+        type: Exclude<
+          ES_FIELD_TYPES,
+          ES_FIELD_TYPES._ID | ES_FIELD_TYPES._INDEX | ES_FIELD_TYPES._SOURCE | ES_FIELD_TYPES._TYPE
+        >;
+        format?: string;
+      };
+    };
+  };
+  quote: string;
+  delimiter: string;
+  need_client_timezone: boolean;
+  num_lines_analyzed: number;
+  column_names?: string[];
+  explanation?: string[];
+  grok_pattern?: string;
+  multiline_start_pattern?: string;
+  exclude_lines_pattern?: string;
+  java_timestamp_formats?: string[];
+  joda_timestamp_formats?: string[];
+  timestamp_field?: string;
+  should_trim_fields?: boolean;
+}
+
+export type InputData = any[];
+
 export interface ImportResponse {
   success: boolean;
   id: string;

diff --git a/...e_data_visualizer/file_data_visualizer.ts → ...ugins/file_upload/server/analyze_file.tsx b/...e_data_visualizer/file_data_visualizer.ts → ...ugins/file_upload/server/analyze_file.tsx
@@ -9,41 +9,38 @@ import { IScopedClusterClient } from 'kibana/server';
 import {
   AnalysisResult,
   FormattedOverrides,
+  InputData,
   InputOverrides,
   FindFileStructureResponse,
-} from '../../../common/types/file_datavisualizer';
-
-export type InputData = any[];
-
-export function fileDataVisualizerProvider(client: IScopedClusterClient) {
-  async function analyzeFile(data: InputData, overrides: InputOverrides): Promise<AnalysisResult> {
-    overrides.explain = overrides.explain === undefined ? 'true' : overrides.explain;
-    const {
-      body,
-    } = await client.asInternalUser.textStructure.findStructure<FindFileStructureResponse>({
-      body: data,
-      ...overrides,
-    });
-
-    const { hasOverrides, reducedOverrides } = formatOverrides(overrides);
-
-    return {
-      ...(hasOverrides && { overrides: reducedOverrides }),
-      results: body,
-    };
-  }
+} from '../common';
+
+export async function analyzeFile(
+  client: IScopedClusterClient,
+  data: InputData,
+  overrides: InputOverrides
+): Promise<AnalysisResult> {
+  overrides.explain = overrides.explain === undefined ? 'true' : overrides.explain;
+  const {
+    body,
+  } = await client.asInternalUser.textStructure.findStructure<FindFileStructureResponse>({
+    body: data,
+    ...overrides,
+  });
+
+  const { hasOverrides, reducedOverrides } = formatOverrides(overrides);
 
   return {
-    analyzeFile,
+    ...(hasOverrides && { overrides: reducedOverrides }),
+    results: body,
   };
 }
 
 function formatOverrides(overrides: InputOverrides) {
   let hasOverrides = false;
 
   const reducedOverrides: FormattedOverrides = Object.keys(overrides).reduce((acc, overrideKey) => {
-    const overrideValue: string = overrides[overrideKey];
-    if (overrideValue !== '') {
+    const overrideValue: string | undefined = overrides[overrideKey];
+    if (overrideValue !== undefined && overrideValue !== '') {
       if (overrideKey === 'column_names') {
         acc.column_names = overrideValue.split(',');
       } else if (overrideKey === 'has_header_row') {

diff --git a/x-pack/plugins/file_upload/server/import_data.ts b/x-pack/plugins/file_upload/server/import_data.ts
@@ -10,13 +10,12 @@ import { INDEX_META_DATA_CREATED_BY } from '../common/constants';
 import {
   ImportResponse,
   ImportFailure,
+  InputData,
   Settings,
   Mappings,
   IngestPipelineWrapper,
 } from '../common';
 
-export type InputData = any[];
-
 export function importDataProvider({ asCurrentUser }: IScopedClusterClient) {
   async function importData(
     id: string | undefined,

diff --git a/x-pack/plugins/file_upload/server/routes.ts b/x-pack/plugins/file_upload/server/routes.ts
@@ -5,13 +5,21 @@
  * 2.0.
  */
 
+import { schema } from '@kbn/config-schema';
 import { IRouter, IScopedClusterClient } from 'kibana/server';
-import { MAX_FILE_SIZE_BYTES, IngestPipelineWrapper, Mappings, Settings } from '../common';
+import {
+  MAX_FILE_SIZE_BYTES,
+  IngestPipelineWrapper,
+  InputData,
+  Mappings,
+  Settings,
+} from '../common';
 import { wrapError } from './error_wrapper';
-import { InputData, importDataProvider } from './import_data';
+import { analyzeFile } from './analyze_file';
+import { importDataProvider } from './import_data';
 
 import { updateTelemetry } from './telemetry';
-import { importFileBodySchema, importFileQuerySchema } from './schemas';
+import { analyzeFileQuerySchema, importFileBodySchema, importFileQuerySchema } from './schemas';
 
 function importData(
   client: IScopedClusterClient,
@@ -30,6 +38,44 @@ function importData(
  * Routes for the file upload.
  */
 export function fileUploadRoutes(router: IRouter) {
+  /**
+   * @apiGroup FileDataVisualizer
+   *
+   * @api {post} /api/file_upload/analyze_file Analyze file data
+   * @apiName AnalyzeFile
+   * @apiDescription Performs analysis of the file data.
+   *
+   * @apiSchema (query) analyzeFileQuerySchema
+   */
+  router.post(
+    {
+      path: '/api/file_upload/analyze_file',
+      validate: {
+        body: schema.any(),
+        query: analyzeFileQuerySchema,
+      },
+      options: {
+        body: {
+          accepts: ['text/*', 'application/json'],
+          maxBytes: MAX_FILE_SIZE_BYTES,
+        },
+        tags: ['access:fileUpload:analyzeFile'],
+      },
+    },
+    async (context, request, response) => {
+      try {
+        const result = await analyzeFile(
+          context.core.elasticsearch.client,
+          request.body,
+          request.query
+        );
+        return response.ok({ body: result });
+      } catch (e) {
+        return response.customError(wrapError(e));
+      }
+    }
+  );
+
   /**
    * @apiGroup FileDataVisualizer
    *

diff --git a/x-pack/plugins/file_upload/server/schemas.ts b/x-pack/plugins/file_upload/server/schemas.ts
@@ -7,6 +7,23 @@
 
 import { schema } from '@kbn/config-schema';
 
+export const analyzeFileQuerySchema = schema.object({
+  charset: schema.maybe(schema.string()),
+  column_names: schema.maybe(schema.string()),
+  delimiter: schema.maybe(schema.string()),
+  explain: schema.maybe(schema.string()),
+  format: schema.maybe(schema.string()),
+  grok_pattern: schema.maybe(schema.string()),
+  has_header_row: schema.maybe(schema.string()),
+  line_merge_size_limit: schema.maybe(schema.string()),
+  lines_to_sample: schema.maybe(schema.string()),
+  quote: schema.maybe(schema.string()),
+  should_trim_fields: schema.maybe(schema.string()),
+  timeout: schema.maybe(schema.string()),
+  timestamp_field: schema.maybe(schema.string()),
+  timestamp_format: schema.maybe(schema.string()),
+});
+
 export const importFileQuerySchema = schema.object({
   id: schema.maybe(schema.string()),
 });

diff --git a/x-pack/plugins/ml/common/types/capabilities.ts b/x-pack/plugins/ml/common/types/capabilities.ts
@@ -102,7 +102,11 @@ export function getPluginPrivileges() {
   return {
     admin: {
       ...privilege,
-      api: ['fileUpload:import', ...allMlCapabilitiesKeys.map((k) => `ml:${k}`)],
+      api: [
+        'fileUpload:import',
+        'fileUpload:analyzeFile',
+        ...allMlCapabilitiesKeys.map((k) => `ml:${k}`),
+      ],
       catalogue: [PLUGIN_ID, `${PLUGIN_ID}_file_data_visualizer`],
       ui: allMlCapabilitiesKeys,
       savedObject: {
@@ -116,7 +120,7 @@ export function getPluginPrivileges() {
     },
     user: {
       ...privilege,
-      api: userMlCapabilitiesKeys.map((k) => `ml:${k}`),
+      api: ['fileUpload:analyzeFile', ...userMlCapabilitiesKeys.map((k) => `ml:${k}`)],
       catalogue: [PLUGIN_ID],
       management: { insightsAndAlerting: [] },
       ui: userMlCapabilitiesKeys,

diff --git a/x-pack/plugins/ml/common/types/file_datavisualizer.ts b/x-pack/plugins/ml/common/types/file_datavisualizer.ts
diff --git a/...ic/application/datavisualizer/file_based/components/analysis_summary/analysis_summary.tsx b/...ic/application/datavisualizer/file_based/components/analysis_summary/analysis_summary.tsx
@@ -9,7 +9,7 @@ import { FormattedMessage } from '@kbn/i18n/react';
 import React, { FC } from 'react';
 
 import { EuiTitle, EuiSpacer, EuiDescriptionList } from '@elastic/eui';
-import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+import { FindFileStructureResponse } from '../../../../../../../file_upload/common';
 
 export const AnalysisSummary: FC<{ results: FindFileStructureResponse }> = ({ results }) => {
   const items = createDisplayItems(results);

diff --git a/...application/datavisualizer/file_based/components/combined_fields/combined_fields_form.tsx b/...application/datavisualizer/file_based/components/combined_fields/combined_fields_form.tsx
@@ -29,7 +29,7 @@ import {
   removeCombinedFieldsFromMappings,
   removeCombinedFieldsFromPipeline,
 } from './utils';
-import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+import { FindFileStructureResponse } from '../../../../../../../file_upload/common';
 
 interface Props {
   mappingsString: string;

diff --git a/.../ml/public/application/datavisualizer/file_based/components/combined_fields/geo_point.tsx b/.../ml/public/application/datavisualizer/file_based/components/combined_fields/geo_point.tsx
@@ -29,7 +29,7 @@ import {
   getFieldNames,
   getNameCollisionMsg,
 } from './utils';
-import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+import { FindFileStructureResponse } from '../../../../../../../file_upload/common';
 
 interface Props {
   addCombinedField: (combinedField: CombinedField) => void;

diff --git a/...ugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts b/...ugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts
@@ -9,8 +9,11 @@ import { i18n } from '@kbn/i18n';
 import { cloneDeep } from 'lodash';
 import uuid from 'uuid/v4';
 import { CombinedField } from './types';
-import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
-import { IngestPipeline, Mappings } from '../../../../../../../file_upload/common';
+import {
+  FindFileStructureResponse,
+  IngestPipeline,
+  Mappings,
+} from '../../../../../../../file_upload/common';
 
 const COMMON_LAT_NAMES = ['latitude', 'lat'];
 const COMMON_LON_NAMES = ['longitude', 'long', 'lon'];

diff --git a/...pplication/datavisualizer/file_based/components/explanation_flyout/explanation_flyout.tsx b/...pplication/datavisualizer/file_based/components/explanation_flyout/explanation_flyout.tsx
@@ -20,7 +20,7 @@ import {
   EuiText,
   EuiSubSteps,
 } from '@elastic/eui';
-import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+import { FindFileStructureResponse } from '../../../../../../../file_upload/common';
 
 interface Props {
   results: FindFileStructureResponse;

diff --git a/...ublic/application/datavisualizer/file_based/components/fields_stats_grid/create_fields.ts b/...ublic/application/datavisualizer/file_based/components/fields_stats_grid/create_fields.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+import { FindFileStructureResponse } from '../../../../../../../file_upload/common';
 import { getFieldNames, getSupportedFieldType } from './get_field_names';
 import { FileBasedFieldVisConfig } from '../../../stats_table/types';
 import { ML_JOB_FIELD_TYPES } from '../../../../../../common/constants/field_types';