[ AutoImport] Introduce automatic log type detection graph (elastic#1…

…90407) ## Summary This PR introduces a new graph in `Auto Import` called - `LogTypeDetection` Currently, only JSON/NDJSON formats are supported to be uploaded for building custom integrations. With this feature the capabilities to upload different log types is allowed. Although parsing of the new log types will be handled separately with a separate [issue.](elastic/security-team#9845) - The logs are initially parsed for JSON/NDJSON types in the UI side. - If it is not JSON/NDJSON format , then a new API `AnalyzeLogs` is triggered. - UI allows any type of logs to be uploaded. - Currently there is a server level content length restriction of `1MB` which needs to be extended. - For any log types other than JSON/NDJSON the handling graphs are not yet implemented , hence a `501 Not implemented` message appears. - The idea is to support `structured` , `csv` , `unstructured` syslog handling graphs. ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com> Co-authored-by: Hanna Tamoudi <hanna.tamoudi@elastic.co> Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
bhapas · Aug 27, 2024 · 9f01f73 · 9f01f73
1 parent 9d967e0
commit 9f01f73
Show file tree

Hide file tree

Showing 33 changed files with 640 additions and 79 deletions.
diff --git a/x-pack/plugins/integration_assistant/__jest__/fixtures/log_type_detection.ts b/x-pack/plugins/integration_assistant/__jest__/fixtures/log_type_detection.ts
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SamplesFormatName } from '../../common/api/model/common_attributes';
+
+export const logFormatDetectionTestState = {
+  lastExecutedChain: 'testchain',
+  logSamples: ['{"test1": "test1"}'],
+  exAnswer: 'testanswer',
+  packageName: 'testPackage',
+  dataStreamName: 'testDatastream',
+  finalized: false,
+  samplesFormat: { name: SamplesFormatName.Values.json },
+  ecsVersion: 'testVersion',
+  results: { test1: 'test1' },
+};
diff --git a/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.schema.yaml b/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.schema.yaml
@@ -0,0 +1,36 @@
+openapi: 3.0.3
+info:
+  title: Auto Import Analyze Logs API endpoint
+  version: "1"
+paths:
+  /api/integration_assistant/analyzelogs:
+    post:
+      summary: Analyzes log samples and processes them.
+      operationId: AnalyzeLogs
+      x-codegen-enabled: false
+      description: Analyzes log samples and processes them
+      tags:
+        - Analyze Logs API
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              required:
+                - logSamples
+                - connectorId
+              properties:
+                logSamples:
+                  $ref: "../model/common_attributes.schema.yaml#/components/schemas/LogSamples"
+                connectorId:
+                  $ref: "../model/common_attributes.schema.yaml#/components/schemas/Connector"
+                langSmithOptions:
+                  $ref: "../model/common_attributes.schema.yaml#/components/schemas/LangSmithOptions"
+      responses:
+        200:
+          description: Indicates a successful call.
+          content:
+            application/json:
+              schema:
+                $ref: "../model/response_schemas.schema.yaml#/components/schemas/AnalyzeLogsAPIResponse"
diff --git a/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.ts b/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.ts
@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Auto Import Analyze Logs API endpoint
+ *   version: 1
+ */
+
+import { z } from '@kbn/zod';
+
+import { LogSamples, Connector, LangSmithOptions } from '../model/common_attributes';
+import { AnalyzeLogsAPIResponse } from '../model/response_schemas';
+
+export type AnalyzeLogsRequestBody = z.infer<typeof AnalyzeLogsRequestBody>;
+export const AnalyzeLogsRequestBody = z.object({
+  logSamples: LogSamples,
+  connectorId: Connector,
+  langSmithOptions: LangSmithOptions.optional(),
+});
+export type AnalyzeLogsRequestBodyInput = z.input<typeof AnalyzeLogsRequestBody>;
+
+export type AnalyzeLogsResponse = z.infer<typeof AnalyzeLogsResponse>;
+export const AnalyzeLogsResponse = AnalyzeLogsAPIResponse;
diff --git a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.schema.yaml
@@ -16,6 +16,12 @@ components:
       minLength: 1
       description: DataStream name for the integration to be built.
 
+    LogSamples:
+      type: array
+      items:
+        type: string
+      description:  String form of the input logsamples.
+
     RawSamples:
       type: array
       items:
@@ -42,6 +48,10 @@ components:
       enum:
         - ndjson
         - json
+        - csv
+        - structured
+        - unstructured
+        - unsupported
 
     SamplesFormat:
       type: object

diff --git a/x-pack/plugins/integration_assistant/common/api/model/common_attributes.ts b/x-pack/plugins/integration_assistant/common/api/model/common_attributes.ts
@@ -21,6 +21,12 @@ export const PackageName = z.string().min(1);
 export type DataStreamName = z.infer<typeof DataStreamName>;
 export const DataStreamName = z.string().min(1);
 
+/**
+ * String form of the input logsamples.
+ */
+export type LogSamples = z.infer<typeof LogSamples>;
+export const LogSamples = z.array(z.string());
+
 /**
  * String array containing the json raw samples that are used for ecs mapping.
  */
@@ -49,7 +55,14 @@ export const Docs = z.array(z.object({}).passthrough());
  * The name of the log samples format.
  */
 export type SamplesFormatName = z.infer<typeof SamplesFormatName>;
-export const SamplesFormatName = z.enum(['ndjson', 'json']);
+export const SamplesFormatName = z.enum([
+  'ndjson',
+  'json',
+  'csv',
+  'structured',
+  'unstructured',
+  'unsupported',
+]);
 export type SamplesFormatNameEnum = typeof SamplesFormatName.enum;
 export const SamplesFormatNameEnum = SamplesFormatName.enum;
 

diff --git a/x-pack/plugins/integration_assistant/common/api/model/response_schemas.schema.yaml b/x-pack/plugins/integration_assistant/common/api/model/response_schemas.schema.yaml
@@ -66,3 +66,20 @@ components:
           properties:
             docs:
               $ref: "./common_attributes.schema.yaml#/components/schemas/Docs"
+
+    AnalyzeLogsAPIResponse:
+      type: object
+      required:
+        - results
+      properties:
+        results:
+          type: object
+          required:
+            - parsedSamples
+          properties:
+            samplesFormat:
+              $ref: "./common_attributes.schema.yaml#/components/schemas/SamplesFormat"
+            parsedSamples:
+              type: array
+              items:
+                type: string
diff --git a/x-pack/plugins/integration_assistant/common/api/model/response_schemas.ts b/x-pack/plugins/integration_assistant/common/api/model/response_schemas.ts
@@ -16,7 +16,7 @@
 
 import { z } from '@kbn/zod';
 
-import { Docs, Mapping, Pipeline } from './common_attributes';
+import { Docs, Mapping, Pipeline, SamplesFormat } from './common_attributes';
 
 export type EcsMappingAPIResponse = z.infer<typeof EcsMappingAPIResponse>;
 export const EcsMappingAPIResponse = z.object({
@@ -48,3 +48,11 @@ export const CheckPipelineAPIResponse = z.object({
     docs: Docs,
   }),
 });
+
+export type AnalyzeLogsAPIResponse = z.infer<typeof AnalyzeLogsAPIResponse>;
+export const AnalyzeLogsAPIResponse = z.object({
+  results: z.object({
+    samplesFormat: SamplesFormat,
+    parsedSamples: z.array(z.string()),
+  }),
+});
diff --git a/x-pack/plugins/integration_assistant/common/constants.ts b/x-pack/plugins/integration_assistant/common/constants.ts
@@ -18,6 +18,7 @@ export const INTEGRATION_ASSISTANT_BASE_PATH = '/api/integration_assistant';
 
 export const ECS_GRAPH_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/ecs`;
 export const CATEGORIZATION_GRAPH_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/categorization`;
+export const ANALYZE_LOGS_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/analyzelogs`;
 export const RELATED_GRAPH_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/related`;
 export const CHECK_PIPELINE_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/pipeline`;
 export const INTEGRATION_BUILDER_PATH = `${INTEGRATION_ASSISTANT_BASE_PATH}/build`;

diff --git a/x-pack/plugins/integration_assistant/common/index.ts b/x-pack/plugins/integration_assistant/common/index.ts
@@ -15,6 +15,7 @@ export {
 } from './api/check_pipeline/check_pipeline';
 export { EcsMappingRequestBody, EcsMappingResponse } from './api/ecs/ecs_route';
 export { RelatedRequestBody, RelatedResponse } from './api/related/related_route';
+export { AnalyzeLogsRequestBody, AnalyzeLogsResponse } from './api/analyze_logs/analyze_logs_route';
 
 export type {
   DataStream,
@@ -35,4 +36,5 @@ export {
   PLUGIN_ID,
   RELATED_GRAPH_PATH,
   CHECK_PIPELINE_PATH,
+  ANALYZE_LOGS_PATH,
 } from './constants';
diff --git a/x-pack/plugins/integration_assistant/public/common/lib/api.ts b/x-pack/plugins/integration_assistant/public/common/lib/api.ts
@@ -16,6 +16,8 @@ import type {
   CheckPipelineRequestBody,
   CheckPipelineResponse,
   BuildIntegrationRequestBody,
+  AnalyzeLogsRequestBody,
+  AnalyzeLogsResponse,
 } from '../../../common';
 import {
   INTEGRATION_BUILDER_PATH,
@@ -24,7 +26,7 @@ import {
   RELATED_GRAPH_PATH,
   CHECK_PIPELINE_PATH,
 } from '../../../common';
-import { FLEET_PACKAGES_PATH } from '../../../common/constants';
+import { ANALYZE_LOGS_PATH, FLEET_PACKAGES_PATH } from '../../../common/constants';
 
 export interface EpmPackageResponse {
   response: [{ id: string; name: string }];
@@ -42,6 +44,16 @@ export interface RequestDeps {
   abortSignal: AbortSignal;
 }
 
+export const runAnalyzeLogsGraph = async (
+  body: AnalyzeLogsRequestBody,
+  { http, abortSignal }: RequestDeps
+): Promise<AnalyzeLogsResponse> =>
+  http.post<AnalyzeLogsResponse>(ANALYZE_LOGS_PATH, {
+    headers: defaultHeaders,
+    body: JSON.stringify(body),
+    signal: abortSignal,
+  });
+
 export const runEcsGraph = async (
   body: EcsMappingRequestBody,
   { http, abortSignal }: RequestDeps

diff --git a/...ssistant/public/components/create_integration/create_integration_assistant/mocks/state.ts b/...ssistant/public/components/create_integration/create_integration_assistant/mocks/state.ts
@@ -5,11 +5,11 @@
  * 2.0.
  */
 
-import type { Pipeline, Docs } from '../../../../../common';
+import type { Pipeline, Docs, SamplesFormat } from '../../../../../common';
 import type { Actions, State } from '../state';
 import type { AIConnector } from '../types';
 
-const result: { pipeline: Pipeline; docs: Docs } = {
+const result: { pipeline: Pipeline; docs: Docs; samplesFormat: SamplesFormat } = {
   pipeline: {
     description: 'Pipeline to process my_integration my_data_stream_title logs',
     processors: [
@@ -389,6 +389,7 @@ const result: { pipeline: Pipeline; docs: Docs } = {
       ],
     },
   ],
+  samplesFormat: { name: 'json' },
 };
 
 const rawSamples = [
@@ -419,8 +420,7 @@ export const mockState: State = {
     dataStreamName: 'mocked_datastream_name',
     dataStreamDescription: 'Mocked Data Stream Description',
     inputTypes: ['filestream'],
-    logsSampleParsed: rawSamples,
-    samplesFormat: { name: 'ndjson', multiline: false },
+    logSamples: rawSamples,
   },
   isGenerating: false,
   result,

diff --git a/...tion_assistant/public/components/create_integration/create_integration_assistant/state.ts b/...tion_assistant/public/components/create_integration/create_integration_assistant/state.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 import { createContext, useContext } from 'react';
-import type { Pipeline, Docs } from '../../../../common';
+import type { Pipeline, Docs, SamplesFormat } from '../../../../common';
 import type { AIConnector, IntegrationSettings } from './types';
 
 export interface State {
@@ -16,6 +16,7 @@ export interface State {
   result?: {
     pipeline: Pipeline;
     docs: Docs;
+    samplesFormat?: SamplesFormat;
   };
 }
 

diff --git a/...integration/create_integration_assistant/steps/data_stream_step/generation_modal.test.tsx b/...integration/create_integration_assistant/steps/data_stream_step/generation_modal.test.tsx
@@ -17,23 +17,27 @@ import { TelemetryEventType } from '../../../../../services/telemetry/types';
 const integrationSettings = mockState.integrationSettings!;
 const connector = mockState.connector!;
 
+const mockAnalyzeLogsResults = {
+  parsedSamples: [{ test: 'analyzeLogsResponse' }],
+  sampleLogsFormat: { name: 'json' },
+};
 const mockEcsMappingResults = { pipeline: { test: 'ecsMappingResponse' }, docs: [] };
 const mockCategorizationResults = { pipeline: { test: 'categorizationResponse' }, docs: [] };
 const mockRelatedResults = { pipeline: { test: 'relatedResponse' }, docs: [] };
+const mockRunAnalyzeLogsGraph = jest.fn((_: unknown) => ({ results: mockAnalyzeLogsResults }));
 const mockRunEcsGraph = jest.fn((_: unknown) => ({ results: mockEcsMappingResults }));
 const mockRunCategorizationGraph = jest.fn((_: unknown) => ({
   results: mockCategorizationResults,
 }));
 const mockRunRelatedGraph = jest.fn((_: unknown) => ({ results: mockRelatedResults }));
 
 const defaultRequest = {
-  packageName: integrationSettings.name ?? '',
-  dataStreamName: integrationSettings.dataStreamName ?? '',
-  rawSamples: integrationSettings.logsSampleParsed ?? [],
   connectorId: connector.id,
+  LangSmithOptions: undefined,
 };
 
 jest.mock('../../../../../common/lib/api', () => ({
+  runAnalyzeLogsGraph: (params: unknown) => mockRunAnalyzeLogsGraph(params),
   runEcsGraph: (params: unknown) => mockRunEcsGraph(params),
   runCategorizationGraph: (params: unknown) => mockRunCategorizationGraph(params),
   runRelatedGraph: (params: unknown) => mockRunRelatedGraph(params),
@@ -74,21 +78,39 @@ describe('GenerationModal', () => {
       expect(result.queryByTestId('generationModal')).toBeInTheDocument();
     });
 
+    it('should call runAnalyzeLogsGraph with correct parameters', () => {
+      expect(mockRunAnalyzeLogsGraph).toHaveBeenCalledWith({
+        ...defaultRequest,
+        logSamples: integrationSettings.logSamples ?? [],
+      });
+    });
+
     it('should call runEcsGraph with correct parameters', () => {
-      expect(mockRunEcsGraph).toHaveBeenCalledWith(defaultRequest);
+      expect(mockRunEcsGraph).toHaveBeenCalledWith({
+        ...defaultRequest,
+        rawSamples: mockAnalyzeLogsResults.parsedSamples,
+        packageName: integrationSettings.name ?? '',
+        dataStreamName: integrationSettings.dataStreamName ?? '',
+      });
     });
 
     it('should call runCategorizationGraph with correct parameters', () => {
       expect(mockRunCategorizationGraph).toHaveBeenCalledWith({
         ...defaultRequest,
         currentPipeline: mockEcsMappingResults.pipeline,
+        rawSamples: mockAnalyzeLogsResults.parsedSamples,
+        packageName: integrationSettings.name ?? '',
+        dataStreamName: integrationSettings.dataStreamName ?? '',
       });
     });
 
     it('should call runRelatedGraph with correct parameters', () => {
       expect(mockRunRelatedGraph).toHaveBeenCalledWith({
         ...defaultRequest,
         currentPipeline: mockCategorizationResults.pipeline,
+        rawSamples: mockAnalyzeLogsResults.parsedSamples,
+        packageName: integrationSettings.name ?? '',
+        dataStreamName: integrationSettings.dataStreamName ?? '',
       });
     });
 
@@ -101,7 +123,7 @@ describe('GenerationModal', () => {
         TelemetryEventType.IntegrationAssistantGenerationComplete,
         {
           sessionId: expect.any(String),
-          sampleRows: integrationSettings.logsSampleParsed?.length ?? 0,
+          sampleRows: integrationSettings.logSamples?.length ?? 0,
           actionTypeId: connector.actionTypeId,
           model: expect.anything(),
           provider: connector.apiProvider ?? 'unknown',
@@ -147,7 +169,7 @@ describe('GenerationModal', () => {
         TelemetryEventType.IntegrationAssistantGenerationComplete,
         {
           sessionId: expect.any(String),
-          sampleRows: integrationSettings.logsSampleParsed?.length ?? 0,
+          sampleRows: integrationSettings.logSamples?.length ?? 0,
           actionTypeId: connector.actionTypeId,
           model: expect.anything(),
           provider: connector.apiProvider ?? 'unknown',