[Automatic Import] Introduce support for structured logs (elastic#191749

) ## Summary This PR introduces `KVGraph` that is used to support `structured` log samples. Examples of structured log samples would be: ``` <134>1 1639132850.430422377 AP1 events type=disassociation radio='1' vap='1' client_mac='B0:A4:60:9B:3B:A6' channel='100' reason='1' instigator='2' duration='223.031691642' auth_neg_dur='0.005054229' last_auth_ago='223.020414600' is_wpa='1' full_conn='0.384002374' ip_resp='0.384002374' ip_src='10.197.39.50' http_resp='0.647356228' arp_resp='0.013562625' arp_src='10.197.39.50' dns_server='10.128.128.128' dns_req_rtt='0.023370084' dns_resp='0.263616104' dhcp_lease_completed='0.009196083' dhcp_server='10.128.128.128' dhcp_server_mac='E0:CB:BC:31:23:60' dhcp_resp='0.009196083' aid='977866432' ``` Currently the tests prove that it works best with the log samples adhering to `RFC5424` and `RFC3164`. The Graph shall be improved to work with `Custom Formats` going forward. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com> Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> (cherry picked from commit 08f70b7) # Conflicts: # x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.ts
bhapas · Sep 11, 2024 · aae8b8e · aae8b8e
1 parent 8457287
commit aae8b8e
Show file tree

Hide file tree

Showing 58 changed files with 1,260 additions and 68 deletions.
diff --git a/x-pack/plugins/integration_assistant/__jest__/fixtures/categorization.ts b/x-pack/plugins/integration_assistant/__jest__/fixtures/categorization.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { SamplesFormatName } from '../../common/api/model/common_attributes';
 import type { Pipeline } from '../../common';
 
 export const categorizationInitialPipeline: Pipeline = {
@@ -191,6 +192,7 @@ export const categorizationTestState = {
   invalidCategorization: [{ test: 'testinvalid' }],
   initialPipeline: categorizationInitialPipeline,
   results: { test: 'testresults' },
+  samplesFormat: { name: SamplesFormatName.Values.json },
 };
 
 export const categorizationMockProcessors = [

diff --git a/x-pack/plugins/integration_assistant/__jest__/fixtures/ecs_mapping.ts b/x-pack/plugins/integration_assistant/__jest__/fixtures/ecs_mapping.ts
@@ -5,6 +5,8 @@
  * 2.0.
  */
 
+import { SamplesFormatName } from '../../common/api/model/common_attributes';
+
 export const ecsMappingExpectedResults = {
   mapping: {
     mysql_enterprise: {
@@ -63,21 +65,35 @@ export const ecsMappingExpectedResults = {
           value: '8.11.0',
         },
       },
+      {
+        set: {
+          copy_from: 'message',
+          field: 'originalMessage',
+          tag: 'copy_original_message',
+        },
+      },
       {
         rename: {
-          field: 'message',
+          field: 'originalMessage',
           target_field: 'event.original',
           tag: 'rename_message',
           ignore_missing: true,
           if: 'ctx.event?.original == null',
         },
       },
+      {
+        remove: {
+          field: 'originalMessage',
+          if: 'ctx.event?.original != null',
+          ignore_missing: true,
+          tag: 'remove_copied_message',
+        },
+      },
       {
         remove: {
           field: 'message',
           ignore_missing: true,
           tag: 'remove_message',
-          if: 'ctx.event?.original != null',
         },
       },
       {
@@ -450,7 +466,7 @@ export const ecsTestState = {
   finalMapping: { test: 'testmapping' },
   sampleChunks: [''],
   results: { test: 'testresults' },
-  samplesFormat: 'testsamplesFormat',
+  samplesFormat: { name: SamplesFormatName.Values.json },
   ecsVersion: 'testversion',
   chunkMapping: { test1: 'test1' },
   useFinalMapping: false,
@@ -462,4 +478,5 @@ export const ecsTestState = {
   packageName: 'testpackage',
   dataStreamName: 'testDataStream',
   combinedSamples: '{"test1": "test1"}',
+  additionalProcessors: [],
 };
diff --git a/x-pack/plugins/integration_assistant/__jest__/fixtures/kv.ts b/x-pack/plugins/integration_assistant/__jest__/fixtures/kv.ts
@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SamplesFormatName } from '../../common/api/model/common_attributes';
+
+export const kvState = {
+  lastExecutedChain: 'testchain',
+  packageName: 'testPackage',
+  dataStreamName: 'testDatastream',
+  kvProcessor: { kv: { field: 'test', target_field: 'newtest' } },
+  logSamples: ['<134>1 dummy="data"'],
+  jsonSamples: ['{"test1": "test1"}'],
+  kvLogMessages: ['{"test1": "test1"}'],
+  finalized: false,
+  samplesFormat: { name: SamplesFormatName.Values.structured },
+  header: true,
+  ecsVersion: 'testVersion',
+  errors: { test: 'testerror' },
+  additionalProcessors: [{ kv: { field: 'test', target_field: 'newtest' } }],
+};
diff --git a/x-pack/plugins/integration_assistant/__jest__/fixtures/log_type_detection.ts b/x-pack/plugins/integration_assistant/__jest__/fixtures/log_type_detection.ts
@@ -10,11 +10,14 @@ import { SamplesFormatName } from '../../common/api/model/common_attributes';
 export const logFormatDetectionTestState = {
   lastExecutedChain: 'testchain',
   logSamples: ['{"test1": "test1"}'],
+  jsonSamples: ['{"test1": "test1"}'],
   exAnswer: 'testanswer',
   packageName: 'testPackage',
   dataStreamName: 'testDatastream',
   finalized: false,
-  samplesFormat: { name: SamplesFormatName.Values.json },
+  samplesFormat: { name: SamplesFormatName.Values.structured },
+  header: true,
   ecsVersion: 'testVersion',
   results: { test1: 'test1' },
+  additionalProcessors: [{ kv: { field: 'test', target_field: 'newtest' } }],
 };
diff --git a/x-pack/plugins/integration_assistant/__jest__/fixtures/related.ts b/x-pack/plugins/integration_assistant/__jest__/fixtures/related.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { SamplesFormatName } from '../../common/api/model/common_attributes';
 import type { Pipeline } from '../../common';
 
 export const relatedInitialPipeline: Pipeline = {
@@ -166,6 +167,7 @@ export const relatedTestState = {
   initialPipeline: relatedInitialPipeline,
   results: { test: 'testresults' },
   lastExecutedChain: 'testchain',
+  samplesFormat: { name: SamplesFormatName.Values.json },
 };
 
 export const relatedMockProcessors = [

diff --git a/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.schema.yaml b/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.schema.yaml
@@ -20,7 +20,13 @@ paths:
               required:
                 - logSamples
                 - connectorId
+                - packageName
+                - dataStreamName
               properties:
+                packageName:
+                  $ref: "../model/common_attributes.schema.yaml#/components/schemas/PackageName"
+                dataStreamName:
+                  $ref: "../model/common_attributes.schema.yaml#/components/schemas/DataStreamName"
                 logSamples:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/LogSamples"
                 connectorId:

diff --git a/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.test.ts b/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.test.ts
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { expectParseSuccess } from '@kbn/zod-helpers';
+import { AnalyzeLogsRequestBody } from './analyze_logs_route';
+import { getAnalyzeLogsRequestBody } from '../model/api_test.mock';
+
+describe('Analyze Logs request schema', () => {
+  test('full request validate', () => {
+    const payload: AnalyzeLogsRequestBody = getAnalyzeLogsRequestBody();
+
+    const result = AnalyzeLogsRequestBody.safeParse(payload);
+    expectParseSuccess(result);
+    expect(result.data).toEqual(payload);
+  });
+});
diff --git a/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.ts b/x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.ts
@@ -16,11 +16,19 @@
 
 import { z } from '@kbn/zod';
 
-import { LogSamples, Connector, LangSmithOptions } from '../model/common_attributes';
+import {
+  LogSamples,
+  Connector,
+  LangSmithOptions,
+  DataStreamName,
+  PackageName,
+} from '../model/common_attributes';
 import { AnalyzeLogsAPIResponse } from '../model/response_schemas';
 
 export type AnalyzeLogsRequestBody = z.infer<typeof AnalyzeLogsRequestBody>;
 export const AnalyzeLogsRequestBody = z.object({
+  packageName: PackageName,
+  dataStreamName: DataStreamName,
   logSamples: LogSamples,
   connectorId: Connector,
   langSmithOptions: LangSmithOptions.optional(),

diff --git a/.../plugins/integration_assistant/common/api/categorization/categorization_route.schema.yaml b/.../plugins/integration_assistant/common/api/categorization/categorization_route.schema.yaml
@@ -23,6 +23,7 @@ paths:
                 - rawSamples
                 - currentPipeline
                 - connectorId
+                - samplesFormat
               properties:
                 packageName:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/PackageName"
@@ -34,6 +35,8 @@ paths:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/Pipeline"
                 connectorId:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/Connector"
+                samplesFormat:
+                  $ref: "../model/common_attributes.schema.yaml#/components/schemas/SamplesFormat"
                 langSmithOptions:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/LangSmithOptions"
       responses:

diff --git a/x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.ts b/x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.ts
@@ -14,6 +14,7 @@ import {
   PackageName,
   Pipeline,
   RawSamples,
+  SamplesFormat,
 } from '../model/common_attributes';
 import { CategorizationAPIResponse } from '../model/response_schemas';
 
@@ -22,6 +23,7 @@ export const CategorizationRequestBody = z.object({
   packageName: PackageName,
   dataStreamName: DataStreamName,
   rawSamples: RawSamples,
+  samplesFormat: SamplesFormat,
   currentPipeline: Pipeline,
   connectorId: Connector,
   langSmithOptions: LangSmithOptions.optional(),

diff --git a/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.schema.yaml b/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.schema.yaml
@@ -21,6 +21,7 @@ paths:
                 - packageName
                 - dataStreamName
                 - rawSamples
+                - samplesFormat
                 - connectorId
               properties:
                 packageName:
@@ -29,8 +30,14 @@ paths:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/DataStreamName"
                 rawSamples:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/RawSamples"
+                samplesFormat:
+                  $ref: "../model/common_attributes.schema.yaml#/components/schemas/SamplesFormat"
                 mapping:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/Mapping"
+                additionalProcessors:
+                  type: array
+                  items:
+                    $ref: "../model/processor_attributes.schema.yaml#/components/schemas/ESProcessorItem"
                 connectorId:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/Connector"
                 langSmithOptions:

diff --git a/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.ts b/x-pack/plugins/integration_assistant/common/api/ecs/ecs_route.ts
@@ -5,24 +5,37 @@
  * 2.0.
  */
 
-import { z } from 'zod';
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Integration Assistatnt ECS Mapping API endpoint
+ *   version: 1
+ */
+
+import { z } from '@kbn/zod';
 
 import {
-  Connector,
-  DataStreamName,
-  LangSmithOptions,
-  Mapping,
   PackageName,
+  DataStreamName,
   RawSamples,
+  Mapping,
+  Connector,
+  LangSmithOptions,
+  SamplesFormat,
 } from '../model/common_attributes';
+import { ESProcessorItem } from '../model/processor_attributes';
 import { EcsMappingAPIResponse } from '../model/response_schemas';
 
 export type EcsMappingRequestBody = z.infer<typeof EcsMappingRequestBody>;
 export const EcsMappingRequestBody = z.object({
   packageName: PackageName,
   dataStreamName: DataStreamName,
   rawSamples: RawSamples,
+  samplesFormat: SamplesFormat,
   mapping: Mapping.optional(),
+  additionalProcessors: z.array(ESProcessorItem).optional(),
   connectorId: Connector,
   langSmithOptions: LangSmithOptions.optional(),
 });

diff --git a/x-pack/plugins/integration_assistant/common/api/model/api_test.mock.ts b/x-pack/plugins/integration_assistant/common/api/model/api_test.mock.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import type { AnalyzeLogsRequestBody } from '../analyze_logs/analyze_logs_route';
 import type { BuildIntegrationRequestBody } from '../build_integration/build_integration';
 import type { CategorizationRequestBody } from '../categorization/categorization_route';
 import type { EcsMappingRequestBody } from '../ecs/ecs_route';
@@ -61,6 +62,7 @@ export const getCategorizationRequestMock = (): CategorizationRequestBody => ({
   dataStreamName: 'test-data-stream-name',
   packageName: 'test-package-name',
   rawSamples,
+  samplesFormat: { name: 'ndjson' },
 });
 
 export const getBuildIntegrationRequestMock = (): BuildIntegrationRequestBody => ({
@@ -72,6 +74,7 @@ export const getEcsMappingRequestMock = (): EcsMappingRequestBody => ({
   dataStreamName: 'test-data-stream-name',
   packageName: 'test-package-name',
   connectorId: 'test-connector-id',
+  samplesFormat: { name: 'json', multiline: false },
 });
 
 export const getRelatedRequestMock = (): RelatedRequestBody => ({
@@ -80,4 +83,12 @@ export const getRelatedRequestMock = (): RelatedRequestBody => ({
   rawSamples,
   connectorId: 'test-connector-id',
   currentPipeline: getPipelineMock(),
+  samplesFormat: { name: 'structured', multiline: false },
+});
+
+export const getAnalyzeLogsRequestBody = (): AnalyzeLogsRequestBody => ({
+  dataStreamName: 'test-data-stream-name',
+  packageName: 'test-package-name',
+  connectorId: 'test-connector-id',
+  logSamples: rawSamples,
 });
diff --git a/x-pack/plugins/integration_assistant/common/api/model/response_schemas.schema.yaml b/x-pack/plugins/integration_assistant/common/api/model/response_schemas.schema.yaml
@@ -72,10 +72,15 @@ components:
       required:
         - results
       properties:
+        additionalProcessors:
+          type: array
+          items:
+            $ref: "./processor_attributes.schema.yaml#/components/schemas/ESProcessorItem"
         results:
           type: object
           required:
             - parsedSamples
+            - samplesFormat
           properties:
             samplesFormat:
               $ref: "./common_attributes.schema.yaml#/components/schemas/SamplesFormat"

diff --git a/x-pack/plugins/integration_assistant/common/api/model/response_schemas.ts b/x-pack/plugins/integration_assistant/common/api/model/response_schemas.ts
@@ -17,6 +17,7 @@
 import { z } from 'zod';
 
 import { Docs, Mapping, Pipeline, SamplesFormat } from './common_attributes';
+import { ESProcessorItem } from './processor_attributes';
 
 export type EcsMappingAPIResponse = z.infer<typeof EcsMappingAPIResponse>;
 export const EcsMappingAPIResponse = z.object({
@@ -55,4 +56,5 @@ export const AnalyzeLogsAPIResponse = z.object({
     samplesFormat: SamplesFormat,
     parsedSamples: z.array(z.string()),
   }),
+  additionalProcessors: z.array(ESProcessorItem).optional(),
 });
diff --git a/x-pack/plugins/integration_assistant/common/api/related/related_route.schema.yaml b/x-pack/plugins/integration_assistant/common/api/related/related_route.schema.yaml
@@ -23,6 +23,7 @@ paths:
                 - rawSamples
                 - currentPipeline
                 - connectorId
+                - samplesFormat
               properties:
                 packageName:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/PackageName"
@@ -34,6 +35,8 @@ paths:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/Pipeline"
                 connectorId:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/Connector"
+                samplesFormat:
+                  $ref: "../model/common_attributes.schema.yaml#/components/schemas/SamplesFormat"
                 langSmithOptions:
                   $ref: "../model/common_attributes.schema.yaml#/components/schemas/LangSmithOptions"
       responses:

diff --git a/x-pack/plugins/integration_assistant/common/api/related/related_route.ts b/x-pack/plugins/integration_assistant/common/api/related/related_route.ts
@@ -14,6 +14,7 @@ import {
   PackageName,
   Pipeline,
   RawSamples,
+  SamplesFormat,
 } from '../model/common_attributes';
 import { RelatedAPIResponse } from '../model/response_schemas';
 
@@ -22,6 +23,7 @@ export const RelatedRequestBody = z.object({
   packageName: PackageName,
   dataStreamName: DataStreamName,
   rawSamples: RawSamples,
+  samplesFormat: SamplesFormat,
   currentPipeline: Pipeline,
   connectorId: Connector,
   langSmithOptions: LangSmithOptions.optional(),