-
Notifications
You must be signed in to change notification settings - Fork 8.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[ML] AIOps: Functional/API integration tests for text field support f…
…or log rate analysis (#168177) This updates the artificial dataset generator for log rate analysis to allow to create variants including text fields. The artificial dataset is now used for 4 variants of functional and API integration tests: Testing spike and dip with both with and without a text field. The new tests surfaced some issues that were fixed as part of this PR: - Getting the counts of log patterns in combination with individual significant terms ended up with to granular groups. This PR adds additional queries to get counts for log patterns in combination with item sets already derived from significant terms. - The `support` value is returned by the frequent item sets agg and is used as a threshold whether to include an item set for grouping. This was missing from significant log patterns and is fixed by this PR. - Adds a check to not get frequent item sets for log patterns if there are no significant terms. - The way we fetched log patterns using a time filter that spans the whole of the baseline start to the deviation end caused problems with analysing dips. This PR updates those queries to only fetch the actual baseline and deviation time range. - The integration tests caught an issue where we'd still fetch the histogram for log patterns even if we'd request grouping information only.
- Loading branch information
Showing
25 changed files
with
603 additions
and
142 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
129 changes: 129 additions & 0 deletions
129
...plugins/aiops/common/__mocks__/artificial_logs/final_significant_term_groups_textfield.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,129 @@ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the Elastic License | ||
* 2.0; you may not use this file except in compliance with the Elastic License | ||
* 2.0. | ||
*/ | ||
|
||
import type { SignificantTermGroup } from '@kbn/ml-agg-utils'; | ||
|
||
export const finalSignificantTermGroupsTextfield: SignificantTermGroup[] = [ | ||
{ | ||
docCount: 636, | ||
group: [ | ||
{ | ||
docCount: 792, | ||
duplicate: 2, | ||
fieldName: 'url', | ||
fieldValue: 'home.php', | ||
key: 'url:home.php', | ||
pValue: 0.00974308761016614, | ||
type: 'keyword', | ||
}, | ||
{ | ||
docCount: 636, | ||
duplicate: 2, | ||
fieldName: 'user', | ||
fieldValue: 'Peter', | ||
key: 'user:Peter', | ||
pValue: 0.00974308761016614, | ||
type: 'keyword', | ||
}, | ||
], | ||
id: '2091742187', | ||
pValue: 0.00974308761016614, | ||
}, | ||
{ | ||
docCount: 634, | ||
group: [ | ||
{ | ||
docCount: 1266, | ||
duplicate: 2, | ||
fieldName: 'response_code', | ||
fieldValue: '500', | ||
key: 'response_code:500', | ||
pValue: 0.012783309213417932, | ||
type: 'keyword', | ||
}, | ||
{ | ||
docCount: 792, | ||
duplicate: 2, | ||
fieldName: 'url', | ||
fieldValue: 'home.php', | ||
key: 'url:home.php', | ||
pValue: 0.00974308761016614, | ||
type: 'keyword', | ||
}, | ||
{ | ||
docCount: 634, | ||
duplicate: 2, | ||
fieldName: 'message', | ||
fieldValue: 'an unexpected error occured', | ||
key: 'an unexpected error occured', | ||
pValue: 0.00974308761016614, | ||
type: 'log_pattern', | ||
}, | ||
], | ||
id: '1528268618', | ||
pValue: 0.00974308761016614, | ||
}, | ||
{ | ||
docCount: 632, | ||
group: [ | ||
{ | ||
docCount: 1266, | ||
duplicate: 2, | ||
fieldName: 'response_code', | ||
fieldValue: '500', | ||
key: 'response_code:500', | ||
pValue: 0.012783309213417932, | ||
type: 'keyword', | ||
}, | ||
{ | ||
docCount: 790, | ||
duplicate: 2, | ||
fieldName: 'url', | ||
fieldValue: 'login.php', | ||
key: 'url:login.php', | ||
pValue: 0.012783309213417932, | ||
type: 'keyword', | ||
}, | ||
{ | ||
docCount: 632, | ||
duplicate: 2, | ||
fieldName: 'message', | ||
fieldValue: 'an unexpected error occured', | ||
key: 'an unexpected error occured', | ||
pValue: 0.012783309213417932, | ||
type: 'log_pattern', | ||
}, | ||
], | ||
id: '2619569380', | ||
pValue: 0.012783309213417932, | ||
}, | ||
{ | ||
docCount: 632, | ||
group: [ | ||
{ | ||
docCount: 790, | ||
duplicate: 2, | ||
fieldName: 'url', | ||
fieldValue: 'login.php', | ||
key: 'url:login.php', | ||
pValue: 0.012783309213417932, | ||
type: 'keyword', | ||
}, | ||
{ | ||
docCount: 632, | ||
duplicate: 2, | ||
fieldName: 'user', | ||
fieldValue: 'Peter', | ||
key: 'user:Peter', | ||
pValue: 0.012783309213417932, | ||
type: 'keyword', | ||
}, | ||
], | ||
id: '1937394803', | ||
pValue: 0.012783309213417932, | ||
}, | ||
]; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
24 changes: 24 additions & 0 deletions
24
x-pack/plugins/aiops/common/__mocks__/artificial_logs/significant_log_patterns.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the Elastic License | ||
* 2.0; you may not use this file except in compliance with the Elastic License | ||
* 2.0. | ||
*/ | ||
|
||
import type { SignificantTerm } from '@kbn/ml-agg-utils'; | ||
|
||
export const significantLogPatterns: SignificantTerm[] = [ | ||
{ | ||
bg_count: 0, | ||
doc_count: 1266, | ||
fieldName: 'message', | ||
fieldValue: 'an unexpected error occured', | ||
key: 'an unexpected error occured', | ||
normalizedScore: 0, | ||
pValue: 0.000001, | ||
score: -13.815510557964274, | ||
total_bg_count: 1975, | ||
total_doc_count: 4669, | ||
type: 'log_pattern', | ||
}, | ||
]; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.