Skip to content

Commit

Permalink
[ML] Functional tests - stabilize job row details validations (elasti…
Browse files Browse the repository at this point in the history 
…c#64503)

This PR stabilizes the AD job row details check by only validating expected properties instead of the whole object.
  • Loading branch information
@pheyos
pheyos committed Apr 27, 2020
1 parent aa01c28 commit 6d05737
Show file tree
Hide file tree
Showing 9 changed files with 19 additions and 55 deletions.
4 changes: 0 additions & 4 deletions x-pack/test/functional/apps/machine_learning/anomaly_detection/advanced_job.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,6 @@ export default function({ getService }: FtrProviderContext) {
modelSizeStats: {
result_type: 'model_size_stats',
model_bytes_exceeded: '0.0 B',
model_bytes_memory_limit: '10.0 MB',
total_by_field_count: '37',
total_over_field_count: '92',
total_partition_field_count: '8',
Expand Down Expand Up @@ -262,7 +261,6 @@ export default function({ getService }: FtrProviderContext) {
modelSizeStats: {
result_type: 'model_size_stats',
model_bytes_exceeded: '0.0 B',
model_bytes_memory_limit: '100.0 MB',
total_by_field_count: '994',
total_over_field_count: '0',
total_partition_field_count: '2',
Expand Down Expand Up @@ -549,7 +547,6 @@ export default function({ getService }: FtrProviderContext) {
job_id: testData.jobId,
result_type: testData.expected.modelSizeStats.result_type,
model_bytes_exceeded: testData.expected.modelSizeStats.model_bytes_exceeded,
model_bytes_memory_limit: testData.expected.modelSizeStats.model_bytes_memory_limit,
total_by_field_count: testData.expected.modelSizeStats.total_by_field_count,
total_over_field_count: testData.expected.modelSizeStats.total_over_field_count,
total_partition_field_count:
Expand Down Expand Up @@ -813,7 +810,6 @@ export default function({ getService }: FtrProviderContext) {
job_id: testData.jobIdClone,
result_type: testData.expected.modelSizeStats.result_type,
model_bytes_exceeded: testData.expected.modelSizeStats.model_bytes_exceeded,
model_bytes_memory_limit: testData.expected.modelSizeStats.model_bytes_memory_limit,
total_by_field_count: testData.expected.modelSizeStats.total_by_field_count,
total_over_field_count: testData.expected.modelSizeStats.total_over_field_count,
total_partition_field_count:
Expand Down
1 change: 0 additions & 1 deletion x-pack/test/functional/apps/machine_learning/anomaly_detection/categorization_job.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,6 @@ export default function({ getService }: FtrProviderContext) {
job_id: expectedJobId,
result_type: 'model_size_stats',
model_bytes_exceeded: '0.0 B',
model_bytes_memory_limit: '15.0 MB',
total_by_field_count: '30',
total_over_field_count: '0',
total_partition_field_count: '2',
Expand Down
2 changes: 0 additions & 2 deletions x-pack/test/functional/apps/machine_learning/anomaly_detection/date_nanos_job.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,6 @@ export default function({ getService }: FtrProviderContext) {
modelSizeStats: {
result_type: 'model_size_stats',
model_bytes_exceeded: '0.0 B',
model_bytes_memory_limit: '10.0 MB',
total_by_field_count: '3',
total_over_field_count: '0',
total_partition_field_count: '2',
Expand Down Expand Up @@ -422,7 +421,6 @@ export default function({ getService }: FtrProviderContext) {
job_id: testData.jobId,
result_type: testData.expected.modelSizeStats.result_type,
model_bytes_exceeded: testData.expected.modelSizeStats.model_bytes_exceeded,
model_bytes_memory_limit: testData.expected.modelSizeStats.model_bytes_memory_limit,
total_by_field_count: testData.expected.modelSizeStats.total_by_field_count,
total_over_field_count: testData.expected.modelSizeStats.total_over_field_count,
total_partition_field_count:
Expand Down
1 change: 0 additions & 1 deletion x-pack/test/functional/apps/machine_learning/anomaly_detection/multi_metric_job.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@ export default function({ getService }: FtrProviderContext) {
job_id: expectedJobId,
result_type: 'model_size_stats',
model_bytes_exceeded: '0.0 B',
model_bytes_memory_limit: '20.0 MB',
total_by_field_count: '59',
total_over_field_count: '0',
total_partition_field_count: '58',
Expand Down
1 change: 0 additions & 1 deletion x-pack/test/functional/apps/machine_learning/anomaly_detection/population_job.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ export default function({ getService }: FtrProviderContext) {
job_id: expectedJobId,
result_type: 'model_size_stats',
model_bytes_exceeded: '0.0 B',
model_bytes_memory_limit: '8.0 MB',
total_by_field_count: '25',
total_over_field_count: '92',
total_partition_field_count: '3',
Expand Down
6 changes: 0 additions & 6 deletions x-pack/test/functional/apps/machine_learning/anomaly_detection/saved_search_job.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ export default function({ getService }: FtrProviderContext) {
modelSizeStats: {
result_type: 'model_size_stats',
model_bytes_exceeded: '0.0 B',
model_bytes_memory_limit: '20.0 MB',
total_by_field_count: '3',
total_over_field_count: '0',
total_partition_field_count: '2',
Expand Down Expand Up @@ -105,7 +104,6 @@ export default function({ getService }: FtrProviderContext) {
modelSizeStats: {
result_type: 'model_size_stats',
model_bytes_exceeded: '0.0 B',
model_bytes_memory_limit: '20.0 MB',
total_by_field_count: '7',
total_over_field_count: '0',
total_partition_field_count: '6',
Expand Down Expand Up @@ -156,7 +154,6 @@ export default function({ getService }: FtrProviderContext) {
modelSizeStats: {
result_type: 'model_size_stats',
model_bytes_exceeded: '0.0 B',
model_bytes_memory_limit: '20.0 MB',
total_by_field_count: '7',
total_over_field_count: '0',
total_partition_field_count: '6',
Expand Down Expand Up @@ -208,7 +205,6 @@ export default function({ getService }: FtrProviderContext) {
modelSizeStats: {
result_type: 'model_size_stats',
model_bytes_exceeded: '0.0 B',
model_bytes_memory_limit: '20.0 MB',
total_by_field_count: '3',
total_over_field_count: '0',
total_partition_field_count: '2',
Expand Down Expand Up @@ -259,7 +255,6 @@ export default function({ getService }: FtrProviderContext) {
modelSizeStats: {
result_type: 'model_size_stats',
model_bytes_exceeded: '0.0 B',
model_bytes_memory_limit: '20.0 MB',
total_by_field_count: '3',
total_over_field_count: '0',
total_partition_field_count: '2',
Expand Down Expand Up @@ -464,7 +459,6 @@ export default function({ getService }: FtrProviderContext) {
job_id: testData.jobId,
result_type: testData.expected.modelSizeStats.result_type,
model_bytes_exceeded: testData.expected.modelSizeStats.model_bytes_exceeded,
model_bytes_memory_limit: testData.expected.modelSizeStats.model_bytes_memory_limit,
total_by_field_count: testData.expected.modelSizeStats.total_by_field_count,
total_over_field_count: testData.expected.modelSizeStats.total_over_field_count,
total_partition_field_count:
Expand Down
1 change: 0 additions & 1 deletion x-pack/test/functional/apps/machine_learning/anomaly_detection/single_metric_job.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,6 @@ export default function({ getService }: FtrProviderContext) {
job_id: expectedJobId,
result_type: 'model_size_stats',
model_bytes_exceeded: '0.0 B',
model_bytes_memory_limit: '15.0 MB',
total_by_field_count: '3',
total_over_field_count: '0',
total_partition_field_count: '2',
Expand Down
2 changes: 1 addition & 1 deletion x-pack/test/functional/apps/machine_learning/pages.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ export default function({ getService }: FtrProviderContext) {
const ml = getService('ml');

describe('page navigation', function() {
this.tags(['includeFirefox', 'mlqa']);
this.tags(['skipFirefox', 'mlqa']);
before(async () => {
await ml.api.cleanMlIndices();
await ml.securityUI.loginAsMlPowerUser();
Expand Down
56 changes: 18 additions & 38 deletions x-pack/test/functional/services/machine_learning/job_table.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,44 +187,24 @@ export function MachineLearningJobTableProvider({ getService }: FtrProviderConte
expectedCounts: object,
expectedModelSizeStats: object
) {
const countDetails = await this.parseJobCounts(jobId);
const counts = countDetails.counts;

// fields that have changing values are only validated
// to be present and then removed so they don't make
// the object validation fail
expect(counts).to.have.property('last_data_time');
delete counts.last_data_time;

expect(counts).to.eql(expectedCounts);

const modelSizeStats = countDetails.modelSizeStats;

// fields that have changing values are only validated
// to be present and then removed so they don't make
// the object validation fail
expect(modelSizeStats).to.have.property('log_time');
delete modelSizeStats.log_time;
expect(modelSizeStats).to.have.property('model_bytes');
delete modelSizeStats.model_bytes;

// remove categorization fields from validation until
// the ES version is updated
delete modelSizeStats.categorization_status;
delete modelSizeStats.categorized_doc_count;
delete modelSizeStats.dead_category_count;
delete modelSizeStats.frequent_category_count;
delete modelSizeStats.rare_category_count;
delete modelSizeStats.total_category_count;

// MML during clone has changed in #61589
// TODO: adjust test code to reflect the new behavior
expect(modelSizeStats).to.have.property('model_bytes_memory_limit');
delete modelSizeStats.model_bytes_memory_limit;
// @ts-ignore
delete expectedModelSizeStats.model_bytes_memory_limit;

expect(modelSizeStats).to.eql(expectedModelSizeStats);
const { counts, modelSizeStats } = await this.parseJobCounts(jobId);

// Only check for expected keys / values, ignore additional properties
// This way the tests stay stable when new properties are added on the ES side
for (const [key, value] of Object.entries(expectedCounts)) {
expect(counts)
.to.have.property(key)
.eql(value, `Expected counts property '${key}' to exist with value '${value}'`);
}

for (const [key, value] of Object.entries(expectedModelSizeStats)) {
expect(modelSizeStats)
.to.have.property(key)
.eql(
value,
`Expected model size stats property '${key}' to exist with value '${value}')`
);
}
}

public async clickActionsMenu(jobId: string) {
Expand Down

0 comments on commit 6d05737

Please sign in to comment.