From 54b33f0170cade957b0387e0f3a0c9d03a956623 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 29 May 2018 21:17:47 -0400 Subject: [PATCH] Use dedicated ML APIs in tests (#30941) ML has dedicated APIs for datafeeds and jobs yet base test classes and some tests were relying on the cluster state for this state. This commit removes this usage in favor of using the dedicated endpoints. --- .../integration/MlRestTestStateCleaner.java | 21 ++++++++----- .../xpack/ml/support/BaseMlIntegTestCase.java | 25 +++++++++------- .../rest-api-spec/test/ml/jobs_crud.yml | 30 ++++++++----------- 3 files changed, 40 insertions(+), 36 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java index 8e326e3556b59..426b9d7edd866 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.core.ml.integration; import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.test.rest.ESRestTestCase; @@ -35,10 +37,12 @@ public void clearMlMetadata() throws IOException { @SuppressWarnings("unchecked") private void deleteAllDatafeeds() throws IOException { - Map clusterStateAsMap = testCase.entityAsMap(adminClient.performRequest("GET", "/_cluster/state", - Collections.singletonMap("filter_path", "metadata.ml.datafeeds"))); - List> datafeeds = - (List>) XContentMapValues.extractValue("metadata.ml.datafeeds", clusterStateAsMap); + final Request datafeedsRequest = new Request("GET", "/_xpack/ml/datafeeds"); + datafeedsRequest.addParameter("filter_path", "datafeeds"); + final Response datafeedsResponse = adminClient.performRequest(datafeedsRequest); + @SuppressWarnings("unchecked") + final List> datafeeds = + (List>) XContentMapValues.extractValue("datafeeds", testCase.entityAsMap(datafeedsResponse)); if (datafeeds == null) { return; } @@ -75,11 +79,12 @@ private void deleteAllDatafeeds() throws IOException { } private void deleteAllJobs() throws IOException { - Map clusterStateAsMap = testCase.entityAsMap(adminClient.performRequest("GET", "/_cluster/state", - Collections.singletonMap("filter_path", "metadata.ml.jobs"))); + final Request jobsRequest = new Request("GET", "/_xpack/ml/anomaly_detectors"); + jobsRequest.addParameter("filter_path", "jobs"); + final Response response = adminClient.performRequest(jobsRequest); @SuppressWarnings("unchecked") - List> jobConfigs = - (List>) XContentMapValues.extractValue("metadata.ml.jobs", clusterStateAsMap); + final List> jobConfigs = + (List>) XContentMapValues.extractValue("jobs", testCase.entityAsMap(response)); if (jobConfigs == null) { return; } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java index 32961680f5b2d..9c1850c167b59 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java @@ -26,6 +26,10 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.discovery.TestZenDiscovery; import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction; +import org.elasticsearch.xpack.core.ml.action.GetJobsAction; +import org.elasticsearch.xpack.core.ml.action.util.QueryPage; +import org.elasticsearch.xpack.core.ml.client.MachineLearningClient; import org.elasticsearch.xpack.ml.LocalStateMachineLearning; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.core.ml.MachineLearningField; @@ -270,7 +274,9 @@ public static GetDatafeedsStatsAction.Response.DatafeedStats getDatafeedStats(St } public static void deleteAllDatafeeds(Logger logger, Client client) throws Exception { - MlMetadata mlMetadata = MlMetadata.getMlMetadata(client.admin().cluster().prepareState().get().getState()); + final MachineLearningClient mlClient = new MachineLearningClient(client); + final QueryPage datafeeds = + mlClient.getDatafeeds(new GetDatafeedsAction.Request(GetDatafeedsAction.ALL)).actionGet().getResponse(); try { logger.info("Closing all datafeeds (using _all)"); StopDatafeedAction.Response stopResponse = client @@ -291,11 +297,10 @@ public static void deleteAllDatafeeds(Logger logger, Client client) throws Excep "Had to resort to force-stopping datafeed, something went wrong?", e1); } - for (DatafeedConfig datafeed : mlMetadata.getDatafeeds().values()) { - String datafeedId = datafeed.getId(); + for (final DatafeedConfig datafeed : datafeeds.results()) { assertBusy(() -> { try { - GetDatafeedsStatsAction.Request request = new GetDatafeedsStatsAction.Request(datafeedId); + GetDatafeedsStatsAction.Request request = new GetDatafeedsStatsAction.Request(datafeed.getId()); GetDatafeedsStatsAction.Response r = client.execute(GetDatafeedsStatsAction.INSTANCE, request).get(); assertThat(r.getResponse().results().get(0).getDatafeedState(), equalTo(DatafeedState.STOPPED)); } catch (InterruptedException | ExecutionException e) { @@ -303,13 +308,14 @@ public static void deleteAllDatafeeds(Logger logger, Client client) throws Excep } }); DeleteDatafeedAction.Response deleteResponse = - client.execute(DeleteDatafeedAction.INSTANCE, new DeleteDatafeedAction.Request(datafeedId)).get(); + client.execute(DeleteDatafeedAction.INSTANCE, new DeleteDatafeedAction.Request(datafeed.getId())).get(); assertTrue(deleteResponse.isAcknowledged()); } } public static void deleteAllJobs(Logger logger, Client client) throws Exception { - MlMetadata mlMetadata = MlMetadata.getMlMetadata(client.admin().cluster().prepareState().get().getState()); + final MachineLearningClient mlClient = new MachineLearningClient(client); + final QueryPage jobs = mlClient.getJobs(new GetJobsAction.Request(MetaData.ALL)).actionGet().getResponse(); try { CloseJobAction.Request closeRequest = new CloseJobAction.Request(MetaData.ALL); @@ -333,15 +339,14 @@ public static void deleteAllJobs(Logger logger, Client client) throws Exception e1); } - for (Map.Entry entry : mlMetadata.getJobs().entrySet()) { - String jobId = entry.getKey(); + for (final Job job : jobs.results()) { assertBusy(() -> { GetJobsStatsAction.Response statsResponse = - client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request(jobId)).actionGet(); + client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request(job.getId())).actionGet(); assertEquals(JobState.CLOSED, statsResponse.getResponse().results().get(0).getState()); }); DeleteJobAction.Response response = - client.execute(DeleteJobAction.INSTANCE, new DeleteJobAction.Request(jobId)).get(); + client.execute(DeleteJobAction.INSTANCE, new DeleteJobAction.Request(job.getId())).get(); assertTrue(response.isAcknowledged()); } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml index 9ed14c2f860ef..ed6b9ead1a3b1 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml @@ -548,10 +548,9 @@ - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser - cluster.state: - metric: [ metadata ] - filter_path: metadata.persistent_tasks - - match: {"metadata.persistent_tasks.tasks.0.task.xpack/ml/job.status.state": opened} + xpack.ml.get_job_stats: + job_id: jobs-crud-close-job + - match: {"jobs.0.state": opened} - do: xpack.ml.close_job: @@ -561,11 +560,9 @@ - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser - cluster.state: - metric: [ metadata ] - filter_path: metadata.persistent_tasks - - match: - metadata.persistent_tasks.tasks: [] + xpack.ml.get_job_stats: + job_id: jobs-crud-close-job + - match: {"jobs.0.state": closed} --- "Test closing a closed job isn't an error": @@ -789,10 +786,9 @@ - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser - cluster.state: - metric: [ metadata ] - filter_path: metadata.persistent_tasks - - match: {"metadata.persistent_tasks.tasks.0.task.xpack/ml/job.status.state": opened} + xpack.ml.get_job_stats: + job_id: jobs-crud-force-close-job + - match: {"jobs.0.state": opened} - do: xpack.ml.close_job: @@ -803,11 +799,9 @@ - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser - cluster.state: - metric: [ metadata ] - filter_path: metadata.persistent_tasks - - match: - metadata.persistent_tasks.tasks: [] + xpack.ml.get_job_stats: + job_id: jobs-crud-force-close-job + - match: {"jobs.0.state": closed} --- "Test force closing a closed job isn't an error":